Survey tool guides

Checking open-text answers

Populiva doesn't read survey answers. Instead, this script runs on your own computer against your survey export (and, optionally, your Populiva CSV) and flags answers worth a second look. It uses only Python's standard library, so there's nothing to install beyond Python 3.

1. 1. Run the check

   Save the script below as open_text_check.py, export your survey as CSV, and run it with the names of your open-text columns. Add your Populiva study export to use in-survey signals (pastes, typing speed, the hidden AI instruction), matched on SESSION_ID:

   python3 open_text_check.py survey.csv --text Q7 Q12 --populiva populiva-export.csv --id SESSION_ID

2. 2. What it flags

   Chatbot-style wording (phrases like “As an AI” or “Certainly! Here's”), answers nearly identical to another participant's, and, if you add your Populiva export, pasting in the survey, typing faster than people can, and following our hidden AI instruction.

   It writes a copy of your CSV with a check_flags column. A flag is a reason to read an answer again, not proof. Review flagged answers yourself before rejecting anyone, and remember that participants can appeal.

3. 3. The script

   Free to use and change. Adjust the phrase list and thresholds at the top for your study.

   #!/usr/bin/env python3
   """Flag open-text answers worth a closer look (Populiva guide, ADR 0003).
   
   Runs on your own computer: answer text never leaves it. Standard library only.
   
     python3 open_text_check.py survey.csv --text Q7 Q12 [--populiva populiva-export.csv] [--id SESSION_ID]
   
   Writes survey-checked.csv with a `check_flags` column. Flags are reasons to
   read an answer again, never proof: review before rejecting anyone.
   """
   import argparse, csv, difflib, re, sys
   
   PHRASES = [  # wording that often appears when a chatbot wrote the answer
       "as an ai", "as a language model", "i don't have personal", "i do not have personal", "i'm just an ai",
       "certainly! here", "here's a", "in conclusion,", "overall, it is important to", "it's important to note",
       "i hope this helps", "delve into",
   ]
   MIN_LENGTH = 40          # characters; shorter answers are too generic to compare
   DUPLICATE_RATIO = 0.9    # near-identical answers from different people
   FAST_TYPING_CPM = 1200   # Populiva snippet: characters per minute while typing
   
   
   def load(path):
       with open(path, newline="", encoding="utf-8-sig") as handle:
           rows = list(csv.DictReader(handle))
       return rows, (list(rows[0].keys()) if rows else [])
   
   
   def main():
       parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
       parser.add_argument("survey")
       parser.add_argument("--text", nargs="+", required=True, help="open-text column names")
       parser.add_argument("--populiva", help="Populiva study CSV export, to use in-survey signals")
       parser.add_argument("--id", default="SESSION_ID", help="column that matches Populiva's SESSION_ID")
       args = parser.parse_args()
   
       rows, headers = load(args.survey)
       missing = [c for c in args.text + [args.id] if c not in headers]
       if missing:
           sys.exit(f"Missing columns in {args.survey}: {', '.join(missing)}")
   
       signals = {}
       if args.populiva:
           export, _ = load(args.populiva)
           signals = {row["SESSION_ID"]: row for row in export}
   
       flags = {i: set() for i in range(len(rows))}
       for column in args.text:
           answers = [(i, re.sub(r"\s+", " ", (row.get(column) or "").strip().lower())) for i, row in enumerate(rows)]
           long_answers = [(i, text) for i, text in answers if len(text) >= MIN_LENGTH]
           for i, text in long_answers:
               if any(phrase in text for phrase in PHRASES):
                   flags[i].add(f"{column}: chatbot-style wording")
           for a in range(len(long_answers)):
               for b in range(a + 1, len(long_answers)):
                   (i, x), (j, y) = long_answers[a], long_answers[b]
                   if difflib.SequenceMatcher(None, x, y).ratio() >= DUPLICATE_RATIO:
                       flags[i].add(f"{column}: near-duplicate of another participant")
                       flags[j].add(f"{column}: near-duplicate of another participant")
   
       for i, row in enumerate(rows):
           signal = signals.get(row.get(args.id, ""))
           if not signal:
               continue
           if signal.get("snippet_pastes") not in (None, "", "0"):
               flags[i].add("pasted text in the survey")
           cpm = signal.get("snippet_chars_per_minute") or ""
           if cpm.isdigit() and int(cpm) > FAST_TYPING_CPM:
               flags[i].add("typed faster than people can")
           if "ai_instruction_followed" in (signal.get("quality_flags") or ""):
               flags[i].add("followed Populiva's hidden AI instruction")
   
       out = args.survey.rsplit(".", 1)[0] + "-checked.csv"
       with open(out, "w", newline="", encoding="utf-8") as handle:
           writer = csv.DictWriter(handle, fieldnames=headers + ["check_flags"])
           writer.writeheader()
           for i, row in enumerate(rows):
               writer.writerow({**row, "check_flags": "; ".join(sorted(flags[i]))})
       print(f"Wrote {out}: {sum(1 for f in flags.values() if f)} of {len(rows)} rows flagged for a closer look.")
   
   
   if __name__ == "__main__":
       main()