"""Exercise 15.1: pick QuALITY questions for the three-person debate exercise (standard library only). Usage python3 ch15_quality_select.py select [--seed N] python3 ch15_quality_select.py check "" `select` reads one QuALITY split (train or dev; the test split has no released answers) and writes, for each of the nine groups G1..G9 of ch15_schedule.py: /G1_judge.txt questions with two options A/B; no story, no answer, no stance /G1_expert.txt the story with numbered paragraphs, the questions, the correct option and the stance of each debater or consultant (open only if you are an expert for this group in the schedule) /answer_key.txt the correct options of all groups (open only after all six sessions) It prints file names and counts, never the text of a record. `check` tells a judge whether a quoted passage occurs verbatim in the story of a group, without showing the story: the exercise's stand-in for the system-verified quotes of Michael et al. (2023). Only the numbered story paragraphs are searched (not the header lines), and quotes shorter than MIN_WORDS words are refused, since they would match almost anywhere. Selection rule (fixed before looking at any question): - stories from Project Gutenberg only (fiction, so the judge cannot answer from prior knowledge); - the two writers' question sets for one article are merged; - a question is eligible if `difficult` == 1, most untimed validators marked it answerable and unambiguous (eval1 == 1), and most said it needs more than a sentence or two of context (eval2 >= 2); - the wrong option shown is the distractor most often named best by the untimed validators (ties: lower option number), as in Michael et al. (2023); - nine articles with at least four eligible questions are drawn at random, four questions each; - within each group the correct option is shown as A on two questions and as B on two, in random order; - consultancy groups: within the group the consultant defends the correct option on exactly two of the four questions, in random order; debate groups: each debater defends the correct option on exactly two of the four questions, in random order. """ import argparse import json import os import random import re from collections import Counter, defaultdict GROUPS = { # group -> (protocol, experts), copied from ch15_schedule.py "G1": ("辩论", ["甲", "乙"]), "G2": ("单方咨询", ["丙"]), "G3": ("直接问答", []), "G4": ("单方咨询", ["乙"]), "G5": ("直接问答", []), "G6": ("辩论", ["甲", "丙"]), "G7": ("辩论", ["乙", "丙"]), "G8": ("单方咨询", ["甲"]), "G9": ("直接问答", []), } PER_GROUP = 4 def majority(values, pred): values = [v for v in values if v is not None] return bool(values) and sum(1 for v in values if pred(v)) * 2 > len(values) def eligible(q): if q.get("difficult") != 1: return False val = q.get("validation", []) ev1 = [v.get("untimed_eval1_answerability") for v in val] ev2 = [v.get("untimed_eval2_context") for v in val] return majority(ev1, lambda x: x == 1) and majority(ev2, lambda x: x >= 2) def wrong_option(q): gold = q["gold_label"] votes = Counter() for v in q.get("validation", []): d = v.get("untimed_best_distractor", v.get("untimed_eval3_distractor")) if isinstance(d, int) and d != gold and 1 <= d <= len(q["options"]): votes[d] += 1 if votes: best = max(votes.values()) return min(d for d, c in votes.items() if c == best) return min(i for i in range(1, len(q["options"]) + 1) if i != gold) def norm(text): return re.sub(r"\s+", " ", text).strip().lower() def select(data, outdir, seed): rng = random.Random(seed) articles = defaultdict(lambda: {"title": "", "article": "", "questions": []}) with open(data, encoding="utf8") as f: for line in f: row = json.loads(line) if row.get("source") != "Gutenberg": continue a = articles[row["article_id"]] a["title"], a["article"] = row["title"], row["article"] a["questions"] += [q for q in row["questions"] if eligible(q)] pool = sorted(k for k, a in articles.items() if len(a["questions"]) >= PER_GROUP) assert len(pool) >= len(GROUPS), f"only {len(pool)} articles qualify" chosen = rng.sample(pool, len(GROUPS)) os.makedirs(outdir, exist_ok=True) key_lines = [] for (g, (proto, experts)), art_id in zip(sorted(GROUPS.items()), chosen): a = articles[art_id] qs = rng.sample(a["questions"], PER_GROUP) if proto == "单方咨询": stance = ["正确", "正确", "错误", "错误"] rng.shuffle(stance) elif proto == "辩论": first = [experts[0], experts[0], experts[1], experts[1]] # who defends the correct option rng.shuffle(first) judge, expert = [f"题组 {g}({proto})\n"], [f"题组 {g}({proto}),专家:{'、'.join(experts) or '无'}\n", f"标题:{a['title']}\n"] paras = [p for p in re.split(r"\n\s*\n", a["article"]) if p.strip()] expert += [f"[{i}] {p.strip()}\n" for i, p in enumerate(paras, 1)] expert.append("\n") letters = ["A", "A", "B", "B"] # where the correct option is shown, balanced within the group rng.shuffle(letters) for i, q in enumerate(qs, 1): right, wrong = q["options"][q["gold_label"] - 1], q["options"][wrong_option(q) - 1] letter = letters[i - 1] pair = [("正确", right), ("错误", wrong)] if letter == "A" else [("错误", wrong), ("正确", right)] judge.append(f"\n第 {i} 题:{q['question']}\nA. {pair[0][1]}\nB. {pair[1][1]}\n") expert.append(f"\n第 {i} 题:{q['question']}\nA. {pair[0][1]}\nB. {pair[1][1]}\n正确选项:{letter}\n") if proto == "单方咨询": side = letter if stance[i - 1] == "正确" else ("B" if letter == "A" else "A") expert.append(f"咨询方({experts[0]})为选项 {side} 辩护\n") elif proto == "辩论": other = experts[1] if first[i - 1] == experts[0] else experts[0] wrong_letter = "B" if letter == "A" else "A" expert.append(f"{first[i - 1]} 为选项 {letter} 辩护,{other} 为选项 {wrong_letter} 辩护\n") key_lines.append(f"{g} 第 {i} 题:{letter}\n") with open(os.path.join(outdir, f"{g}_judge.txt"), "w", encoding="utf8") as f: f.writelines(judge) with open(os.path.join(outdir, f"{g}_expert.txt"), "w", encoding="utf8") as f: f.writelines(expert) with open(os.path.join(outdir, "answer_key.txt"), "w", encoding="utf8") as f: f.writelines(key_lines) print(f"{len(pool)} Gutenberg articles have >= {PER_GROUP} eligible questions; drew {len(GROUPS)}") print("wrote", ", ".join(sorted(os.listdir(outdir)))) MIN_WORDS = 5 # shorter quotes match almost anywhere and prove nothing def check(outdir, group, quote): if len(quote.split()) < MIN_WORDS: print(f"引文太短(少于 {MIN_WORDS} 个词),无法说明它出自原文;请引用更长的原话") return with open(os.path.join(outdir, f"{group}_expert.txt"), encoding="utf8") as f: text = f.read() head = text.split("\n\n第 1 题")[0] start = re.search(r"^\[1\] ", head, flags=re.M) # skip the group, expert and title lines story = re.sub(r"^\[\d+\] ", "", head[start.start():] if start else "", flags=re.M) print("已核对:原文中有这段话" if norm(quote) in norm(story) else "未核对:原文中找不到这段话") if __name__ == "__main__": ap = argparse.ArgumentParser() sub = ap.add_subparsers(dest="cmd", required=True) s = sub.add_parser("select") s.add_argument("data") s.add_argument("outdir") s.add_argument("--seed", type=int, default=15) c = sub.add_parser("check") c.add_argument("outdir") c.add_argument("group") c.add_argument("quote") args = ap.parse_args() if args.cmd == "select": select(args.data, args.outdir, args.seed) else: check(args.outdir, args.group, args.quote)