"""Exercise 24.1: do three models make the same mistakes? (constructed teaching data) Three models (甲, 乙, 丙) answer the same 20 questions. 1 = correct, 0 = wrong. Each question also has a difficulty from an independent source: the share of human test-takers who answered it correctly (constructed numbers). Questions with a human accuracy of at least 60% form the "easy" stratum, the rest the "hard" stratum. For each pair of models we compute observed -- share of questions both got wrong; expected -- what independent errors would give: (error rate of one) x (error rate of the other); first over all 20 questions, then within each stratum (the stratified expectation adds up the within-stratum expectations). Finally we show what goes wrong if the strata are formed from the three models' own answers instead of from the independent difficulty. The script first computes the small example of section 24.3: two models whose errors are independent within each difficulty level, pooled over easy and hard questions. Run: python3 tools/exercises/ch24_error_overlap.py """ from itertools import combinations print("=== section 24.3: difficulty alone makes pooled co-errors look correlated ===") share_easy, err_easy = 0.5, 0.10 # half the questions are easy; each model errs on 10% of them share_hard, err_hard = 0.5, 0.50 # half are hard; each model errs on 50% of them both = share_easy * err_easy ** 2 + share_hard * err_hard ** 2 # independent within each level err_all = share_easy * err_easy + share_hard * err_hard print(f"within easy: both wrong {err_easy ** 2:.0%}; within hard: both wrong {err_hard ** 2:.0%}") print(f"overall error rate of each model {err_all:.0%}; pooled both-wrong {both:.0%}; " f"pooled independence expectation {err_all:.2f} x {err_all:.2f} = {err_all ** 2:.0%}") print() print("=== exercise 24.1 ===") # (question, human accuracy %, 甲, 乙, 丙) ROWS = [ (1, 25, 0, 0, 0), (2, 41, 1, 1, 1), (3, 63, 1, 1, 1), (4, 66, 1, 1, 0), (5, 70, 1, 1, 1), (6, 45, 0, 0, 1), (7, 38, 0, 0, 1), (8, 92, 1, 1, 1), (9, 79, 1, 1, 0), (10, 34, 1, 1, 0), (11, 88, 1, 1, 1), (12, 57, 1, 0, 0), (13, 22, 0, 0, 0), (14, 85, 0, 0, 1), (15, 81, 1, 1, 1), (16, 48, 1, 1, 1), (17, 74, 0, 0, 1), (18, 76, 1, 1, 1), (19, 52, 0, 1, 0), (20, 30, 0, 0, 0), ] MODELS = ["甲", "乙", "丙"] EASY_CUTOFF = 60 def wrong(row, m): return row[2 + MODELS.index(m)] == 0 def summary(rows, label): n = len(rows) print(f"--- {label}: {n} questions ---") for m in MODELS: e = sum(wrong(r, m) for r in rows) print(f" {m}: wrong on {e}, error rate {e / n:.0%}") out = {} for a, b in combinations(MODELS, 2): both = sum(wrong(r, a) and wrong(r, b) for r in rows) ea = sum(wrong(r, a) for r in rows) / n eb = sum(wrong(r, b) for r in rows) / n exp = ea * eb * n out[(a, b)] = (both, exp) print(f" {a}{b}: both wrong on {both} ({both / n:.0%}); independent expectation " f"{ea:.2f} x {eb:.2f} = {ea * eb:.0%}, i.e. {exp:.2f} questions") all3 = sum(all(wrong(r, m) for m in MODELS) for r in rows) rates = [sum(wrong(r, m) for r in rows) / n for m in MODELS] exp3 = rates[0] * rates[1] * rates[2] * n print(f" all three wrong on {all3}; independent expectation {exp3:.2f} questions") return out, all3, exp3 pooled, all3, exp3_pooled = summary(ROWS, "all questions, no strata") easy = [r for r in ROWS if r[1] >= EASY_CUTOFF] hard = [r for r in ROWS if r[1] < EASY_CUTOFF] print(f"easy questions: {[r[0] for r in easy]}") print(f"hard questions: {[r[0] for r in hard]}") s_easy, all3_e, exp3_e = summary(easy, f"easy stratum (human accuracy >= {EASY_CUTOFF}%)") s_hard, all3_h, exp3_h = summary(hard, f"hard stratum (human accuracy < {EASY_CUTOFF}%)") print("--- comparison: observed both-wrong vs expectation (in questions) ---") for pair in pooled: obs = pooled[pair][0] exp_pooled = pooled[pair][1] exp_strat = s_easy[pair][1] + s_hard[pair][1] print(f" {pair[0]}{pair[1]}: observed {obs}; pooled expectation {exp_pooled:.2f} (ratio {obs / exp_pooled:.2f}); " f"stratified expectation {s_easy[pair][1]:.2f} + {s_hard[pair][1]:.2f} = {exp_strat:.2f} " f"(ratio {obs / exp_strat:.2f})") print(f" all three: observed {all3}; pooled expectation {exp3_pooled:.2f}; " f"stratified expectation {exp3_e:.2f} + {exp3_h:.2f} = {exp3_e + exp3_h:.2f}") print() print("--- the wrong way: strata from the models' own answers ---") own_hard = [r for r in ROWS if sum(wrong(r, m) for m in MODELS) >= 2] own_easy = [r for r in ROWS if sum(wrong(r, m) for m in MODELS) < 2] print(f"'hard' = at least two of the three wrong: {[r[0] for r in own_hard]}") o_easy, *_ = summary(own_easy, "'easy' by the models' own answers") o_hard, *_ = summary(own_hard, "'hard' by the models' own answers")