"""Exercise 18.1: reading three results of one truth probe (constructed data). The same logistic-regression probe on the same layer, three ways of splitting the statements: R1 random split of statements into training and held-out test sets R2 split by template: no template appears on both sides R3 the same template split as R2, training labels randomly shuffled before training; the test set and its true labels are exactly those of R2 The counts below are constructed for teaching. The script computes each accuracy with its 95% confidence interval (section 3.3, sqrt(p(1-p)/n)), the difference R1 - R2 (two different test sets, so the unpaired interval of section 3.4 applies), and compares R2 and R3 with the chance rate of the balanced test set. R1 - R3 is printed only to show why that pair is not compared. Standard library only. Run: python3 tools/exercises/ch18_probe_readings.py """ import math Z = 1.96 results = { # name: (correct, test-set size, true statements in test set) "R1 random split": (388, 400, 200), "R2 template split": (292, 360, 180), "R3 template split, shuffled training labels": (187, 360, 180), } def ci(k, n): p = k / n se = math.sqrt(p * (1 - p) / n) return p, se, p - Z * se, p + Z * se acc = {} for name, (k, n, n_true) in results.items(): p, se, lo, hi = ci(k, n) acc[name] = (p, se, n) majority = max(n_true, n - n_true) / n print(f"{name}: {k}/{n} = {p:.1%}, SE {se:.2%}, 95% CI [{lo:.1%}, {hi:.1%}]; " f"true/false in test = {n_true}/{n - n_true}, majority-class rate {majority:.0%}") p1, se1, _ = acc["R1 random split"] p2, se2, _ = acc["R2 template split"] p3, se3, n3 = acc["R3 template split, shuffled training labels"] d = p1 - p2 se_d = math.sqrt(se1 ** 2 + se2 ** 2) print() print(f"R1 - R2 (different test sets, unpaired): {d:.1%}, " f"95% CI [{d - Z * se_d:.1%}, {d + Z * se_d:.1%}]") print(f"R1 - R3: {p1 - p3:.1%} (split and training labels both differ, so not attributable " f"to either; compare pairs that change one thing)") chance = 0.5 for label, p, se in (("R2", p2, se2), ("R3", p3, se3)): lo, hi = p - Z * se, p + Z * se inside = lo <= chance <= hi print(f"{label} vs chance {chance:.0%}: CI [{lo:.1%}, {hi:.1%}] " f"{'contains' if inside else 'excludes'} {chance:.0%}")