"""Exercise 17.2: performance gap recovered on three tasks (constructed data). For each task: test accuracy of the weak teacher, of the strong student trained on the teacher's labels, and of the strong reference trained on reliable labels. Task B has a student worse than its teacher; on task C the strong reference beats the teacher by only 1 point. For task C the script also builds the paired 2x2 table of the strong reference and the weak teacher on the same test set and computes the 95% confidence interval of their accuracy difference with the paired method of section 3.4: only the items on which the two disagree move the difference. For comparison it also prints the (wider) interval that ignores the pairing. Standard library only. Run: python3 tools/exercises/ch17_pgr.py """ import math Z = 1.96 # 95% two-sided normal quantile tasks = { # task: (weak teacher, strong student on weak labels, strong reference), percent "A": (62.0, 71.0, 86.0), "B": (70.0, 66.0, 82.0), "C": (75.0, 77.0, 76.0), } def pgr(weak, student, ref): gap = ref - weak if gap == 0: return None # denominator zero: do not compute return (student - weak) / gap print("Performance gap recovered, PGR = (student - weak) / (reference - weak)") for name, (w, s, r) in tasks.items(): p = pgr(w, s, r) print(f" task {name}: weak {w:.1f}%, student {s:.1f}%, reference {r:.1f}%; " f"numerator {s - w:+.1f} pt, denominator {r - w:+.1f} pt, PGR = {p:.1%}") # ---- Task C: paired 2x2 table on the same test set ------------------------------------- n = 1000 both_right = 700 # reference right, teacher right ref_only = 60 # reference right, teacher wrong teacher_only = 50 # reference wrong, teacher right both_wrong = n - both_right - ref_only - teacher_only assert both_wrong == 190 acc_ref = (both_right + ref_only) / n acc_weak = (both_right + teacher_only) / n acc_student = 770 / n assert abs(acc_ref - tasks["C"][2] / 100) < 1e-12 assert abs(acc_weak - tasks["C"][0] / 100) < 1e-12 assert abs(acc_student - tasks["C"][1] / 100) < 1e-12 print() print(f"Task C test set: n = {n}") print(f" both right {both_right}, reference only {ref_only}, teacher only {teacher_only}, " f"both wrong {both_wrong}") print(f" reference {acc_ref:.1%}, teacher {acc_weak:.1%}") def paired_ci(b, c, n): """Paired difference of two accuracies on the same n items. Per item, D = +1 (only the first is right), -1 (only the second is right), 0 otherwise. Mean d = (b - c)/n; variance of D = (b + c)/n - d^2; standard error = sqrt(variance / n). """ d = (b - c) / n var = (b + c) / n - d * d se = math.sqrt(var / n) return d, se, d - Z * se, d + Z * se d, se, lo, hi = paired_ci(ref_only, teacher_only, n) print(f" paired: difference {d:.2%}, SE {se:.4%}, 95% CI [{lo:.2%}, {hi:.2%}]") se_unpaired = math.sqrt(acc_ref * (1 - acc_ref) / n + acc_weak * (1 - acc_weak) / n) print(f" unpaired (ignores pairing, for comparison): SE {se_unpaired:.4%}, " f"95% CI [{d - Z * se_unpaired:.2%}, {d + Z * se_unpaired:.2%}]") # What the ratio does across the plausible range of the denominator (student fixed at 77%). num = (acc_student - acc_weak) * 100 print(" PGR of task C if the true gap were (student fixed at 77.0%):") for gap in (hi * 100, 1.0, 0.5): print(f" gap {gap:.2f} pt -> PGR {num / gap:.0%}") print(f" gap 0 pt -> not computed; gap below 0 (lower end {lo * 100:.2f} pt) -> sign flips") # Same rates of disagreement on a much larger test set: the 1-point gap becomes reliable. n_big = 20000 d2, se2, lo2, hi2 = paired_ci(1200, 1000, n_big) print() print(f"Same disagreement rates on n = {n_big} (reference only 1200, teacher only 1000):") print(f" difference {d2:.2%}, SE {se2:.4%}, 95% CI [{lo2:.2%}, {hi2:.2%}]")