"""Chapter 3, section 3.4 and exercise 3.1: comparing models A and B on the same questions. Constructed data (for teaching): models A and B answer the same 200 questions. The per-question 0/1 results are built from four category counts (both right 124, A right B wrong 18, A wrong B right 14, both wrong 44), shuffled with a fixed seed, so A scores 142/200 = 71% and B 138/200 = 69%. The script prints the two 200-character strings used in the exercise, counts the four categories back from them, and computes every number quoted in section 3.4 and in the solution. Standard library only. """ import math import random N = 200 counts = {"11": 124, "10": 18, "01": 14, "00": 44} # first digit A, second digit B cells = [k for k, v in counts.items() for _ in range(v)] random.Random(20261116).shuffle(cells) A = "".join(c[0] for c in cells) B = "".join(c[1] for c in cells) print("A strings (4 lines of 50):") for i in range(0, N, 50): print(f' "{A[i:i + 50]}"') print("B strings (4 lines of 50):") for i in range(0, N, 50): print(f' "{B[i:i + 50]}"') # count the four categories back from the strings both = sum(a == "1" and b == "1" for a, b in zip(A, B)) a_only = sum(a == "1" and b == "0" for a, b in zip(A, B)) b_only = sum(a == "0" and b == "1" for a, b in zip(A, B)) neither = sum(a == "0" and b == "0" for a, b in zip(A, B)) print(f"both right {both}, A right B wrong {a_only}, A wrong B right {b_only}, both wrong {neither}") pa, pb = A.count("1") / N, B.count("1") / N print(f"accuracy A = {A.count('1')}/{N} = {pa:.1%}, B = {B.count('1')}/{N} = {pb:.1%}") # first ten questions, for the table shown in the exercise print("first ten questions:", list(zip(A[:10], B[:10]))) # separate standard errors and intervals se_a = math.sqrt(pa * (1 - pa) / N) se_b = math.sqrt(pb * (1 - pb) / N) print(f"SE A = {se_a:.4f}, CI A = [{pa - 1.96 * se_a:.1%}, {pa + 1.96 * se_a:.1%}]") print(f"SE B = {se_b:.4f}, CI B = [{pb - 1.96 * se_b:.1%}, {pb + 1.96 * se_b:.1%}]") print(f"SE A + SE B = {se_a + se_b:.4f}") se_unpaired = math.sqrt(se_a ** 2 + se_b ** 2) diff = pa - pb # a difference of two accuracies is in percentage points, not percent print(f"difference = {100 * diff:.1f} points; unpaired SE = {se_unpaired:.4f}, " f"CI = [{100 * (diff - 1.96 * se_unpaired):.1f}, {100 * (diff + 1.96 * se_unpaired):.1f}] points") # paired analysis: per-question differences +1, 0, -1 d = [int(a) - int(b) for a, b in zip(A, B)] mean_d = sum(d) / N var_d = sum((x - mean_d) ** 2 for x in d) / N # divide by n (plug-in) var_formula = (a_only + b_only) / N - ((a_only - b_only) / N) ** 2 se_paired = math.sqrt(var_d / N) print(f"paired: mean diff = ({a_only}-{b_only})/{N} = {mean_d:.3f}; var of d = {var_d:.4f} " f"(formula (b+c)/n - ((b-c)/n)^2 = {var_formula:.4f}); SE = {se_paired:.4f}") print(f" paired CI = [{100 * (mean_d - 1.96 * se_paired):.1f}, {100 * (mean_d + 1.96 * se_paired):.1f}] points") var_d1 = sum((x - mean_d) ** 2 for x in d) / (N - 1) print(f" (with n-1 instead of n: SE = {math.sqrt(var_d1 / N):.4f})") corr_num = sum((int(a) - pa) * (int(b) - pb) for a, b in zip(A, B)) / N corr = corr_num / math.sqrt(pa * (1 - pa) * pb * (1 - pb)) print(f" correlation of A and B scores = {corr:.2f}") def mcnemar_exact(b, c): """Two-sided exact McNemar p-value: under 'equally good', b ~ Binomial(b + c, 1/2).""" n = b + c k = max(b, c) tail = sum(math.comb(n, i) for i in range(k, n + 1)) / 2 ** n return min(1.0, 2 * tail) print(f"McNemar exact p (b={a_only}, c={b_only}) = {mcnemar_exact(a_only, b_only):.3f}") chi2 = (abs(a_only - b_only) - 1) ** 2 / (a_only + b_only) p_chi2 = math.erfc(math.sqrt(chi2 / 2)) # chi-square with 1 df print(f"McNemar chi-square with continuity correction = {chi2:.3f}, p = {p_chi2:.3f}") # paired bootstrap (same code as in the exercise solution) rng = random.Random(0) reps = 10000 boot = [] for _ in range(reps): idx = [rng.randrange(N) for _ in range(N)] boot.append(sum(d[i] for i in idx) / N) boot.sort() print(f"paired bootstrap ({reps} resamples, seed 0): 2.5% = {boot[int(0.025 * reps)]:.3f}, " f"97.5% = {boot[int(0.975 * reps) - 1]:.3f}") # how many questions would a 2-point difference need, at this disagreement rate? r = (a_only + b_only) / N need = 1.96 ** 2 * (r - 0.02 ** 2) / 0.02 ** 2 print(f"disagreement rate {r:.0%}: n for paired half-width <= 2 points = {need:.0f}") print("\n--- section 3.4: what a 2-point gap on 200 questions needs ---") for b, c in [(4, 0), (5, 1), (6, 2), (10, 6), (18, 14), (30, 26)]: m = (b - c) / N se = math.sqrt(((b + c) / N - m ** 2) / N) print(f"b={b:2d} c={c:2d}: diff {100 * m:.1f} points, paired SE {se:.4f}, CI [{100 * (m - 1.96 * se):.1f}, {100 * (m + 1.96 * se):.1f}] points, " f"exact McNemar p = {mcnemar_exact(b, c):.3f}") print("\n--- section 3.4: constructed 2000-question illustration ---") n2, b2, c2 = 2000, 70, 30 pa2, pb2 = 0.71, 0.69 sa2, sb2 = math.sqrt(pa2 * (1 - pa2) / n2), math.sqrt(pb2 * (1 - pb2) / n2) print(f"A 71%: SE {sa2:.4f}, CI [{pa2 - 1.96 * sa2:.1%}, {pa2 + 1.96 * sa2:.1%}]") print(f"B 69%: SE {sb2:.4f}, CI [{pb2 - 1.96 * sb2:.1%}, {pb2 + 1.96 * sb2:.1%}]") su2 = math.sqrt(sa2 ** 2 + sb2 ** 2) print(f"unpaired SE {su2:.4f}, CI [{100 * (0.02 - 1.96 * su2):.1f}, {100 * (0.02 + 1.96 * su2):.1f}] points") m2 = (b2 - c2) / n2 sp2 = math.sqrt(((b2 + c2) / n2 - m2 ** 2) / n2) print(f"paired: diff {100 * m2:.1f} points, SE {sp2:.4f}, CI [{100 * (m2 - 1.96 * sp2):.1f}, {100 * (m2 + 1.96 * sp2):.1f}] points") print(f"McNemar exact p (70 vs 30) = {mcnemar_exact(b2, c2):.6f}") print("\n--- section 3.4: questions needed so that the 95% half-width of a 2-point gap is <= 2 points ---") for r in (0.05, 0.10, 0.20, 0.30): print(f"paired, disagreement rate {r:.0%}: n >= {1.96 ** 2 * (r - 0.0004) / 0.0004:.0f}") print(f"unpaired (two models near 70%): n >= {1.96 ** 2 * (0.71 * 0.29 + 0.69 * 0.31) / 0.0004:.0f}")