"""Chapter 3, section 3.5 and exercise 3.3: samples that are not independent (constructed examples). Part 1 (section 3.5): a question pool in which 40% of questions are always answered correctly, 20% always wrongly, and 40% correctly half the time. 200 questions are drawn; each is answered K times. Prints the standard error of the overall score for K = 1, 5, and unlimited, the wrong "pooled" standard error that treats 200 x 5 answers as 1000 independent questions, and a simulation check. Part 2 (section 3.5): 200 questions from 50 articles, 4 per article, at 71%: the two extremes. Part 3 (exercise 3.3): a report that pools 1000 answers, and a small clustered data set. Standard library only. """ import math import random import statistics print("--- part 1: resampling the same question ---") pool = [(0.4, 1.0), (0.2, 0.0), (0.4, 0.5)] # (share of questions, chance this question is answered correctly) mu = sum(w * x for w, x in pool) between = sum(w * (x - mu) ** 2 for w, x in pool) # variation between questions within = sum(w * x * (1 - x) for w, x in pool) # average variation within a question print(f"overall accuracy {mu:.2f}; between-question variance {between:.3f}; " f"average within-question variance {within:.3f}; total {between + within:.3f} = 0.6*0.4") n = 200 for K in (1, 5, 1000000): se = math.sqrt((between + within / K) / n) label = "unlimited" if K == 1000000 else K print(f"K = {label}: SE = {se:.4f}") print(f"pooled (wrong, 1000 'independent' answers): SE = {math.sqrt(0.6 * 0.4 / 1000):.4f}") print(f"1000 different questions, answered once each: SE = {math.sqrt((between + within) / 1000):.4f}") rng = random.Random(5) for K in (1, 5): scores = [] for _ in range(4000): total = 0.0 for _ in range(n): u = rng.random() x = 1.0 if u < 0.4 else (0.0 if u < 0.6 else 0.5) total += sum(rng.random() < x for _ in range(K)) / K scores.append(total / n) print(f"simulation, K = {K}: SD of overall score over 4000 repeats = {statistics.pstdev(scores):.4f}") print("\n--- part 2: questions drawn in clusters (50 articles x 4 questions, 71%) ---") p = 0.71 print(f"independent questions: SE = {math.sqrt(p * (1 - p) / 200):.4f}") print(f"perfectly correlated within an article (50 effective observations): SE = {math.sqrt(p * (1 - p) / 50):.4f}") print("\n--- part 3a: a report that pools 200 questions x 5 samples ---") groups = {1.0: 80, 0.8: 15, 0.6: 25, 0.4: 25, 0.2: 15, 0.0: 40} # question-level mean of 5 answers -> number of questions nq = sum(groups.values()) mean = sum(s * k for s, k in groups.items()) / nq var = sum(k * (s - mean) ** 2 for s, k in groups.items()) / nq print(f"questions {nq}, answers {5 * nq}, correct answers {round(5 * sum(s * k for s, k in groups.items()))}, mean {mean:.3f}") print(f"wrong pooled SE = sqrt(0.6*0.4/1000) = {math.sqrt(mean * (1 - mean) / (5 * nq)):.4f}") print(f"variance of question means = {var:.4f}, SD = {math.sqrt(var):.4f}, correct SE = SD/sqrt(200) = {math.sqrt(var / nq):.4f}") print(f" (if each question were scored 0/1 once: sqrt(0.24/200) = {math.sqrt(0.24 / 200):.4f})") print("\n--- part 3b: 10 articles x 4 questions ---") k = [4, 4, 1, 3, 4, 0, 4, 2, 4, 2] # correct answers per article m, n3 = 4, 4 * len(k) p3 = sum(k) / n3 naive = math.sqrt(p3 * (1 - p3) / n3) dev = [kc - m * p3 for kc in k] clustered = math.sqrt(sum(x * x for x in dev)) / n3 print(f"correct {sum(k)}/{n3} = {p3:.0%}; naive SE = {naive:.4f}, CI half-width {1.96 * naive:.1%}") print("per-article deviations k_c - 4*0.7:", [round(x, 1) for x in dev]) print(f"sum of squared deviations = {sum(x * x for x in dev):.2f}; clustered SE = {clustered:.4f}, " f"CI half-width {1.96 * clustered:.1%}; ratio {clustered / naive:.2f}") print(f"articles with all {m} right: {k.count(m)}; with all {m} wrong: {k.count(0)}") print(f"if all 4 questions of an article always agreed (10 observations): SE = {math.sqrt(p3 * (1 - p3) / 10):.4f}")