"""Section 23.3 and exercise 23.2: choosing the best of many methods on one test set (a simulation). Every method has the same true accuracy, 70%. Each is scored on the same test set of N questions; every question is answered correctly with probability 0.7, independently. We keep the method with the highest test score. Then we score the winner once more on a fresh N-question set it was not chosen on. Repeat 10,000 times with a fixed seed and average. Run: python3 tools/exercises/ch23_benchmark_reuse.py [N] (default N = 200) """ import math import random import statistics import sys SEED = 23 TRUE_ACC = 0.70 N_QUESTIONS = int(sys.argv[1]) if len(sys.argv) > 1 else 200 TRIALS = 10_000 rng = random.Random(SEED) def score(): return rng.binomialvariate(N_QUESTIONS, TRUE_ACC) / N_QUESTIONS se = math.sqrt(TRUE_ACC * (1 - TRUE_ACC) / N_QUESTIONS) print(f"{N_QUESTIONS} questions; standard error of one score: " f"sqrt(0.7*0.3/{N_QUESTIONS}) = {se:.4f} ({se * 100:.1f} percentage points)") for n_methods in (1, 5, 20): winner_scores, fresh_scores = [], [] for _ in range(TRIALS): winner_scores.append(max(score() for _ in range(n_methods))) fresh_scores.append(score()) # the winner's true accuracy is still 0.7 print(f"{n_methods:2d} methods: winner's score on the reused test set {statistics.mean(winner_scores):.1%}, " f"on a fresh test set {statistics.mean(fresh_scores):.1%}")