"""Chapter 31.4 and exercises 31.1 to 31.3: how much evidence a safety claim needs. Uses the zero-failure bound of section 3.7: after n independent clean tests from one distribution, the one-sided 95% upper bound on the failure rate is 1 - 0.05**(1/n). All scenario numbers below are constructed for teaching. """ import math ALPHA = 0.05 def upper_bound(n: int) -> float: return 1 - ALPHA ** (1 / n) def tests_needed(target: float) -> int: n = math.ceil(math.log(ALPHA) / math.log(1 - target)) while upper_bound(n) > target: n += 1 return n print("1. Clean independent tests needed for a claimed failure rate (section 31.4)") for target in (0.01, 0.001, 0.0001): print(f" rate below {target:.2%}: {tests_needed(target):,} tests") print() print("2. Two checks in a row, each missing 5% of bad changes (section 31.4)") miss = 0.05 independent = miss * miss print(f" independent misses: combined miss rate {independent:.2%}") print(f" the same 5% missed by both: combined miss rate {miss:.2%}") hard = 0.04 # share of bad changes that both checks always miss r = (miss - hard) / (1 - hard) # each check's independent miss rate on the other changes partial = hard + (1 - hard) * r * r print(f" {hard:.0%} hard cases missed by both, the rest independent:") print(f" each check misses {r:.4%} of the other cases; combined miss rate {partial:.2%}") print(f" seeded bad changes, all caught, needed to support {independent:.2%}: {tests_needed(independent):,}") print() print("3. Exercise 31.2: 500 red-team prompts with no privacy leak") print(f" upper bound {upper_bound(500):.2%}") print() print("4. Exercise 31.1 demonstration framework (constructed placeholders)") for n in (200, 3000): print(f" {n} clean tasks: upper bound {upper_bound(n):.2%}")