tests/test_probed.py
122 lines · View on GitHub · Linked from 2. Probe
1"""
2What docs/02-probe.md claims, checked against the record under runs/probe/.
3The readers have a closed form and run in float64 on the CPU, so a repeated
4run gives the same file; if a number here moves, the note has to follow.
5"""
6
7import json
8
9import numpy as np
10import pytest
11import torch
12
13from conftest import needs_model
14
15ROLES, TARGETS = ("A", "B", "S", "S+1", "end"), ("due", "acts", "first", "second", "control")
16
17
18def run(seed):
19 with open(f"runs/probe/probe-{seed}.json") as f:
20 return json.load(f)
21
22
23def score(seed, rule, target, role):
24 """Percent right, mean over the held-out patterns: (versions, depth)."""
25 a = np.array(run(seed)[rule]["accuracy"])[:, :, :, ROLES.index(role), TARGETS.index(target)]
26 return np.round(100 * a.mean(0)).astype(int)
27
28
29def test_the_design():
30 for seed in ("seed0", "seed1"):
31 r = run(seed)
32 assert r["pairs"] == {"held out": 44, "trained": 88}
33 assert (r["new"]["train rows"], r["new"]["test rows"]) == (704, 88)
34 assert tuple(r["roles"]) == ROLES and tuple(r["targets"]) == TARGETS and len(r["names"]) == 12
35 assert r["penalties"] == [0.001, 0.01, 0.1, 1.0, 10.0, 100.0, 1000.0]
36
37
38@pytest.mark.parametrize("seed", ["seed0", "seed1"])
39def test_the_two_free_checks(seed):
40
41 for target in TARGETS[:4]:
42 assert score(seed, "new", target, "end")[:, 0].tolist() == [7, 7]
43
44 a = np.array(run(seed)["new"]["accuracy"])[..., TARGETS.index("due")].mean(0)
45 for role in ("A", "B"):
46 assert (a[:, :, ROLES.index(role)].mean(0) <= 0.5 + 1e-9).all()
47 assert a[0, 1, 0] == 0.0 and a[1, 1, 0] == 1.0
48
49
50def test_where_the_due_name_can_be_read():
51 assert score("seed0", "new", "due", "end").tolist() == [[7, 8, 12, 32, 27, 33, 23, 31, 95], [7, 7, 10, 15, 15, 18, 17, 28, 73]]
52 assert score("seed0", "new", "due", "S").tolist() == [[0, 6, 11, 53, 95, 80, 68, 60, 40], [0, 6, 11, 58, 90, 74, 65, 60, 36]]
53 assert score("seed1", "new", "due", "end").tolist() == [[7, 7, 70, 44, 39, 27, 21, 83, 87], [7, 7, 10, 10, 9, 11, 14, 13, 65]]
54 assert score("seed1", "new", "due", "S").tolist() == [[0, 31, 60, 76, 80, 69, 67, 66, 51], [0, 36, 45, 62, 71, 68, 61, 60, 46]]
55
56 for seed, low, high in (("seed0", 77, 100), ("seed1", 48, 95)):
57 folds = 100 * np.array(run(seed)["new"]["accuracy"])[:, :, 4, ROLES.index("S"), 0]
58 assert (round(folds.min()), round(folds.max())) == (low, high)
59
60
61def test_what_the_last_position_holds_before_the_final_block():
62 assert score("seed0", "new", "acts", "end").tolist() == [[7, 9, 8, 65, 52, 48, 28, 57, 50], [7, 9, 7, 70, 59, 50, 27, 55, 60]]
63 assert score("seed1", "new", "second", "end").tolist() == [[7, 8, 69, 47, 38, 33, 23, 94, 87], [7, 8, 74, 51, 42, 35, 31, 89, 81]]
64
65
66def test_on_new_pairs_the_reader_finds_no_more_than_the_lens():
67 lens = lambda seed: np.round(100 * np.array(run(seed)["new"]["lens"]).mean(0)).astype(int)
68 assert lens("seed0").tolist() == [[9, 7, 11, 39, 37, 32, 30, 29, 96], [9, 7, 9, 20, 19, 19, 19, 19, 75]]
69 assert lens("seed1").tolist() == [[9, 9, 62, 49, 49, 47, 47, 66, 88], [9, 9, 7, 9, 11, 12, 10, 8, 70]]
70
71 def against(seed, rule, depths):
72 reader = np.array(run(seed)[rule]["accuracy"])[:, :, depths, ROLES.index("end"), 0].mean(0)
73 other = np.array(run(seed)[rule]["lens"]).mean(0)[:, depths]
74 return int((reader > other + 1e-9).sum()), int((reader < other - 1e-9).sum()), 100 * (reader - other).mean()
75
76 before = slice(1, 8)
77 new = [against(seed, "new", before) for seed in ("seed0", "seed1")]
78 assert (new[0][0] + new[1][0], new[0][1] + new[1][1]) == (13, 15)
79 assert [round(n[2], 1) for n in new] == [-1.4, -2.2]
80
81 assert [against(seed, "new", slice(8, 9))[1] for seed in ("seed0", "seed1")] == [2, 2]
82
83 seen = [against(seed, "seen", before) for seed in ("seed0", "seed1")]
84 assert (seen[0][0] + seen[1][0], seen[0][1] + seen[1][1]) == (20, 7)
85
86
87def test_a_reader_can_do_the_work():
88 for seed in ("seed0", "seed1"):
89 tokens = run(seed)["tokens only"]
90 assert tokens["first, second and acts"] == {"clean": 1.0, "flipped": 1.0, "both": 1.0, "tied": False}
91
92 assert tokens["first and second"]["both"] == 0.5 and tokens["first and second"]["tied"]
93
94
95def test_a_reader_can_remember():
96 made_up = lambda seed, rule: 100 * np.array(run(seed)[rule]["accuracy"])[..., 4].mean((0, 1))
97 b, end = ROLES.index("B"), ROLES.index("end")
98 for seed, block1, later, last, anywhere in (("seed0", 67, (37, 59), (14, 24), (5, 15)), ("seed1", 88, (37, 63), (14, 26), (5, 14))):
99 seen, new = made_up(seed, "seen"), made_up(seed, "new")
100 assert round(seen[0, end]) == 15 and round(new[0, end]) == 11
101 assert round(seen[1, b]) == block1 and (round(seen[2:, b].min()), round(seen[2:, b].max())) == later
102 assert (round(seen[:, end].min()), round(seen[:, end].max())) == last
103 assert (round(new.min()), round(new.max())) == anywhere
104
105 at = lambda seed, rule, depth: score(seed, rule, "due", "S")[:, depth].tolist()
106 assert at("seed0", "seen", 2) == [53, 60] and at("seed0", "new", 2) == [11, 11]
107 assert at("seed1", "seen", 2) == [79, 69] and at("seed1", "new", 2) == [60, 45]
108 assert at("seed0", "seen", 4) == [95, 92] and at("seed0", "new", 4) == [95, 90]
109 assert (run("seed0")["seen"]["train rows"], run("seed0")["new"]["train rows"]) == (1056, 704)
110
111
112@needs_model
113def test_the_script_reproduces_the_record(tmp_path, monkeypatch):
114 """Two runs give the same file: nothing in the fit depends on chance or on the machine's GPU."""
115 import os
116 import subprocess
117 import sys
118
119 env = dict(os.environ, MINTERP_RUNS=str(tmp_path))
120 subprocess.run([sys.executable, "scripts/probe.py"], check=True, env=env, capture_output=True)
121 again = json.load(open(tmp_path / "probe" / "probe-seed0.json"))
122 assert torch.allclose(torch.tensor(again["new"]["accuracy"]), torch.tensor(run("seed0")["new"]["accuracy"]))