minterp

tests/test_probed.py

122 lines · View on GitHub · Linked from 2. Probe

1"""
2What docs/02-probe.md claims, checked against the record under runs/probe/.
3The readers have a closed form and run in float64 on the CPU, so a repeated
4run gives the same file; if a number here moves, the note has to follow.
5"""
6
7import json
8
9import numpy as np
10import pytest
11import torch
12
13from conftest import needs_model
14
15ROLES, TARGETS = ("A", "B", "S", "S+1", "end"), ("due", "acts", "first", "second", "control")
16
17
18def run(seed):
19    with open(f"runs/probe/probe-{seed}.json") as f:
20        return json.load(f)
21
22
23def score(seed, rule, target, role):
24    """Percent right, mean over the held-out patterns: (versions, depth)."""
25    a = np.array(run(seed)[rule]["accuracy"])[:, :, :, ROLES.index(role), TARGETS.index(target)]
26    return np.round(100 * a.mean(0)).astype(int)
27
28
29def test_the_design():
30    for seed in ("seed0", "seed1"):
31        r = run(seed)
32        assert r["pairs"] == {"held out": 44, "trained": 88}  # ordered pairs: 22 and 44 of the 66
33        assert (r["new"]["train rows"], r["new"]["test rows"]) == (704, 88)
34        assert tuple(r["roles"]) == ROLES and tuple(r["targets"]) == TARGETS and len(r["names"]) == 12
35        assert r["penalties"] == [0.001, 0.01, 0.1, 1.0, 10.0, 100.0, 1000.0]
36
37
38@pytest.mark.parametrize("seed", ["seed0", "seed1"])
39def test_the_two_free_checks(seed):
40    # After the embedding the last position is " to" in every prompt: nothing can be read.
41    for target in TARGETS[:4]:
42        assert score(seed, "new", target, "end")[:, 0].tolist() == [7, 7]
43    # Before the acting name the two versions are the same tokens: at most one in two.
44    a = np.array(run(seed)["new"]["accuracy"])[..., TARGETS.index("due")].mean(0)  # (versions, depth, roles)
45    for role in ("A", "B"):
46        assert (a[:, :, ROLES.index(role)].mean(0) <= 0.5 + 1e-9).all()
47    assert a[0, 1, 0] == 0.0 and a[1, 1, 0] == 1.0  # at the first name: wrong on clean, right on flipped
48
49
50def test_where_the_due_name_can_be_read():
51    assert score("seed0", "new", "due", "end").tolist() == [[7, 8, 12, 32, 27, 33, 23, 31, 95], [7, 7, 10, 15, 15, 18, 17, 28, 73]]
52    assert score("seed0", "new", "due", "S").tolist() == [[0, 6, 11, 53, 95, 80, 68, 60, 40], [0, 6, 11, 58, 90, 74, 65, 60, 36]]
53    assert score("seed1", "new", "due", "end").tolist() == [[7, 7, 70, 44, 39, 27, 21, 83, 87], [7, 7, 10, 10, 9, 11, 14, 13, 65]]
54    assert score("seed1", "new", "due", "S").tolist() == [[0, 31, 60, 76, 80, 69, 67, 66, 51], [0, 36, 45, 62, 71, 68, 61, 60, 46]]
55    # The spread over the held-out patterns at the acting name after block 4.
56    for seed, low, high in (("seed0", 77, 100), ("seed1", 48, 95)):
57        folds = 100 * np.array(run(seed)["new"]["accuracy"])[:, :, 4, ROLES.index("S"), 0]
58        assert (round(folds.min()), round(folds.max())) == (low, high)
59
60
61def test_what_the_last_position_holds_before_the_final_block():
62    assert score("seed0", "new", "acts", "end").tolist() == [[7, 9, 8, 65, 52, 48, 28, 57, 50], [7, 9, 7, 70, 59, 50, 27, 55, 60]]
63    assert score("seed1", "new", "second", "end").tolist() == [[7, 8, 69, 47, 38, 33, 23, 94, 87], [7, 8, 74, 51, 42, 35, 31, 89, 81]]
64
65
66def test_on_new_pairs_the_reader_finds_no_more_than_the_lens():
67    lens = lambda seed: np.round(100 * np.array(run(seed)["new"]["lens"]).mean(0)).astype(int)
68    assert lens("seed0").tolist() == [[9, 7, 11, 39, 37, 32, 30, 29, 96], [9, 7, 9, 20, 19, 19, 19, 19, 75]]
69    assert lens("seed1").tolist() == [[9, 9, 62, 49, 49, 47, 47, 66, 88], [9, 9, 7, 9, 11, 12, 10, 8, 70]]
70
71    def against(seed, rule, depths):
72        reader = np.array(run(seed)[rule]["accuracy"])[:, :, depths, ROLES.index("end"), 0].mean(0)
73        other = np.array(run(seed)[rule]["lens"]).mean(0)[:, depths]
74        return int((reader > other + 1e-9).sum()), int((reader < other - 1e-9).sum()), 100 * (reader - other).mean()
75
76    before = slice(1, 8)  # the seven depths before the last block
77    new = [against(seed, "new", before) for seed in ("seed0", "seed1")]
78    assert (new[0][0] + new[1][0], new[0][1] + new[1][1]) == (13, 15)  # of 28 comparisons
79    assert [round(n[2], 1) for n in new] == [-1.4, -2.2]
80    # After the final block, where the answer is there, the reader is still behind in all four rows.
81    assert [against(seed, "new", slice(8, 9))[1] for seed in ("seed0", "seed1")] == [2, 2]
82    # On pairs it trained on, the same reader is ahead in 20 of the 28.
83    seen = [against(seed, "seen", before) for seed in ("seed0", "seed1")]
84    assert (seen[0][0] + seen[1][0], seen[0][1] + seen[1][1]) == (20, 7)
85
86
87def test_a_reader_can_do_the_work():
88    for seed in ("seed0", "seed1"):
89        tokens = run(seed)["tokens only"]
90        assert tokens["first, second and acts"] == {"clean": 1.0, "flipped": 1.0, "both": 1.0, "tied": False}
91        # Without the acting name the two versions are one input: right on one of them, and the two names tie.
92        assert tokens["first and second"]["both"] == 0.5 and tokens["first and second"]["tied"]
93
94
95def test_a_reader_can_remember():
96    made_up = lambda seed, rule: 100 * np.array(run(seed)[rule]["accuracy"])[..., 4].mean((0, 1))  # (depth, roles)
97    b, end = ROLES.index("B"), ROLES.index("end")
98    for seed, block1, later, last, anywhere in (("seed0", 67, (37, 59), (14, 24), (5, 15)), ("seed1", 88, (37, 63), (14, 26), (5, 14))):
99        seen, new = made_up(seed, "seen"), made_up(seed, "new")
100        assert round(seen[0, end]) == 15 and round(new[0, end]) == 11  # what always naming the commonest label gives
101        assert round(seen[1, b]) == block1 and (round(seen[2:, b].min()), round(seen[2:, b].max())) == later
102        assert (round(seen[:, end].min()), round(seen[:, end].max())) == last
103        assert (round(new.min()), round(new.max())) == anywhere
104    # The due name at the acting name, pairs it trained on against new pairs.
105    at = lambda seed, rule, depth: score(seed, rule, "due", "S")[:, depth].tolist()
106    assert at("seed0", "seen", 2) == [53, 60] and at("seed0", "new", 2) == [11, 11]
107    assert at("seed1", "seen", 2) == [79, 69] and at("seed1", "new", 2) == [60, 45]
108    assert at("seed0", "seen", 4) == [95, 92] and at("seed0", "new", 4) == [95, 90]
109    assert (run("seed0")["seen"]["train rows"], run("seed0")["new"]["train rows"]) == (1056, 704)
110
111
112@needs_model
113def test_the_script_reproduces_the_record(tmp_path, monkeypatch):
114    """Two runs give the same file: nothing in the fit depends on chance or on the machine's GPU."""
115    import os
116    import subprocess
117    import sys
118
119    env = dict(os.environ, MINTERP_RUNS=str(tmp_path))
120    subprocess.run([sys.executable, "scripts/probe.py"], check=True, env=env, capture_output=True)
121    again = json.load(open(tmp_path / "probe" / "probe-seed0.json"))
122    assert torch.allclose(torch.tensor(again["new"]["accuracy"]), torch.tensor(run("seed0")["new"]["accuracy"]))