scripts/probe.py
190 lines · View on GitHub · Linked from 2. Probe
1"""
2Step 2: where can the answer be read out? See docs/02-probe.md.
3
4For every depth (after the embedding and after every block) and five
5positions named by their role (A, B, S, the token after S, and the last
6one), this fits linear readers from the stream to four properties of the
7prompt, each a name out of twelve:
8
9 due the name that is due acts the name that acts
10 first the first name, A second the second name, B
11
12and to a fifth that is made up, `control`: a random name drawn once for
13every ordered pair, the same in both versions and all patterns. Nothing in
14the text determines it, so a reader can only get it right by remembering
15pairs it was trained on.
16
17Readers are trained on four sentence patterns and tested on the fifth, each
18pattern in turn, under two rules:
19
20 new tested on name pairs that were held out of training as well
21 seen tested on the same pairs it was trained on
22
23It also records, on the same test prompts, how often the lens of step 1
24ranks the due name first among the twelve; how well the due name is read by
25combining the readers for first, second and acts without training on it;
26and how well it can be read from the three tokens' identities alone, with
27no model at all.
28
29 uv run scripts/probe.py
30 uv run scripts/probe.py --model seed1
31"""
32
33import argparse
34import os
35
36import torch
37
38from minterp import lens, probe, tasks
39from minterp.common import load, rounded, save
40from minterp.tap import Tap
41
42parser = argparse.ArgumentParser()
43parser.add_argument("--model", default="seed0", help="seed0, seed1, or the path of a checkpoint")
44args = parser.parse_args()
45
46ROLES = ("A", "B", "S", "S+1", "end")
47TARGETS = ("due", "acts", "first", "second", "control")
48VERSIONS = ("clean", "flipped")
49
50model, tok, meta = load(args.model, device="cpu")
51people = tasks.names(tok)
52k, depth = len(people), model.cfg.n_layers + 1
53name_ids = torch.tensor([tok.encode_ordinary(" " + n)[0] for n in people])
54batches = tasks.batches(tok, people=people)
55
56
57streams, picks = [], []
58for batch in batches:
59 at = [batch.pos["A"], batch.pos["B"], batch.pos["S"], batch.pos["S"] + 1, batch.pos["end"]]
60 per_version, per_pick = [], []
61 for version in VERSIONS:
62 with Tap(model, keep=("resid",)) as tap, torch.no_grad():
63 model(getattr(batch, version))
64 stream = torch.stack([tap["resid", i][:, at] for i in range(depth)], dim=1)
65 per_version.append(stream)
66 per_pick.append(lens.decode(model, stream[:, :, -1])[..., name_ids].argmax(-1))
67 streams.append(torch.stack(per_version))
68 picks.append(torch.stack(per_pick))
69X = torch.stack(streams).double()
70picks = torch.stack(picks)
71patterns, prompts = X.shape[0], X.shape[2]
72
73
74index = {n: i for i, n in enumerate(people)}
75first = torch.tensor([index[a] for a, _, _ in batches[0].names])
76second = torch.tensor([index[b] for _, b, _ in batches[0].names])
77control = torch.randint(0, k, (prompts,), generator=torch.Generator().manual_seed(0)).repeat(2, 1)
78labels = torch.stack([
79 torch.stack([second, first]),
80 torch.stack([first, second]),
81 torch.stack([first, first]),
82 torch.stack([second, second]),
83 control,
84])
85
86
87held = (first + second) % 3 == 0
88assert set(first[~held].tolist()) == set(second[~held].tolist()) == set(range(k)), "a name never trains"
89
90
91def rows(which, mask):
92 """The chosen patterns and pairs as rows: streams (places, rows, dim), labels, pattern and version of every row."""
93 x = X[which][:, :, mask]
94 p, v, m = x.shape[:3]
95 x = x.permute(3, 4, 0, 1, 2, 5).reshape(depth * len(ROLES), p * v * m, -1)
96 lab = labels[:, :, mask][:, None].expand(-1, p, -1, -1).reshape(len(TARGETS), -1)
97 group = torch.tensor(which)[:, None, None].expand(p, v, m).reshape(-1)
98 version = torch.arange(v)[None, :, None].expand(p, v, m).reshape(-1)
99 return x, lab, group, version
100
101
102def by_version(right, version):
103 """(places, rows, ...) of hits -> (versions, depth, roles, ...) of accuracies."""
104 out = torch.stack([right[:, version == v].double().mean(1) for v in range(len(VERSIONS))])
105 return out.unflatten(1, (depth, len(ROLES)))
106
107
108result = {
109 "model": meta, "names": people, "roles": ROLES, "targets": TARGETS, "versions": VERSIONS, "penalties": probe.LAMS,
110 "pairs": {"held out": int(held.sum()), "trained": int((~held).sum())},
111}
112every = torch.ones(prompts, dtype=torch.bool)
113for rule, train_mask, test_mask in (("new", ~held, held), ("seen", every, every)):
114 acc, composite, lens_acc, chosen = [], [], [], []
115 for fold in range(patterns):
116 others = [p for p in range(patterns) if p != fold]
117 xtr, ltr, gtr, _ = rows(others, train_mask)
118 xte, lte, _, vte = rows([fold], test_mask)
119 out, pick = probe.read(xtr, ltr, gtr, xte, k)
120 acc.append(by_version(out.argmax(-1) == lte.T, vte))
121 chosen.append(pick.unflatten(0, (depth, len(ROLES))))
122
123 t = TARGETS.index
124 combined = out[:, :, t("first")] + out[:, :, t("second")] - out[:, :, t("acts")]
125 composite.append(by_version(combined.argmax(-1) == lte[t("due")], vte))
126 due = labels[t("due")][:, test_mask]
127 lens_acc.append((picks[fold][:, test_mask] == due[:, :, None]).double().mean(1))
128 result[rule] = {
129 "train rows": xtr.shape[1], "test rows": xte.shape[1],
130 "accuracy": rounded(torch.stack(acc)),
131 "composite": rounded(torch.stack(composite)),
132 "lens": rounded(torch.stack(lens_acc)),
133 "penalty": torch.stack(chosen)[..., 0].tolist(),
134 }
135
136
137eye = torch.eye(k, dtype=torch.float64)
138tokens = {
139 "first, second and acts": lambda v: torch.cat([eye[first], eye[second], eye[labels[1, v]]], dim=1),
140 "first and second": lambda v: torch.cat([eye[first], eye[second]], dim=1),
141}
142result["tokens only"] = {}
143for name, make in tokens.items():
144 feats = torch.stack([make(v) for v in range(2)])
145 xtr, xte = feats[:, ~held].reshape(1, -1, feats.shape[-1]), feats[:, held].reshape(1, -1, feats.shape[-1])
146 ltr, lte = labels[:1, :, ~held].reshape(1, -1), labels[0][:, held]
147 groups = torch.arange(xtr.shape[1]) % 4
148 out = probe.read(xtr, ltr, groups, xte, k)[0][0, :, 0]
149 right = (out.argmax(-1).view(2, -1) == lte).double()
150
151
152
153 top2 = out.topk(2, dim=-1).values
154 result["tokens only"][name] = {
155 "clean": right[0].mean().item(), "flipped": right[1].mean().item(), "both": right.mean().item(),
156 "tied": bool(((top2[:, 0] - top2[:, 1]).abs() < 1e-9).all()),
157 }
158
159path = save(f"runs/probe/probe-{meta['name']}.json", result)
160
161
162x, lab, _, _ = rows(list(range(patterns)), every)
163end = [d * len(ROLES) + ROLES.index("end") for d in range(depth)]
164mean, spread = x[end].mean(1, keepdim=True), x[end].std(1, keepdim=True).clamp_min(1e-8)
165W, b = probe.fit((x[end] - mean) / spread, probe.onehot(lab[:1], k), 0.1)
166os.makedirs("out/probe", exist_ok=True)
167torch.save({"W": W, "b": b, "mean": mean, "spread": spread, "names": people}, f"out/probe/due-{meta['name']}.pt")
168
169new, seen = torch.tensor(result["new"]["accuracy"]).mean(0), torch.tensor(result["seen"]["accuracy"]).mean(0)
170t, e = TARGETS.index, ROLES.index("end")
171show = lambda row: " ".join(f"{100 * v:4.0f}" for v in row.tolist())
172print(f"\nthe due name at the last position, percent right (chance {100 / k:.0f}), mean of {patterns} held-out patterns")
173print(" after block emb 1 2 3 4 5 6 7 8")
174for v, version in enumerate(VERSIONS):
175 print(f" reader, new pairs, {version:8s}" + show(new[v, :, e, t("due")]))
176for v, version in enumerate(VERSIONS):
177 print(f" lens, {version:8s} " + show(torch.tensor(result["new"]["lens"]).mean(0)[v]))
178for v, version in enumerate(VERSIONS):
179 print(f" combined readers, {version:8s} " + show(torch.tensor(result["new"]["composite"]).mean(0)[v, :, e]))
180for v, version in enumerate(VERSIONS):
181 print(f" seen pairs: due, {version:8s} " + show(seen[v, :, e, t("due")]))
182print(" seen pairs: control " + show(seen[:, :, e, t("control")].mean(0)))
183print(" new pairs: control " + show(new[:, :, e, t("control")].mean(0)))
184print("\nthe worse of the two versions, new pairs, by depth (rows) and position (columns: " + ", ".join(ROLES) + ")")
185for target in TARGETS[:4]:
186 print(f" {target}")
187 worse = new[:, :, :, t(target)].min(0).values
188 for d in range(depth):
189 print(f" {'emb' if d == 0 else d:>3} " + show(worse[d]))
190print("\nfrom the tokens' identities alone, no model:", result["tokens only"])