scripts/patch.py
203 lines · View on GitHub · Linked from 3. Patch
1"""
2Step 3: which places does the answer depend on? See docs/03-patch.md.
3
4Four experiments, each per sentence pattern, on every ordered pair of names.
5
6travel The stream at one depth and one group of positions, moved between
7 the clean and the flipped prompt, in both directions:
8 carry clean into the flipped run: does the clean answer come along?
9 break flipped into the clean run: does the clean answer go?
10 Measured on the logit of B minus the logit of A, as the share of
11 the way from the target run's value to the source run's.
12
13name The stream moved from the prompt with C in place of B into the
14 clean run, measured on the logit of C minus the logit of B: where
15 is the second name, at each depth? And its mirror: the stream of
16 the prompt with C in place of A moved into the flipped run,
17 measured on the logit of C minus the logit of A.
18
19parts Every head and MLP, its output moved between the clean and the
20 flipped prompt: at all positions, at the last one only, at all but
21 the last, and at the acting name only. The four parts with the
22 largest share are also moved two at a time.
23
24choice Every head and MLP of the clean run replaced in four ways: by the
25 flipped prompt's, by the neutral prompt's, by the average over the
26 three versions, and by nothing. Measured as the change of the logit
27 difference, in logits.
28
29 uv run scripts/patch.py
30 uv run scripts/patch.py --model seed1
31 uv run scripts/patch.py --model qwen --stream-only # step 8: travel and name only, minutes
32 uv run scripts/patch.py --model qwen # step 8: with parts and choice, hours
33"""
34
35import argparse
36
37import torch
38
39from minterp import patch, tasks
40from minterp.common import load, rounded, save
41
42parser = argparse.ArgumentParser()
43parser.add_argument("--model", default="seed0", help="seed0, seed1, qwen, or the path of a checkpoint")
44parser.add_argument("--stream-only", action="store_true", help="travel and name only, without the heads and MLPs: for a larger model (step 8)")
45args = parser.parse_args()
46
47model, tok, meta = load(args.model)
48device = next(model.parameters()).device
49n, heads = model.cfg.n_layers, model.cfg.n_heads
50PARTS = [("heads", b, h) for b in range(1, n + 1) for h in range(heads)] + [("mlp", b, None) for b in range(1, n + 1)]
51TRAVEL = ("before", "A", "B", "S", "S+1", "between", "end", "all")
52NAME = ("B", "B to S", "S", "S+1", "between", "end")
53FIRST = ("A", "A to S", "S", "S+1", "between", "end")
54SCOPES = ("all", "last", "not last", "S")
55CHOICES = ("flipped", "neutral", "mean", "zero")
56result = {"model": meta, "travel groups": TRAVEL, "name groups": NAME, "first name groups": FIRST, "scopes": SCOPES, "choices": CHOICES, "templates": []}
57
58
59def grid(parts_values):
60 """A value per part -> (blocks, heads + 1), the MLP in the last column."""
61 t = torch.tensor(parts_values)
62 return torch.cat([t[: n * heads].view(n, heads), t[n * heads:, None]], dim=1)
63
64
65def swap_part(part, source, positions=None):
66 kind, block, head = part
67 return {(kind, block): patch.swap(source[kind, block], positions=positions, head=head)}
68
69
70for batch in tasks.batches(tok, device):
71 p, T = batch.pos, batch.clean.shape[1]
72 ld = lambda logits: tasks.logit_diff(logits.float(), batch.right, batch.wrong)
73 cb = lambda logits: tasks.logit_diff(logits.float(), batch.third, batch.right)
74 ca = lambda logits: tasks.logit_diff(logits.float(), batch.third, batch.wrong)
75 rms = lambda x: x.float().pow(2).mean(-1).sqrt().mean().item()
76 store, value = {}, {}
77 for version in ("clean", "flipped", "neutral", "other", "other_first"):
78 store[version], logits = patch.record(model, getattr(batch, version))
79 value[version] = logits
80 entry = {"template": batch.template, "logit_diff": {v: ld(value[v]).mean().item() for v in ("clean", "flipped", "neutral")}}
81
82
83 groups = {
84 "before": [i for i in range(p["S"]) if i not in (p["A"], p["B"])], "A": [p["A"]], "B": [p["B"]], "S": [p["S"]],
85 "S+1": [p["S"] + 1], "between": list(range(p["S"] + 2, p["end"])), "end": [p["end"]], "all": list(range(T)),
86 }
87 entry["travel"] = {}
88 for name, src, dst in (("carry", "clean", "flipped"), ("break", "flipped", "clean")):
89 entry["travel"][name] = rounded(torch.tensor([[
90 patch.share(ld(patch.run(model, getattr(batch, dst), {("resid", d): patch.swap(store[src]["resid", d], positions=groups[g])})),
91 ld(value[dst]), ld(value[src]))
92 for g in TRAVEL] for d in range(n + 1)]), 3)
93
94
95 groups = {
96 "B": [p["B"]], "B to S": list(range(p["B"] + 1, p["S"])), "S": [p["S"]], "S+1": [p["S"] + 1],
97 "between": list(range(p["S"] + 2, p["end"])), "end": [p["end"]],
98 }
99 entry["name"] = rounded(torch.tensor([[
100 patch.share(cb(patch.run(model, batch.clean, {("resid", d): patch.swap(store["other"]["resid", d], positions=groups[g])})),
101 cb(value["clean"]), cb(value["other"]))
102 for g in NAME] for d in range(n + 1)]), 3)
103 entry["name measure"] = {"clean": cb(value["clean"]).mean().item(), "other": cb(value["other"]).mean().item()}
104 groups = {
105 "A": [p["A"]], "A to S": list(range(p["A"] + 1, p["S"])), "S": [p["S"]], "S+1": [p["S"] + 1],
106 "between": list(range(p["S"] + 2, p["end"])), "end": [p["end"]],
107 }
108 entry["first name"] = rounded(torch.tensor([[
109 patch.share(ca(patch.run(model, batch.flipped, {("resid", d): patch.swap(store["other_first"]["resid", d], positions=groups[g])})),
110 ca(value["flipped"]), ca(value["other_first"]))
111 for g in FIRST] for d in range(n + 1)]), 3)
112 entry["first name measure"] = {"flipped": ca(value["flipped"]).mean().item(), "other": ca(value["other_first"]).mean().item()}
113
114 at = store["clean"]
115 entry["size at S"] = {"embedding": rms(at["resid", 0][:, p["S"]]), "after block 1": rms(at["resid", 1][:, p["S"]]),
116 "attention 1": rms(at["attn", 1][:, p["S"]]), "MLP 1": rms(at["mlp", 1][:, p["S"]])}
117
118 if args.stream_only:
119 result["templates"].append(entry)
120 print(batch.template, {k: round(v, 2) for k, v in entry["logit_diff"].items()})
121 continue
122
123
124 scopes = {"all": None, "last": [p["end"]], "not last": list(range(p["end"])), "S": [p["S"]]}
125 entry["parts"] = {}
126 for name, src, dst in (("carry", "clean", "flipped"), ("break", "flipped", "clean")):
127 entry["parts"][name] = {
128 scope: rounded(grid([
129 patch.share(ld(patch.run(model, getattr(batch, dst), swap_part(part, store[src], positions))), ld(value[dst]), ld(value[src]))
130 for part in PARTS]), 3)
131 for scope, positions in scopes.items()
132 }
133
134 entry["carry by part"] = [patch.share(ld(patch.run(model, batch.flipped, swap_part(part, store["clean"]))), ld(value["flipped"]), ld(value["clean"])) for part in PARTS]
135
136
137 three = [store[v] for v in ("clean", "flipped", "neutral")]
138 entry["choice"] = {}
139 for choice in CHOICES:
140 changes = []
141 for part in PARTS:
142 key = part[:2]
143 source = {"flipped": store["flipped"][key], "neutral": store["neutral"][key],
144 "mean": patch.mean(three, key).expand_as(store["clean"][key]), "zero": patch.zero(store["clean"][key])}[choice]
145 changes.append((ld(patch.run(model, batch.clean, swap_part(part, {key: source}))) - ld(value["clean"])).mean().item())
146 entry["choice"][choice] = rounded(grid(changes), 3)
147 result["templates"].append(entry)
148 print(batch.template, {k: round(v, 2) for k, v in entry["logit_diff"].items()})
149
150if not args.stream_only:
151
152 single = torch.tensor([t.pop("carry by part") for t in result["templates"]]).mean(0)
153 single[PARTS.index(("mlp", 1, None))] = 0
154 top = [PARTS[i] for i in single.abs().argsort(descending=True)[:4].tolist()]
155 name_of = lambda part: f"block {part[1]} " + ("MLP" if part[0] == "mlp" else f"head {part[2]}")
156 pairs = [(a, b) for i, a in enumerate(top) for b in top[i + 1:]]
157 together = {pair: [] for pair in pairs}
158 for batch in tasks.batches(tok, device):
159 ld = lambda logits: tasks.logit_diff(logits.float(), batch.right, batch.wrong)
160 clean, clean_logits = patch.record(model, batch.clean)
161 _, flipped_logits = patch.record(model, batch.flipped)
162 for a, b in pairs:
163 edits = {}
164 for part in (a, b):
165 key, edit = next(iter(swap_part(part, clean).items()))
166 edits[key] = (lambda x, first=edits[key], second=edit: second(first(x))) if key in edits else edit
167 together[a, b].append(patch.share(ld(patch.run(model, batch.flipped, edits)), ld(flipped_logits), ld(clean_logits)))
168 result["pairs"] = [{"parts": [name_of(a), name_of(b)], "each": [round(single[PARTS.index(a)].item(), 3), round(single[PARTS.index(b)].item(), 3)],
169 "together": round(sum(v) / len(v), 3)} for (a, b), v in together.items()]
170
171mean_of = lambda pick: torch.tensor([pick(t) for t in result["templates"]]).mean(0)
172show = lambda row: " ".join(f"{v:6.2f}" for v in row.tolist())
173for name in ("carry", "break"):
174 print(f"\ntravel, {name}: share of the gap, mean of the patterns (rows: after block; columns: {', '.join(TRAVEL)})")
175 for d, row in enumerate(mean_of(lambda t: t["travel"][name])):
176 print(f" {'emb' if d == 0 else d:>3} " + show(row))
177print(f"\nname: share of the way from B to C (columns: {', '.join(NAME)})")
178for d, row in enumerate(mean_of(lambda t: t["name"])):
179 print(f" {'emb' if d == 0 else d:>3} " + show(row))
180print(f"\nfirst name: share of the way from A to C (columns: {', '.join(FIRST)})")
181for d, row in enumerate(mean_of(lambda t: t["first name"])):
182 print(f" {'emb' if d == 0 else d:>3} " + show(row))
183if args.stream_only:
184 save(f"runs/patch/patch-{meta['name']}.json", result)
185 raise SystemExit(0)
186print("\ntwo parts moved together, against the sum of their shares")
187for pair in result["pairs"]:
188 print(f" {pair['parts'][0]:16s} + {pair['parts'][1]:16s} together {pair['together']:.2f}, sum {sum(pair['each']):.2f}")
189for name in ("carry", "break"):
190 print(f"\nparts, {name}, all positions (rows: blocks; columns: heads, then the MLP)")
191 for b, row in enumerate(mean_of(lambda t: t["parts"][name]["all"]), start=1):
192 print(f" {b:>3} " + show(row))
193carry = {s: mean_of(lambda t, s=s: t["parts"]["carry"][s]) for s in SCOPES}
194choice = {c: mean_of(lambda t, c=c: t["choice"][c]) for c in CHOICES}
195ranked = sorted(((carry["all"][b, h].abs().item(), b, h) for b in range(n) for h in range(heads + 1)), reverse=True)[:10]
196print("\nthe ten parts with the largest carry, where they act, and the four replacements on the clean run (logits)")
197print(f" {'':16s} {'carry':>6} {'last':>6} {'before':>6} | " + " ".join(f"{c:>8}" for c in CHOICES))
198for _, b, h in ranked:
199 label = f"block {b + 1} " + ("MLP" if h == heads else f"head {h}")
200 print(f" {label:16s} {carry['all'][b, h]:6.2f} {carry['last'][b, h]:6.2f} {carry['not last'][b, h]:6.2f} | "
201 + " ".join(f"{choice[c][b, h]:+8.2f}" for c in CHOICES))
202
203save(f"runs/patch/patch-{meta['name']}.json", result)