minterp

scripts/patch.py

203 lines · View on GitHub · Linked from 3. Patch

1"""
2Step 3: which places does the answer depend on? See docs/03-patch.md.
3
4Four experiments, each per sentence pattern, on every ordered pair of names.
5
6travel   The stream at one depth and one group of positions, moved between
7         the clean and the flipped prompt, in both directions:
8           carry   clean into the flipped run: does the clean answer come along?
9           break   flipped into the clean run: does the clean answer go?
10         Measured on the logit of B minus the logit of A, as the share of
11         the way from the target run's value to the source run's.
12
13name     The stream moved from the prompt with C in place of B into the
14         clean run, measured on the logit of C minus the logit of B: where
15         is the second name, at each depth? And its mirror: the stream of
16         the prompt with C in place of A moved into the flipped run,
17         measured on the logit of C minus the logit of A.
18
19parts    Every head and MLP, its output moved between the clean and the
20         flipped prompt: at all positions, at the last one only, at all but
21         the last, and at the acting name only. The four parts with the
22         largest share are also moved two at a time.
23
24choice   Every head and MLP of the clean run replaced in four ways: by the
25         flipped prompt's, by the neutral prompt's, by the average over the
26         three versions, and by nothing. Measured as the change of the logit
27         difference, in logits.
28
29    uv run scripts/patch.py
30    uv run scripts/patch.py --model seed1
31    uv run scripts/patch.py --model qwen --stream-only   # step 8: travel and name only, minutes
32    uv run scripts/patch.py --model qwen                 # step 8: with parts and choice, hours
33"""
34
35import argparse
36
37import torch
38
39from minterp import patch, tasks
40from minterp.common import load, rounded, save
41
42parser = argparse.ArgumentParser()
43parser.add_argument("--model", default="seed0", help="seed0, seed1, qwen, or the path of a checkpoint")
44parser.add_argument("--stream-only", action="store_true", help="travel and name only, without the heads and MLPs: for a larger model (step 8)")
45args = parser.parse_args()
46
47model, tok, meta = load(args.model)
48device = next(model.parameters()).device
49n, heads = model.cfg.n_layers, model.cfg.n_heads
50PARTS = [("heads", b, h) for b in range(1, n + 1) for h in range(heads)] + [("mlp", b, None) for b in range(1, n + 1)]
51TRAVEL = ("before", "A", "B", "S", "S+1", "between", "end", "all")
52NAME = ("B", "B to S", "S", "S+1", "between", "end")
53FIRST = ("A", "A to S", "S", "S+1", "between", "end")
54SCOPES = ("all", "last", "not last", "S")
55CHOICES = ("flipped", "neutral", "mean", "zero")
56result = {"model": meta, "travel groups": TRAVEL, "name groups": NAME, "first name groups": FIRST, "scopes": SCOPES, "choices": CHOICES, "templates": []}
57
58
59def grid(parts_values):
60    """A value per part -> (blocks, heads + 1), the MLP in the last column."""
61    t = torch.tensor(parts_values)
62    return torch.cat([t[: n * heads].view(n, heads), t[n * heads:, None]], dim=1)
63
64
65def swap_part(part, source, positions=None):
66    kind, block, head = part
67    return {(kind, block): patch.swap(source[kind, block], positions=positions, head=head)}
68
69
70for batch in tasks.batches(tok, device):
71    p, T = batch.pos, batch.clean.shape[1]
72    ld = lambda logits: tasks.logit_diff(logits.float(), batch.right, batch.wrong)  # B minus A
73    cb = lambda logits: tasks.logit_diff(logits.float(), batch.third, batch.right)  # C minus B
74    ca = lambda logits: tasks.logit_diff(logits.float(), batch.third, batch.wrong)  # C minus A
75    rms = lambda x: x.float().pow(2).mean(-1).sqrt().mean().item()
76    store, value = {}, {}
77    for version in ("clean", "flipped", "neutral", "other", "other_first"):
78        store[version], logits = patch.record(model, getattr(batch, version))
79        value[version] = logits
80    entry = {"template": batch.template, "logit_diff": {v: ld(value[v]).mean().item() for v in ("clean", "flipped", "neutral")}}
81
82    # travel
83    groups = {
84        "before": [i for i in range(p["S"]) if i not in (p["A"], p["B"])], "A": [p["A"]], "B": [p["B"]], "S": [p["S"]],
85        "S+1": [p["S"] + 1], "between": list(range(p["S"] + 2, p["end"])), "end": [p["end"]], "all": list(range(T)),
86    }
87    entry["travel"] = {}
88    for name, src, dst in (("carry", "clean", "flipped"), ("break", "flipped", "clean")):
89        entry["travel"][name] = rounded(torch.tensor([[
90            patch.share(ld(patch.run(model, getattr(batch, dst), {("resid", d): patch.swap(store[src]["resid", d], positions=groups[g])})),
91                        ld(value[dst]), ld(value[src]))
92            for g in TRAVEL] for d in range(n + 1)]), 3)  # (depth, groups)
93
94    # name
95    groups = {
96        "B": [p["B"]], "B to S": list(range(p["B"] + 1, p["S"])), "S": [p["S"]], "S+1": [p["S"] + 1],
97        "between": list(range(p["S"] + 2, p["end"])), "end": [p["end"]],
98    }
99    entry["name"] = rounded(torch.tensor([[
100        patch.share(cb(patch.run(model, batch.clean, {("resid", d): patch.swap(store["other"]["resid", d], positions=groups[g])})),
101                    cb(value["clean"]), cb(value["other"]))
102        for g in NAME] for d in range(n + 1)]), 3)
103    entry["name measure"] = {"clean": cb(value["clean"]).mean().item(), "other": cb(value["other"]).mean().item()}
104    groups = {
105        "A": [p["A"]], "A to S": list(range(p["A"] + 1, p["S"])), "S": [p["S"]], "S+1": [p["S"] + 1],
106        "between": list(range(p["S"] + 2, p["end"])), "end": [p["end"]],
107    }
108    entry["first name"] = rounded(torch.tensor([[
109        patch.share(ca(patch.run(model, batch.flipped, {("resid", d): patch.swap(store["other_first"]["resid", d], positions=groups[g])})),
110                    ca(value["flipped"]), ca(value["other_first"]))
111        for g in FIRST] for d in range(n + 1)]), 3)
112    entry["first name measure"] = {"flipped": ca(value["flipped"]).mean().item(), "other": ca(value["other_first"]).mean().item()}
113    # How large the stream is at the acting name: the embedding, the stream after block 1, and what block 1 added.
114    at = store["clean"]
115    entry["size at S"] = {"embedding": rms(at["resid", 0][:, p["S"]]), "after block 1": rms(at["resid", 1][:, p["S"]]),
116                          "attention 1": rms(at["attn", 1][:, p["S"]]), "MLP 1": rms(at["mlp", 1][:, p["S"]])}
117
118    if args.stream_only:
119        result["templates"].append(entry)
120        print(batch.template, {k: round(v, 2) for k, v in entry["logit_diff"].items()})
121        continue
122
123    # parts
124    scopes = {"all": None, "last": [p["end"]], "not last": list(range(p["end"])), "S": [p["S"]]}
125    entry["parts"] = {}
126    for name, src, dst in (("carry", "clean", "flipped"), ("break", "flipped", "clean")):
127        entry["parts"][name] = {
128            scope: rounded(grid([
129                patch.share(ld(patch.run(model, getattr(batch, dst), swap_part(part, store[src], positions))), ld(value[dst]), ld(value[src]))
130                for part in PARTS]), 3)
131            for scope, positions in scopes.items()
132        }
133
134    entry["carry by part"] = [patch.share(ld(patch.run(model, batch.flipped, swap_part(part, store["clean"]))), ld(value["flipped"]), ld(value["clean"])) for part in PARTS]
135
136    # choice
137    three = [store[v] for v in ("clean", "flipped", "neutral")]
138    entry["choice"] = {}
139    for choice in CHOICES:
140        changes = []
141        for part in PARTS:
142            key = part[:2]
143            source = {"flipped": store["flipped"][key], "neutral": store["neutral"][key],
144                      "mean": patch.mean(three, key).expand_as(store["clean"][key]), "zero": patch.zero(store["clean"][key])}[choice]
145            changes.append((ld(patch.run(model, batch.clean, swap_part(part, {key: source}))) - ld(value["clean"])).mean().item())
146        entry["choice"][choice] = rounded(grid(changes), 3)
147    result["templates"].append(entry)
148    print(batch.template, {k: round(v, 2) for k, v in entry["logit_diff"].items()})
149
150if not args.stream_only:
151    # The four parts with the largest share, block 1's MLP aside, moved two at a time: do their shares add?
152    single = torch.tensor([t.pop("carry by part") for t in result["templates"]]).mean(0)
153    single[PARTS.index(("mlp", 1, None))] = 0
154    top = [PARTS[i] for i in single.abs().argsort(descending=True)[:4].tolist()]
155    name_of = lambda part: f"block {part[1]} " + ("MLP" if part[0] == "mlp" else f"head {part[2]}")
156    pairs = [(a, b) for i, a in enumerate(top) for b in top[i + 1:]]
157    together = {pair: [] for pair in pairs}
158    for batch in tasks.batches(tok, device):
159        ld = lambda logits: tasks.logit_diff(logits.float(), batch.right, batch.wrong)
160        clean, clean_logits = patch.record(model, batch.clean)
161        _, flipped_logits = patch.record(model, batch.flipped)
162        for a, b in pairs:
163            edits = {}
164            for part in (a, b):  # two heads of one block share a key, so their edits are chained
165                key, edit = next(iter(swap_part(part, clean).items()))
166                edits[key] = (lambda x, first=edits[key], second=edit: second(first(x))) if key in edits else edit
167            together[a, b].append(patch.share(ld(patch.run(model, batch.flipped, edits)), ld(flipped_logits), ld(clean_logits)))
168    result["pairs"] = [{"parts": [name_of(a), name_of(b)], "each": [round(single[PARTS.index(a)].item(), 3), round(single[PARTS.index(b)].item(), 3)],
169                        "together": round(sum(v) / len(v), 3)} for (a, b), v in together.items()]
170
171mean_of = lambda pick: torch.tensor([pick(t) for t in result["templates"]]).mean(0)
172show = lambda row: " ".join(f"{v:6.2f}" for v in row.tolist())
173for name in ("carry", "break"):
174    print(f"\ntravel, {name}: share of the gap, mean of the patterns (rows: after block; columns: {', '.join(TRAVEL)})")
175    for d, row in enumerate(mean_of(lambda t: t["travel"][name])):
176        print(f"  {'emb' if d == 0 else d:>3}  " + show(row))
177print(f"\nname: share of the way from B to C (columns: {', '.join(NAME)})")
178for d, row in enumerate(mean_of(lambda t: t["name"])):
179    print(f"  {'emb' if d == 0 else d:>3}  " + show(row))
180print(f"\nfirst name: share of the way from A to C (columns: {', '.join(FIRST)})")
181for d, row in enumerate(mean_of(lambda t: t["first name"])):
182    print(f"  {'emb' if d == 0 else d:>3}  " + show(row))
183if args.stream_only:
184    save(f"runs/patch/patch-{meta['name']}.json", result)
185    raise SystemExit(0)
186print("\ntwo parts moved together, against the sum of their shares")
187for pair in result["pairs"]:
188    print(f"  {pair['parts'][0]:16s} + {pair['parts'][1]:16s} together {pair['together']:.2f}, sum {sum(pair['each']):.2f}")
189for name in ("carry", "break"):
190    print(f"\nparts, {name}, all positions (rows: blocks; columns: heads, then the MLP)")
191    for b, row in enumerate(mean_of(lambda t: t["parts"][name]["all"]), start=1):
192        print(f"  {b:>3}  " + show(row))
193carry = {s: mean_of(lambda t, s=s: t["parts"]["carry"][s]) for s in SCOPES}
194choice = {c: mean_of(lambda t, c=c: t["choice"][c]) for c in CHOICES}
195ranked = sorted(((carry["all"][b, h].abs().item(), b, h) for b in range(n) for h in range(heads + 1)), reverse=True)[:10]
196print("\nthe ten parts with the largest carry, where they act, and the four replacements on the clean run (logits)")
197print(f"  {'':16s} {'carry':>6} {'last':>6} {'before':>6} | " + " ".join(f"{c:>8}" for c in CHOICES))
198for _, b, h in ranked:
199    label = f"block {b + 1} " + ("MLP" if h == heads else f"head {h}")
200    print(f"  {label:16s} {carry['all'][b, h]:6.2f} {carry['last'][b, h]:6.2f} {carry['not last'][b, h]:6.2f} | "
201          + " ".join(f"{choice[c][b, h]:+8.2f}" for c in CHOICES))
202
203save(f"runs/patch/patch-{meta['name']}.json", result)