scripts/heads.py
169 lines · View on GitHub · Linked from 8. Limits
1"""
2Step 8, the open model part by part: where the heads that step 3's patch
3found look, what they write, and which of them the head at the acting name
4works through. See docs/08-limits.md.
5
6The heads are not chosen by hand. They are every head whose share of the
7gap between the clean and the flipped prompt is 0.05 or more in either
8direction, in the record of scripts/patch.py. For each of them, per
9sentence pattern and version of the prompt, this records
10
11attention from the position where it acts (the acting name if step 3
12 found its share there, the last position otherwise), how much
13 it reads from the first token, the first name, the second name,
14 the full stop that ends the first sentence, the acting name,
15 the token after it and the last position; and the position it
16 reads most from among all the others
17writes for a head read at the last position, what it adds to the
18 logit of A, of B and of C, each against the mean of the nine
19 other names
20by kind the same on clean and flipped prompts for the name that acts
21 and the name that is due, each against the uninvolved names of
22 its own kind, where a kind is the names the model follows with
23 " She" or with " He" (step 6)
24holds for a head that acts at the acting name and every later head:
25 its share alone, and what is left of it when the later head is
26 held at the flipped run's own output (step 4)
27
28The first two and the last are measures of step 4 (scripts/circuit.py), on
29another choice of heads: step 4 takes the heads with a positive share or a
30large direct write. tests/test_limits.py checks that on the story models
31the two scripts give the same numbers for the heads they share.
32
33 uv run scripts/heads.py # the open model: under a minute
34 uv run scripts/heads.py --model seed0 # a story model, as a check against step 4
35"""
36
37import argparse
38import json
39
40import torch
41
42from minterp import attention, patch, tasks
43from minterp.common import eot, load, save
44from minterp.tap import Tap, head_writes
45
46parser = argparse.ArgumentParser()
47parser.add_argument("--model", default="qwen", help="qwen, seed0, seed1 or seed2: scripts/patch.py must have run on it, with its parts")
48args = parser.parse_args()
49
50model, tok, meta = load(args.model)
51device = next(model.parameters()).device
52n, H = model.cfg.n_layers, model.cfg.n_heads
53VERSIONS = ("clean", "flipped", "neutral")
54PLACES = ("first token", "A", "B", "stop", "S", "S+1", "end")
55
56with open(f"runs/patch/patch-{meta['name']}.json") as f:
57 step3 = json.load(f)["templates"]
58carry = {s: torch.tensor([t["parts"]["carry"][s] for t in step3]).mean(0)[:, :H] for s in ("all", "S")}
59heads = [{
60 "block": b + 1, "head": h, "carry": round(carry["all"][b, h].item(), 3),
61
62 "acts at": "S" if carry["S"][b, h].abs() >= 0.05 else "end",
63} for b in range(n) for h in range(H) if carry["all"][b, h].abs() >= 0.05]
64name_of = lambda x: f"block {x['block']} head {x['head']}"
65key = lambda x: ("heads", x["block"])
66people = tasks.names(tok)
67ids = torch.tensor([tok.encode_ordinary(" " + p)[0] for p in people], device=device)
68stop_id = tok.encode_ordinary(".")[0]
69
70
71she_id, he_id = (tok.encode_ordinary(w)[0] for w in (" She", " He"))
72with torch.no_grad():
73 after = model(torch.tensor([[eot(tok)] + tok.encode_ordinary(f"One day, {p} went to the park.") for p in people], device=device))[0][:, -1].float()
74she = after[:, she_id] > after[:, he_id]
75
76result = {"model": meta, "heads": heads, "versions": VERSIONS, "places": PLACES,
77 "names that go on with She": [p for p, g in zip(people, she.tolist()) if g], "templates": []}
78print("the heads:", ", ".join(f"{name_of(x)} (at {x['acts at']})" for x in heads))
79
80for batch in tasks.batches(tok, device, people=people):
81 p = batch.pos
82 entry = {"template": batch.template, "attention": {}, "elsewhere": {}, "writes": {}, "by kind": {}, "holds": []}
83
84 stop = next(i for i in range(p["B"] + 1, p["S"]) if batch.clean[0, i].item() == stop_id)
85 place = {"first token": 0, "A": p["A"], "B": p["B"], "stop": stop, "S": p["S"], "S+1": p["S"] + 1, "end": p["end"]}
86 index = {name: i for i, name in enumerate(people)}
87 rows = torch.arange(len(batch), device=device)
88 who = {k: torch.tensor([index[t[i]] for t in batch.names], device=device) for i, k in enumerate(("A", "B", "C"))}
89 uninvolved = torch.ones(len(batch), len(people), dtype=torch.bool, device=device)
90 for k in who:
91 uninvolved[rows, who[k]] = False
92
93 for version in VERSIONS:
94 with torch.no_grad(), Tap(model, keep=("resid", "heads")) as tap:
95 model(getattr(batch, version))
96 final = tap["resid", n][:, -1].float()
97 size = torch.rsqrt(final.pow(2).mean(-1, keepdim=True) + model.norm.eps)
98 weights = {}
99 for x in heads:
100 b = x["block"]
101 if b not in weights:
102 with torch.no_grad():
103 weights = {b: attention.weights(model, model.layers[b - 1].attn, model.layers[b - 1].attn_norm(tap["resid", b - 1])).float()}
104 row = weights[b][:, x["head"], p[x["acts at"]]]
105 entry["attention"].setdefault(name_of(x), {})[version] = {k: row[:, i].mean().item() for k, i in place.items()}
106
107 others = [i for i in range(p[x["acts at"]] + 1) if i not in place.values()]
108 most = max(others, key=lambda i: row[:, i].mean().item())
109 entry["elsewhere"].setdefault(name_of(x), {})[version] = {"token": batch.labels[most], "weight": row[:, most].mean().item()}
110 if x["acts at"] == "end":
111 write = head_writes(model, tap, b)[:, -1, x["head"]].float()
112 logits = (write * model.norm.weight.float() * size) @ model.lm_head.weight[ids].float().T
113 rest = (logits * uninvolved).sum(1) / uninvolved.sum(1)
114 entry["writes"].setdefault(name_of(x), {})[version] = {k: (logits[rows, who[k]] - rest).mean().item() for k in who}
115 if version != "neutral":
116
117 acts, due = (who["A"], who["B"]) if version == "clean" else (who["B"], who["A"])
118 split = {}
119 for role, target in (("acts", acts), ("due", due)):
120 own = uninvolved & (she[None, :] == she[target][:, None])
121 against_own = logits[rows, target] - (logits * own).sum(1) / own.sum(1).clamp(min=1)
122 usable = own.sum(1) > 0
123 split[role] = {label: against_own[mask & usable].mean().item() for label, mask in (("She", she[target]), ("He", ~she[target]))}
124 entry["by kind"].setdefault(name_of(x), {})[version] = split
125 del tap, weights
126
127
128 ld = lambda logits: tasks.logit_diff(logits.float(), batch.right, batch.wrong)
129 clean, clean_logits = patch.record(model, batch.clean, keep=("heads",))
130 flipped, flipped_logits = patch.record(model, batch.flipped, keep=("heads",))
131 base, goal = ld(flipped_logits), ld(clean_logits)
132 for s in heads:
133 if s["acts at"] != "S":
134 continue
135 send = {key(s): patch.swap(clean[key(s)], head=s["head"])}
136 alone = patch.share(ld(patch.run(model, batch.flipped, send)), base, goal)
137 for r in heads:
138 if r["block"] <= s["block"]:
139 continue
140 edits = dict(send)
141 edits[key(r)] = patch.swap(flipped[key(r)], head=r["head"])
142 entry["holds"].append({"sender": name_of(s), "receiver": name_of(r), "alone": alone,
143 "held": patch.share(ld(patch.run(model, batch.flipped, edits)), base, goal)})
144 del clean, flipped
145 result["templates"].append(entry)
146
147T = result["templates"]
148mean = lambda pick: sum(pick(t) for t in T) / len(T)
149print(f"\ngoes on with She after {', '.join(result['names that go on with She'])}")
150print("\nwhere each head reads from, at the position where it acts (clean | flipped | neutral): " + ", ".join(PLACES) + "; then the most-read other token on clean prompts, by pattern")
151for x in heads:
152 print(f" {name_of(x):17s} {x['carry']:6.2f} at {x['acts at']:3s} " + " | ".join(
153 " ".join(f"{mean(lambda t, v=v, k=k: t['attention'][name_of(x)][v][k]):.2f}" for k in PLACES) for v in VERSIONS)
154 + " " + ", ".join(f"{t['elsewhere'][name_of(x)]['clean']['token'].strip()} {t['elsewhere'][name_of(x)]['clean']['weight']:.2f}" for t in T))
155print("\nwhat the heads at the last position add to the logit of A, B, C against the other names (clean | flipped | neutral)")
156for x in heads:
157 if x["acts at"] == "end":
158 print(f" {name_of(x):17s} " + " | ".join(
159 " ".join(f"{mean(lambda t, v=v, k=k: t['writes'][name_of(x)][v][k]):+.2f}" for k in ("A", "B", "C")) for v in VERSIONS))
160print("\nthe same against the uninvolved names of the name's own kind: the name that acts if She, if He; the name that is due if She, if He (clean | flipped)")
161for x in heads:
162 if x["acts at"] == "end":
163 print(f" {name_of(x):17s} " + " | ".join(
164 " ".join(f"{mean(lambda t, v=v, r=r, g=g: t['by kind'][name_of(x)][v][r][g]):+.2f}" for r in ("acts", "due") for g in ("She", "He")) for v in ("clean", "flipped")))
165print("\nholds: the share of a head at the acting name alone, and what is left when a later head is held at the flipped run's own output")
166for i, h in enumerate(T[0]["holds"]):
167 print(f" {h['sender']} alone {mean(lambda t: t['holds'][i]['alone']):.2f}; with {h['receiver']:17s} held {mean(lambda t: t['holds'][i]['held']):.2f}")
168
169save(f"runs/open/heads-{meta['name']}.json", json.loads(json.dumps(result), parse_float=lambda s: round(float(s), 4)))