scripts/features.py
328 lines · View on GitHub · Linked from 6. Feature circuit
1"""
2Step 6: which latents does the answer depend on? See docs/06-feature-circuit.md.
3
4Uses the dictionary of step 5 (out/sae/<model>-b4.pt), the heads of step 4
5and the latents step 5 found for the names. Everything happens at the
6acting name's position after block 4, where step 3 found that the answer's
7dependence on who acts sits.
8
9parts the stream there is what the dictionary rebuilds plus what it
10 misses. Each part is moved from one prompt into another, for
11 real: how much of who acts does each carry, and how much of a
12 name (step 3's two name experiments)?
13estimate the same for every latent at once, estimated from one gradient
14check the latents the estimate ranks highest, moved for real, one to
15 thirty-two of them, and the estimate set against the real
16 effect latent by latent
17head what the veto head of step 4 adds to the acting name: against
18 the nine uninvolved names, as in step 4, and against the
19 uninvolved names of its own kind, where a kind is the six names
20 the model follows with "She" or the six it follows with "He";
21 what the gradient says this rests on, and what really happens
22 when latents are switched off
23names for each name, two latents: the one step 5 picked by how often
24 it fires, and the one the veto head rests on most. What moving
25 just the two names' latents carries, by the kinds of the pair
26
27 uv run scripts/features.py
28 uv run scripts/features.py --model seed1
29 uv run scripts/features.py --dict s1 # step 8: another dictionary
30 uv run scripts/features.py --model seed2 --head 6.5 # step 8: another head than the one with the largest share
31"""
32
33import argparse
34import json
35
36import torch
37
38from minterp import attribute, tasks
39from minterp import sae as S
40from minterp.common import load, rounded, save
41from minterp.tap import Tap, head_writes
42
43parser = argparse.ArgumentParser()
44parser.add_argument("--model", default="seed0", help="seed0, seed1 or seed2: the steps before must have run on it")
45parser.add_argument("--block", type=int, default=4)
46parser.add_argument("--dict", default="", help="another dictionary of the same place, by the name scripts/sae.py gave it (step 8)")
47parser.add_argument("--head", default="", help="another head read at the last position, as block.head, in place of the one with the largest share (step 8)")
48args = parser.parse_args()
49
50model, tok, meta = load(args.model)
51device = next(model.parameters()).device
52n = model.cfg.n_layers
53sae = S.load(S.path(meta["name"], args.block, args.dict), device)
54tag = f"-{args.dict}" if args.dict else ""
55key = ("resid", args.block)
56with open(f"runs/sae/sae-{meta['name']}{tag}.json") as f:
57 step5 = json.load(f)["task"]
58name_latent = {row["name"]: row["latent"] for row in step5["names"]["acts"]}
59with open(f"runs/circuit/circuit-{meta['name']}.json") as f:
60 veto = max((h for h in json.load(f)["heads"] if h["acts at"] == "end"), key=lambda h: h["carry"])
61if args.head:
62 veto = dict(zip(("block", "head"), map(int, args.head.split("."))))
63people = tasks.names(tok)
64name_ids = torch.tensor([tok.encode_ordinary(" " + p)[0] for p in people], device=device)
65she = torch.tensor([p in step5["names that go on with She"] for p in people], device=device)
66COUNTS = (1, 2, 4, 8, 16, 32)
67batches = tasks.batches(tok, device, people=people)
68result = {"model": meta, "block": args.block, "dictionary": args.dict or "first", "counts": COUNTS, "veto head": f"block {veto['block']} head {veto['head']}",
69 "names that go on with She": step5["names that go on with She"], "templates": []}
70
71
72def index(batch, i):
73 return torch.tensor([people.index(t[i]) for t in batch.names], device=device)
74
75
76def head_names(tap):
77 """What the veto head adds at the last position to the logit of each of the twelve names: (prompts, names), on the
78 scale of the final stream, whose size is held fixed so that the gradient is that of the head's write alone."""
79 final = tap["resid", n][:, -1].detach()
80 size = torch.rsqrt(final.pow(2).mean(-1, keepdim=True) + model.norm.eps)
81 write = head_writes(model, tap, veto["block"])[:, -1, veto["head"]]
82 return ((write * model.norm.weight * size) @ model.lm_head.weight[name_ids].T).float()
83
84
85def masks(batch):
86 """For every prompt: its row index, and the nine names that are none of A, B and C."""
87 rows = torch.arange(len(batch), device=device)
88 nine = torch.ones(len(batch), len(people), dtype=torch.bool, device=device)
89 for i in range(3):
90 nine[rows, index(batch, i)] = False
91 return rows, nine
92
93
94def against(names, target, pool):
95 """A name's logit minus the mean over a pool of names, per prompt."""
96 rows = torch.arange(names.shape[0], device=names.device)
97 return names[rows, target] - (names * pool).sum(1) / pool.sum(1)
98
99
100def adds_to(batch, target):
101 """A measure for attribute.through: what the veto head adds to one name per prompt, against the nine uninvolved names."""
102 _, nine = masks(batch)
103 return lambda logits, tap: against(head_names(tap), target, nine)
104
105
106
107
108tops, actors, fired, where = [], [], [], []
109for p, batch in enumerate(batches):
110 for version, col in (("clean", 0), ("flipped", 1)):
111 actor = index(batch, col)
112 z, _, g, _, _ = attribute.through(model, sae, getattr(batch, version), args.block, adds_to(batch, actor), keep=("heads", "resid"))
113 at = batch.pos["S"]
114 tops.append((z[:, at] * g[:, at]).abs().argmax(1).cpu())
115 actors.append(actor.cpu())
116 fired.append((z[:, at] > 0).cpu())
117 where.append(torch.full((len(batch),), p))
118tops, actors, fired, where = torch.cat(tops), torch.cat(actors), torch.cat(fired), torch.cat(where)
119head_latent, result["names"] = {}, []
120for i, name in enumerate(people):
121 mine = actors == i
122 mode = tops[mine].mode().values.item()
123 marks = (fired[mine].float().mean(0) >= 0.9) & (fired[~mine].float().mean(0) <= 0.1)
124 head_latent[name] = mode
125
126 left_out = []
127 for p in range(len(batches)):
128 seen, unseen = mine & (where != p), mine & (where == p)
129 left_out.append((tops[unseen] == tops[seen].mode().values).float().mean().item())
130 result["names"].append({
131 "name": name, "step 5's latent": name_latent[name], "the head's latent": mode,
132 "the head rests on it most in": (tops[mine] == mode).float().mean().item(),
133 "on a pattern left out": sum(left_out) / len(left_out),
134 "latents that mark the name by firing": int(marks.sum()),
135 "the head's latent marks the name by firing": bool(marks[mode]), "step 5's latent marks the name by firing": bool(marks[name_latent[name]]),
136 "names for which the head's latent fires on 90% or more": [people[j] for j in range(len(people)) if fired[actors == j][:, mode].float().mean() >= 0.9],
137 })
138she_latents = {head_latent[p] for p in step5["names that go on with She"]}
139result["one latent for the six"] = sorted(she_latents)
140
141
142theirs = [head_latent[p] for p in step5["names that go on with She"]]
143the_shared = max(sorted(she_latents), key=theirs.count)
144result["the shared latent"] = the_shared
145
146for batch in batches:
147 s = batch.pos["S"]
148 rows, nine = masks(batch)
149 A, B, C = (index(batch, i) for i in range(3))
150 ld = lambda logits, tap=None: tasks.logit_diff(logits.float(), batch.right, batch.wrong)
151 zc, ec, _, _, vc = attribute.through(model, sae, batch.clean, args.block, ld)
152 zf, ef, gz, ge, vf = attribute.through(model, sae, batch.flipped, args.block, ld)
153 gap = (vc.mean() - vf.mean()).item()
154
155 @torch.no_grad()
156 def run(tokens, z, err, keep=()):
157 """A forward pass with the stream after the block set to decode(z) + err."""
158 with Tap(model, edit={key: attribute.rebuilt_with(sae, z, err)}, keep=keep) as tap:
159 logits, _ = model(tokens)
160 return logits, tap
161
162 moved = lambda z, err: ld(run(batch.flipped, z, err)[0])
163 share = lambda values, pick=slice(None): ((values[pick].mean() - vf[pick].mean()) / (vc[pick].mean() - vf[pick].mean())).item()
164
165 def at_s(target, source):
166 out = target.clone()
167 out[:, s] = source[:, s]
168 return out
169
170 entry = {"template": batch.template, "gap": gap}
171
172 entry["parts"] = {"rebuilt": share(moved(at_s(zf, zc), ef)), "missed": share(moved(zf, at_s(ef, ec))), "both": share(moved(at_s(zf, zc), at_s(ef, ec)))}
173
174 entry["name parts"] = {}
175 for label, source, target, old in (("second name", batch.other, batch.clean, B), ("first name", batch.other_first, batch.flipped, A)):
176 to_c = lambda logits, old=old: tasks.logit_diff(logits.float(), name_ids[C], name_ids[old])
177 zo, eo, _, _, _ = attribute.through(model, sae, source, args.block, lambda logits, tap: to_c(logits))
178 zt, et, _, _, _ = attribute.through(model, sae, target, args.block, lambda logits, tap: to_c(logits))
179 base, goal = to_c(run(target, zt, et)[0]).mean(), to_c(run(source, zo, eo)[0]).mean()
180 way = lambda z, err: ((to_c(run(target, z, err)[0]).mean() - base) / (goal - base)).item()
181 entry["name parts"][label] = {"rebuilt": way(at_s(zt, zo), et), "missed": way(zt, at_s(et, eo)), "both": way(at_s(zt, zo), at_s(et, eo))}
182
183
184 est = (zc[:, s] - zf[:, s]) * gz[:, s]
185 est_missed = ((ec[:, s] - ef[:, s]) * ge[:, s]).sum(1)
186 on_c, on_f = zc[:, s] > 0, zf[:, s] > 0
187 order = est.abs().argsort(dim=1, descending=True)
188 entry["estimate"] = {"rebuilt": (est.sum(1).mean() / gap).item(), "missed": (est_missed.mean() / gap).item(),
189 "latents that differ": (zc[:, s] != zf[:, s]).sum(1).float().mean().item(),
190 "on in the clean prompt only": (on_c & ~on_f).sum(1).float().mean().item(),
191 "on in the flipped prompt only": (~on_c & on_f).sum(1).float().mean().item(),
192 "on in both": (on_c & on_f & (zc[:, s] != zf[:, s])).sum(1).float().mean().item()}
193
194
195 by_count = {"estimated": [], "real": []}
196 for count in COUNTS:
197 top = order[:, :count]
198 z = zf.clone()
199 z[rows[:, None], s, top] = zc[rows[:, None], s, top]
200 by_count["real"].append(share(moved(z, ef)))
201 by_count["estimated"].append((est.gather(1, top).sum(1).mean() / gap).item())
202 entry["by count"] = by_count
203
204 pairs = []
205 for rank in range(5):
206 j = order[:, rank]
207 z = zf.clone()
208 z[rows, s, j] = zc[rows, s, j]
209 pairs.append(torch.stack([est[rows, j], moved(z, ef) - vf]))
210 pairs = torch.stack(pairs)
211 flat = pairs.permute(1, 0, 2).reshape(2, -1)
212 entry["one latent"] = {"estimated": rounded(pairs[:, 0].mean(1), 3), "real": rounded(pairs[:, 1].mean(1), 3),
213 "correlation": torch.corrcoef(flat)[0, 1].item(), "same sign": (flat[0].sign() == flat[1].sign()).float().mean().item()}
214
215
216 kinds = {"both She": she[A] & she[B], "both He": ~she[A] & ~she[B], "one of each": she[A] != she[B]}
217 entry["pairs"] = {k: {"prompts": int(m.sum()), "gap": (vc[m].mean() - vf[m].mean()).item()} for k, m in kinds.items()}
218 entry["names"] = {}
219 for choice, table in (("step 5's latents", name_latent), ("the head's latents", head_latent)):
220 la = torch.tensor([table[a] for a, _, _ in batch.names], device=device)
221 lb = torch.tensor([table[b] for _, b, _ in batch.names], device=device)
222 z = zf.clone()
223 z[rows, s, la] = zc[rows, s, la]
224 z[rows, s, lb] = zc[rows, s, lb]
225 values = moved(z, ef)
226 hit = (order[:, 0] == la) | (order[:, 0] == lb)
227 entry["names"][choice] = {"moved for real": share(values), "estimated": ((est[rows, la] + est[rows, lb]).mean() / gap).item(),
228 "the highest is one of them": hit.float().mean().item(),
229 "by pair": {k: {"moved for real": share(values, m), "the highest is one of them": hit[m].float().mean().item()} for k, m in kinds.items()}}
230
231
232 own = nine & (she[None, :] == she[A][:, None])
233 other_kind = nine & ~own
234 ha = torch.tensor([head_latent[a] for a, _, _ in batch.names], device=device)
235 z, err, g, gerr, value = attribute.through(model, sae, batch.clean, args.block, adds_to(batch, A), keep=("heads", "resid"))
236 rest_on = z[:, s] * g[:, s]
237 ranked = rest_on.abs().argsort(dim=1, descending=True)
238 plain = head_names(run(batch.clean, z, err, keep=("heads", "resid"))[1])
239
240 def off(mask):
241 """The head's write with some latents at the acting name switched off, for real."""
242 cut = z.clone()
243 cut[:, s] = cut[:, s] * (~mask)
244 return head_names(run(batch.clean, cut, err, keep=("heads", "resid"))[1])
245
246 one_hot = lambda j: torch.zeros_like(z[:, s], dtype=torch.bool).scatter_(1, j[:, None], True)
247 everything = torch.ones_like(z[:, s], dtype=torch.bool)
248 shared = torch.zeros_like(z[:, s], dtype=torch.bool)
249 shared[:, the_shared] = True
250 acts_she = she[A]
251 kind_gap = lambda names: (names * own).sum(1) / own.sum(1) - (names * other_kind).sum(1) / other_kind.sum(1)
252 head = {"by kind of the acting name": {}}
253 for label, m in (("She", acts_she), ("He", ~acts_she)):
254 head["by kind of the acting name"][label] = {
255 "prompts": int(m.sum()),
256 "against the nine": against(plain, A, nine)[m].mean().item(),
257 "against its own kind": against(plain, A, own)[m].mean().item(),
258 "its own kind against the other": kind_gap(plain)[m].mean().item(),
259 }
260 cut = off(shared)
261 m = acts_she
262 head["the shared latent switched off, where a She name acts"] = {
263 "the acting name against the nine": (against(cut, A, nine) - against(plain, A, nine))[m].mean().item(),
264 "estimated": -rest_on[:, the_shared][m].mean().item(),
265 "the acting name against its own kind": (against(cut, A, own) - against(plain, A, own))[m].mean().item(),
266 "its own kind against the other": (kind_gap(cut) - kind_gap(plain))[m].mean().item(),
267 }
268 head["adds to the acting name"] = value.mean().item()
269 head["estimated to rest on"] = {
270 "the latents at the acting name": rest_on.sum(1).mean().item(),
271 "what the dictionary misses there": (err[:, s] * gerr[:, s]).sum(1).mean().item(),
272 "the other positions": ((z * g).sum((1, 2)) + (err * gerr).sum((1, 2)) - rest_on.sum(1) - (err[:, s] * gerr[:, s]).sum(1)).mean().item(),
273 "its highest latent": rest_on.gather(1, ranked[:, :1]).mean().item(),
274 "the latents below zero": rest_on.clamp(max=0).sum(1).mean().item(), "the latents above zero": rest_on.clamp(min=0).sum(1).mean().item(),
275 }
276 head["really lost when switched off"] = {
277 "its highest latent": -(against(off(one_hot(ranked[:, 0])), A, nine) - value).mean().item(),
278 "all latents at the acting name": -(against(off(everything), A, nine) - value).mean().item(),
279 }
280 head["its highest latent is the head's latent for the name"] = (ranked[:, 0] == ha).float().mean().item()
281
282 z2, err2, g2, _, value2 = attribute.through(model, sae, batch.clean, args.block, adds_to(batch, B), keep=("heads", "resid"))
283 rest2 = z2[:, s] * g2[:, s]
284 top2 = rest2.gather(1, rest2.abs().argsort(dim=1, descending=True)[:, :1])[:, 0]
285 same = she[A] == she[B]
286 cut_all = against(off(everything), B, nine)
287 head["the other name"] = {
288 "adds": value2.mean().item(), "adds, same kind": value2[same].mean().item(), "adds, other kind": value2[~same].mean().item(),
289 "estimated to rest on the latents at the acting name": rest2.sum(1).mean().item(),
290 "really lost when all latents there are switched off": -(cut_all - value2).mean().item(),
291 "its highest latent, size": top2.abs().mean().item(), "its highest latent, same kind": top2[same].mean().item(), "its highest latent, other kind": top2[~same].mean().item(),
292 }
293 entry["head"] = head
294 result["templates"].append(entry)
295 print(batch.template, {k: round(v, 2) for k, v in entry["parts"].items()})
296
297T = result["templates"]
298mean = lambda pick: sum(pick(t) for t in T) / len(T)
299show = lambda d, digits=2: ", ".join(f"{k} {v:+.{digits}f}" for k, v in d.items() if isinstance(v, float))
300print(f"\nat the acting name after block {args.block} (shares of a gap, moved for real):")
301print(f" who acts: what the dictionary rebuilds {mean(lambda t: t['parts']['rebuilt']):.2f}, what it misses {mean(lambda t: t['parts']['missed']):.2f}, both {mean(lambda t: t['parts']['both']):.2f}; "
302 f"estimated {mean(lambda t: t['estimate']['rebuilt']):.2f} and {mean(lambda t: t['estimate']['missed']):.2f}")
303for label in ("second name", "first name"):
304 print(f" the {label}: rebuilt {mean(lambda t: t['name parts'][label]['rebuilt']):.3f}, missed {mean(lambda t: t['name parts'][label]['missed']):.3f}, both {mean(lambda t: t['name parts'][label]['both']):.3f}")
305e = {k: mean(lambda t, k=k: t["estimate"][k]) for k in T[0]["estimate"]}
306print(f" {e['latents that differ']:.0f} latents differ: {e['on in the clean prompt only']:.0f} on in the clean prompt only, {e['on in the flipped prompt only']:.0f} in the flipped only, {e['on in both']:.0f} in both")
307print(" the highest-ranked latents: " + " ".join(f"{c:>6}" for c in COUNTS))
308print(" estimated " + " ".join(f"{mean(lambda t, i=i: t['by count']['estimated'][i]):6.2f}" for i in range(len(COUNTS))))
309print(" moved for real " + " ".join(f"{mean(lambda t, i=i: t['by count']['real'][i]):6.2f}" for i in range(len(COUNTS))))
310print(f" one latent at a time: correlation {mean(lambda t: t['one latent']['correlation']):.2f}, same sign {100 * mean(lambda t: t['one latent']['same sign']):.1f}%")
311for choice in T[0]["names"]:
312 nm = T[0]["names"][choice]
313 print(f" {choice}: moved for real {mean(lambda t: t['names'][choice]['moved for real']):.2f}, the highest is one of them {100 * mean(lambda t: t['names'][choice]['the highest is one of them']):.0f}%; by pair: "
314 + "; ".join(f"{k} {mean(lambda t, k=k: t['names'][choice]['by pair'][k]['moved for real']):+.2f} ({100 * mean(lambda t, k=k: t['names'][choice]['by pair'][k]['the highest is one of them']):.0f}%)" for k in nm["by pair"]))
315print(" gap by pair: " + ", ".join(f"{k} {mean(lambda t, k=k: t['pairs'][k]['gap']):.2f} ({T[0]['pairs'][k]['prompts']} prompts)" for k in T[0]["pairs"]))
316print(" per name: step 5's latent / the head's (share of prompts; on a pattern left out), names the head's latent fires for")
317for x in result["names"]:
318 print(f" {x['name']:5s} {x['step 5' + chr(39) + 's latent']:4d} / {x['the head' + chr(39) + 's latent']:4d} ({100 * x['the head rests on it most in']:.0f}%; {100 * x['on a pattern left out']:.0f}%) {', '.join(x['names for which the head' + chr(39) + 's latent fires on 90% or more'])}")
319for label in ("She", "He"):
320 print(f" where a {label} name acts, {result['veto head']} adds to it: " + show({k: mean(lambda t, k=k: t['head']['by kind of the acting name'][label][k]) for k in T[0]['head']['by kind of the acting name'][label] if k != 'prompts'}))
321k1 = "the shared latent switched off, where a She name acts"
322print(f" {k1}: " + show({k: mean(lambda t, k=k: t['head'][k1][k]) for k in T[0]['head'][k1]}))
323print(f" adds to the acting name {mean(lambda t: t['head']['adds to the acting name']):+.2f}; estimated to rest on: " + show({k: mean(lambda t, k=k: t['head']['estimated to rest on'][k]) for k in T[0]['head']['estimated to rest on']}))
324print(" really lost when switched off: " + show({k: mean(lambda t, k=k: t['head']['really lost when switched off'][k]) for k in T[0]['head']['really lost when switched off']})
325 + f"; its highest latent is the head's latent for the name in {100 * mean(lambda t: t['head']['its highest latent is the head' + chr(39) + 's latent for the name']):.0f}%")
326print(" the other name: " + show({k: mean(lambda t, k=k: t['head']['the other name'][k]) for k in T[0]['head']['the other name']}))
327
328save(f"runs/features/features-{meta['name']}{tag}{'-h' + args.head if args.head else ''}.json", result)