scripts/sae.py
368 lines · View on GitHub · Linked from 5. Features
1"""
2Step 5: in what units is the stream written? See docs/05-features.md.
3
4Trains a sparse autoencoder on the stream after one block, on ordinary
5story text, and asks four things of it.
6
7rebuilds how much of the stream's variance it misses on text it did not
8 train on, against 32 fixed directions; how often latents fire
9costs the model's loss on that text with the rebuilt stream in place
10 of the real one, against the plain model and against a stream
11 replaced by its mean
12fires on for a sample of latents taken at evenly spaced ranks of how
13 often they fire, the text on which each fires most
14the task at the acting name's position, where step 4 found the other
15 name being fetched:
16 - for each of the twelve names, the latent that best marks it
17 as the name that acts, and as the other name
18 - how well step 2's reader reads the other name from the
19 stream, from the rebuilt stream and from what the dictionary
20 misses, and how the score of the reader fitted on the stream
21 splits over the two parts
22 - what the answer and the veto head of step 4 do when the
23 stream there is rebuilt, and when the other name is averaged
24 out of it
25
26 uv run scripts/sae.py # the model of minllm's stage 3, after block 4
27 uv run scripts/sae.py --model seed1
28 uv run scripts/sae.py --seed 1 --dict s1 # a second dictionary, for step 8
29"""
30
31import argparse
32import json
33import os
34
35import numpy as np
36import torch
37import torch.nn.functional as F
38from minllm.data import load_tokens
39
40from minterp import probe, tasks
41from minterp import sae as S
42from minterp.common import MINLLM, eot, load, rounded, save
43from minterp.tap import Tap, head_writes
44
45parser = argparse.ArgumentParser()
46parser.add_argument("--model", default="seed0", help="seed0, seed1 or seed2: the steps before must have run on it")
47parser.add_argument("--block", type=int, default=4, help="the stream after this block")
48parser.add_argument("--width", type=int, default=4096)
49parser.add_argument("--k", type=int, default=32)
50parser.add_argument("--steps", type=int, default=4000)
51parser.add_argument("--seed", type=int, default=0)
52parser.add_argument("--dict", default="", help="a name for another dictionary of the same place, kept beside the first (step 8)")
53args = parser.parse_args()
54
55model, tok, meta = load(args.model)
56device = next(model.parameters()).device
57SEQ, key, n = model.cfg.max_seq_len, ("resid", args.block), model.cfg.n_layers
58
59
60@torch.no_grad()
61def collect(split, windows, seed):
62 """The stream after the block at every position of random windows of story text, and the tokens."""
63 data = load_tokens(os.path.join(MINLLM, f"data/tinystories/{split}.bin"))
64 starts = np.random.default_rng(seed).integers(0, len(data) - SEQ - 1, windows)
65 vectors, tokens = [], []
66 for i in range(0, windows, 100):
67 idx = torch.from_numpy(np.stack([data[s:s + SEQ].astype(np.int64) for s in starts[i:i + 100]])).to(device)
68 with Tap(model, keep=("resid",)) as tap:
69 model(idx)
70 vectors.append(tap[key].float().cpu())
71 tokens.append(idx.cpu())
72 return torch.cat(vectors), torch.cat(tokens)
73
74
75train_vectors, _ = collect("train", 4000, seed=1)
76held, held_tokens = collect("valid", 300, seed=2)
77dim = train_vectors.shape[-1]
78flat_train, flat = train_vectors.reshape(-1, dim), held.reshape(-1, dim)
79print(f"{flat_train.shape[0]:,} training vectors, {flat.shape[0]:,} held out")
80sae = S.train(flat_train, args.width, args.k, steps=args.steps, seed=args.seed, device=device,
81 log=lambda step, loss: print(f" step {step:5d} loss {loss:.3f}"))
82os.makedirs("out/sae", exist_ok=True)
83S.save(sae, S.path(meta["name"], args.block, args.dict))
84rebuild = lambda x: sae(x.float())[0].to(x.dtype)
85
86result = {"model": meta, "block": args.block, "width": args.width, "k": args.k, "steps": args.steps, "seed": args.seed,
87 "training vectors": flat_train.shape[0], "held-out vectors": flat.shape[0]}
88
89
90with torch.no_grad():
91 z = torch.cat([sae.encode(flat[i:i + 8192].to(device)).cpu() for i in range(0, flat.shape[0], 8192)])
92often = (z > 0).float().mean(0)
93story_mean = flat_train[:200_000].mean(0)
94
95centered = (flat_train[:200_000] - story_mean).double()
96basis = torch.linalg.eigh(centered.T @ centered).eigenvectors[:, -args.k:].float()
97off = flat - story_mean
98result["rebuilds"] = {
99 "log10 of how often each latent fires": rounded(torch.histc(often.clamp_min(1e-6).log10(), bins=24, min=-5, max=0), 0),
100 "unexplained variance": S.unexplained(sae, flat),
101 "unexplained variance with the same number of fixed directions": ((off - off @ basis @ basis.T).pow(2).sum() / (flat - flat.mean(0)).pow(2).sum()).item(),
102 "latents that never fire": int((often == 0).sum()),
103 "fires on share of tokens": {q: often.quantile(p).item() for q, p in (("5%", 0.05), ("median", 0.5), ("95%", 0.95), ("most", 1.0))},
104 "active per token": (z > 0).sum(1).float().mean().item(),
105}
106
107
108
109@torch.no_grad()
110def loss_with(edit=None):
111 idx = held_tokens[:100].to(device)
112 with Tap(model, edit={key: edit} if edit else None, keep=()):
113 logits, _ = model(idx)
114 return F.cross_entropy(logits[:, :-1].reshape(-1, logits.shape[-1]).float(), idx[:, 1:].reshape(-1)).item()
115
116
117mean_vector = story_mean.to(device)
118plain, spliced, meaned = loss_with(), loss_with(rebuild), loss_with(lambda x: mean_vector.to(x.dtype).expand_as(x).clone())
119result["costs"] = {"loss": plain, "loss with the rebuilt stream": spliced, "loss with the mean stream": meaned,
120 "share of the loss recovered": (meaned - spliced) / (meaned - plain), "loss rises by": spliced / plain - 1}
121
122
123ids = held_tokens.reshape(-1)
124alive = torch.nonzero(often > 0)[:, 0]
125order = alive[often[alive].argsort(descending=True)]
126
127
128def strongest(j, count):
129 return z[:, j].topk(min(count, int((z[:, j] > 0).sum()))).indices
130
131
132def one_token(j):
133 """Of a latent's 30 strongest activations (or all, if it has fewer), the share on its commonest token."""
134 at = strongest(j, 30)
135 return torch.bincount(ids[at], minlength=model.cfg.vocab_size).max().item() / len(at)
136
137
138def examples(j, count):
139 out = []
140 for at in strongest(j, count).tolist():
141 start = max(at - 8, (at // SEQ) * SEQ)
142 out.append([tok.decode(ids[start:at].tolist()).replace("\n", " "), tok.decode([ids[at].item()])])
143 return out
144
145
146result["fires on"] = []
147for q in np.linspace(0.02, 0.98, 16):
148 j = order[int(q * (len(order) - 1))].item()
149 fired = ids[z[:, j] > 0]
150 shown = ids[strongest(j, 1)[0]].item()
151 result["fires on"].append({"latent": j, "fires on share of tokens": often[j].item(), "examples": examples(j, 8),
152 "share of its 30 strongest on one token": one_token(j),
153 "share of all its firings on its commonest token": torch.bincount(fired).max().item() / len(fired),
154 "share of all its firings on the token of its strongest example": (fired == shown).float().mean().item()})
155purity = torch.tensor([one_token(j) for j in order.tolist()])
156result["one token"] = {"latents whose 30 strongest are one token in 80% or more": (purity >= 0.8).float().mean().item(),
157 "in under 30%": (purity < 0.3).float().mean().item()}
158
159
160people = tasks.names(tok)
161k = len(people)
162name_ids = torch.tensor([tok.encode_ordinary(" " + p)[0] for p in people], device=device)
163batches = tasks.batches(tok, device, people=people)
164first = torch.tensor([people.index(a) for a, _, _ in batches[0].names])
165second = torch.tensor([people.index(b) for _, b, _ in batches[0].names])
166third = torch.tensor([people.index(c) for _, _, c in batches[0].names])
167acts = torch.stack([first, second])
168other = torch.stack([second, first])
169
170
171she, he = (tok.encode_ordinary(w)[0] for w in (" She", " He"))
172with torch.no_grad():
173 after = model(torch.tensor([[eot(tok)] + tok.encode_ordinary(f"One day, {p} went to the park.") for p in people], device=device))[0][:, -1]
174kind = (after[:, she] > after[:, he]).cpu()
175
176
177with open(f"runs/circuit/circuit-{meta['name']}.json") as f:
178 veto = max((h for h in json.load(f)["heads"] if h["acts at"] == "end"), key=lambda h: h["carry"])
179
180at_name = {x: [] for x in ("stream", "rebuilt", "missed", "averaged", "latents")}
181effects = {}
182MEASURES = ("logit difference", "right side", "change per prompt", "veto head adds to the acting name",
183 "veto head adds to the other name", "other name largest in what the veto head writes")
184for batch in batches:
185 s_pos = batch.pos["S"]
186 rows = torch.arange(len(batch), device=device)
187 for v, version in enumerate(("clean", "flipped")):
188 tokens = getattr(batch, version)
189 with torch.no_grad(), Tap(model, keep=("resid",)) as tap:
190 model(tokens)
191 x = tap[key][:, s_pos].float()
192 at_name["stream"].append(x.cpu())
193 at_name["rebuilt"].append(sae(x)[0].cpu())
194 at_name["missed"].append((x - sae(x)[0]).cpu())
195 at_name["latents"].append(sae.encode(x).cpu())
196
197
198 same_actor = torch.stack([x[(acts[v] == a).to(device)].mean(0) for a in acts[v].tolist()])
199 at_name["averaged"].append(same_actor.cpu())
200
201 def put(replacement, s_pos=s_pos):
202 def edit(stream):
203 stream = stream.clone()
204 stream[:, s_pos] = replacement(stream[:, s_pos])
205 return stream
206 return edit
207
208 ways = {"plain": None, "rebuilt at the acting name": put(rebuild), "rebuilt everywhere": rebuild,
209 "the other name averaged out at the acting name": put(lambda cur, m=same_actor: m.to(cur.dtype))}
210 base = None
211 for where, edit in ways.items():
212 with torch.no_grad(), Tap(model, edit={key: edit} if edit else None) as t2:
213 logits, _ = model(tokens)
214 ld = tasks.logit_diff(logits.float(), batch.right, batch.wrong)
215 base = ld if where == "plain" else base
216 final = t2["resid", n][:, -1]
217 size = torch.rsqrt(final.pow(2).mean(-1, keepdim=True) + model.norm.eps)
218 write = head_writes(model, t2, veto["block"])[:, -1, veto["head"]]
219 names = ((write * model.norm.weight * size) @ model.lm_head.weight[name_ids].T).float()
220 nine = torch.ones_like(names, dtype=torch.bool)
221 for who in (first, second, third):
222 nine[rows, who.to(device)] = False
223 rest = (names * nine).sum(1) / nine.sum(1)
224 a, o = acts[v].to(device), other[v].to(device)
225 without_actor = names.clone()
226 without_actor[rows, a] = float("-inf")
227 sign = 1 if version == "clean" else -1
228 d = effects.setdefault(where, {}).setdefault(version, {m: [] for m in MEASURES})
229 d["logit difference"].append(ld.mean().item())
230 d["right side"].append((sign * ld > 0).float().mean().item())
231 d["change per prompt"].append((ld - base).abs().mean().item())
232 d["veto head adds to the acting name"].append((names[rows, a] - rest).mean().item())
233 d["veto head adds to the other name"].append((names[rows, o] - rest).mean().item())
234 d["other name largest in what the veto head writes"].append((without_actor.argmax(1) == o).float().mean().item())
235
236patterns, prompts = len(batches), len(first)
237stacked = {x: torch.stack(parts).view(patterns, 2, prompts, -1) for x, parts in at_name.items()}
238result["task"] = {"prompts": patterns * 2 * prompts, "veto head": f"block {veto['block']} head {veto['head']}",
239 "names that go on with She": [p for p, g in zip(people, kind.tolist()) if g],
240 "effects": {where: {v: {m: {"mean": float(np.mean(x)), "by pattern": rounded(torch.tensor(x), 3)} for m, x in d.items()}
241 for v, d in by.items()} for where, by in effects.items()}}
242
243
244xs, miss = stacked["stream"].reshape(-1, dim), stacked["missed"].reshape(-1, dim)
245result["task"]["misses at the acting name"] = {
246 "of the variance around the stories' mean": (miss.pow(2).sum() / (xs - story_mean).pow(2).sum()).item(),
247 "of the variance among these prompts": (miss.pow(2).sum() / (xs - xs.mean(0)).pow(2).sum()).item(),
248 "of the stream's size": (miss.norm(dim=1) / xs.norm(dim=1)).mean().item(),
249}
250
251
252Z = stacked["latents"].reshape(-1, args.width)
253fires = Z > 0
254role = {"acts": acts[None].expand(patterns, -1, -1).reshape(-1), "other": other[None].expand(patterns, -1, -1).reshape(-1)}
255busy = torch.nonzero(fires.float().mean(0) >= 0.02)[:, 0]
256result["task"]["names"] = {}
257for r, lab in role.items():
258 table = []
259 for i, name in enumerate(people):
260 yes, no = lab == i, lab != i
261 gap = fires[yes].float().mean(0) - fires[no].float().mean(0)
262 j = gap.argmax().item()
263
264 larger = (Z[yes][:, None, busy] > Z[no][None, :, busy]).float().mean((0, 1))
265 equal = (Z[yes][:, None, busy] == Z[no][None, :, busy]).float().mean((0, 1))
266 auroc = larger + equal / 2
267 on_token = ids == name_ids[i].item()
268 both = ((z[:, j] > 0) & on_token).sum().item()
269 table.append({
270 "name": name, "latent": j, "fires when it is": fires[yes, j].float().mean().item(), "fires when it is not": fires[no, j].float().mean().item(),
271 "fires on share of story tokens": often[j].item(),
272 "of its story firings, on the name's token": both / max(1, int((z[:, j] > 0).sum())),
273 "of the name's story tokens, fires on": both / max(1, int(on_token.sum())),
274 "the name's story tokens": int(on_token.sum()),
275 "share of its 30 strongest on one token": one_token(j),
276 "share of its 30 strongest on the name's token": (ids[strongest(j, 30)] == name_ids[i].item()).float().mean().item(),
277 "a latent that fires on 80% with and under 20% without": bool(((fires[yes].float().mean(0) > 0.8) & (fires[no].float().mean(0) < 0.2)).any()),
278 "best separation by size": auroc.max().item(), "its latent": busy[auroc.argmax()].item(),
279 })
280 result["task"]["names"][r] = table
281
282
283
284held_pairs = (first + second) % 3 == 0
285result["task"]["reader"] = {}
286
287
288def fold(X, f):
289 others = [q for q in range(patterns) if q != f]
290 xtr = X[others][:, :, ~held_pairs].reshape(-1, X.shape[-1])
291 ltr = other[:, ~held_pairs][None].expand(len(others), -1, -1).reshape(-1)
292 gtr = torch.tensor(others)[:, None, None].expand(len(others), 2, int((~held_pairs).sum())).reshape(-1)
293 return xtr, ltr, gtr, X[f][:, held_pairs]
294
295
296truth = other[:, held_pairs]
297for name in ("stream", "rebuilt", "missed", "averaged"):
298 X = stacked[name].double()
299 right, same_kind = [], []
300 for f in range(patterns):
301 xtr, ltr, gtr, xte = fold(X, f)
302 guess = probe.read(xtr[None], ltr[None], gtr, xte.reshape(1, -1, X.shape[-1]), k)[0].argmax(-1)[0, :, 0].view(2, -1)
303 right.append((guess == truth).double().mean(1))
304 same_kind.append((kind[guess] == kind[truth]).double().mean(1))
305 right, same_kind = torch.stack(right), torch.stack(same_kind)
306 result["task"]["reader"][name] = {"clean": right[:, 0].mean().item(), "flipped": right[:, 1].mean().item(), "by pattern": rounded(right),
307 "a name of the right kind": rounded(same_kind.mean(0))}
308
309
310split = {m: [] for m in ("from the missed part: share of the margin", "right from the rebuilt part alone", "right from the missed part alone")}
311for f in range(patterns):
312 xtr, ltr, gtr, _ = fold(stacked["stream"].double(), f)
313 W, b, mean, spread = probe.weights(xtr, ltr, gtr, k)
314 kept = ((stacked["rebuilt"][f][:, held_pairs].double() - mean) / spread) @ W + b
315 lost = (stacked["missed"][f][:, held_pairs].double() / spread) @ W
316
317 def margin(scores):
318 true = scores.gather(2, truth[:, :, None])[:, :, 0]
319 return (true - (scores.sum(2) - true) / (k - 1)).mean(1)
320
321 split["from the missed part: share of the margin"].append(margin(lost) / (margin(kept) + margin(lost)))
322 split["right from the rebuilt part alone"].append((kept.argmax(2) == truth).double().mean(1))
323 split["right from the missed part alone"].append((lost.argmax(2) == truth).double().mean(1))
324result["task"]["reader fitted on the stream"] = {m: rounded(torch.stack(x).mean(0)) for m, x in split.items()}
325
326
327shared = {}
328for row in result["task"]["names"]["other"]:
329 shared.setdefault(row["latent"], []).append(row["name"])
330result["task"]["other-name latents"] = [{"latent": j, "names": names, "fires on share of story tokens": often[j].item(), "examples": examples(j, 6)}
331 for j, names in shared.items()]
332
333
334r, c, t = result["rebuilds"], result["costs"], result["task"]
335pc = lambda x: f"{100 * x:.0f}"
336print(f"\nmisses {100 * r['unexplained variance']:.1f}% of the variance on held-out text ({args.k} fixed directions miss "
337 f"{100 * r['unexplained variance with the same number of fixed directions']:.0f}%); {r['latents that never fire']} latents never fire; "
338 f"a latent fires on {100 * r['fires on share of tokens']['median']:.2f}% of tokens (median)")
339print(f"loss {c['loss']:.3f}, with the rebuilt stream {c['loss with the rebuilt stream']:.3f} (+{100 * c['loss rises by']:.1f}%), with the mean stream "
340 f"{c['loss with the mean stream']:.3f}: {100 * c['share of the loss recovered']:.1f}% recovered")
341print(f"one-token latents {pc(result['one token']['latents whose 30 strongest are one token in 80% or more'])}%")
342for row in result["fires on"]:
343 print(f" latent {row['latent']:4d}, {100 * row['fires on share of tokens']:.2f}% of tokens, {pc(row['share of its 30 strongest on one token'])}% of its strongest and "
344 f"{pc(row['share of all its firings on its commonest token'])}% of all on one token: " + " | ".join(f"{a[-24:]}[{b}]" for a, b in row["examples"][:3]))
345m = list(t["misses at the acting name"].values())
346print(f"\nat the acting name the dictionary misses {100 * m[0]:.1f}% of the variance around the stories' mean, "
347 f"{100 * m[1]:.1f}% of the variance among the prompts, {100 * m[2]:.0f}% of the stream's size")
348print("the model goes on with She after:", ", ".join(t["names that go on with She"]))
349for role_name in ("acts", "other"):
350 rows_ = t["names"][role_name]
351 print(f"a latent for each name when it {'acts' if role_name == 'acts' else 'is the other name'}: fires when it is / is not; best separation by size")
352 print(" " + " ".join(f"{x['name']} {pc(x['fires when it is'])}/{pc(x['fires when it is not'])} {x['best separation by size']:.2f}" for x in rows_))
353 print(f" {len({x['latent'] for x in rows_})} different latents; names with a latent at 80% and under 20%: {sum(x['a latent that fires on 80% with and under 20% without'] for x in rows_)}; "
354 f"with separation by size of 0.9 or more: {sum(x['best separation by size'] >= 0.9 for x in rows_)}")
355print(" on stories, the latents for the acting name: share of their firings on the name's token: " + " ".join(pc(x["of its story firings, on the name's token"]) for x in t["names"]["acts"]))
356print("the other name read by step 2's reader (clean / flipped, percent right; a name of the right kind):")
357for name, v in t["reader"].items():
358 print(f" from the {name:8s} {pc(v['clean'])}/{pc(v['flipped'])}; kind {pc(v['a name of the right kind'][0])}/{pc(v['a name of the right kind'][1])}")
359print("the reader fitted on the stream: " + "; ".join(f"{name} {pc(v[0])}/{pc(v[1])}" for name, v in t["reader fitted on the stream"].items()))
360print(f"what happens to the answer and to {t['veto head']} (clean | flipped):")
361for where, by in t["effects"].items():
362 g = lambda v, name: by[v][name]["mean"]
363 print(f" {where:46s} " + " | ".join(
364 f"difference {g(v, 'logit difference'):+.2f} (right {pc(g(v, 'right side'))}%, moves {g(v, 'change per prompt'):.2f} a prompt), "
365 f"acting {g(v, 'veto head adds to the acting name'):+.2f}, other {g(v, 'veto head adds to the other name'):+.2f}, "
366 f"other largest {pc(g(v, 'other name largest in what the veto head writes'))}%" for v in ("clean", "flipped")))
367
368save(f"runs/sae/sae-{meta['name']}{'-' + args.dict if args.dict else ''}.json", result)