minterp

scripts/sae.py

368 lines · View on GitHub · Linked from 5. Features

1"""
2Step 5: in what units is the stream written? See docs/05-features.md.
3
4Trains a sparse autoencoder on the stream after one block, on ordinary
5story text, and asks four things of it.
6
7rebuilds    how much of the stream's variance it misses on text it did not
8            train on, against 32 fixed directions; how often latents fire
9costs       the model's loss on that text with the rebuilt stream in place
10            of the real one, against the plain model and against a stream
11            replaced by its mean
12fires on    for a sample of latents taken at evenly spaced ranks of how
13            often they fire, the text on which each fires most
14the task    at the acting name's position, where step 4 found the other
15            name being fetched:
16            - for each of the twelve names, the latent that best marks it
17              as the name that acts, and as the other name
18            - how well step 2's reader reads the other name from the
19              stream, from the rebuilt stream and from what the dictionary
20              misses, and how the score of the reader fitted on the stream
21              splits over the two parts
22            - what the answer and the veto head of step 4 do when the
23              stream there is rebuilt, and when the other name is averaged
24              out of it
25
26    uv run scripts/sae.py                    # the model of minllm's stage 3, after block 4
27    uv run scripts/sae.py --model seed1
28    uv run scripts/sae.py --seed 1 --dict s1 # a second dictionary, for step 8
29"""
30
31import argparse
32import json
33import os
34
35import numpy as np
36import torch
37import torch.nn.functional as F
38from minllm.data import load_tokens
39
40from minterp import probe, tasks
41from minterp import sae as S
42from minterp.common import MINLLM, eot, load, rounded, save
43from minterp.tap import Tap, head_writes
44
45parser = argparse.ArgumentParser()
46parser.add_argument("--model", default="seed0", help="seed0, seed1 or seed2: the steps before must have run on it")
47parser.add_argument("--block", type=int, default=4, help="the stream after this block")
48parser.add_argument("--width", type=int, default=4096)
49parser.add_argument("--k", type=int, default=32)
50parser.add_argument("--steps", type=int, default=4000)
51parser.add_argument("--seed", type=int, default=0)
52parser.add_argument("--dict", default="", help="a name for another dictionary of the same place, kept beside the first (step 8)")
53args = parser.parse_args()
54
55model, tok, meta = load(args.model)
56device = next(model.parameters()).device
57SEQ, key, n = model.cfg.max_seq_len, ("resid", args.block), model.cfg.n_layers
58
59
60@torch.no_grad()
61def collect(split, windows, seed):
62    """The stream after the block at every position of random windows of story text, and the tokens."""
63    data = load_tokens(os.path.join(MINLLM, f"data/tinystories/{split}.bin"))
64    starts = np.random.default_rng(seed).integers(0, len(data) - SEQ - 1, windows)
65    vectors, tokens = [], []
66    for i in range(0, windows, 100):
67        idx = torch.from_numpy(np.stack([data[s:s + SEQ].astype(np.int64) for s in starts[i:i + 100]])).to(device)
68        with Tap(model, keep=("resid",)) as tap:
69            model(idx)
70        vectors.append(tap[key].float().cpu())
71        tokens.append(idx.cpu())
72    return torch.cat(vectors), torch.cat(tokens)
73
74
75train_vectors, _ = collect("train", 4000, seed=1)
76held, held_tokens = collect("valid", 300, seed=2)  # (windows, seq, dim), (windows, seq)
77dim = train_vectors.shape[-1]
78flat_train, flat = train_vectors.reshape(-1, dim), held.reshape(-1, dim)
79print(f"{flat_train.shape[0]:,} training vectors, {flat.shape[0]:,} held out")
80sae = S.train(flat_train, args.width, args.k, steps=args.steps, seed=args.seed, device=device,
81              log=lambda step, loss: print(f"  step {step:5d}  loss {loss:.3f}"))
82os.makedirs("out/sae", exist_ok=True)
83S.save(sae, S.path(meta["name"], args.block, args.dict))
84rebuild = lambda x: sae(x.float())[0].to(x.dtype)
85
86result = {"model": meta, "block": args.block, "width": args.width, "k": args.k, "steps": args.steps, "seed": args.seed,
87          "training vectors": flat_train.shape[0], "held-out vectors": flat.shape[0]}
88
89# rebuilds
90with torch.no_grad():
91    z = torch.cat([sae.encode(flat[i:i + 8192].to(device)).cpu() for i in range(0, flat.shape[0], 8192)])  # (vectors, width)
92often = (z > 0).float().mean(0)  # how often each latent fires
93story_mean = flat_train[:200_000].mean(0)
94# A yardstick: the best 32 fixed directions (principal components of the training vectors), against 32 chosen per vector.
95centered = (flat_train[:200_000] - story_mean).double()
96basis = torch.linalg.eigh(centered.T @ centered).eigenvectors[:, -args.k:].float()  # (dim, k)
97off = flat - story_mean
98result["rebuilds"] = {
99    "log10 of how often each latent fires": rounded(torch.histc(often.clamp_min(1e-6).log10(), bins=24, min=-5, max=0), 0),
100    "unexplained variance": S.unexplained(sae, flat),
101    "unexplained variance with the same number of fixed directions": ((off - off @ basis @ basis.T).pow(2).sum() / (flat - flat.mean(0)).pow(2).sum()).item(),
102    "latents that never fire": int((often == 0).sum()),
103    "fires on share of tokens": {q: often.quantile(p).item() for q, p in (("5%", 0.05), ("median", 0.5), ("95%", 0.95), ("most", 1.0))},
104    "active per token": (z > 0).sum(1).float().mean().item(),
105}
106
107
108# costs
109@torch.no_grad()
110def loss_with(edit=None):
111    idx = held_tokens[:100].to(device)
112    with Tap(model, edit={key: edit} if edit else None, keep=()):
113        logits, _ = model(idx)
114    return F.cross_entropy(logits[:, :-1].reshape(-1, logits.shape[-1]).float(), idx[:, 1:].reshape(-1)).item()
115
116
117mean_vector = story_mean.to(device)
118plain, spliced, meaned = loss_with(), loss_with(rebuild), loss_with(lambda x: mean_vector.to(x.dtype).expand_as(x).clone())
119result["costs"] = {"loss": plain, "loss with the rebuilt stream": spliced, "loss with the mean stream": meaned,
120                   "share of the loss recovered": (meaned - spliced) / (meaned - plain), "loss rises by": spliced / plain - 1}
121
122# fires on: latents at evenly spaced ranks of how often they fire, so that the sample is not picked by eye
123ids = held_tokens.reshape(-1)
124alive = torch.nonzero(often > 0)[:, 0]
125order = alive[often[alive].argsort(descending=True)]
126
127
128def strongest(j, count):
129    return z[:, j].topk(min(count, int((z[:, j] > 0).sum()))).indices
130
131
132def one_token(j):
133    """Of a latent's 30 strongest activations (or all, if it has fewer), the share on its commonest token."""
134    at = strongest(j, 30)
135    return torch.bincount(ids[at], minlength=model.cfg.vocab_size).max().item() / len(at)
136
137
138def examples(j, count):
139    out = []
140    for at in strongest(j, count).tolist():
141        start = max(at - 8, (at // SEQ) * SEQ)
142        out.append([tok.decode(ids[start:at].tolist()).replace("\n", " "), tok.decode([ids[at].item()])])
143    return out
144
145
146result["fires on"] = []
147for q in np.linspace(0.02, 0.98, 16):
148    j = order[int(q * (len(order) - 1))].item()
149    fired = ids[z[:, j] > 0]
150    shown = ids[strongest(j, 1)[0]].item()  # the token of its strongest example
151    result["fires on"].append({"latent": j, "fires on share of tokens": often[j].item(), "examples": examples(j, 8),
152                               "share of its 30 strongest on one token": one_token(j),
153                               "share of all its firings on its commonest token": torch.bincount(fired).max().item() / len(fired),
154                               "share of all its firings on the token of its strongest example": (fired == shown).float().mean().item()})
155purity = torch.tensor([one_token(j) for j in order.tolist()])
156result["one token"] = {"latents whose 30 strongest are one token in 80% or more": (purity >= 0.8).float().mean().item(),
157                       "in under 30%": (purity < 0.3).float().mean().item()}
158
159# ---------------------------------------------------------------- the task, at the acting name
160people = tasks.names(tok)
161k = len(people)
162name_ids = torch.tensor([tok.encode_ordinary(" " + p)[0] for p in people], device=device)
163batches = tasks.batches(tok, device, people=people)
164first = torch.tensor([people.index(a) for a, _, _ in batches[0].names])
165second = torch.tensor([people.index(b) for _, b, _ in batches[0].names])
166third = torch.tensor([people.index(c) for _, _, c in batches[0].names])
167acts = torch.stack([first, second])  # (versions, prompts): the first name acts on clean prompts, the second on flipped
168other = torch.stack([second, first])
169
170# Two kinds of names, as the model sorts them: after a sentence about the name, does it go on with " She" or " He"?
171she, he = (tok.encode_ordinary(w)[0] for w in (" She", " He"))
172with torch.no_grad():
173    after = model(torch.tensor([[eot(tok)] + tok.encode_ordinary(f"One day, {p} went to the park.") for p in people], device=device))[0][:, -1]
174kind = (after[:, she] > after[:, he]).cpu()  # (names,): True where the model goes on with " She"
175
176# The head of step 4 that is studied at the last position and carries most: 7.0 in seed0, 8.0 in seed1.
177with open(f"runs/circuit/circuit-{meta['name']}.json") as f:
178    veto = max((h for h in json.load(f)["heads"] if h["acts at"] == "end"), key=lambda h: h["carry"])
179
180at_name = {x: [] for x in ("stream", "rebuilt", "missed", "averaged", "latents")}  # per pattern and version: (prompts, dim)
181effects = {}
182MEASURES = ("logit difference", "right side", "change per prompt", "veto head adds to the acting name",
183            "veto head adds to the other name", "other name largest in what the veto head writes")
184for batch in batches:
185    s_pos = batch.pos["S"]
186    rows = torch.arange(len(batch), device=device)
187    for v, version in enumerate(("clean", "flipped")):
188        tokens = getattr(batch, version)
189        with torch.no_grad(), Tap(model, keep=("resid",)) as tap:
190            model(tokens)
191        x = tap[key][:, s_pos].float()
192        at_name["stream"].append(x.cpu())
193        at_name["rebuilt"].append(sae(x)[0].cpu())
194        at_name["missed"].append((x - sae(x)[0]).cpu())
195        at_name["latents"].append(sae.encode(x).cpu())
196        # The other name averaged out: every prompt gets the mean stream of the prompts of its pattern and version in
197        # which the same name acts. Those eleven prompts differ in one token, the other name.
198        same_actor = torch.stack([x[(acts[v] == a).to(device)].mean(0) for a in acts[v].tolist()])
199        at_name["averaged"].append(same_actor.cpu())
200
201        def put(replacement, s_pos=s_pos):
202            def edit(stream):
203                stream = stream.clone()
204                stream[:, s_pos] = replacement(stream[:, s_pos])
205                return stream
206            return edit
207
208        ways = {"plain": None, "rebuilt at the acting name": put(rebuild), "rebuilt everywhere": rebuild,
209                "the other name averaged out at the acting name": put(lambda cur, m=same_actor: m.to(cur.dtype))}
210        base = None
211        for where, edit in ways.items():
212            with torch.no_grad(), Tap(model, edit={key: edit} if edit else None) as t2:
213                logits, _ = model(tokens)
214            ld = tasks.logit_diff(logits.float(), batch.right, batch.wrong)
215            base = ld if where == "plain" else base
216            final = t2["resid", n][:, -1]
217            size = torch.rsqrt(final.pow(2).mean(-1, keepdim=True) + model.norm.eps)
218            write = head_writes(model, t2, veto["block"])[:, -1, veto["head"]]
219            names = ((write * model.norm.weight * size) @ model.lm_head.weight[name_ids].T).float()  # (prompts, names)
220            nine = torch.ones_like(names, dtype=torch.bool)
221            for who in (first, second, third):  # against the nine names that are none of A, B and C, as in step 4
222                nine[rows, who.to(device)] = False
223            rest = (names * nine).sum(1) / nine.sum(1)
224            a, o = acts[v].to(device), other[v].to(device)
225            without_actor = names.clone()
226            without_actor[rows, a] = float("-inf")
227            sign = 1 if version == "clean" else -1
228            d = effects.setdefault(where, {}).setdefault(version, {m: [] for m in MEASURES})
229            d["logit difference"].append(ld.mean().item())
230            d["right side"].append((sign * ld > 0).float().mean().item())
231            d["change per prompt"].append((ld - base).abs().mean().item())
232            d["veto head adds to the acting name"].append((names[rows, a] - rest).mean().item())
233            d["veto head adds to the other name"].append((names[rows, o] - rest).mean().item())
234            d["other name largest in what the veto head writes"].append((without_actor.argmax(1) == o).float().mean().item())
235
236patterns, prompts = len(batches), len(first)
237stacked = {x: torch.stack(parts).view(patterns, 2, prompts, -1) for x, parts in at_name.items()}  # (patterns, versions, prompts, dim)
238result["task"] = {"prompts": patterns * 2 * prompts, "veto head": f"block {veto['block']} head {veto['head']}",
239                  "names that go on with She": [p for p, g in zip(people, kind.tolist()) if g],
240                  "effects": {where: {v: {m: {"mean": float(np.mean(x)), "by pattern": rounded(torch.tensor(x), 3)} for m, x in d.items()}
241                                      for v, d in by.items()} for where, by in effects.items()}}
242
243# How much of the stream the dictionary misses at this place, where the stories' figure need not hold.
244xs, miss = stacked["stream"].reshape(-1, dim), stacked["missed"].reshape(-1, dim)
245result["task"]["misses at the acting name"] = {
246    "of the variance around the stories' mean": (miss.pow(2).sum() / (xs - story_mean).pow(2).sum()).item(),
247    "of the variance among these prompts": (miss.pow(2).sum() / (xs - xs.mean(0)).pow(2).sum()).item(),
248    "of the stream's size": (miss.norm(dim=1) / xs.norm(dim=1)).mean().item(),
249}
250
251# A latent for each name: as the name that acts, and as the other name.
252Z = stacked["latents"].reshape(-1, args.width)
253fires = Z > 0
254role = {"acts": acts[None].expand(patterns, -1, -1).reshape(-1), "other": other[None].expand(patterns, -1, -1).reshape(-1)}
255busy = torch.nonzero(fires.float().mean(0) >= 0.02)[:, 0]  # latents that fire on some of these prompts: the candidates for the size test
256result["task"]["names"] = {}
257for r, lab in role.items():
258    table = []
259    for i, name in enumerate(people):
260        yes, no = lab == i, lab != i
261        gap = fires[yes].float().mean(0) - fires[no].float().mean(0)  # fires when the name has the role, and not otherwise
262        j = gap.argmax().item()
263        # By size instead of firing: how often a prompt with the name in the role has a larger value than one without.
264        larger = (Z[yes][:, None, busy] > Z[no][None, :, busy]).float().mean((0, 1))
265        equal = (Z[yes][:, None, busy] == Z[no][None, :, busy]).float().mean((0, 1))
266        auroc = larger + equal / 2
267        on_token = ids == name_ids[i].item()
268        both = ((z[:, j] > 0) & on_token).sum().item()
269        table.append({
270            "name": name, "latent": j, "fires when it is": fires[yes, j].float().mean().item(), "fires when it is not": fires[no, j].float().mean().item(),
271            "fires on share of story tokens": often[j].item(),
272            "of its story firings, on the name's token": both / max(1, int((z[:, j] > 0).sum())),
273            "of the name's story tokens, fires on": both / max(1, int(on_token.sum())),
274            "the name's story tokens": int(on_token.sum()),
275            "share of its 30 strongest on one token": one_token(j),
276            "share of its 30 strongest on the name's token": (ids[strongest(j, 30)] == name_ids[i].item()).float().mean().item(),
277            "a latent that fires on 80% with and under 20% without": bool(((fires[yes].float().mean(0) > 0.8) & (fires[no].float().mean(0) < 0.2)).any()),
278            "best separation by size": auroc.max().item(), "its latent": busy[auroc.argmax()].item(),
279        })
280    result["task"]["names"][r] = table
281
282# The other name, which at this position is the due name of step 2, read by step 2's reader:
283# fitted on four patterns, tested on the fifth, on name pairs it did not train on.
284held_pairs = (first + second) % 3 == 0
285result["task"]["reader"] = {}
286
287
288def fold(X, f):
289    others = [q for q in range(patterns) if q != f]
290    xtr = X[others][:, :, ~held_pairs].reshape(-1, X.shape[-1])
291    ltr = other[:, ~held_pairs][None].expand(len(others), -1, -1).reshape(-1)
292    gtr = torch.tensor(others)[:, None, None].expand(len(others), 2, int((~held_pairs).sum())).reshape(-1)
293    return xtr, ltr, gtr, X[f][:, held_pairs]  # the test rows stay (versions, pairs, dim)
294
295
296truth = other[:, held_pairs]  # (versions, pairs)
297for name in ("stream", "rebuilt", "missed", "averaged"):  # the last is the floor: a stream that holds nothing of the other name
298    X = stacked[name].double()
299    right, same_kind = [], []
300    for f in range(patterns):
301        xtr, ltr, gtr, xte = fold(X, f)
302        guess = probe.read(xtr[None], ltr[None], gtr, xte.reshape(1, -1, X.shape[-1]), k)[0].argmax(-1)[0, :, 0].view(2, -1)
303        right.append((guess == truth).double().mean(1))
304        same_kind.append((kind[guess] == kind[truth]).double().mean(1))
305    right, same_kind = torch.stack(right), torch.stack(same_kind)  # (patterns, versions)
306    result["task"]["reader"][name] = {"clean": right[:, 0].mean().item(), "flipped": right[:, 1].mean().item(), "by pattern": rounded(right),
307                                      "a name of the right kind": rounded(same_kind.mean(0))}
308
309# The reader fitted on the stream, applied to the two parts of the stream. Its scores are linear, so they add.
310split = {m: [] for m in ("from the missed part: share of the margin", "right from the rebuilt part alone", "right from the missed part alone")}
311for f in range(patterns):
312    xtr, ltr, gtr, _ = fold(stacked["stream"].double(), f)
313    W, b, mean, spread = probe.weights(xtr, ltr, gtr, k)
314    kept = ((stacked["rebuilt"][f][:, held_pairs].double() - mean) / spread) @ W + b  # (versions, pairs, names)
315    lost = (stacked["missed"][f][:, held_pairs].double() / spread) @ W
316
317    def margin(scores):  # the true name's score above the mean of the other eleven
318        true = scores.gather(2, truth[:, :, None])[:, :, 0]
319        return (true - (scores.sum(2) - true) / (k - 1)).mean(1)
320
321    split["from the missed part: share of the margin"].append(margin(lost) / (margin(kept) + margin(lost)))
322    split["right from the rebuilt part alone"].append((kept.argmax(2) == truth).double().mean(1))
323    split["right from the missed part alone"].append((lost.argmax(2) == truth).double().mean(1))
324result["task"]["reader fitted on the stream"] = {m: rounded(torch.stack(x).mean(0)) for m, x in split.items()}  # (versions,)
325
326# What the latents that stand for the other name fire on in stories.
327shared = {}
328for row in result["task"]["names"]["other"]:
329    shared.setdefault(row["latent"], []).append(row["name"])
330result["task"]["other-name latents"] = [{"latent": j, "names": names, "fires on share of story tokens": often[j].item(), "examples": examples(j, 6)}
331                                        for j, names in shared.items()]
332
333# ---------------------------------------------------------------- what it found
334r, c, t = result["rebuilds"], result["costs"], result["task"]
335pc = lambda x: f"{100 * x:.0f}"
336print(f"\nmisses {100 * r['unexplained variance']:.1f}% of the variance on held-out text ({args.k} fixed directions miss "
337      f"{100 * r['unexplained variance with the same number of fixed directions']:.0f}%); {r['latents that never fire']} latents never fire; "
338      f"a latent fires on {100 * r['fires on share of tokens']['median']:.2f}% of tokens (median)")
339print(f"loss {c['loss']:.3f}, with the rebuilt stream {c['loss with the rebuilt stream']:.3f} (+{100 * c['loss rises by']:.1f}%), with the mean stream "
340      f"{c['loss with the mean stream']:.3f}: {100 * c['share of the loss recovered']:.1f}% recovered")
341print(f"one-token latents {pc(result['one token']['latents whose 30 strongest are one token in 80% or more'])}%")
342for row in result["fires on"]:
343    print(f"  latent {row['latent']:4d}, {100 * row['fires on share of tokens']:.2f}% of tokens, {pc(row['share of its 30 strongest on one token'])}% of its strongest and "
344          f"{pc(row['share of all its firings on its commonest token'])}% of all on one token: " + " | ".join(f"{a[-24:]}[{b}]" for a, b in row["examples"][:3]))
345m = list(t["misses at the acting name"].values())
346print(f"\nat the acting name the dictionary misses {100 * m[0]:.1f}% of the variance around the stories' mean, "
347      f"{100 * m[1]:.1f}% of the variance among the prompts, {100 * m[2]:.0f}% of the stream's size")
348print("the model goes on with She after:", ", ".join(t["names that go on with She"]))
349for role_name in ("acts", "other"):
350    rows_ = t["names"][role_name]
351    print(f"a latent for each name when it {'acts' if role_name == 'acts' else 'is the other name'}: fires when it is / is not; best separation by size")
352    print("  " + "  ".join(f"{x['name']} {pc(x['fires when it is'])}/{pc(x['fires when it is not'])} {x['best separation by size']:.2f}" for x in rows_))
353    print(f"  {len({x['latent'] for x in rows_})} different latents; names with a latent at 80% and under 20%: {sum(x['a latent that fires on 80% with and under 20% without'] for x in rows_)}; "
354          f"with separation by size of 0.9 or more: {sum(x['best separation by size'] >= 0.9 for x in rows_)}")
355print("  on stories, the latents for the acting name: share of their firings on the name's token: " + " ".join(pc(x["of its story firings, on the name's token"]) for x in t["names"]["acts"]))
356print("the other name read by step 2's reader (clean / flipped, percent right; a name of the right kind):")
357for name, v in t["reader"].items():
358    print(f"  from the {name:8s} {pc(v['clean'])}/{pc(v['flipped'])}; kind {pc(v['a name of the right kind'][0])}/{pc(v['a name of the right kind'][1])}")
359print("the reader fitted on the stream: " + "; ".join(f"{name} {pc(v[0])}/{pc(v[1])}" for name, v in t["reader fitted on the stream"].items()))
360print(f"what happens to the answer and to {t['veto head']} (clean | flipped):")
361for where, by in t["effects"].items():
362    g = lambda v, name: by[v][name]["mean"]
363    print(f"  {where:46s} " + " | ".join(
364        f"difference {g(v, 'logit difference'):+.2f} (right {pc(g(v, 'right side'))}%, moves {g(v, 'change per prompt'):.2f} a prompt), "
365        f"acting {g(v, 'veto head adds to the acting name'):+.2f}, other {g(v, 'veto head adds to the other name'):+.2f}, "
366        f"other largest {pc(g(v, 'other name largest in what the veto head writes'))}%" for v in ("clean", "flipped")))
367
368save(f"runs/sae/sae-{meta['name']}{'-' + args.dict if args.dict else ''}.json", result)