scripts/stack.py
240 lines · View on GitHub · Linked from 9. The practitioner's stack
1"""
2Step 9: the same numbers from a library. See docs/09-stack.md.
3
4minterp writes its own hooks, lens, patches and attention so that nothing
5is hidden. Libraries exist for each of these. This script runs one of
6them, TransformerLens, on the open model of step 8 and sets its numbers
7beside minterp's, piece by piece: the same weights (the files minllm
8downloaded), the same token ids.
9
10logits minllm's loader against Hugging Face transformers, before the
11 library touches that model; then the library's wrapped model
12 against both
13pieces every piece a Tap records, against the library's hook of that
14 name, in three blocks
15pattern the attention weights minterp recomputes, against the library's
16attribute step 1's split of the logit difference, with its heads summed
17 within each block, against the library's helper for that
18 split: on the weights as published, with the final norm's gain
19 multiplied in first, and in the library's compatibility mode,
20 which folds the norms into the weights
21mode what else compatibility mode changes: the logits, and what the
22 heads record
23patch the stream of the flipped run put into the clean run at the
24 acting name, by minterp/patch.py and by the library's hooks
25steer a vector added there, by minterp/steer.py and by the library's
26 hooks
27
28The library is not a dependency of minterp. Run this with it laid over the
29project for one command, at the versions the record was made with:
30
31 uv run --with transformer-lens==4.4.0 --with transformers==5.24.0 scripts/stack.py
32"""
33
34import os
35from importlib.metadata import version
36
37import torch
38
39from minterp import attention, lens, patch, steer, tasks
40from minterp.common import MINLLM, OPEN, load, rounded, save
41from minterp.tap import Tap
42
43try:
44 from transformer_lens.model_bridge import TransformerBridge
45 from transformers import AutoModelForCausalLM, AutoTokenizer
46except ImportError:
47 raise SystemExit("TransformerLens is not installed. Run: uv run --with transformer-lens==4.4.0 --with transformers==5.24.0 scripts/stack.py")
48
49NAME, COUNT = "qwen", 16
50folder, repo = os.path.join(MINLLM, OPEN[NAME][0]), OPEN[NAME][1]
51model, tok, meta = load(NAME, device="cpu")
52n = model.cfg.n_layers
53half = n // 2
54batch = tasks.batches(tok, "cpu", people=tasks.names(tok))[0]
55clean, flipped, s = batch.clean[:COUNT], batch.flipped[:COUNT], batch.pos["S"]
56right, wrong = batch.right[:COUNT], batch.wrong[:COUNT]
57ld = lambda logits: tasks.logit_diff(logits[:, -1].float(), right, wrong).mean().item()
58gap = lambda a, b: (a - b).abs().max().item()
59size = lambda a: a.abs().max().item()
60
61with torch.no_grad(), Tap(model) as tap:
62 mine, _ = model(clean)
63
64def counted(hf_model):
65 """Count the calls of transformers' own attention and MLP modules, on the objects as they are before any wrapping."""
66 calls = {"attention": 0, "MLP": 0}
67
68 def count(kind):
69 def hook(module, args, output):
70 calls[kind] += 1
71 return hook
72
73 for layer in hf_model.model.layers:
74 layer.self_attn.register_forward_hook(count("attention"))
75 layer.mlp.register_forward_hook(count("MLP"))
76 return calls
77
78
79
80
81hf = AutoModelForCausalLM.from_pretrained(folder, torch_dtype=torch.float32)
82hf_tok = AutoTokenizer.from_pretrained(folder)
83paths = {"model.layers": len(hf.model.layers), "lm_head": hasattr(hf, "lm_head"), "transformer.h": hasattr(hf, "transformer")}
84before = hf.config._attn_implementation
85calls = counted(hf)
86with torch.no_grad():
87 theirs = hf(clean).logits
88unwrapped = dict(calls)
89bridge = TransformerBridge.boot_transformers(repo, hf_model=hf, tokenizer=hf_tok, device="cpu")
90calls.update(attention=0, MLP=0)
91with torch.no_grad():
92 logits, cache = bridge.run_with_cache(clean)
93wrapped = dict(calls)
94with torch.no_grad():
95 _, other = bridge.run_with_cache(flipped)
96
97
98PIECES = (("resid", "blocks.{}.hook_resid_post"), ("attn", "blocks.{}.hook_attn_out"), ("mlp", "blocks.{}.hook_mlp_out"), ("heads", "blocks.{}.attn.hook_z"))
99first = [t[0] for t in batch.names[:COUNT]]
100result = {"model": meta, "versions": {p: version(p) for p in ("transformer-lens", "transformers", "torch")},
101 "prompts": {"count": COUNT, "of": len(batch), "pattern": batch.template, "first names": {p: first.count(p) for p in sorted(set(first))}},
102 "the tokenizers decode the prompt alike": hf_tok.decode(clean[0].tolist()) == tok.decode(clean[0].tolist()),
103 "a string gets no first token from the library": len(bridge.to_tokens("One day")[0]) == len(tok.encode_ordinary("One day")),
104 "module paths of the transformers model": paths, "the attention setting in transformers' config": {"before wrapping": before, "after": hf.config._attn_implementation},
105 "calls of transformers' own modules in one pass": {"before wrapping": unwrapped, "wrapped": wrapped},
106 "hook points in the cache": len(cache),
107 "logits": {"largest entry": size(mine), "minllm's loader against transformers, before wrapping": gap(mine, theirs),
108 "the library against minterp": gap(logits, mine), "the library against transformers, before wrapping": gap(logits, theirs)},
109 "pieces": {'("resid", 0) against hook_embed': {"difference": gap(tap["resid", 0], cache["hook_embed"]), "largest entry": size(tap["resid", 0])}},
110 "names": {'("resid", 0)': "hook_embed"}, "canonical names": {"hook_embed": bridge.hook_dict["hook_embed"].name}}
111for kind, theirs_name in PIECES:
112 result["names"][f'("{kind}", i)'] = theirs_name.format("i-1")
113 result["canonical names"][theirs_name.format("i")] = bridge.hook_dict[theirs_name.format(0)].name.replace(".0.", ".i.")
114 for block in (1, half, n):
115 result["pieces"][f'("{kind}", {block}) against {theirs_name.format(block - 1)}'] = {"difference": gap(tap[kind, block], cache[theirs_name.format(block - 1)]),
116 "largest entry": size(tap[kind, block])}
117result["names"]["attention.patterns(model, tap)[:, i-1]"] = "blocks.i-1.attn.hook_pattern"
118weights = attention.patterns(model, tap)
119result["pattern"] = {f"block {block}": gap(weights[:, block - 1], cache[f"blocks.{block - 1}.attn.hook_pattern"]) for block in (1, half, n)}
120
121
122
123parts = lens.attribute(model, tap, right, wrong)
124named = {"embed": parts["embed"], **{f"{i}_attn_out": parts["heads"].sum(2)[:, i] for i in range(n)}, **{f"{i}_mlp_out": parts["mlp"][:, i] for i in range(n)}}
125worst = lambda attrs, labels: max(gap(attrs[j], named[label]) for j, label in enumerate(labels))
126with torch.no_grad():
127 stack, labels = cache.decompose_resid(pos_slice=-1, return_labels=True)
128 raw = cache.logit_attrs(stack, tokens=right, incorrect_tokens=wrong, pos_slice=-1)
129
130 gained = cache.logit_attrs(stack * model.norm.weight, tokens=right, incorrect_tokens=wrong, pos_slice=-1)
131 second = AutoModelForCausalLM.from_pretrained(folder, torch_dtype=torch.float32)
132 calls_in_mode = counted(second)
133 folded = TransformerBridge.boot_transformers(repo, hf_model=second, tokenizer=hf_tok, device="cpu")
134 folded.enable_compatibility_mode()
135 folded_logits, folded_cache = folded.run_with_cache(clean)
136 result["calls of transformers' own modules in one pass"]["wrapped, in compatibility mode"] = dict(calls_in_mode)
137 folded_stack, folded_labels = folded_cache.decompose_resid(pos_slice=-1, return_labels=True)
138 in_mode = folded_cache.logit_attrs(folded_stack, tokens=right, incorrect_tokens=wrong, pos_slice=-1)
139 one_token = folded_cache.logit_attrs(folded_stack, tokens=right, pos_slice=-1).sum(0)
140mean_parts = lambda attrs: torch.stack([attrs[j].mean() for j in range(len(labels))])
141ours = torch.stack([named[label].mean() for label in labels])
142published = mine[:, -1].gather(1, right[:, None])[:, 0]
143result["attribute"] = {
144 "parts": len(labels), "parts of lens.attribute before the heads are summed": 1 + parts["heads"].shape[1] * parts["heads"].shape[2] + parts["mlp"].shape[1],
145 "the logit difference": ld(mine), "minterp's parts sum to": sum(v.mean().item() for v in named.values()),
146 "the library's helper on the published weights": {"sums to": raw.sum(0).mean().item(), "largest difference to minterp, per part": worst(raw, labels),
147 "correlation of its mean parts with minterp's": torch.corrcoef(torch.stack([mean_parts(raw), ours]))[0, 1].item()},
148 "the final norm's gain": {"smallest": model.norm.weight.min().item(), "largest": model.norm.weight.max().item(), "mean": model.norm.weight.mean().item()},
149 "with the stack multiplied by the gain first": {"sums to": gained.sum(0).mean().item(), "largest difference to minterp, per part": worst(gained, labels)},
150 "in compatibility mode": {"sums to": in_mode.sum(0).mean().item(), "largest difference to minterp, per part": worst(in_mode, folded_labels),
151 "for one token's logit it sums to": one_token.mean().item(), "that token's published logit": published.mean().item(),
152 "the published logit less its mean over the vocabulary": (published - mine[:, -1].mean(-1)).mean().item()},
153}
154
155
156shift = mine - folded_logits
157result["mode"] = {
158 "logits, largest difference to the published model's": gap(folded_logits, mine), "log-probabilities, largest difference": gap(folded_logits.log_softmax(-1), mine.log_softmax(-1)),
159 "the shift within one prompt and position, largest spread over the vocabulary": (shift.max(-1).values - shift.min(-1).values).max().item(),
160 "the shift over prompts and positions": {"smallest": shift.mean(-1).min().item(), "largest": shift.mean(-1).max().item()},
161 "at the last position": {"smallest": shift.mean(-1)[:, -1].min().item(), "largest": shift.mean(-1)[:, -1].max().item()},
162 "the shift against the mean of the published logits over the vocabulary": gap(shift.mean(-1), mine.mean(-1)),
163 f"pieces after block {half} against minterp's, on the published weights": {kind: gap(tap[kind, half], folded_cache[name.format(half - 1)]) for kind, name in PIECES},
164 "largest entry of the heads there": size(tap["heads", half]),
165}
166
167moved = tap["heads", half] - folded_cache[f"blocks.{half - 1}.attn.hook_z"]
168result["mode"]["the heads' change"] = {"largest entry": size(moved), "largest spread over prompts and positions": (moved.amax((0, 1)) - moved.amin((0, 1))).max().item(),
169 "largest entry of that block's value bias": size(model.layers[half - 1].attn.v_proj.bias)}
170
171seen = {}
172
173
174def note(label):
175 def fn(activation, hook):
176 seen[label] = hook.name
177 return activation
178 return fn
179
180
181asked = ("hook_embed", f"blocks.{half - 1}.hook_resid_post", f"blocks.{half - 1}.attn.hook_z", f"blocks.{half - 1}.attn.hook_pattern")
182with torch.no_grad():
183 for label, which in (("as wrapped", bridge), ("in compatibility mode", folded)):
184 which.run_with_hooks(clean[:1], fwd_hooks=[(name, note(f"{label}: {name}")) for name in asked])
185result["the name a hook function sees"] = seen
186result["sizes"] = {"the stream after the middle block, without the first position": size(tap["resid", half][:, 1:]), "attention weights": size(weights)}
187
188
189there = f"blocks.{half - 1}.hook_resid_post"
190store, _ = patch.record(model, flipped)
191by_minterp = patch.run(model, clean, {("resid", half): patch.swap(store["resid", half], positions=[s])})
192
193
194def put(activation, hook):
195 activation[:, s] = other[there][:, s]
196 return activation
197
198
199with torch.no_grad():
200 by_library = bridge.run_with_hooks(clean, fwd_hooks=[(there, put)])
201each = tasks.logit_diff(mine[:, -1].float(), right, wrong)
202result["patch"] = {"where": f"after block {half}, at the acting name", "plain": ld(mine), "plain, per prompt": [each.min().item(), each.max().item()],
203 "by minterp": ld(by_minterp), "by the library": ld(by_library), "logits, largest difference": gap(by_minterp, by_library)}
204
205
206direction = 0.5 * (store["resid", half][:, s] - tap["resid", half][:, s])
207steered = steer.steered(model, clean, half, direction, [s], 1.0)
208
209
210def add(activation, hook):
211 activation[:, s] = activation[:, s] + direction
212 return activation
213
214
215with torch.no_grad():
216 steered_there = bridge.run_with_hooks(clean, fwd_hooks=[(there, add)])[:, -1]
217pick = lambda last: (last.gather(1, right[:, None]) - last.gather(1, wrong[:, None])).mean().item()
218result["steer"] = {"where": result["patch"]["where"], "by minterp": pick(steered), "by the library": pick(steered_there.float()), "logits, largest difference": gap(steered, steered_there)}
219
220L, A, M = result["logits"], result["attribute"], result["mode"]
221print(f"{repo} with " + ", ".join(f"{p} {v}" for p, v in result["versions"].items()) + f"; the first {COUNT} of {len(batch)} clean prompts, first names {result['prompts']['first names']}")
222print(f"logits up to {L['largest entry']:.1f}: " + "; ".join(f"{k} {v:.1e}" for k, v in L.items() if k != "largest entry"))
223print("calls of transformers' own attention and MLP modules in one pass:", result["calls of transformers' own modules in one pass"], "| the config's attention setting", before, "->", hf.config._attn_implementation)
224print("the name a hook function sees:", seen)
225print("sizes:", {k: round(v, 2) for k, v in result["sizes"].items()}, "| the heads' change in the mode:", {k: round(v, 5) for k, v in M["the heads' change"].items()})
226print("pieces: largest difference, and the largest entry")
227for name, value in result["pieces"].items():
228 print(f" {name:55s} {value['difference']:.1e} {value['largest entry']:8.2f}")
229print("attention weights: " + ", ".join(f"{name} {value:.1e}" for name, value in result["pattern"].items()))
230print("canonical names:", result["canonical names"])
231print(f"attribution over {A['parts']} parts ({A['parts of lens.attribute before the heads are summed']} before the heads are summed): the logit difference {A['the logit difference']:+.3f}; minterp's parts sum to {A['minterp' + chr(39) + 's parts sum to']:+.3f}")
232for key in ("the library's helper on the published weights", "with the stack multiplied by the gain first", "in compatibility mode"):
233 print(f" {key}: sums to {A[key]['sums to']:+.3f}, per part within {A[key]['largest difference to minterp, per part']:.2e} of minterp")
234print(f" the gain runs from {A['the final norm' + chr(39) + 's gain']['smallest']:.3f} to {A['the final norm' + chr(39) + 's gain']['largest']:.2f}; on the published weights the helper's mean parts correlate {A['the library' + chr(39) + 's helper on the published weights']['correlation of its mean parts with minterp' + chr(39) + 's']:.2f} with minterp's")
235print(f" in the mode, one token's logit: the parts sum to {A['in compatibility mode']['for one token' + chr(39) + 's logit it sums to']:.3f}; the published logit is {A['in compatibility mode']['that token' + chr(39) + 's published logit']:.3f}, less its mean over the vocabulary {A['in compatibility mode']['the published logit less its mean over the vocabulary']:.3f}")
236print("in compatibility mode: " + "; ".join(f"{k} {v if not isinstance(v, float) else round(v, 5)}" for k, v in M.items()))
237print(f"a patch {result['patch']['where']}: the logit difference {result['patch']['plain']:+.3f} becomes {result['patch']['by minterp']:+.3f} by minterp and {result['patch']['by the library']:+.3f} by the library")
238print(f"a vector added there: {result['steer']['by minterp']:+.3f} by minterp and {result['steer']['by the library']:+.3f} by the library")
239result["pattern"] = rounded(torch.tensor(list(result["pattern"].values())), 8)
240save(f"runs/stack/stack-{NAME}.json", result)