minterp

scripts/stack.py

240 lines · View on GitHub · Linked from 9. The practitioner's stack

1"""
2Step 9: the same numbers from a library. See docs/09-stack.md.
3
4minterp writes its own hooks, lens, patches and attention so that nothing
5is hidden. Libraries exist for each of these. This script runs one of
6them, TransformerLens, on the open model of step 8 and sets its numbers
7beside minterp's, piece by piece: the same weights (the files minllm
8downloaded), the same token ids.
9
10logits     minllm's loader against Hugging Face transformers, before the
11           library touches that model; then the library's wrapped model
12           against both
13pieces     every piece a Tap records, against the library's hook of that
14           name, in three blocks
15pattern    the attention weights minterp recomputes, against the library's
16attribute  step 1's split of the logit difference, with its heads summed
17           within each block, against the library's helper for that
18           split: on the weights as published, with the final norm's gain
19           multiplied in first, and in the library's compatibility mode,
20           which folds the norms into the weights
21mode       what else compatibility mode changes: the logits, and what the
22           heads record
23patch      the stream of the flipped run put into the clean run at the
24           acting name, by minterp/patch.py and by the library's hooks
25steer      a vector added there, by minterp/steer.py and by the library's
26           hooks
27
28The library is not a dependency of minterp. Run this with it laid over the
29project for one command, at the versions the record was made with:
30
31    uv run --with transformer-lens==4.4.0 --with transformers==5.24.0 scripts/stack.py
32"""
33
34import os
35from importlib.metadata import version
36
37import torch
38
39from minterp import attention, lens, patch, steer, tasks
40from minterp.common import MINLLM, OPEN, load, rounded, save
41from minterp.tap import Tap
42
43try:
44    from transformer_lens.model_bridge import TransformerBridge
45    from transformers import AutoModelForCausalLM, AutoTokenizer
46except ImportError:
47    raise SystemExit("TransformerLens is not installed. Run:  uv run --with transformer-lens==4.4.0 --with transformers==5.24.0 scripts/stack.py")
48
49NAME, COUNT = "qwen", 16
50folder, repo = os.path.join(MINLLM, OPEN[NAME][0]), OPEN[NAME][1]
51model, tok, meta = load(NAME, device="cpu")  # the CPU for both, so that the difference is not the device's
52n = model.cfg.n_layers
53half = n // 2
54batch = tasks.batches(tok, "cpu", people=tasks.names(tok))[0]
55clean, flipped, s = batch.clean[:COUNT], batch.flipped[:COUNT], batch.pos["S"]
56right, wrong = batch.right[:COUNT], batch.wrong[:COUNT]
57ld = lambda logits: tasks.logit_diff(logits[:, -1].float(), right, wrong).mean().item()
58gap = lambda a, b: (a - b).abs().max().item()  # the largest absolute difference, over every entry
59size = lambda a: a.abs().max().item()  # the largest absolute entry
60
61with torch.no_grad(), Tap(model) as tap:
62    mine, _ = model(clean)
63
64def counted(hf_model):
65    """Count the calls of transformers' own attention and MLP modules, on the objects as they are before any wrapping."""
66    calls = {"attention": 0, "MLP": 0}
67
68    def count(kind):
69        def hook(module, args, output):
70            calls[kind] += 1
71        return hook
72
73    for layer in hf_model.model.layers:
74        layer.self_attn.register_forward_hook(count("attention"))
75        layer.mlp.register_forward_hook(count("MLP"))
76    return calls
77
78
79# The published weights again, read by transformers. Its logits are taken before the library wraps that very model,
80# because wrapping changes it in place: the library then computes the attention itself, from transformers' weights.
81hf = AutoModelForCausalLM.from_pretrained(folder, torch_dtype=torch.float32)
82hf_tok = AutoTokenizer.from_pretrained(folder)
83paths = {"model.layers": len(hf.model.layers), "lm_head": hasattr(hf, "lm_head"), "transformer.h": hasattr(hf, "transformer")}  # what a wrapper without names of its own exposes
84before = hf.config._attn_implementation
85calls = counted(hf)
86with torch.no_grad():
87    theirs = hf(clean).logits
88unwrapped = dict(calls)
89bridge = TransformerBridge.boot_transformers(repo, hf_model=hf, tokenizer=hf_tok, device="cpu")
90calls.update(attention=0, MLP=0)
91with torch.no_grad():
92    logits, cache = bridge.run_with_cache(clean)  # token ids, not text
93wrapped = dict(calls)
94with torch.no_grad():
95    _, other = bridge.run_with_cache(flipped)
96
97# What a Tap records, and the library's name for it. Blocks are counted from 1 in minterp and from 0 there.
98PIECES = (("resid", "blocks.{}.hook_resid_post"), ("attn", "blocks.{}.hook_attn_out"), ("mlp", "blocks.{}.hook_mlp_out"), ("heads", "blocks.{}.attn.hook_z"))
99first = [t[0] for t in batch.names[:COUNT]]
100result = {"model": meta, "versions": {p: version(p) for p in ("transformer-lens", "transformers", "torch")},
101          "prompts": {"count": COUNT, "of": len(batch), "pattern": batch.template, "first names": {p: first.count(p) for p in sorted(set(first))}},
102          "the tokenizers decode the prompt alike": hf_tok.decode(clean[0].tolist()) == tok.decode(clean[0].tolist()),
103          "a string gets no first token from the library": len(bridge.to_tokens("One day")[0]) == len(tok.encode_ordinary("One day")),
104          "module paths of the transformers model": paths, "the attention setting in transformers' config": {"before wrapping": before, "after": hf.config._attn_implementation},
105          "calls of transformers' own modules in one pass": {"before wrapping": unwrapped, "wrapped": wrapped},
106          "hook points in the cache": len(cache),
107          "logits": {"largest entry": size(mine), "minllm's loader against transformers, before wrapping": gap(mine, theirs),
108                     "the library against minterp": gap(logits, mine), "the library against transformers, before wrapping": gap(logits, theirs)},
109          "pieces": {'("resid", 0) against hook_embed': {"difference": gap(tap["resid", 0], cache["hook_embed"]), "largest entry": size(tap["resid", 0])}},
110          "names": {'("resid", 0)': "hook_embed"}, "canonical names": {"hook_embed": bridge.hook_dict["hook_embed"].name}}
111for kind, theirs_name in PIECES:
112    result["names"][f'("{kind}", i)'] = theirs_name.format("i-1")
113    result["canonical names"][theirs_name.format("i")] = bridge.hook_dict[theirs_name.format(0)].name.replace(".0.", ".i.")
114    for block in (1, half, n):
115        result["pieces"][f'("{kind}", {block}) against {theirs_name.format(block - 1)}'] = {"difference": gap(tap[kind, block], cache[theirs_name.format(block - 1)]),
116                                                                                           "largest entry": size(tap[kind, block])}
117result["names"]["attention.patterns(model, tap)[:, i-1]"] = "blocks.i-1.attn.hook_pattern"
118weights = attention.patterns(model, tap)
119result["pattern"] = {f"block {block}": gap(weights[:, block - 1], cache[f"blocks.{block - 1}.attn.hook_pattern"]) for block in (1, half, n)}
120
121# Step 1's attribution, and the library's helper for the same split. lens.attribute goes down to single heads; the
122# helper stops at a block's attention, so the heads are summed within each block.
123parts = lens.attribute(model, tap, right, wrong)
124named = {"embed": parts["embed"], **{f"{i}_attn_out": parts["heads"].sum(2)[:, i] for i in range(n)}, **{f"{i}_mlp_out": parts["mlp"][:, i] for i in range(n)}}
125worst = lambda attrs, labels: max(gap(attrs[j], named[label]) for j, label in enumerate(labels))
126with torch.no_grad():
127    stack, labels = cache.decompose_resid(pos_slice=-1, return_labels=True)  # (parts, prompts, dim) at the last position
128    raw = cache.logit_attrs(stack, tokens=right, incorrect_tokens=wrong, pos_slice=-1)
129    # The helper divides by the final norm's scale and expects the norm's learned gain to be in the unembedding already.
130    gained = cache.logit_attrs(stack * model.norm.weight, tokens=right, incorrect_tokens=wrong, pos_slice=-1)
131    second = AutoModelForCausalLM.from_pretrained(folder, torch_dtype=torch.float32)
132    calls_in_mode = counted(second)
133    folded = TransformerBridge.boot_transformers(repo, hf_model=second, tokenizer=hf_tok, device="cpu")
134    folded.enable_compatibility_mode()
135    folded_logits, folded_cache = folded.run_with_cache(clean)
136    result["calls of transformers' own modules in one pass"]["wrapped, in compatibility mode"] = dict(calls_in_mode)
137    folded_stack, folded_labels = folded_cache.decompose_resid(pos_slice=-1, return_labels=True)
138    in_mode = folded_cache.logit_attrs(folded_stack, tokens=right, incorrect_tokens=wrong, pos_slice=-1)
139    one_token = folded_cache.logit_attrs(folded_stack, tokens=right, pos_slice=-1).sum(0)  # one token's logit, not a difference
140mean_parts = lambda attrs: torch.stack([attrs[j].mean() for j in range(len(labels))])
141ours = torch.stack([named[label].mean() for label in labels])
142published = mine[:, -1].gather(1, right[:, None])[:, 0]
143result["attribute"] = {
144    "parts": len(labels), "parts of lens.attribute before the heads are summed": 1 + parts["heads"].shape[1] * parts["heads"].shape[2] + parts["mlp"].shape[1],
145    "the logit difference": ld(mine), "minterp's parts sum to": sum(v.mean().item() for v in named.values()),
146    "the library's helper on the published weights": {"sums to": raw.sum(0).mean().item(), "largest difference to minterp, per part": worst(raw, labels),
147                                                       "correlation of its mean parts with minterp's": torch.corrcoef(torch.stack([mean_parts(raw), ours]))[0, 1].item()},
148    "the final norm's gain": {"smallest": model.norm.weight.min().item(), "largest": model.norm.weight.max().item(), "mean": model.norm.weight.mean().item()},
149    "with the stack multiplied by the gain first": {"sums to": gained.sum(0).mean().item(), "largest difference to minterp, per part": worst(gained, labels)},
150    "in compatibility mode": {"sums to": in_mode.sum(0).mean().item(), "largest difference to minterp, per part": worst(in_mode, folded_labels),
151                              "for one token's logit it sums to": one_token.mean().item(), "that token's published logit": published.mean().item(),
152                              "the published logit less its mean over the vocabulary": (published - mine[:, -1].mean(-1)).mean().item()},
153}
154
155# What else the mode changes. The logits move by one number per prompt and position, their mean over the vocabulary.
156shift = mine - folded_logits  # (prompts, positions, vocabulary)
157result["mode"] = {
158    "logits, largest difference to the published model's": gap(folded_logits, mine), "log-probabilities, largest difference": gap(folded_logits.log_softmax(-1), mine.log_softmax(-1)),
159    "the shift within one prompt and position, largest spread over the vocabulary": (shift.max(-1).values - shift.min(-1).values).max().item(),
160    "the shift over prompts and positions": {"smallest": shift.mean(-1).min().item(), "largest": shift.mean(-1).max().item()},
161    "at the last position": {"smallest": shift.mean(-1)[:, -1].min().item(), "largest": shift.mean(-1)[:, -1].max().item()},
162    "the shift against the mean of the published logits over the vocabulary": gap(shift.mean(-1), mine.mean(-1)),
163    f"pieces after block {half} against minterp's, on the published weights": {kind: gap(tap[kind, half], folded_cache[name.format(half - 1)]) for kind, name in PIECES},
164    "largest entry of the heads there": size(tap["heads", half]),
165}
166# What the heads record changes by the same vector at every prompt and position: the attention's value bias.
167moved = tap["heads", half] - folded_cache[f"blocks.{half - 1}.attn.hook_z"]  # (prompts, positions, heads, head_dim)
168result["mode"]["the heads' change"] = {"largest entry": size(moved), "largest spread over prompts and positions": (moved.amax((0, 1)) - moved.amin((0, 1))).max().item(),
169                                       "largest entry of that block's value bias": size(model.layers[half - 1].attn.v_proj.bias)}
170# The name a hook function is given, for a hook registered under a name of the table: as wrapped, and in the mode.
171seen = {}
172
173
174def note(label):
175    def fn(activation, hook):
176        seen[label] = hook.name
177        return activation
178    return fn
179
180
181asked = ("hook_embed", f"blocks.{half - 1}.hook_resid_post", f"blocks.{half - 1}.attn.hook_z", f"blocks.{half - 1}.attn.hook_pattern")
182with torch.no_grad():
183    for label, which in (("as wrapped", bridge), ("in compatibility mode", folded)):
184        which.run_with_hooks(clean[:1], fwd_hooks=[(name, note(f"{label}: {name}")) for name in asked])
185result["the name a hook function sees"] = seen
186result["sizes"] = {"the stream after the middle block, without the first position": size(tap["resid", half][:, 1:]), "attention weights": size(weights)}
187
188# A patch: the flipped run's stream after the middle block, at the acting name, in the clean run.
189there = f"blocks.{half - 1}.hook_resid_post"
190store, _ = patch.record(model, flipped)
191by_minterp = patch.run(model, clean, {("resid", half): patch.swap(store["resid", half], positions=[s])})
192
193
194def put(activation, hook):  # the library passes the second argument by the name `hook`
195    activation[:, s] = other[there][:, s]
196    return activation
197
198
199with torch.no_grad():
200    by_library = bridge.run_with_hooks(clean, fwd_hooks=[(there, put)])
201each = tasks.logit_diff(mine[:, -1].float(), right, wrong)
202result["patch"] = {"where": f"after block {half}, at the acting name", "plain": ld(mine), "plain, per prompt": [each.min().item(), each.max().item()],
203                   "by minterp": ld(by_minterp), "by the library": ld(by_library), "logits, largest difference": gap(by_minterp, by_library)}
204
205# A vector added there: half of the difference between the two runs' streams at that place, one vector per prompt.
206direction = 0.5 * (store["resid", half][:, s] - tap["resid", half][:, s])
207steered = steer.steered(model, clean, half, direction, [s], 1.0)
208
209
210def add(activation, hook):
211    activation[:, s] = activation[:, s] + direction
212    return activation
213
214
215with torch.no_grad():
216    steered_there = bridge.run_with_hooks(clean, fwd_hooks=[(there, add)])[:, -1]
217pick = lambda last: (last.gather(1, right[:, None]) - last.gather(1, wrong[:, None])).mean().item()
218result["steer"] = {"where": result["patch"]["where"], "by minterp": pick(steered), "by the library": pick(steered_there.float()), "logits, largest difference": gap(steered, steered_there)}
219
220L, A, M = result["logits"], result["attribute"], result["mode"]
221print(f"{repo} with " + ", ".join(f"{p} {v}" for p, v in result["versions"].items()) + f"; the first {COUNT} of {len(batch)} clean prompts, first names {result['prompts']['first names']}")
222print(f"logits up to {L['largest entry']:.1f}: " + "; ".join(f"{k} {v:.1e}" for k, v in L.items() if k != "largest entry"))
223print("calls of transformers' own attention and MLP modules in one pass:", result["calls of transformers' own modules in one pass"], "| the config's attention setting", before, "->", hf.config._attn_implementation)
224print("the name a hook function sees:", seen)
225print("sizes:", {k: round(v, 2) for k, v in result["sizes"].items()}, "| the heads' change in the mode:", {k: round(v, 5) for k, v in M["the heads' change"].items()})
226print("pieces: largest difference, and the largest entry")
227for name, value in result["pieces"].items():
228    print(f"  {name:55s} {value['difference']:.1e}   {value['largest entry']:8.2f}")
229print("attention weights: " + ", ".join(f"{name} {value:.1e}" for name, value in result["pattern"].items()))
230print("canonical names:", result["canonical names"])
231print(f"attribution over {A['parts']} parts ({A['parts of lens.attribute before the heads are summed']} before the heads are summed): the logit difference {A['the logit difference']:+.3f}; minterp's parts sum to {A['minterp' + chr(39) + 's parts sum to']:+.3f}")
232for key in ("the library's helper on the published weights", "with the stack multiplied by the gain first", "in compatibility mode"):
233    print(f"  {key}: sums to {A[key]['sums to']:+.3f}, per part within {A[key]['largest difference to minterp, per part']:.2e} of minterp")
234print(f"  the gain runs from {A['the final norm' + chr(39) + 's gain']['smallest']:.3f} to {A['the final norm' + chr(39) + 's gain']['largest']:.2f}; on the published weights the helper's mean parts correlate {A['the library' + chr(39) + 's helper on the published weights']['correlation of its mean parts with minterp' + chr(39) + 's']:.2f} with minterp's")
235print(f"  in the mode, one token's logit: the parts sum to {A['in compatibility mode']['for one token' + chr(39) + 's logit it sums to']:.3f}; the published logit is {A['in compatibility mode']['that token' + chr(39) + 's published logit']:.3f}, less its mean over the vocabulary {A['in compatibility mode']['the published logit less its mean over the vocabulary']:.3f}")
236print("in compatibility mode: " + "; ".join(f"{k} {v if not isinstance(v, float) else round(v, 5)}" for k, v in M.items()))
237print(f"a patch {result['patch']['where']}: the logit difference {result['patch']['plain']:+.3f} becomes {result['patch']['by minterp']:+.3f} by minterp and {result['patch']['by the library']:+.3f} by the library")
238print(f"a vector added there: {result['steer']['by minterp']:+.3f} by minterp and {result['steer']['by the library']:+.3f} by the library")
239result["pattern"] = rounded(torch.tensor(list(result["pattern"].values())), 8)
240save(f"runs/stack/stack-{NAME}.json", result)