scripts/open.py
187 lines · View on GitHub · Linked from 8. Limits
1"""
2Step 8: the same task in a published model. See docs/08-limits.md.
3
4Qwen2.5-0.5B-Instruct, read from its published weights by minllm's stage 5
5loader: 24 blocks, 14 heads, width 896, 0.5 billion parameters. It goes
6into the same Transformer class as the story model, so the scripts of
7steps 1 and 3 run on it as they are:
8
9 uv run scripts/behavior.py --model qwen
10 uv run scripts/lens.py --model qwen
11 uv run scripts/patch.py --model qwen --stream-only # minutes; without the flag every part is moved, in hours
12 uv run scripts/heads.py # what the heads found by the full run read and write
13
14This script repeats the first experiment of step 7 there, without a
15dictionary, at the acting name's position.
16
17kinds which names the model follows with " She" and which with " He",
18 found as step 5 found them
19steer on a clean prompt, after the block in the middle of the model
20 (--block): the directions of step 7 that need no dictionary and
21 its controls, at doses from -2 to 2. The random control is as
22 long as the pair direction.
23depth at dose 1 after every second block: the pair direction, beside
24 what step 3's patch at the acting name carries at that depth, and
25 the names' means, the vector every flipped prompt shares, and the
26 two together
27
28 uv run scripts/open.py
29 uv run scripts/open.py --model seed0 --block 4 # the story models, for the comparison by depth
30 uv run scripts/open.py --model seed1 --block 4
31"""
32
33import argparse
34import json
35
36import torch
37
38from minterp import steer, tasks
39from minterp.common import eot, load, rounded, save
40from minterp.tap import Tap
41
42parser = argparse.ArgumentParser()
43parser.add_argument("--model", default="qwen")
44parser.add_argument("--block", type=int, default=None, help="where to steer: by default after the block in the middle")
45args = parser.parse_args()
46
47model, tok, meta = load(args.model)
48device = next(model.parameters()).device
49n = model.cfg.n_layers
50block = args.block if args.block is not None else n // 2
51people = tasks.names(tok)
52k = len(people)
53name_ids = torch.tensor([tok.encode_ordinary(" " + p)[0] for p in people], device=device)
54DOSES = (-2.0, -1.0, -0.5, 0.0, 0.25, 0.5, 0.75, 1.0, 1.5, 2.0)
55generator = torch.Generator().manual_seed(0)
56more = torch.Generator().manual_seed(1)
57series = lambda values: rounded(torch.tensor(values))
58
59
60she_id, he_id = (tok.encode_ordinary(w)[0] for w in (" She", " He"))
61with torch.no_grad():
62 after = model(torch.tensor([[eot(tok)] + tok.encode_ordinary(f"One day, {p} went to the park.") for p in people], device=device))[0][:, -1].float()
63pronoun = after[:, she_id] - after[:, he_id]
64she = pronoun > 0
65result = {"model": meta, "block": block, "doses": DOSES, "names": people, "names that go on with She": [p for p, g in zip(people, she.tolist()) if g],
66 "logit of She minus logit of He after each name": rounded(pronoun, 2)}
67
68batches = tasks.batches(tok, device, people=people)
69A, B, C = (torch.tensor([people.index(t[i]) for t in batches[0].names], device=device) for i in range(3))
70rows = torch.arange(len(A), device=device)
71acts = torch.stack([A, B])
72D, E = steer.stand_ins(A, B, C, she)
73apart = torch.ones(len(A), k, dtype=torch.bool, device=device)
74for names in (A, B, C, D, E):
75 apart[rows, names] = False
76own_kind = {"A": apart & (she[None] == she[A][:, None]), "B": apart & (she[None] == she[B][:, None])}
77additive = steer.term_per_name(A, B, k)
78kinds = {"both She": she[A] & she[B], "both He": ~she[A] & ~she[B], "one of each": she[A] != she[B]}
79result["pairs"] = {g: int(m.sum()) for g, m in kinds.items()}
80
81
82at_name, plain, third = [], [], []
83for batch in batches:
84 per = []
85 for version in ("clean", "flipped"):
86 with torch.no_grad(), Tap(model, keep=("resid",)) as tap:
87 logits, _ = model(getattr(batch, version))
88 per.append((torch.stack([tap["resid", d][:, batch.pos["S"]].float() for d in range(n + 1)]), logits[:, -1].float()))
89 at_name.append(torch.stack([x for x, _ in per]))
90 plain.append(torch.stack([l for _, l in per]))
91 with torch.no_grad():
92 third.append(model(batch.neutral)[0][:, -1].float())
93at_name, plain = torch.stack(at_name), torch.stack(plain)
94
95
96def directions_at(p, depth, controls=True):
97 """The directions of step 7 for pattern p after one block, built on the other four patterns."""
98 others = [q for q in range(len(batches)) if q != p]
99 x = at_name[others][:, :, depth]
100 actor = acts[None].expand(len(others), -1, -1)
101 mean_when = torch.stack([x[actor == i].mean(0) for i in range(k)])
102 pair = (x[:, 1] - x[:, 0]).mean(0)
103 order = pair.mean(0, keepdim=True).expand_as(pair)
104 means = mean_when[B] - mean_when[A]
105 if not controls:
106 return {"pair": pair, "means": means, "order": order, "means + order": means + order}
107 here = at_name[p][:, depth]
108 return {"own": here[1] - here[0], "pair": pair, "names": additive @ pair, "means + order": means + order, "means": means, "order": order,
109 "kind": steer.kind_only(mean_when, A, B, she), "other names": mean_when[D] - mean_when[E],
110 "random, long": steer.like(pair, generator), "means + random": means + steer.like(order, more), "wipe": here.mean((0, 1))[None] - here[0]}
111
112
113OUTSIDE = "to the flipped prompt, outside the two names"
114NAMES = ("own", "pair", "names", "means + order", "means", "order", "kind", "other names", "random, long", "means + random", "wipe")
115BY_DEPTH = ("pair", "means", "order", "means + order")
116result["directions"], result["templates"] = NAMES, []
117for p, batch in enumerate(batches):
118 ld = lambda logits: tasks.logit_diff(logits, batch.right, batch.wrong)
119 clean, flipped = ld(plain[p, 0]), ld(plain[p, 1])
120 gap = (clean - flipped).mean()
121 names_out = torch.stack([batch.right, batch.wrong], dim=1)
122 own = {x: lambda logits, x=x, t=t: steer.against(logits[:, name_ids], t, own_kind[x]) for x, t in (("A", A), ("B", B))}
123 between = steer.elsewhere(plain[p, 1], plain[p, 0], names_out)
124 directions = directions_at(p, block)
125 entry = {"template": batch.template, "gap": gap.item(), "stream size at the acting name": at_name[p, 0, block].norm(dim=1).mean().item(),
126 "pairs": {g: {"gap": (clean - flipped)[m].mean().item()} for g, m in kinds.items()},
127 "length": {name: {"all": d.norm(dim=1).mean().item(), **{g: d[m].norm(dim=1).mean().item() for g, m in kinds.items()}} for name, d in directions.items()},
128 "plain": {"clean": clean.mean().item(), "flipped": flipped.mean().item(), "a third name acts, share": ((clean - ld(third[p])).mean() / gap).item(),
129 "the flipped prompt turned": (flipped < 0).float().mean().item(), "the clean prompt turned": (clean < 0).float().mean().item(),
130 "from the clean prompt to the flipped, outside the two names": between.mean().item(),
131 "the same, by pair": {g: between[m].mean().item() for g, m in kinds.items()}},
132 "steered": {}, "depth": {name: {"share": [], "turned": []} for name in BY_DEPTH}}
133 for name, d in directions.items():
134 out = {m: [] for m in ("share", "turned", "A against its own kind", "B against its own kind", OUTSIDE)}
135 by = {g: [] for g in kinds}
136 for dose in DOSES:
137 logits = steer.steered(model, batch.clean, block, d, [batch.pos["S"]], dose)
138 moved = clean - ld(logits)
139 out["share"].append((moved.mean() / gap).item())
140 out["turned"].append((ld(logits) < 0).float().mean().item())
141 for x in ("A", "B"):
142 out[f"{x} against its own kind"].append(((own[x](logits) - own[x](plain[p, 0])).mean() / (own[x](plain[p, 1]) - own[x](plain[p, 0])).mean()).item())
143 out[OUTSIDE].append(steer.elsewhere(plain[p, 1], logits, names_out).mean().item())
144 for g, m in kinds.items():
145 by[g].append(moved[m].mean().item())
146 entry["steered"][name] = {**{m: series(v) for m, v in out.items()}, "moved, in logits, by pair": {g: series(v) for g, v in by.items()}}
147
148 for depth in range(0, n + 1, 2):
149 for name, d in directions_at(p, depth, controls=False).items():
150 logits = steer.steered(model, batch.clean, depth, d, [batch.pos["S"]], 1.0)
151 entry["depth"][name]["share"].append(((clean - ld(logits)).mean() / gap).item())
152 entry["depth"][name]["turned"].append((ld(logits) < 0).float().mean().item())
153 result["templates"].append(entry)
154 print(batch.template, f"clean {clean.mean():+.2f}, flipped {flipped.mean():+.2f}")
155result["depths"] = list(range(0, n + 1, 2))
156try:
157 with open(f"runs/patch/patch-{meta['name']}.json") as f:
158 travel = json.load(f)
159 at = travel["travel groups"].index("S")
160 result["the patch at the acting name, by depth"] = rounded(torch.tensor([[t["travel"]["break"][d][at] for d in result["depths"]] for t in travel["templates"]]).mean(0), 3)
161except FileNotFoundError:
162 pass
163
164T = result["templates"]
165mean = lambda pick: sum(pick(t) for t in T) / len(T)
166row = lambda values: " ".join(f"{v:6.2f}" for v in values)
167at = DOSES.index(1.0)
168print(f"\n{meta['name']}: {n} blocks; goes on with She after {', '.join(result['names that go on with She'])}; steering at the acting name after block {block}, where the stream's size is {mean(lambda t: t['stream size at the acting name']):.1f}")
169print(f"plain: clean {mean(lambda t: t['plain']['clean']):+.2f}, flipped {mean(lambda t: t['plain']['flipped']):+.2f}; a third name acts {mean(lambda t: t['plain']['a third name acts, share']):.2f} of the way; A above B in "
170 f"{100 * mean(lambda t: t['plain']['the flipped prompt turned']):.1f}% of flipped and {100 * mean(lambda t: t['plain']['the clean prompt turned']):.1f}% of clean prompts; outside the two names the clean prompt is "
171 f"{mean(lambda t: t['plain']['from the clean prompt to the flipped, outside the two names']):.2f} nats from the flipped (" + ", ".join(f"{g} {mean(lambda t, g=g: t['plain']['the same, by pair'][g]):.2f}" for g in kinds) + ")")
172for measure in ("share", "turned"):
173 print(f" {measure}\n dose " + row(DOSES))
174 for name in NAMES:
175 print(f" {name:14s} " + row([mean(lambda t, i=i: t["steered"][name][measure][i]) for i in range(len(DOSES))]))
176print(" at dose 1: A and B against their own kind; nats from the flipped prompt; length; share of the gap by pair (" + ", ".join(f"{g} {mean(lambda t, g=g: t['pairs'][g]['gap']):.2f}" for g in kinds) + ")")
177for name in NAMES:
178 print(f" {name:14s} {mean(lambda t: t['steered'][name]['A against its own kind'][at]):+.2f} {mean(lambda t: t['steered'][name]['B against its own kind'][at]):+.2f}; {mean(lambda t: t['steered'][name][OUTSIDE][at]):.2f}; "
179 f"{mean(lambda t: t['length'][name]['all']):.1f}; " + ", ".join(f"{mean(lambda t, g=g: t['steered'][name]['moved, in logits, by pair'][g][at]) / mean(lambda t, g=g: t['pairs'][g]['gap']):+.2f}" for g in kinds))
180print(" at dose 1 by depth, share of the way " + " ".join(f"{d:>5}" for d in result["depths"]))
181for name in BY_DEPTH:
182 print(f" {name:34s} " + " ".join(f"{mean(lambda t, i=i: t['depth'][name]['share'][i]):5.2f}" for i in range(len(result["depths"]))))
183print(" pair, A above B " + " ".join(f"{mean(lambda t, i=i: t['depth']['pair']['turned'][i]):5.2f}" for i in range(len(result["depths"]))))
184if "the patch at the acting name, by depth" in result:
185 print(" the patch of step 3 " + " ".join(f"{v:5.2f}" for v in result["the patch at the acting name, by depth"]))
186
187save(f"runs/open/steer-{meta['name']}.json", result)