minterp

tests/test_probe.py

77 lines · View on GitHub · Linked from 2. Probe

1import torch
2
3from minterp import probe
4
5
6def toy(n=240, d=20, k=4, places=3, noise=0.3, seed=0):
7    """Rows whose label is written along k fixed directions, at every place."""
8    g = torch.Generator().manual_seed(seed)
9    labels = torch.randint(0, k, (n,), generator=g)
10    directions = torch.randn(places, k, d, generator=g, dtype=torch.float64)
11    X = directions[:, labels] + noise * torch.randn(places, n, d, generator=g, dtype=torch.float64)
12    return X, labels
13
14
15def test_fit_solves_the_ridge_problem():
16    X, labels = toy()
17    (X,) = probe.standardize(X)
18    Y = probe.onehot(labels[None], 4)
19    W, b = probe.fit(X, Y, 0.1)
20    # At the solution the gradient of |XW + b - Y|^2 + n * lam * |W|^2 is zero.
21    n = X.shape[1]
22    gradient = X.transpose(-1, -2) @ (X @ W + b - Y) + n * 0.1 * W
23    assert gradient.abs().max() < 1e-8
24    assert torch.allclose(b, Y.mean(0))
25    # A stronger penalty gives smaller weights.
26    assert probe.fit(X, Y, 10.0)[0].norm() < W.norm()
27
28
29def test_a_written_label_is_read_and_a_random_one_is_not():
30    X, labels = toy()
31    g = torch.Generator().manual_seed(1)
32    random = torch.randint(0, 4, labels.shape, generator=g)
33    groups = torch.arange(len(labels)) % 4
34    train, test = slice(0, 180), slice(180, 240)
35    out, chosen = probe.read(X[:, train], torch.stack([labels, random])[:, train], groups[train], X[:, test], 4)
36    assert out.shape == (3, 60, 2, 4) and chosen.shape == (3, 2)
37    guess = out.argmax(-1)
38    assert (guess[:, :, 0] == labels[test]).double().mean() > 0.95
39    assert (guess[:, :, 1] == random[test]).double().mean() < 0.45  # chance is 0.25
40
41
42def test_two_runs_give_the_same_numbers_and_scale_does_not_matter():
43    X, labels = toy()
44    groups = torch.arange(len(labels)) % 4
45    args = (labels[None, :180], groups[:180])
46    one, _ = probe.read(X[:, :180], *args, X[:, 180:], 4)
47    two, _ = probe.read(X[:, :180], *args, X[:, 180:], 4)
48    assert torch.equal(one, two)
49    # Standardizing by the training rows makes the reader blind to the stream's size.
50    big, _ = probe.read(60 * X[:, :180], *args, 60 * X[:, 180:], 4)
51    assert torch.allclose(one, big, atol=1e-9)
52
53
54def test_standardize_uses_the_training_rows_only():
55    X, _ = toy()
56    train, test = probe.standardize(X[:, :100], X[:, 100:])
57    assert train.mean(1).abs().max() < 1e-12 and torch.allclose(train.std(1), torch.ones(3, 20, dtype=torch.float64))
58    shifted = probe.standardize(X[:, :100], X[:, 100:] + 5.0)[1]
59    assert torch.allclose(shifted - test, 5.0 / X[:, :100].std(1, keepdim=True).expand_as(test))
60
61
62def test_onehot_puts_the_targets_side_by_side():
63    y = probe.onehot(torch.tensor([[0, 2], [1, 1]]), 3)
64    assert y.tolist() == [[1, 0, 0, 0, 1, 0], [0, 0, 1, 0, 1, 0]]
65
66
67def test_weights_are_the_reader_that_read_applies():
68    X, labels = toy()
69    groups = torch.arange(len(labels)) % 4
70    out, _ = probe.read(X[:1, :180], labels[None, :180], groups[:180], X[:1, 180:], 4)
71    W, b, mean, spread = probe.weights(X[0, :180], labels[:180], groups[:180], 4)
72    again = ((X[0, 180:] - mean) / spread) @ W + b
73    assert torch.allclose(again, out[0, :, 0], atol=1e-9)
74    # The scores of two parts of an input add up to the score of the input.
75    part = 0.3 * X[0, 180:]
76    rest = X[0, 180:] - part
77    assert torch.allclose(((part - mean) / spread) @ W + b + (rest / spread) @ W, again, atol=1e-9)