minterp

tests/test_tasks.py

57 lines · View on GitHub · Linked from 1. Read

1import torch
2
3from minterp import tasks
4from minterp.common import eot
5
6
7def test_names_keeps_single_tokens(tok, people):
8    assert tasks.names(tok, people) == people
9
10    class Splits(type(tok)):
11        def encode_ordinary(self, text):
12            return [1, 2] if text == " " + people[0] else super().encode_ordinary(text)
13
14    two = Splits.__new__(Splits)
15    two.__dict__ = tok.__dict__
16    assert tasks.names(two, people) == people[1:]
17
18
19def test_the_three_versions_differ_in_one_token(tok, people):
20    for batch in tasks.batches(tok, people=people):
21        n = len(people) * (len(people) - 1)
22        assert len(batch) == n and batch.clean.shape == batch.flipped.shape == batch.neutral.shape
23        assert (batch.clean[:, 0] == eot(tok)).all()
24        s = batch.pos["S"]
25        for other in (batch.flipped, batch.neutral):
26            differs = other != batch.clean
27            assert differs[:, s].all() and differs.sum() == n
28        assert batch.pos["A"] < batch.pos["B"] < s < batch.pos["end"] == batch.clean.shape[1] - 1
29        assert (batch.clean[:, batch.pos["A"]] == batch.wrong).all()
30        assert (batch.clean[:, batch.pos["B"]] == batch.right).all()
31        assert (batch.clean[:, s] == batch.wrong).all()  # A acts, so B is the answer
32        assert (batch.flipped[:, s] == batch.right).all()
33        for (a, b, c), row in zip(batch.names, batch.neutral):
34            assert len({a, b, c}) == 3 and tok.decode([row[s].item()]) == " " + c
35        # The third name is not tied to the second: it stands at every distance from it in the list.
36        offsets = {(people.index(c) - people.index(b)) % len(people) for _, b, c in batch.names}
37        assert offsets == set(range(1, len(people)))
38        counts = [sum(c == n for _, _, c in batch.names) for n in people]
39        assert max(counts) - min(counts) <= 2
40        # The fourth version differs from the clean one at the second name only, and has C there.
41        differs = batch.other != batch.clean
42        assert differs[:, batch.pos["B"]].all() and differs.sum() == n
43        assert (batch.other[:, batch.pos["B"]] == batch.third).all()
44        differs = batch.other_first != batch.flipped
45        assert differs[:, batch.pos["A"]].all() and differs.sum() == n
46        assert (batch.other_first[:, batch.pos["A"]] == batch.third).all()
47        assert [batch.labels[batch.pos[k]] for k in ("A", "B", "S")] == ["A", "B", "S"]
48        assert len(batch.labels) == batch.clean.shape[1]
49
50
51def test_logit_diff_reads_the_last_position():
52    logits = torch.zeros(2, 3, 5)
53    logits[0, -1, 4], logits[0, -1, 1] = 3.0, 1.0
54    logits[1, 0, 4] = 9.0  # not the last position
55    right, wrong = torch.tensor([4, 4]), torch.tensor([1, 1])
56    assert tasks.logit_diff(logits, right, wrong).tolist() == [2.0, 0.0]
57    assert tasks.logit_diff(logits[:, -1], right, wrong).tolist() == [2.0, 0.0]