PyTorch Ensemble Results, Honestly

Ensemble results are the record of where a group of models agrees and where it splits. The split cases are the only cases where averaging can change the answer.

Count the agreements

Three prediction rows stack into one table. A column agrees when all three entries match.

preds = torch.tensor([[0, 1, 2, 1], [0, 2, 2, 1], [0, 1, 2, 0]])
agree = (preds[0] == preds[1]) & (preds[1] == preds[2])
print("All agree on:", int(agree.sum()), "of", preds.shape[1])

Agreeing and wrong is final

The true class here is 0, and one model's row peaks at class 1. Averaging three copies of that row changes nothing.

p = torch.tensor([0.1, 0.8, 0.1])
avg = (p + p + p) / 3
print("Averaged pick:", avg.argmax().item())

Count one pair's splits

A pair disagrees wherever their prediction rows differ. The count is a plain sum over the comparison.

a = torch.tensor([0, 1, 2, 1, 0])
b = torch.tensor([0, 2, 2, 1, 1])
print("Disagree on:", int((a != b).sum()))

Example

import torch
from torch import nn

def make_shapes(per_class, seed):
    g = torch.Generator().manual_seed(seed)
    rows = torch.arange(16).reshape(16, 1)
    cols = torch.arange(16).reshape(1, 16)
    images = []
    labels = []
    for label in range(3):
        for _ in range(per_class):
            img = torch.zeros(16, 16)
            if label == 0:
                top = int(torch.randint(6, 9, (1,), generator=g))
                img[top:top + 3, 2:14] = 1.0
            elif label == 1:
                dist = ((rows - 7.5) ** 2 + (cols - 7.5) ** 2).sqrt()
                img[(dist >= 4.0) & (dist <= 5.5)] = 1.0
            else:
                img[4:12, 4:6] = 1.0
                img[4:12, 10:12] = 1.0
                img[4:6, 4:12] = 1.0
                img[10:12, 4:12] = 1.0
            img = img + 0.05 * torch.rand(16, 16, generator=g)
            images.append(img)
            labels.append(label)
    x = torch.stack(images).unsqueeze(1)
    y = torch.tensor(labels)
    perm = torch.randperm(len(labels), generator=g)
    return x[perm], y[perm]


class ShapeCNN(nn.Module):
    def __init__(self):
        super().__init__()
        self.features = nn.Sequential(
            nn.Conv2d(1, 8, 3, padding=1),
            nn.ReLU(),
            nn.MaxPool2d(2),
            nn.Conv2d(8, 16, 3, padding=1),
            nn.ReLU(),
            nn.AdaptiveAvgPool2d(1),
        )
        self.classifier = nn.Linear(16, 3)

    def forward(self, x):
        x = self.features(x)
        x = torch.flatten(x, 1)
        return self.classifier(x)


def train_model(seed, x, y, epochs=15):
    torch.manual_seed(seed)
    model = ShapeCNN()
    loss_fn = nn.CrossEntropyLoss()
    optimizer = torch.optim.Adam(model.parameters(), lr=0.01)
    for epoch in range(epochs):
        loss = loss_fn(model(x), y)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
    return model

x_train, y_train = make_shapes(30, 100)
x_test, y_test = make_shapes(15, 200)

models = [train_model(seed, x_train, y_train) for seed in (10, 11, 12)]

probs = []
for model in models:
    model.eval()
    with torch.no_grad():
        probs.append(torch.softmax(model(x_test), dim=1))

preds = torch.stack([p.argmax(dim=1) for p in probs])
ensemble_pred = ((probs[0] + probs[1] + probs[2]) / 3).argmax(dim=1)

agree = (preds[0] == preds[1]) & (preds[1] == preds[2])
print("Test images:", len(y_test))
print("All agree, right:", int((agree & (preds[0] == y_test)).sum()))
print("All agree, wrong:", int((agree & (preds[0] != y_test)).sum()))
print("Split vote, ensemble right:", int(((~agree) & (ensemble_pred == y_test)).sum()))
print("Split vote, ensemble wrong:", int(((~agree) & (ensemble_pred != y_test)).sum()))
for a, b in ((0, 1), (0, 2), (1, 2)):
    print(f"Models {a + 1} and {b + 1} disagree on:",
          int((preds[a] != preds[b]).sum()))

Key points

Agreement: every model gives the same prediction for one input
Where all models agree and are wrong, averaging returns the same wrong class
Where models split, the averaged probabilities decide the final class
An ensemble can only improve the split cases, never the unanimous ones
A disagreement table counts agree-right, agree-wrong, split-right, and split-wrong cases
Pair counts show which two models disagree most often

In this example, the same three models from the last lesson face the same 45 test images. The training seeds, data seeds, and epochs match that lesson exactly. The three prediction rows sort every test image into one of four boxes. All three models agree and are right on 30 images. They split on 15 images, and the averaged probabilities get all 15 right. No image lands in either wrong box in this run. The pair counts show why, since models 1 and 2 never disagree. Every split comes from the third model, and the ensemble repairs each one.

Note

In this run the agree-and-wrong count is zero, so that failure mode does not appear in the table

Try it Yourself

Edit the code, then press Run.

Running is simulated · Python Playground

import torch
from torch import nn

def make_shapes(per_class, seed):
    g = torch.Generator().manual_seed(seed)
    rows = torch.arange(16).reshape(16, 1)
    cols = torch.arange(16).reshape(1, 16)
    images = []
    labels = []
    for label in range(3):
        for _ in range(per_class):
            img = torch.zeros(16, 16)
            if label == 0:
                top = int(torch.randint(6, 9, (1,), generator=g))
                img[top:top + 3, 2:14] = 1.0
            elif label == 1:
                dist = ((rows - 7.5) ** 2 + (cols - 7.5) ** 2).sqrt()
                img[(dist >= 4.0) & (dist <= 5.5)] = 1.0
            else:
                img[4:12, 4:6] = 1.0
                img[4:12, 10:12] = 1.0
                img[4:6, 4:12] = 1.0
                img[10:12, 4:12] = 1.0
            img = img + 0.05 * torch.rand(16, 16, generator=g)
            images.append(img)
            labels.append(label)
    x = torch.stack(images).unsqueeze(1)
    y = torch.tensor(labels)
    perm = torch.randperm(len(labels), generator=g)
    return x[perm], y[perm]


class ShapeCNN(nn.Module):
    def __init__(self):
        super().__init__()
        self.features = nn.Sequential(
            nn.Conv2d(1, 8, 3, padding=1),
            nn.ReLU(),
            nn.MaxPool2d(2),
            nn.Conv2d(8, 16, 3, padding=1),
            nn.ReLU(),
            nn.AdaptiveAvgPool2d(1),
        )
        self.classifier = nn.Linear(16, 3)

    def forward(self, x):
        x = self.features(x)
        x = torch.flatten(x, 1)
        return self.classifier(x)


def train_model(seed, x, y, epochs=15):
    torch.manual_seed(seed)
    model = ShapeCNN()
    loss_fn = nn.CrossEntropyLoss()
    optimizer = torch.optim.Adam(model.parameters(), lr=0.01)
    for epoch in range(epochs):
        loss = loss_fn(model(x), y)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
    return model

x_train, y_train = make_shapes(30, 100)
x_test, y_test = make_shapes(15, 200)

models = [train_model(seed, x_train, y_train) for seed in (10, 11, 12)]

probs = []
for model in models:
    model.eval()
    with torch.no_grad():
        probs.append(torch.softmax(model(x_test), dim=1))

preds = torch.stack([p.argmax(dim=1) for p in probs])
ensemble_pred = ((probs[0] + probs[1] + probs[2]) / 3).argmax(dim=1)

agree = (preds[0] == preds[1]) & (preds[1] == preds[2])
print("Test images:", len(y_test))
print("All agree, right:", int((agree & (preds[0] == y_test)).sum()))
print("All agree, wrong:", int((agree & (preds[0] != y_test)).sum()))
print("Split vote, ensemble right:", int(((~agree) & (ensemble_pred == y_test)).sum()))
print("Split vote, ensemble wrong:", int(((~agree) & (ensemble_pred != y_test)).sum()))
for a, b in ((0, 1), (0, 2), (1, 2)):
    print(f"Models {a + 1} and {b + 1} disagree on:",
          int((preds[a] != preds[b]).sum()))

Exercise

All three models predict class 1 for an image whose label is 0. What does the ensemble predict?

Class 1 Class 0 A new class It refuses to answer