PyTorch Reading Sweep Results

Reading sweep results means choosing a setting from validation scores without trusting noise. Noise here is score movement caused by the seed alone, with the settings held fixed.

One seed can mislead

The pain first. The top three under seed 4 look settled, until seed 5 reorders two of them.

seed4 = ["lr=0.03 width=12", "lr=0.03 width=6", "lr=0.01 width=12"]
seed5 = ["lr=0.03 width=12", "lr=0.01 width=12", "lr=0.03 width=6"]
print("Same order:", seed4 == seed5)

Compare every rank

A winner check alone misses the movement below it. Comparing rank by rank shows four of the six runs moving.

seed4 = ["0.03/12", "0.03/6", "0.01/12", "0.01/6", "0.003/12", "0.003/6"]
seed5 = ["0.03/12", "0.01/12", "0.03/6", "0.003/12", "0.01/6", "0.003/6"]
for run in seed4:
    if seed4.index(run) != seed5.index(run):
        print(run, "rank", seed4.index(run) + 1, "->", seed5.index(run) + 1)

Pick by validation

The pick uses the validation column only. Training loss can reach zero for a setting that validates worse.

results = [
    {"lr": 0.03, "width": 12, "val_loss": 0.02, "train_loss": 0.01},
    {"lr": 0.01, "width": 12, "val_loss": 0.05, "train_loss": 0.0},
]
best = min(results, key=lambda run: run["val_loss"])
print("Picked lr:", best["lr"])

Example

import torch
from torch import nn
from torch.utils.data import DataLoader, TensorDataset

def make_shapes(per_class, seed):
    g = torch.Generator().manual_seed(seed)
    rows = torch.arange(16).reshape(16, 1)
    cols = torch.arange(16).reshape(1, 16)
    images = []
    labels = []
    for label in range(3):
        for _ in range(per_class):
            img = torch.zeros(16, 16)
            if label == 0:
                top = int(torch.randint(6, 9, (1,), generator=g))
                img[top:top + 3, 2:14] = 1.0
            elif label == 1:
                dist = ((rows - 7.5) ** 2 + (cols - 7.5) ** 2).sqrt()
                img[(dist >= 4.0) & (dist <= 5.5)] = 1.0
            else:
                img[4:12, 4:6] = 1.0
                img[4:12, 10:12] = 1.0
                img[4:6, 4:12] = 1.0
                img[10:12, 4:12] = 1.0
            img = img + 0.05 * torch.rand(16, 16, generator=g)
            images.append(img)
            labels.append(label)
    x = torch.stack(images).unsqueeze(1)
    y = torch.tensor(labels)
    perm = torch.randperm(len(labels), generator=g)
    return x[perm], y[perm]



def run_one(lr, width, seed, x_val, y_val):
    x, y = make_shapes(12, 100 + seed)
    x = x.flatten(1)
    torch.manual_seed(seed)
    model = nn.Sequential(
        nn.Linear(256, width),
        nn.ReLU(),
        nn.Linear(width, 3),
    )
    loss_fn = nn.CrossEntropyLoss()
    optimizer = torch.optim.Adam(model.parameters(), lr=lr)
    generator = torch.Generator().manual_seed(seed)
    loader = DataLoader(
        TensorDataset(x, y), batch_size=12, shuffle=True, generator=generator
    )
    for epoch in range(4):
        for xb, yb in loader:
            loss = loss_fn(model(xb), yb)
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
    model.eval()
    with torch.no_grad():
        scores = model(x_val)
        val_loss = loss_fn(scores, y_val).item()
        val_acc = (scores.argmax(dim=1) == y_val).float().mean().item()
    return round(val_loss, 4), round(val_acc, 4)


x_val, y_val = make_shapes(15, 200)
x_val = x_val.flatten(1)

orders = {}
for seed in (4, 5):
    results = []
    for lr in (0.003, 0.01, 0.03):
        for width in (6, 12):
            val_loss, val_acc = run_one(lr, width, seed, x_val, y_val)
            results.append((val_loss, val_acc, lr, width))
    results.sort()
    orders[seed] = [(lr, width) for val_loss, val_acc, lr, width in results]
    print(f"Seed {seed} ranking:")
    for rank, (val_loss, val_acc, lr, width) in enumerate(results, start=1):
        print(f"  {rank}. lr={lr} width={width} loss={val_loss} acc={val_acc}")

for lr, width in orders[4]:
    rank4 = orders[4].index((lr, width)) + 1
    rank5 = orders[5].index((lr, width)) + 1
    if rank4 != rank5:
        print(f"Moved: lr={lr} width={width} rank {rank4} -> {rank5}")
print("Rankings identical:", orders[4] == orders[5])

Key points

Seed noise: score changes that come from the seed alone, not from a better setting
Use validation scores to pick a setting, never training scores
Wide gap: a lead much larger than the movement between seeds
Close runs swap ranks between seeds, as the closing example shows
Re-running a sweep with a second seed tests whether its ranking is stable
A ranking is a measurement with noise, not a verdict

In this example, the six sweep settings run twice, once under seed 4 and once under seed 5. Each seed redraws the 36 training images, restarts the model, and reshuffles the batches. The small training set and four short epochs keep each run noisy on purpose. Both rankings print in full, ranked by validation loss. Learning rate 0.03 with width 12 wins under both seeds. Below it, four runs change rank, and the moved lines print each change. The last line confirms the two rankings are not identical. The honest pick is still the winner, because its lead survived the seed change.

Note

The winner stayed first under both seeds in this run, because its lead was wider than the seed noise

Try it Yourself

Edit the code, then press Run.

Running is simulated · Python Playground

import torch
from torch import nn
from torch.utils.data import DataLoader, TensorDataset

def make_shapes(per_class, seed):
    g = torch.Generator().manual_seed(seed)
    rows = torch.arange(16).reshape(16, 1)
    cols = torch.arange(16).reshape(1, 16)
    images = []
    labels = []
    for label in range(3):
        for _ in range(per_class):
            img = torch.zeros(16, 16)
            if label == 0:
                top = int(torch.randint(6, 9, (1,), generator=g))
                img[top:top + 3, 2:14] = 1.0
            elif label == 1:
                dist = ((rows - 7.5) ** 2 + (cols - 7.5) ** 2).sqrt()
                img[(dist >= 4.0) & (dist <= 5.5)] = 1.0
            else:
                img[4:12, 4:6] = 1.0
                img[4:12, 10:12] = 1.0
                img[4:6, 4:12] = 1.0
                img[10:12, 4:12] = 1.0
            img = img + 0.05 * torch.rand(16, 16, generator=g)
            images.append(img)
            labels.append(label)
    x = torch.stack(images).unsqueeze(1)
    y = torch.tensor(labels)
    perm = torch.randperm(len(labels), generator=g)
    return x[perm], y[perm]



def run_one(lr, width, seed, x_val, y_val):
    x, y = make_shapes(12, 100 + seed)
    x = x.flatten(1)
    torch.manual_seed(seed)
    model = nn.Sequential(
        nn.Linear(256, width),
        nn.ReLU(),
        nn.Linear(width, 3),
    )
    loss_fn = nn.CrossEntropyLoss()
    optimizer = torch.optim.Adam(model.parameters(), lr=lr)
    generator = torch.Generator().manual_seed(seed)
    loader = DataLoader(
        TensorDataset(x, y), batch_size=12, shuffle=True, generator=generator
    )
    for epoch in range(4):
        for xb, yb in loader:
            loss = loss_fn(model(xb), yb)
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
    model.eval()
    with torch.no_grad():
        scores = model(x_val)
        val_loss = loss_fn(scores, y_val).item()
        val_acc = (scores.argmax(dim=1) == y_val).float().mean().item()
    return round(val_loss, 4), round(val_acc, 4)


x_val, y_val = make_shapes(15, 200)
x_val = x_val.flatten(1)

orders = {}
for seed in (4, 5):
    results = []
    for lr in (0.003, 0.01, 0.03):
        for width in (6, 12):
            val_loss, val_acc = run_one(lr, width, seed, x_val, y_val)
            results.append((val_loss, val_acc, lr, width))
    results.sort()
    orders[seed] = [(lr, width) for val_loss, val_acc, lr, width in results]
    print(f"Seed {seed} ranking:")
    for rank, (val_loss, val_acc, lr, width) in enumerate(results, start=1):
        print(f"  {rank}. lr={lr} width={width} loss={val_loss} acc={val_acc}")

for lr, width in orders[4]:
    rank4 = orders[4].index((lr, width)) + 1
    rank5 = orders[5].index((lr, width)) + 1
    if rank4 != rank5:
        print(f"Moved: lr={lr} width={width} rank {rank4} -> {rank5}")
print("Rankings identical:", orders[4] == orders[5])

Exercise

Two settings score 0.401 and 0.404 under one seed, then swap under a second seed. What is the honest reading?

The two settings are effectively tied at this data size The second seed must be wrong The lower number always wins Noise never affects rankings