Experiment Comparison and Hyperparameter Optimization

1. Comparing Runs in W&B

W&B's web UI provides built-in comparison tools: parallel coordinates plots, scatter plots, and grouped metric charts. You can also build comparisons programmatically using the W&B API.

import wandb

api = wandb.Api()

# Fetch runs from a project
runs = api.runs(
    path="my-team/llm-fine-tuning",
    filters={"tags": {"$in": ["lora"]}, "state": "finished"},
    order="-summary_metrics.best_val_loss",
)

# Build a comparison table
print(f"{'Run Name':<30} {'LR':<12} {'LoRA Rank':<10} {'Val Loss':<10}")
print("-" * 62)
for run in runs[:10]:
    print(
        f"{run.name:<30} "
        f"{run.config.get('learning_rate', 'N/A'):<12} "
        f"{run.config.get('lora_rank', 'N/A'):<10} "
        f"{run.summary.get('best_val_loss', 'N/A'):<10.4f}"
    )

The API returns run objects with full access to configs, metrics, summary values, and artifacts. This programmatic access lets you build custom dashboards, generate reports, or feed results into downstream automation.

2. Comparing Runs in MLflow

MLflow provides a similar comparison capability through its search API and UI. The search_runs function returns a Pandas DataFrame, making it easy to analyze results with standard data science tools.

import mlflow
import pandas as pd

# Search runs and get a DataFrame
df = mlflow.search_runs(
    experiment_names=["llm-fine-tuning"],
    filter_string="status = 'FINISHED'",
    order_by=["metrics.val_loss ASC"],
)

# Compare key metrics
comparison = df[
    ["run_id", "params.model", "params.learning_rate",
     "params.lora_rank", "metrics.val_loss", "metrics.val_accuracy"]
].head(10)

print(comparison.to_string(index=False))

# Find the best configuration
best = df.iloc[0]
print(f"\nBest run: {best['run_id'][:8]}")
print(f"  Model: {best['params.model']}")
print(f"  LR: {best['params.learning_rate']}")
print(f"  Val loss: {best['metrics.val_loss']:.4f}")

3. W&B Sweeps: Automated Search

Sweeps automate the process of trying many hyperparameter combinations. The sweep controller runs on W&B's servers and dispatches configurations to your local agents.

import wandb

# Define sweep configuration
sweep_config = {
    "method": "bayes",
    "metric": {"name": "val/loss", "goal": "minimize"},
    "parameters": {
        "learning_rate": {
            "distribution": "log_uniform_values",
            "min": 1e-6,
            "max": 1e-3,
        },
        "batch_size": {"values": [8, 16, 32, 64]},
        "warmup_steps": {"min": 0, "max": 500},
        "weight_decay": {
            "distribution": "uniform",
            "min": 0.0,
            "max": 0.3,
        },
    },
    "early_terminate": {
        "type": "hyperband",
        "min_iter": 3,
        "eta": 3,
    },
}

# Create and run the sweep
sweep_id = wandb.sweep(sweep_config, project="llm-fine-tuning")

def train_fn():
    with wandb.init() as run:
        config = wandb.config
        model = train_model(
            lr=config.learning_rate,
            bs=config.batch_size,
            warmup=config.warmup_steps,
            wd=config.weight_decay,
        )

wandb.agent(sweep_id, function=train_fn, count=50)

The early_terminate option uses Hyperband to stop unpromising runs early, saving compute. This is especially valuable for LLM fine-tuning where each run can take hours.

4. Optuna Integration

Optuna is a dedicated hyperparameter optimization framework with sophisticated search algorithms. Both W&B and MLflow integrate with Optuna, giving you the best of both worlds: Optuna's search intelligence with your tracking platform's visualization and logging.

import optuna
import mlflow

def objective(trial):
    # Suggest hyperparameters
    lr = trial.suggest_float("learning_rate", 1e-6, 1e-3, log=True)
    batch_size = trial.suggest_categorical("batch_size", [8, 16, 32])
    lora_rank = trial.suggest_categorical("lora_rank", [4, 8, 16, 32])
    warmup_ratio = trial.suggest_float("warmup_ratio", 0.0, 0.1)

    with mlflow.start_run(nested=True):
        mlflow.log_params({
            "learning_rate": lr,
            "batch_size": batch_size,
            "lora_rank": lora_rank,
            "warmup_ratio": warmup_ratio,
        })

        # Train and evaluate
        val_loss = train_and_evaluate(lr, batch_size, lora_rank, warmup_ratio)
        mlflow.log_metric("val_loss", val_loss)

    return val_loss

# Run optimization
with mlflow.start_run(run_name="optuna-sweep"):
    study = optuna.create_study(direction="minimize")
    study.optimize(objective, n_trials=30)

    # Log best parameters
    mlflow.log_params(study.best_params)
    mlflow.log_metric("best_val_loss", study.best_value)

print(f"Best params: {study.best_params}")

5. Statistical Comparison

When comparing two model configurations, a single run per configuration is insufficient because results vary due to random initialization, data shuffling, and GPU non-determinism. Run multiple seeds and use statistical tests to determine if differences are significant.

import numpy as np
from scipy import stats

# Run each configuration multiple times with different seeds
config_a_scores = [0.87, 0.89, 0.86, 0.88, 0.90]  # 5 seeds
config_b_scores = [0.91, 0.90, 0.92, 0.89, 0.91]  # 5 seeds

# Paired t-test (same data splits, different configs)
t_stat, p_value = stats.ttest_rel(config_a_scores, config_b_scores)
print(f"Mean A: {np.mean(config_a_scores):.3f} +/- {np.std(config_a_scores):.3f}")
print(f"Mean B: {np.mean(config_b_scores):.3f} +/- {np.std(config_b_scores):.3f}")
print(f"p-value: {p_value:.4f}")

if p_value < 0.05:
    print("Difference is statistically significant")
else:
    print("Difference is NOT statistically significant")

6. Visualization: Loss Curves and Parameter Importance

Both platforms offer rich visualization. For custom analysis, export data and use matplotlib or plotly.

import matplotlib.pyplot as plt
import mlflow

# Fetch training curves from multiple runs
client = mlflow.MlflowClient()
run_ids = ["abc123", "def456", "ghi789"]

fig, axes = plt.subplots(1, 2, figsize=(14, 5))

for run_id in run_ids:
    run = client.get_run(run_id)
    name = run.data.params.get("model", run_id[:8])

    # Get metric history
    train_history = client.get_metric_history(run_id, "train_loss")
    val_history = client.get_metric_history(run_id, "val_loss")

    steps = [m.step for m in train_history]
    train_vals = [m.value for m in train_history]
    val_vals = [m.value for m in val_history]

    axes[0].plot(steps, train_vals, label=name)
    axes[1].plot(steps, val_vals, label=name)

axes[0].set_title("Training Loss")
axes[1].set_title("Validation Loss")
for ax in axes:
    ax.legend()
    ax.set_xlabel("Step")
    ax.set_ylabel("Loss")

plt.tight_layout()
plt.savefig("loss_comparison.png")

7. Building a Comparison Workflow

A systematic comparison workflow has four stages: define the comparison, run the experiments, analyze the results, and document the decision.

# Automated comparison workflow
def compare_configs(configs: list[dict], num_seeds: int = 3):
    """Run multiple configs with multiple seeds and compare."""
    results = {}

    for config in configs:
        config_name = config.pop("name")
        scores = []

        for seed in range(num_seeds):
            config["seed"] = seed
            with mlflow.start_run(run_name=f"{config_name}_seed{seed}"):
                mlflow.log_params(config)
                score = train_and_evaluate(**config)
                mlflow.log_metric("val_accuracy", score)
                scores.append(score)

        results[config_name] = scores

    # Statistical comparison
    names = list(results.keys())
    for i in range(len(names)):
        for j in range(i + 1, len(names)):
            _, p = stats.ttest_rel(results[names[i]], results[names[j]])
            print(f"{names[i]} vs {names[j]}: p={p:.4f}")

    return results

Documenting the decision (which configuration was chosen and why) is the most overlooked step. Use MLflow tags or W&B notes to record the rationale alongside the experiment data. Your future self will thank you.