LLM Experiment Reproducibility

# implement run_experiment, evaluate_pipeline, save_results
import hydra
from omegaconf import DictConfig, OmegaConf
@hydra.main(version_base=None, config_path="config", config_name="experiment")
def run_experiment(cfg: DictConfig):
    """Run an LLM experiment with full configuration tracking."""
    # Hydra automatically saves the full resolved config
    print(OmegaConf.to_yaml(cfg))
    # Access nested config values
    model_name = cfg.model.model_name
    temperature = cfg.model.temperature
    top_k = cfg.retrieval.top_k
    # Run evaluation with tracked configuration
    results = evaluate_pipeline(cfg)
    save_results(results, cfg)
def evaluate_pipeline(cfg: DictConfig) -> dict:
    """Run the evaluation pipeline with the given config."""
    # Pipeline implementation using config values...
    return {"accuracy": 0.847, "faithfulness": 0.91}
def save_results(results: dict, cfg: DictConfig):
    """Save results alongside the full config for reproducibility."""
    import json
    output = {
        "config": OmegaConf.to_container(cfg, resolve=True),
        "results": results,
        }
    with open("experiment_results.json", "w") as f:
        json.dump(output, f, indent=2)
        # Run with overrides from command line:
            # python experiment.py model=gpt4o_mini retrieval.top_k=20

# Initialize DVC in your project
$ dvc init
$ dvc remote add -d storage s3://my-bucket/llm-experiments
# Track evaluation dataset
$ dvc add data/eval_{dataset}_v3.jsonl
$ git add data/eval_dataset_v3.jsonl.dvc data/.gitignore
$ git commit -m "Track eval dataset v3"
# Track knowledge base snapshot
$ dvc add data/knowledge_base/
$ git add data/knowledge_{base}.dvc
$ git commit -m "Track knowledge base snapshot 2024-Q3"
# Push data to remote storage
$ dvc push
# Reproduce exact data state from any git commit
$ git checkout experiment-2024-08-15
$ dvc checkout # restores data files to match that commit

# implement track_experiment_mlflow
import mlflow
from omegaconf import DictConfig, OmegaConf
def track_experiment_mlflow(cfg: DictConfig, results: dict):
    """Track an LLM experiment run with MLflow."""
    mlflow.set_experiment(cfg.experiment.name)
    with mlflow.start_run():
        # Log all configuration parameters
        flat_cfg = {
            k: str(v) for k, v
            in OmegaConf.to_container(cfg, resolve=True).items()
            }
        mlflow.log_params({
            "model": cfg.model.model_name,
            "temperature": cfg.model.temperature,
            "top_k": cfg.retrieval.top_k,
            "reranker": cfg.retrieval.reranker,
            "seed": cfg.experiment.seed,
            })
        # Log evaluation metrics
        mlflow.log_metrics({
            "accuracy": results["accuracy"],
            "faithfulness": results["faithfulness"],
            "latency_p50_ms": results["latency_p50"],
            "cost_per_query_usd": results["cost_per_query"],
            })
        # Log prompt template as artifact
        mlflow.log_text(cfg.prompt.template, "prompt_template.txt")
        # Log full config
        mlflow.log_text(OmegaConf.to_yaml(cfg), "full_config.yaml")
        # Tag the run for easy filtering
        mlflow.set_tags({
            "experiment_type": "ablation",
            "git_sha": get_git_sha(),
            })

# Dockerfile for reproducible LLM experiments
FROM python:3.11-slim
# Pin system dependencies
RUN apt-get update && apt-get install -y --no-install-recommends \
 git curl && rm -rf /var/lib/apt/lists/*
WORKDIR /app
# Copy and install pinned Python dependencies first (cache-friendly)
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
# Copy application code
COPY . .
# Record build metadata for traceability
ARG GIT_SHA=unknown
ARG BUILD_DATE=unknown
ENV GIT_SHA=${GIT_SHA} BUILD_DATE=${BUILD_DATE}
# Default command: run experiment with Hydra
ENTRYPOINT ["python", "experiment.py"]

# Build with metadata
$ docker build \
 --build-arg GIT_SHA=$(git rev-parse HEAD) \
 --build-arg BUILD_{DATE}=$(date -u +%Y-%m-%dT%H:%M:%SZ) \
 -t llm-experiment:v1.2.0 .
# Run experiment with config overrides
$ docker run \
 -e OPENAI_{API}_{KEY}=$OPENAI_API_KEY \
 -v $(pwd)/data:/app/data \
 -v $(pwd)/outputs:/app/outputs \
 llm-experiment:v1.2.0 \
 model=gpt4o retrieval.top_k=5 experiment.seed=42