LLM Experiment Reproducibility

# config/experiment.yaml
defaults:
 - model: gpt4o
 - prompt: rag_v2
 - retrieval: dense_rerank
 - eval: standard

experiment:
 name: rag_ablation_cot
 seed: 42
 num_eval_seeds: 5

# config/model/gpt4o.yaml
provider: openai
model_name: gpt-4o-2024-08-06
temperature: 0.0
max_tokens: 1024
seed: 42

# config/retrieval/dense_rerank.yaml
embedding_model: text-embedding-3-small
top_k: 10
reranker: cohere-rerank-v3
rerank_top_n: 3
chunk_size: 512
chunk_overlap: 50

# implement run_experiment, evaluate_pipeline, save_results
# Key operations: results display, retrieval pipeline, evaluation logic
import hydra
from omegaconf import DictConfig, OmegaConf

@hydra.main(version_base=None, config_path="config", config_name="experiment")
def run_experiment(cfg: DictConfig):
 """Run an LLM experiment with full configuration tracking."""
 # Hydra automatically saves the full resolved config
 print(OmegaConf.to_yaml(cfg))

 # Access nested config values
 model_name = cfg.model.model_name
 temperature = cfg.model.temperature
 top_k = cfg.retrieval.top_k

 # Run evaluation with tracked configuration
 results = evaluate_pipeline(cfg)
 save_results(results, cfg)

def evaluate_pipeline(cfg: DictConfig) -> dict:
 """Run the evaluation pipeline with the given config."""
 # Pipeline implementation using config values...
 return {"accuracy": 0.847, "faithfulness": 0.91}

def save_results(results: dict, cfg: DictConfig):
 """Save results alongside the full config for reproducibility."""
 import json
 output = {
 "config": OmegaConf.to_container(cfg, resolve=True),
 "results": results,
 }
 with open("experiment_results.json", "w") as f:
 json.dump(output, f, indent=2)

# Run with overrides from command line:
# python experiment.py model=gpt4o_mini retrieval.top_k=20

# Initialize DVC in your project
$ dvc init
$ dvc remote add -d storage s3://my-bucket/llm-experiments

# Track evaluation dataset
$ dvc add data/eval_{dataset}_v3.jsonl
$ git add data/eval_dataset_v3.jsonl.dvc data/.gitignore
$ git commit -m "Track eval dataset v3"

# Track knowledge base snapshot
$ dvc add data/knowledge_base/
$ git add data/knowledge_{base}.dvc
$ git commit -m "Track knowledge base snapshot 2024-Q3"

# Push data to remote storage
$ dvc push

# Reproduce exact data state from any git commit
$ git checkout experiment-2024-08-15
$ dvc checkout # restores data files to match that commit

# implement track_experiment_mlflow
# Key operations: retrieval pipeline, prompt construction, evaluation logic
import mlflow
from omegaconf import DictConfig, OmegaConf

def track_experiment_mlflow(cfg: DictConfig, results: dict):
 """Track an LLM experiment run with MLflow."""
 mlflow.set_experiment(cfg.experiment.name)

 with mlflow.start_run():
 # Log all configuration parameters
 flat_cfg = {
 k: str(v) for k, v
 in OmegaConf.to_container(cfg, resolve=True).items()
 }
 mlflow.log_params({
 "model": cfg.model.model_name,
 "temperature": cfg.model.temperature,
 "top_k": cfg.retrieval.top_k,
 "reranker": cfg.retrieval.reranker,
 "seed": cfg.experiment.seed,
 })

 # Log evaluation metrics
 mlflow.log_metrics({
 "accuracy": results["accuracy"],
 "faithfulness": results["faithfulness"],
 "latency_p50_ms": results["latency_p50"],
 "cost_per_query_usd": results["cost_per_query"],
 })

 # Log prompt template as artifact
 mlflow.log_text(cfg.prompt.template, "prompt_template.txt")

 # Log full config
 mlflow.log_text(OmegaConf.to_yaml(cfg), "full_config.yaml")

 # Tag the run for easy filtering
 mlflow.set_tags({
 "experiment_type": "ablation",
 "git_sha": get_git_sha(),
 })

# Dockerfile for reproducible LLM experiments
FROM python:3.11-slim

# Pin system dependencies
RUN apt-get update && apt-get install -y --no-install-recommends \
 git curl && rm -rf /var/lib/apt/lists/*

WORKDIR /app

# Copy and install pinned Python dependencies first (cache-friendly)
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt

# Copy application code
COPY . .

# Record build metadata for traceability
ARG GIT_SHA=unknown
ARG BUILD_DATE=unknown
ENV GIT_SHA=${GIT_SHA} BUILD_DATE=${BUILD_DATE}

# Default command: run experiment with Hydra
ENTRYPOINT ["python", "experiment.py"]

# Build with metadata
$ docker build \
 --build-arg GIT_SHA=$(git rev-parse HEAD) \
 --build-arg BUILD_{DATE}=$(date -u +%Y-%m-%dT%H:%M:%SZ) \
 -t llm-experiment:v1.2.0 .

# Run experiment with config overrides
$ docker run \
 -e OPENAI_{API}_{KEY}=$OPENAI_API_KEY \
 -v $(pwd)/data:/app/data \
 -v $(pwd)/outputs:/app/outputs \
 llm-experiment:v1.2.0 \
 model=gpt4o retrieval.top_k=5 experiment.seed=42