LLM Evaluation Fundamentals

# implement compute_perplexity_and_bpb
# Key operations: loss calculation, training loop, results display
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer

def compute_perplexity_and_bpb(model_name: str, text: str):
 """Compute perplexity and bits-per-byte for a text sample."""
 tokenizer = AutoTokenizer.from_pretrained(model_name)
 model = AutoModelForCausalLM.from_pretrained(
 model_name, torch_dtype=torch.float16, device_map="auto"
 )

 encodings = tokenizer(text, return_tensors="pt")
 input_ids = encodings.input_ids.to(model.device)

 with torch.no_grad():
 outputs = model(input_ids, labels=input_ids)
 neg_log_likelihood = outputs.loss # average NLL per token

 num_tokens = input_ids.size(1)
 perplexity = torch.exp(neg_log_likelihood).item()

 # Bits-per-byte: convert nats to bits, normalize by bytes
 total_nll_bits = neg_log_likelihood.item() * num_tokens / torch.log(torch.tensor(2.0))
 num_bytes = len(text.encode("utf-8"))
 bpb = total_nll_bits / num_bytes

 return {
 "perplexity": round(perplexity, 2),
 "bits_per_byte": round(bpb.item(), 4),
 "num_tokens": num_tokens,
 "num_bytes": num_bytes
 }

# Example usage
text = "The transformer architecture revolutionized natural language processing."
result = compute_perplexity_and_bpb("gpt2", text)
print(result)

# Implementation example
# Key operations: results display, evaluation logic
from rouge_score import rouge_scorer
from bert_score import score as bert_score
import evaluate

# Reference and candidate texts
reference = "The cat sat on the mat and watched the birds outside."
candidate = "A cat was sitting on a mat, observing birds through the window."

# ROUGE scores
scorer = rouge_scorer.RougeScorer(["rouge1", "rouge2", "rougeL"], use_stemmer=True)
rouge_results = scorer.score(reference, candidate)
for key, value in rouge_results.items():
 print(f"{key}: precision={value.precision:.3f}, recall={value.recall:.3f}, f1={value.fmeasure:.3f}")

# BLEU score
bleu = evaluate.load("bleu")
bleu_result = bleu.compute(
 predictions=[candidate],
 references=[[reference]]
)
print(f"BLEU: {bleu_result['bleu']:.4f}")

# BERTScore
P, R, F1 = bert_score([candidate], [reference], lang="en", verbose=False)
print(f"BERTScore: precision={P[0]:.4f}, recall={R[0]:.4f}, f1={F1[0]:.4f}")

# implement llm_judge_evaluate
# Key operations: results display, prompt construction, evaluation logic
from openai import OpenAI
import json

client = OpenAI()

def llm_judge_evaluate(question: str, answer: str, criteria: str) -> dict:
 """Use an LLM as a judge to evaluate answer quality."""
 judge_prompt = f"""You are an expert evaluator. Assess the following answer to a question.

QUESTION: {question}

ANSWER: {answer}

EVALUATION CRITERIA: {criteria}

RUBRIC:
5 = Excellent: Fully addresses the question with accurate, comprehensive, well-organized content
4 = Good: Addresses the question well with minor gaps or imprecisions
3 = Adequate: Partially addresses the question but has notable gaps
2 = Poor: Mostly fails to address the question or contains significant errors
1 = Very Poor: Completely off-topic, factually wrong, or incoherent

Provide your evaluation as JSON with these fields:
- "reasoning": Your step-by-step analysis (2-3 sentences)
- "score": Integer from 1 to 5
- "strengths": List of strengths
- "weaknesses": List of weaknesses"""

 response = client.chat.completions.create(
 model="gpt-4o",
 messages=[{"role": "user", "content": judge_prompt}],
 response_format={"type": "json_object"},
 temperature=0.0
 )

 return json.loads(response.choices[0].message.content)

# Example evaluation
result = llm_judge_evaluate(
 question="What causes seasons on Earth?",
 answer="Seasons happen because Earth's axis is tilted at 23.5 degrees relative to its orbital plane. This means different hemispheres receive more direct sunlight at different times of year.",
 criteria="Factual accuracy, completeness, and clarity"
)
print(json.dumps(result, indent=2))

# implement compute_inter_annotator_agreement
# Key operations: results display
import numpy as np
from sklearn.metrics import cohen_kappa_score

def compute_inter_annotator_agreement(annotations: dict) -> dict:
 """Compute pairwise Cohen's kappa between annotators.

 Args:
 annotations: dict mapping annotator_id to list of scores
 """
 annotator_ids = list(annotations.keys())
 kappas = {}

 for i in range(len(annotator_ids)):
 for j in range(i + 1, len(annotator_ids)):
 a1, a2 = annotator_ids[i], annotator_ids[j]
 kappa = cohen_kappa_score(
 annotations[a1], annotations[a2], weights="quadratic"
 )
 kappas[f"{a1}_vs_{a2}"] = round(kappa, 3)

 avg_kappa = np.mean(list(kappas.values()))
 return {"pairwise_kappas": kappas, "mean_kappa": round(avg_kappa, 3)}

# Three annotators rating 10 examples on a 1-5 scale
annotations = {
 "annotator_A": [4, 3, 5, 2, 4, 3, 5, 4, 3, 4],
 "annotator_B": [4, 3, 4, 2, 5, 3, 5, 3, 3, 4],
 "annotator_C": [5, 3, 5, 3, 4, 2, 4, 4, 3, 5],
}

agreement = compute_inter_annotator_agreement(annotations)
print(f"Pairwise kappas: {agreement['pairwise_kappas']}")
print(f"Mean kappa: {agreement['mean_kappa']}")
# Interpretation: >0.8 = excellent, 0.6-0.8 = good, 0.4-0.6 = moderate

# implement run_mmlu_sample
# Key operations: prompt construction, evaluation logic
import json
from datasets import load_dataset

def run_mmlu_sample(model_fn, num_samples: int = 20):
 """Evaluate a model on a sample of MMLU questions.

 Args:
 model_fn: callable that takes a prompt and returns A/B/C/D
 num_samples: number of questions to evaluate
 """
 dataset = load_dataset("cais/mmlu", "all", split="test")
 dataset = dataset.shuffle(seed=42).select(range(num_samples))

 correct = 0
 results_by_subject = {}

 for example in dataset:
 choices = example["choices"]
 prompt = f"""Question: {example['question']}

A) {choices[0]}
B) {choices[1]}
C) {choices[2]}
D) {choices[3]}

Answer with just the letter (A, B, C, or D):"""

 prediction = model_fn(prompt).strip().upper()
 answer_map = {0: "A", 1: "B", 2: "C", 3: "D"}
 correct_letter = answer_map[example["answer"]]
 is_correct = prediction[0] == correct_letter if prediction else False

 subject = example["subject"]
 if subject not in results_by_subject:
 results_by_subject[subject] = {"correct": 0, "total": 0}
 results_by_subject[subject]["total"] += 1
 if is_correct:
 results_by_subject[subject]["correct"] += 1
 correct += 1

 accuracy = correct / num_samples
 return {
 "overall_accuracy": round(accuracy, 3),
 "num_correct": correct,
 "num_total": num_samples,
 "by_subject": results_by_subject
 }

# Define EvalCase, EvalResult, EvalHarness; implement __init__, evaluate, summary
# Key operations: prompt construction, evaluation logic
from dataclasses import dataclass, field
from typing import Callable
import json, time

@dataclass
class EvalCase:
 """A single evaluation test case."""
 question: str
 reference: str = ""
 metadata: dict = field(default_factory=dict)

@dataclass
class EvalResult:
 """Result of evaluating one test case."""
 case: EvalCase
 model_output: str
 scores: dict
 latency_ms: float

class EvalHarness:
 """Lightweight evaluation harness for LLM applications."""

 def __init__(self, model_fn: Callable, scorers: dict[str, Callable]):
 self.model_fn = model_fn # callable: prompt -> response
 self.scorers = scorers # dict of name -> scoring function

 def evaluate(self, cases: list[EvalCase]) -> list[EvalResult]:
 """Run all test cases through the model and score them."""
 results = []
 for case in cases:
 start = time.time()
 output = self.model_fn(case.question)
 latency = (time.time() - start) * 1000

 scores = {}
 for name, scorer in self.scorers.items():
 scores[name] = scorer(
 question=case.question,
 output=output,
 reference=case.reference
 )

 results.append(EvalResult(
 case=case, model_output=output,
 scores=scores, latency_ms=round(latency, 1)
 ))
 return results

 def summary(self, results: list[EvalResult]) -> dict:
 """Aggregate results into a summary report."""
 import numpy as np
 metric_names = list(results[0].scores.keys())
 summary = {}
 for metric in metric_names:
 values = [r.scores[metric] for r in results]
 summary[metric] = {
 "mean": round(np.mean(values), 3),
 "std": round(np.std(values), 3),
 "min": round(min(values), 3),
 "max": round(max(values), 3),
 }
 summary["mean_latency_ms"] = round(
 np.mean([r.latency_ms for r in results]), 1
 )
 return summary