LLM Product Management

# Define RiskLevel, LLMProductSpec; implement model_tier_recommendation
# Key operations: training loop, results display, safety guardrails
from dataclasses import dataclass, field
from typing import List, Optional
from enum import Enum

class RiskLevel(Enum):
 LOW = "low" # Wrong answer is inconvenient
 MEDIUM = "medium" # Wrong answer causes rework
 HIGH = "high" # Wrong answer causes financial or legal harm
 CRITICAL = "critical" # Wrong answer endangers safety

@dataclass
class LLMProductSpec:
 """Structured LLM product specification."""
 name: str
 user_persona: str
 job_to_be_done: str

 # Functional requirements
 input_types: List[str] # text, image, document, audio
 output_format: str # free text, structured JSON, classification
 max_latency_seconds: float
 context_window_needs: str # "short" (<4K), "medium" (4-32K), "long" (>32K)

 # Risk and quality
 hallucination_risk: RiskLevel
 requires_citations: bool
 human_review_required: bool
 accuracy_target: float # 0.0 to 1.0

 # Scale
 daily_requests_estimate: int
 concurrent_users_peak: int

 # Constraints
 data_residency: Optional[str] = None # "US", "EU", etc.
 pii_handling: str = "none" # "none", "redact", "allowed"

 def model_tier_recommendation(self) -> str:
 if self.hallucination_risk in (RiskLevel.HIGH, RiskLevel.CRITICAL):
 return "Frontier model (GPT-4o, Claude 3.5 Sonnet) with guardrails"
 elif self.context_window_needs == "long":
 return "Long-context model (Gemini 1.5 Pro, Claude 3.5)"
 elif self.daily_requests_estimate > 10000:
 return "Fine-tuned small model (Llama 3.1 8B) for cost efficiency"
 else:
 return "Mid-tier model (GPT-4o-mini, Claude 3.5 Haiku)"

# Example: Customer support copilot
spec = LLMProductSpec(
 name="Support Copilot",
 user_persona="Tier-1 support agent handling billing and account inquiries",
 job_to_be_done="Draft accurate responses to customer tickets using knowledge base",
 input_types=["text", "document"],
 output_format="free text with inline citations",
 max_latency_seconds=3.0,
 context_window_needs="medium",
 hallucination_risk=RiskLevel.HIGH,
 requires_citations=True,
 human_review_required=True,
 accuracy_target=0.92,
 daily_requests_estimate=5000,
 concurrent_users_peak=200,
 data_residency="US",
 pii_handling="redact",
)

print(f"Product: {spec.name}")
print(f"Model recommendation: {spec.model_tier_recommendation()}")
print(f"Accuracy target: {spec.accuracy_target:.0%}")

# Define LLMProductMetrics; implement health_check
# Key operations: results display, health checking, monitoring and metrics
from dataclasses import dataclass

@dataclass
class LLMProductMetrics:
 """Weekly metrics dashboard for an LLM product."""
 # Model quality
 accuracy: float
 hallucination_rate: float
 latency_p95: float

 # Product usage
 csat: float
 adoption_rate: float
 edit_rate: float

 # Business impact
 resolution_rate: float
 deflection_rate: float
 cost_per_resolution: float

 def health_check(self) -> dict:
 return {
 "accuracy": "PASS" if self.accuracy >= 0.85 else "FAIL",
 "hallucination": "PASS" if self.hallucination_rate < 0.05 else "FAIL",
 "csat": "PASS" if self.csat >= 4.0 else "WARN",
 "adoption": "PASS" if self.adoption_rate >= 0.60 else "WARN",
 "deflection": "PASS" if self.deflection_rate >= 0.40 else "WARN",
 }

# Week 8 metrics for Support Copilot
week8 = LLMProductMetrics(
 accuracy=0.91, hallucination_rate=0.03, latency_p95=2.8,
 csat=4.2, adoption_rate=0.72, edit_rate=0.24,
 resolution_rate=0.76, deflection_rate=0.38, cost_per_resolution=4.20
)

for metric, status in week8.health_check().items():
 print(f" {metric:15s} {status}")