Human-AI Interaction Patterns & Evaluation

# Framework for task-based human-AI evaluation
import time
import json
from dataclasses import dataclass, field
from enum import Enum
from typing import Optional

class Condition(Enum):
 HUMAN_ONLY = "human_only"
 AI_ONLY = "ai_only"
 HUMAN_AI = "human_ai"

@dataclass
class TaskTrial:
 participant_id: str
 condition: Condition
 task_id: str
 start_time: float = 0.0
 end_time: float = 0.0
 ai_suggestions_shown: int = 0
 ai_suggestions_accepted: int = 0
 ai_suggestions_modified: int = 0
 ai_suggestions_rejected: int = 0
 task_output: str = ""
 expert_quality_score: Optional[float] = None
 participant_confidence: Optional[int] = None # 1-7 Likert

 @property
 def duration_seconds(self) -> float:
 return self.end_time - self.start_time

 @property
 def acceptance_rate(self) -> float:
 total = self.ai_suggestions_shown
 if total == 0:
 return 0.0
 return self.ai_suggestions_accepted / total

 @property
 def modification_rate(self) -> float:
 total = self.ai_suggestions_shown
 if total == 0:
 return 0.0
 return self.ai_suggestions_modified / total

@dataclass
class StudyResults:
 trials: list[TaskTrial] = field(default_factory=list)

 def summary_by_condition(self) -> dict:
 """Compute summary statistics per condition."""
 from collections import defaultdict
 import statistics

 grouped = defaultdict(list)
 for trial in self.trials:
 grouped[trial.condition.value].append(trial)

 summary = {}
 for condition, condition_trials in grouped.items():
 durations = [t.duration_seconds for t in condition_trials]
 qualities = [
 t.expert_quality_score for t in condition_trials
 if t.expert_quality_score is not None
 ]
 summary[condition] = {
 "n": len(condition_trials),
 "mean_duration_s": round(statistics.mean(durations), 1),
 "median_duration_s": round(statistics.median(durations), 1),
 "mean_quality": (
 round(statistics.mean(qualities), 2) if qualities else None
 ),
 "std_quality": (
 round(statistics.stdev(qualities), 2)
 if len(qualities) > 1 else None
 ),
 }

 # Human-AI specific metrics
 if condition == "human_ai":
 acceptance_rates = [
 t.acceptance_rate for t in condition_trials
 ]
 summary[condition]["mean_acceptance_rate"] = round(
 statistics.mean(acceptance_rates), 3
 )

 return summary

# RealHumanEval-style productivity measurement
import statistics
from dataclasses import dataclass
from typing import Optional
import json

@dataclass
class ProductivityMetrics:
 """Metrics for a single participant across conditions."""
 participant_id: str

 # Time metrics (seconds)
 human_only_time: float
 human_ai_time: float
 ai_only_time: Optional[float] = None

 # Quality metrics (0-100 expert rating)
 human_only_quality: float = 0.0
 human_ai_quality: float = 0.0
 ai_only_quality: Optional[float] = None

 # Error metrics
 human_only_errors: int = 0
 human_ai_errors: int = 0
 ai_introduced_errors: int = 0 # Errors from accepted bad suggestions

 # Subjective metrics (1-7 Likert scales)
 perceived_usefulness: Optional[int] = None
 perceived_ease: Optional[int] = None
 trust_in_ai: Optional[int] = None
 cognitive_load: Optional[int] = None # NASA-TLX derived

 @property
 def speedup_ratio(self) -> float:
 """How much faster is human+AI vs human-only."""
 if self.human_ai_time == 0:
 return 0.0
 return self.human_only_time / self.human_ai_time

 @property
 def quality_delta(self) -> float:
 """Quality improvement from AI assistance."""
 return self.human_ai_quality - self.human_only_quality

 @property
 def net_error_impact(self) -> int:
 """Net change in errors with AI assistance."""
 return self.human_ai_errors - self.human_only_errors

def analyze_study(participants: list[ProductivityMetrics]) -> dict:
 """Compute aggregate productivity metrics."""
 speedups = [p.speedup_ratio for p in participants]
 quality_deltas = [p.quality_delta for p in participants]
 net_errors = [p.net_error_impact for p in participants]

 # Trust calibration: correlation between trust and actual quality
 trust_scores = [
 p.trust_in_ai for p in participants
 if p.trust_in_ai is not None
 ]
 quality_scores = [
 p.human_ai_quality for p in participants
 if p.trust_in_ai is not None
 ]

 return {
 "n_participants": len(participants),
 "mean_speedup": round(statistics.mean(speedups), 2),
 "median_speedup": round(statistics.median(speedups), 2),
 "pct_faster_with_ai": round(
 sum(1 for s in speedups if s > 1.0) / len(speedups) * 100, 1
 ),
 "mean_quality_delta": round(statistics.mean(quality_deltas), 2),
 "pct_quality_improved": round(
 sum(1 for d in quality_deltas if d > 0) / len(quality_deltas) * 100, 1
 ),
 "mean_net_errors": round(statistics.mean(net_errors), 2),
 "ai_introduced_errors_total": sum(
 p.ai_introduced_errors for p in participants
 ),
 }

# Example output:
# {
# "n_participants": 48,
# "mean_speedup": 1.34, # 34% faster with AI
# "pct_faster_with_ai": 79.2, # 79% of participants were faster
# "mean_quality_delta": 4.2, # 4.2 points higher quality
# "mean_net_errors": -0.8, # Fewer errors with AI (good)
# }

# Tracking trust calibration over time
import numpy as np
from dataclasses import dataclass

@dataclass
class WeeklySnapshot:
 week: int
 acceptance_rate: float # Rate of accepting AI suggestions
 accepted_accuracy: float # Accuracy of accepted suggestions
 rejected_accuracy: float # Accuracy of rejected suggestions
 override_rate: float # Rate of modifying AI suggestions
 self_reported_trust: float # 1-7 Likert scale

 @property
 def calibration_error(self) -> float:
 """Measures how well trust tracks actual accuracy.
 Low values indicate well-calibrated trust."""
 # Normalize trust to 0-1 range
 normalized_trust = (self.self_reported_trust - 1) / 6
 return abs(normalized_trust - self.accepted_accuracy)

 @property
 def discrimination(self) -> float:
 """Can the user tell good suggestions from bad ones?
 High values mean user accepts good and rejects bad."""
 return self.accepted_accuracy - (1 - self.rejected_accuracy)

def analyze_trust_trajectory(snapshots: list[WeeklySnapshot]) -> dict:
 """Analyze trust calibration over a longitudinal study."""
 early = [s for s in snapshots if s.week <= 2]
 late = [s for s in snapshots if s.week >= max(s.week for s in snapshots) - 1]

 early_cal = np.mean([s.calibration_error for s in early])
 late_cal = np.mean([s.calibration_error for s in late])

 early_disc = np.mean([s.discrimination for s in early])
 late_disc = np.mean([s.discrimination for s in late])

 return {
 "early_calibration_error": round(float(early_cal), 3),
 "late_calibration_error": round(float(late_cal), 3),
 "calibration_improved": late_cal < early_cal,
 "early_discrimination": round(float(early_disc), 3),
 "late_discrimination": round(float(late_disc), 3),
 "discrimination_improved": late_disc > early_disc,
 }

# Over-reliance detection system
from enum import Enum

class RelianceLevel(Enum):
 UNDERTRUST = "undertrust" # Rejecting good suggestions
 CALIBRATED = "calibrated" # Acceptance tracks accuracy
 SLIGHT_OVER = "slight_over" # Moderate over-acceptance
 SEVERE_OVER = "severe_over" # Accepting most suggestions uncritically

def classify_reliance(
 acceptance_rate: float,
 accepted_accuracy: float,
 rejection_of_correct: float, # Rate of rejecting correct suggestions
) -> RelianceLevel:
 """Classify a user's reliance pattern."""
 if rejection_of_correct > 0.3:
 return RelianceLevel.UNDERTRUST
 if acceptance_rate > 0.9 and accepted_accuracy < 0.8:
 return RelianceLevel.SEVERE_OVER
 if acceptance_rate > 0.8 and accepted_accuracy < 0.85:
 return RelianceLevel.SLIGHT_OVER
 return RelianceLevel.CALIBRATED

def generate_intervention(level: RelianceLevel) -> dict:
 """Suggest UX interventions based on reliance pattern."""
 interventions = {
 RelianceLevel.UNDERTRUST: {
 "action": "increase_transparency",
 "details": (
 "Show confidence scores, provide explanations, "
 "highlight the AI's track record on similar tasks"
 ),
 },
 RelianceLevel.CALIBRATED: {
 "action": "maintain_current",
 "details": "No intervention needed; trust is well-calibrated",
 },
 RelianceLevel.SLIGHT_OVER: {
 "action": "friction_nudge",
 "details": (
 "Add brief verification prompts for high-stakes "
 "suggestions; show accuracy statistics periodically"
 ),
 },
 RelianceLevel.SEVERE_OVER: {
 "action": "mandatory_review",
 "details": (
 "Require explicit confirmation for important outputs; "
 "introduce deliberate errors to calibrate vigilance; "
 "display warning when acceptance rate exceeds threshold"
 ),
 },
 }
 return interventions[level]

# UX pattern: structured AI response with metadata
from dataclasses import dataclass
from typing import Optional

@dataclass
class AIResponseMetadata:
 """Metadata to display alongside an AI response."""
 confidence: str # "high", "medium", "low"
 confidence_factors: list[str] # Why this confidence level
 sources: list[dict] # Documents/data that informed response
 reasoning_summary: str # One-sentence chain of thought
 alternatives: list[str] # Other plausible answers
 caveats: list[str] # Known limitations of this response

@dataclass
class AIResponse:
 content: str
 metadata: AIResponseMetadata

def format_response_for_ui(response: AIResponse) -> dict:
 """Structure the response for a progressive-disclosure UI."""
 return {
 # Layer 1: Primary content (always visible)
 "primary": {
 "text": response.content,
 "confidence_badge": response.metadata.confidence,
 },
 # Layer 2: Quick details (visible on hover/tap)
 "details": {
 "reasoning": response.metadata.reasoning_summary,
 "source_count": len(response.metadata.sources),
 "caveats": response.metadata.caveats,
 },
 # Layer 3: Full transparency (visible on expand)
 "deep_dive": {
 "sources": response.metadata.sources,
 "confidence_factors": response.metadata.confidence_factors,
 "alternatives": response.metadata.alternatives,
 },
 }

# Example usage
response = AIResponse(
 content="The quarterly revenue increased by 12% year-over-year.",
 metadata=AIResponseMetadata(
 confidence="high",
 confidence_factors=[
 "Multiple consistent data sources",
 "Simple factual lookup (low reasoning complexity)",
 ],
 sources=[
 {"title": "Q3 2025 Earnings Report", "page": 4},
 {"title": "Revenue Dashboard", "date": "2025-10-01"},
 ],
 reasoning_summary=(
 "Compared Q3 2025 ($4.2B) to Q3 2024 ($3.75B) from earnings report"
 ),
 alternatives=[
 "11.8% if adjusted for currency effects",
 ],
 caveats=[
 "Does not account for one-time items",
 ],
 ),
)
ui_data = format_response_for_ui(response)

# Statistical analysis of human-AI collaborative outcomes
import numpy as np
from scipy import stats
from dataclasses import dataclass

@dataclass
class CollaborativeOutcome:
 participant_id: str
 skill_level: str # "novice", "intermediate", "expert"
 task_difficulty: str # "easy", "medium", "hard"
 human_only_score: float # 0-100
 ai_only_score: float # 0-100
 human_ai_score: float # 0-100
 human_only_time_s: float
 human_ai_time_s: float

 @property
 def synergy(self) -> float:
 """Positive synergy means the team exceeds the best individual."""
 best_individual = max(self.human_only_score, self.ai_only_score)
 return self.human_ai_score - best_individual

 @property
 def complementarity(self) -> float:
 """How much does the team cover weaknesses of each?"""
 return self.human_ai_score - (
 (self.human_only_score + self.ai_only_score) / 2
 )

def analyze_collaborative_outcomes(
 outcomes: list[CollaborativeOutcome],
) -> dict:
 """Comprehensive analysis of human-AI collaboration."""
 synergies = [o.synergy for o in outcomes]
 complementarities = [o.complementarity for o in outcomes]
 speedups = [
 o.human_only_time_s / max(o.human_ai_time_s, 1)
 for o in outcomes
 ]

 # Test if human-AI significantly outperforms human-only
 human_scores = [o.human_only_score for o in outcomes]
 team_scores = [o.human_ai_score for o in outcomes]
 t_stat, p_value = stats.ttest_rel(team_scores, human_scores)

 # Analyze by skill level
 skill_analysis = {}
 for level in ["novice", "intermediate", "expert"]:
 level_outcomes = [o for o in outcomes if o.skill_level == level]
 if level_outcomes:
 skill_analysis[level] = {
 "n": len(level_outcomes),
 "mean_synergy": round(
 np.mean([o.synergy for o in level_outcomes]), 2
 ),
 "mean_speedup": round(
 np.mean([
 o.human_only_time_s / max(o.human_ai_time_s, 1)
 for o in level_outcomes
 ]), 2
 ),
 }

 return {
 "overall": {
 "mean_synergy": round(float(np.mean(synergies)), 2),
 "pct_positive_synergy": round(
 sum(1 for s in synergies if s > 0) / len(synergies) * 100, 1
 ),
 "mean_speedup": round(float(np.mean(speedups)), 2),
 "t_statistic": round(float(t_stat), 3),
 "p_value": round(float(p_value), 4),
 "significant": p_value < 0.05,
 },
 "by_skill_level": skill_analysis,
 }