AI Copilots Across the Lifecycle

# Stress-test a product hypothesis using an LLM as devil's advocate
import openai

client = openai.OpenAI()

def stress_test_hypothesis(hypothesis: str, perspectives: int = 5) -> str:
 """Ask the LLM to critique a product hypothesis from multiple angles."""
 system_prompt = (
 "You are a rigorous product strategy advisor. "
 "Given a product hypothesis, provide structured criticism:\n"
 "1. List the strongest counter-arguments.\n"
 "2. Identify hidden assumptions the founder may not realize they are making.\n"
 "3. Name the riskiest dependency (technical, market, or regulatory).\n"
 "4. Suggest one pivot that preserves the core insight but avoids the biggest risk."
 )
 response = client.chat.completions.create(
 model="gpt-4o",
 messages=[
 {"role": "system", "content": system_prompt},
 {"role": "user", "content": f"Hypothesis: {hypothesis}"},
 ],
 temperature=0.7,
 max_tokens=800,
 )
 return response.choices[0].message.content

# Example usage
idea = (
 "Small law firms will pay $200/month for an AI assistant that drafts "
 "client intake summaries from recorded phone calls."
)
critique = stress_test_hypothesis(idea)
print(critique)

# Generate acceptance criteria from a feature description
ACCEPTANCE_CRITERIA_PROMPT = """\
You are a senior product manager writing acceptance criteria.

Feature: {feature_description}

For each user story, produce acceptance criteria in this format:
- GIVEN [precondition]
- WHEN [action]
- THEN [expected outcome]

Also list:
- Two edge cases that are easy to overlook.
- One potential abuse scenario and its mitigation.
"""

def generate_acceptance_criteria(feature: str) -> str:
 """Expand a feature description into structured acceptance criteria."""
 response = client.chat.completions.create(
 model="gpt-4o",
 messages=[
 {"role": "user", "content": ACCEPTANCE_CRITERIA_PROMPT.format(
 feature_description=feature
 )},
 ],
 temperature=0.4,
 max_tokens=1000,
 )
 return response.choices[0].message.content

# Example: generate criteria for the intake-summary feature
criteria = generate_acceptance_criteria(
 "AI assistant transcribes a recorded client phone call and produces "
 "a structured intake summary with client name, issue category, "
 "key dates, and recommended next steps."
)
print(criteria)

# Meta-prompting: use an LLM to critique and improve a draft prompt
META_CRITIC_SYSTEM = """\
You are a prompt engineering expert. Given a draft system prompt,
provide a structured critique:
1. AMBIGUITIES: phrases a model might misinterpret.
2. MISSING CONSTRAINTS: important boundaries not stated.
3. JAILBREAK SURFACE: ways a user could bypass the instructions.
4. REVISED PROMPT: an improved version addressing the issues above.
"""

def meta_prompt_critique(draft_prompt: str) -> str:
 """Critique a draft system prompt and return an improved version."""
 response = client.chat.completions.create(
 model="gpt-4o",
 messages=[
 {"role": "system", "content": META_CRITIC_SYSTEM},
 {"role": "user", "content": f"Draft prompt:\n\n{draft_prompt}"},
 ],
 temperature=0.5,
 max_tokens=1200,
 )
 return response.choices[0].message.content

# Example: critique a draft system prompt for the intake assistant
draft = (
 "You are a legal intake assistant. Summarize the client's phone call. "
 "Include the client name, issue type, and next steps."
)
improved = meta_prompt_critique(draft)
print(improved)

# Summarize evaluation failures and suggest next experiments
import json

def summarize_eval_failures(failures: list[dict], top_k: int = 5) -> str:
 """Cluster eval failures by root cause and suggest fixes.

 Each failure dict should have 'input', 'expected', 'actual', and 'score'.
 """
 # Truncate to avoid exceeding context limits
 sample = failures[:100]
 payload = json.dumps(sample, indent=2)

 response = client.chat.completions.create(
 model="gpt-4o",
 messages=[
 {"role": "system", "content": (
 "You are an evaluation analyst. Given a list of failed test cases, "
 "group them into root-cause clusters. For each cluster:\n"
 "1. Name the failure pattern.\n"
 "2. Count how many cases belong to it.\n"
 "3. Suggest a concrete fix (prompt change, guardrail, or data addition).\n"
 "4. Rank clusters by impact (most failures first).\n"
 f"Return the top {top_k} clusters."
 )},
 {"role": "user", "content": f"Failed test cases:\n\n{payload}"},
 ],
 temperature=0.3,
 max_tokens=1500,
 )
 return response.choices[0].message.content

# Example usage with synthetic failure data
sample_failures = [
 {"input": "Call about car accident", "expected": "Personal Injury",
 "actual": "Property Damage", "score": 0.0},
 {"input": "Llamada sobre accidente", "expected": "Personal Injury",
 "actual": "I don't understand", "score": 0.0},
 # ... imagine 198 more entries
]
analysis = summarize_eval_failures(sample_failures)
print(analysis)