cd /home/graham/workspace/experiments/pi-mono/.pi/skills/prompt-lab

# Find the smallest model that works (NEW!)
./run.sh find-minimum --ground-truth queryspec.json --threshold 0.80

# Run evaluation with self-correction enabled (default)
./run.sh eval --prompt taxonomy_v1 --model deepseek

# Compare multiple models on same prompt
./run.sh compare --prompt taxonomy_v1 --models "deepseek,gpt-4o"



# View evaluation history
./run.sh history --prompt taxonomy_v1

┌─────────────────────────────────────────────────────────────┐
│  Stage 1: LLM Call with Vocabulary in Prompt                │
│     - Vocabulary clearly defined (Tier 0, Tier 1 tags)      │
│     - JSON response format enforced                         │
└─────────────────────────────────────────────────────────────┘
                              ↓
┌─────────────────────────────────────────────────────────────┐
│  Stage 2: Pydantic Validation                               │
│     - Parse JSON response                                   │
│     - Detect invalid/hallucinated tags                      │
│     - Track rejected tags for metrics                       │
└─────────────────────────────────────────────────────────────┘
                              ↓
┌─────────────────────────────────────────────────────────────┐
│  Stage 3: Self-Correction Loop (if invalid tags detected)   │
│     - Send assistant message: "Invalid tags: X"             │
│     - Ask LLM to fix: "Valid options are: Y"                │
│     - Retry up to N times (default: 2)                      │
│     - Track correction rounds in metrics                    │
└─────────────────────────────────────────────────────────────┘
                              ↓
┌─────────────────────────────────────────────────────────────┐
│  Stage 4: Evaluation Metrics                                │
│     - Precision/Recall/F1 vs ground truth                   │
│     - Correction rounds needed                              │
│     - Correction success rate                               │
└─────────────────────────────────────────────────────────────┘

# Find smallest model for QuerySpec with 80% accuracy threshold
./run.sh find-minimum --ground-truth queryspec.json --threshold 0.80

# Prefer local Ollama models (default)
./run.sh find-minimum -g queryspec.json -t 0.80 --prefer-local

# Test Chutes API models instead
./run.sh find-minimum -g queryspec.json -t 0.80 --no-prefer-local

# Options:
#   --ground-truth, -g   Ground truth JSON file (required)
#   --threshold, -t      Minimum accuracy threshold (default: 0.80)
#   --prompt, -p         Optional prompt file
#   --prefer-local       Prefer local Ollama (default: true)
#   --max-models         Max models to test (default: 10)
#   --verbose, -v        Show per-case details

./run.sh eval --prompt taxonomy_v1 --model deepseek

# Options:
#   --prompt NAME           Prompt version to test
#   --model NAME            Model to use (deepseek, gpt-4o, etc.)
#   --cases N               Number of test cases (default: all)
#   --max-corrections N     Max self-correction rounds (default: 2)
#   --no-correction         Disable self-correction loop
#   --task-name NAME        Task-monitor task name for quality gate
#   --verbose               Show per-case details

./run.sh compare --prompt taxonomy_v1 --models "deepseek,gpt-4o"

# Output:
# | Model    | F1    | Corrections | Time  |
# |----------|-------|-------------|-------|
# | deepseek | 0.90  | 3           | 2.3s  |
# | gpt-4o   | 0.93  | 1           | 1.1s  |

./run.sh extract-prompts --file /path/to/12_qra.py --output prompts/

./run.sh test-sparta --db-path /path/to/sparta.duckdb --cases 100

# Run test with task-monitor (enabled by default)
./run.sh test-sparta --cases 100

# View progress in task-monitor TUI
cd ~/.pi/skills/task-monitor
uv run python monitor.py tui --filter prompt-lab

# Or check state file directly
cat /path/to/prompt-lab/prompt_lab_task_state.json | jq

{
  "completed": 50,
  "total": 100,
  "progress_pct": 50.0,
  "gates": {
    "ambiguity": {"passed": 50, "failed": 0, "rate": 1.0},
    "anchoring": {"passed": 49, "failed": 1, "rate": 0.98},
    "grounding": {"passed": 48, "failed": 2, "rate": 0.96}
  },
  "failures": [...],
  "status": "running"
}

./run.sh test-sparta --cases 500 --json-stream | tee results.jsonl

# Each line:
# {"case_id": "control_T1110", "status": "pass", "qra_count": 5, "metrics": {...}, "latency_ms": 1234.5}

./run.sh history --prompt taxonomy_v1

# Shows all evaluation runs with scores over time

./run.sh analyze --prompt taxonomy_v1

# Analyzes error patterns across all previous evaluations:
# - Most common invalid tags
# - Cases needing correction
# - Performance trend over time
# - Suggests improvements based on patterns

./run.sh optimize --prompt taxonomy_v1

# Uses LLM to analyze error cases and suggest prompt improvements:
# - Reviews cases with low F1 scores
# - Identifies ambiguous tag definitions
# - Generates revised prompt sections
# - Saves suggestions for review

Your response contained invalid tags that are not in the allowed vocabulary.

Invalid tags you used: {rejected_tags}

Valid conceptual tags (Tier 0): Corruption, Fragility, Loyalty, Precision, Resilience, Stealth
Valid tactical tags (Tier 1): Detect, Evade, Exploit, Harden, Isolate, Model, Persist, Restore

Please correct your response. Return ONLY valid JSON with tags from the allowed vocabulary above.
Do NOT invent new categories. Only use the exact tag names listed.

# Notify task-monitor of evaluation result
./run.sh eval --prompt taxonomy_v1 --task-name "prompt-validation"

# Task-monitor receives:
# - Task name
# - Pass/fail status
# - Metrics (F1, correction rounds, rejected tags)

prompt-lab/
├── SKILL.md
├── run.sh
├── prompt_lab.py           # Main CLI with self-correction
├── ground_truth/
│   └── taxonomy.json       # Test cases with expected outputs
├── prompts/
│   ├── taxonomy_v1.txt     # Prompt version 1
│   └── taxonomy_v2.txt     # Prompt version 2
├── results/
│   └── taxonomy_v1_deepseek_20260128.json  # Evaluation results
├── models.json             # Model configurations
└── pyproject.toml

{
  "deepseek": {
    "provider": "chutes",
    "model": "deepseek-ai/DeepSeek-V3-0324-TEE",
    "api_base": "$CHUTES_API_BASE",
    "api_key": "$CHUTES_API_KEY"
  },
  "gpt-4o": {
    "provider": "openai",
    "model": "gpt-4o",
    "api_key": "$OPENAI_API_KEY"
  }
}

./run.sh eval-qra --prompt qra_v2 --model deepseek --cases 5

def validate_citation(citation: str, source_text: str, threshold: float = 0.85) -> bool:
    from rapidfuzz import fuzz
    return fuzz.partial_ratio(citation.lower(), source_text.lower()) >= threshold * 100

You are a space-based cybersecurity expert generating Question-Reasoning-Answer pairs
for SPARTA. Think like a LEGAL LLM: every claim MUST cite precedent from the provided text.

STRICT GROUNDING RULE: If information is NOT in the provided text, you CANNOT include
it in your answer. Do NOT add external knowledge, mitigation advice, or implications
not directly stated.

Return JSON: {"items": [{"question": "...",
  "question_type": "simple|medium|complex|reversal_curse",
  "questioner_persona": "lay_person|project_manager|cybersecurity_expert",
  "reasoning": "...", "answer": "...",
  "citations": ["EXACT verbatim excerpt from text"],
  "confidence": "strong|partial",
  "conceptual_tags": [], "tactical_tags": []}]}

GENERATE ALL REASONABLE NON-DUPLICATE QUESTIONS:
- Simple, Medium, Complex levels + Reversal curse where applicable
- Each extracts DIFFERENT information from the text
- NEVER add information not in the source text

TAXONOMY: C=[Corruption,Fragility,Loyalty,Precision,Resilience,Stealth],
T=[Detect,Evade,Exploit,Harden,Isolate,Model,Persist,Restore]

# Run QRA evaluation with default settings
./run.sh eval-qra --prompt qra_grounded_v1 --model deepseek

# With custom threshold and verbose output
./run.sh eval-qra --prompt qra_grounded_v1 --model deepseek --threshold 0.90 --verbose

# Test on limited cases
./run.sh eval-qra --prompt qra_grounded_v1 --model deepseek --cases 2