poimen-memory/verify/known-answers.yaml

# M3.4 — Known-answer questions for gate verification
# 
# These questions are drawn from real infrastructure findings in the poimen corpus.
# Each names a fact that genuinely appears in ingested sessions.
# Used to measure hit rate and provenance precision of the retrieval pipeline.

questions:
  - id: db_query_timeout
    question: "why are database queries timing out?"
    expected_node_text: "Missing index on queries table"
    expected_source_substring: "sequential scan"
    expected_query: "infra-root-causes"
    level: "L1"

  - id: model_load_timeout
    question: "why does the model fail to load on cold start?"
    expected_node_text: "Model loading exceeds 60s timeout"
    expected_source_substring: "torch compile"
    expected_query: "infra-root-causes"
    level: "L1"

  - id: memory_pressure
    question: "what causes out of memory errors?"
    expected_node_text: "GPU VRAM exhaustion"
    expected_source_substring: "loaded models eviction"
    expected_query: "infra-root-causes"
    level: "L1"

# Thresholds for gate
thresholds:
  hit_rate_at_5: 0.8  # ≥ 80% of questions should return the right node in top-5
  provenance_precision: 0.9  # ≥ 90% of cited sources should contain the fact
  max_failed_questions: 1  # Allow 1 failing question out of 3 (due to incomplete seeds)