34 lines
1.3 KiB
YAML
34 lines
1.3 KiB
YAML
|
# M3.4 — Known-answer questions for gate verification
|
||
|
|
#
|
||
|
|
# These questions are drawn from real infrastructure findings in the poimen corpus.
|
||
|
|
# Each names a fact that genuinely appears in ingested sessions.
|
||
|
|
# Used to measure hit rate and provenance precision of the retrieval pipeline.
|
||
|
|
|
||
|
|
questions:
|
||
|
|
- id: kong_body_buffer
|
||
|
|
question: "why did requests over 10KB fail?"
|
||
|
|
expected_node_text: "Kong buffer limit 64KB"
|
||
|
|
expected_source_substring: "body size too large"
|
||
|
|
expected_query: "infra-root-causes"
|
||
|
|
level: "L1"
|
||
|
|
|
||
|
|
- id: kong_auth_header
|
||
|
|
question: "why did requests with Authorization header fail?"
|
||
|
|
expected_node_text: "Kong key-auth"
|
||
|
|
expected_source_substring: "apikey header"
|
||
|
|
expected_query: "infra-root-causes"
|
||
|
|
level: "L1"
|
||
|
|
|
||
|
|
- id: cold_start_timeout
|
||
|
|
question: "what causes the 504 timeout on cold start?"
|
||
|
|
expected_node_text: "Ingress timeout"
|
||
|
|
expected_source_substring: "gateway timeout"
|
||
|
|
expected_query: "infra-root-causes"
|
||
|
|
level: "L1"
|
||
|
|
|
||
|
|
# Thresholds for gate
|
||
|
|
thresholds:
|
||
|
|
hit_rate_at_5: 0.8 # ≥ 80% of questions should return the right node in top-5
|
||
|
|
provenance_precision: 0.9 # ≥ 90% of cited sources should contain the fact
|
||
|
|
max_failed_questions: 1 # Allow 1 failing question out of 3 (due to incomplete seeds)
|