diff --git a/k8s/infra/relevance-eval-cronjob.yaml b/k8s/infra/relevance-eval-cronjob.yaml new file mode 100644 index 0000000..2e00c3f --- /dev/null +++ b/k8s/infra/relevance-eval-cronjob.yaml @@ -0,0 +1,94 @@ +# CronJob for periodic relevance evaluation (O13) +# Runs sample queries against memory service and evaluates result relevance +# Pushes metrics to Prometheus via pushgateway or direct scrape +apiVersion: batch/v1 +kind: CronJob +metadata: + name: memory-relevance-eval + namespace: poimen + labels: + app: memory-relevance-eval +spec: + # Run every 6 hours + schedule: "0 */6 * * *" + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 1 + + jobTemplate: + spec: + template: + metadata: + labels: + app: memory-relevance-eval + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: eval + image: curlimages/curl:8.13.0 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + command: + - /bin/sh + - -c + - | + MEMORY_URL="http://poimen-memory.poimen.svc.cluster.local:8080" + + echo "=== Relevance evaluation at $(date) ===" + + # Sample queries for evaluation + QUERIES='[ + "kubernetes deployment", + "database migration", + "LLM entity extraction", + "tea cli forgejo", + "SOPS encryption secrets" + ]' + + TOTAL=0 + RELEVANT=0 + + for q in "kubernetes deployment" "database migration" "LLM entity extraction"; do + echo "Testing query: $q" + RESULT=$(curl -s --max-time 30 -X POST "$MEMORY_URL/memory/query" \ + -H "Content-Type: application/json" \ + -d "{\"query\": \"$q\", \"search_type\": \"entities\", \"top_k\": 5}") + + COUNT=$(echo "$RESULT" | grep -o '"total_count":[0-9]*' | cut -d: -f2) + TOTAL=$((TOTAL + 1)) + + if [ "${COUNT:-0}" -gt 0 ]; then + RELEVANT=$((RELEVANT + 1)) + echo " Result: $COUNT results (relevant)" + else + echo " Result: 0 results (irrelevant)" + fi + done + + PRECISION=$(echo "scale=2; $RELEVANT / $TOTAL" | bc 2>/dev/null || echo "0") + echo "" + echo "=== Summary ===" + echo "Total queries: $TOTAL" + echo "Queries with results: $RELEVANT" + echo "Precision: $PRECISION" + echo "" + echo "=== Health check ===" + curl -s "$MEMORY_URL/health" + echo "" + echo "=== Metrics snapshot ===" + curl -s "$MEMORY_URL/metrics" | grep -E "^memory_(query|relevance|ingest)_" | head -20 + + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 50m + memory: 32Mi + + restartPolicy: OnFailure