rock
9e85b1a063
CI / CI (pull_request) Successful in 12m14s
- Runs every 6 hours with sample queries - Tests query endpoint with known queries - Reports precision (queries with results / total) - Snapshots /metrics endpoint for monitoring - Lightweight: curl-based, 16Mi memory - Deploy: kubectl apply -f k8s/infra/relevance-eval-cronjob.yaml
95 lines
3.3 KiB
YAML
95 lines
3.3 KiB
YAML
# CronJob for periodic relevance evaluation (O13)
|
|
# Runs sample queries against memory service and evaluates result relevance
|
|
# Pushes metrics to Prometheus via pushgateway or direct scrape
|
|
apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
name: memory-relevance-eval
|
|
namespace: poimen
|
|
labels:
|
|
app: memory-relevance-eval
|
|
spec:
|
|
# Run every 6 hours
|
|
schedule: "0 */6 * * *"
|
|
successfulJobsHistoryLimit: 3
|
|
failedJobsHistoryLimit: 1
|
|
|
|
jobTemplate:
|
|
spec:
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: memory-relevance-eval
|
|
spec:
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 1000
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
containers:
|
|
- name: eval
|
|
image: curlimages/curl:8.13.0
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop: ["ALL"]
|
|
command:
|
|
- /bin/sh
|
|
- -c
|
|
- |
|
|
MEMORY_URL="http://poimen-memory.poimen.svc.cluster.local:8080"
|
|
|
|
echo "=== Relevance evaluation at $(date) ==="
|
|
|
|
# Sample queries for evaluation
|
|
QUERIES='[
|
|
"kubernetes deployment",
|
|
"database migration",
|
|
"LLM entity extraction",
|
|
"tea cli forgejo",
|
|
"SOPS encryption secrets"
|
|
]'
|
|
|
|
TOTAL=0
|
|
RELEVANT=0
|
|
|
|
for q in "kubernetes deployment" "database migration" "LLM entity extraction"; do
|
|
echo "Testing query: $q"
|
|
RESULT=$(curl -s --max-time 30 -X POST "$MEMORY_URL/memory/query" \
|
|
-H "Content-Type: application/json" \
|
|
-d "{\"query\": \"$q\", \"search_type\": \"entities\", \"top_k\": 5}")
|
|
|
|
COUNT=$(echo "$RESULT" | grep -o '"total_count":[0-9]*' | cut -d: -f2)
|
|
TOTAL=$((TOTAL + 1))
|
|
|
|
if [ "${COUNT:-0}" -gt 0 ]; then
|
|
RELEVANT=$((RELEVANT + 1))
|
|
echo " Result: $COUNT results (relevant)"
|
|
else
|
|
echo " Result: 0 results (irrelevant)"
|
|
fi
|
|
done
|
|
|
|
PRECISION=$(echo "scale=2; $RELEVANT / $TOTAL" | bc 2>/dev/null || echo "0")
|
|
echo ""
|
|
echo "=== Summary ==="
|
|
echo "Total queries: $TOTAL"
|
|
echo "Queries with results: $RELEVANT"
|
|
echo "Precision: $PRECISION"
|
|
echo ""
|
|
echo "=== Health check ==="
|
|
curl -s "$MEMORY_URL/health"
|
|
echo ""
|
|
echo "=== Metrics snapshot ==="
|
|
curl -s "$MEMORY_URL/metrics" | grep -E "^memory_(query|relevance|ingest)_" | head -20
|
|
|
|
resources:
|
|
requests:
|
|
cpu: 10m
|
|
memory: 16Mi
|
|
limits:
|
|
cpu: 50m
|
|
memory: 32Mi
|
|
|
|
restartPolicy: OnFailure
|