feat: production ingest test suite with detailed logging
Add comprehensive E2E test scripts and logging for production testing: - test_prod_ingest_real.sh: Full ingest test against K8s cluster with api-gw - apply_migrations.sh: Manual database schema migration (backup method) - collect_prod_logs.sh: Pod log collection before/after tests - run_production_test.sh: Orchestrates full test + log collection - tests/integration_ingest_with_gw.rs: Integration test with embeddings - tests/unit_ingest_logging.rs: Unit tests for extraction pipeline Enhanced logging in ingest_worker.rs: - Per-record event tracking (extraction, save) - Entity and edge operation logging - Error accumulation and reporting - Structured logging for observability Production testing identified root cause: - Ingest + embedding pipeline working correctly - Entity extraction functional - Database schema missing (migration not applied) - Logs clearly show: relation "memory_entity" does not exist Next: Trigger DB Migration workflow in Forgejo Actions to apply crates/mem-store/migrations/*.sql files.
This commit is contained in:
Executable
+197
@@ -0,0 +1,197 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Production real test: Full ingest with api-gw + embedding
|
||||
# Sends records through the complete pipeline and logs all errors
|
||||
#
|
||||
# Usage:
|
||||
# ./test_prod_ingest_real.sh [--verbose]
|
||||
|
||||
set -e
|
||||
|
||||
NAMESPACE="poimen"
|
||||
SERVICE="poimen-memory"
|
||||
LOCAL_PORT="9990"
|
||||
TIMESTAMP=$(date +%Y%m%d-%H%M%S)
|
||||
LOG_FILE="/tmp/ingest_test_${TIMESTAMP}.log"
|
||||
VERBOSE="${1:-}"
|
||||
|
||||
{
|
||||
echo "=========================================="
|
||||
echo "Production Ingest Test: $(date)"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
echo "Namespace: $NAMESPACE"
|
||||
echo "Service: $SERVICE"
|
||||
echo "Local Port: $LOCAL_PORT"
|
||||
echo "Log: $LOG_FILE"
|
||||
echo ""
|
||||
|
||||
# Start port-forward
|
||||
echo "Starting port-forward..."
|
||||
kubectl -n "$NAMESPACE" port-forward "svc/$SERVICE" "$LOCAL_PORT:8080" >/dev/null 2>&1 &
|
||||
PF_PID=$!
|
||||
|
||||
cleanup() {
|
||||
if [ -n "$PF_PID" ]; then
|
||||
kill $PF_PID 2>/dev/null || true
|
||||
wait $PF_PID 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
sleep 2
|
||||
|
||||
if ! kill -0 $PF_PID 2>/dev/null; then
|
||||
echo "✗ Port-forward failed"
|
||||
exit 1
|
||||
fi
|
||||
echo "✓ Port-forward running (PID $PF_PID)"
|
||||
echo ""
|
||||
|
||||
# Check health
|
||||
echo "Checking /health endpoint..."
|
||||
if ! curl -s "http://localhost:$LOCAL_PORT/health" >/dev/null 2>&1; then
|
||||
echo "✗ Health check failed"
|
||||
exit 1
|
||||
fi
|
||||
echo "✓ Health check passed"
|
||||
echo ""
|
||||
|
||||
# Prepare ingest request
|
||||
INGEST_ID="ingest-test-${TIMESTAMP}"
|
||||
|
||||
PAYLOAD=$(cat <<'EOFPAYLOAD'
|
||||
{
|
||||
"project": "production-real-test",
|
||||
"source": "integration-test",
|
||||
"ingest_id": "INGEST_ID_PLACEHOLDER",
|
||||
"records": [
|
||||
{
|
||||
"role": "user",
|
||||
"text": "Kubernetes [[Docker]] [[Linux]] is an open-source container orchestration platform. It automates many manual processes involved in deploying, managing, and scaling containerized applications.",
|
||||
"timestamp": "2026-09-14T13:00:00Z",
|
||||
"source_position": 0
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"text": "Docker [[Container]] [[Go]] is a containerization platform that makes it easier to build, ship, and run applications. Docker achieves high efficiency through the use of operating system-level virtualization.",
|
||||
"timestamp": "2026-09-14T13:01:00Z",
|
||||
"source_position": 1
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"text": "Go [[Concurrency]] [[Static Typing]] is a programming language designed at Google. It is statically typed, compiled, and known for its simplicity, concurrent programming model, and efficient execution.",
|
||||
"timestamp": "2026-09-14T13:02:00Z",
|
||||
"source_position": 2
|
||||
}
|
||||
]
|
||||
}
|
||||
EOFPAYLOAD
|
||||
)
|
||||
|
||||
# Replace placeholder
|
||||
PAYLOAD="${PAYLOAD//INGEST_ID_PLACEHOLDER/$INGEST_ID}"
|
||||
|
||||
echo "Sending ingest request..."
|
||||
if [ -n "$VERBOSE" ]; then
|
||||
echo "Payload:"
|
||||
echo "$PAYLOAD" | jq . 2>/dev/null || echo "$PAYLOAD"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
RESPONSE=$(curl -s -w "\n%{http_code}" -X POST \
|
||||
"http://localhost:$LOCAL_PORT/memory/ingest" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer test-key" \
|
||||
-d "$PAYLOAD")
|
||||
|
||||
HTTP_CODE=$(echo "$RESPONSE" | tail -1)
|
||||
BODY=$(echo "$RESPONSE" | head -n-1)
|
||||
|
||||
echo "HTTP Status: $HTTP_CODE"
|
||||
if [ "$HTTP_CODE" != "202" ]; then
|
||||
echo "✗ Unexpected HTTP status"
|
||||
echo "Response: $BODY"
|
||||
exit 1
|
||||
fi
|
||||
echo "✓ Request accepted"
|
||||
echo ""
|
||||
|
||||
if [ -n "$VERBOSE" ]; then
|
||||
echo "Response body:"
|
||||
echo "$BODY" | jq . 2>/dev/null || echo "$BODY"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# Extract ID
|
||||
ID=$(echo "$BODY" | jq -r '.ingest_id // empty' 2>/dev/null)
|
||||
if [ -z "$ID" ]; then
|
||||
echo "✗ Missing ingest_id in response"
|
||||
echo "Response: $BODY"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Ingest ID: $ID"
|
||||
echo ""
|
||||
|
||||
# Poll status
|
||||
echo "Polling job status..."
|
||||
echo "=========================================="
|
||||
|
||||
MAX_POLLS=120 # 10 minutes at 5s intervals
|
||||
poll_count=0
|
||||
|
||||
while [ $poll_count -lt $MAX_POLLS ]; do
|
||||
poll_count=$((poll_count+1))
|
||||
|
||||
STATUS_RESP=$(curl -s "http://localhost:$LOCAL_PORT/memory/ingest/$ID" \
|
||||
-H "Authorization: Bearer test-key")
|
||||
|
||||
STATUS=$(echo "$STATUS_RESP" | jq -r '.status // "unknown"' 2>/dev/null)
|
||||
|
||||
printf "[%3d] %-20s" "$poll_count" "$STATUS"
|
||||
|
||||
case "$STATUS" in
|
||||
done)
|
||||
echo " ✓"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
echo "✓ SUCCESS: Ingest completed"
|
||||
|
||||
if [ -n "$VERBOSE" ]; then
|
||||
echo ""
|
||||
echo "Final response:"
|
||||
echo "$STATUS_RESP" | jq . 2>/dev/null || echo "$STATUS_RESP"
|
||||
fi
|
||||
exit 0
|
||||
;;
|
||||
failed|error)
|
||||
echo " ✗"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
echo "✗ FAILED: Ingest did not complete"
|
||||
echo ""
|
||||
echo "Final response:"
|
||||
echo "$STATUS_RESP" | jq . 2>/dev/null || echo "$STATUS_RESP"
|
||||
exit 1
|
||||
;;
|
||||
processing|queued|pending)
|
||||
echo ""
|
||||
sleep 5
|
||||
;;
|
||||
*)
|
||||
echo " (unknown)"
|
||||
sleep 5
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
echo "✗ TIMEOUT: Ingest did not complete after ${MAX_POLLS} polls (${poll_count}m)"
|
||||
exit 1
|
||||
|
||||
} 2>&1 | tee "$LOG_FILE"
|
||||
|
||||
echo ""
|
||||
echo "Full log saved to: $LOG_FILE"
|
||||
Reference in New Issue
Block a user