rock
6915dc2462
Add comprehensive E2E test scripts and logging for production testing: - test_prod_ingest_real.sh: Full ingest test against K8s cluster with api-gw - apply_migrations.sh: Manual database schema migration (backup method) - collect_prod_logs.sh: Pod log collection before/after tests - run_production_test.sh: Orchestrates full test + log collection - tests/integration_ingest_with_gw.rs: Integration test with embeddings - tests/unit_ingest_logging.rs: Unit tests for extraction pipeline Enhanced logging in ingest_worker.rs: - Per-record event tracking (extraction, save) - Entity and edge operation logging - Error accumulation and reporting - Structured logging for observability Production testing identified root cause: - Ingest + embedding pipeline working correctly - Entity extraction functional - Database schema missing (migration not applied) - Logs clearly show: relation "memory_entity" does not exist Next: Trigger DB Migration workflow in Forgejo Actions to apply crates/mem-store/migrations/*.sql files.
82 lines
2.2 KiB
Bash
Executable File
82 lines
2.2 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Master script: Run full production ingest test with logging
|
|
#
|
|
# Usage:
|
|
# ./run_production_test.sh
|
|
# ./run_production_test.sh [api-key]
|
|
#
|
|
# What it does:
|
|
# 1. Collect baseline logs
|
|
# 2. Run ingest test
|
|
# 3. Collect post-test logs
|
|
# 4. Analyze for errors
|
|
# 5. Display results
|
|
|
|
set -e
|
|
|
|
API_KEY="${1:-test-key}"
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
echo ""
|
|
echo "╔═════════════════════════════════════════╗"
|
|
echo "║ Production Ingest Test with api-gw ║"
|
|
echo "║ (Full root-cause error logging) ║"
|
|
echo "╚═════════════════════════════════════════╝"
|
|
echo ""
|
|
|
|
# Verify scripts exist
|
|
for script in test_production_ingest.sh collect_prod_logs.sh; do
|
|
if [ ! -f "$SCRIPT_DIR/$script" ]; then
|
|
echo "✗ Missing: $script"
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
echo "Step 1: Collecting baseline logs..."
|
|
"$SCRIPT_DIR/collect_prod_logs.sh" before
|
|
|
|
echo ""
|
|
echo "Step 2: Running ingest test..."
|
|
echo " (Sending records through embedding pipeline to api-gw)"
|
|
echo ""
|
|
|
|
if MEM_API_KEY="$API_KEY" "$SCRIPT_DIR/test_production_ingest.sh"; then
|
|
echo ""
|
|
echo "✓ Test passed!"
|
|
test_status=0
|
|
else
|
|
echo ""
|
|
echo "✗ Test failed!"
|
|
test_status=1
|
|
fi
|
|
|
|
echo ""
|
|
echo "Step 3: Collecting post-test logs..."
|
|
"$SCRIPT_DIR/collect_prod_logs.sh" after
|
|
|
|
echo ""
|
|
echo "Step 4: Analyzing logs for errors..."
|
|
echo ""
|
|
"$SCRIPT_DIR/collect_prod_logs.sh" analyze
|
|
|
|
echo ""
|
|
echo "════════════════════════════════════════"
|
|
if [ $test_status -eq 0 ]; then
|
|
echo "✓ INGEST TEST PASSED"
|
|
else
|
|
echo "✗ INGEST TEST FAILED"
|
|
echo ""
|
|
echo "Next steps:"
|
|
echo " 1. Check logs in prod_logs_*/ directory"
|
|
echo " 2. Look for errors in:"
|
|
echo " - /memory/ingest endpoint response"
|
|
echo " - Embedding service (LLM_ENDPOINT)"
|
|
echo " - api-gw gateway logs"
|
|
echo " - Database connection"
|
|
fi
|
|
echo "════════════════════════════════════════"
|
|
echo ""
|
|
|
|
exit $test_status
|