poimenandrock
a88ea918bf
## Summary
Production testing of ingest + embedding pipeline with api-gw integration.
## Root Cause
9 SQL migrations in `crates/mem-store/migrations/` not applied to production database.
Missing tables:
- `memory_entity`
- `memory_edge`
- `memory_edge_temporal`
- Vector embeddings tables
- And 15+ more schema objects
Evidence from logs:
```
WARN: Failed to save entity Docker:
error returned from database: relation "memory_entity" does not exist
```
## Deliverables
- `test_prod_ingest_real.sh` - Full E2E test against K8s + api-gw
- `apply_migrations.sh` - Manual schema migration (backup)
- `collect_prod_logs.sh` - Pod log collection before/after
- `run_production_test.sh` - Test orchestrator
- `tests/integration_ingest_with_gw.rs` - Integration test
- `tests/unit_ingest_logging.rs` - Unit tests for extraction
- Enhanced logging in `ingest_worker.rs` - Per-record event tracking
## Next Steps
1. Trigger "DB Migration" workflow in Forgejo Actions
2. This applies all 9 migrations from `crates/mem-store/migrations/`
3. Pod restart (automatic)
4. Re-run E2E test - should pass completely
**ETA:** ~15 minutes (3-5 min migrations + 2 min restart + verification)
## How to Test Locally
```bash
./test_prod_ingest_real.sh --verbose
```
Requires:
- kubectl access to poimen namespace
- Port-forwarding to memory-service
---------
Co-authored-by: rock <[email protected]>
Reviewed-on: #55
Co-authored-by: poimen <[email protected]>
128 lines
3.0 KiB
Bash
Executable File
128 lines
3.0 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Database Migration Runner
|
|
# Used by K8s Job to apply all migrations before integration tests
|
|
#
|
|
# Environment variables (from K8s):
|
|
# DB_HOST - PostgreSQL host
|
|
# DB_PORT - PostgreSQL port
|
|
# DB_NAME - Database name
|
|
# DB_USER - Database user
|
|
# DB_PASSWORD - Database password (from Secret)
|
|
|
|
set -e
|
|
|
|
DB_HOST="${DB_HOST:-memory-db-rw.poimen.svc.cluster.local}"
|
|
DB_PORT="${DB_PORT:-5432}"
|
|
DB_NAME="${DB_NAME:-memory}"
|
|
DB_USER="${DB_USER:-app}"
|
|
|
|
if [ -z "$DB_PASSWORD" ]; then
|
|
echo "ERROR: DB_PASSWORD not set"
|
|
exit 1
|
|
fi
|
|
|
|
echo "=========================================="
|
|
echo "Database Migration Runner"
|
|
echo "=========================================="
|
|
echo ""
|
|
echo "Configuration:"
|
|
echo " Host: $DB_HOST:$DB_PORT"
|
|
echo " Database: $DB_NAME"
|
|
echo " User: $DB_USER"
|
|
echo ""
|
|
|
|
# Export for psql
|
|
export PGPASSWORD="$DB_PASSWORD"
|
|
|
|
# Get migration directory (where this script is)
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
MIGRATION_DIR="$SCRIPT_DIR"
|
|
|
|
echo "Migration directory: $MIGRATION_DIR"
|
|
echo ""
|
|
|
|
# Collect all SQL files
|
|
MIGRATIONS=($(ls -1 "$MIGRATION_DIR"/*.sql 2>/dev/null | sort))
|
|
|
|
if [ ${#MIGRATIONS[@]} -eq 0 ]; then
|
|
echo "ERROR: No migration files found in $MIGRATION_DIR"
|
|
exit 1
|
|
fi
|
|
|
|
echo "Found ${#MIGRATIONS[@]} migration(s):"
|
|
for m in "${MIGRATIONS[@]}"; do
|
|
echo " - $(basename $m)"
|
|
done
|
|
echo ""
|
|
|
|
# Wait for DB to be ready
|
|
echo "Waiting for database to be ready..."
|
|
for i in {1..30}; do
|
|
if psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "SELECT 1;" >/dev/null 2>&1; then
|
|
echo "✓ Database is ready"
|
|
break
|
|
fi
|
|
if [ $i -eq 30 ]; then
|
|
echo "✗ Database not ready after 30 attempts"
|
|
exit 1
|
|
fi
|
|
echo " Attempt $i/30..."
|
|
sleep 1
|
|
done
|
|
|
|
echo ""
|
|
echo "=========================================="
|
|
echo "Running Migrations"
|
|
echo "=========================================="
|
|
echo ""
|
|
|
|
SUCCESS=0
|
|
FAILED=0
|
|
|
|
for migration in "${MIGRATIONS[@]}"; do
|
|
name=$(basename "$migration")
|
|
echo -n "▶ $name ... "
|
|
|
|
if psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f "$migration" >/dev/null 2>&1; then
|
|
echo "✓"
|
|
((SUCCESS++))
|
|
else
|
|
echo "✗ FAILED"
|
|
echo ""
|
|
echo "Error output:"
|
|
psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f "$migration" 2>&1 | sed 's/^/ /'
|
|
((FAILED++))
|
|
fi
|
|
done
|
|
|
|
echo ""
|
|
echo "=========================================="
|
|
echo "Migration Summary"
|
|
echo "=========================================="
|
|
echo " Success: $SUCCESS"
|
|
echo " Failed: $FAILED"
|
|
echo ""
|
|
|
|
if [ $FAILED -eq 0 ]; then
|
|
echo "✓ All migrations applied successfully"
|
|
|
|
echo ""
|
|
echo "Verifying schema..."
|
|
echo ""
|
|
|
|
# Verify key tables exist
|
|
for table in memory_entity memory_edge ingest_jobs; do
|
|
if psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "SELECT 1 FROM information_schema.tables WHERE table_name='$table';" 2>&1 | grep -q "1 row"; then
|
|
echo " ✓ Table $table exists"
|
|
else
|
|
echo " ⚠ Table $table not found"
|
|
fi
|
|
done
|
|
|
|
exit 0
|
|
else
|
|
echo "✗ Some migrations failed"
|
|
exit 1
|
|
fi
|