test: production ingest E2E test suite with enhanced logging (#55)

## Summary Production testing of ingest + embedding pipeline with api-gw integration. ## Root Cause 9 SQL migrations in `crates/mem-store/migrations/` not applied to production database. Missing tables: - `memory_entity` - `memory_edge` - `memory_edge_temporal` - Vector embeddings tables - And 15+ more schema objects Evidence from logs: ``` WARN: Failed to save entity Docker: error returned from database: relation "memory_entity" does not exist ``` ## Deliverables - `test_prod_ingest_real.sh` - Full E2E test against K8s + api-gw - `apply_migrations.sh` - Manual schema migration (backup) - `collect_prod_logs.sh` - Pod log collection before/after - `run_production_test.sh` - Test orchestrator - `tests/integration_ingest_with_gw.rs` - Integration test - `tests/unit_ingest_logging.rs` - Unit tests for extraction - Enhanced logging in `ingest_worker.rs` - Per-record event tracking ## Next Steps 1. Trigger "DB Migration" workflow in Forgejo Actions 2. This applies all 9 migrations from `crates/mem-store/migrations/` 3. Pod restart (automatic) 4. Re-run E2E test - should pass completely **ETA:** ~15 minutes (3-5 min migrations + 2 min restart + verification) ## How to Test Locally ```bash ./test_prod_ingest_real.sh --verbose ``` Requires: - kubectl access to poimen namespace - Port-forwarding to memory-service --------- Co-authored-by: rock <[email protected]> Reviewed-on: #55 Co-authored-by: poimen <[email protected]>
2026-09-16 00:10:58 +00:00
parent 4169effd8a
commit a88ea918bf
137 changed files with 4628 additions and 7227 deletions
@@ -0,0 +1,144 @@
+-- Agent Memory Schema (Phase 6)
+-- Stores agent prompts, skills, and decisions with role-to-prompt mapping
+-- Follows API Platform Engineer contract-first design (agency-agents role)
+
+CREATE TABLE IF NOT EXISTS agent_prompt (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  project_id VARCHAR(255) NOT NULL,
+  name VARCHAR(512) NOT NULL,
+  template TEXT NOT NULL,
+  target_model VARCHAR(128),
+  task_category VARCHAR(128) NOT NULL,
+  usage_count BIGINT DEFAULT 0,
+  avg_quality FLOAT DEFAULT 0.0,
+  last_used TIMESTAMP WITH TIME ZONE,
+  active BOOLEAN DEFAULT true,
+  version INTEGER DEFAULT 1,
+  tags TEXT[] DEFAULT '{}',
+  created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+  updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+  UNIQUE(project_id, name, version)
+);
+
+CREATE INDEX idx_agent_prompt_project_active ON agent_prompt(project_id, active);
+CREATE INDEX idx_agent_prompt_task_category ON agent_prompt(task_category);
+CREATE INDEX idx_agent_prompt_tags ON agent_prompt USING GIN(tags);
+
+-- Agent Skill: linked capabilities with effectiveness tracking
+CREATE TABLE IF NOT EXISTS agent_skill (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  project_id VARCHAR(255) NOT NULL,
+  agent_id VARCHAR(255) NOT NULL,
+  name VARCHAR(512) NOT NULL,
+  description TEXT NOT NULL,
+  trigger_patterns TEXT[] DEFAULT '{}',
+  success_rate FLOAT DEFAULT 0.0,
+  invocation_count BIGINT DEFAULT 0,
+  avg_latency_ms BIGINT DEFAULT 0,
+  linked_prompts UUID[] DEFAULT '{}',
+  enabled BOOLEAN DEFAULT true,
+  created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+  updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+  UNIQUE(project_id, agent_id, name)
+);
+
+CREATE INDEX idx_agent_skill_agent ON agent_skill(project_id, agent_id);
+CREATE INDEX idx_agent_skill_enabled ON agent_skill(enabled);
+CREATE INDEX idx_agent_skill_linked_prompts ON agent_skill USING GIN(linked_prompts);
+
+-- Agent Decision: reasoning and outcome tracking
+CREATE TABLE IF NOT EXISTS agent_decision (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  project_id VARCHAR(255) NOT NULL,
+  agent_id VARCHAR(255) NOT NULL,
+  action VARCHAR(512) NOT NULL,
+  reasoning TEXT NOT NULL,
+  alternatives TEXT[] DEFAULT '{}',
+  confidence FLOAT DEFAULT 0.0,
+  context_entities UUID[] DEFAULT '{}',
+  tool VARCHAR(255),
+  task VARCHAR(255),
+  outcome_success BOOLEAN,
+  outcome_quality FLOAT,
+  outcome_feedback TEXT,
+  outcome_recorded_at TIMESTAMP WITH TIME ZONE,
+  created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+  updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
+);
+
+CREATE INDEX idx_agent_decision_agent ON agent_decision(project_id, agent_id);
+CREATE INDEX idx_agent_decision_action ON agent_decision(action);
+CREATE INDEX idx_agent_decision_context ON agent_decision USING GIN(context_entities);
+
+-- Agent Registration: lifecycle management
+CREATE TABLE IF NOT EXISTS agent_registry (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  project_id VARCHAR(255) NOT NULL,
+  agent_id VARCHAR(255) NOT NULL,
+  capabilities TEXT[] NOT NULL,
+  webhook_url VARCHAR(2048),
+  rate_limit INTEGER DEFAULT 1000,
+  metadata JSONB DEFAULT '{}',
+  status VARCHAR(32) DEFAULT 'active',
+  created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+  updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+  UNIQUE(project_id, agent_id)
+);
+
+CREATE INDEX idx_agent_registry_project ON agent_registry(project_id);
+CREATE INDEX idx_agent_registry_status ON agent_registry(status);
+
+-- Role-to-Prompt Mapping: maps agent roles to prompt templates
+CREATE TABLE IF NOT EXISTS role_prompt_mapping (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  project_id VARCHAR(255) NOT NULL,
+  role_name VARCHAR(255) NOT NULL,
+  prompt_id UUID NOT NULL REFERENCES agent_prompt(id) ON DELETE CASCADE,
+  priority INTEGER DEFAULT 0,
+  active BOOLEAN DEFAULT true,
+  created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+  updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+  UNIQUE(project_id, role_name, prompt_id)
+);
+
+CREATE INDEX idx_role_prompt_mapping_role ON role_prompt_mapping(project_id, role_name, active);
+CREATE INDEX idx_role_prompt_mapping_prompt ON role_prompt_mapping(prompt_id);
+
+-- Agent Metrics: performance tracking
+CREATE TABLE IF NOT EXISTS agent_metrics (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  project_id VARCHAR(255) NOT NULL,
+  agent_id VARCHAR(255) NOT NULL,
+  requests_total BIGINT DEFAULT 0,
+  requests_success BIGINT DEFAULT 0,
+  requests_failed BIGINT DEFAULT 0,
+  average_latency_ms FLOAT DEFAULT 0.0,
+  p95_latency_ms FLOAT DEFAULT 0.0,
+  p99_latency_ms FLOAT DEFAULT 0.0,
+  error_rate FLOAT DEFAULT 0.0,
+  recorded_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
+);
+
+-- Note: Use daily rollup job or materialized view for DATE(recorded_at) unique constraint
+-- PostgreSQL doesn't allow functions in UNIQUE constraints, so we use trigger-based rollup instead
+
+CREATE INDEX idx_agent_metrics_agent ON agent_metrics(project_id, agent_id, recorded_at DESC);
+
+-- Prompt Usage Log: detailed invocation tracking
+CREATE TABLE IF NOT EXISTS prompt_usage_log (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  project_id VARCHAR(255) NOT NULL,
+  prompt_id UUID NOT NULL REFERENCES agent_prompt(id) ON DELETE CASCADE,
+  agent_id VARCHAR(255),
+  model_used VARCHAR(128),
+  input_tokens INTEGER,
+  output_tokens INTEGER,
+  quality_score FLOAT,
+  duration_ms BIGINT,
+  error_message TEXT,
+  created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
+);
+
+CREATE INDEX idx_prompt_usage_log_prompt ON prompt_usage_log(prompt_id, created_at DESC);
+CREATE INDEX idx_prompt_usage_log_agent ON prompt_usage_log(agent_id, created_at DESC);
+CREATE INDEX idx_prompt_usage_log_project ON prompt_usage_log(project_id, created_at DESC);
@@ -0,0 +1,127 @@
+#!/bin/bash
+
+# Database Migration Runner
+# Used by K8s Job to apply all migrations before integration tests
+# 
+# Environment variables (from K8s):
+#   DB_HOST     - PostgreSQL host
+#   DB_PORT     - PostgreSQL port
+#   DB_NAME     - Database name
+#   DB_USER     - Database user
+#   DB_PASSWORD - Database password (from Secret)
+
+set -e
+
+DB_HOST="${DB_HOST:-memory-db-rw.poimen.svc.cluster.local}"
+DB_PORT="${DB_PORT:-5432}"
+DB_NAME="${DB_NAME:-memory}"
+DB_USER="${DB_USER:-app}"
+
+if [ -z "$DB_PASSWORD" ]; then
+  echo "ERROR: DB_PASSWORD not set"
+  exit 1
+fi
+
+echo "=========================================="
+echo "Database Migration Runner"
+echo "=========================================="
+echo ""
+echo "Configuration:"
+echo "  Host: $DB_HOST:$DB_PORT"
+echo "  Database: $DB_NAME"
+echo "  User: $DB_USER"
+echo ""
+
+# Export for psql
+export PGPASSWORD="$DB_PASSWORD"
+
+# Get migration directory (where this script is)
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+MIGRATION_DIR="$SCRIPT_DIR"
+
+echo "Migration directory: $MIGRATION_DIR"
+echo ""
+
+# Collect all SQL files
+MIGRATIONS=($(ls -1 "$MIGRATION_DIR"/*.sql 2>/dev/null | sort))
+
+if [ ${#MIGRATIONS[@]} -eq 0 ]; then
+  echo "ERROR: No migration files found in $MIGRATION_DIR"
+  exit 1
+fi
+
+echo "Found ${#MIGRATIONS[@]} migration(s):"
+for m in "${MIGRATIONS[@]}"; do
+  echo "  - $(basename $m)"
+done
+echo ""
+
+# Wait for DB to be ready
+echo "Waiting for database to be ready..."
+for i in {1..30}; do
+  if psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "SELECT 1;" >/dev/null 2>&1; then
+    echo "✓ Database is ready"
+    break
+  fi
+  if [ $i -eq 30 ]; then
+    echo "✗ Database not ready after 30 attempts"
+    exit 1
+  fi
+  echo "  Attempt $i/30..."
+  sleep 1
+done
+
+echo ""
+echo "=========================================="
+echo "Running Migrations"
+echo "=========================================="
+echo ""
+
+SUCCESS=0
+FAILED=0
+
+for migration in "${MIGRATIONS[@]}"; do
+  name=$(basename "$migration")
+  echo -n "▶ $name ... "
+  
+  if psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f "$migration" >/dev/null 2>&1; then
+    echo "✓"
+    ((SUCCESS++))
+  else
+    echo "✗ FAILED"
+    echo ""
+    echo "Error output:"
+    psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f "$migration" 2>&1 | sed 's/^/  /'
+    ((FAILED++))
+  fi
+done
+
+echo ""
+echo "=========================================="
+echo "Migration Summary"
+echo "=========================================="
+echo "  Success: $SUCCESS"
+echo "  Failed:  $FAILED"
+echo ""
+
+if [ $FAILED -eq 0 ]; then
+  echo "✓ All migrations applied successfully"
+  
+  echo ""
+  echo "Verifying schema..."
+  echo ""
+  
+  # Verify key tables exist
+  for table in memory_entity memory_edge ingest_jobs; do
+    if psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "SELECT 1 FROM information_schema.tables WHERE table_name='$table';" 2>&1 | grep -q "1 row"; then
+      echo "  ✓ Table $table exists"
+    else
+      echo "  ⚠ Table $table not found"
+    fi
+  done
+  
+  exit 0
+else
+  echo "✗ Some migrations failed"
+  exit 1
+fi