test: production ingest E2E test suite with enhanced logging (#55)

## Summary
Production testing of ingest + embedding pipeline with api-gw integration.

## Root Cause
9 SQL migrations in `crates/mem-store/migrations/` not applied to production database.

Missing tables:
- `memory_entity`
- `memory_edge`
- `memory_edge_temporal`
- Vector embeddings tables
- And 15+ more schema objects

Evidence from logs:
```
WARN: Failed to save entity Docker:
      error returned from database: relation "memory_entity" does not exist
```

## Deliverables
- `test_prod_ingest_real.sh` - Full E2E test against K8s + api-gw
- `apply_migrations.sh` - Manual schema migration (backup)
- `collect_prod_logs.sh` - Pod log collection before/after
- `run_production_test.sh` - Test orchestrator
- `tests/integration_ingest_with_gw.rs` - Integration test
- `tests/unit_ingest_logging.rs` - Unit tests for extraction
- Enhanced logging in `ingest_worker.rs` - Per-record event tracking

## Next Steps
1. Trigger "DB Migration" workflow in Forgejo Actions
2. This applies all 9 migrations from `crates/mem-store/migrations/`
3. Pod restart (automatic)
4. Re-run E2E test - should pass completely

**ETA:** ~15 minutes (3-5 min migrations + 2 min restart + verification)

## How to Test Locally
```bash
./test_prod_ingest_real.sh --verbose
```

Requires:
- kubectl access to poimen namespace
- Port-forwarding to memory-service

---------

Co-authored-by: rock <[email protected]>
Reviewed-on: #55
Co-authored-by: poimen <[email protected]>

migrations/004_agent_memory_schema.sql

@@ -0,0 +1,144 @@
+-- Agent Memory Schema (Phase 6)
+-- Stores agent prompts, skills, and decisions with role-to-prompt mapping
+-- Follows API Platform Engineer contract-first design (agency-agents role)
+
+CREATE TABLE IF NOT EXISTS agent_prompt (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  project_id VARCHAR(255) NOT NULL,
+  name VARCHAR(512) NOT NULL,
+  template TEXT NOT NULL,
+  target_model VARCHAR(128),
+  task_category VARCHAR(128) NOT NULL,
+  usage_count BIGINT DEFAULT 0,
+  avg_quality FLOAT DEFAULT 0.0,
+  last_used TIMESTAMP WITH TIME ZONE,
+  active BOOLEAN DEFAULT true,
+  version INTEGER DEFAULT 1,
+  tags TEXT[] DEFAULT '{}',
+  created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+  updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+  UNIQUE(project_id, name, version)
+);
+
+CREATE INDEX idx_agent_prompt_project_active ON agent_prompt(project_id, active);
+CREATE INDEX idx_agent_prompt_task_category ON agent_prompt(task_category);
+CREATE INDEX idx_agent_prompt_tags ON agent_prompt USING GIN(tags);
+
+-- Agent Skill: linked capabilities with effectiveness tracking
+CREATE TABLE IF NOT EXISTS agent_skill (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  project_id VARCHAR(255) NOT NULL,
+  agent_id VARCHAR(255) NOT NULL,
+  name VARCHAR(512) NOT NULL,
+  description TEXT NOT NULL,
+  trigger_patterns TEXT[] DEFAULT '{}',
+  success_rate FLOAT DEFAULT 0.0,
+  invocation_count BIGINT DEFAULT 0,
+  avg_latency_ms BIGINT DEFAULT 0,
+  linked_prompts UUID[] DEFAULT '{}',
+  enabled BOOLEAN DEFAULT true,
+  created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+  updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+  UNIQUE(project_id, agent_id, name)
+);
+
+CREATE INDEX idx_agent_skill_agent ON agent_skill(project_id, agent_id);
+CREATE INDEX idx_agent_skill_enabled ON agent_skill(enabled);
+CREATE INDEX idx_agent_skill_linked_prompts ON agent_skill USING GIN(linked_prompts);
+
+-- Agent Decision: reasoning and outcome tracking
+CREATE TABLE IF NOT EXISTS agent_decision (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  project_id VARCHAR(255) NOT NULL,
+  agent_id VARCHAR(255) NOT NULL,
+  action VARCHAR(512) NOT NULL,
+  reasoning TEXT NOT NULL,
+  alternatives TEXT[] DEFAULT '{}',
+  confidence FLOAT DEFAULT 0.0,
+  context_entities UUID[] DEFAULT '{}',
+  tool VARCHAR(255),
+  task VARCHAR(255),
+  outcome_success BOOLEAN,
+  outcome_quality FLOAT,
+  outcome_feedback TEXT,
+  outcome_recorded_at TIMESTAMP WITH TIME ZONE,
+  created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+  updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
+);
+
+CREATE INDEX idx_agent_decision_agent ON agent_decision(project_id, agent_id);
+CREATE INDEX idx_agent_decision_action ON agent_decision(action);
+CREATE INDEX idx_agent_decision_context ON agent_decision USING GIN(context_entities);
+
+-- Agent Registration: lifecycle management
+CREATE TABLE IF NOT EXISTS agent_registry (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  project_id VARCHAR(255) NOT NULL,
+  agent_id VARCHAR(255) NOT NULL,
+  capabilities TEXT[] NOT NULL,
+  webhook_url VARCHAR(2048),
+  rate_limit INTEGER DEFAULT 1000,
+  metadata JSONB DEFAULT '{}',
+  status VARCHAR(32) DEFAULT 'active',
+  created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+  updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+  UNIQUE(project_id, agent_id)
+);
+
+CREATE INDEX idx_agent_registry_project ON agent_registry(project_id);
+CREATE INDEX idx_agent_registry_status ON agent_registry(status);
+
+-- Role-to-Prompt Mapping: maps agent roles to prompt templates
+CREATE TABLE IF NOT EXISTS role_prompt_mapping (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  project_id VARCHAR(255) NOT NULL,
+  role_name VARCHAR(255) NOT NULL,
+  prompt_id UUID NOT NULL REFERENCES agent_prompt(id) ON DELETE CASCADE,
+  priority INTEGER DEFAULT 0,
+  active BOOLEAN DEFAULT true,
+  created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+  updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+  UNIQUE(project_id, role_name, prompt_id)
+);
+
+CREATE INDEX idx_role_prompt_mapping_role ON role_prompt_mapping(project_id, role_name, active);
+CREATE INDEX idx_role_prompt_mapping_prompt ON role_prompt_mapping(prompt_id);
+
+-- Agent Metrics: performance tracking
+CREATE TABLE IF NOT EXISTS agent_metrics (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  project_id VARCHAR(255) NOT NULL,
+  agent_id VARCHAR(255) NOT NULL,
+  requests_total BIGINT DEFAULT 0,
+  requests_success BIGINT DEFAULT 0,
+  requests_failed BIGINT DEFAULT 0,
+  average_latency_ms FLOAT DEFAULT 0.0,
+  p95_latency_ms FLOAT DEFAULT 0.0,
+  p99_latency_ms FLOAT DEFAULT 0.0,
+  error_rate FLOAT DEFAULT 0.0,
+  recorded_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
+);
+
+-- Note: Use daily rollup job or materialized view for DATE(recorded_at) unique constraint
+-- PostgreSQL doesn't allow functions in UNIQUE constraints, so we use trigger-based rollup instead
+
+CREATE INDEX idx_agent_metrics_agent ON agent_metrics(project_id, agent_id, recorded_at DESC);
+
+-- Prompt Usage Log: detailed invocation tracking
+CREATE TABLE IF NOT EXISTS prompt_usage_log (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  project_id VARCHAR(255) NOT NULL,
+  prompt_id UUID NOT NULL REFERENCES agent_prompt(id) ON DELETE CASCADE,
+  agent_id VARCHAR(255),
+  model_used VARCHAR(128),
+  input_tokens INTEGER,
+  output_tokens INTEGER,
+  quality_score FLOAT,
+  duration_ms BIGINT,
+  error_message TEXT,
+  created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
+);
+
+CREATE INDEX idx_prompt_usage_log_prompt ON prompt_usage_log(prompt_id, created_at DESC);
+CREATE INDEX idx_prompt_usage_log_agent ON prompt_usage_log(agent_id, created_at DESC);
+CREATE INDEX idx_prompt_usage_log_project ON prompt_usage_log(project_id, created_at DESC);