test: production ingest E2E test suite with enhanced logging (#55)

## Summary
Production testing of ingest + embedding pipeline with api-gw integration.

## Root Cause
9 SQL migrations in `crates/mem-store/migrations/` not applied to production database.

Missing tables:
- `memory_entity`
- `memory_edge`
- `memory_edge_temporal`
- Vector embeddings tables
- And 15+ more schema objects

Evidence from logs:
```
WARN: Failed to save entity Docker:
      error returned from database: relation "memory_entity" does not exist
```

## Deliverables
- `test_prod_ingest_real.sh` - Full E2E test against K8s + api-gw
- `apply_migrations.sh` - Manual schema migration (backup)
- `collect_prod_logs.sh` - Pod log collection before/after
- `run_production_test.sh` - Test orchestrator
- `tests/integration_ingest_with_gw.rs` - Integration test
- `tests/unit_ingest_logging.rs` - Unit tests for extraction
- Enhanced logging in `ingest_worker.rs` - Per-record event tracking

## Next Steps
1. Trigger "DB Migration" workflow in Forgejo Actions
2. This applies all 9 migrations from `crates/mem-store/migrations/`
3. Pod restart (automatic)
4. Re-run E2E test - should pass completely

**ETA:** ~15 minutes (3-5 min migrations + 2 min restart + verification)

## How to Test Locally
```bash
./test_prod_ingest_real.sh --verbose
```

Requires:
- kubectl access to poimen namespace
- Port-forwarding to memory-service

---------

Co-authored-by: rock <[email protected]>
Reviewed-on: #55
Co-authored-by: poimen <[email protected]>

crates/mem-ingest/src/contradiction_detector.rs

@@ -83,6 +83,7 @@ impl ContradictionPreFilter {
 
 /// LLM-based contradiction detector (stage 2)
 /// Only called if pre-filter returns true (cost optimization)
+#[allow(dead_code)]
 pub struct LlmContradictionDetector {
     model_name: String,
     auto_confirm_threshold: f32,

crates/mem-ingest/src/entity_extractor.rs

@@ -44,10 +44,15 @@ impl ExtractedEntity {
 #[async_trait]
 pub trait EntityExtractor: Send + Sync {
     async fn extract(&self, text: &str) -> Result<Vec<ExtractedEntity>>;
+    async fn extract_with_auth(&self, text: &str, _x_forward_user: Option<&str>) -> Result<Vec<ExtractedEntity>> {
+        // Default: ignore auth header, use regular extract
+        self.extract(text).await
+    }
 }
 
 /// LLM-based extractor with reflection verification (stage 1 + 2)
 /// Uses Authentik JWT tokens for authentication to LLM gateway
+#[allow(dead_code)]
 pub struct LlmEntityExtractor {
     model_name: String,
     enable_reflection: bool,
@@ -120,29 +125,42 @@ impl LlmEntityExtractor {
         Ok(parsed.verified.into_iter().map(|v| (v.name, v.present)).collect())
     }
 
-    /// Call LLM via api.riotpiao.com using Authentik JWT
-    /// Token is fetched from Authentik service account and cached
-    async fn call_llm_endpoint(&self, prompt: &str) -> Result<String> {
+    /// Call LLM via api.riotpiao.com using X-Forward-User auth/exchange
+    /// Supports: Authentik JWT, X-Forward-User header, or API key fallback
+    async fn call_llm_endpoint(&self, prompt: &str, x_forward_user: Option<&str>) -> Result<String> {
         let endpoint = std::env::var("LLM_ENDPOINT")
             .unwrap_or_else(|_| "http://api-internal.riotpiao.com:8000/v1/chat/completions".to_string());
         let model = std::env::var("LLM_MODEL")
             .unwrap_or_else(|_| "qwen:7b".to_string());
         
-        // Get JWT token from Authentik
-        let auth_header = if let Some(jwt_issuer) = &self.jwt_issuer {
+        // Get auth header: prefer X-Forward-User, fallback to Authentik JWT, then API key
+        let auth_header = if let Some(user) = x_forward_user {
+            // Use X-Forward-User directly (API Gateway pattern)
+            tracing::info!("Using X-Forward-User for LLM auth: {}", user);
+            format!("X-Forward-User: {}", user)
+        } else if let Some(jwt_issuer) = &self.jwt_issuer {
             let issuer = jwt_issuer.lock().await;
             match issuer.get_access_token().await {
-                Ok(token) => format!("Bearer {}", token),
+                Ok(token) => {
+                    tracing::info!("Using Authentik JWT for LLM auth");
+                    format!("Bearer {}", token)
+                },
                 Err(e) => {
                     tracing::warn!("Failed to get Authentik JWT: {}", e);
-                    return Err(e);
+                    // Fallback to env var
+                    let api_key = std::env::var("LLM_API_KEY")
+                        .or_else(|_| std::env::var("MEM_API_KEY"))
+                        .unwrap_or_else(|_| "test-key".to_string());
+                    tracing::info!("Falling back to LLM_API_KEY");
+                    format!("Bearer {}", api_key)
                 }
             }
         } else {
             // Fallback to env var if Authentik not configured
             let api_key = std::env::var("LLM_API_KEY")
                 .or_else(|_| std::env::var("MEM_API_KEY"))
-                .unwrap_or_else(|_| "default-key".to_string());
+                .unwrap_or_else(|_| "test-key".to_string());
+            tracing::info!("Using LLM_API_KEY for LLM auth");
             format!("Bearer {}", api_key)
         };
         
@@ -159,23 +177,33 @@ impl LlmEntityExtractor {
             "max_tokens": 12000
         });
         
-        let response = client
+        let mut request = client
             .post(&endpoint)
-            .header("Authorization", auth_header)
-            .header("Content-Type", "application/json")
+            .header("Content-Type", "application/json");
+        
+        // Set auth header (varies by auth method)
+        if auth_header.starts_with("X-Forward-User") {
+            request = request.header("X-Forward-User", auth_header.split(": ").nth(1).unwrap_or("unknown"));
+        } else {
+            request = request.header("Authorization", auth_header);
+        }
+        
+        let response = request
             .json(&payload)
             .timeout(std::time::Duration::from_secs(90))
             .send()
             .await?;
         
-        if !response.status().is_success() {
-            tracing::warn!(
+        let status = response.status();
+        if !status.is_success() {
+            let error_text = response.text().await.unwrap_or_default();
+            tracing::error!(
                 "LLM API error: {} - {}",
-                response.status(),
-                response.text().await.unwrap_or_default()
+                status,
+                error_text
             );
-            // Fallback to mock response on error
-            return Ok(r#"{"entities": []}"#.to_string());
+            // Return error instead of silently returning empty array
+            return Err(anyhow::anyhow!("LLM API failed with status {}: {}", status, error_text));
         }
         
         let data: serde_json::Value = response.json().await?;
@@ -257,7 +285,10 @@ Respond in JSON:
 
         // Try real LLM first, fallback to mock if not configured
         let extraction_response = if std::env::var("LLM_ENDPOINT").is_ok() {
-            self.call_llm_endpoint(&prompt).await.unwrap_or_else(|_| self.simulate_llm(&prompt).unwrap_or_default())
+            self.call_llm_endpoint(&prompt, None).await.unwrap_or_else(|e| {
+                tracing::error!("LLM entity extraction failed: {}, using mock", e);
+                self.simulate_llm(&prompt).unwrap_or_default()
+            })
         } else {
             self.simulate_llm(&prompt)?
         };
@@ -282,7 +313,7 @@ Respond in JSON:
             );
 
             let reflection = if std::env::var("LLM_ENDPOINT").is_ok() {
-                self.call_llm_endpoint(&reflection_prompt).await.unwrap_or_else(|e| {
+                self.call_llm_endpoint(&reflection_prompt, None).await.unwrap_or_else(|e| {
                     tracing::warn!("Reflection LLM call failed: {}, skipping verification", e);
                     String::new()
                 })
@@ -312,6 +343,85 @@ Respond in JSON:
 
         Ok(entities)
     }
+
+    /// Extract with X-Forward-User auth header (API Gateway pattern)
+    async fn extract_with_auth(&self, text: &str, x_forward_user: Option<&str>) -> Result<Vec<ExtractedEntity>> {
+        let mut entities = vec![];
+        
+        // Extract speaker if available
+        use crate::speaker_extractor::{HeuristicSpeakerExtractor, SpeakerConfig};
+        if let Ok(speaker_extractor) = HeuristicSpeakerExtractor::new(SpeakerConfig::default()) {
+            if let Ok(Some(speaker)) = speaker_extractor.extract_speaker(text).await {
+                entities.push(ExtractedEntity {
+                    name: speaker.name,
+                    entity_type: mem_core::entity::EntityType::Person,
+                    summary: "Speaker in this episode".to_string(),
+                    confidence: speaker.confidence,
+                });
+            }
+        }
+
+        // Extract entities with auth header
+        let prompt = format!(
+            r#"Extract named entities from this text.
+
+For each entity provide:
+- name: Canonical name (proper capitalization)
+- type: One of [person, tool, concept, location, event, organization]
+- summary: One sentence
+
+CRITICAL: Only extract entities EXPLICITLY mentioned. No inference.
+
+Text:
+"{}"
+
+Respond in JSON:
+{{"entities": [{{"name": "...", "type": "...", "summary": "..."}}, ...]}}
+"#,
+            text
+        );
+
+        // Use provided X-Forward-User for auth
+        let extraction_response = if std::env::var("LLM_ENDPOINT").is_ok() {
+            self.call_llm_endpoint(&prompt, x_forward_user).await.unwrap_or_else(|e| {
+                tracing::error!("LLM entity extraction with auth failed: {}", e);
+                self.simulate_llm(&prompt).unwrap_or_default()
+            })
+        } else {
+            self.simulate_llm(&prompt)?
+        };
+        
+        let extracted = Self::parse_extraction(&extraction_response)?;
+        entities.extend(extracted);
+
+        // Optional: reflection verification with auth
+        if self.enable_reflection && std::env::var("LLM_ENDPOINT").is_ok() {
+            let reflection_prompt = format!(
+                r#"Verify these entities are explicitly in the text:
+
+Text:
+"{}"
+
+Entities:
+{:?}
+
+Respond in JSON:
+{{"verified": [{{"name": "...", "present": true/false}}, ...]}}
+"#,
+                text, entities
+            );
+
+            if let Ok(reflection) = self.call_llm_endpoint(&reflection_prompt, x_forward_user).await {
+                if !reflection.is_empty() {
+                    if let Ok(verified) = Self::parse_reflection(&reflection) {
+                        entities.retain(|e| verified.iter().any(|(name, present)| name == &e.name && *present));
+                    }
+                }
+            }
+        }
+
+        Ok(entities)
+    }
 }
 
 /// Fallback extractor: Use wiki_links if LLM fails (stage 3)
@@ -330,7 +440,7 @@ impl EntityExtractor for WikiLinkFallbackExtractor {
                 entities.push(ExtractedEntity {
                     name: name_str.to_string(),
                     entity_type: EntityType::Unknown,
-                    summary: format!("Mentioned in episode"),
+                    summary: "Mentioned in episode".to_string(),
                     confidence: 0.7, // Lower confidence for fallback
                 });
             }
@@ -418,6 +528,6 @@ mod tests {
         let text = "[[Entity1]] and [[Entity2]]";
 
         let entities = composite.extract(text).await.unwrap();
-        assert!(entities.len() > 0);
+        assert!(!entities.is_empty());
     }
 }

crates/mem-ingest/src/grm_retriever.rs

@@ -10,10 +10,7 @@
 use anyhow::Result;
 use async_trait::async_trait;
 use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
-use tracing::{debug, info};
-use mem_core::entity::Entity;
-use mem_core::edge::Edge;
+use tracing::debug;
 
 /// Memorability decision for entity or fact
 #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq)]

crates/mem-ingest/src/ingest_pipeline.rs

@@ -59,10 +59,15 @@ impl IngestPipeline {
     /// Execute extraction pipeline for episode
     /// CRAP: 14 (Low: orchestration only, delegates to stages)
     pub async fn ingest(&self, episode: &Episode) -> Result<ExtractionResult> {
+        self.ingest_with_auth(episode, None).await
+    }
+
+    /// Ingest with optional X-Forward-User auth header
+    pub async fn ingest_with_auth(&self, episode: &Episode, x_forward_user: Option<&str>) -> Result<ExtractionResult> {
         debug!("Starting ingest for episode: {}", episode.id);
 
-        // Stage 1: Extract entities
-        let extracted_entities = self.entity_extractor.extract(&episode.text).await?;
+        // Stage 1: Extract entities (with optional auth header)
+        let extracted_entities = self.entity_extractor.extract_with_auth(&episode.text, x_forward_user).await?;
         debug!("Extracted {} entities", extracted_entities.len());
 
         // Convert to domain entities
@@ -144,6 +149,7 @@ impl IngestPipeline {
 
 /// Async queue worker: Process episodes from queue
 /// CRAP: 12 (Async loop, straightforward)
+#[allow(dead_code)]
 pub struct QueueWorker {
     pipeline: Arc<IngestPipeline>,
     batch_size: usize,

crates/mem-ingest/src/memorability_gate.rs

@@ -14,7 +14,7 @@ use tracing::{debug, info};
 use crate::grm_retriever::{
     EntityContext, FactContext, GraphContextRetriever, MemorabilityDecision, GrmConfig, MockGrmRetriever,
 };
-use mem_core::entity::{Entity, EntityType};
+use mem_core::entity::Entity;
 use mem_core::edge::Edge;
 
 /// Entity filtering result
@@ -88,7 +88,7 @@ impl MemorabilityGate {
         let (filtered, reason) = match context.decision {
             MemorabilityDecision::Keep => {
                 if context.matched_entity_id.is_some() {
-                    (true, format!("Existing entity (merge required)"))
+                    (true, "Existing entity (merge required)".to_string())
                 } else {
                     (false, format!("New entity (score: {:.2})", context.memorability_score))
                 }
@@ -243,7 +243,7 @@ pub struct FilterStatistics {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use mem_core::entity::Entity;
+    use mem_core::entity::{Entity, EntityType};
 
     fn create_test_entity(name: &str) -> Entity {
         Entity::new("poimen", name, EntityType::Person)

crates/mem-ingest/src/obsidian_ref_source.rs

@@ -20,6 +20,7 @@ pub struct RefMetadata {
 }
 
 /// Obsidian REST API client
+#[allow(dead_code)]
 pub struct ObsidianClient {
     base_url: String,
 }
@@ -47,6 +48,7 @@ impl ObsidianClient {
 }
 
 /// ObsidianRefSource: Fetches & chunks reference documents from Obsidian vault
+#[allow(dead_code)]
 pub struct ObsidianRefSource {
     client: ObsidianClient,
     project: String,
@@ -68,11 +70,13 @@ impl ObsidianRefSource {
     }
 
     /// Check if a file path is allowed (matches configured prefixes)
+    #[allow(dead_code)]
     fn is_allowed_path(&self, path: &str) -> bool {
         self.allowed_paths.iter().any(|prefix| path.starts_with(prefix))
     }
 
     /// Chunk reference document via heading-boundary logic
+    #[allow(dead_code)]
     fn chunk_document(&self, path: &str, content: &str) -> Vec<Record> {
         // M3.6.1 heading-boundary chunking
         // - Split by headings
@@ -203,7 +207,7 @@ mod tests {
         let chunks = source.chunk_document("docs/test.md", content);
 
         // Should split by headings
-        assert!(chunks.len() > 0);
+        assert!(!chunks.is_empty());
     }
 
     #[test]

crates/mem-ingest/src/optimizer_metrics.rs

@@ -60,8 +60,7 @@ impl MetricsCollector {
         self.by_project
             .lock()
             .unwrap()
-            .get(project)
-            .map(|m| m.clone())
+            .get(project).cloned()
     }
 
     /// Get all project metrics.

crates/mem-ingest/src/query_metrics.rs

@@ -306,7 +306,7 @@ impl QueryMetricsRepository {
         let mut repo = self.metrics.lock().unwrap();
         repo.get_mut(query_id)
             .ok_or_else(|| format!("Query {} not found", query_id))
-            .map(|metrics| f(metrics))
+            .map(f)
     }
 
     /// Get progress for a query

crates/mem-ingest/src/wiki_link.rs

@@ -4,9 +4,10 @@
 ///
 /// Used to scope queries to project namespaces and enable graph traversal.
 /// For example: poimen/tools/kubectl.md [[debugging.md]] creates an edge
+#[allow(clippy::empty_line_after_doc_comments)]
 /// from tools/kubectl to debugging (within same project).
 
-use anyhow::{anyhow, Result};
+use anyhow::Result;
 use regex::Regex;
 use std::collections::{HashMap, HashSet};
 use std::path::{Path, PathBuf};
@@ -79,6 +80,7 @@ impl WikiLinkParser {
 }
 
 /// Graph Index: Stores and queries wiki-link relationships
+#[allow(dead_code)]
 pub struct WikiLinkGraph {
     /// Forward links: source -> [targets]
     forward_links: HashMap<String, Vec<String>>,
@@ -100,11 +102,11 @@ impl WikiLinkGraph {
     /// Add a wiki-link edge
     pub fn add_link(&mut self, source: &str, target: &str) {
         self.forward_links.entry(source.to_string())
-            .or_insert_with(Vec::new)
+            .or_default()
             .push(target.to_string());
 
         self.backward_links.entry(target.to_string())
-            .or_insert_with(Vec::new)
+            .or_default()
             .push(source.to_string());
     }