test: production ingest E2E test suite with enhanced logging (#55)

## Summary
Production testing of ingest + embedding pipeline with api-gw integration.

## Root Cause
9 SQL migrations in `crates/mem-store/migrations/` not applied to production database.

Missing tables:
- `memory_entity`
- `memory_edge`
- `memory_edge_temporal`
- Vector embeddings tables
- And 15+ more schema objects

Evidence from logs:
```
WARN: Failed to save entity Docker:
      error returned from database: relation "memory_entity" does not exist
```

## Deliverables
- `test_prod_ingest_real.sh` - Full E2E test against K8s + api-gw
- `apply_migrations.sh` - Manual schema migration (backup)
- `collect_prod_logs.sh` - Pod log collection before/after
- `run_production_test.sh` - Test orchestrator
- `tests/integration_ingest_with_gw.rs` - Integration test
- `tests/unit_ingest_logging.rs` - Unit tests for extraction
- Enhanced logging in `ingest_worker.rs` - Per-record event tracking

## Next Steps
1. Trigger "DB Migration" workflow in Forgejo Actions
2. This applies all 9 migrations from `crates/mem-store/migrations/`
3. Pod restart (automatic)
4. Re-run E2E test - should pass completely

**ETA:** ~15 minutes (3-5 min migrations + 2 min restart + verification)

## How to Test Locally
```bash
./test_prod_ingest_real.sh --verbose
```

Requires:
- kubectl access to poimen namespace
- Port-forwarding to memory-service

---------

Co-authored-by: rock <[email protected]>
Reviewed-on: #55
Co-authored-by: poimen <[email protected]>

crates/mem-llm/src/chat.rs

@@ -34,7 +34,7 @@ pub enum AuthMode {
 
 impl AuthMode {
     /// Detect from base URL or explicit env var.
-    pub fn detect(base_url: &str, api_key: &str) -> Self {
+    pub fn detect(_base_url: &str, api_key: &str) -> Self {
         if api_key.is_empty() {
             return Self::None;
         }
@@ -87,6 +87,7 @@ struct Choice {
 }
 
 #[derive(Debug, Deserialize)]
+#[allow(dead_code)]
 struct MessageResponse {
     role: String,
     content: String,
@@ -208,12 +209,11 @@ impl ChatClient {
                 Ok(r) => r,
                 Err(e) => {
                     last_error = Some(anyhow!("Request failed: {}", e));
-                    if e.is_timeout() || e.is_status() {
-                        if attempt < self.max_retries - 1 {
+                    if (e.is_timeout() || e.is_status())
+                        && attempt < self.max_retries - 1 {
                             tokio::time::sleep(Duration::from_millis(100 * 2_u64.pow(attempt))).await;
                             continue;
                         }
-                    }
                     return Err(last_error.unwrap());
                 }
             };

crates/mem-llm/src/embeddings.rs

@@ -27,6 +27,7 @@ struct EmbeddingRequest {
 }
 
 #[derive(Debug, Deserialize)]
+#[allow(dead_code)]
 #[serde(untagged)]
 enum EmbeddingResponse {
     Success {
@@ -42,6 +43,7 @@ enum EmbeddingResponse {
 }
 
 #[derive(Debug, Deserialize)]
+#[allow(dead_code)]
 struct EmbeddingData {
     embedding: Vec<f32>,
     #[serde(default)]
@@ -62,9 +64,15 @@ impl EmbeddingsClient {
     /// - `LLM_API_BASE`: Gateway endpoint (default: https://api.riotpiao.com)
     /// - `LLM_API_KEY`: API key (optional)
     pub fn from_env() -> Result<Self> {
-        let base_url = env::var("LLM_API_BASE")
+        let mut base_url = env::var("LLM_API_BASE")
             .unwrap_or_else(|_| "https://api.riotpiao.com".to_string());
         
+        // Strip trailing /v1 to avoid double /v1/v1/embeddings
+        base_url = base_url.trim_end_matches('/').to_string();
+        if base_url.ends_with("/v1") {
+            base_url = base_url[..base_url.len() - 3].to_string();
+        }
+        
         let model = env::var("EMBEDDINGS_MODEL")
             .unwrap_or_else(|_| "nomic-ai/nomic-embed-text-v2-moe".to_string());
         
@@ -120,10 +128,10 @@ impl EmbeddingsClient {
     /// Embed a single text string, returning a 768-dim vector
     pub async fn embed_one(&self, text: &str) -> Result<Vector> {
         let embeddings = self.embed(&[text.to_string()]).await?;
-        Ok(embeddings
+        embeddings
             .into_iter()
             .next()
-            .ok_or_else(|| anyhow!("empty embedding response"))?)
+            .ok_or_else(|| anyhow!("empty embedding response"))
     }
 
     /// Embed multiple texts, batched at ≤32 per request, preserving input order
@@ -159,10 +167,9 @@ impl EmbeddingsClient {
         let url = format!("{}/v1/embeddings", self.base_url);
         let mut builder = self.http.post(&url);
 
-        // Send apikey header even though route currently doesn't require auth
-        // This future-proofs for when the route's auth plugin gets enabled
+        // Send as Bearer token (gateway expects Authorization: Bearer <key>)
         if !self.api_key.is_empty() {
-            builder = builder.header("apikey", &self.api_key);
+            builder = builder.header("Authorization", format!("Bearer {}", &self.api_key));
         }
 
         let resp = builder.json(&req).send().await?;
@@ -212,4 +219,95 @@ mod tests {
         assert_eq!(BATCH_SIZE, 32);
         assert_eq!(EMBEDDINGS_DIM, 768);
     }
+
+    #[test]
+    fn test_strip_trailing_v1() {
+        // Simulates LLM_API_BASE=https://api.riotpiao.com/v1
+        let mut base = "https://api.riotpiao.com/v1".to_string();
+        base = base.trim_end_matches('/').to_string();
+        if base.ends_with("/v1") {
+            base = base[..base.len() - 3].to_string();
+        }
+        assert_eq!(base, "https://api.riotpiao.com");
+        assert_eq!(format!("{}/v1/embeddings", base), "https://api.riotpiao.com/v1/embeddings");
+    }
+
+    #[test]
+    fn test_no_strip_when_no_v1() {
+        let mut base = "https://api.riotpiao.com".to_string();
+        base = base.trim_end_matches('/').to_string();
+        if base.ends_with("/v1") {
+            base = base[..base.len() - 3].to_string();
+        }
+        assert_eq!(base, "https://api.riotpiao.com");
+    }
+
+    #[test]
+    fn test_parse_real_embedding_response() {
+        // Exact format returned by embeddings-predictor service
+        let raw = r#"{"object":"list","data":[{"object":"embedding","embedding":[0.1,0.2,0.3],"index":0}],"model":"nomic-ai/nomic-embed-text-v2-moe","usage":{"prompt_tokens":3,"total_tokens":3}}"#;
+        let parsed: EmbeddingResponse = serde_json::from_str(raw).expect("should parse");
+        match parsed {
+            EmbeddingResponse::Success { data, .. } => {
+                assert_eq!(data.len(), 1);
+                assert_eq!(data[0].embedding.len(), 3);
+                assert_eq!(data[0].index, 0);
+            }
+            EmbeddingResponse::Error { error } => panic!("parsed as error: {:?}", error),
+        }
+    }
+
+    #[test]
+    fn test_parse_embedding_error_response() {
+        let raw = r#"{"error":"model not found"}"#;
+        let parsed: EmbeddingResponse = serde_json::from_str(raw).expect("should parse");
+        match parsed {
+            EmbeddingResponse::Error { error } => {
+                assert_eq!(error.as_str().unwrap(), "model not found");
+            }
+            EmbeddingResponse::Success { .. } => panic!("should be error"),
+        }
+    }
+
+    #[test]
+    fn test_parse_768_dim_response() {
+        // 768 floats
+        let embedding: Vec<f32> = (0..768).map(|i| i as f32 * 0.001).collect();
+        let raw = format!(
+            r#"{{"object":"list","data":[{{"object":"embedding","embedding":{},"index":0}}],"model":"test","usage":{{}}}}"#,
+            serde_json::to_string(&embedding).unwrap()
+        );
+        let parsed: EmbeddingResponse = serde_json::from_str(&raw).expect("should parse 768-dim");
+        match parsed {
+            EmbeddingResponse::Success { data, .. } => {
+                assert_eq!(data[0].embedding.len(), 768);
+            }
+            _ => panic!("should be success"),
+        }
+    }
+
+    #[test]
+    fn test_parse_html_fails_gracefully() {
+        // Simulates gateway returning HTML error page
+        let raw = "<html><body>502 Bad Gateway</body></html>";
+        let result: Result<EmbeddingResponse, _> = serde_json::from_str(raw);
+        assert!(result.is_err(), "HTML should fail to parse as JSON");
+        let err_msg = result.unwrap_err().to_string();
+        assert!(err_msg.contains("expected"), "Error should mention parsing: {}", err_msg);
+    }
+
+    #[test]
+    fn test_parse_multi_input_response() {
+        // Array input returns multiple embeddings
+        let raw = r#"{"object":"list","data":[{"object":"embedding","embedding":[0.1,0.2,0.3],"index":0},{"object":"embedding","embedding":[0.4,0.5,0.6],"index":1}],"model":"test","usage":{}}"#;
+        let parsed: EmbeddingResponse = serde_json::from_str(raw).expect("should parse");
+        match parsed {
+            EmbeddingResponse::Success { data, .. } => {
+                assert_eq!(data.len(), 2);
+                assert_eq!(data[0].index, 0);
+                assert_eq!(data[1].index, 1);
+            }
+            _ => panic!("should be success"),
+        }
+    }
 }