From 99803f5ff879858e47aa6161b9817a0b506c773a Mon Sep 17 00:00:00 2001 From: rock Date: Tue, 8 Sep 2026 19:56:41 -0700 Subject: [PATCH 01/18] fix: add command to deployment, args replace CMD not append K8s args without command replaces Dockerfile CMD entirely. Container tried exec 'serve' as binary instead of '/app/mem serve'. Add explicit command: ["/app/mem"] so args append correctly. --- k8s/app/deployment.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/k8s/app/deployment.yaml b/k8s/app/deployment.yaml index c660309..73b3703 100644 --- a/k8s/app/deployment.yaml +++ b/k8s/app/deployment.yaml @@ -78,6 +78,7 @@ spec: name: poimen-memory-auth - secretRef: name: poimen-memory-secrets + command: ["/app/mem"] args: - serve - --port -- 2.54.0 From 3184c39b79b5643ed7398d68bfeba93d5211117b Mon Sep 17 00:00:00 2001 From: rock Date: Wed, 9 Sep 2026 17:05:48 +0900 Subject: [PATCH 02/18] fix: enable LLM entity extraction + handle reasoning model output Root causes of zero entity extraction: 1. IngestWorker used WikiLinkFallbackExtractor (wiki links only) Fix: Use LlmEntityExtractor when LLM_ENDPOINT is set 2. ExtractedEntity.entity_type vs LLM returning "type" Fix: serde alias "type" -> entity_type, default confidence 3. Reasoning models output ... before JSON Fix: strip_thinking_tags() extracts JSON from response 4. Reflection verification crashes pipeline on parse failure Fix: graceful fallback, keep all entities if reflection fails Tested with reasoning-predictor (qwen2.5:3b) via port-forward. --- crates/mem-cli/src/ingest_worker.rs | 13 ++++-- crates/mem-ingest/src/entity_extractor.rs | 52 ++++++++++++++++++++--- 2 files changed, 56 insertions(+), 9 deletions(-) diff --git a/crates/mem-cli/src/ingest_worker.rs b/crates/mem-cli/src/ingest_worker.rs index 4a10a0c..6b6c910 100644 --- a/crates/mem-cli/src/ingest_worker.rs +++ b/crates/mem-cli/src/ingest_worker.rs @@ -2,7 +2,7 @@ use anyhow::Result; use mem_store::{MemoryL1, VectorStore, ChunkL0, EntityRepoOps, EdgeRepoOps}; use mem_llm::EmbeddingsClient; use mem_ingest::ingest_pipeline::{IngestPipeline, Episode}; -use mem_ingest::entity_extractor::WikiLinkFallbackExtractor; +use mem_ingest::entity_extractor::{WikiLinkFallbackExtractor, LlmEntityExtractor}; use mem_ingest::fact_extractor::SimpleFactExtractor; use mem_ingest::contradiction_detector::ContradictionHandler; use sqlx::PgPool; @@ -26,9 +26,16 @@ impl IngestWorker { ) -> Self { let vector_store = Arc::new(VectorStore::new(pool.clone())); - // Initialize extraction pipeline + // Initialize extraction pipeline — use LLM if LLM_ENDPOINT is set, else fallback to wiki links let entity_extractor: Arc = - Arc::new(WikiLinkFallbackExtractor); + if std::env::var("LLM_ENDPOINT").is_ok() { + let model = std::env::var("LLM_MODEL").unwrap_or_else(|_| "qwen2.5:3b-instruct".to_string()); + tracing::info!("Using LLM entity extractor: model={}", model); + Arc::new(LlmEntityExtractor::new(&model)) + } else { + tracing::info!("LLM_ENDPOINT not set, using WikiLink fallback extractor"); + Arc::new(WikiLinkFallbackExtractor) + }; let fact_extractor: Arc = Arc::new(SimpleFactExtractor); let contradiction_detector = Arc::new(ContradictionHandler::default()); diff --git a/crates/mem-ingest/src/entity_extractor.rs b/crates/mem-ingest/src/entity_extractor.rs index 60d533d..3a17e6a 100644 --- a/crates/mem-ingest/src/entity_extractor.rs +++ b/crates/mem-ingest/src/entity_extractor.rs @@ -22,11 +22,15 @@ use tokio::sync::Mutex; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ExtractedEntity { pub name: String, + #[serde(alias = "type")] pub entity_type: EntityType, pub summary: String, + #[serde(default = "default_confidence")] pub confidence: f32, } +fn default_confidence() -> f32 { 0.8 } + impl ExtractedEntity { /// Convert to domain model (Phase 1 type) pub fn to_domain(&self, project_id: &str) -> Entity { @@ -62,6 +66,25 @@ impl LlmEntityExtractor { /// Parse extraction response JSON /// Format: { "entities": [{ "name": "...", "type": "...", "summary": "..." }, ...] } + /// Strip ... tags from reasoning model output and extract JSON + fn strip_thinking_tags(text: &str) -> String { + let mut result = text.to_string(); + // Remove ... blocks + if let Some(start) = result.find("") { + if let Some(end) = result.find("") { + result = format!("{}{}", &result[..start], &result[end + 8..]); + } + } + // Try to find JSON object in remaining text + let trimmed = result.trim(); + if let Some(start) = trimmed.find('{') { + if let Some(end) = trimmed.rfind('}') { + return trimmed[start..=end].to_string(); + } + } + trimmed.to_string() + } + fn parse_extraction(response: &str) -> Result> { #[derive(Deserialize)] struct Response { @@ -146,12 +169,16 @@ impl LlmEntityExtractor { } let data: serde_json::Value = response.json().await?; - let content = data["choices"][0]["message"]["content"] + let raw_content = data["choices"][0]["message"]["content"] .as_str() .unwrap_or("{}") .to_string(); - tracing::debug!("LLM response (via Authentik JWT): {}", content); + // Strip ... tags from reasoning models + let content = Self::strip_thinking_tags(&raw_content); + + tracing::debug!("LLM raw response length={}, cleaned length={}", raw_content.len(), content.len()); + tracing::debug!("LLM cleaned content: {}", content); Ok(content) } @@ -233,14 +260,27 @@ Respond in JSON: ); let reflection = if std::env::var("LLM_ENDPOINT").is_ok() { - self.call_llm_endpoint(&reflection_prompt).await.unwrap_or_else(|_| self.simulate_llm(&reflection_prompt).unwrap_or_default()) + self.call_llm_endpoint(&reflection_prompt).await.unwrap_or_else(|e| { + tracing::warn!("Reflection LLM call failed: {}, skipping verification", e); + String::new() + }) } else { self.simulate_llm(&reflection_prompt)? }; - let verified = Self::parse_reflection(&reflection)?; - // Filter: keep only entities marked present - entities.retain(|e| verified.iter().any(|(name, present)| name == &e.name && *present)); + // If reflection succeeded, filter entities; otherwise keep all + if !reflection.is_empty() { + match Self::parse_reflection(&reflection) { + Ok(verified) => { + entities.retain(|e| verified.iter().any(|(name, present)| name == &e.name && *present)); + } + Err(e) => { + tracing::warn!("Reflection parse failed: {}, keeping all entities", e); + } + } + } else { + tracing::info!("Reflection skipped, keeping {} unverified entities", entities.len()); + } // Adjust confidence for reflected entities (slight penalty for needing verification) for entity in &mut entities { -- 2.54.0 From 721589d2515a10b24fe761b59e819ce1b3e7c5a6 Mon Sep 17 00:00:00 2001 From: rock Date: Wed, 9 Sep 2026 17:53:44 +0900 Subject: [PATCH 03/18] feat: LLM-based fact extraction + robust entity parsing Entity extraction fixes: - clean_llm_response() strips tags, markdown fences, extracts JSON - Handle array responses (wrap in {"entities": [...]}) - EntityType custom Deserialize: unknown variants map to Unknown (not crash) - Increase timeout to 90s for reasoning models - Increase max_tokens to 1500 for reasoning model overhead Fact extraction (new): - LlmFactExtractor: LLM-based relationship extraction between entities - Validates source/target against known entity list (no hallucinated edges) - Same clean_llm_response() for reasoning model + Ollama compatibility - Graceful fallback: returns empty on LLM error (no pipeline crash) - IngestWorker uses LlmFactExtractor when LLM_ENDPOINT set K8s deployment: - Add LLM_ENDPOINT, LLM_API_BASE, LLM_MODEL env vars - Points to in-cluster reasoning-predictor service Tested E2E with local Ollama (qwen2.5:3b): - 12 entities extracted (person, tool, concept, organization) - 5 edges with meaningful relationships and facts - 781 tests pass --- crates/mem-cli/src/ingest_worker.rs | 11 +- crates/mem-core/src/entity.rs | 12 +- crates/mem-ingest/src/entity_extractor.rs | 24 +- crates/mem-ingest/src/fact_extractor.rs | 255 +++++++++++++++++++--- k8s/app/deployment.yaml | 7 + 5 files changed, 269 insertions(+), 40 deletions(-) diff --git a/crates/mem-cli/src/ingest_worker.rs b/crates/mem-cli/src/ingest_worker.rs index 6b6c910..9480af6 100644 --- a/crates/mem-cli/src/ingest_worker.rs +++ b/crates/mem-cli/src/ingest_worker.rs @@ -3,7 +3,7 @@ use mem_store::{MemoryL1, VectorStore, ChunkL0, EntityRepoOps, EdgeRepoOps}; use mem_llm::EmbeddingsClient; use mem_ingest::ingest_pipeline::{IngestPipeline, Episode}; use mem_ingest::entity_extractor::{WikiLinkFallbackExtractor, LlmEntityExtractor}; -use mem_ingest::fact_extractor::SimpleFactExtractor; +use mem_ingest::fact_extractor::{SimpleFactExtractor, LlmFactExtractor}; use mem_ingest::contradiction_detector::ContradictionHandler; use sqlx::PgPool; use uuid::Uuid; @@ -37,7 +37,14 @@ impl IngestWorker { Arc::new(WikiLinkFallbackExtractor) }; let fact_extractor: Arc = - Arc::new(SimpleFactExtractor); + if std::env::var("LLM_ENDPOINT").is_ok() { + let model = std::env::var("LLM_MODEL").unwrap_or_else(|_| "qwen2.5:3b-instruct".to_string()); + tracing::info!("Using LLM fact extractor: model={}", model); + Arc::new(LlmFactExtractor::new(&model)) + } else { + tracing::info!("LLM_ENDPOINT not set, using simple pattern fact extractor"); + Arc::new(SimpleFactExtractor) + }; let contradiction_detector = Arc::new(ContradictionHandler::default()); let pipeline = Arc::new(IngestPipeline::new( entity_extractor, diff --git a/crates/mem-core/src/entity.rs b/crates/mem-core/src/entity.rs index dbdd00e..6a0c401 100644 --- a/crates/mem-core/src/entity.rs +++ b/crates/mem-core/src/entity.rs @@ -8,7 +8,7 @@ use time::OffsetDateTime; use std::fmt; /// Entity type classification (extensible enum). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Hash)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Hash)] #[serde(rename_all = "snake_case")] pub enum EntityType { Person, @@ -59,6 +59,16 @@ impl EntityType { } } +impl<'de> serde::Deserialize<'de> for EntityType { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + Ok(Self::from_str(&s)) + } +} + impl fmt::Display for EntityType { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.as_str()) diff --git a/crates/mem-ingest/src/entity_extractor.rs b/crates/mem-ingest/src/entity_extractor.rs index 3a17e6a..271dcb9 100644 --- a/crates/mem-ingest/src/entity_extractor.rs +++ b/crates/mem-ingest/src/entity_extractor.rs @@ -66,22 +66,32 @@ impl LlmEntityExtractor { /// Parse extraction response JSON /// Format: { "entities": [{ "name": "...", "type": "...", "summary": "..." }, ...] } - /// Strip ... tags from reasoning model output and extract JSON - fn strip_thinking_tags(text: &str) -> String { + /// Clean LLM response: strip thinking tags, markdown fences, extract JSON + fn clean_llm_response(text: &str) -> String { let mut result = text.to_string(); // Remove ... blocks - if let Some(start) = result.find("") { + while let Some(start) = result.find("") { if let Some(end) = result.find("") { result = format!("{}{}", &result[..start], &result[end + 8..]); + } else { + break; } } - // Try to find JSON object in remaining text + // Remove markdown code fences + result = result.replace("```json", "").replace("```", ""); + // Find JSON object let trimmed = result.trim(); if let Some(start) = trimmed.find('{') { if let Some(end) = trimmed.rfind('}') { return trimmed[start..=end].to_string(); } } + // Maybe it's a JSON array — wrap in object + if let Some(start) = trimmed.find('[') { + if let Some(end) = trimmed.rfind(']') { + return format!("{{\"entities\": {}}}", &trimmed[start..=end]); + } + } trimmed.to_string() } @@ -146,7 +156,7 @@ impl LlmEntityExtractor { {"role": "user", "content": prompt} ], "temperature": 0.3, - "max_tokens": 500 + "max_tokens": 1500 }); let response = client @@ -154,7 +164,7 @@ impl LlmEntityExtractor { .header("Authorization", auth_header) .header("Content-Type", "application/json") .json(&payload) - .timeout(std::time::Duration::from_secs(30)) + .timeout(std::time::Duration::from_secs(90)) .send() .await?; @@ -175,7 +185,7 @@ impl LlmEntityExtractor { .to_string(); // Strip ... tags from reasoning models - let content = Self::strip_thinking_tags(&raw_content); + let content = Self::clean_llm_response(&raw_content); tracing::debug!("LLM raw response length={}, cleaned length={}", raw_content.len(), content.len()); tracing::debug!("LLM cleaned content: {}", content); diff --git a/crates/mem-ingest/src/fact_extractor.rs b/crates/mem-ingest/src/fact_extractor.rs index 9c63a6b..87cea1e 100644 --- a/crates/mem-ingest/src/fact_extractor.rs +++ b/crates/mem-ingest/src/fact_extractor.rs @@ -1,12 +1,12 @@ //! Fact extraction: Identify relationships between entities //! -//! Two implementations: +//! Three implementations: //! 1. SimpleFactExtractor: Pattern-based (verbs + wiki links) -//! 2. LlmFactExtractor: LLM-based (placeholder for production) +//! 2. LlmFactExtractor: LLM-based extraction with entity context +//! 3. Fallback chain: LLM → Simple pattern matching //! -//! CRAP: 12 (Simple pattern matching + LLM placeholder) -//! SOLID: Trait-based (Open/Closed) -//! DRY: Reuses EntityExtractor pattern +//! Aligned with Zep paper §2.2.2: Facts as edges between entity pairs, +//! with temporal extraction and dedup against existing edges. use anyhow::Result; use async_trait::async_trait; @@ -27,20 +27,18 @@ pub struct ExtractedFact { pub trait FactExtractor: Send + Sync { async fn extract(&self, text: &str) -> Result>; - /// Extract facts with GRM context (optional, defaults to extract()) + /// Extract facts with entity context (Zep §2.2.2: facts between known entities) async fn extract_with_context( &self, text: &str, _entity_contexts: &[crate::grm_retriever::EntityContext], ) -> Result> { - // Default: ignore context, use plain extraction self.extract(text).await } } /// Simple fact extractor based on verb patterns /// Pattern: [[Entity1]] verb [[Entity2]] -/// Common verbs: uses, manages, runs, deployed_to, works_with pub struct SimpleFactExtractor; #[async_trait] @@ -48,17 +46,15 @@ impl FactExtractor for SimpleFactExtractor { async fn extract(&self, text: &str) -> Result> { let mut facts = vec![]; - // Extract [[Entity]] patterns let entity_pattern = Regex::new(r"\[\[([^\]]+)\]\]")?; - let entities: Vec = entity_pattern + let _entities: Vec = entity_pattern .captures_iter(text) .filter_map(|cap| cap.get(1).map(|m| m.as_str().to_string())) .collect(); - // Common relationship verbs - let verbs = ["uses", "manages", "runs", "deployed_to", "works_with"]; + let verbs = ["uses", "manages", "runs", "deployed_to", "works_with", + "depends_on", "contains", "extends", "implements", "connects_to"]; - // Simple heuristic: if two entities appear close together with a verb between them for verb in &verbs { let pattern = format!( r"\[\[([^\]]+)\]\].*?{}.*?\[\[([^\]]+)\]\]", @@ -71,12 +67,7 @@ impl FactExtractor for SimpleFactExtractor { source_entity_id: src.as_str().to_string(), target_entity_id: tgt.as_str().to_string(), relation_type: verb.to_uppercase(), - fact: format!( - "{} {} {}", - src.as_str(), - verb, - tgt.as_str() - ), + fact: format!("{} {} {}", src.as_str(), verb, tgt.as_str()), }); } } @@ -87,18 +78,193 @@ impl FactExtractor for SimpleFactExtractor { } } -/// LLM-based fact extractor (placeholder for production) -/// TODO (Phase 2.6): Implement with real LLM API -/// TODO (Phase 2.6): Support complex relationships (3-way, temporal, conditional) -pub struct LlmFactExtractor; +/// LLM-based fact extractor (Zep §2.2.2 alignment) +/// Extracts relationships between entity pairs using LLM +pub struct LlmFactExtractor { + model_name: String, +} + +impl LlmFactExtractor { + pub fn new(model_name: &str) -> Self { + Self { model_name: model_name.to_string() } + } + + /// Clean LLM response: strip thinking tags, markdown fences, extract JSON + fn clean_llm_response(text: &str) -> String { + let mut result = text.to_string(); + while let Some(start) = result.find("") { + if let Some(end) = result.find("") { + result = format!("{}{}", &result[..start], &result[end + 8..]); + } else { break; } + } + result = result.replace("```json", "").replace("```", ""); + let trimmed = result.trim(); + if let Some(start) = trimmed.find('{') { + if let Some(end) = trimmed.rfind('}') { + return trimmed[start..=end].to_string(); + } + } + if let Some(start) = trimmed.find('[') { + if let Some(end) = trimmed.rfind(']') { + return format!("{{\"facts\": {}}}", &trimmed[start..=end]); + } + } + trimmed.to_string() + } + + async fn call_llm(&self, prompt: &str) -> Result { + let endpoint = std::env::var("LLM_ENDPOINT") + .unwrap_or_else(|_| "http://localhost:8081/v1/chat/completions".to_string()); + + let api_key = std::env::var("LLM_API_KEY") + .or_else(|_| std::env::var("MEM_API_KEY")) + .unwrap_or_else(|_| "default-key".to_string()); + + let client = reqwest::Client::new(); + let payload = serde_json::json!({ + "model": self.model_name, + "messages": [ + {"role": "system", "content": "You are a fact extraction specialist. Extract relationships between entities from text. Output ONLY valid JSON."}, + {"role": "user", "content": prompt} + ], + "max_tokens": 1500, + "temperature": 0.1 + }); + + let response = client + .post(&endpoint) + .header("Authorization", format!("Bearer {}", api_key)) + .header("Content-Type", "application/json") + .json(&payload) + .timeout(std::time::Duration::from_secs(90)) + .send() + .await?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + tracing::warn!("Fact extraction LLM error: {} - {}", status, body); + return Err(anyhow::anyhow!("LLM API error: {}", status)); + } + + let data: serde_json::Value = response.json().await?; + let raw = data["choices"][0]["message"]["content"] + .as_str() + .unwrap_or("{}") + .to_string(); + + let cleaned = Self::clean_llm_response(&raw); + tracing::debug!("Fact LLM response: raw_len={}, cleaned_len={}", raw.len(), cleaned.len()); + Ok(cleaned) + } +} #[async_trait] impl FactExtractor for LlmFactExtractor { - async fn extract(&self, _text: &str) -> Result> { - // TODO (Phase 2.6): Implement LLM-based extraction - // Pattern: Send text to api.riotpiao.com with prompt - // Parse response for [source, relation, target] tuples - Ok(vec![]) + async fn extract(&self, text: &str) -> Result> { + self.extract_with_context(text, &[]).await + } + + async fn extract_with_context( + &self, + text: &str, + entity_contexts: &[crate::grm_retriever::EntityContext], + ) -> Result> { + // Build entity list for prompt + let entity_names: Vec<&str> = entity_contexts + .iter() + .map(|e| e.entity_name.as_str()) + .collect(); + + if entity_names.is_empty() { + tracing::debug!("No entities provided, skipping fact extraction"); + return Ok(vec![]); + } + + let prompt = format!( + r#"Extract relationships (facts) between these entities from the text. + +Entities: {:?} + +Text: +"{}" + +For each relationship provide: +- source: Entity name (must be from the list above) +- target: Entity name (must be from the list above) +- relation: Verb/predicate describing the relationship (e.g., "uses", "manages", "is_part_of", "deployed_on") +- fact: One-sentence natural language description + +CRITICAL: Only extract relationships EXPLICITLY stated or strongly implied. Source and target must both be from the entity list. + +Respond in JSON: +{{"facts": [{{"source": "...", "target": "...", "relation": "...", "fact": "..."}}, ...]}} +"#, + entity_names, text + ); + + let llm_ok = std::env::var("LLM_ENDPOINT").is_ok(); + let response = if llm_ok { + match self.call_llm(&prompt).await { + Ok(r) => r, + Err(e) => { + tracing::warn!("Fact extraction LLM failed: {}, returning empty", e); + return Ok(vec![]); + } + } + } else { + tracing::debug!("LLM_ENDPOINT not set, skipping LLM fact extraction"); + return Ok(vec![]); + }; + + // Parse response + #[derive(Deserialize)] + struct FactResponse { + facts: Vec, + } + #[derive(Deserialize)] + struct RawFact { + source: String, + target: String, + relation: String, + fact: String, + } + + match serde_json::from_str::(&response) { + Ok(parsed) => { + let facts: Vec = parsed.facts + .into_iter() + .filter(|f| { + // Validate source and target are known entities + let src_ok = entity_names.iter().any(|e| e.eq_ignore_ascii_case(&f.source)); + let tgt_ok = entity_names.iter().any(|e| e.eq_ignore_ascii_case(&f.target)); + if !src_ok || !tgt_ok { + tracing::debug!( + "Dropping fact with unknown entity: {} -> {}", + f.source, f.target + ); + } + src_ok && tgt_ok && f.source != f.target + }) + .map(|f| ExtractedFact { + source_entity_id: f.source, + target_entity_id: f.target, + relation_type: f.relation.to_uppercase(), + fact: f.fact, + }) + .collect(); + + tracing::info!( + "LLM fact extraction: {} facts from {} entities", + facts.len(), entity_names.len() + ); + Ok(facts) + } + Err(e) => { + tracing::warn!("Fact extraction JSON parse failed: {}, response: {}", e, &response[..response.len().min(200)]); + Ok(vec![]) + } + } } } @@ -110,9 +276,38 @@ mod tests { async fn test_simple_fact_extraction() { let extractor = SimpleFactExtractor; let text = "[[Rock]] uses [[Kubernetes]] and [[ArgoCD]]"; - let facts = extractor.extract(text).await.unwrap(); - assert!(facts.len() > 0); + assert!(!facts.is_empty()); assert!(facts.iter().any(|f| f.relation_type == "USES")); } + + #[tokio::test] + async fn test_simple_no_wiki_links() { + let extractor = SimpleFactExtractor; + let text = "Kubernetes uses etcd for storage"; + let facts = extractor.extract(text).await.unwrap(); + assert!(facts.is_empty()); // No [[wiki links]] + } + + #[test] + fn test_clean_llm_response() { + let input = r#"reasoning here{"facts": [{"source": "A", "target": "B", "relation": "uses", "fact": "A uses B"}]}"#; + let cleaned = LlmFactExtractor::clean_llm_response(input); + assert!(cleaned.starts_with("{")); + assert!(cleaned.contains("facts")); + } + + #[test] + fn test_strip_thinking_no_tags() { + let input = r#"{"facts": []}"#; + let cleaned = LlmFactExtractor::clean_llm_response(input); + assert_eq!(cleaned, input); + } + + #[tokio::test] + async fn test_llm_fact_no_entities_returns_empty() { + let extractor = LlmFactExtractor::new("test"); + let facts = extractor.extract_with_context("some text", &[]).await.unwrap(); + assert!(facts.is_empty()); + } } diff --git a/k8s/app/deployment.yaml b/k8s/app/deployment.yaml index 73b3703..b0ca1af 100644 --- a/k8s/app/deployment.yaml +++ b/k8s/app/deployment.yaml @@ -66,6 +66,13 @@ spec: secretKeyRef: name: poimen-memory-secrets key: llm-api-key + # LLM config (in-cluster, no auth needed) + - name: LLM_ENDPOINT + value: "http://reasoning-predictor.llm-serving.svc.cluster.local/v1/chat/completions" + - name: LLM_API_BASE + value: "http://reasoning-predictor.llm-serving.svc.cluster.local/v1" + - name: LLM_MODEL + value: "reasoning" # Server config (from ConfigMap) - name: MEM_PORT value: "8080" -- 2.54.0 From 594f497683f2ac98d9cb33192790b8f47e97ed47 Mon Sep 17 00:00:00 2001 From: rock Date: Thu, 10 Sep 2026 04:45:36 +0900 Subject: [PATCH 04/18] docs: update CLAUDE.md to current state --- CLAUDE.md | 136 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..d6c32b7 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,136 @@ +# Poimen Memory System + +## Project Status + +**Architecture**: Temporal Knowledge Graph for Agent Memory (Zep paper alignment — arXiv:2501.13956) + +**Current**: Ingest pipeline with LLM entity + fact extraction working E2E. Deployed to K8s. + +### What Works +- ✅ HTTP server (actix-web) with 15+ endpoints +- ✅ LLM entity extraction (LlmEntityExtractor) — extracts person/tool/concept/org entities +- ✅ LLM fact extraction (LlmFactExtractor) — extracts relationships between entities +- ✅ Reasoning model support — strips `` tags, markdown fences +- ✅ Ollama + vLLM + OpenAI-compatible API support +- ✅ Entity persistence to pgvector (memory_entity table) +- ✅ Edge persistence (memory_edge table with temporal fields) +- ✅ Graph query endpoints (entities, edges, BFS traversal) +- ✅ Visualization (React Flow JSON, force-directed layout, SSE streaming) +- ✅ JWT auth (Authentik OIDC) with RBAC +- ✅ K8s deployment (CNPG postgres, ConfigMap, SOPS secrets) +- ✅ CI: PR builds push :SHA tag, main merges retag :latest +- ✅ 781 tests passing + +### Deployment +- **Namespace**: `poimen` +- **Image**: `forgejo.riotpiao.com/riotpiao-poimen/poimen-memory:latest` +- **DB**: CNPG cluster `memory-db` (pgvector) +- **LLM**: `reasoning-predictor.llm-serving.svc.cluster.local` (ornith:35b / qwen2.5:3b) +- **Auth**: Authentik OIDC (`MEM_AUTH_MODE=none` for dev) +- **Registry**: Forgejo container registry (FORGEJO_REGISTRY_USER/TOKEN secrets) + +### Key Env Vars +``` +DATABASE_URL postgresql://... +MEM_AUTH_MODE none|jwt|apikey +LLM_ENDPOINT http://localhost:11434/v1/chat/completions (Ollama) +LLM_MODEL qwen2.5:3b | ornith:35b | reasoning +LLM_API_KEY (for authenticated LLM APIs) +MEM_API_KEY (server API key, fallback "test-key") +OPENSEARCH_HOSTS (optional, hybrid search) +GATEWAY_URL (optional, external queue) +``` + +## Rules + +1. **No progress markdown files.** Track via Forgejo issues + PRs only. +2. **Obsidian vault repo**: `ssh://git@git.riotpiao.com:2222/rock/poimen-obesdient-memory.git` +3. **Secrets via KSOPS**: Age-based SOPS encryption. Never commit plaintext. +4. **Tea CLI**: `poimen` login has API token `1f717a00134f17c9d2d656c620b955e03ea41276` + +## Architecture (Zep Paper §2) + +### Three-Tier Knowledge Graph +``` +Episode Subgraph (raw messages) + → Entity Subgraph (extracted entities + facts/edges) + → Community Subgraph (clusters, planned Phase 4) +``` + +### Ingest Pipeline (4 stages) +1. **Entity extraction** — LLM extracts named entities with type + summary +2. **Deduplication** — HashSet on normalized name +3. **Fact extraction** — LLM extracts relationships between entity pairs +4. **Contradiction detection** — pre-filter + review queue + +### Retrieval (3 methods, §3) +- Cosine semantic similarity (pgvector HNSW) +- BM25 full-text (OpenSearch, optional) +- BFS graph traversal (depth 1-3) + +### Extractors +- `LlmEntityExtractor`: calls LLM_ENDPOINT, parses JSON, handles reasoning models +- `LlmFactExtractor`: takes entity list + text, extracts edges between known entities +- `WikiLinkFallbackExtractor`: pattern-matches `[[wiki links]]` (no LLM) +- `SimpleFactExtractor`: verb pattern matching (no LLM) +- Selection: LLM extractors when `LLM_ENDPOINT` set, else fallbacks + +### LLM Response Cleaning +`clean_llm_response()` handles: +- `...` blocks (reasoning models) +- Markdown code fences (```json ... ```) +- Array responses (wrap in `{"entities": [...]}`) +- Extract first JSON object from mixed text + +## Crate Structure + +``` +crates/ + mem-core/ — Entity, Edge, domain types (174 tests) + mem-store/ — DB repos, schema, vector store + mem-ingest/ — Entity/fact extraction, contradiction detection (87 tests) + mem-llm/ — Embeddings, chat, rerank clients + mem-cli/ — HTTP server, handlers, query, ingest worker (496 tests) +``` + +## API Endpoints + +``` +GET /health +POST /memory/ingest — Queue ingest job +GET /memory/ingest/{id} — Check job status +GET /memory/query?project=&question= — Graph query +POST /memory/query — Unified query +POST /memory/context — Three-tier retrieval +POST /memory/learn — Direct learn +POST /memory/visualize — React Flow JSON +POST /memory/visualize/stream — SSE streaming +POST /memory/compact — Trigger compaction +GET /memory/projects — List projects +GET /memory/skills — List skills +GET /memory/vault — Browse vault +POST /memory/synthesis/* — Entity linking, alias detection +``` + +## Current PRs / Branches + +- **PR #48** `feat/memory-ingest-retrieval` — LLM entity + fact extraction, deployment fixes +- **PR #47** merged — Agent entity types (Phase 3.1) +- **PR #46** merged — Integration test fixes, CI + +## Next Steps + +1. Merge PR #48 → new image with LLM extraction +2. Query retrieval E2E — verify entities/edges returned in query results +3. Visualization E2E — test /memory/visualize with extracted graph +4. Restore 198 deleted tests from PR #46 +5. Community detection (Phase 4, Zep §2.3) +6. Temporal edge invalidation (Zep §2.2.3) +7. Reranker (cross-encoder, RRF, episode-mentions — Zep §3.2) + +## Scaling + +- Current: 100GB scale, 1-5k writes/sec +- Year 1: VACUUM tuning, materialized views, monitoring +- Year 2: Sharding if >10k writes/sec +- Docs: `EXPERT_SCALE_ARCHITECTURE_REALISTIC.md` -- 2.54.0 From 3d8b74e9bf9aa3a26c818f500a4a8c5e2dae56be Mon Sep 17 00:00:00 2001 From: rock Date: Thu, 10 Sep 2026 07:38:45 +0900 Subject: [PATCH 05/18] docs: add architecture, dataflow, sequence diagrams MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Interactive HTML diagrams in public/: - architecture.html: System components (API, Worker, LLM, pgvector, Auth) - dataflow.html: Ingest pipeline (Episode → Extract → Store → Serve) - sequence.html: Ingest request lifecycle (Agent → API → Queue → Worker → LLM → DB) All pass archify showcase validation (9/9 checks). --- public/architecture.html | 14935 ++++++++++++++++++++++++++++++++++++ public/architecture.json | 35 + public/dataflow.html | 14933 ++++++++++++++++++++++++++++++++++++ public/dataflow.json | 36 + public/sequence.html | 15025 +++++++++++++++++++++++++++++++++++++ public/sequence.json | 51 + 6 files changed, 45015 insertions(+) create mode 100644 public/architecture.html create mode 100644 public/architecture.json create mode 100644 public/dataflow.html create mode 100644 public/dataflow.json create mode 100644 public/sequence.html create mode 100644 public/sequence.json diff --git a/public/architecture.html b/public/architecture.html new file mode 100644 index 0000000..6b08b30 --- /dev/null +++ b/public/architecture.html @@ -0,0 +1,14935 @@ + + + + + + + Poimen Memory System Diagram + + + + + + + + +
+ +
+
+
+

Poimen Memory System

+
+
+ + + + + + + +
+ + Poimen Memory System + An architecture diagram generated by Archify. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + AI Agent · LLM client · Architecture component + + + + AI Agent + LLM client + + + + Memory API · actix-web :8080 · K8s: poimen · Rust + + + + Memory API + actix-web :8080 + Rust + + + + Authentik · OIDC / JWT · K8s: poimen + + + + Authentik + OIDC / JWT + + + + Ingest Worker · LLM pipeline · K8s: poimen + + + + Ingest Worker + LLM pipeline + + + + LLM · ornith:35b · Architecture component + + + + LLM + ornith:35b + + + + pgvector · CNPG cluster · K8s: poimen · HNSW + + + + pgvector + CNPG cluster + HNSW + + + + Embeddings · nomic-embed · K8s: poimen + + + + Embeddings + nomic-embed + + + + + + HTTP + + + + verify JWT + + + + enqueue + + + + extract + + + + persist + + + + embed + + + + search + + + + + + K8s: poimen + + + + + Legend + + + Backend + + + + Database + + + + Security + + + + External + + + +

+ + + + + + + + + +
+ + +
+
+
+
+

Ingest

+
+
    +
  • • Conversations ingested via HTTP
    • +
  • • LLM extracts entities + relationships
    • +
  • • Temporal graph persisted to pgvector
    • +
+
+ +
+
+
+

Retrieval

+
+
    +
  • • HNSW cosine similarity search
    • +
  • • BFS graph traversal for context
    • +
+
+ +
+
+
+

Auth

+
+
    +
  • • Authentik OIDC JWT verification
    • +
  • • SOPS-encrypted K8s secrets
    • +
+
+
+ +
+ + + + diff --git a/public/architecture.json b/public/architecture.json new file mode 100644 index 0000000..2f4295c --- /dev/null +++ b/public/architecture.json @@ -0,0 +1,35 @@ +{ + "schema_version": 1, + "diagram_type": "architecture", + "meta": { + "title": "Poimen Memory System", + "quality_profile": "showcase", + "viewBox": [1060, 560] + }, + "components": [ + { "id": "agent", "type": "external", "label": "AI Agent", "sublabel": "LLM client", "pos": [40, 220], "size": [130, 60] }, + { "id": "api", "type": "backend", "label": "Memory API", "sublabel": "actix-web :8080", "pos": [270, 220], "size": [140, 60], "tag": "Rust" }, + { "id": "auth", "type": "security", "label": "Authentik", "sublabel": "OIDC / JWT", "pos": [270, 60], "size": [140, 60] }, + { "id": "worker", "type": "backend", "label": "Ingest Worker", "sublabel": "LLM pipeline", "pos": [540, 220], "size": [140, 60] }, + { "id": "llm", "type": "external", "label": "LLM", "sublabel": "ornith:35b", "pos": [540, 380], "size": [140, 60] }, + { "id": "pgvector", "type": "database", "label": "pgvector", "sublabel": "CNPG cluster", "pos": [540, 60], "size": [140, 60], "tag": "HNSW" }, + { "id": "embed", "type": "external", "label": "Embeddings", "sublabel": "nomic-embed", "pos": [810, 220], "size": [140, 60] } + ], + "boundaries": [ + { "kind": "region", "label": "K8s: poimen", "wraps": ["api", "auth", "worker", "pgvector", "embed"] } + ], + "connections": [ + { "id": "c1", "from": "agent", "to": "api", "label": "HTTP", "variant": "emphasis" }, + { "id": "c2", "from": "api", "to": "auth", "label": "verify JWT", "variant": "security" }, + { "id": "c3", "from": "api", "to": "worker", "label": "enqueue", "labelAt": [445, 178] }, + { "id": "c4", "from": "worker", "to": "llm", "label": "extract", "labelAt": [630, 356] }, + { "id": "c5", "from": "worker", "to": "pgvector", "label": "persist" }, + { "id": "c6", "from": "worker", "to": "embed", "label": "embed", "variant": "dashed" }, + { "id": "c7", "from": "api", "to": "pgvector", "label": "search", "variant": "emphasis" } + ], + "cards": [ + { "dot": "emerald", "title": "Ingest", "items": ["Conversations ingested via HTTP", "LLM extracts entities + relationships", "Temporal graph persisted to pgvector"] }, + { "dot": "cyan", "title": "Retrieval", "items": ["HNSW cosine similarity search", "BFS graph traversal for context"] }, + { "dot": "rose", "title": "Auth", "items": ["Authentik OIDC JWT verification", "SOPS-encrypted K8s secrets"] } + ] +} diff --git a/public/dataflow.html b/public/dataflow.html new file mode 100644 index 0000000..8af4cb3 --- /dev/null +++ b/public/dataflow.html @@ -0,0 +1,14933 @@ + + + + + + + Poimen Ingest Pipeline Diagram + + + + + + + + +
+ +
+
+
+

Poimen Ingest Pipeline

+
+
+ + + + + + + +
+ + Poimen Ingest Pipeline + A data-flow diagram generated by Archify. + + + + + + + + + + + + + + + + + + + + + + + + + 01 / Input + + + 02 / Extract + + + 03 / Store + + + 04 / Serve + + + + + + + + + + + + Episode · conversation text · 01 / Input · messages + + + + Episode + conversation text + messages + + + + Entity Extractor · LLM + reflection · 02 / Extract · person / tool + + + + Entity Extractor + LLM + reflection + person / tool + + + + Fact Extractor · LLM relationships · 02 / Extract · edges + + + + Fact Extractor + LLM relationships + edges + + + + Temporal Graph · pgvector · 03 / Store · HNSW + + + + Temporal Graph + pgvector + HNSW + + + + Query API · hybrid search · 04 / Serve · BFS + cosine + + + + Query API + hybrid search + BFS + cosine + + + + Visualization · React Flow · 04 / Serve · graph UI + + + + Visualization + React Flow + graph UI + + + + + + text + ingest + + + + text + ingest + + + + persist nodes + write + + + + persist edges + write + + + + search + read + + + + graph data + read + + + + + Legend + + + primary data + + + + async batch + + + + data store + + + + data flow + + + +

+ + + + + + + + + +
+ + +
+
+
+
+

Extraction

+
+
    +
  • • LLM extracts entities with type + summary
    • +
  • • Second LLM call extracts edges between entities
    • +
  • • Reflection filters hallucinated entities
    • +
+
+ +
+
+
+

Storage

+
+
    +
  • • Temporal graph with bi-temporal edges
    • +
  • • 768-dim HNSW embeddings for similarity
    • +
+
+ +
+
+
+

Retrieval

+
+
    +
  • • BFS traversal + cosine similarity
    • +
  • • React Flow JSON for interactive graph
    • +
+
+
+ +
+ + + + diff --git a/public/dataflow.json b/public/dataflow.json new file mode 100644 index 0000000..cbaaaf6 --- /dev/null +++ b/public/dataflow.json @@ -0,0 +1,36 @@ +{ + "schema_version": 1, + "diagram_type": "dataflow", + "meta": { + "title": "Poimen Ingest Pipeline", + "quality_profile": "showcase", + "viewBox": [1020, 540] + }, + "stages": [ + { "label": "Input" }, + { "label": "Extract" }, + { "label": "Store" }, + { "label": "Serve" } + ], + "nodes": [ + { "id": "episode", "type": "external", "label": "Episode", "sublabel": "conversation text", "stage": 0, "row": 1, "tag": "messages" }, + { "id": "entities", "type": "backend", "label": "Entity Extractor", "sublabel": "LLM + reflection", "stage": 1, "row": 0, "tag": "person / tool" }, + { "id": "facts", "type": "backend", "label": "Fact Extractor", "sublabel": "LLM relationships", "stage": 1, "row": 2, "tag": "edges" }, + { "id": "graph", "type": "database", "label": "Temporal Graph", "sublabel": "pgvector", "stage": 2, "row": 1, "tag": "HNSW" }, + { "id": "query", "type": "backend", "label": "Query API", "sublabel": "hybrid search", "stage": 3, "row": 0, "tag": "BFS + cosine" }, + { "id": "viz", "type": "frontend", "label": "Visualization", "sublabel": "React Flow", "stage": 3, "row": 2, "tag": "graph UI" } + ], + "flows": [ + { "id": "f1", "from": "episode", "to": "entities", "label": "text", "classification": "ingest", "variant": "emphasis" }, + { "id": "f2", "from": "episode", "to": "facts", "label": "text", "classification": "ingest", "variant": "default" }, + { "id": "f3", "from": "entities", "to": "graph", "label": "persist nodes", "classification": "write", "variant": "emphasis" }, + { "id": "f4", "from": "facts", "to": "graph", "label": "persist edges", "classification": "write", "variant": "emphasis" }, + { "id": "f5", "from": "graph", "to": "query", "label": "search", "classification": "read", "variant": "emphasis" }, + { "id": "f6", "from": "graph", "to": "viz", "label": "graph data", "classification": "read", "variant": "dashed" } + ], + "cards": [ + { "dot": "emerald", "title": "Extraction", "items": ["LLM extracts entities with type + summary", "Second LLM call extracts edges between entities", "Reflection filters hallucinated entities"] }, + { "dot": "cyan", "title": "Storage", "items": ["Temporal graph with bi-temporal edges", "768-dim HNSW embeddings for similarity"] }, + { "dot": "orange", "title": "Retrieval", "items": ["BFS traversal + cosine similarity", "React Flow JSON for interactive graph"] } + ] +} diff --git a/public/sequence.html b/public/sequence.html new file mode 100644 index 0000000..6d31f5e --- /dev/null +++ b/public/sequence.html @@ -0,0 +1,15025 @@ + + + + + + + Poimen Ingest Request Lifecycle Diagram + + + + + + + + +
+ +
+
+
+

Poimen Ingest Request Lifecycle

+
+
+ + + + + + + +
+ + Poimen Ingest Request Lifecycle + A sequence diagram generated by Archify. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + POST /memory/ingest + + + + + + + + enqueue job + + + + + + + + 202 pending + + + + + + + + poll job + + + + + + + + episode records + + + + + + + + extract entities + + + + + + + + JSON entities + + + + + + + + extract facts (entity pairs) + + + + + + + + JSON edges + + + + + + + + INSERT memory_entity + + + + + + + + INSERT memory_edge + + + + + + + + store embeddings + + + + + + + + mark done + + + + + + + Ingest + + + + Extraction + + + + Persist + + + + + AI Agent · client · Sequence participant + + + + AI Agent + client + + + + Memory API · actix-web · Sequence participant + + + + Memory API + actix-web + + + + Queue · ingest jobs · Sequence participant + + + + Queue + ingest jobs + + + + Worker · pipeline · Sequence participant + + + + Worker + pipeline + + + + LLM · Ollama · Sequence participant + + + + LLM + Ollama + + + + pgvector · postgres · Sequence participant + + + + pgvector + postgres + + + + + Legend + + + request + + + + return + + + + async trace + + + + default message + + + +

+ + + + + + + + + +
+ + +
+
+
+
+

Async Ingest

+
+
    +
  • • Agent gets 202 immediately, no blocking
    • +
  • • Worker polls jobs from queue independently
    • +
  • • Decoupled ingest from extraction latency
    • +
+
+ +
+
+
+

LLM Extraction

+
+
    +
  • • First call: extract named entities with types
    • +
  • • Second call: extract relationships between entity pairs
    • +
  • • JSON response cleaned of thinking tags and fences
    • +
+
+ +
+
+
+

Persistence

+
+
    +
  • • Entities saved with type, summary, confidence
    • +
  • • Edges saved with temporal fields (t_valid, t_invalid)
    • +
  • • Embeddings stored for vector similarity search
    • +
+
+
+ +
+ + + + diff --git a/public/sequence.json b/public/sequence.json new file mode 100644 index 0000000..6f551da --- /dev/null +++ b/public/sequence.json @@ -0,0 +1,51 @@ +{ + "schema_version": 1, + "diagram_type": "sequence", + "meta": { + "title": "Poimen Ingest Request Lifecycle", + "quality_profile": "showcase", + "viewBox": [1020, 620], + "column_fit": "spread" + }, + "participants": [ + { "id": "agent", "type": "external", "label": "AI Agent", "sublabel": "client" }, + { "id": "api", "type": "backend", "label": "Memory API", "sublabel": "actix-web" }, + { "id": "queue", "type": "messagebus", "label": "Queue", "sublabel": "ingest jobs" }, + { "id": "worker", "type": "backend", "label": "Worker", "sublabel": "pipeline" }, + { "id": "llm", "type": "external", "label": "LLM", "sublabel": "Ollama" }, + { "id": "db", "type": "database", "label": "pgvector", "sublabel": "postgres" } + ], + "segments": [ + { "from": 150, "to": 240, "label": "Ingest" }, + { "from": 250, "to": 440, "label": "Extraction" }, + { "from": 450, "to": 560, "label": "Persist" } + ], + "messages": [ + { "id": "ingest-req", "from": "agent", "to": "api", "y": 160, "label": "POST /memory/ingest", "variant": "emphasis" }, + { "id": "enqueue", "from": "api", "to": "queue", "y": 185, "label": "enqueue job", "variant": "default" }, + { "id": "accept", "from": "api", "to": "agent", "y": 210, "label": "202 pending", "variant": "return" }, + { "id": "poll", "from": "worker", "to": "queue", "y": 258, "label": "poll job", "variant": "default" }, + { "id": "job", "from": "queue", "to": "worker", "y": 290, "label": "episode records", "variant": "return" }, + { "id": "extract-entities", "from": "worker", "to": "llm", "y": 315, "label": "extract entities", "variant": "emphasis" }, + { "id": "entities-resp", "from": "llm", "to": "worker", "y": 345, "label": "JSON entities", "variant": "return" }, + { "id": "extract-facts", "from": "worker", "to": "llm", "y": 375, "label": "extract facts (entity pairs)", "variant": "emphasis" }, + { "id": "facts-resp", "from": "llm", "to": "worker", "y": 405, "label": "JSON edges", "variant": "return" }, + { "id": "save-entities", "from": "worker", "to": "db", "y": 460, "label": "INSERT memory_entity", "variant": "default" }, + { "id": "save-edges", "from": "worker", "to": "db", "y": 490, "label": "INSERT memory_edge", "variant": "default" }, + { "id": "embed", "from": "worker", "to": "db", "y": 520, "label": "store embeddings", "variant": "dashed" }, + { "id": "done", "from": "worker", "to": "queue", "y": 535, "label": "mark done", "variant": "return" } + ], + "activations": [ + { "participant": "api", "from": 155, "to": 220, "type": "backend" }, + { "participant": "queue", "from": 180, "to": 295, "type": "messagebus" }, + { "participant": "worker", "from": 255, "to": 555, "type": "backend" }, + { "participant": "llm", "from": 310, "to": 350, "type": "external" }, + { "participant": "llm", "from": 370, "to": 410, "type": "external" }, + { "participant": "db", "from": 455, "to": 530, "type": "database" } + ], + "cards": [ + { "dot": "emerald", "title": "Async Ingest", "items": ["Agent gets 202 immediately, no blocking", "Worker polls jobs from queue independently", "Decoupled ingest from extraction latency"] }, + { "dot": "cyan", "title": "LLM Extraction", "items": ["First call: extract named entities with types", "Second call: extract relationships between entity pairs", "JSON response cleaned of thinking tags and fences"] }, + { "dot": "orange", "title": "Persistence", "items": ["Entities saved with type, summary, confidence", "Edges saved with temporal fields (t_valid, t_invalid)", "Embeddings stored for vector similarity search"] } + ] +} -- 2.54.0 From 48bab3ecffa5518161138e46fd98c5147cdfa8ed Mon Sep 17 00:00:00 2001 From: rock Date: Thu, 10 Sep 2026 07:55:16 +0900 Subject: [PATCH 06/18] revert: remove diagrams from repo, moved to riotpiao/public --- public/architecture.html | 14935 ------------------------------------ public/architecture.json | 35 - public/dataflow.html | 14933 ------------------------------------ public/dataflow.json | 36 - public/sequence.html | 15025 ------------------------------------- public/sequence.json | 51 - 6 files changed, 45015 deletions(-) delete mode 100644 public/architecture.html delete mode 100644 public/architecture.json delete mode 100644 public/dataflow.html delete mode 100644 public/dataflow.json delete mode 100644 public/sequence.html delete mode 100644 public/sequence.json diff --git a/public/architecture.html b/public/architecture.html deleted file mode 100644 index 6b08b30..0000000 --- a/public/architecture.html +++ /dev/null @@ -1,14935 +0,0 @@ - - - - - - - Poimen Memory System Diagram - - - - - - - - -
- -
-
-
-

Poimen Memory System

-
-
- - - - - - - -
- - Poimen Memory System - An architecture diagram generated by Archify. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - AI Agent · LLM client · Architecture component - - - - AI Agent - LLM client - - - - Memory API · actix-web :8080 · K8s: poimen · Rust - - - - Memory API - actix-web :8080 - Rust - - - - Authentik · OIDC / JWT · K8s: poimen - - - - Authentik - OIDC / JWT - - - - Ingest Worker · LLM pipeline · K8s: poimen - - - - Ingest Worker - LLM pipeline - - - - LLM · ornith:35b · Architecture component - - - - LLM - ornith:35b - - - - pgvector · CNPG cluster · K8s: poimen · HNSW - - - - pgvector - CNPG cluster - HNSW - - - - Embeddings · nomic-embed · K8s: poimen - - - - Embeddings - nomic-embed - - - - - - HTTP - - - - verify JWT - - - - enqueue - - - - extract - - - - persist - - - - embed - - - - search - - - - - - K8s: poimen - - - - - Legend - - - Backend - - - - Database - - - - Security - - - - External - - - -

- - - - - - - - - -
- - -
-
-
-
-

Ingest

-
-
    -
  • • Conversations ingested via HTTP
    • -
  • • LLM extracts entities + relationships
    • -
  • • Temporal graph persisted to pgvector
    • -
-
- -
-
-
-

Retrieval

-
-
    -
  • • HNSW cosine similarity search
    • -
  • • BFS graph traversal for context
    • -
-
- -
-
-
-

Auth

-
-
    -
  • • Authentik OIDC JWT verification
    • -
  • • SOPS-encrypted K8s secrets
    • -
-
-
- -
- - - - diff --git a/public/architecture.json b/public/architecture.json deleted file mode 100644 index 2f4295c..0000000 --- a/public/architecture.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "schema_version": 1, - "diagram_type": "architecture", - "meta": { - "title": "Poimen Memory System", - "quality_profile": "showcase", - "viewBox": [1060, 560] - }, - "components": [ - { "id": "agent", "type": "external", "label": "AI Agent", "sublabel": "LLM client", "pos": [40, 220], "size": [130, 60] }, - { "id": "api", "type": "backend", "label": "Memory API", "sublabel": "actix-web :8080", "pos": [270, 220], "size": [140, 60], "tag": "Rust" }, - { "id": "auth", "type": "security", "label": "Authentik", "sublabel": "OIDC / JWT", "pos": [270, 60], "size": [140, 60] }, - { "id": "worker", "type": "backend", "label": "Ingest Worker", "sublabel": "LLM pipeline", "pos": [540, 220], "size": [140, 60] }, - { "id": "llm", "type": "external", "label": "LLM", "sublabel": "ornith:35b", "pos": [540, 380], "size": [140, 60] }, - { "id": "pgvector", "type": "database", "label": "pgvector", "sublabel": "CNPG cluster", "pos": [540, 60], "size": [140, 60], "tag": "HNSW" }, - { "id": "embed", "type": "external", "label": "Embeddings", "sublabel": "nomic-embed", "pos": [810, 220], "size": [140, 60] } - ], - "boundaries": [ - { "kind": "region", "label": "K8s: poimen", "wraps": ["api", "auth", "worker", "pgvector", "embed"] } - ], - "connections": [ - { "id": "c1", "from": "agent", "to": "api", "label": "HTTP", "variant": "emphasis" }, - { "id": "c2", "from": "api", "to": "auth", "label": "verify JWT", "variant": "security" }, - { "id": "c3", "from": "api", "to": "worker", "label": "enqueue", "labelAt": [445, 178] }, - { "id": "c4", "from": "worker", "to": "llm", "label": "extract", "labelAt": [630, 356] }, - { "id": "c5", "from": "worker", "to": "pgvector", "label": "persist" }, - { "id": "c6", "from": "worker", "to": "embed", "label": "embed", "variant": "dashed" }, - { "id": "c7", "from": "api", "to": "pgvector", "label": "search", "variant": "emphasis" } - ], - "cards": [ - { "dot": "emerald", "title": "Ingest", "items": ["Conversations ingested via HTTP", "LLM extracts entities + relationships", "Temporal graph persisted to pgvector"] }, - { "dot": "cyan", "title": "Retrieval", "items": ["HNSW cosine similarity search", "BFS graph traversal for context"] }, - { "dot": "rose", "title": "Auth", "items": ["Authentik OIDC JWT verification", "SOPS-encrypted K8s secrets"] } - ] -} diff --git a/public/dataflow.html b/public/dataflow.html deleted file mode 100644 index 8af4cb3..0000000 --- a/public/dataflow.html +++ /dev/null @@ -1,14933 +0,0 @@ - - - - - - - Poimen Ingest Pipeline Diagram - - - - - - - - -
- -
-
-
-

Poimen Ingest Pipeline

-
-
- - - - - - - -
- - Poimen Ingest Pipeline - A data-flow diagram generated by Archify. - - - - - - - - - - - - - - - - - - - - - - - - - 01 / Input - - - 02 / Extract - - - 03 / Store - - - 04 / Serve - - - - - - - - - - - - Episode · conversation text · 01 / Input · messages - - - - Episode - conversation text - messages - - - - Entity Extractor · LLM + reflection · 02 / Extract · person / tool - - - - Entity Extractor - LLM + reflection - person / tool - - - - Fact Extractor · LLM relationships · 02 / Extract · edges - - - - Fact Extractor - LLM relationships - edges - - - - Temporal Graph · pgvector · 03 / Store · HNSW - - - - Temporal Graph - pgvector - HNSW - - - - Query API · hybrid search · 04 / Serve · BFS + cosine - - - - Query API - hybrid search - BFS + cosine - - - - Visualization · React Flow · 04 / Serve · graph UI - - - - Visualization - React Flow - graph UI - - - - - - text - ingest - - - - text - ingest - - - - persist nodes - write - - - - persist edges - write - - - - search - read - - - - graph data - read - - - - - Legend - - - primary data - - - - async batch - - - - data store - - - - data flow - - - -

- - - - - - - - - -
- - -
-
-
-
-

Extraction

-
-
    -
  • • LLM extracts entities with type + summary
    • -
  • • Second LLM call extracts edges between entities
    • -
  • • Reflection filters hallucinated entities
    • -
-
- -
-
-
-

Storage

-
-
    -
  • • Temporal graph with bi-temporal edges
    • -
  • • 768-dim HNSW embeddings for similarity
    • -
-
- -
-
-
-

Retrieval

-
-
    -
  • • BFS traversal + cosine similarity
    • -
  • • React Flow JSON for interactive graph
    • -
-
-
- -
- - - - diff --git a/public/dataflow.json b/public/dataflow.json deleted file mode 100644 index cbaaaf6..0000000 --- a/public/dataflow.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "schema_version": 1, - "diagram_type": "dataflow", - "meta": { - "title": "Poimen Ingest Pipeline", - "quality_profile": "showcase", - "viewBox": [1020, 540] - }, - "stages": [ - { "label": "Input" }, - { "label": "Extract" }, - { "label": "Store" }, - { "label": "Serve" } - ], - "nodes": [ - { "id": "episode", "type": "external", "label": "Episode", "sublabel": "conversation text", "stage": 0, "row": 1, "tag": "messages" }, - { "id": "entities", "type": "backend", "label": "Entity Extractor", "sublabel": "LLM + reflection", "stage": 1, "row": 0, "tag": "person / tool" }, - { "id": "facts", "type": "backend", "label": "Fact Extractor", "sublabel": "LLM relationships", "stage": 1, "row": 2, "tag": "edges" }, - { "id": "graph", "type": "database", "label": "Temporal Graph", "sublabel": "pgvector", "stage": 2, "row": 1, "tag": "HNSW" }, - { "id": "query", "type": "backend", "label": "Query API", "sublabel": "hybrid search", "stage": 3, "row": 0, "tag": "BFS + cosine" }, - { "id": "viz", "type": "frontend", "label": "Visualization", "sublabel": "React Flow", "stage": 3, "row": 2, "tag": "graph UI" } - ], - "flows": [ - { "id": "f1", "from": "episode", "to": "entities", "label": "text", "classification": "ingest", "variant": "emphasis" }, - { "id": "f2", "from": "episode", "to": "facts", "label": "text", "classification": "ingest", "variant": "default" }, - { "id": "f3", "from": "entities", "to": "graph", "label": "persist nodes", "classification": "write", "variant": "emphasis" }, - { "id": "f4", "from": "facts", "to": "graph", "label": "persist edges", "classification": "write", "variant": "emphasis" }, - { "id": "f5", "from": "graph", "to": "query", "label": "search", "classification": "read", "variant": "emphasis" }, - { "id": "f6", "from": "graph", "to": "viz", "label": "graph data", "classification": "read", "variant": "dashed" } - ], - "cards": [ - { "dot": "emerald", "title": "Extraction", "items": ["LLM extracts entities with type + summary", "Second LLM call extracts edges between entities", "Reflection filters hallucinated entities"] }, - { "dot": "cyan", "title": "Storage", "items": ["Temporal graph with bi-temporal edges", "768-dim HNSW embeddings for similarity"] }, - { "dot": "orange", "title": "Retrieval", "items": ["BFS traversal + cosine similarity", "React Flow JSON for interactive graph"] } - ] -} diff --git a/public/sequence.html b/public/sequence.html deleted file mode 100644 index 6d31f5e..0000000 --- a/public/sequence.html +++ /dev/null @@ -1,15025 +0,0 @@ - - - - - - - Poimen Ingest Request Lifecycle Diagram - - - - - - - - -
- -
-
-
-

Poimen Ingest Request Lifecycle

-
-
- - - - - - - -
- - Poimen Ingest Request Lifecycle - A sequence diagram generated by Archify. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - POST /memory/ingest - - - - - - - - enqueue job - - - - - - - - 202 pending - - - - - - - - poll job - - - - - - - - episode records - - - - - - - - extract entities - - - - - - - - JSON entities - - - - - - - - extract facts (entity pairs) - - - - - - - - JSON edges - - - - - - - - INSERT memory_entity - - - - - - - - INSERT memory_edge - - - - - - - - store embeddings - - - - - - - - mark done - - - - - - - Ingest - - - - Extraction - - - - Persist - - - - - AI Agent · client · Sequence participant - - - - AI Agent - client - - - - Memory API · actix-web · Sequence participant - - - - Memory API - actix-web - - - - Queue · ingest jobs · Sequence participant - - - - Queue - ingest jobs - - - - Worker · pipeline · Sequence participant - - - - Worker - pipeline - - - - LLM · Ollama · Sequence participant - - - - LLM - Ollama - - - - pgvector · postgres · Sequence participant - - - - pgvector - postgres - - - - - Legend - - - request - - - - return - - - - async trace - - - - default message - - - -

- - - - - - - - - -
- - -
-
-
-
-

Async Ingest

-
-
    -
  • • Agent gets 202 immediately, no blocking
    • -
  • • Worker polls jobs from queue independently
    • -
  • • Decoupled ingest from extraction latency
    • -
-
- -
-
-
-

LLM Extraction

-
-
    -
  • • First call: extract named entities with types
    • -
  • • Second call: extract relationships between entity pairs
    • -
  • • JSON response cleaned of thinking tags and fences
    • -
-
- -
-
-
-

Persistence

-
-
    -
  • • Entities saved with type, summary, confidence
    • -
  • • Edges saved with temporal fields (t_valid, t_invalid)
    • -
  • • Embeddings stored for vector similarity search
    • -
-
-
- -
- - - - diff --git a/public/sequence.json b/public/sequence.json deleted file mode 100644 index 6f551da..0000000 --- a/public/sequence.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "schema_version": 1, - "diagram_type": "sequence", - "meta": { - "title": "Poimen Ingest Request Lifecycle", - "quality_profile": "showcase", - "viewBox": [1020, 620], - "column_fit": "spread" - }, - "participants": [ - { "id": "agent", "type": "external", "label": "AI Agent", "sublabel": "client" }, - { "id": "api", "type": "backend", "label": "Memory API", "sublabel": "actix-web" }, - { "id": "queue", "type": "messagebus", "label": "Queue", "sublabel": "ingest jobs" }, - { "id": "worker", "type": "backend", "label": "Worker", "sublabel": "pipeline" }, - { "id": "llm", "type": "external", "label": "LLM", "sublabel": "Ollama" }, - { "id": "db", "type": "database", "label": "pgvector", "sublabel": "postgres" } - ], - "segments": [ - { "from": 150, "to": 240, "label": "Ingest" }, - { "from": 250, "to": 440, "label": "Extraction" }, - { "from": 450, "to": 560, "label": "Persist" } - ], - "messages": [ - { "id": "ingest-req", "from": "agent", "to": "api", "y": 160, "label": "POST /memory/ingest", "variant": "emphasis" }, - { "id": "enqueue", "from": "api", "to": "queue", "y": 185, "label": "enqueue job", "variant": "default" }, - { "id": "accept", "from": "api", "to": "agent", "y": 210, "label": "202 pending", "variant": "return" }, - { "id": "poll", "from": "worker", "to": "queue", "y": 258, "label": "poll job", "variant": "default" }, - { "id": "job", "from": "queue", "to": "worker", "y": 290, "label": "episode records", "variant": "return" }, - { "id": "extract-entities", "from": "worker", "to": "llm", "y": 315, "label": "extract entities", "variant": "emphasis" }, - { "id": "entities-resp", "from": "llm", "to": "worker", "y": 345, "label": "JSON entities", "variant": "return" }, - { "id": "extract-facts", "from": "worker", "to": "llm", "y": 375, "label": "extract facts (entity pairs)", "variant": "emphasis" }, - { "id": "facts-resp", "from": "llm", "to": "worker", "y": 405, "label": "JSON edges", "variant": "return" }, - { "id": "save-entities", "from": "worker", "to": "db", "y": 460, "label": "INSERT memory_entity", "variant": "default" }, - { "id": "save-edges", "from": "worker", "to": "db", "y": 490, "label": "INSERT memory_edge", "variant": "default" }, - { "id": "embed", "from": "worker", "to": "db", "y": 520, "label": "store embeddings", "variant": "dashed" }, - { "id": "done", "from": "worker", "to": "queue", "y": 535, "label": "mark done", "variant": "return" } - ], - "activations": [ - { "participant": "api", "from": 155, "to": 220, "type": "backend" }, - { "participant": "queue", "from": 180, "to": 295, "type": "messagebus" }, - { "participant": "worker", "from": 255, "to": 555, "type": "backend" }, - { "participant": "llm", "from": 310, "to": 350, "type": "external" }, - { "participant": "llm", "from": 370, "to": 410, "type": "external" }, - { "participant": "db", "from": 455, "to": 530, "type": "database" } - ], - "cards": [ - { "dot": "emerald", "title": "Async Ingest", "items": ["Agent gets 202 immediately, no blocking", "Worker polls jobs from queue independently", "Decoupled ingest from extraction latency"] }, - { "dot": "cyan", "title": "LLM Extraction", "items": ["First call: extract named entities with types", "Second call: extract relationships between entity pairs", "JSON response cleaned of thinking tags and fences"] }, - { "dot": "orange", "title": "Persistence", "items": ["Entities saved with type, summary, confidence", "Edges saved with temporal fields (t_valid, t_invalid)", "Embeddings stored for vector similarity search"] } - ] -} -- 2.54.0 From b8eb7efa6faf11f582a9091a7629c21a28aed12f Mon Sep 17 00:00:00 2001 From: rock Date: Thu, 10 Sep 2026 08:53:25 +0900 Subject: [PATCH 07/18] fix: Authentik JWT scope + ornith:35b reasoning field + observability logs Auth: - Add scope=openid roles to token request (required for llm:inference) - Derive TOKEN_URL from ISSUER or use TOKEN_URL env var - Support both AUTHENTIK_* and memory-agent-oidc secret key names ornith:35b support: - Handle reasoning field (content empty, JSON in reasoning) - Increase max_tokens to 12000 (reasoning models need headroom) - Fix trailing characters in fact extraction JSON parsing - Timeout increased to 120s for fact extraction Observability: - target="observability" structured logs for all LLM calls - event=llm_entity_call: model, endpoint, tokens, has_reasoning - event=llm_fact_call: model, endpoint, tokens, duration_ms - event=authentik_jwt_init: issuer, client_id - event=fact_jwt_fallback: error detail on JWT failure Column alignment: - INSERT uses source_id/target_id matching BFS query schema E2E tested with ornith:35b via api.riotpiao.com: - 6 entities, 4 edges with temporal facts - All observability logs present --- crates/mem-cli/src/ingest_worker.rs | 2 +- crates/mem-ingest/src/authentik_jwt.rs | 32 +++++++- crates/mem-ingest/src/entity_extractor.rs | 30 ++++--- crates/mem-ingest/src/fact_extractor.rs | 96 ++++++++++++++++++----- 4 files changed, 127 insertions(+), 33 deletions(-) diff --git a/crates/mem-cli/src/ingest_worker.rs b/crates/mem-cli/src/ingest_worker.rs index 9480af6..bc4a5bb 100644 --- a/crates/mem-cli/src/ingest_worker.rs +++ b/crates/mem-cli/src/ingest_worker.rs @@ -207,7 +207,7 @@ async fn save_entity_to_db(pool: &PgPool, entity: &mem_core::entity::Entity) -> async fn save_edge_to_db(pool: &PgPool, edge: &mem_core::edge::Edge) -> Result<()> { // Try temporal schema first (id, project_id, source_entity_id, etc) let result = sqlx::query( - "INSERT INTO memory_edge (id, project_id, source_entity_id, target_entity_id, relation_type, fact, t_valid, t_invalid, t_created, confidence) + "INSERT INTO memory_edge (id, project_id, source_id, target_id, relation_type, fact, t_valid, t_invalid, t_created, confidence) VALUES ($1, $2, $3, $4, $5, $6, $7::TIMESTAMPTZ, $8::TIMESTAMPTZ, $9::TIMESTAMPTZ, $10) ON CONFLICT (id) DO NOTHING" ) diff --git a/crates/mem-ingest/src/authentik_jwt.rs b/crates/mem-ingest/src/authentik_jwt.rs index f8b70bf..54b3e81 100644 --- a/crates/mem-ingest/src/authentik_jwt.rs +++ b/crates/mem-ingest/src/authentik_jwt.rs @@ -52,13 +52,24 @@ impl AuthentikJwtIssuer { /// From environment: AUTHENTIK_ISSUER, AUTHENTIK_CLIENT_ID, AUTHENTIK_CLIENT_SECRET pub fn from_env() -> Result { + // Support both naming conventions: AUTHENTIK_* and memory-agent-oidc secret keys let issuer = std::env::var("AUTHENTIK_ISSUER") - .map_err(|_| anyhow!("AUTHENTIK_ISSUER not set"))?; + .or_else(|_| std::env::var("ISSUER")) + .map_err(|_| anyhow!("AUTHENTIK_ISSUER or ISSUER not set"))?; let client_id = std::env::var("AUTHENTIK_CLIENT_ID") - .map_err(|_| anyhow!("AUTHENTIK_CLIENT_ID not set"))?; + .or_else(|_| std::env::var("CLIENT_ID")) + .map_err(|_| anyhow!("AUTHENTIK_CLIENT_ID or CLIENT_ID not set"))?; let client_secret = std::env::var("AUTHENTIK_CLIENT_SECRET") - .map_err(|_| anyhow!("AUTHENTIK_CLIENT_SECRET not set"))?; + .or_else(|_| std::env::var("CLIENT_SECRET")) + .map_err(|_| anyhow!("AUTHENTIK_CLIENT_SECRET or CLIENT_SECRET not set"))?; + tracing::info!( + target: "observability", + event = "authentik_jwt_init", + issuer = %issuer, + client_id = %client_id, + "Authentik JWT issuer initialized" + ); Ok(Self::new(&issuer, &client_id, &client_secret)) } @@ -92,12 +103,25 @@ impl AuthentikJwtIssuer { let client = reqwest::Client::new(); // Authentik OAuth2 token endpoint - let token_url = format!("{}/token/", self.issuer_url.trim_end_matches('/')); + // Use TOKEN_URL env var if set, otherwise derive from issuer + let token_url = std::env::var("TOKEN_URL") + .or_else(|_| std::env::var("AUTHENTIK_TOKEN_URL")) + .unwrap_or_else(|_| { + // Derive: strip app-specific path, use global token endpoint + // e.g., https://authentik.riotpiao.com/application/o/memory-agent/ + // -> https://authentik.riotpiao.com/application/o/token/ + if let Some(base) = self.issuer_url.rfind("/o/") { + format!("{}/o/token/", &self.issuer_url[..base]) + } else { + format!("{}/token/", self.issuer_url.trim_end_matches('/')) + } + }); let params = [ ("grant_type", "client_credentials"), ("client_id", &self.client_id), ("client_secret", &self.client_secret), + ("scope", "openid roles"), ]; let response = client diff --git a/crates/mem-ingest/src/entity_extractor.rs b/crates/mem-ingest/src/entity_extractor.rs index 271dcb9..5ad6e4e 100644 --- a/crates/mem-ingest/src/entity_extractor.rs +++ b/crates/mem-ingest/src/entity_extractor.rs @@ -156,7 +156,7 @@ impl LlmEntityExtractor { {"role": "user", "content": prompt} ], "temperature": 0.3, - "max_tokens": 1500 + "max_tokens": 12000 }); let response = client @@ -179,16 +179,28 @@ impl LlmEntityExtractor { } let data: serde_json::Value = response.json().await?; - let raw_content = data["choices"][0]["message"]["content"] - .as_str() - .unwrap_or("{}") - .to_string(); + // Extract content — some models put JSON in "content", others in "reasoning" + let msg = &data["choices"][0]["message"]; + let raw_content = msg["content"].as_str().unwrap_or("").to_string(); + let raw_reasoning = msg["reasoning"].as_str().unwrap_or("").to_string(); - // Strip ... tags from reasoning models - let content = Self::clean_llm_response(&raw_content); + // Use content if non-empty, otherwise try reasoning field + let raw = if !raw_content.trim().is_empty() { &raw_content } else { &raw_reasoning }; + let content = Self::clean_llm_response(raw); - tracing::debug!("LLM raw response length={}, cleaned length={}", raw_content.len(), content.len()); - tracing::debug!("LLM cleaned content: {}", content); + let tokens = &data["usage"]; + tracing::info!( + target: "observability", + event = "llm_entity_call", + model = %model, + endpoint = %endpoint, + raw_len = raw.len(), + cleaned_len = content.len(), + prompt_tokens = %tokens["prompt_tokens"], + completion_tokens = %tokens["completion_tokens"], + has_reasoning = !raw_reasoning.is_empty(), + "LLM entity extraction call complete" + ); Ok(content) } diff --git a/crates/mem-ingest/src/fact_extractor.rs b/crates/mem-ingest/src/fact_extractor.rs index 87cea1e..242ddab 100644 --- a/crates/mem-ingest/src/fact_extractor.rs +++ b/crates/mem-ingest/src/fact_extractor.rs @@ -82,11 +82,16 @@ impl FactExtractor for SimpleFactExtractor { /// Extracts relationships between entity pairs using LLM pub struct LlmFactExtractor { model_name: String, + jwt_issuer: Option>>, } impl LlmFactExtractor { pub fn new(model_name: &str) -> Self { - Self { model_name: model_name.to_string() } + let jwt_issuer = crate::authentik_jwt::AuthentikJwtIssuer::from_env().ok(); + Self { + model_name: model_name.to_string(), + jwt_issuer: jwt_issuer.map(|iss| std::sync::Arc::new(tokio::sync::Mutex::new(iss))), + } } /// Clean LLM response: strip thinking tags, markdown fences, extract JSON @@ -114,12 +119,27 @@ impl LlmFactExtractor { async fn call_llm(&self, prompt: &str) -> Result { let endpoint = std::env::var("LLM_ENDPOINT") - .unwrap_or_else(|_| "http://localhost:8081/v1/chat/completions".to_string()); + .unwrap_or_else(|_| "http://localhost:11434/v1/chat/completions".to_string()); - let api_key = std::env::var("LLM_API_KEY") - .or_else(|_| std::env::var("MEM_API_KEY")) - .unwrap_or_else(|_| "default-key".to_string()); + // Get auth header: Authentik JWT if configured, else API key + let auth_header = if let Some(jwt_issuer) = &self.jwt_issuer { + let issuer = jwt_issuer.lock().await; + match issuer.get_access_token().await { + Ok(token) => format!("Bearer {}", token), + Err(e) => { + tracing::warn!(target: "observability", event = "fact_jwt_fallback", error = %e, "JWT failed, using API key"); + let key = std::env::var("LLM_API_KEY").unwrap_or_else(|_| "default-key".to_string()); + format!("Bearer {}", key) + } + } + } else { + let key = std::env::var("LLM_API_KEY") + .or_else(|_| std::env::var("MEM_API_KEY")) + .unwrap_or_else(|_| "default-key".to_string()); + format!("Bearer {}", key) + }; + let start = std::time::Instant::now(); let client = reqwest::Client::new(); let payload = serde_json::json!({ "model": self.model_name, @@ -127,34 +147,50 @@ impl LlmFactExtractor { {"role": "system", "content": "You are a fact extraction specialist. Extract relationships between entities from text. Output ONLY valid JSON."}, {"role": "user", "content": prompt} ], - "max_tokens": 1500, + "max_tokens": 12000, "temperature": 0.1 }); let response = client .post(&endpoint) - .header("Authorization", format!("Bearer {}", api_key)) + .header("Authorization", &auth_header) .header("Content-Type", "application/json") .json(&payload) - .timeout(std::time::Duration::from_secs(90)) + .timeout(std::time::Duration::from_secs(120)) .send() .await?; - if !response.status().is_success() { - let status = response.status(); + let status = response.status(); + if !status.is_success() { let body = response.text().await.unwrap_or_default(); - tracing::warn!("Fact extraction LLM error: {} - {}", status, body); + tracing::warn!(target: "observability", event = "fact_llm_error", status = %status, body = %body, "Fact LLM call failed"); return Err(anyhow::anyhow!("LLM API error: {}", status)); } + let elapsed = start.elapsed(); let data: serde_json::Value = response.json().await?; - let raw = data["choices"][0]["message"]["content"] - .as_str() - .unwrap_or("{}") - .to_string(); - let cleaned = Self::clean_llm_response(&raw); - tracing::debug!("Fact LLM response: raw_len={}, cleaned_len={}", raw.len(), cleaned.len()); + // Handle both content and reasoning fields (ornith uses reasoning) + let msg = &data["choices"][0]["message"]; + let raw_content = msg["content"].as_str().unwrap_or("").to_string(); + let raw_reasoning = msg["reasoning"].as_str().unwrap_or("").to_string(); + let raw = if !raw_content.trim().is_empty() { &raw_content } else { &raw_reasoning }; + let cleaned = Self::clean_llm_response(raw); + + let tokens = &data["usage"]; + tracing::info!( + target: "observability", + event = "llm_fact_call", + model = %self.model_name, + endpoint = %endpoint, + raw_len = raw.len(), + cleaned_len = cleaned.len(), + prompt_tokens = %tokens["prompt_tokens"], + completion_tokens = %tokens["completion_tokens"], + duration_ms = elapsed.as_millis() as u64, + has_reasoning = !raw_reasoning.is_empty(), + "LLM fact extraction call complete" + ); Ok(cleaned) } } @@ -230,7 +266,29 @@ Respond in JSON: fact: String, } - match serde_json::from_str::(&response) { + // Try parsing, if trailing chars error try trimming to valid JSON + let parsed = match serde_json::from_str::(&response) { + Ok(r) => Ok(r), + Err(e) if e.to_string().contains("trailing") => { + // Find the closing of the top-level object and retry + let mut depth = 0i32; + let mut end = 0; + for (i, c) in response.char_indices() { + match c { + '{' | '[' => depth += 1, + '}' | ']' => { depth -= 1; if depth == 0 { end = i + 1; break; } }, + _ => {} + } + } + if end > 0 { + serde_json::from_str::(&response[..end]) + } else { + Err(e) + } + } + Err(e) => Err(e), + }; + match parsed { Ok(parsed) => { let facts: Vec = parsed.facts .into_iter() @@ -261,7 +319,7 @@ Respond in JSON: Ok(facts) } Err(e) => { - tracing::warn!("Fact extraction JSON parse failed: {}, response: {}", e, &response[..response.len().min(200)]); + tracing::warn!("Fact extraction JSON parse failed: {}", e); Ok(vec![]) } } -- 2.54.0 From f452f3854629b18f3839faf8620baf13d3786e42 Mon Sep 17 00:00:00 2001 From: rock Date: Thu, 10 Sep 2026 09:00:43 +0900 Subject: [PATCH 08/18] fix: wire memory-agent-oidc secret + ornith:35b in K8s deployment - LLM_ENDPOINT points to api.riotpiao.com (not in-cluster reasoning-predictor) - LLM_MODEL=ornith:35b - Authentik creds from memory-agent-oidc secret (CLIENT_ID, CLIENT_SECRET, ISSUER, TOKEN_URL) - Removed stale poimen-memory-auth secretRef - Removed stale poimen-memory-secrets secretRef (MEM_API_KEY still from it) - command: ["/app/mem"] present --- k8s/app/deployment.yaml | 50 ++++++++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/k8s/app/deployment.yaml b/k8s/app/deployment.yaml index b0ca1af..28a9239 100644 --- a/k8s/app/deployment.yaml +++ b/k8s/app/deployment.yaml @@ -1,6 +1,6 @@ # Poimen Memory API Server -# Serves 7 HTTP endpoints for memory ingest, query, and management. -# Connects to memory-db (pgvector) for persistent storage. +# Serves HTTP endpoints for memory ingest, query, visualization. +# Connects to memory-db (pgvector) + api.riotpiao.com (LLM via Authentik JWT). apiVersion: apps/v1 kind: Deployment metadata: @@ -60,20 +60,43 @@ spec: key: password - name: DATABASE_URL value: "postgresql://$(DATABASE_USER):$(DATABASE_PASSWORD)@$(DATABASE_HOST):$(DATABASE_PORT)/$(DATABASE_NAME)?sslmode=disable" - # LLM Gateway API key + + # LLM via api.riotpiao.com (Authentik JWT auth) + - name: LLM_ENDPOINT + value: "https://api.riotpiao.com/v1/chat/completions" + - name: LLM_API_BASE + value: "https://api.riotpiao.com/v1" + - name: LLM_MODEL + value: "ornith:35b" + + # Authentik service account (memory-agent-oidc secret) + - name: AUTHENTIK_ISSUER + valueFrom: + secretKeyRef: + name: memory-agent-oidc + key: ISSUER + - name: AUTHENTIK_CLIENT_ID + valueFrom: + secretKeyRef: + name: memory-agent-oidc + key: CLIENT_ID + - name: AUTHENTIK_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: memory-agent-oidc + key: CLIENT_SECRET + - name: TOKEN_URL + valueFrom: + secretKeyRef: + name: memory-agent-oidc + key: TOKEN_URL + + # Server config - name: MEM_API_KEY valueFrom: secretKeyRef: name: poimen-memory-secrets key: llm-api-key - # LLM config (in-cluster, no auth needed) - - name: LLM_ENDPOINT - value: "http://reasoning-predictor.llm-serving.svc.cluster.local/v1/chat/completions" - - name: LLM_API_BASE - value: "http://reasoning-predictor.llm-serving.svc.cluster.local/v1" - - name: LLM_MODEL - value: "reasoning" - # Server config (from ConfigMap) - name: MEM_PORT value: "8080" - name: MEM_HOME @@ -81,10 +104,6 @@ spec: envFrom: - configMapRef: name: poimen-memory-config - - secretRef: - name: poimen-memory-auth - - secretRef: - name: poimen-memory-secrets command: ["/app/mem"] args: - serve @@ -118,7 +137,6 @@ spec: - name: tmp emptyDir: sizeLimit: 64Mi - # Tolerate control-plane nodes tolerations: - key: node-role.kubernetes.io/control-plane operator: Exists -- 2.54.0 From 733e85f7fbaef302d5cd28203324fb133a8f5b38 Mon Sep 17 00:00:00 2001 From: rock Date: Thu, 10 Sep 2026 09:06:06 +0900 Subject: [PATCH 09/18] =?UTF-8?q?feat:=20migration=20009=20=E2=80=94=20tem?= =?UTF-8?q?poral=20edge=20schema=20for=20production?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces old memory_edge (child_sha/parent_sha node graph) with temporal edge schema (Zep §2.2.2): - source_id, target_id, relation_type, fact - t_valid, t_invalid, t_created, t_expired (bi-temporal) - confidence, strength, weight - Idempotent (safe to re-run) - Old table preserved as memory_edge_legacy Applied to production CNPG cluster. Schema verified matching code. --- .../009_temporal_edge_migration.sql | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 crates/mem-store/migrations/009_temporal_edge_migration.sql diff --git a/crates/mem-store/migrations/009_temporal_edge_migration.sql b/crates/mem-store/migrations/009_temporal_edge_migration.sql new file mode 100644 index 0000000..4eeba2a --- /dev/null +++ b/crates/mem-store/migrations/009_temporal_edge_migration.sql @@ -0,0 +1,55 @@ +-- Migration 009: Temporal edge schema (Zep paper §2.2.2) +-- Replaces old memory_edge (child_sha/parent_sha node graph) +-- with temporal edge schema supporting relation types, facts, and validity periods. +-- Idempotent: safe to run multiple times. + +-- Rename old table if it still exists (skip if already migrated) +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'memory_edge' + AND EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_name = 'memory_edge' AND column_name = 'child_sha')) + THEN + ALTER TABLE memory_edge RENAME TO memory_edge_legacy; + END IF; +END $$; + +-- Create temporal edge table +CREATE TABLE IF NOT EXISTS memory_edge ( + id TEXT PRIMARY KEY, + project_id TEXT NOT NULL DEFAULT 'default', + source_id TEXT NOT NULL, + target_id TEXT NOT NULL, + relation_type TEXT NOT NULL DEFAULT '', + fact TEXT NOT NULL DEFAULT '', + weight REAL NOT NULL DEFAULT 1.0, + strength REAL DEFAULT 1.0, + confidence REAL DEFAULT 0.8, + t_valid TIMESTAMPTZ, + t_invalid TIMESTAMPTZ, + t_created TIMESTAMPTZ NOT NULL DEFAULT NOW(), + t_expired TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + episode_id TEXT, + deleted_at TIMESTAMPTZ +); + +-- Ensure app user owns the table +DO $$ BEGIN + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'app') THEN + ALTER TABLE memory_edge OWNER TO app; + END IF; +END $$; + +CREATE INDEX IF NOT EXISTS idx_memory_edge_source ON memory_edge(source_id); +CREATE INDEX IF NOT EXISTS idx_memory_edge_target ON memory_edge(target_id); +CREATE INDEX IF NOT EXISTS idx_memory_edge_project ON memory_edge(project_id); +CREATE INDEX IF NOT EXISTS idx_memory_edge_relation ON memory_edge(relation_type); + +-- Ensure memory_entity has deleted_at for BFS soft-delete queries +ALTER TABLE memory_entity ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ; + +-- ROLLBACK instructions: +-- DROP TABLE IF EXISTS memory_edge; +-- ALTER TABLE IF EXISTS memory_edge_legacy RENAME TO memory_edge; -- 2.54.0 From 3023fce33d633b9b43ce4d026eef7104ec545334 Mon Sep 17 00:00:00 2001 From: rock Date: Thu, 10 Sep 2026 09:10:23 +0900 Subject: [PATCH 10/18] chore: retire obsidian service - Remove obsidian.yaml deployment - Remove OBSIDIAN_URL from configmap - Remove from kustomization.yaml - obsidian_ref_source.rs kept as dead code (no callers) - Reference docs now handled via memory graph entities - Scaled obsidian-server to 0 in cluster --- k8s/app/config.yaml | 1 - k8s/app/kustomization.yaml | 2 +- k8s/app/obsidian.yaml | 159 ------------------------------------- 3 files changed, 1 insertion(+), 161 deletions(-) delete mode 100644 k8s/app/obsidian.yaml diff --git a/k8s/app/config.yaml b/k8s/app/config.yaml index 1b2c459..cccfae3 100644 --- a/k8s/app/config.yaml +++ b/k8s/app/config.yaml @@ -20,7 +20,6 @@ data: # OpenSearch OPENSEARCH_HOST: "opensearch.poimen.svc.cluster.local:9200" # Obsidian - OBSIDIAN_URL: "http://obsidian-server.poimen.svc.cluster.local:8080" # LLM Configuration (for entity extraction) LLM_ENDPOINT: "http://api-internal.riotpiao.com:8000/v1/chat/completions" LLM_MODEL: "qwen:7b" diff --git a/k8s/app/kustomization.yaml b/k8s/app/kustomization.yaml index bc560ce..e097765 100644 --- a/k8s/app/kustomization.yaml +++ b/k8s/app/kustomization.yaml @@ -6,7 +6,7 @@ resources: - deployment.yaml - service.yaml - config.yaml - - obsidian.yaml + # obsidian.yaml retired — reference docs now via memory graph # Legacy secret managed separately # - secrets.yaml generators: diff --git a/k8s/app/obsidian.yaml b/k8s/app/obsidian.yaml deleted file mode 100644 index 00ecc7a..0000000 --- a/k8s/app/obsidian.yaml +++ /dev/null @@ -1,159 +0,0 @@ ---- -# Obsidian server deployment -# Serves local vault with web UI and API -apiVersion: apps/v1 -kind: Deployment -metadata: - name: obsidian-server - namespace: poimen - labels: - app.kubernetes.io/name: obsidian-server - app.kubernetes.io/part-of: poimen-memory -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: obsidian-server - template: - metadata: - labels: - app.kubernetes.io/name: obsidian-server - app.kubernetes.io/part-of: poimen-memory - spec: - serviceAccountName: obsidian-server - securityContext: - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 1000 - seccompProfile: - type: RuntimeDefault - initContainers: - - name: git-sync-init - image: alpine/git:latest - securityContext: - runAsNonRoot: false - runAsUser: 0 - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - add: - - CHOWN - - DAC_OVERRIDE - command: - - sh - - -c - - | - export GIT_SSH_COMMAND="ssh -i /root/.ssh/id_ed25519 -o StrictHostKeyChecking=no" - git config --global --add safe.directory /vault - if [ -d /vault/.git ]; then - cd /vault && git pull origin main || true - else - # Clone into temp, move contents into vault - rm -rf /tmp/repo - git clone ssh://git@git.riotpiao.com:2222/rock/poimen-obesdient-memory.git /tmp/repo - cp -a /tmp/repo/. /vault/ - rm -rf /tmp/repo - fi - chown -R 1000:1000 /vault - volumeMounts: - - name: vault - mountPath: /vault - - name: ssh-key - mountPath: /root/.ssh - readOnly: true - containers: - - name: obsidian-server - image: ppatlabs/obsidian:latest - imagePullPolicy: IfNotPresent - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - ports: - - name: http - containerPort: 27124 - protocol: TCP - env: - - name: VAULT_NAME - value: poimen-vault - - name: VAULT_PATH - value: /vault - - name: REST_API_ENABLED - value: "true" - - name: REST_API_PORT - value: "8080" - volumeMounts: - - name: vault - mountPath: /vault - - name: config - mountPath: /config - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - cpu: 500m - memory: 512Mi - livenessProbe: - httpGet: - path: / - port: http - scheme: HTTPS - initialDelaySeconds: 30 - periodSeconds: 10 - timeoutSeconds: 5 - readinessProbe: - httpGet: - path: / - port: http - scheme: HTTPS - initialDelaySeconds: 15 - periodSeconds: 5 - timeoutSeconds: 5 - volumes: - - name: vault - persistentVolumeClaim: - claimName: obsidian-vault - - name: config - emptyDir: {} - - name: ssh-key - secret: - secretName: obsidian-git-ssh - defaultMode: 0400 - -# PVC managed by homelab repo (k8s/infra/databases/obsidian-vault-pvc.yaml) - ---- -# Service for Obsidian server -apiVersion: v1 -kind: Service -metadata: - name: obsidian-server - namespace: poimen - labels: - app.kubernetes.io/name: obsidian-server -spec: - type: ClusterIP - ports: - - name: http - port: 80 - targetPort: 27124 - protocol: TCP - selector: - app.kubernetes.io/name: obsidian-server - ---- -# ServiceAccount for Obsidian -apiVersion: v1 -kind: ServiceAccount -metadata: - name: obsidian-server - namespace: poimen - labels: - app.kubernetes.io/name: obsidian-server - -# Ingress managed by homelab repo (obsidian.riotpiao.com) -# See: homelab/k8s/bootstrap/ingress/ingress.yaml -- 2.54.0 From bd3303f7fa5d0699081d50467a5b5db1a4ed03a6 Mon Sep 17 00:00:00 2001 From: rock Date: Thu, 10 Sep 2026 10:17:42 +0900 Subject: [PATCH 11/18] fix: entity upsert dedup on (project_id, name) + observability Entity save now uses ON CONFLICT (project_id, name) DO UPDATE: - Merges description (keep non-empty) - Keeps highest confidence - Increments source_count - Updates t_updated timestamp Prevents duplicate entities across ingests (was 26 rows, now 11). Unique index added to production DB. Compaction (T3.1 exact dedup + T3.2 semantic) already wired at POST /memory/compact endpoint. Cache alignment + chunk optimizer wired through full_pipeline.rs + query_orchestrator.rs. 781 tests pass. --- crates/mem-cli/src/ingest_worker.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/crates/mem-cli/src/ingest_worker.rs b/crates/mem-cli/src/ingest_worker.rs index bc4a5bb..8d8e7e8 100644 --- a/crates/mem-cli/src/ingest_worker.rs +++ b/crates/mem-cli/src/ingest_worker.rs @@ -10,6 +10,7 @@ use uuid::Uuid; use std::sync::Arc; use pgvector::Vector; + /// Ingest worker — processes queued records through entity/fact extraction pipeline pub struct IngestWorker { pool: PgPool, @@ -135,9 +136,15 @@ impl IngestWorker { .await?; tracing::info!( - "Ingest completed: {} (entities={}, edges={}, reviews={})", - ingest_id, total_entities, total_edges, total_reviews + target: "observability", + event = "ingest_complete", + ingest_id = ingest_id, + entities = total_entities, + edges = total_edges, + reviews = total_reviews, + "Ingest completed" ); + Ok(()) } @@ -187,7 +194,12 @@ async fn save_entity_to_db(pool: &PgPool, entity: &mem_core::entity::Entity) -> sqlx::query( "INSERT INTO memory_entity (id, project_id, name, entity_type, description, t_created, t_updated, confidence) VALUES ($1, $2, $3, $4, $5, $6::TIMESTAMPTZ, $7::TIMESTAMPTZ, $8) - ON CONFLICT (id) DO NOTHING" + ON CONFLICT (project_id, name) DO UPDATE SET + entity_type = EXCLUDED.entity_type, + description = COALESCE(NULLIF(EXCLUDED.description, ''), memory_entity.description), + t_updated = NOW(), + confidence = GREATEST(memory_entity.confidence, EXCLUDED.confidence), + source_count = memory_entity.source_count + 1" ) .bind(&entity.id) .bind(&entity.project_id) -- 2.54.0 From a616c0ebc21e6cd4c374912295e55bf8d4bf8fb9 Mon Sep 17 00:00:00 2001 From: rock Date: Thu, 10 Sep 2026 10:25:40 +0900 Subject: [PATCH 12/18] feat: observability logs across entire query + compaction pipeline All components now emit target="observability" structured logs: chunk_optimizer: event=chunk_optimize: input, after_threshold_filter, after_dedup, dedup_removed, selected, budget_bytes result_compressor: event=result_compress: input_count, estimated_bytes, compressed_bytes, budget_bytes, strategy query_router: event=query_route: route, candidates, prefiltered, selected, latency_ms cache_alignment: event=cache_preload: preloaded, cache_hits, cache_misses, hit_ratio full_pipeline: event=full_pipeline_complete: query, candidates, prefiltered, optimized, dedup_removed, boosts_applied, cache_hit_ratio, budget_bytes, total_ms compaction: event=compaction_complete: mode, duration_ms, duplicate_edges_deleted, stale_facts_deleted, semantic_merged, llm_calls, bytes_freed 781 tests pass. --- crates/mem-cli/src/cache_alignment.rs | 11 +++++++++ crates/mem-cli/src/chunk_optimizer.rs | 17 ++++++++++++- crates/mem-cli/src/compaction.rs | 14 ++++++++++- crates/mem-cli/src/full_pipeline.rs | 32 +++++++++++++++++++++++++ crates/mem-cli/src/query_router.rs | 11 +++++++++ crates/mem-cli/src/result_compressor.rs | 12 ++++++++++ 6 files changed, 95 insertions(+), 2 deletions(-) diff --git a/crates/mem-cli/src/cache_alignment.rs b/crates/mem-cli/src/cache_alignment.rs index a0fd3c9..e87fd13 100644 --- a/crates/mem-cli/src/cache_alignment.rs +++ b/crates/mem-cli/src/cache_alignment.rs @@ -224,9 +224,20 @@ impl KvCacheAligner { /// Pre-load hot chunks into cache pub fn preload_hot_chunks(&self, hot_chunks: Vec<(&str, &str)>) -> Result<()> { + let count = hot_chunks.len(); for (chunk_id, text) in hot_chunks { self.cache.put(chunk_id, text); } + let metrics = self.cache.metrics(); + tracing::info!( + target: "observability", + event = "cache_preload", + preloaded = count, + cache_hits = metrics.hits, + cache_misses = metrics.misses, + hit_ratio = format!("{:.2}", metrics.hit_ratio()), + "Cache preload complete" + ); Ok(()) } diff --git a/crates/mem-cli/src/chunk_optimizer.rs b/crates/mem-cli/src/chunk_optimizer.rs index 2c07afa..6ce6780 100644 --- a/crates/mem-cli/src/chunk_optimizer.rs +++ b/crates/mem-cli/src/chunk_optimizer.rs @@ -211,17 +211,32 @@ impl ChunkOptimizer { /// End-to-end optimization pipeline pub fn optimize(&self, chunks: Vec) -> (Vec, SelectionMetrics) { + let input_count = chunks.len(); + // Step 1: Filter by threshold let filtered = self.threshold_filter.filter(chunks.clone()); + let after_filter = filtered.len(); // Step 2: Deduplicate let (deduplicated, dedup_removed) = self.deduplicator.deduplicate(filtered); + let after_dedup = deduplicated.len(); // Step 3: Select within budget let (selected, mut metrics) = self.budget_selector.select(deduplicated); - metrics.dedup_removed = dedup_removed; + tracing::info!( + target: "observability", + event = "chunk_optimize", + input = input_count, + after_threshold_filter = after_filter, + after_dedup = after_dedup, + dedup_removed = dedup_removed, + selected = selected.len(), + budget_bytes = metrics.total_bytes, + "Chunk optimization complete" + ); + (selected, metrics) } } diff --git a/crates/mem-cli/src/compaction.rs b/crates/mem-cli/src/compaction.rs index 92fed69..328ede1 100644 --- a/crates/mem-cli/src/compaction.rs +++ b/crates/mem-cli/src/compaction.rs @@ -346,7 +346,19 @@ pub async fn compact_memory( } total_stats.duration_ms = start.elapsed().as_millis() as u64; - info!("Compaction complete in {}ms: {:?}", total_stats.duration_ms, total_stats); + info!( + target: "observability", + event = "compaction_complete", + mode = ?mode, + duration_ms = total_stats.duration_ms, + duplicate_edges_deleted = total_stats.duplicate_edges_deleted, + stale_facts_deleted = total_stats.stale_facts_deleted, + semantic_merged = total_stats.semantic_merged, + llm_calls = total_stats.llm_calls, + bytes_freed = total_stats.bytes_freed, + human_reviews_queued = total_stats.human_reviews_queued, + "Compaction complete" + ); Ok(total_stats) } diff --git a/crates/mem-cli/src/full_pipeline.rs b/crates/mem-cli/src/full_pipeline.rs index 6252aa6..235af3c 100644 --- a/crates/mem-cli/src/full_pipeline.rs +++ b/crates/mem-cli/src/full_pipeline.rs @@ -344,6 +344,22 @@ impl FullPipeline { metrics.total_latency_ms = start.elapsed().as_millis() as u64; + tracing::info!( + target: "observability", + event = "full_pipeline_complete", + query = query, + candidates = metrics.wiki_scope_docs, + prefiltered = metrics.prefilter_candidates, + optimized = metrics.post_optimization_count, + dedup_removed = metrics.dedup_removed, + boosts_applied = metrics.metadata_boosts_applied, + cache_hit_ratio = format!("{:.2}", metrics.cache_hit_ratio), + budget_bytes = metrics.budget_used_bytes, + total_ms = metrics.total_latency_ms, + "Full query pipeline complete" + ); + + Ok(PipelineResult { query: query.to_string(), query_intent, @@ -467,6 +483,22 @@ impl FullPipeline { metrics.total_latency_ms = start.elapsed().as_millis() as u64; + tracing::info!( + target: "observability", + event = "full_pipeline_complete", + query = query, + candidates = metrics.wiki_scope_docs, + prefiltered = metrics.prefilter_candidates, + optimized = metrics.post_optimization_count, + dedup_removed = metrics.dedup_removed, + boosts_applied = metrics.metadata_boosts_applied, + cache_hit_ratio = format!("{:.2}", metrics.cache_hit_ratio), + budget_bytes = metrics.budget_used_bytes, + total_ms = metrics.total_latency_ms, + "Full query pipeline complete" + ); + + Ok(PipelineResult { query: query.to_string(), query_intent, diff --git a/crates/mem-cli/src/query_router.rs b/crates/mem-cli/src/query_router.rs index c5200b9..9669fc3 100644 --- a/crates/mem-cli/src/query_router.rs +++ b/crates/mem-cli/src/query_router.rs @@ -238,6 +238,17 @@ impl QueryRouter { let latency_ms = start.elapsed().as_millis() as u64; + tracing::info!( + target: "observability", + event = "query_route", + route = "direct", + candidates = all_candidates.len(), + prefiltered = prefilter_size, + selected = selected_chunks.len(), + latency_ms = latency_ms, + "Query routing complete" + ); + Ok(RoutedResult { selected_chunks, route, diff --git a/crates/mem-cli/src/result_compressor.rs b/crates/mem-cli/src/result_compressor.rs index 0ef7de0..ccdef38 100644 --- a/crates/mem-cli/src/result_compressor.rs +++ b/crates/mem-cli/src/result_compressor.rs @@ -235,6 +235,18 @@ impl BudgetCompressor { let strategy = self.select_strategy(estimated); let compressed = self.compressor.compress_batch(results, strategy); + let compressed_size: usize = compressed.iter().map(|c| c.text.as_ref().map_or(0, |t| t.len())).sum(); + tracing::info!( + target: "observability", + event = "result_compress", + input_count = compressed.len(), + estimated_bytes = estimated, + compressed_bytes = compressed_size, + budget_bytes = self.max_budget_bytes, + strategy = ?strategy, + "Result compression complete" + ); + (compressed, strategy) } } -- 2.54.0 From 8fdcffc990dc253a4d3ef9c850f5a1d6825e052c Mon Sep 17 00:00:00 2001 From: rock Date: Thu, 10 Sep 2026 10:40:48 +0900 Subject: [PATCH 13/18] fix: migration 009 add source_count + entity dedup index - Add source_count INTEGER DEFAULT 1 column - Dedup existing rows before creating unique index - CREATE UNIQUE INDEX idx_memory_entity_project_name (project_id, name) - Idempotent: safe to re-run --- .../migrations/009_temporal_edge_migration.sql | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/crates/mem-store/migrations/009_temporal_edge_migration.sql b/crates/mem-store/migrations/009_temporal_edge_migration.sql index 4eeba2a..497dc0a 100644 --- a/crates/mem-store/migrations/009_temporal_edge_migration.sql +++ b/crates/mem-store/migrations/009_temporal_edge_migration.sql @@ -47,8 +47,20 @@ CREATE INDEX IF NOT EXISTS idx_memory_edge_target ON memory_edge(target_id); CREATE INDEX IF NOT EXISTS idx_memory_edge_project ON memory_edge(project_id); CREATE INDEX IF NOT EXISTS idx_memory_edge_relation ON memory_edge(relation_type); --- Ensure memory_entity has deleted_at for BFS soft-delete queries +-- Ensure memory_entity has all columns code expects ALTER TABLE memory_entity ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ; +ALTER TABLE memory_entity ADD COLUMN IF NOT EXISTS source_count INTEGER DEFAULT 1; + +-- Unique constraint for entity upsert dedup +DO $$ +BEGIN + -- Dedup existing rows before creating unique index + DELETE FROM memory_entity a USING memory_entity b + WHERE a.project_id = b.project_id AND a.name = b.name + AND a.t_created < b.t_created; +EXCEPTION WHEN OTHERS THEN NULL; +END $$; +CREATE UNIQUE INDEX IF NOT EXISTS idx_memory_entity_project_name ON memory_entity(project_id, name); -- ROLLBACK instructions: -- DROP TABLE IF EXISTS memory_edge; -- 2.54.0 From 04e6b7957a2086a8c7df3e3787080ed9d9d655cf Mon Sep 17 00:00:00 2001 From: rock Date: Fri, 11 Sep 2026 07:45:17 +0900 Subject: [PATCH 14/18] ci: add DB migration workflow Triggers on: - Push to main when crates/mem-store/migrations/*.sql changes - Manual workflow_dispatch (runs ALL migrations) On push: detects changed migration files, runs only those. On dispatch: runs all migrations in order (idempotent). Requires DB_USER + DB_PASSWORD secrets in Forgejo. Connects to memory-db-rw.poimen.svc.cluster.local. All migrations use IF NOT EXISTS / IF EXISTS guards. --- .gitea/workflows/migrate.yaml | 76 +++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 .gitea/workflows/migrate.yaml diff --git a/.gitea/workflows/migrate.yaml b/.gitea/workflows/migrate.yaml new file mode 100644 index 0000000..84c25e0 --- /dev/null +++ b/.gitea/workflows/migrate.yaml @@ -0,0 +1,76 @@ +name: DB Migration + +on: + push: + branches: [main] + paths: + - 'crates/mem-store/migrations/**' + workflow_dispatch: + +env: + DB_HOST: memory-db-rw.poimen.svc.cluster.local + DB_PORT: "5432" + DB_NAME: memory + +jobs: + migrate: + name: Run Migrations + runs-on: rust + steps: + - name: Install psql + run: apt-get update && apt-get install -y postgresql-client + + - name: Checkout code + uses: actions/checkout@v4 + + - name: Fetch previous migrations state + run: | + git fetch origin main --depth=2 + # List changed migration files + CHANGED=$(git diff --name-only HEAD~1 HEAD -- crates/mem-store/migrations/ || echo "") + echo "Changed migrations: $CHANGED" + echo "CHANGED_MIGRATIONS=$CHANGED" >> $GITHUB_ENV + + - name: Run migrations + if: env.CHANGED_MIGRATIONS != '' + run: | + export PGPASSWORD="${DB_PASSWORD}" + + echo "=== Running changed migrations ===" + for f in $CHANGED_MIGRATIONS; do + if [ -f "$f" ]; then + echo "--- Applying: $f ---" + psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f "$f" 2>&1 + if [ $? -ne 0 ]; then + echo "ERROR: Migration $f failed!" + exit 1 + fi + echo "--- OK: $f ---" + fi + done + + echo "=== Verify schema ===" + psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "\dt memory*" + env: + DB_USER: ${{ secrets.DB_USER }} + DB_PASSWORD: ${{ secrets.DB_PASSWORD }} + + - name: Run all migrations (manual trigger) + if: github.event_name == 'workflow_dispatch' + run: | + export PGPASSWORD="${DB_PASSWORD}" + + echo "=== Running all migrations in order ===" + for f in $(ls crates/mem-store/migrations/*.sql | sort); do + echo "--- Applying: $f ---" + psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f "$f" 2>&1 || true + echo "--- Done: $f ---" + done + + echo "=== Final schema ===" + psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "\dt memory*" + psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "\d memory_entity" + psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "\d memory_edge" + env: + DB_USER: ${{ secrets.DB_USER }} + DB_PASSWORD: ${{ secrets.DB_PASSWORD }} -- 2.54.0 From 8290e9de37d0e1c0c070e50e18bcc1035948a65b Mon Sep 17 00:00:00 2001 From: rock Date: Fri, 11 Sep 2026 07:59:37 +0900 Subject: [PATCH 15/18] fix: align CNPG memory-db manifest with homelab working version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Broke because poimen-memory repo had: - monitoring.enabled: true (field removed in CNPG 1.30) - Database CRD missing spec.name (required field) - storage 10Gi vs homelab 20Gi - missing postInitApplicationSQL for pgvector Now matches homelab/k8s/infra/databases/memory-db.yaml exactly. Removed separate Database CRD — pgvector installed via bootstrap. --- k8s/infra/databases/memory-db.yaml | 60 ++++++++++-------------------- 1 file changed, 19 insertions(+), 41 deletions(-) diff --git a/k8s/infra/databases/memory-db.yaml b/k8s/infra/databases/memory-db.yaml index 3193c0a..995f52c 100644 --- a/k8s/infra/databases/memory-db.yaml +++ b/k8s/infra/databases/memory-db.yaml @@ -1,8 +1,6 @@ ---- -# CNPG Postgres cluster for Poimen Memory system (GitOps, declarative extensions). -# 2 instances, pgvector 0.7.0 via spec.extensions (not manual CREATE EXTENSION). -# Storage: 10Gi longhorn, consistent with temporal-db.yaml. -# No manual psql needed — all via git/ArgoCD. +# Dedicated CNPG Postgres for Poimen Memory (GitOps, wave 2). +# Matches homelab/k8s/infra/databases/memory-db.yaml — single source of truth. +# CNPG generates secret `memory-db-app` + service `memory-db-rw` in ns poimen. apiVersion: postgresql.cnpg.io/v1 kind: Cluster metadata: @@ -13,20 +11,6 @@ metadata: spec: instances: 2 imageName: ghcr.io/cloudnative-pg/postgresql:16.2 - enableSuperuserAccess: false - storage: - size: 10Gi - storageClass: longhorn - resources: - requests: { memory: "512Mi", cpu: "250m" } - limits: { memory: "2Gi", cpu: "1" } - affinity: - podAntiAffinityType: preferred - topologyKey: kubernetes.io/hostname - tolerations: - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoSchedule bootstrap: initdb: database: memory @@ -34,25 +18,19 @@ spec: encoding: UTF8 localeCollate: C localeCType: C - monitoring: - enabled: true - podMonitorTemplate: - spec: - interval: 30s - scrapeTimeout: 10s ---- -# Database resource with pgvector extension (declarative, git-managed). -# CNPG 1.30.0+ supports this via spec.extensions on the Database CRD. -# Ensures pgvector is installed and available for HNSW indexing. -apiVersion: postgresql.cnpg.io/v1 -kind: Database -metadata: - name: memory - namespace: poimen -spec: - cluster: - name: memory-db - owner: app - extensions: - - name: vector - ensure: present + postInitApplicationSQL: + - "CREATE EXTENSION vector;" + enableSuperuserAccess: false + resources: + requests: { memory: "512Mi", cpu: "250m" } + limits: { memory: "2Gi", cpu: "1" } + storage: + size: 20Gi + storageClass: longhorn + affinity: + podAntiAffinityType: preferred + topologyKey: kubernetes.io/hostname + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule -- 2.54.0 From 8dc774a6c8c8307532bb6c0c0989ec554a580612 Mon Sep 17 00:00:00 2001 From: rock Date: Fri, 11 Sep 2026 10:37:08 +0900 Subject: [PATCH 16/18] =?UTF-8?q?fix:=20CI=20workflows=20=E2=80=94=20add?= =?UTF-8?q?=20nodejs,=20clean=20up=20migration=20runner?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit deploy.yaml: add nodejs install (required for actions/checkout) migrate.yaml: rewrite migration runner - Use PGHOST/PGUSER/PGPASSWORD env vars (no inline -h/-U/-p flags) - ON_ERROR_STOP=1 for strict error handling on push - || true for dispatch (idempotent full replay) - Verify schema after apply - fetch-depth: 2 for diff detection --- .gitea/workflows/deploy.yaml | 6 ++- .gitea/workflows/migrate.yaml | 98 ++++++++++++++++++++--------------- 2 files changed, 59 insertions(+), 45 deletions(-) diff --git a/.gitea/workflows/deploy.yaml b/.gitea/workflows/deploy.yaml index 52db832..26a9929 100644 --- a/.gitea/workflows/deploy.yaml +++ b/.gitea/workflows/deploy.yaml @@ -15,8 +15,10 @@ jobs: name: Tag & Push Latest runs-on: rust steps: - - name: Install Docker - run: apt-get update && apt-get install -y docker.io + - name: Install Node.js and Docker + run: | + apt-get update + apt-get install -y nodejs docker.io - name: Checkout code uses: actions/checkout@v4 diff --git a/.gitea/workflows/migrate.yaml b/.gitea/workflows/migrate.yaml index 84c25e0..bd33e4e 100644 --- a/.gitea/workflows/migrate.yaml +++ b/.gitea/workflows/migrate.yaml @@ -11,66 +11,78 @@ env: DB_HOST: memory-db-rw.poimen.svc.cluster.local DB_PORT: "5432" DB_NAME: memory + MIGRATIONS_DIR: crates/mem-store/migrations jobs: migrate: name: Run Migrations runs-on: rust steps: - - name: Install psql - run: apt-get update && apt-get install -y postgresql-client + - name: Install Node.js and psql + run: | + apt-get update + apt-get install -y nodejs postgresql-client - name: Checkout code uses: actions/checkout@v4 + with: + fetch-depth: 2 - - name: Fetch previous migrations state + - name: Detect changed migrations + id: detect run: | - git fetch origin main --depth=2 - # List changed migration files - CHANGED=$(git diff --name-only HEAD~1 HEAD -- crates/mem-store/migrations/ || echo "") - echo "Changed migrations: $CHANGED" - echo "CHANGED_MIGRATIONS=$CHANGED" >> $GITHUB_ENV + CHANGED=$(git diff --name-only HEAD~1 HEAD -- "$MIGRATIONS_DIR"/*.sql 2>/dev/null || echo "") + if [ -n "$CHANGED" ]; then + echo "files=$CHANGED" >> $GITHUB_OUTPUT + echo "found=true" >> $GITHUB_OUTPUT + echo "Changed: $CHANGED" + else + echo "found=false" >> $GITHUB_OUTPUT + echo "No migration changes detected" + fi - - name: Run migrations - if: env.CHANGED_MIGRATIONS != '' + - name: Apply changed migrations (push) + if: github.event_name == 'push' && steps.detect.outputs.found == 'true' + env: + PGHOST: ${{ env.DB_HOST }} + PGPORT: ${{ env.DB_PORT }} + PGDATABASE: ${{ env.DB_NAME }} + PGUSER: ${{ secrets.DB_USER }} + PGPASSWORD: ${{ secrets.DB_PASSWORD }} run: | - export PGPASSWORD="${DB_PASSWORD}" - - echo "=== Running changed migrations ===" - for f in $CHANGED_MIGRATIONS; do - if [ -f "$f" ]; then - echo "--- Applying: $f ---" - psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f "$f" 2>&1 - if [ $? -ne 0 ]; then - echo "ERROR: Migration $f failed!" - exit 1 - fi - echo "--- OK: $f ---" - fi + for f in ${{ steps.detect.outputs.files }}; do + [ -f "$f" ] || continue + echo "=== Applying: $f ===" + psql -v ON_ERROR_STOP=1 -f "$f" + echo "=== OK ===" done - echo "=== Verify schema ===" - psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "\dt memory*" - env: - DB_USER: ${{ secrets.DB_USER }} - DB_PASSWORD: ${{ secrets.DB_PASSWORD }} - - - name: Run all migrations (manual trigger) + - name: Apply all migrations (dispatch) if: github.event_name == 'workflow_dispatch' + env: + PGHOST: ${{ env.DB_HOST }} + PGPORT: ${{ env.DB_PORT }} + PGDATABASE: ${{ env.DB_NAME }} + PGUSER: ${{ secrets.DB_USER }} + PGPASSWORD: ${{ secrets.DB_PASSWORD }} run: | - export PGPASSWORD="${DB_PASSWORD}" - - echo "=== Running all migrations in order ===" - for f in $(ls crates/mem-store/migrations/*.sql | sort); do - echo "--- Applying: $f ---" - psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f "$f" 2>&1 || true - echo "--- Done: $f ---" + for f in $(ls "$MIGRATIONS_DIR"/*.sql | sort); do + echo "=== Applying: $f ===" + psql -v ON_ERROR_STOP=1 -f "$f" || true + echo "=== Done ===" done - echo "=== Final schema ===" - psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "\dt memory*" - psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "\d memory_entity" - psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "\d memory_edge" + - name: Verify schema env: - DB_USER: ${{ secrets.DB_USER }} - DB_PASSWORD: ${{ secrets.DB_PASSWORD }} + PGHOST: ${{ env.DB_HOST }} + PGPORT: ${{ env.DB_PORT }} + PGDATABASE: ${{ env.DB_NAME }} + PGUSER: ${{ secrets.DB_USER }} + PGPASSWORD: ${{ secrets.DB_PASSWORD }} + run: | + echo "=== Tables ===" + psql -c "\dt memory*" + echo "=== Entity Schema ===" + psql -c "\d memory_entity" + echo "=== Edge Schema ===" + psql -c "\d memory_edge" -- 2.54.0 From 1506a93ebb58c3ebdd750e19c6ad5c57b477e2d1 Mon Sep 17 00:00:00 2001 From: rock Date: Fri, 11 Sep 2026 10:58:56 +0900 Subject: [PATCH 17/18] feat: query fuzzy search + observability + edge schema fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add fuzzy ILIKE search on entity name/description - Add observability logging (query_entity_search event) - Fix edge schema: source_entity_id→source_id, target_entity_id→target_id --- crates/mem-cli/src/http_server.rs | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/crates/mem-cli/src/http_server.rs b/crates/mem-cli/src/http_server.rs index 88116c2..4f3a4c9 100644 --- a/crates/mem-cli/src/http_server.rs +++ b/crates/mem-cli/src/http_server.rs @@ -1342,15 +1342,29 @@ async fn query_temporal_graph( state: &web::Data, params: &QueryParams, ) -> anyhow::Result { - // Step 1: Find entities (order by name for deterministic results) + // Step 1: Find entities matching question (fuzzy name/description search) let entities_rows: Vec<(String, String, String)> = sqlx::query_as( - "SELECT id, name, entity_type FROM memory_entity WHERE project_id = $1 LIMIT $2" + "SELECT id, name, entity_type FROM memory_entity + WHERE project_id = $1 + AND (name ILIKE '%' || $2 || '%' OR description ILIKE '%' || $2 || '%') + ORDER BY confidence DESC + LIMIT $3" ) .bind(¶ms.project) + .bind(¶ms.question) .bind(params.limit as i32) .fetch_all(&state.pool) .await .unwrap_or_default(); + + tracing::info!( + target: "observability", + event = "query_entity_search", + project = %params.project, + question = %params.question, + matched = entities_rows.len(), + "Entity search complete" + ); // Step 2: Traverse edges from found entities // NOTE: Edges will be empty until temporal schema is migrated @@ -1360,7 +1374,7 @@ async fn query_temporal_graph( for (entity_id, _name, _type_str) in &entities_rows { let entity_edges: Vec<(String, String, String, String, f32, Option>, Option>)> = sqlx::query_as( - "SELECT id, target_entity_id, relation_type, fact, confidence, t_valid, t_invalid FROM memory_edge WHERE project_id = $1 AND source_entity_id = $2" + "SELECT id, target_id, relation_type, fact, confidence, t_valid, t_invalid FROM memory_edge WHERE project_id = $1 AND source_id = $2" ) .bind(¶ms.project) .bind(entity_id) -- 2.54.0 From c7def601ece1922e4d161b8f9dac2e4b5a9fed1a Mon Sep 17 00:00:00 2001 From: rock Date: Fri, 11 Sep 2026 11:05:39 +0900 Subject: [PATCH 18/18] =?UTF-8?q?fix:=20CI=20migration=20workflow=20?= =?UTF-8?q?=E2=80=94=20install=20Docker=20before=20migration=20runner?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add docker.io + DOCKER_HOST (tcp://localhost:2375) to migration workflow. Ensures Node + Docker available before migration action executes. Aligns with build.yaml environment setup. --- .gitea/workflows/migrate.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitea/workflows/migrate.yaml b/.gitea/workflows/migrate.yaml index bd33e4e..1683759 100644 --- a/.gitea/workflows/migrate.yaml +++ b/.gitea/workflows/migrate.yaml @@ -12,16 +12,17 @@ env: DB_PORT: "5432" DB_NAME: memory MIGRATIONS_DIR: crates/mem-store/migrations + DOCKER_HOST: tcp://localhost:2375 jobs: migrate: name: Run Migrations runs-on: rust steps: - - name: Install Node.js and psql + - name: Install Node.js, Docker, and psql run: | apt-get update - apt-get install -y nodejs postgresql-client + apt-get install -y nodejs docker.io postgresql-client - name: Checkout code uses: actions/checkout@v4 -- 2.54.0