fix: enable LLM entity extraction + handle reasoning model output

Root causes of zero entity extraction:
1. IngestWorker used WikiLinkFallbackExtractor (wiki links only)
   Fix: Use LlmEntityExtractor when LLM_ENDPOINT is set
2. ExtractedEntity.entity_type vs LLM returning "type"
   Fix: serde alias "type" -> entity_type, default confidence
3. Reasoning models output <think>...</think> before JSON
   Fix: strip_thinking_tags() extracts JSON from response
4. Reflection verification crashes pipeline on parse failure
   Fix: graceful fallback, keep all entities if reflection fails

Tested with reasoning-predictor (qwen2.5:3b) via port-forward.

crates/mem-cli/src/ingest_worker.rs

@@ -2,7 +2,7 @@ use anyhow::Result;
 use mem_store::{MemoryL1, VectorStore, ChunkL0, EntityRepoOps, EdgeRepoOps};
 use mem_llm::EmbeddingsClient;
 use mem_ingest::ingest_pipeline::{IngestPipeline, Episode};
-use mem_ingest::entity_extractor::WikiLinkFallbackExtractor;
+use mem_ingest::entity_extractor::{WikiLinkFallbackExtractor, LlmEntityExtractor};
 use mem_ingest::fact_extractor::SimpleFactExtractor;
 use mem_ingest::contradiction_detector::ContradictionHandler;
 use sqlx::PgPool;
@@ -26,9 +26,16 @@ impl IngestWorker {
     ) -> Self {
         let vector_store = Arc::new(VectorStore::new(pool.clone()));
         
-        // Initialize extraction pipeline
+        // Initialize extraction pipeline — use LLM if LLM_ENDPOINT is set, else fallback to wiki links
         let entity_extractor: Arc<dyn mem_ingest::entity_extractor::EntityExtractor> = 
-            Arc::new(WikiLinkFallbackExtractor);
+            if std::env::var("LLM_ENDPOINT").is_ok() {
+                let model = std::env::var("LLM_MODEL").unwrap_or_else(|_| "qwen2.5:3b-instruct".to_string());
+                tracing::info!("Using LLM entity extractor: model={}", model);
+                Arc::new(LlmEntityExtractor::new(&model))
+            } else {
+                tracing::info!("LLM_ENDPOINT not set, using WikiLink fallback extractor");
+                Arc::new(WikiLinkFallbackExtractor)
+            };
         let fact_extractor: Arc<dyn mem_ingest::fact_extractor::FactExtractor> = 
             Arc::new(SimpleFactExtractor);
         let contradiction_detector = Arc::new(ContradictionHandler::default());

crates/mem-core/src/agent_entity.rs

@@ -0,0 +1,300 @@
+/// Agent-specific entity metadata for Phase 3 Agent Self-Awareness.
+///
+/// These structures attach to Entity via entity_type discriminator.
+/// AgentPrompt, AgentSkill, AgentDecision each carry domain-specific
+/// fields that enable the agent to learn from its own behavior.
+
+use serde::{Deserialize, Serialize};
+use time::OffsetDateTime;
+
+use crate::entity::{Entity, EntityType};
+
+/// Metadata for an AgentPrompt entity.
+/// Tracks prompt templates, their usage frequency, and effectiveness.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct AgentPromptMeta {
+    /// The prompt template text (may contain {{placeholders}}).
+    pub template: String,
+    /// Which LLM model this prompt targets (e.g. "claude-3-sonnet").
+    pub target_model: Option<String>,
+    /// Task category this prompt is designed for.
+    pub task_category: String,
+    /// Number of times this prompt has been used.
+    pub usage_count: u64,
+    /// Average quality score from outcomes (0.0-1.0).
+    pub avg_quality: f32,
+    /// Last time this prompt was used.
+    #[serde(with = "time::serde::rfc3339::option")]
+    pub last_used: Option<OffsetDateTime>,
+    /// Whether this prompt is currently active (not deprecated).
+    pub active: bool,
+    /// Version for tracking prompt evolution.
+    pub version: u32,
+    /// Tags for categorization.
+    pub tags: Vec<String>,
+}
+
+/// Metadata for an AgentSkill entity.
+/// Tracks learned capabilities and their effectiveness.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct AgentSkillMeta {
+    /// Description of what this skill does.
+    pub description: String,
+    /// Trigger conditions that activate this skill.
+    pub trigger_patterns: Vec<String>,
+    /// Success rate over all invocations (0.0-1.0).
+    pub success_rate: f32,
+    /// Number of times this skill was invoked.
+    pub invocation_count: u64,
+    /// Average latency in milliseconds.
+    pub avg_latency_ms: u64,
+    /// Linked prompt entity IDs that this skill uses.
+    pub linked_prompts: Vec<String>,
+    /// Whether this skill is currently enabled.
+    pub enabled: bool,
+}
+
+/// Metadata for an AgentDecision entity.
+/// Records a decision the agent made, including reasoning and outcome.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct AgentDecisionMeta {
+    /// What the agent decided to do.
+    pub action: String,
+    /// Why the agent chose this action.
+    pub reasoning: String,
+    /// Available alternatives that were considered.
+    pub alternatives: Vec<String>,
+    /// Confidence in the decision (0.0-1.0).
+    pub confidence: f32,
+    /// Outcome of the decision (set after execution).
+    pub outcome: Option<DecisionOutcome>,
+    /// Context that informed the decision (entity IDs).
+    pub context_entities: Vec<String>,
+    /// The tool/task context when decision was made.
+    pub tool: Option<String>,
+    pub task: Option<String>,
+}
+
+/// Outcome of an agent decision.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct DecisionOutcome {
+    /// Whether the decision led to success.
+    pub success: bool,
+    /// Quality score of the outcome (0.0-1.0).
+    pub quality: f32,
+    /// Feedback or error message.
+    pub feedback: Option<String>,
+    /// When the outcome was recorded.
+    #[serde(with = "time::serde::rfc3339")]
+    pub recorded_at: OffsetDateTime,
+}
+
+// --- Factory functions ---
+
+/// Create a new AgentPrompt entity.
+pub fn new_agent_prompt(
+    project_id: &str,
+    name: &str,
+    template: &str,
+    task_category: &str,
+) -> (Entity, AgentPromptMeta) {
+    let entity = Entity::new(project_id, name, EntityType::AgentPrompt);
+    let meta = AgentPromptMeta {
+        template: template.to_string(),
+        target_model: None,
+        task_category: task_category.to_string(),
+        usage_count: 0,
+        avg_quality: 0.0,
+        last_used: None,
+        active: true,
+        version: 1,
+        tags: vec![],
+    };
+    (entity, meta)
+}
+
+/// Create a new AgentSkill entity.
+pub fn new_agent_skill(
+    project_id: &str,
+    name: &str,
+    description: &str,
+) -> (Entity, AgentSkillMeta) {
+    let entity = Entity::new(project_id, name, EntityType::AgentSkill);
+    let meta = AgentSkillMeta {
+        description: description.to_string(),
+        trigger_patterns: vec![],
+        success_rate: 0.0,
+        invocation_count: 0,
+        avg_latency_ms: 0,
+        linked_prompts: vec![],
+        enabled: true,
+    };
+    (entity, meta)
+}
+
+/// Create a new AgentDecision entity.
+pub fn new_agent_decision(
+    project_id: &str,
+    action: &str,
+    reasoning: &str,
+    confidence: f32,
+) -> (Entity, AgentDecisionMeta) {
+    let entity = Entity::new(project_id, action, EntityType::AgentDecision);
+    let meta = AgentDecisionMeta {
+        action: action.to_string(),
+        reasoning: reasoning.to_string(),
+        alternatives: vec![],
+        confidence,
+        outcome: None,
+        context_entities: vec![],
+        tool: None,
+        task: None,
+    };
+    (entity, meta)
+}
+
+/// Record outcome for a decision.
+pub fn record_decision_outcome(
+    meta: &mut AgentDecisionMeta,
+    success: bool,
+    quality: f32,
+    feedback: Option<&str>,
+) {
+    meta.outcome = Some(DecisionOutcome {
+        success,
+        quality,
+        feedback: feedback.map(|s| s.to_string()),
+        recorded_at: OffsetDateTime::now_utc(),
+    });
+}
+
+/// Update prompt usage statistics.
+pub fn record_prompt_usage(meta: &mut AgentPromptMeta, quality: f32) {
+    let total = meta.avg_quality * meta.usage_count as f32 + quality;
+    meta.usage_count += 1;
+    meta.avg_quality = total / meta.usage_count as f32;
+    meta.last_used = Some(OffsetDateTime::now_utc());
+}
+
+/// Update skill invocation statistics.
+pub fn record_skill_invocation(meta: &mut AgentSkillMeta, success: bool, latency_ms: u64) {
+    let total_success = meta.success_rate * meta.invocation_count as f32
+        + if success { 1.0 } else { 0.0 };
+    let total_latency = meta.avg_latency_ms * meta.invocation_count + latency_ms;
+    meta.invocation_count += 1;
+    meta.success_rate = total_success / meta.invocation_count as f32;
+    meta.avg_latency_ms = total_latency / meta.invocation_count;
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_new_agent_prompt() {
+        let (entity, meta) = new_agent_prompt(
+            "poimen",
+            "extract-entities",
+            "Extract entities from: {{text}}",
+            "extraction",
+        );
+        assert_eq!(entity.entity_type, EntityType::AgentPrompt);
+        assert_eq!(entity.name, "extract-entities");
+        assert_eq!(meta.template, "Extract entities from: {{text}}");
+        assert_eq!(meta.task_category, "extraction");
+        assert_eq!(meta.usage_count, 0);
+        assert!(meta.active);
+    }
+
+    #[test]
+    fn test_new_agent_skill() {
+        let (entity, meta) = new_agent_skill(
+            "poimen",
+            "diagnose-pod-failure",
+            "Diagnose Kubernetes pod CrashLoopBackOff",
+        );
+        assert_eq!(entity.entity_type, EntityType::AgentSkill);
+        assert_eq!(meta.description, "Diagnose Kubernetes pod CrashLoopBackOff");
+        assert!(meta.enabled);
+        assert_eq!(meta.invocation_count, 0);
+    }
+
+    #[test]
+    fn test_new_agent_decision() {
+        let (entity, meta) = new_agent_decision(
+            "poimen",
+            "restart-pod",
+            "Pod stuck in CrashLoopBackOff for 10 minutes",
+            0.85,
+        );
+        assert_eq!(entity.entity_type, EntityType::AgentDecision);
+        assert_eq!(meta.action, "restart-pod");
+        assert_eq!(meta.confidence, 0.85);
+        assert!(meta.outcome.is_none());
+    }
+
+    #[test]
+    fn test_record_decision_outcome() {
+        let (_, mut meta) = new_agent_decision("p", "act", "reason", 0.9);
+        assert!(meta.outcome.is_none());
+
+        record_decision_outcome(&mut meta, true, 0.95, Some("Pod recovered"));
+        assert!(meta.outcome.is_some());
+        let outcome = meta.outcome.unwrap();
+        assert!(outcome.success);
+        assert_eq!(outcome.quality, 0.95);
+        assert_eq!(outcome.feedback, Some("Pod recovered".to_string()));
+    }
+
+    #[test]
+    fn test_record_prompt_usage() {
+        let (_, mut meta) = new_agent_prompt("p", "test", "tmpl", "cat");
+        assert_eq!(meta.usage_count, 0);
+        assert_eq!(meta.avg_quality, 0.0);
+
+        record_prompt_usage(&mut meta, 0.8);
+        assert_eq!(meta.usage_count, 1);
+        assert_eq!(meta.avg_quality, 0.8);
+
+        record_prompt_usage(&mut meta, 1.0);
+        assert_eq!(meta.usage_count, 2);
+        assert!((meta.avg_quality - 0.9).abs() < 0.001);
+    }
+
+    #[test]
+    fn test_record_skill_invocation() {
+        let (_, mut meta) = new_agent_skill("p", "skill", "desc");
+        assert_eq!(meta.invocation_count, 0);
+
+        record_skill_invocation(&mut meta, true, 100);
+        assert_eq!(meta.invocation_count, 1);
+        assert_eq!(meta.success_rate, 1.0);
+        assert_eq!(meta.avg_latency_ms, 100);
+
+        record_skill_invocation(&mut meta, false, 200);
+        assert_eq!(meta.invocation_count, 2);
+        assert_eq!(meta.success_rate, 0.5);
+        assert_eq!(meta.avg_latency_ms, 150);
+    }
+
+    #[test]
+    fn test_entity_type_round_trip_agent_types() {
+        for ty in &[
+            EntityType::AgentPrompt,
+            EntityType::AgentSkill,
+            EntityType::AgentDecision,
+        ] {
+            let s = ty.as_str();
+            assert_eq!(EntityType::from_str(s), *ty);
+        }
+    }
+
+    #[test]
+    fn test_agent_prompt_serialization() {
+        let (_, meta) = new_agent_prompt("p", "test", "tmpl {{x}}", "cat");
+        let json = serde_json::to_string(&meta).unwrap();
+        let deserialized: AgentPromptMeta = serde_json::from_str(&json).unwrap();
+        assert_eq!(deserialized.template, "tmpl {{x}}");
+        assert_eq!(deserialized.task_category, "cat");
+    }
+}

crates/mem-core/src/entity.rs

@@ -17,6 +17,13 @@ pub enum EntityType {
     Location,
     Event,
     Organization,
+    /// Agent prompt template tracked as a first-class entity.
+    /// Enables the agent to learn which prompts produce good results.
+    AgentPrompt,
+    /// Agent skill — a reusable capability the agent has learned.
+    AgentSkill,
+    /// Agent decision — a recorded choice with reasoning and outcome.
+    AgentDecision,
     Unknown,
 }
 
@@ -29,6 +36,9 @@ impl EntityType {
             Self::Location => "location",
             Self::Event => "event",
             Self::Organization => "organization",
+            Self::AgentPrompt => "agent_prompt",
+            Self::AgentSkill => "agent_skill",
+            Self::AgentDecision => "agent_decision",
             Self::Unknown => "unknown",
         }
     }
@@ -41,6 +51,9 @@ impl EntityType {
             "location" => Self::Location,
             "event" => Self::Event,
             "organization" => Self::Organization,
+            "agent_prompt" => Self::AgentPrompt,
+            "agent_skill" => Self::AgentSkill,
+            "agent_decision" => Self::AgentDecision,
             _ => Self::Unknown,
         }
     }
@@ -175,6 +188,9 @@ mod tests {
             EntityType::Person,
             EntityType::Tool,
             EntityType::Concept,
+            EntityType::AgentPrompt,
+            EntityType::AgentSkill,
+            EntityType::AgentDecision,
         ] {
             let s = ty.as_str();
             assert_eq!(EntityType::from_str(s), *ty);

crates/mem-core/src/lib.rs

@@ -12,6 +12,7 @@ pub mod scoring;
 pub mod entity;
 pub mod edge;
 pub mod community;
+pub mod agent_entity;
 
 pub use gate_parser::{GateResponse, ParseError, parse_gate_response};
 
@@ -30,3 +31,4 @@ pub use scoring::{DocumentScorer, ScoringPipeline, GlobalTfIdfScorer, ProjectTfI
 pub use entity::{Entity, EntityType};
 pub use edge::{Edge, ContradictionStatus};
 pub use community::Community;
+pub use agent_entity::{AgentPromptMeta, AgentSkillMeta, AgentDecisionMeta, DecisionOutcome};

crates/mem-ingest/src/entity_extractor.rs

@@ -22,11 +22,15 @@ use tokio::sync::Mutex;
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct ExtractedEntity {
     pub name: String,
+    #[serde(alias = "type")]
     pub entity_type: EntityType,
     pub summary: String,
+    #[serde(default = "default_confidence")]
     pub confidence: f32,
 }
 
+fn default_confidence() -> f32 { 0.8 }
+
 impl ExtractedEntity {
     /// Convert to domain model (Phase 1 type)
     pub fn to_domain(&self, project_id: &str) -> Entity {
@@ -62,6 +66,25 @@ impl LlmEntityExtractor {
 
     /// Parse extraction response JSON
     /// Format: { "entities": [{ "name": "...", "type": "...", "summary": "..." }, ...] }
+    /// Strip <think>...</think> tags from reasoning model output and extract JSON
+    fn strip_thinking_tags(text: &str) -> String {
+        let mut result = text.to_string();
+        // Remove <think>...</think> blocks
+        if let Some(start) = result.find("<think>") {
+            if let Some(end) = result.find("</think>") {
+                result = format!("{}{}", &result[..start], &result[end + 8..]);
+            }
+        }
+        // Try to find JSON object in remaining text
+        let trimmed = result.trim();
+        if let Some(start) = trimmed.find('{') {
+            if let Some(end) = trimmed.rfind('}') {
+                return trimmed[start..=end].to_string();
+            }
+        }
+        trimmed.to_string()
+    }
+
     fn parse_extraction(response: &str) -> Result<Vec<ExtractedEntity>> {
         #[derive(Deserialize)]
         struct Response {
@@ -146,12 +169,16 @@ impl LlmEntityExtractor {
         }
         
         let data: serde_json::Value = response.json().await?;
-        let content = data["choices"][0]["message"]["content"]
+        let raw_content = data["choices"][0]["message"]["content"]
             .as_str()
             .unwrap_or("{}")
             .to_string();
         
-        tracing::debug!("LLM response (via Authentik JWT): {}", content);
+        // Strip <think>...</think> tags from reasoning models
+        let content = Self::strip_thinking_tags(&raw_content);
+        
+        tracing::debug!("LLM raw response length={}, cleaned length={}", raw_content.len(), content.len());
+        tracing::debug!("LLM cleaned content: {}", content);
         Ok(content)
     }
     
@@ -233,14 +260,27 @@ Respond in JSON:
             );
 
             let reflection = if std::env::var("LLM_ENDPOINT").is_ok() {
-                self.call_llm_endpoint(&reflection_prompt).await.unwrap_or_else(|_| self.simulate_llm(&reflection_prompt).unwrap_or_default())
+                self.call_llm_endpoint(&reflection_prompt).await.unwrap_or_else(|e| {
+                    tracing::warn!("Reflection LLM call failed: {}, skipping verification", e);
+                    String::new()
+                })
             } else {
                 self.simulate_llm(&reflection_prompt)?
             };
-            let verified = Self::parse_reflection(&reflection)?;
 
-            // Filter: keep only entities marked present
-            entities.retain(|e| verified.iter().any(|(name, present)| name == &e.name && *present));
+            // If reflection succeeded, filter entities; otherwise keep all
+            if !reflection.is_empty() {
+                match Self::parse_reflection(&reflection) {
+                    Ok(verified) => {
+                        entities.retain(|e| verified.iter().any(|(name, present)| name == &e.name && *present));
+                    }
+                    Err(e) => {
+                        tracing::warn!("Reflection parse failed: {}, keeping all entities", e);
+                    }
+                }
+            } else {
+                tracing::info!("Reflection skipped, keeping {} unverified entities", entities.len());
+            }
 
             // Adjust confidence for reflected entities (slight penalty for needing verification)
             for entity in &mut entities {

k8s/app/deployment.yaml

@@ -78,6 +78,7 @@ spec:
                 name: poimen-memory-auth
             - secretRef:
                 name: poimen-memory-secrets
+          command: ["/app/mem"]
           args:
             - serve
             - --port