feat: LLM entity + fact extraction pipeline (Zep paper alignment) (#48)
## Changes
### Entity Extraction
- Switch from WikiLinkFallbackExtractor to LlmEntityExtractor when LLM_ENDPOINT set
- `clean_llm_response()`: strips `<think>` tags, markdown fences, extracts JSON
- Handle array responses (Ollama returns `[...]` not `{entities: [...]}`)
- EntityType custom Deserialize: unknown variants → Unknown (no crash)
- Increase timeout 30s→90s, max_tokens 500→1500 for reasoning models
- Graceful reflection fallback: keep entities if verification fails
### Fact Extraction (NEW)
- LlmFactExtractor: LLM-based relationship extraction between entity pairs
- Validates source/target against known entity list (drops hallucinated edges)
- Same robust JSON cleaning for reasoning models + Ollama
- IngestWorker auto-selects LLM vs Simple based on LLM_ENDPOINT env
### K8s Deployment
- Add `command: ["/app/mem"]` (fix args replacing CMD)
- Add LLM_ENDPOINT, LLM_MODEL env vars for in-cluster LLM
## E2E Tested (local Ollama qwen2.5:3b)
- 12 entities extracted (person, tool, concept, organization)
- 5 edges with relationships and facts
- 781 tests pass
## Zep Paper Alignment (§2.2)
- Entity extraction + resolution (§2.2.1)
- Fact extraction between entity pairs (§2.2.2)
- Temporal edge invalidation ready (t_valid/t_invalid schema)
- Reflection verification (§2.2.1, graceful fallback)
---------
Co-authored-by: rock <[email protected]>
Reviewed-on: #48
Co-authored-by: poimen <[email protected]>
This commit was merged in pull request #48.
This commit is contained in:
@@ -52,13 +52,24 @@ impl AuthentikJwtIssuer {
|
||||
|
||||
/// From environment: AUTHENTIK_ISSUER, AUTHENTIK_CLIENT_ID, AUTHENTIK_CLIENT_SECRET
|
||||
pub fn from_env() -> Result<Self> {
|
||||
// Support both naming conventions: AUTHENTIK_* and memory-agent-oidc secret keys
|
||||
let issuer = std::env::var("AUTHENTIK_ISSUER")
|
||||
.map_err(|_| anyhow!("AUTHENTIK_ISSUER not set"))?;
|
||||
.or_else(|_| std::env::var("ISSUER"))
|
||||
.map_err(|_| anyhow!("AUTHENTIK_ISSUER or ISSUER not set"))?;
|
||||
let client_id = std::env::var("AUTHENTIK_CLIENT_ID")
|
||||
.map_err(|_| anyhow!("AUTHENTIK_CLIENT_ID not set"))?;
|
||||
.or_else(|_| std::env::var("CLIENT_ID"))
|
||||
.map_err(|_| anyhow!("AUTHENTIK_CLIENT_ID or CLIENT_ID not set"))?;
|
||||
let client_secret = std::env::var("AUTHENTIK_CLIENT_SECRET")
|
||||
.map_err(|_| anyhow!("AUTHENTIK_CLIENT_SECRET not set"))?;
|
||||
.or_else(|_| std::env::var("CLIENT_SECRET"))
|
||||
.map_err(|_| anyhow!("AUTHENTIK_CLIENT_SECRET or CLIENT_SECRET not set"))?;
|
||||
|
||||
tracing::info!(
|
||||
target: "observability",
|
||||
event = "authentik_jwt_init",
|
||||
issuer = %issuer,
|
||||
client_id = %client_id,
|
||||
"Authentik JWT issuer initialized"
|
||||
);
|
||||
Ok(Self::new(&issuer, &client_id, &client_secret))
|
||||
}
|
||||
|
||||
@@ -92,12 +103,25 @@ impl AuthentikJwtIssuer {
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
// Authentik OAuth2 token endpoint
|
||||
let token_url = format!("{}/token/", self.issuer_url.trim_end_matches('/'));
|
||||
// Use TOKEN_URL env var if set, otherwise derive from issuer
|
||||
let token_url = std::env::var("TOKEN_URL")
|
||||
.or_else(|_| std::env::var("AUTHENTIK_TOKEN_URL"))
|
||||
.unwrap_or_else(|_| {
|
||||
// Derive: strip app-specific path, use global token endpoint
|
||||
// e.g., https://authentik.riotpiao.com/application/o/memory-agent/
|
||||
// -> https://authentik.riotpiao.com/application/o/token/
|
||||
if let Some(base) = self.issuer_url.rfind("/o/") {
|
||||
format!("{}/o/token/", &self.issuer_url[..base])
|
||||
} else {
|
||||
format!("{}/token/", self.issuer_url.trim_end_matches('/'))
|
||||
}
|
||||
});
|
||||
|
||||
let params = [
|
||||
("grant_type", "client_credentials"),
|
||||
("client_id", &self.client_id),
|
||||
("client_secret", &self.client_secret),
|
||||
("scope", "openid roles"),
|
||||
];
|
||||
|
||||
let response = client
|
||||
|
||||
Reference in New Issue
Block a user