feat: LLM entity + fact extraction pipeline (Zep paper alignment) (#48)
## Changes
### Entity Extraction
- Switch from WikiLinkFallbackExtractor to LlmEntityExtractor when LLM_ENDPOINT set
- `clean_llm_response()`: strips `<think>` tags, markdown fences, extracts JSON
- Handle array responses (Ollama returns `[...]` not `{entities: [...]}`)
- EntityType custom Deserialize: unknown variants → Unknown (no crash)
- Increase timeout 30s→90s, max_tokens 500→1500 for reasoning models
- Graceful reflection fallback: keep entities if verification fails
### Fact Extraction (NEW)
- LlmFactExtractor: LLM-based relationship extraction between entity pairs
- Validates source/target against known entity list (drops hallucinated edges)
- Same robust JSON cleaning for reasoning models + Ollama
- IngestWorker auto-selects LLM vs Simple based on LLM_ENDPOINT env
### K8s Deployment
- Add `command: ["/app/mem"]` (fix args replacing CMD)
- Add LLM_ENDPOINT, LLM_MODEL env vars for in-cluster LLM
## E2E Tested (local Ollama qwen2.5:3b)
- 12 entities extracted (person, tool, concept, organization)
- 5 edges with relationships and facts
- 781 tests pass
## Zep Paper Alignment (§2.2)
- Entity extraction + resolution (§2.2.1)
- Fact extraction between entity pairs (§2.2.2)
- Temporal edge invalidation ready (t_valid/t_invalid schema)
- Reflection verification (§2.2.1, graceful fallback)
---------
Co-authored-by: rock <[email protected]>
Reviewed-on: #48
Co-authored-by: poimen <[email protected]>
This commit was merged in pull request #48.
This commit is contained in:
@@ -0,0 +1,67 @@
|
||||
-- Migration 009: Temporal edge schema (Zep paper §2.2.2)
|
||||
-- Replaces old memory_edge (child_sha/parent_sha node graph)
|
||||
-- with temporal edge schema supporting relation types, facts, and validity periods.
|
||||
-- Idempotent: safe to run multiple times.
|
||||
|
||||
-- Rename old table if it still exists (skip if already migrated)
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'memory_edge'
|
||||
AND EXISTS (SELECT 1 FROM information_schema.columns
|
||||
WHERE table_name = 'memory_edge' AND column_name = 'child_sha'))
|
||||
THEN
|
||||
ALTER TABLE memory_edge RENAME TO memory_edge_legacy;
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
-- Create temporal edge table
|
||||
CREATE TABLE IF NOT EXISTS memory_edge (
|
||||
id TEXT PRIMARY KEY,
|
||||
project_id TEXT NOT NULL DEFAULT 'default',
|
||||
source_id TEXT NOT NULL,
|
||||
target_id TEXT NOT NULL,
|
||||
relation_type TEXT NOT NULL DEFAULT '',
|
||||
fact TEXT NOT NULL DEFAULT '',
|
||||
weight REAL NOT NULL DEFAULT 1.0,
|
||||
strength REAL DEFAULT 1.0,
|
||||
confidence REAL DEFAULT 0.8,
|
||||
t_valid TIMESTAMPTZ,
|
||||
t_invalid TIMESTAMPTZ,
|
||||
t_created TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
t_expired TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
episode_id TEXT,
|
||||
deleted_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
-- Ensure app user owns the table
|
||||
DO $$ BEGIN
|
||||
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'app') THEN
|
||||
ALTER TABLE memory_edge OWNER TO app;
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_memory_edge_source ON memory_edge(source_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_memory_edge_target ON memory_edge(target_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_memory_edge_project ON memory_edge(project_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_memory_edge_relation ON memory_edge(relation_type);
|
||||
|
||||
-- Ensure memory_entity has all columns code expects
|
||||
ALTER TABLE memory_entity ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ;
|
||||
ALTER TABLE memory_entity ADD COLUMN IF NOT EXISTS source_count INTEGER DEFAULT 1;
|
||||
|
||||
-- Unique constraint for entity upsert dedup
|
||||
DO $$
|
||||
BEGIN
|
||||
-- Dedup existing rows before creating unique index
|
||||
DELETE FROM memory_entity a USING memory_entity b
|
||||
WHERE a.project_id = b.project_id AND a.name = b.name
|
||||
AND a.t_created < b.t_created;
|
||||
EXCEPTION WHEN OTHERS THEN NULL;
|
||||
END $$;
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_memory_entity_project_name ON memory_entity(project_id, name);
|
||||
|
||||
-- ROLLBACK instructions:
|
||||
-- DROP TABLE IF EXISTS memory_edge;
|
||||
-- ALTER TABLE IF EXISTS memory_edge_legacy RENAME TO memory_edge;
|
||||
Reference in New Issue
Block a user