feat: LLM entity + fact extraction pipeline (Zep paper alignment) (#48)

## Changes ### Entity Extraction - Switch from WikiLinkFallbackExtractor to LlmEntityExtractor when LLM_ENDPOINT set - `clean_llm_response()`: strips `<think>` tags, markdown fences, extracts JSON - Handle array responses (Ollama returns `[...]` not `{entities: [...]}`) - EntityType custom Deserialize: unknown variants → Unknown (no crash) - Increase timeout 30s→90s, max_tokens 500→1500 for reasoning models - Graceful reflection fallback: keep entities if verification fails ### Fact Extraction (NEW) - LlmFactExtractor: LLM-based relationship extraction between entity pairs - Validates source/target against known entity list (drops hallucinated edges) - Same robust JSON cleaning for reasoning models + Ollama - IngestWorker auto-selects LLM vs Simple based on LLM_ENDPOINT env ### K8s Deployment - Add `command: ["/app/mem"]` (fix args replacing CMD) - Add LLM_ENDPOINT, LLM_MODEL env vars for in-cluster LLM ## E2E Tested (local Ollama qwen2.5:3b) - 12 entities extracted (person, tool, concept, organization) - 5 edges with relationships and facts - 781 tests pass ## Zep Paper Alignment (§2.2) - Entity extraction + resolution (§2.2.1) - Fact extraction between entity pairs (§2.2.2) - Temporal edge invalidation ready (t_valid/t_invalid schema) - Reflection verification (§2.2.1, graceful fallback) --------- Co-authored-by: rock <[email protected]> Reviewed-on: #48 Co-authored-by: poimen <[email protected]>
2026-09-11 01:11:15 +00:00
parent 6b18d81421
commit fb61de6b47
19 changed files with 859 additions and 268 deletions
@@ -20,7 +20,6 @@ data:
  # OpenSearch
  OPENSEARCH_HOST: "opensearch.poimen.svc.cluster.local:9200"
  # Obsidian
-  OBSIDIAN_URL: "http://obsidian-server.poimen.svc.cluster.local:8080"
  # LLM Configuration (for entity extraction)
  LLM_ENDPOINT: "http://api-internal.riotpiao.com:8000/v1/chat/completions"
  LLM_MODEL: "qwen:7b"
@@ -1,6 +1,6 @@
 # Poimen Memory API Server
-# Serves 7 HTTP endpoints for memory ingest, query, and management.
-# Connects to memory-db (pgvector) for persistent storage.
+# Serves HTTP endpoints for memory ingest, query, visualization.
+# Connects to memory-db (pgvector) + api.riotpiao.com (LLM via Authentik JWT).
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -60,13 +60,43 @@ spec:
                  key: password
            - name: DATABASE_URL
              value: "postgresql://$(DATABASE_USER):$(DATABASE_PASSWORD)@$(DATABASE_HOST):$(DATABASE_PORT)/$(DATABASE_NAME)?sslmode=disable"
-            # LLM Gateway API key
+
+            # LLM via api.riotpiao.com (Authentik JWT auth)
+            - name: LLM_ENDPOINT
+              value: "https://api.riotpiao.com/v1/chat/completions"
+            - name: LLM_API_BASE
+              value: "https://api.riotpiao.com/v1"
+            - name: LLM_MODEL
+              value: "ornith:35b"
+
+            # Authentik service account (memory-agent-oidc secret)
+            - name: AUTHENTIK_ISSUER
+              valueFrom:
+                secretKeyRef:
+                  name: memory-agent-oidc
+                  key: ISSUER
+            - name: AUTHENTIK_CLIENT_ID
+              valueFrom:
+                secretKeyRef:
+                  name: memory-agent-oidc
+                  key: CLIENT_ID
+            - name: AUTHENTIK_CLIENT_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: memory-agent-oidc
+                  key: CLIENT_SECRET
+            - name: TOKEN_URL
+              valueFrom:
+                secretKeyRef:
+                  name: memory-agent-oidc
+                  key: TOKEN_URL
+
+            # Server config
            - name: MEM_API_KEY
              valueFrom:
                secretKeyRef:
                  name: poimen-memory-secrets
                  key: llm-api-key
-            # Server config (from ConfigMap)
            - name: MEM_PORT
              value: "8080"
            - name: MEM_HOME
@@ -74,10 +104,7 @@ spec:
          envFrom:
            - configMapRef:
                name: poimen-memory-config
-            - secretRef:
-                name: poimen-memory-auth
-            - secretRef:
-                name: poimen-memory-secrets
+          command: ["/app/mem"]
          args:
            - serve
            - --port
@@ -110,7 +137,6 @@ spec:
        - name: tmp
          emptyDir:
            sizeLimit: 64Mi
-      # Tolerate control-plane nodes
      tolerations:
        - key: node-role.kubernetes.io/control-plane
          operator: Exists
@@ -6,7 +6,7 @@ resources:
  - deployment.yaml
  - service.yaml
  - config.yaml
-  - obsidian.yaml
+  # obsidian.yaml retired — reference docs now via memory graph
  # Legacy secret managed separately
  # - secrets.yaml
 generators:
@@ -1,159 +0,0 @@
---
-# Obsidian server deployment
-# Serves local vault with web UI and API
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: obsidian-server
-  namespace: poimen
-  labels:
-    app.kubernetes.io/name: obsidian-server
-    app.kubernetes.io/part-of: poimen-memory
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: obsidian-server
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: obsidian-server
-        app.kubernetes.io/part-of: poimen-memory
-    spec:
-      serviceAccountName: obsidian-server
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 1000
-        runAsGroup: 1000
-        fsGroup: 1000
-        seccompProfile:
-          type: RuntimeDefault
-      initContainers:
-      - name: git-sync-init
-        image: alpine/git:latest
-        securityContext:
-          runAsNonRoot: false
-          runAsUser: 0
-          allowPrivilegeEscalation: false
-          capabilities:
-            drop:
-              - ALL
-            add:
-              - CHOWN
-              - DAC_OVERRIDE
-        command:
-        - sh
-        - -c
-        - |
-          export GIT_SSH_COMMAND="ssh -i /root/.ssh/id_ed25519 -o StrictHostKeyChecking=no"
-          git config --global --add safe.directory /vault
-          if [ -d /vault/.git ]; then
-            cd /vault && git pull origin main || true
-          else
-            # Clone into temp, move contents into vault
-            rm -rf /tmp/repo
-            git clone ssh://[email protected]:2222/rock/poimen-obesdient-memory.git /tmp/repo
-            cp -a /tmp/repo/. /vault/
-            rm -rf /tmp/repo
-          fi
-          chown -R 1000:1000 /vault
-        volumeMounts:
-        - name: vault
-          mountPath: /vault
-        - name: ssh-key
-          mountPath: /root/.ssh
-          readOnly: true
-      containers:
-      - name: obsidian-server
-        image: ppatlabs/obsidian:latest
-        imagePullPolicy: IfNotPresent
-        securityContext:
-          allowPrivilegeEscalation: false
-          capabilities:
-            drop:
-              - ALL
-        ports:
-        - name: http
-          containerPort: 27124
-          protocol: TCP
-        env:
-        - name: VAULT_NAME
-          value: poimen-vault
-        - name: VAULT_PATH
-          value: /vault
-        - name: REST_API_ENABLED
-          value: "true"
-        - name: REST_API_PORT
-          value: "8080"
-        volumeMounts:
-        - name: vault
-          mountPath: /vault
-        - name: config
-          mountPath: /config
-        resources:
-          requests:
-            cpu: 100m
-            memory: 256Mi
-          limits:
-            cpu: 500m
-            memory: 512Mi
-        livenessProbe:
-          httpGet:
-            path: /
-            port: http
-            scheme: HTTPS
-          initialDelaySeconds: 30
-          periodSeconds: 10
-          timeoutSeconds: 5
-        readinessProbe:
-          httpGet:
-            path: /
-            port: http
-            scheme: HTTPS
-          initialDelaySeconds: 15
-          periodSeconds: 5
-          timeoutSeconds: 5
-      volumes:
-      - name: vault
-        persistentVolumeClaim:
-          claimName: obsidian-vault
-      - name: config
-        emptyDir: {}
-      - name: ssh-key
-        secret:
-          secretName: obsidian-git-ssh
-          defaultMode: 0400
-
-# PVC managed by homelab repo (k8s/infra/databases/obsidian-vault-pvc.yaml)
-
---
-# Service for Obsidian server
-apiVersion: v1
-kind: Service
-metadata:
-  name: obsidian-server
-  namespace: poimen
-  labels:
-    app.kubernetes.io/name: obsidian-server
-spec:
-  type: ClusterIP
-  ports:
-  - name: http
-    port: 80
-    targetPort: 27124
-    protocol: TCP
-  selector:
-    app.kubernetes.io/name: obsidian-server
-
---
-# ServiceAccount for Obsidian
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: obsidian-server
-  namespace: poimen
-  labels:
-    app.kubernetes.io/name: obsidian-server
-
-# Ingress managed by homelab repo (obsidian.riotpiao.com)
-# See: homelab/k8s/bootstrap/ingress/ingress.yaml
@@ -1,8 +1,6 @@
---
-# CNPG Postgres cluster for Poimen Memory system (GitOps, declarative extensions).
-# 2 instances, pgvector 0.7.0 via spec.extensions (not manual CREATE EXTENSION).
-# Storage: 10Gi longhorn, consistent with temporal-db.yaml.
-# No manual psql needed — all via git/ArgoCD.
+# Dedicated CNPG Postgres for Poimen Memory (GitOps, wave 2).
+# Matches homelab/k8s/infra/databases/memory-db.yaml — single source of truth.
+# CNPG generates secret `memory-db-app` + service `memory-db-rw` in ns poimen.
 apiVersion: postgresql.cnpg.io/v1
 kind: Cluster
 metadata:
@@ -13,20 +11,6 @@ metadata:
 spec:
  instances: 2
  imageName: ghcr.io/cloudnative-pg/postgresql:16.2
-  enableSuperuserAccess: false
-  storage:
-    size: 10Gi
-    storageClass: longhorn
-  resources:
-    requests: { memory: "512Mi", cpu: "250m" }
-    limits: { memory: "2Gi", cpu: "1" }
-  affinity:
-    podAntiAffinityType: preferred
-    topologyKey: kubernetes.io/hostname
-    tolerations:
-      - key: node-role.kubernetes.io/control-plane
-        operator: Exists
-        effect: NoSchedule
  bootstrap:
    initdb:
      database: memory
@@ -34,25 +18,19 @@ spec:
      encoding: UTF8
      localeCollate: C
      localeCType: C
-  monitoring:
-    enabled: true
-    podMonitorTemplate:
-      spec:
-        interval: 30s
-        scrapeTimeout: 10s
---
-# Database resource with pgvector extension (declarative, git-managed).
-# CNPG 1.30.0+ supports this via spec.extensions on the Database CRD.
-# Ensures pgvector is installed and available for HNSW indexing.
-apiVersion: postgresql.cnpg.io/v1
-kind: Database
-metadata:
-  name: memory
-  namespace: poimen
-spec:
-  cluster:
-    name: memory-db
-  owner: app
-  extensions:
-    - name: vector
-      ensure: present
+      postInitApplicationSQL:
+        - "CREATE EXTENSION vector;"
+  enableSuperuserAccess: false
+  resources:
+    requests: { memory: "512Mi", cpu: "250m" }
+    limits: { memory: "2Gi", cpu: "1" }
+  storage:
+    size: 20Gi
+    storageClass: longhorn
+  affinity:
+    podAntiAffinityType: preferred
+    topologyKey: kubernetes.io/hostname
+    tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+        effect: NoSchedule