feat: LLM entity + fact extraction pipeline (Zep paper alignment) (#48)
## Changes
### Entity Extraction
- Switch from WikiLinkFallbackExtractor to LlmEntityExtractor when LLM_ENDPOINT set
- `clean_llm_response()`: strips `<think>` tags, markdown fences, extracts JSON
- Handle array responses (Ollama returns `[...]` not `{entities: [...]}`)
- EntityType custom Deserialize: unknown variants → Unknown (no crash)
- Increase timeout 30s→90s, max_tokens 500→1500 for reasoning models
- Graceful reflection fallback: keep entities if verification fails
### Fact Extraction (NEW)
- LlmFactExtractor: LLM-based relationship extraction between entity pairs
- Validates source/target against known entity list (drops hallucinated edges)
- Same robust JSON cleaning for reasoning models + Ollama
- IngestWorker auto-selects LLM vs Simple based on LLM_ENDPOINT env
### K8s Deployment
- Add `command: ["/app/mem"]` (fix args replacing CMD)
- Add LLM_ENDPOINT, LLM_MODEL env vars for in-cluster LLM
## E2E Tested (local Ollama qwen2.5:3b)
- 12 entities extracted (person, tool, concept, organization)
- 5 edges with relationships and facts
- 781 tests pass
## Zep Paper Alignment (§2.2)
- Entity extraction + resolution (§2.2.1)
- Fact extraction between entity pairs (§2.2.2)
- Temporal edge invalidation ready (t_valid/t_invalid schema)
- Reflection verification (§2.2.1, graceful fallback)
---------
Co-authored-by: rock <[email protected]>
Reviewed-on: #48
Co-authored-by: poimen <[email protected]>
This commit was merged in pull request #48.
This commit is contained in:
@@ -20,7 +20,6 @@ data:
|
||||
# OpenSearch
|
||||
OPENSEARCH_HOST: "opensearch.poimen.svc.cluster.local:9200"
|
||||
# Obsidian
|
||||
OBSIDIAN_URL: "http://obsidian-server.poimen.svc.cluster.local:8080"
|
||||
# LLM Configuration (for entity extraction)
|
||||
LLM_ENDPOINT: "http://api-internal.riotpiao.com:8000/v1/chat/completions"
|
||||
LLM_MODEL: "qwen:7b"
|
||||
|
||||
+35
-9
@@ -1,6 +1,6 @@
|
||||
# Poimen Memory API Server
|
||||
# Serves 7 HTTP endpoints for memory ingest, query, and management.
|
||||
# Connects to memory-db (pgvector) for persistent storage.
|
||||
# Serves HTTP endpoints for memory ingest, query, visualization.
|
||||
# Connects to memory-db (pgvector) + api.riotpiao.com (LLM via Authentik JWT).
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
@@ -60,13 +60,43 @@ spec:
|
||||
key: password
|
||||
- name: DATABASE_URL
|
||||
value: "postgresql://$(DATABASE_USER):$(DATABASE_PASSWORD)@$(DATABASE_HOST):$(DATABASE_PORT)/$(DATABASE_NAME)?sslmode=disable"
|
||||
# LLM Gateway API key
|
||||
|
||||
# LLM via api.riotpiao.com (Authentik JWT auth)
|
||||
- name: LLM_ENDPOINT
|
||||
value: "https://api.riotpiao.com/v1/chat/completions"
|
||||
- name: LLM_API_BASE
|
||||
value: "https://api.riotpiao.com/v1"
|
||||
- name: LLM_MODEL
|
||||
value: "ornith:35b"
|
||||
|
||||
# Authentik service account (memory-agent-oidc secret)
|
||||
- name: AUTHENTIK_ISSUER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: memory-agent-oidc
|
||||
key: ISSUER
|
||||
- name: AUTHENTIK_CLIENT_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: memory-agent-oidc
|
||||
key: CLIENT_ID
|
||||
- name: AUTHENTIK_CLIENT_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: memory-agent-oidc
|
||||
key: CLIENT_SECRET
|
||||
- name: TOKEN_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: memory-agent-oidc
|
||||
key: TOKEN_URL
|
||||
|
||||
# Server config
|
||||
- name: MEM_API_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: poimen-memory-secrets
|
||||
key: llm-api-key
|
||||
# Server config (from ConfigMap)
|
||||
- name: MEM_PORT
|
||||
value: "8080"
|
||||
- name: MEM_HOME
|
||||
@@ -74,10 +104,7 @@ spec:
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: poimen-memory-config
|
||||
- secretRef:
|
||||
name: poimen-memory-auth
|
||||
- secretRef:
|
||||
name: poimen-memory-secrets
|
||||
command: ["/app/mem"]
|
||||
args:
|
||||
- serve
|
||||
- --port
|
||||
@@ -110,7 +137,6 @@ spec:
|
||||
- name: tmp
|
||||
emptyDir:
|
||||
sizeLimit: 64Mi
|
||||
# Tolerate control-plane nodes
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
|
||||
@@ -6,7 +6,7 @@ resources:
|
||||
- deployment.yaml
|
||||
- service.yaml
|
||||
- config.yaml
|
||||
- obsidian.yaml
|
||||
# obsidian.yaml retired — reference docs now via memory graph
|
||||
# Legacy secret managed separately
|
||||
# - secrets.yaml
|
||||
generators:
|
||||
|
||||
@@ -1,159 +0,0 @@
|
||||
---
|
||||
# Obsidian server deployment
|
||||
# Serves local vault with web UI and API
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: obsidian-server
|
||||
namespace: poimen
|
||||
labels:
|
||||
app.kubernetes.io/name: obsidian-server
|
||||
app.kubernetes.io/part-of: poimen-memory
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: obsidian-server
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: obsidian-server
|
||||
app.kubernetes.io/part-of: poimen-memory
|
||||
spec:
|
||||
serviceAccountName: obsidian-server
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
runAsGroup: 1000
|
||||
fsGroup: 1000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
initContainers:
|
||||
- name: git-sync-init
|
||||
image: alpine/git:latest
|
||||
securityContext:
|
||||
runAsNonRoot: false
|
||||
runAsUser: 0
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
add:
|
||||
- CHOWN
|
||||
- DAC_OVERRIDE
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
export GIT_SSH_COMMAND="ssh -i /root/.ssh/id_ed25519 -o StrictHostKeyChecking=no"
|
||||
git config --global --add safe.directory /vault
|
||||
if [ -d /vault/.git ]; then
|
||||
cd /vault && git pull origin main || true
|
||||
else
|
||||
# Clone into temp, move contents into vault
|
||||
rm -rf /tmp/repo
|
||||
git clone ssh://[email protected]:2222/rock/poimen-obesdient-memory.git /tmp/repo
|
||||
cp -a /tmp/repo/. /vault/
|
||||
rm -rf /tmp/repo
|
||||
fi
|
||||
chown -R 1000:1000 /vault
|
||||
volumeMounts:
|
||||
- name: vault
|
||||
mountPath: /vault
|
||||
- name: ssh-key
|
||||
mountPath: /root/.ssh
|
||||
readOnly: true
|
||||
containers:
|
||||
- name: obsidian-server
|
||||
image: ppatlabs/obsidian:latest
|
||||
imagePullPolicy: IfNotPresent
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 27124
|
||||
protocol: TCP
|
||||
env:
|
||||
- name: VAULT_NAME
|
||||
value: poimen-vault
|
||||
- name: VAULT_PATH
|
||||
value: /vault
|
||||
- name: REST_API_ENABLED
|
||||
value: "true"
|
||||
- name: REST_API_PORT
|
||||
value: "8080"
|
||||
volumeMounts:
|
||||
- name: vault
|
||||
mountPath: /vault
|
||||
- name: config
|
||||
mountPath: /config
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: http
|
||||
scheme: HTTPS
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: http
|
||||
scheme: HTTPS
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 5
|
||||
volumes:
|
||||
- name: vault
|
||||
persistentVolumeClaim:
|
||||
claimName: obsidian-vault
|
||||
- name: config
|
||||
emptyDir: {}
|
||||
- name: ssh-key
|
||||
secret:
|
||||
secretName: obsidian-git-ssh
|
||||
defaultMode: 0400
|
||||
|
||||
# PVC managed by homelab repo (k8s/infra/databases/obsidian-vault-pvc.yaml)
|
||||
|
||||
---
|
||||
# Service for Obsidian server
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: obsidian-server
|
||||
namespace: poimen
|
||||
labels:
|
||||
app.kubernetes.io/name: obsidian-server
|
||||
spec:
|
||||
type: ClusterIP
|
||||
ports:
|
||||
- name: http
|
||||
port: 80
|
||||
targetPort: 27124
|
||||
protocol: TCP
|
||||
selector:
|
||||
app.kubernetes.io/name: obsidian-server
|
||||
|
||||
---
|
||||
# ServiceAccount for Obsidian
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: obsidian-server
|
||||
namespace: poimen
|
||||
labels:
|
||||
app.kubernetes.io/name: obsidian-server
|
||||
|
||||
# Ingress managed by homelab repo (obsidian.riotpiao.com)
|
||||
# See: homelab/k8s/bootstrap/ingress/ingress.yaml
|
||||
Reference in New Issue
Block a user