Compare commits

..
Author SHA1 Message Date
rock 78e7aa8302 feat: scale memory-db to 3 replicas for HA
Sync with homelab/k8s/infra/databases/memory-db.yaml.
Update CNPG Cluster instances from 2 to 3 for high availability.
2026-09-12 04:52:45 +09:00
poimenandrock fb61de6b47 feat: LLM entity + fact extraction pipeline (Zep paper alignment) (#48)
CI / CI (push) Successful in 12m9s
Deploy / Tag & Push Latest (push) Failing after 41s
DB Migration / Run Migrations (push) Failing after 18s
## Changes

### Entity Extraction
- Switch from WikiLinkFallbackExtractor to LlmEntityExtractor when LLM_ENDPOINT set
- `clean_llm_response()`: strips `<think>` tags, markdown fences, extracts JSON
- Handle array responses (Ollama returns `[...]` not `{entities: [...]}`)
- EntityType custom Deserialize: unknown variants → Unknown (no crash)
- Increase timeout 30s→90s, max_tokens 500→1500 for reasoning models
- Graceful reflection fallback: keep entities if verification fails

### Fact Extraction (NEW)
- LlmFactExtractor: LLM-based relationship extraction between entity pairs
- Validates source/target against known entity list (drops hallucinated edges)
- Same robust JSON cleaning for reasoning models + Ollama
- IngestWorker auto-selects LLM vs Simple based on LLM_ENDPOINT env

### K8s Deployment
- Add `command: ["/app/mem"]` (fix args replacing CMD)
- Add LLM_ENDPOINT, LLM_MODEL env vars for in-cluster LLM

## E2E Tested (local Ollama qwen2.5:3b)
- 12 entities extracted (person, tool, concept, organization)
- 5 edges with relationships and facts
- 781 tests pass

## Zep Paper Alignment (§2.2)
- Entity extraction + resolution (§2.2.1)
- Fact extraction between entity pairs (§2.2.2)
- Temporal edge invalidation ready (t_valid/t_invalid schema)
- Reflection verification (§2.2.1, graceful fallback)

---------

Co-authored-by: rock <[email protected]>
Reviewed-on: #48
Co-authored-by: poimen <[email protected]>
2026-09-11 01:11:15 +00:00
4 changed files with 49 additions and 78 deletions
+2 -4
View File
@@ -15,10 +15,8 @@ jobs:
name: Tag & Push Latest
runs-on: rust
steps:
- name: Install Node.js and Docker
run: |
apt-get update
apt-get install -y nodejs docker.io
- name: Install Docker
run: apt-get update && apt-get install -y docker.io
- name: Checkout code
uses: actions/checkout@v4
+43 -56
View File
@@ -11,79 +11,66 @@ env:
DB_HOST: memory-db-rw.poimen.svc.cluster.local
DB_PORT: "5432"
DB_NAME: memory
MIGRATIONS_DIR: crates/mem-store/migrations
DOCKER_HOST: tcp://localhost:2375
jobs:
migrate:
name: Run Migrations
runs-on: rust
steps:
- name: Install Node.js, Docker, and psql
run: |
apt-get update
apt-get install -y nodejs docker.io postgresql-client
- name: Install psql
run: apt-get update && apt-get install -y postgresql-client
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 2
- name: Detect changed migrations
id: detect
- name: Fetch previous migrations state
run: |
CHANGED=$(git diff --name-only HEAD~1 HEAD -- "$MIGRATIONS_DIR"/*.sql 2>/dev/null || echo "")
if [ -n "$CHANGED" ]; then
echo "files=$CHANGED" >> $GITHUB_OUTPUT
echo "found=true" >> $GITHUB_OUTPUT
echo "Changed: $CHANGED"
else
echo "found=false" >> $GITHUB_OUTPUT
echo "No migration changes detected"
fi
git fetch origin main --depth=2
# List changed migration files
CHANGED=$(git diff --name-only HEAD~1 HEAD -- crates/mem-store/migrations/ || echo "")
echo "Changed migrations: $CHANGED"
echo "CHANGED_MIGRATIONS=$CHANGED" >> $GITHUB_ENV
- name: Apply changed migrations (push)
if: github.event_name == 'push' && steps.detect.outputs.found == 'true'
env:
PGHOST: ${{ env.DB_HOST }}
PGPORT: ${{ env.DB_PORT }}
PGDATABASE: ${{ env.DB_NAME }}
PGUSER: ${{ secrets.DB_USER }}
PGPASSWORD: ${{ secrets.DB_PASSWORD }}
- name: Run migrations
if: env.CHANGED_MIGRATIONS != ''
run: |
for f in ${{ steps.detect.outputs.files }}; do
[ -f "$f" ] || continue
echo "=== Applying: $f ==="
psql -v ON_ERROR_STOP=1 -f "$f"
echo "=== OK ==="
export PGPASSWORD="${DB_PASSWORD}"
echo "=== Running changed migrations ==="
for f in $CHANGED_MIGRATIONS; do
if [ -f "$f" ]; then
echo "--- Applying: $f ---"
psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f "$f" 2>&1
if [ $? -ne 0 ]; then
echo "ERROR: Migration $f failed!"
exit 1
fi
echo "--- OK: $f ---"
fi
done
- name: Apply all migrations (dispatch)
echo "=== Verify schema ==="
psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "\dt memory*"
env:
DB_USER: ${{ secrets.DB_USER }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
- name: Run all migrations (manual trigger)
if: github.event_name == 'workflow_dispatch'
env:
PGHOST: ${{ env.DB_HOST }}
PGPORT: ${{ env.DB_PORT }}
PGDATABASE: ${{ env.DB_NAME }}
PGUSER: ${{ secrets.DB_USER }}
PGPASSWORD: ${{ secrets.DB_PASSWORD }}
run: |
for f in $(ls "$MIGRATIONS_DIR"/*.sql | sort); do
echo "=== Applying: $f ==="
psql -v ON_ERROR_STOP=1 -f "$f" || true
echo "=== Done ==="
export PGPASSWORD="${DB_PASSWORD}"
echo "=== Running all migrations in order ==="
for f in $(ls crates/mem-store/migrations/*.sql | sort); do
echo "--- Applying: $f ---"
psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f "$f" 2>&1 || true
echo "--- Done: $f ---"
done
- name: Verify schema
echo "=== Final schema ==="
psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "\dt memory*"
psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "\d memory_entity"
psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "\d memory_edge"
env:
PGHOST: ${{ env.DB_HOST }}
PGPORT: ${{ env.DB_PORT }}
PGDATABASE: ${{ env.DB_NAME }}
PGUSER: ${{ secrets.DB_USER }}
PGPASSWORD: ${{ secrets.DB_PASSWORD }}
run: |
echo "=== Tables ==="
psql -c "\dt memory*"
echo "=== Entity Schema ==="
psql -c "\d memory_entity"
echo "=== Edge Schema ==="
psql -c "\d memory_edge"
DB_USER: ${{ secrets.DB_USER }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
+3 -17
View File
@@ -1342,29 +1342,15 @@ async fn query_temporal_graph(
state: &web::Data<AppState>,
params: &QueryParams,
) -> anyhow::Result<serde_json::Value> {
// Step 1: Find entities matching question (fuzzy name/description search)
// Step 1: Find entities (order by name for deterministic results)
let entities_rows: Vec<(String, String, String)> = sqlx::query_as(
"SELECT id, name, entity_type FROM memory_entity
WHERE project_id = $1
AND (name ILIKE '%' || $2 || '%' OR description ILIKE '%' || $2 || '%')
ORDER BY confidence DESC
LIMIT $3"
"SELECT id, name, entity_type FROM memory_entity WHERE project_id = $1 LIMIT $2"
)
.bind(&params.project)
.bind(&params.question)
.bind(params.limit as i32)
.fetch_all(&state.pool)
.await
.unwrap_or_default();
tracing::info!(
target: "observability",
event = "query_entity_search",
project = %params.project,
question = %params.question,
matched = entities_rows.len(),
"Entity search complete"
);
// Step 2: Traverse edges from found entities
// NOTE: Edges will be empty until temporal schema is migrated
@@ -1374,7 +1360,7 @@ async fn query_temporal_graph(
for (entity_id, _name, _type_str) in &entities_rows {
let entity_edges: Vec<(String, String, String, String, f32, Option<chrono::DateTime<chrono::Utc>>, Option<chrono::DateTime<chrono::Utc>>)> =
sqlx::query_as(
"SELECT id, target_id, relation_type, fact, confidence, t_valid, t_invalid FROM memory_edge WHERE project_id = $1 AND source_id = $2"
"SELECT id, target_entity_id, relation_type, fact, confidence, t_valid, t_invalid FROM memory_edge WHERE project_id = $1 AND source_entity_id = $2"
)
.bind(&params.project)
.bind(entity_id)
+1 -1
View File
@@ -9,7 +9,7 @@ metadata:
annotations:
argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
spec:
instances: 2
instances: 3
imageName: ghcr.io/cloudnative-pg/postgresql:16.2
bootstrap:
initdb: