Compare commits
13
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
90f21f3db3 | ||
|
|
88234ac927 | ||
|
|
168fd41fd2 | ||
|
|
16e3ff16f1 | ||
|
|
800d9d8ae2 | ||
|
|
88027b1a72 | ||
|
|
52b037f788 | ||
|
|
25dde42ea4 | ||
|
|
e6e67408cd | ||
|
|
02fe15726a | ||
|
|
a0cb3f9211 | ||
|
|
b564ad2a66 | ||
|
|
83e3206dcd |
+8
-51
@@ -1,55 +1,12 @@
|
|||||||
# Git
|
|
||||||
.git
|
.git
|
||||||
.gitignore
|
.gitignore
|
||||||
.gitattributes
|
|
||||||
|
|
||||||
# CI/CD
|
|
||||||
.github
|
|
||||||
.gitea
|
|
||||||
.gitlab-ci.yml
|
|
||||||
|
|
||||||
# Kubernetes
|
|
||||||
k8s/
|
|
||||||
helm/
|
|
||||||
|
|
||||||
# Documentation
|
|
||||||
*.md
|
*.md
|
||||||
docs/
|
__pycache__
|
||||||
|
*.pyc
|
||||||
# IDE
|
|
||||||
.vscode
|
|
||||||
.idea
|
|
||||||
*.swp
|
|
||||||
*.swo
|
|
||||||
*~
|
|
||||||
|
|
||||||
# OS
|
|
||||||
.DS_Store
|
|
||||||
Thumbs.db
|
|
||||||
|
|
||||||
# Build artifacts
|
|
||||||
target/
|
|
||||||
dist/
|
|
||||||
build/
|
|
||||||
|
|
||||||
# Dependencies (will be downloaded fresh)
|
|
||||||
.cargo/
|
|
||||||
Cargo.lock.bak
|
|
||||||
|
|
||||||
# Testing
|
|
||||||
.coverage
|
|
||||||
coverage/
|
|
||||||
|
|
||||||
# Secrets
|
|
||||||
.env
|
|
||||||
.env.local
|
.env.local
|
||||||
.env.*.local
|
.venv
|
||||||
|
venv/
|
||||||
# Archives
|
.pytest_cache
|
||||||
*.tar
|
.coverage
|
||||||
*.tar.gz
|
htmlcov
|
||||||
*.zip
|
.DS_Store
|
||||||
|
|
||||||
# Node (if any)
|
|
||||||
node_modules/
|
|
||||||
*.log
|
|
||||||
|
|||||||
@@ -1,19 +0,0 @@
|
|||||||
MEM_AUTH_MODE=none
|
|
||||||
MEM_RATE_LIMIT_INGEST=1000
|
|
||||||
MEM_RATE_LIMIT_QUERY=10000
|
|
||||||
MEM_IDEMPOTENCY_TTL_SECS=86400
|
|
||||||
MEM_EMBEDDING_BATCH_SIZE=4
|
|
||||||
|
|
||||||
DATABASE_URL=postgresql://app:***REMOVED***@127.0.0.1:5433/memory
|
|
||||||
|
|
||||||
# Embedding via direct port-forward (skip gateway auth)
|
|
||||||
LLM_ENDPOINT=http://localhost:9090/v1/chat/completions
|
|
||||||
LLM_API_BASE=http://localhost:9090
|
|
||||||
LLM_MODEL=nomic-ai/nomic-embed-text-v2-moe
|
|
||||||
LLM_TIMEOUT_SECS=60
|
|
||||||
ENABLE_LLM_EXTRACTION=true
|
|
||||||
EMBEDDINGS_MODEL=nomic-ai/nomic-embed-text-v2-moe
|
|
||||||
|
|
||||||
MEM_PORT=8081
|
|
||||||
MEM_API_KEY=test-key
|
|
||||||
MEM_HOME=/tmp
|
|
||||||
@@ -1,50 +0,0 @@
|
|||||||
# Local development environment (.env file)
|
|
||||||
# Copy to .env and fill in your local/dev URLs
|
|
||||||
# .env is gitignored - never commit
|
|
||||||
|
|
||||||
# Auth mode: jwt | apikey | none
|
|
||||||
MEM_AUTH_MODE=none
|
|
||||||
|
|
||||||
# Rate limiting
|
|
||||||
MEM_RATE_LIMIT_INGEST=1000
|
|
||||||
MEM_RATE_LIMIT_QUERY=10000
|
|
||||||
MEM_IDEMPOTENCY_TTL_SECS=86400
|
|
||||||
|
|
||||||
# Embeddings
|
|
||||||
MEM_EMBEDDING_BATCH_SIZE=32
|
|
||||||
|
|
||||||
# Database (local or remote)
|
|
||||||
DATABASE_URL=postgresql://user:password@localhost:5432/memory
|
|
||||||
|
|
||||||
# Downstream services - point to your local/dev endpoints
|
|
||||||
|
|
||||||
# LLM Service (entity extraction, fact extraction)
|
|
||||||
LLM_ENDPOINT=http://localhost:11434/v1/chat/completions
|
|
||||||
LLM_API_BASE=http://localhost:11434/v1
|
|
||||||
LLM_MODEL=qwen:7b
|
|
||||||
LLM_TIMEOUT_SECS=60
|
|
||||||
ENABLE_LLM_EXTRACTION=true
|
|
||||||
|
|
||||||
# OpenSearch (vector store, BM25)
|
|
||||||
OPENSEARCH_HOST=localhost:9200
|
|
||||||
OPENSEARCH_SCHEME=http
|
|
||||||
OPENSEARCH_VERIFY_CERTS=false
|
|
||||||
|
|
||||||
# Authentik (OIDC - optional for local dev)
|
|
||||||
AUTHENTIK_ISSUER=https://authentik.riotpiao.com/application/o/poimen/
|
|
||||||
AUTHENTIK_CLIENT_ID=
|
|
||||||
AUTHENTIK_CLIENT_SECRET=
|
|
||||||
TOKEN_URL=https://authentik.riotpiao.com/application/o/token/
|
|
||||||
AUTHENTIK_VERIFY_SSL=false
|
|
||||||
|
|
||||||
# Temporal (workflow orchestration - future)
|
|
||||||
TEMPORAL_ENDPOINT=localhost:7233
|
|
||||||
TEMPORAL_NAMESPACE=poimen
|
|
||||||
|
|
||||||
# API Gateway (route optimization - future)
|
|
||||||
GATEWAY_URL=http://localhost:8080
|
|
||||||
|
|
||||||
# Server config
|
|
||||||
MEM_PORT=8080
|
|
||||||
MEM_API_KEY=test-key
|
|
||||||
MEM_HOME=/tmp
|
|
||||||
+17
-137
@@ -18,15 +18,6 @@ jobs:
|
|||||||
name: CI
|
name: CI
|
||||||
runs-on: rust
|
runs-on: rust
|
||||||
steps:
|
steps:
|
||||||
- name: Clean disk space (runner GC)
|
|
||||||
run: |
|
|
||||||
df -h /
|
|
||||||
echo "Cleaning docker, cargo cache..."
|
|
||||||
docker system prune -af --volumes || true
|
|
||||||
rm -rf ~/.cargo/registry/cache ~/.cargo/registry/index ~/.cargo/git || true
|
|
||||||
rm -rf /tmp/* || true
|
|
||||||
df -h /
|
|
||||||
|
|
||||||
- name: Install Node.js and Docker
|
- name: Install Node.js and Docker
|
||||||
run: |
|
run: |
|
||||||
apt-get update
|
apt-get update
|
||||||
@@ -35,11 +26,14 @@ jobs:
|
|||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: Cargo build, test, clippy (single compile pass)
|
- name: Cargo build all
|
||||||
run: |
|
run: cargo build --all --verbose
|
||||||
cargo build --all --verbose
|
|
||||||
cargo test --all --lib --verbose 2>&1 | tail -150 || true
|
- name: Cargo test all
|
||||||
cargo clippy --all --all-targets -- -D warnings 2>&1 | tail -50 || true
|
run: cargo test --all --lib --verbose 2>&1 | tail -150 || true
|
||||||
|
|
||||||
|
- name: Cargo clippy
|
||||||
|
run: cargo clippy --all --all-targets -- -D warnings 2>&1 | tail -50 || true
|
||||||
|
|
||||||
- name: Get short SHA
|
- name: Get short SHA
|
||||||
id: sha
|
id: sha
|
||||||
@@ -47,139 +41,25 @@ jobs:
|
|||||||
|
|
||||||
- name: Registry login
|
- name: Registry login
|
||||||
run: |
|
run: |
|
||||||
if [ -z "${REGISTRY_USER}" ] || [ -z "${REGISTRY_TOKEN}" ]; then
|
|
||||||
echo "ERROR: Missing REGISTRY_USER or REGISTRY_TOKEN secrets"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
echo "${REGISTRY_TOKEN}" | docker login "${REGISTRY}" \
|
echo "${REGISTRY_TOKEN}" | docker login "${REGISTRY}" \
|
||||||
--username "${REGISTRY_USER}" --password-stdin
|
--username "${REGISTRY_USER}" --password-stdin
|
||||||
env:
|
env:
|
||||||
REGISTRY_USER: ${{ secrets.FORGEJO_REGISTRY_USER }}
|
REGISTRY_USER: ${{ secrets.FORGEJO_REGISTRY_USER }}
|
||||||
REGISTRY_TOKEN: ${{ secrets.FORGEJO_REGISTRY_TOKEN }}
|
REGISTRY_TOKEN: ${{ secrets.FORGEJO_REGISTRY_TOKEN }}
|
||||||
|
|
||||||
- name: Clean cargo before Docker build
|
- name: Build Docker image
|
||||||
run: |
|
|
||||||
cargo clean || true
|
|
||||||
rm -rf ~/.cargo/registry/cache ~/.cargo/registry/index ~/.cargo/git || true
|
|
||||||
df -h /
|
|
||||||
|
|
||||||
- name: Build and push Docker image (SHA tag only)
|
|
||||||
run: |
|
run: |
|
||||||
docker build --no-cache --progress=plain \
|
docker build --no-cache --progress=plain \
|
||||||
-t "${IMAGE}:${{ steps.sha.outputs.short_sha }}" \
|
-t "${IMAGE}:${{ steps.sha.outputs.short_sha }}" \
|
||||||
|
-t "${IMAGE}:latest" \
|
||||||
-f Dockerfile .
|
-f Dockerfile .
|
||||||
|
|
||||||
|
- name: Push Docker image
|
||||||
|
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
|
||||||
|
run: |
|
||||||
docker push "${IMAGE}:${{ steps.sha.outputs.short_sha }}"
|
docker push "${IMAGE}:${{ steps.sha.outputs.short_sha }}"
|
||||||
echo "Pushed: ${IMAGE}:${{ steps.sha.outputs.short_sha }}"
|
|
||||||
|
|
||||||
- name: Install kubectl
|
|
||||||
run: |
|
|
||||||
apt-get update
|
|
||||||
apt-get install -y curl
|
|
||||||
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
|
|
||||||
chmod +x kubectl
|
|
||||||
mv kubectl /usr/local/bin/
|
|
||||||
|
|
||||||
- name: Setup kubeconfig for Tekton
|
|
||||||
run: |
|
|
||||||
mkdir -p ~/.kube
|
|
||||||
echo "${KUBECONFIG_B64}" | base64 -d > ~/.kube/config
|
|
||||||
chmod 600 ~/.kube/config
|
|
||||||
kubectl cluster-info 2>&1 | head -3
|
|
||||||
echo "✓ kubeconfig ready"
|
|
||||||
env:
|
|
||||||
KUBECONFIG_B64: ${{ secrets.KUBECONFIG_B64 }}
|
|
||||||
|
|
||||||
- name: Trigger Tekton PipelineRun (CI/CD)
|
|
||||||
id: tekton
|
|
||||||
run: |
|
|
||||||
SHA="${{ steps.sha.outputs.short_sha }}"
|
|
||||||
RUN_NAME="poimen-ci-${SHA}"
|
|
||||||
NAMESPACE="poimen"
|
|
||||||
IMAGE="${REGISTRY}/riotpiao-poimen/poimen-memory:${SHA}"
|
|
||||||
REGISTRY_USER="${{ secrets.FORGEJO_REGISTRY_USER }}"
|
|
||||||
REGISTRY_TOKEN="${{ secrets.FORGEJO_REGISTRY_TOKEN }}"
|
|
||||||
|
|
||||||
echo "Triggering Tekton PipelineRun: ${RUN_NAME}"
|
|
||||||
echo "Image: ${IMAGE}"
|
|
||||||
echo ""
|
|
||||||
|
|
||||||
# Create PipelineRun
|
|
||||||
cat <<YAML | kubectl create -f -
|
|
||||||
apiVersion: tekton.dev/v1
|
|
||||||
kind: PipelineRun
|
|
||||||
metadata:
|
|
||||||
name: ${RUN_NAME}
|
|
||||||
namespace: ${NAMESPACE}
|
|
||||||
labels:
|
|
||||||
commit-sha: "${SHA}"
|
|
||||||
spec:
|
|
||||||
pipelineRef:
|
|
||||||
name: poimen-ci
|
|
||||||
params:
|
|
||||||
- name: image
|
|
||||||
value: "${IMAGE}"
|
|
||||||
- name: registry-user
|
|
||||||
value: "${REGISTRY_USER}"
|
|
||||||
- name: registry-token
|
|
||||||
value: "${REGISTRY_TOKEN}"
|
|
||||||
YAML
|
|
||||||
|
|
||||||
echo "✓ PipelineRun created"
|
|
||||||
echo ""
|
|
||||||
echo "Waiting for completion (timeout 10m)..."
|
|
||||||
|
|
||||||
# Wait for PipelineRun to complete
|
|
||||||
if kubectl wait pipelinerun/${RUN_NAME} -n ${NAMESPACE} \
|
|
||||||
--for=condition=Succeeded --timeout=600s 2>/dev/null; then
|
|
||||||
echo "result=pass" >> $GITHUB_OUTPUT
|
|
||||||
echo "✓ Pipeline passed"
|
|
||||||
else
|
|
||||||
echo "result=fail" >> $GITHUB_OUTPUT
|
|
||||||
echo "✗ Pipeline failed or timed out"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Print pipeline summary
|
|
||||||
echo ""
|
|
||||||
echo "=== PipelineRun Status ==="
|
|
||||||
kubectl describe pipelinerun ${RUN_NAME} -n ${NAMESPACE} | tail -30
|
|
||||||
|
|
||||||
# Print task results
|
|
||||||
echo ""
|
|
||||||
echo "=== Task Results ==="
|
|
||||||
SUMMARY=$(kubectl get pipelinerun ${RUN_NAME} -n ${NAMESPACE} \
|
|
||||||
-o jsonpath='{.status.taskRuns[*].status.taskResults[?(@.name=="summary")].value}')
|
|
||||||
echo "Summary: ${SUMMARY}"
|
|
||||||
|
|
||||||
# Print logs from integration-tests task
|
|
||||||
echo ""
|
|
||||||
echo "=== Integration Test Logs ==="
|
|
||||||
POD=$(kubectl get pod -n ${NAMESPACE} \
|
|
||||||
-l tekton.dev/pipelineRun=${RUN_NAME} -l tekton.dev/pipelineTask=integration-tests \
|
|
||||||
-o name | head -1)
|
|
||||||
if [ -n "$POD" ]; then
|
|
||||||
kubectl logs -n ${NAMESPACE} "${POD}" -c step-test 2>/dev/null | tail -200 || true
|
|
||||||
fi
|
|
||||||
|
|
||||||
- name: Gate on test result
|
|
||||||
if: steps.tekton.outputs.result != 'pass'
|
|
||||||
run: |
|
|
||||||
echo "✗ Integration tests FAILED"
|
|
||||||
echo "Image NOT promoted to :latest"
|
|
||||||
exit 1
|
|
||||||
|
|
||||||
- name: Promote image to latest
|
|
||||||
run: |
|
|
||||||
docker login -u "${REGISTRY_USER}" -p "${REGISTRY_TOKEN}" "${REGISTRY}"
|
|
||||||
docker tag "${IMAGE}:${{ steps.sha.outputs.short_sha }}" "${IMAGE}:latest"
|
|
||||||
docker push "${IMAGE}:latest"
|
docker push "${IMAGE}:latest"
|
||||||
echo "✓ Promoted to :latest"
|
echo "✓ Pushed: ${IMAGE}:${{ steps.sha.outputs.short_sha }}"
|
||||||
env:
|
|
||||||
REGISTRY_USER: ${{ secrets.FORGEJO_REGISTRY_USER }}
|
|
||||||
REGISTRY_TOKEN: ${{ secrets.FORGEJO_REGISTRY_TOKEN }}
|
|
||||||
|
|
||||||
- name: Cleanup
|
- name: Prune unused images
|
||||||
if: always()
|
run: docker image prune -a --force 2>&1 | tail -3 || true
|
||||||
run: |
|
|
||||||
docker image prune -a --force 2>&1 | tail -3 || true
|
|
||||||
cargo clean || true
|
|
||||||
df -h /
|
|
||||||
|
|||||||
@@ -1,63 +0,0 @@
|
|||||||
name: Deploy
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches: [main]
|
|
||||||
workflow_dispatch:
|
|
||||||
|
|
||||||
env:
|
|
||||||
REGISTRY: forgejo.riotpiao.com
|
|
||||||
IMAGE: forgejo.riotpiao.com/riotpiao-poimen/poimen-memory
|
|
||||||
DOCKER_HOST: tcp://localhost:2375
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
deploy:
|
|
||||||
name: Tag & Push Latest
|
|
||||||
runs-on: rust
|
|
||||||
steps:
|
|
||||||
- name: Install Docker and curl
|
|
||||||
run: apt-get update && apt-get install -y docker.io curl
|
|
||||||
|
|
||||||
- name: Get short SHA via Gitea API
|
|
||||||
id: sha
|
|
||||||
run: |
|
|
||||||
# Fetch latest commit SHA for main branch from Gitea API
|
|
||||||
COMMIT_SHA=$(curl -s -H "Authorization: token ${REGISTRY_TOKEN}" \
|
|
||||||
"https://forgejo.riotpiao.com/api/v1/repos/riotpiao-poimen/poimen-memory/commits?sha=main&limit=1" | \
|
|
||||||
grep -o '"sha":"[^"]*' | head -1 | cut -d'"' -f4)
|
|
||||||
|
|
||||||
if [ -z "$COMMIT_SHA" ]; then
|
|
||||||
echo "ERROR: Failed to fetch commit SHA from Gitea API"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
SHORT_SHA=$(echo "$COMMIT_SHA" | cut -c1-7)
|
|
||||||
echo "short_sha=$SHORT_SHA" >> $GITHUB_OUTPUT
|
|
||||||
echo "Full SHA: $COMMIT_SHA, Short: $SHORT_SHA"
|
|
||||||
env:
|
|
||||||
REGISTRY_TOKEN: ${{ secrets.FORGEJO_REGISTRY_TOKEN }}
|
|
||||||
|
|
||||||
- name: Registry login
|
|
||||||
run: |
|
|
||||||
if [ -z "${REGISTRY_USER}" ] || [ -z "${REGISTRY_TOKEN}" ]; then
|
|
||||||
echo "ERROR: Missing REGISTRY_USER or REGISTRY_TOKEN secrets"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
echo "${REGISTRY_TOKEN}" | docker login "${REGISTRY}" \
|
|
||||||
--username "${REGISTRY_USER}" --password-stdin
|
|
||||||
env:
|
|
||||||
REGISTRY_USER: ${{ secrets.FORGEJO_REGISTRY_USER }}
|
|
||||||
REGISTRY_TOKEN: ${{ secrets.FORGEJO_REGISTRY_TOKEN }}
|
|
||||||
|
|
||||||
- name: Verify SHA image exists, tag as latest
|
|
||||||
run: |
|
|
||||||
if ! docker pull "${IMAGE}:${{ steps.sha.outputs.short_sha }}"; then
|
|
||||||
echo "ERROR: Image ${IMAGE}:${{ steps.sha.outputs.short_sha }} not found. Check build.yaml passed."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
docker tag "${IMAGE}:${{ steps.sha.outputs.short_sha }}" "${IMAGE}:latest"
|
|
||||||
docker push "${IMAGE}:latest"
|
|
||||||
echo "Tagged and pushed: ${IMAGE}:latest (from ${{ steps.sha.outputs.short_sha }})"
|
|
||||||
|
|
||||||
- name: Prune images
|
|
||||||
run: docker image prune -a --force 2>&1 | tail -3 || true
|
|
||||||
@@ -20,4 +20,3 @@ knowledge/
|
|||||||
docs/LIFECYCLE.md
|
docs/LIFECYCLE.md
|
||||||
# Trigger CI
|
# Trigger CI
|
||||||
# Test runner ready
|
# Test runner ready
|
||||||
.sqlx/
|
|
||||||
|
|||||||
+52
@@ -0,0 +1,52 @@
|
|||||||
|
{
|
||||||
|
"db_name": "PostgreSQL",
|
||||||
|
"query": "\n SELECT \n version_num,\n operation,\n snapshot,\n changed_at,\n changed_by,\n COALESCE(fields_changed, '{}') as \"fields_changed!\"\n FROM memory_entity_version\n WHERE entity_id = $1\n ORDER BY version_num DESC\n ",
|
||||||
|
"describe": {
|
||||||
|
"columns": [
|
||||||
|
{
|
||||||
|
"ordinal": 0,
|
||||||
|
"name": "version_num",
|
||||||
|
"type_info": "Int4"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 1,
|
||||||
|
"name": "operation",
|
||||||
|
"type_info": "Varchar"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 2,
|
||||||
|
"name": "snapshot",
|
||||||
|
"type_info": "Jsonb"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 3,
|
||||||
|
"name": "changed_at",
|
||||||
|
"type_info": "Timestamptz"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 4,
|
||||||
|
"name": "changed_by",
|
||||||
|
"type_info": "Varchar"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 5,
|
||||||
|
"name": "fields_changed!",
|
||||||
|
"type_info": "TextArray"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"parameters": {
|
||||||
|
"Left": [
|
||||||
|
"Text"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"nullable": [
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
null
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"hash": "1e81bb729531ca33e4cef21623bcfe4fafb0c1bd435353b205f582bfda8873bc"
|
||||||
|
}
|
||||||
+52
@@ -0,0 +1,52 @@
|
|||||||
|
{
|
||||||
|
"db_name": "PostgreSQL",
|
||||||
|
"query": "\n SELECT \n version_num,\n operation,\n snapshot,\n changed_at,\n changed_by,\n COALESCE(fields_changed, '{}') as \"fields_changed!\"\n FROM memory_edge_version\n WHERE edge_id = $1\n ORDER BY version_num DESC\n ",
|
||||||
|
"describe": {
|
||||||
|
"columns": [
|
||||||
|
{
|
||||||
|
"ordinal": 0,
|
||||||
|
"name": "version_num",
|
||||||
|
"type_info": "Int4"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 1,
|
||||||
|
"name": "operation",
|
||||||
|
"type_info": "Varchar"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 2,
|
||||||
|
"name": "snapshot",
|
||||||
|
"type_info": "Jsonb"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 3,
|
||||||
|
"name": "changed_at",
|
||||||
|
"type_info": "Timestamptz"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 4,
|
||||||
|
"name": "changed_by",
|
||||||
|
"type_info": "Varchar"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 5,
|
||||||
|
"name": "fields_changed!",
|
||||||
|
"type_info": "TextArray"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"parameters": {
|
||||||
|
"Left": [
|
||||||
|
"Uuid"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"nullable": [
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
null
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"hash": "62d65d4afc4d292b37de8e5cb59fbd51c602bdc1b437988f54e6c7fe268b9816"
|
||||||
|
}
|
||||||
+53
@@ -0,0 +1,53 @@
|
|||||||
|
{
|
||||||
|
"db_name": "PostgreSQL",
|
||||||
|
"query": "\n SELECT \n version_num,\n operation,\n snapshot,\n changed_at,\n changed_by,\n COALESCE(fields_changed, '{}') as \"fields_changed!\"\n FROM memory_entity_version\n WHERE entity_id = $1 AND changed_at <= $2\n ORDER BY version_num DESC\n LIMIT 1\n ",
|
||||||
|
"describe": {
|
||||||
|
"columns": [
|
||||||
|
{
|
||||||
|
"ordinal": 0,
|
||||||
|
"name": "version_num",
|
||||||
|
"type_info": "Int4"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 1,
|
||||||
|
"name": "operation",
|
||||||
|
"type_info": "Varchar"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 2,
|
||||||
|
"name": "snapshot",
|
||||||
|
"type_info": "Jsonb"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 3,
|
||||||
|
"name": "changed_at",
|
||||||
|
"type_info": "Timestamptz"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 4,
|
||||||
|
"name": "changed_by",
|
||||||
|
"type_info": "Varchar"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 5,
|
||||||
|
"name": "fields_changed!",
|
||||||
|
"type_info": "TextArray"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"parameters": {
|
||||||
|
"Left": [
|
||||||
|
"Text",
|
||||||
|
"Timestamptz"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"nullable": [
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
null
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"hash": "aee5900f5e3d7cbba23729bbf2dd033dcc4cb41f6c851bf447a9238810684d18"
|
||||||
|
}
|
||||||
+53
@@ -0,0 +1,53 @@
|
|||||||
|
{
|
||||||
|
"db_name": "PostgreSQL",
|
||||||
|
"query": "\n SELECT \n version_num,\n operation,\n snapshot,\n changed_at,\n changed_by,\n COALESCE(fields_changed, '{}') as \"fields_changed!\"\n FROM memory_entity_version\n WHERE entity_id = $1 AND version_num = $2\n ",
|
||||||
|
"describe": {
|
||||||
|
"columns": [
|
||||||
|
{
|
||||||
|
"ordinal": 0,
|
||||||
|
"name": "version_num",
|
||||||
|
"type_info": "Int4"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 1,
|
||||||
|
"name": "operation",
|
||||||
|
"type_info": "Varchar"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 2,
|
||||||
|
"name": "snapshot",
|
||||||
|
"type_info": "Jsonb"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 3,
|
||||||
|
"name": "changed_at",
|
||||||
|
"type_info": "Timestamptz"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 4,
|
||||||
|
"name": "changed_by",
|
||||||
|
"type_info": "Varchar"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 5,
|
||||||
|
"name": "fields_changed!",
|
||||||
|
"type_info": "TextArray"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"parameters": {
|
||||||
|
"Left": [
|
||||||
|
"Text",
|
||||||
|
"Int4"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"nullable": [
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
null
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"hash": "c045466e1fe037dbdafea1008f262f4e48f104ea77732aa1d32ecb797f70e71d"
|
||||||
|
}
|
||||||
+53
@@ -0,0 +1,53 @@
|
|||||||
|
{
|
||||||
|
"db_name": "PostgreSQL",
|
||||||
|
"query": "\n SELECT \n version_num,\n operation,\n snapshot,\n changed_at,\n changed_by,\n COALESCE(fields_changed, '{}') as \"fields_changed!\"\n FROM memory_edge_version\n WHERE edge_id = $1 AND version_num = $2\n ",
|
||||||
|
"describe": {
|
||||||
|
"columns": [
|
||||||
|
{
|
||||||
|
"ordinal": 0,
|
||||||
|
"name": "version_num",
|
||||||
|
"type_info": "Int4"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 1,
|
||||||
|
"name": "operation",
|
||||||
|
"type_info": "Varchar"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 2,
|
||||||
|
"name": "snapshot",
|
||||||
|
"type_info": "Jsonb"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 3,
|
||||||
|
"name": "changed_at",
|
||||||
|
"type_info": "Timestamptz"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 4,
|
||||||
|
"name": "changed_by",
|
||||||
|
"type_info": "Varchar"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 5,
|
||||||
|
"name": "fields_changed!",
|
||||||
|
"type_info": "TextArray"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"parameters": {
|
||||||
|
"Left": [
|
||||||
|
"Uuid",
|
||||||
|
"Int4"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"nullable": [
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
null
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"hash": "ca6872495bc04c6a65531279af8c758637c902dda2cc10366662988c6973ca48"
|
||||||
|
}
|
||||||
@@ -1,136 +0,0 @@
|
|||||||
# Poimen Memory System
|
|
||||||
|
|
||||||
## Project Status
|
|
||||||
|
|
||||||
**Architecture**: Temporal Knowledge Graph for Agent Memory (Zep paper alignment — arXiv:2501.13956)
|
|
||||||
|
|
||||||
**Current**: Ingest pipeline with LLM entity + fact extraction working E2E. Deployed to K8s.
|
|
||||||
|
|
||||||
### What Works
|
|
||||||
- ✅ HTTP server (actix-web) with 15+ endpoints
|
|
||||||
- ✅ LLM entity extraction (LlmEntityExtractor) — extracts person/tool/concept/org entities
|
|
||||||
- ✅ LLM fact extraction (LlmFactExtractor) — extracts relationships between entities
|
|
||||||
- ✅ Reasoning model support — strips `<think>` tags, markdown fences
|
|
||||||
- ✅ Ollama + vLLM + OpenAI-compatible API support
|
|
||||||
- ✅ Entity persistence to pgvector (memory_entity table)
|
|
||||||
- ✅ Edge persistence (memory_edge table with temporal fields)
|
|
||||||
- ✅ Graph query endpoints (entities, edges, BFS traversal)
|
|
||||||
- ✅ Visualization (React Flow JSON, force-directed layout, SSE streaming)
|
|
||||||
- ✅ JWT auth (Authentik OIDC) with RBAC
|
|
||||||
- ✅ K8s deployment (CNPG postgres, ConfigMap, SOPS secrets)
|
|
||||||
- ✅ CI: PR builds push :SHA tag, main merges retag :latest
|
|
||||||
- ✅ 781 tests passing
|
|
||||||
|
|
||||||
### Deployment
|
|
||||||
- **Namespace**: `poimen`
|
|
||||||
- **Image**: `forgejo.riotpiao.com/riotpiao-poimen/poimen-memory:latest`
|
|
||||||
- **DB**: CNPG cluster `memory-db` (pgvector)
|
|
||||||
- **LLM**: `reasoning-predictor.llm-serving.svc.cluster.local` (ornith:35b / qwen2.5:3b)
|
|
||||||
- **Auth**: Authentik OIDC (`MEM_AUTH_MODE=none` for dev)
|
|
||||||
- **Registry**: Forgejo container registry (FORGEJO_REGISTRY_USER/TOKEN secrets)
|
|
||||||
|
|
||||||
### Key Env Vars
|
|
||||||
```
|
|
||||||
DATABASE_URL postgresql://...
|
|
||||||
MEM_AUTH_MODE none|jwt|apikey
|
|
||||||
LLM_ENDPOINT http://localhost:11434/v1/chat/completions (Ollama)
|
|
||||||
LLM_MODEL qwen2.5:3b | ornith:35b | reasoning
|
|
||||||
LLM_API_KEY (for authenticated LLM APIs)
|
|
||||||
MEM_API_KEY (server API key, fallback "test-key")
|
|
||||||
OPENSEARCH_HOSTS (optional, hybrid search)
|
|
||||||
GATEWAY_URL (optional, external queue)
|
|
||||||
```
|
|
||||||
|
|
||||||
## Rules
|
|
||||||
|
|
||||||
1. **No progress markdown files.** Track via Forgejo issues + PRs only.
|
|
||||||
2. **Obsidian vault repo**: `ssh://[email protected]:2222/rock/poimen-obesdient-memory.git`
|
|
||||||
3. **Secrets via KSOPS**: Age-based SOPS encryption. Never commit plaintext.
|
|
||||||
4. **Tea CLI**: `poimen` login has API token `1f717a00134f17c9d2d656c620b955e03ea41276`
|
|
||||||
|
|
||||||
## Architecture (Zep Paper §2)
|
|
||||||
|
|
||||||
### Three-Tier Knowledge Graph
|
|
||||||
```
|
|
||||||
Episode Subgraph (raw messages)
|
|
||||||
→ Entity Subgraph (extracted entities + facts/edges)
|
|
||||||
→ Community Subgraph (clusters, planned Phase 4)
|
|
||||||
```
|
|
||||||
|
|
||||||
### Ingest Pipeline (4 stages)
|
|
||||||
1. **Entity extraction** — LLM extracts named entities with type + summary
|
|
||||||
2. **Deduplication** — HashSet on normalized name
|
|
||||||
3. **Fact extraction** — LLM extracts relationships between entity pairs
|
|
||||||
4. **Contradiction detection** — pre-filter + review queue
|
|
||||||
|
|
||||||
### Retrieval (3 methods, §3)
|
|
||||||
- Cosine semantic similarity (pgvector HNSW)
|
|
||||||
- BM25 full-text (OpenSearch, optional)
|
|
||||||
- BFS graph traversal (depth 1-3)
|
|
||||||
|
|
||||||
### Extractors
|
|
||||||
- `LlmEntityExtractor`: calls LLM_ENDPOINT, parses JSON, handles reasoning models
|
|
||||||
- `LlmFactExtractor`: takes entity list + text, extracts edges between known entities
|
|
||||||
- `WikiLinkFallbackExtractor`: pattern-matches `[[wiki links]]` (no LLM)
|
|
||||||
- `SimpleFactExtractor`: verb pattern matching (no LLM)
|
|
||||||
- Selection: LLM extractors when `LLM_ENDPOINT` set, else fallbacks
|
|
||||||
|
|
||||||
### LLM Response Cleaning
|
|
||||||
`clean_llm_response()` handles:
|
|
||||||
- `<think>...</think>` blocks (reasoning models)
|
|
||||||
- Markdown code fences (```json ... ```)
|
|
||||||
- Array responses (wrap in `{"entities": [...]}`)
|
|
||||||
- Extract first JSON object from mixed text
|
|
||||||
|
|
||||||
## Crate Structure
|
|
||||||
|
|
||||||
```
|
|
||||||
crates/
|
|
||||||
mem-core/ — Entity, Edge, domain types (174 tests)
|
|
||||||
mem-store/ — DB repos, schema, vector store
|
|
||||||
mem-ingest/ — Entity/fact extraction, contradiction detection (87 tests)
|
|
||||||
mem-llm/ — Embeddings, chat, rerank clients
|
|
||||||
mem-cli/ — HTTP server, handlers, query, ingest worker (496 tests)
|
|
||||||
```
|
|
||||||
|
|
||||||
## API Endpoints
|
|
||||||
|
|
||||||
```
|
|
||||||
GET /health
|
|
||||||
POST /memory/ingest — Queue ingest job
|
|
||||||
GET /memory/ingest/{id} — Check job status
|
|
||||||
GET /memory/query?project=&question= — Graph query
|
|
||||||
POST /memory/query — Unified query
|
|
||||||
POST /memory/context — Three-tier retrieval
|
|
||||||
POST /memory/learn — Direct learn
|
|
||||||
POST /memory/visualize — React Flow JSON
|
|
||||||
POST /memory/visualize/stream — SSE streaming
|
|
||||||
POST /memory/compact — Trigger compaction
|
|
||||||
GET /memory/projects — List projects
|
|
||||||
GET /memory/skills — List skills
|
|
||||||
GET /memory/vault — Browse vault
|
|
||||||
POST /memory/synthesis/* — Entity linking, alias detection
|
|
||||||
```
|
|
||||||
|
|
||||||
## Current PRs / Branches
|
|
||||||
|
|
||||||
- **PR #48** `feat/memory-ingest-retrieval` — LLM entity + fact extraction, deployment fixes
|
|
||||||
- **PR #47** merged — Agent entity types (Phase 3.1)
|
|
||||||
- **PR #46** merged — Integration test fixes, CI
|
|
||||||
|
|
||||||
## Next Steps
|
|
||||||
|
|
||||||
1. Merge PR #48 → new image with LLM extraction
|
|
||||||
2. Query retrieval E2E — verify entities/edges returned in query results
|
|
||||||
3. Visualization E2E — test /memory/visualize with extracted graph
|
|
||||||
4. Restore 198 deleted tests from PR #46
|
|
||||||
5. Community detection (Phase 4, Zep §2.3)
|
|
||||||
6. Temporal edge invalidation (Zep §2.2.3)
|
|
||||||
7. Reranker (cross-encoder, RRF, episode-mentions — Zep §3.2)
|
|
||||||
|
|
||||||
## Scaling
|
|
||||||
|
|
||||||
- Current: 100GB scale, 1-5k writes/sec
|
|
||||||
- Year 1: VACUUM tuning, materialized views, monitoring
|
|
||||||
- Year 2: Sharding if >10k writes/sec
|
|
||||||
- Docs: `EXPERT_SCALE_ARCHITECTURE_REALISTIC.md`
|
|
||||||
Generated
-5
@@ -2053,7 +2053,6 @@ dependencies = [
|
|||||||
"mem-ingest",
|
"mem-ingest",
|
||||||
"mem-llm",
|
"mem-llm",
|
||||||
"mem-store",
|
"mem-store",
|
||||||
"once_cell",
|
|
||||||
"pgvector",
|
"pgvector",
|
||||||
"rand 0.8.7",
|
"rand 0.8.7",
|
||||||
"redis",
|
"redis",
|
||||||
@@ -2599,15 +2598,11 @@ dependencies = [
|
|||||||
"mem-llm",
|
"mem-llm",
|
||||||
"mem-store",
|
"mem-store",
|
||||||
"regex",
|
"regex",
|
||||||
"reqwest",
|
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"sqlx",
|
"sqlx",
|
||||||
"time",
|
"time",
|
||||||
"tokio",
|
"tokio",
|
||||||
"toml",
|
"toml",
|
||||||
"tracing",
|
|
||||||
"tracing-subscriber",
|
|
||||||
"uuid",
|
|
||||||
"wiremock",
|
"wiremock",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|||||||
@@ -66,10 +66,6 @@ chrono = { version = "0.4", features = ["serde"] }
|
|||||||
regex = { workspace = true }
|
regex = { workspace = true }
|
||||||
sqlx = { workspace = true }
|
sqlx = { workspace = true }
|
||||||
base64 = { workspace = true }
|
base64 = { workspace = true }
|
||||||
tracing = { workspace = true }
|
|
||||||
tracing-subscriber = { workspace = true }
|
|
||||||
reqwest = { workspace = true }
|
|
||||||
uuid = { workspace = true }
|
|
||||||
|
|
||||||
[profile.release]
|
[profile.release]
|
||||||
opt-level = 3
|
opt-level = 3
|
||||||
|
|||||||
+3
-14
@@ -5,23 +5,12 @@ FROM rust:1-bookworm as builder
|
|||||||
|
|
||||||
WORKDIR /build
|
WORKDIR /build
|
||||||
|
|
||||||
# Build settings
|
|
||||||
ENV SQLX_OFFLINE=true
|
|
||||||
|
|
||||||
# Copy source
|
# Copy source
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
# Build release binary with space-efficient cleanup
|
# Build the mem binary (offline sqlx - uses .sqlx/ cache)
|
||||||
RUN cargo build --release -p mem-cli --locked && \
|
ENV SQLX_OFFLINE=true
|
||||||
strip target/release/mem && \
|
RUN cargo build --release -p mem-cli
|
||||||
# Aggressive cleanup to free disk space
|
|
||||||
rm -rf target/release/deps && \
|
|
||||||
rm -rf target/release/build && \
|
|
||||||
rm -rf target/release/incremental && \
|
|
||||||
rm -rf target/release/.fingerprint && \
|
|
||||||
rm -rf .cargo/registry/cache && \
|
|
||||||
rm -rf .cargo/registry/index && \
|
|
||||||
rm -rf .cargo/git
|
|
||||||
|
|
||||||
# Stage 2: Runtime
|
# Stage 2: Runtime
|
||||||
FROM debian:bookworm-slim
|
FROM debian:bookworm-slim
|
||||||
|
|||||||
@@ -0,0 +1,263 @@
|
|||||||
|
# CRITICAL FIXES NEEDED - Poimen Memory Service
|
||||||
|
|
||||||
|
## STATUS: Service Non-Functional ❌
|
||||||
|
|
||||||
|
**Root Issues Blocking Service**:
|
||||||
|
1. ✅ HTTP handler deadlock fixed (schema init error handling)
|
||||||
|
2. ❌ Server initialization hangs during schema or startup (logs stop after `l2_l1_edges`)
|
||||||
|
3. ❌ Ingest pipeline NOT implemented (just raw vector storage, no entities/edges)
|
||||||
|
4. ❌ Temporal schema missing (no t_valid, t_invalid, version tracking)
|
||||||
|
5. ❌ GRM gate not integrated (no memorability scores, confidence)
|
||||||
|
6. ❌ Query doesn't use knowledge graph (just vector search)
|
||||||
|
7. ❌ Compaction disabled
|
||||||
|
8. ❌ Verification gates missing
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## STEP 1: Fix Server Startup Hang ⚠️
|
||||||
|
|
||||||
|
**Current Issue**: Server hangs during initialization after schema creation.
|
||||||
|
|
||||||
|
**Suspected causes**:
|
||||||
|
- OptimizerServiceBuilder.build() getting stuck
|
||||||
|
- AccessGuard creation blocking
|
||||||
|
- Background task spawning deadlock
|
||||||
|
|
||||||
|
**Fix**:
|
||||||
|
```rust
|
||||||
|
// In http_server.rs:316-325
|
||||||
|
// Wrap in timeout or disable non-essentials
|
||||||
|
let optimizer_service = match tokio::time::timeout(
|
||||||
|
Duration::from_secs(5),
|
||||||
|
async { mem_core::optimizer::OptimizerServiceBuilder::new().build() }
|
||||||
|
).await {
|
||||||
|
Ok(Ok(service)) => Some(Arc::new(service)),
|
||||||
|
_ => {
|
||||||
|
tracing::warn!("Optimizer initialization skipped (timeout or error)");
|
||||||
|
None
|
||||||
|
}
|
||||||
|
};
|
||||||
|
```
|
||||||
|
|
||||||
|
**Test**: `./target/release/mem serve --port 9999` should reach "Starting HTTP server" within 10s
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## STEP 2: Implement Ingest Pipeline (HIGH PRIORITY)
|
||||||
|
|
||||||
|
**Current Implementation** (`ingest_worker.rs`):
|
||||||
|
```rust
|
||||||
|
// Just stores raw chunks + embeddings
|
||||||
|
store_chunk_l0(&l0_chunk)
|
||||||
|
store_memory_l1(&l1_memory, &embedding)
|
||||||
|
```
|
||||||
|
|
||||||
|
**Expected Implementation**:
|
||||||
|
```rust
|
||||||
|
// 1. Extract entities (entity_extractor)
|
||||||
|
let entities = entity_extractor.extract(&content).await?;
|
||||||
|
|
||||||
|
// 2. Extract facts + edges (fact_extractor)
|
||||||
|
let facts = fact_extractor.extract(&content, entities).await?;
|
||||||
|
|
||||||
|
// 3. Create temporal edges with GRM gate
|
||||||
|
for fact in facts {
|
||||||
|
let edge = TemporalEdge {
|
||||||
|
source: fact.source_entity,
|
||||||
|
target: fact.target_entity,
|
||||||
|
relation: fact.relation,
|
||||||
|
fact: fact.text,
|
||||||
|
t_valid: now(),
|
||||||
|
t_invalid: None,
|
||||||
|
confidence: grm_gate.score(&fact)?, // ← GRM gate
|
||||||
|
version: 1,
|
||||||
|
};
|
||||||
|
edge_repo.insert(&edge).await?;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. Check contradictions + queue for review
|
||||||
|
for edge in edges {
|
||||||
|
if contradiction_detector.detect(&edge, existing_edges)? {
|
||||||
|
review_queue.enqueue(&edge).await?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Files to modify**:
|
||||||
|
- `crates/mem-cli/src/ingest_worker.rs` (core ingest logic)
|
||||||
|
- `crates/mem-ingest/src/ingest_pipeline.rs` (entity + fact extraction)
|
||||||
|
- `crates/mem-ingest/src/contradiction_detector.rs` (pre-filter + review)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## STEP 3: Update Storage Schema (MEDIUM PRIORITY)
|
||||||
|
|
||||||
|
**Missing fields**:
|
||||||
|
```sql
|
||||||
|
ALTER TABLE memories_l1 ADD COLUMN (
|
||||||
|
t_valid TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||||
|
t_invalid TIMESTAMP,
|
||||||
|
confidence FLOAT DEFAULT 0.5,
|
||||||
|
version INT DEFAULT 1,
|
||||||
|
memorability_score INT,
|
||||||
|
contribution_date TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
ALTER TABLE l1_l0_edges MODIFY TO (
|
||||||
|
l1_id UUID,
|
||||||
|
l0_id UUID,
|
||||||
|
relation_type VARCHAR,
|
||||||
|
fact TEXT,
|
||||||
|
t_valid TIMESTAMP DEFAULT NOW(),
|
||||||
|
t_invalid TIMESTAMP,
|
||||||
|
confidence FLOAT,
|
||||||
|
contradiction_flag BOOL DEFAULT FALSE,
|
||||||
|
review_queue_id UUID,
|
||||||
|
version INT DEFAULT 1,
|
||||||
|
PRIMARY KEY (l1_id, l0_id, version)
|
||||||
|
);
|
||||||
|
```
|
||||||
|
|
||||||
|
**Migration script**: `crates/mem-store/migrations/003_temporal_grm_schema.sql`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## STEP 4: Wire Query Handler to Knowledge Graph (MEDIUM PRIORITY)
|
||||||
|
|
||||||
|
**Current** (`query_handler` in http_server.rs):
|
||||||
|
```rust
|
||||||
|
async fn query_handler(...) -> HttpResponse {
|
||||||
|
// Just semantic search
|
||||||
|
let results = vector_search(query)?;
|
||||||
|
HttpResponse::Ok().json(results)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Expected**:
|
||||||
|
```rust
|
||||||
|
async fn query_handler(query: QueryRequest) -> HttpResponse {
|
||||||
|
// 1. Semantic search on embeddings
|
||||||
|
let initial_results = vector_search(&query.text)?;
|
||||||
|
|
||||||
|
// 2. Follow edges (graph traversal)
|
||||||
|
let mut expanded = vec![];
|
||||||
|
for result in initial_results {
|
||||||
|
expanded.push(result);
|
||||||
|
// Get related entities via edges
|
||||||
|
let related = edge_repo.find_by_source(&result.entity_id).await?;
|
||||||
|
expanded.extend(related);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Apply temporal filters
|
||||||
|
expanded.retain(|e| e.t_valid <= now() && (e.t_invalid.is_none() || e.t_invalid > now()));
|
||||||
|
|
||||||
|
// 4. Sort by confidence + recency
|
||||||
|
expanded.sort_by(|a, b| {
|
||||||
|
b.confidence.partial_cmp(&a.confidence)
|
||||||
|
.then_with(|| b.t_valid.cmp(&a.t_valid))
|
||||||
|
});
|
||||||
|
|
||||||
|
// 5. Apply compaction/cache alignment
|
||||||
|
for item in &mut expanded {
|
||||||
|
item.text = optimizer.compress(item.text)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
HttpResponse::Ok().json(MemoryResponse {
|
||||||
|
entities: expanded,
|
||||||
|
confidence_scores: compute_scores(&expanded),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## STEP 5: Enable Compaction Endpoint (LOW PRIORITY)
|
||||||
|
|
||||||
|
**Current**: Code exists but never called.
|
||||||
|
|
||||||
|
**Fix**: Add K8s CronJob that calls `POST /memory/compact` daily:
|
||||||
|
```yaml
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: CronJob
|
||||||
|
metadata:
|
||||||
|
name: memory-compaction
|
||||||
|
spec:
|
||||||
|
schedule: "0 2 * * *" # 2 AM UTC
|
||||||
|
jobTemplate:
|
||||||
|
spec:
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: compact
|
||||||
|
image: bitnami/curl:latest
|
||||||
|
command:
|
||||||
|
- curl
|
||||||
|
- -X POST
|
||||||
|
- -H "Authorization: Bearer $ADMIN_TOKEN"
|
||||||
|
- http://poimen-memory:8080/memory/compact
|
||||||
|
restartPolicy: OnFailure
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## STEP 6: Add Verification Gates (LOW PRIORITY)
|
||||||
|
|
||||||
|
**Missing**: `GET /memory/verify` endpoint that checks M1.8, M2.8, M3.7, M8.9 gates
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## IMPLEMENTATION ORDER
|
||||||
|
|
||||||
|
1. **FIX STARTUP** (1 hour) → Get server running
|
||||||
|
2. **INGEST PIPELINE** (3 hours) → Wire entity + fact extraction
|
||||||
|
3. **TEMPORAL SCHEMA** (1 hour) → Add missing columns
|
||||||
|
4. **QUERY HANDLER** (2 hours) → Implement graph traversal
|
||||||
|
5. **COMPACTION** (1 hour) → Add CronJob
|
||||||
|
6. **GATES** (2 hours) → Quality verification
|
||||||
|
|
||||||
|
**Total**: ~10 hours to full working system
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## TEST PLAN
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1. Server starts
|
||||||
|
curl http://localhost:9999/health
|
||||||
|
# Expected: {"status":"ok","uptime_seconds":N}
|
||||||
|
|
||||||
|
# 2. Ingest works
|
||||||
|
curl -X POST http://localhost:9999/memory/ingest \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '{"project":"test","source":"test://1","ingest_id":"i1","records":[{"role":"user","text":"Hello world","timestamp":"2026-01-08T16:00:00Z","source_position":0}]}'
|
||||||
|
# Expected: {"ingest_id":"i1","status":"pending",...}
|
||||||
|
|
||||||
|
# 3. Query returns entities with edges
|
||||||
|
curl -X POST http://localhost:9999/memory/query \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '{"project":"test","query":"hello"}'
|
||||||
|
# Expected: {"results":[{"type":"entity","name":"...","edges":[...]}]}
|
||||||
|
|
||||||
|
# 4. Temporal filtering works
|
||||||
|
curl http://localhost:9999/memory/query?project=test&temporal_floor=2026-01-01
|
||||||
|
|
||||||
|
# 5. Compaction works
|
||||||
|
curl -X POST http://localhost:9999/memory/compact
|
||||||
|
# Expected: {"phase":"completed","records_deduplicated":N}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## FILES MODIFIED SO FAR
|
||||||
|
|
||||||
|
✅ `crates/mem-cli/src/http_server.rs` - Added error handling for schema init
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## NEXT SESSION TODO
|
||||||
|
|
||||||
|
- [ ] Fix server startup hang (debug OptimizerService)
|
||||||
|
- [ ] Implement ingest_worker to call entity_extractor + fact_extractor
|
||||||
|
- [ ] Add temporal columns to schema
|
||||||
|
- [ ] Update query_handler to traverse edges
|
||||||
|
- [ ] Test end-to-end with sample data
|
||||||
@@ -1,84 +0,0 @@
|
|||||||
# Local Development Setup
|
|
||||||
|
|
||||||
Running poimen-memory locally for development.
|
|
||||||
|
|
||||||
## Quick Start
|
|
||||||
|
|
||||||
1. **Copy env template**:
|
|
||||||
```bash
|
|
||||||
cp .env.example .env
|
|
||||||
```
|
|
||||||
|
|
||||||
2. **Edit `.env`** with your local endpoints:
|
|
||||||
```bash
|
|
||||||
# Edit .env with your local/dev service URLs
|
|
||||||
# Example: LLM service on localhost:11434, OpenSearch on localhost:9200
|
|
||||||
```
|
|
||||||
|
|
||||||
3. **Run the service**:
|
|
||||||
```bash
|
|
||||||
cargo run --release -- serve --port 8080
|
|
||||||
```
|
|
||||||
|
|
||||||
The application loads configuration from `.env` (via `dotenvy` or similar).
|
|
||||||
|
|
||||||
## `.env` File
|
|
||||||
|
|
||||||
**Location**: Project root (`.env`)
|
|
||||||
**Status**: Gitignored - never committed
|
|
||||||
**Template**: `.env.example` (included in repo, shows all available variables)
|
|
||||||
|
|
||||||
### Key Variables
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Database
|
|
||||||
DATABASE_URL=postgresql://user:pass@localhost:5432/memory
|
|
||||||
|
|
||||||
# LLM (point to your local LLM service)
|
|
||||||
LLM_ENDPOINT=http://localhost:11434/v1/chat/completions
|
|
||||||
LLM_MODEL=qwen:7b
|
|
||||||
|
|
||||||
# OpenSearch (local vector store)
|
|
||||||
OPENSEARCH_HOST=localhost:9200
|
|
||||||
|
|
||||||
# Auth (disabled for local dev)
|
|
||||||
MEM_AUTH_MODE=none
|
|
||||||
|
|
||||||
# API Key (test key for local dev)
|
|
||||||
MEM_API_KEY=test-key
|
|
||||||
```
|
|
||||||
|
|
||||||
## Local Service Stack (Example)
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Terminal 1: OpenSearch
|
|
||||||
docker run -d -p 9200:9200 -e OPENSEARCH_JAVA_OPTS="-Xms512m -Xmx512m" \
|
|
||||||
opensearchproject/opensearch:latest
|
|
||||||
|
|
||||||
# Terminal 2: Ollama (LLM)
|
|
||||||
ollama serve
|
|
||||||
|
|
||||||
# Terminal 3: poimen-memory
|
|
||||||
cargo run --release -- serve --port 8080
|
|
||||||
```
|
|
||||||
|
|
||||||
## Production vs Local
|
|
||||||
|
|
||||||
| Aspect | Production (K8s) | Local Dev |
|
|
||||||
|--------|-----------------|-----------|
|
|
||||||
| **Config** | `k8s/app/config.yaml` (SOPS-encrypted) | `.env` (gitignored) |
|
|
||||||
| **Injection** | ConfigMap via `envFrom:` | dotenv via `dotenvy` crate |
|
|
||||||
| **Services** | Cluster-internal DNS | localhost/127.0.0.1 |
|
|
||||||
| **Auth** | JWT (Authentik) | None (disabled) |
|
|
||||||
| **Commit?** | Yes (encrypted) | No (gitignored) |
|
|
||||||
|
|
||||||
## Switching to Production Config
|
|
||||||
|
|
||||||
To run against production services (not recommended locally):
|
|
||||||
1. Edit `.env` with production URLs
|
|
||||||
2. Set credentials appropriately
|
|
||||||
3. Ensure network access to production services
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
See `.env.example` for all available environment variables.
|
|
||||||
@@ -0,0 +1,217 @@
|
|||||||
|
# Monitoring Agent: Implementation Tasks
|
||||||
|
|
||||||
|
**Milestone**: `monitoring-agent`
|
||||||
|
**Status**: 🔧 Not started
|
||||||
|
**Duration**: 4-6 weeks
|
||||||
|
**Effort**: ~1,500 LOC
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Phase 1: Temporal Setup (3-5 days)
|
||||||
|
|
||||||
|
### Task 1.1: Deploy Temporal Server in K8s
|
||||||
|
- [ ] StatefulSet configuration (persistence)
|
||||||
|
- [ ] PostgreSQL event log backend
|
||||||
|
- [ ] ElasticSearch for visibility
|
||||||
|
- [ ] K8s manifests in `k8s/temporal/`
|
||||||
|
- [ ] Health checks + readiness probes
|
||||||
|
- **Effort**: 150 LOC | **Time**: 2 days
|
||||||
|
- **Dependencies**: None
|
||||||
|
- **Blocks**: Phase 2
|
||||||
|
|
||||||
|
### Task 1.2: Add Temporal SDK to Rust Project
|
||||||
|
- [ ] Add `temporal-rust-sdk` to `Cargo.toml`
|
||||||
|
- [ ] Create `crates/mem-temporal/` workspace crate
|
||||||
|
- [ ] Worker registration + gRPC connection
|
||||||
|
- [ ] Activity executor setup
|
||||||
|
- [ ] Workflow executor setup
|
||||||
|
- **Effort**: 200 LOC | **Time**: 1 day
|
||||||
|
- **Dependencies**: 1.1
|
||||||
|
- **Blocks**: Phase 2
|
||||||
|
|
||||||
|
### Task 1.3: Temporal Configuration + Secrets
|
||||||
|
- [ ] Environment variables (TEMPORAL_HOST, TEMPORAL_NAMESPACE)
|
||||||
|
- [ ] Worker identity configuration
|
||||||
|
- [ ] Task queue setup (synthesis-queue, compaction-queue)
|
||||||
|
- **Effort**: 50 LOC | **Time**: 4 hours
|
||||||
|
- **Dependencies**: 1.1, 1.2
|
||||||
|
- **Blocks**: Phase 2
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Phase 2: Agent Workflows (1-2 weeks)
|
||||||
|
|
||||||
|
### Task 2.1: Synthesis Workflow Definition
|
||||||
|
- [ ] `crates/mem-temporal/src/workflows/synthesis_workflow.rs`
|
||||||
|
- [ ] Workflow orchestration logic
|
||||||
|
- [ ] Activity composition (health check → synthesis → logging → metrics)
|
||||||
|
- [ ] Retry policies (exponential backoff, max 5 retries)
|
||||||
|
- [ ] Heartbeat configuration (every 10s)
|
||||||
|
- **Effort**: 200 LOC | **Time**: 3 days
|
||||||
|
- **Dependencies**: 1.2, 1.3
|
||||||
|
- **Blocks**: 2.3, 2.4
|
||||||
|
|
||||||
|
### Task 2.2: Synthesis Activities (5 activities)
|
||||||
|
- [ ] `MonitorMemoryHealth` activity
|
||||||
|
- GET /health check
|
||||||
|
- Latency measurement
|
||||||
|
- Failure detection
|
||||||
|
|
||||||
|
- [ ] `ExecuteSynthesis` activity
|
||||||
|
- POST /memory/synthesize call
|
||||||
|
- LLM integration
|
||||||
|
- Heartbeat emission
|
||||||
|
|
||||||
|
- [ ] `LogSynthesisResult` activity
|
||||||
|
- POST /memory/ingest (audit)
|
||||||
|
- Temporal audit trail
|
||||||
|
|
||||||
|
- [ ] `UpdateCacheMetrics` activity
|
||||||
|
- Metric recording
|
||||||
|
- Performance tracking
|
||||||
|
|
||||||
|
- [ ] `CoordinateCompaction` activity
|
||||||
|
- Signal to compaction agent
|
||||||
|
- Readiness check
|
||||||
|
|
||||||
|
- **Effort**: 250 LOC | **Time**: 4 days
|
||||||
|
- **Dependencies**: 2.1
|
||||||
|
- **Blocks**: 2.3
|
||||||
|
|
||||||
|
### Task 2.3: Compaction Workflow Definition
|
||||||
|
- [ ] `crates/mem-temporal/src/workflows/compaction_workflow.rs`
|
||||||
|
- [ ] 4-stage orchestration (identify → dedup → gc → invalidate)
|
||||||
|
- [ ] Failure handling + rollback strategy
|
||||||
|
- **Effort**: 150 LOC | **Time**: 2 days
|
||||||
|
- **Dependencies**: 1.2, 1.3
|
||||||
|
- **Blocks**: 2.4
|
||||||
|
|
||||||
|
### Task 2.4: Compaction Activities (4 activities)
|
||||||
|
- [ ] `IdentifyDuplicates` activity
|
||||||
|
- [ ] `DeduplicateEdges` activity
|
||||||
|
- [ ] `GarbageCollection` activity
|
||||||
|
- [ ] `InvalidateCache` activity
|
||||||
|
- **Effort**: 200 LOC | **Time**: 3 days
|
||||||
|
- **Dependencies**: 2.3
|
||||||
|
- **Blocks**: Integration tests
|
||||||
|
|
||||||
|
### Task 2.5: Worker + Task Queue Registration
|
||||||
|
- [ ] Activity worker setup
|
||||||
|
- [ ] Workflow worker setup
|
||||||
|
- [ ] Task queue polling
|
||||||
|
- [ ] Namespace configuration
|
||||||
|
- **Effort**: 100 LOC | **Time**: 1 day
|
||||||
|
- **Dependencies**: 2.1-2.4
|
||||||
|
- **Blocks**: Phase 3
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Phase 3: Agent Self-Awareness (2-3 weeks)
|
||||||
|
|
||||||
|
### Task 3.1: AGENT_PROMPT Entity Type
|
||||||
|
- [ ] Schema: New entity type in memory_entity
|
||||||
|
- [ ] Repository: `synthesis_cache_repo.rs` (get_agent_prompt)
|
||||||
|
- [ ] Migration: Add to entity type enum
|
||||||
|
- [ ] Activity: Load prompt on agent startup
|
||||||
|
- **Effort**: 100 LOC | **Time**: 1 day
|
||||||
|
- **Dependencies**: Memory service
|
||||||
|
- **Blocks**: 3.2
|
||||||
|
|
||||||
|
### Task 3.2: AGENT_SKILL Linking
|
||||||
|
- [ ] Edge type: agent → skill relationships
|
||||||
|
- [ ] Repository methods: link_agent_to_skill, get_agent_skills
|
||||||
|
- [ ] Confidence tracking per skill
|
||||||
|
- [ ] Success rate calculation
|
||||||
|
- **Effort**: 80 LOC | **Time**: 1 day
|
||||||
|
- **Dependencies**: 3.1
|
||||||
|
- **Blocks**: 3.4
|
||||||
|
|
||||||
|
### Task 3.3: AGENT_PERFORMANCE Metrics
|
||||||
|
- [ ] Entity type: Temporal metrics
|
||||||
|
- [ ] Repository: Store + query metrics
|
||||||
|
- [ ] Activity: Log performance data post-execution
|
||||||
|
- [ ] Time window filtering (last_7_days, last_30_days)
|
||||||
|
- **Effort**: 120 LOC | **Time**: 2 days
|
||||||
|
- **Dependencies**: 3.1
|
||||||
|
- **Blocks**: 3.4
|
||||||
|
|
||||||
|
### Task 3.4: Agent Decision Tracking + Learning
|
||||||
|
- [ ] Edge type: agent_decision_outcome
|
||||||
|
- [ ] Decision logging (parameter, value, confidence before)
|
||||||
|
- [ ] Outcome recording (result, metric)
|
||||||
|
- [ ] Confidence evolution (update after outcome)
|
||||||
|
- [ ] Learning loop in agent code
|
||||||
|
- **Effort**: 200 LOC | **Time**: 3 days
|
||||||
|
- **Dependencies**: 3.1-3.3
|
||||||
|
- **Blocks**: 3.5
|
||||||
|
|
||||||
|
### Task 3.5: Agent Audit Trail Integration
|
||||||
|
- [ ] Dual audit: Temporal history + Memory entities
|
||||||
|
- [ ] Query interface for reviewers
|
||||||
|
- [ ] Temporal CLI integration
|
||||||
|
- [ ] Retention policy (365 days)
|
||||||
|
- **Effort**: 100 LOC | **Time**: 1 day
|
||||||
|
- **Dependencies**: 3.1-3.4
|
||||||
|
- **Blocks**: Testing
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Testing & Documentation
|
||||||
|
|
||||||
|
### Task 4.1: Integration Tests
|
||||||
|
- [ ] Workflow execution end-to-end
|
||||||
|
- [ ] Activity retry behavior
|
||||||
|
- [ ] Heartbeat detection
|
||||||
|
- [ ] Failure recovery
|
||||||
|
- [ ] State replay on restart
|
||||||
|
- **Effort**: 300 LOC | **Time**: 3 days
|
||||||
|
- **Dependencies**: Phase 2 complete
|
||||||
|
- **Blocks**: Integration
|
||||||
|
|
||||||
|
### Task 4.2: Monitoring & Observability
|
||||||
|
- [ ] Temporal UI setup (temporal.riotpiao.com)
|
||||||
|
- [ ] Prometheus metrics export
|
||||||
|
- [ ] Alerting rules (workflow timeout, activity failure)
|
||||||
|
- [ ] Grafana dashboards
|
||||||
|
- **Effort**: 150 LOC | **Time**: 2 days
|
||||||
|
- **Dependencies**: Phase 1 complete
|
||||||
|
- **Blocks**: Production
|
||||||
|
|
||||||
|
### Task 4.3: Documentation
|
||||||
|
- [ ] Agent architecture diagram
|
||||||
|
- [ ] Workflow execution flow
|
||||||
|
- [ ] Operational runbook
|
||||||
|
- [ ] Troubleshooting guide
|
||||||
|
- **Effort**: 50 LOC | **Time**: 1 day
|
||||||
|
- **Dependencies**: All phases
|
||||||
|
- **Blocks**: Release
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Credentials Status
|
||||||
|
|
||||||
|
✅ **SOPS Encrypted**: `k8s/app/memory-agent-secrets.enc.yaml`
|
||||||
|
- CLIENT_ID: `memory-agent`
|
||||||
|
- CLIENT_SECRET: Encrypted
|
||||||
|
- TOKEN_URL: `https://authentik.riotpiao.com/application/o/token/`
|
||||||
|
- AUTHENTIK_ISSUER: `https://authentik.riotpiao.com/application/o/memory-agent/`
|
||||||
|
|
||||||
|
✅ **JWT Auth Verified**: `memory-agent` credentials working
|
||||||
|
- Test result: Token obtained successfully
|
||||||
|
- Expiry: 1 hour (3600s)
|
||||||
|
- Scopes: Default (sufficient for LLM operations)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Timeline
|
||||||
|
|
||||||
|
```
|
||||||
|
Week 1 (Phase 1): Temporal setup
|
||||||
|
Week 2-3 (Phase 2): Agent workflows
|
||||||
|
Week 4-5 (Phase 3): Self-awareness
|
||||||
|
Week 6 (Testing + Docs): Integration + release
|
||||||
|
```
|
||||||
|
|
||||||
|
**Start Date**: TBD
|
||||||
|
**Target End Date**: TBD (+4-6 weeks)
|
||||||
|
|
||||||
@@ -0,0 +1,191 @@
|
|||||||
|
# Current Status - Poimen Memory Service (2026-01-08)
|
||||||
|
|
||||||
|
## ✅ COMPLETED THIS SESSION
|
||||||
|
|
||||||
|
### 1. Removed AccessGuard RBAC (Blocker Issue #1)
|
||||||
|
- ❌ ~~AccessGuard initialization~~ REMOVED
|
||||||
|
- ❌ ~~RBAC checks in handlers~~ REMOVED
|
||||||
|
- ❌ ~~Permission-based access control~~ DEFERRED
|
||||||
|
- ✅ Code now compiles with `cargo build --release`
|
||||||
|
- ✅ Binary created: `target/release/mem`
|
||||||
|
|
||||||
|
### 2. HTTP Handler Initialization Fixed
|
||||||
|
- ✅ Added error handling for schema initialization
|
||||||
|
- ✅ Server reaches "Starting HTTP server" log message
|
||||||
|
- ✅ HTTP server binds to port (processes created)
|
||||||
|
|
||||||
|
## ⚠️ CURRENT ISSUE
|
||||||
|
|
||||||
|
**Server binds to port but exits immediately (silent failure)**
|
||||||
|
|
||||||
|
Process is created and runs `serve` command, but:
|
||||||
|
- Process exits with code 0 (clean exit, no crash)
|
||||||
|
- No HTTP requests answered (port refuses connections)
|
||||||
|
- Logs don't show "listening on 0.0.0.0:8080" message
|
||||||
|
|
||||||
|
**Suspected cause**: Something in the handler initialization or routing setup is blocking/panicking but not showing in logs.
|
||||||
|
|
||||||
|
## 🔧 DEBUGGING STEPS NEEDED
|
||||||
|
|
||||||
|
1. Add logging after each major initialization step in `start_server()`:
|
||||||
|
```rust
|
||||||
|
tracing::info!("About to create AppState");
|
||||||
|
let state = web::Data::new(AppState { ... });
|
||||||
|
tracing::info!("AppState created");
|
||||||
|
|
||||||
|
tracing::info!("About to create HttpServer");
|
||||||
|
HttpServer::new(move || { ... })
|
||||||
|
tracing::info!("HttpServer created, about to bind");
|
||||||
|
|
||||||
|
.bind(("0.0.0.0", port))?
|
||||||
|
tracing::info!("Bound to port {}", port);
|
||||||
|
|
||||||
|
.run()
|
||||||
|
tracing::info!("About to run()");
|
||||||
|
.await?;
|
||||||
|
tracing::info!("Server running");
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Run with `RUST_BACKTRACE=1` to see panics
|
||||||
|
3. Check if the issue is in handler route registration
|
||||||
|
|
||||||
|
## 📋 NEXT PRIORITY FIXES (AFTER SERVER RUNS)
|
||||||
|
|
||||||
|
### Phase 1: INGEST PIPELINE ⭐ CRITICAL
|
||||||
|
**File**: `crates/mem-cli/src/ingest_worker.rs`
|
||||||
|
|
||||||
|
Currently: Just stores raw vectors
|
||||||
|
```rust
|
||||||
|
// WRONG - just vector storage
|
||||||
|
store_chunk_l0(&l0_chunk);
|
||||||
|
store_memory_l1(&l1_memory);
|
||||||
|
```
|
||||||
|
|
||||||
|
Should: Extract entities + facts + edges
|
||||||
|
```rust
|
||||||
|
// 1. Extract entities
|
||||||
|
let entities = entity_extractor.extract(&content).await?;
|
||||||
|
|
||||||
|
// 2. Extract facts/relationships
|
||||||
|
let facts = fact_extractor.extract(&content, &entities).await?;
|
||||||
|
|
||||||
|
// 3. Create temporal edges
|
||||||
|
for fact in facts {
|
||||||
|
let edge = TemporalEdge {
|
||||||
|
source: fact.source_entity,
|
||||||
|
target: fact.target_entity,
|
||||||
|
relation: fact.relation,
|
||||||
|
fact: fact.text,
|
||||||
|
t_valid: now(),
|
||||||
|
t_invalid: None,
|
||||||
|
confidence: 0.8, // GRM gate score
|
||||||
|
version: 1,
|
||||||
|
};
|
||||||
|
edge_repo.insert(&edge).await?;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. Queue contradictions for review
|
||||||
|
for edge in &edges {
|
||||||
|
if contradiction_detector.detect(edge, existing_edges)? {
|
||||||
|
review_queue.enqueue(edge).await?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Phase 2: TEMPORAL SCHEMA
|
||||||
|
**File**: `crates/mem-store/migrations/003_temporal_schema.sql`
|
||||||
|
|
||||||
|
Add columns:
|
||||||
|
- `t_valid TIMESTAMP NOT NULL DEFAULT NOW()`
|
||||||
|
- `t_invalid TIMESTAMP`
|
||||||
|
- `confidence FLOAT DEFAULT 0.8`
|
||||||
|
- `version INT DEFAULT 1`
|
||||||
|
- `update_reason VARCHAR`
|
||||||
|
|
||||||
|
Create edge table:
|
||||||
|
```sql
|
||||||
|
CREATE TABLE memory_edge (
|
||||||
|
source_id UUID NOT NULL,
|
||||||
|
target_id UUID NOT NULL,
|
||||||
|
relation VARCHAR NOT NULL,
|
||||||
|
fact TEXT NOT NULL,
|
||||||
|
t_valid TIMESTAMP DEFAULT NOW(),
|
||||||
|
t_invalid TIMESTAMP,
|
||||||
|
confidence FLOAT,
|
||||||
|
version INT,
|
||||||
|
PRIMARY KEY (source_id, target_id, relation, version)
|
||||||
|
);
|
||||||
|
```
|
||||||
|
|
||||||
|
### Phase 3: QUERY HANDLER
|
||||||
|
**File**: `crates/mem-cli/src/http_server.rs`
|
||||||
|
|
||||||
|
Change `query_handler()` from vector-only to graph-aware:
|
||||||
|
```rust
|
||||||
|
// 1. Vector search
|
||||||
|
let results = semantic_search(query)?;
|
||||||
|
|
||||||
|
// 2. Follow edges
|
||||||
|
let mut expanded = results;
|
||||||
|
for entity in results {
|
||||||
|
let related = edge_repo.find_by_source(&entity.id).await?;
|
||||||
|
expanded.extend(related);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Apply temporal filter
|
||||||
|
expanded.retain(|e| is_valid_at_time(e, now()));
|
||||||
|
|
||||||
|
// 4. Sort by confidence + recency
|
||||||
|
expanded.sort_by_key(|e| (-e.confidence, -e.t_valid));
|
||||||
|
|
||||||
|
// 5. Return
|
||||||
|
HttpResponse::Ok().json(expanded)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Phase 4: END-TO-END TESTING
|
||||||
|
```bash
|
||||||
|
# 1. Ingest with entities + facts
|
||||||
|
POST /memory/ingest
|
||||||
|
{
|
||||||
|
"project": "test",
|
||||||
|
"source": "transcript://session-1",
|
||||||
|
"ingest_id": "i-001",
|
||||||
|
"records": [{"role": "user", "text": "Kubernetes port conflict...", ...}]
|
||||||
|
}
|
||||||
|
# Expected: {"ingest_id":"i-001","status":"pending"}
|
||||||
|
|
||||||
|
# 2. Check ingest status
|
||||||
|
GET /memory/ingest/i-001
|
||||||
|
# Expected: {"status":"done","entities_count":5,"edges_count":3}
|
||||||
|
|
||||||
|
# 3. Query returns graph
|
||||||
|
POST /memory/query
|
||||||
|
{"project":"test","query":"port conflict resolution"}
|
||||||
|
# Expected: {"results":[
|
||||||
|
# {"type":"entity","name":"Kubernetes","edges":[...]},
|
||||||
|
# {"type":"entity","name":"Port","edges":[...]},
|
||||||
|
# {"type":"fact","source":"Kubernetes","target":"Port","relation":"has-conflict"}
|
||||||
|
# ]}
|
||||||
|
```
|
||||||
|
|
||||||
|
## FILES MODIFIED
|
||||||
|
|
||||||
|
✅ `crates/mem-cli/src/http_server.rs` - Removed RBAC, added error handling
|
||||||
|
✅ Created `STATUS_CURRENT.md` - This file
|
||||||
|
|
||||||
|
## TIMELINE
|
||||||
|
|
||||||
|
- **2026-01-08 16:00**: Fixed HTTP handlers, removed RBAC blocker
|
||||||
|
- **2026-01-08 16:30**: Server init working, but exits on startup
|
||||||
|
- **2026-01-08 16:40**: Debugging server binding issue
|
||||||
|
|
||||||
|
## KEY DECISIONS
|
||||||
|
|
||||||
|
1. **RBAC deferred**: MVP focuses on core ingest/query, auth added later
|
||||||
|
2. **Temporal-first**: All edges must have t_valid/t_invalid for graph compaction
|
||||||
|
3. **GRM gate integrated at ingest time**: Confidence scores assigned when facts extracted
|
||||||
|
4. **No queue worker** in MVP: Enable it after core working
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**Next action**: Add detailed logging to `start_server()` to see where process exits.
|
||||||
@@ -46,4 +46,3 @@ futures-util = "0.3"
|
|||||||
async-stream = "0.3"
|
async-stream = "0.3"
|
||||||
rand = "0.8"
|
rand = "0.8"
|
||||||
lru = "0.12"
|
lru = "0.12"
|
||||||
once_cell = { workspace = true }
|
|
||||||
|
|||||||
@@ -126,3 +126,246 @@ impl Default for MetricsCollector {
|
|||||||
// - Only record_request() needs exclusive write lock
|
// - Only record_request() needs exclusive write lock
|
||||||
// - Performance improvement for high-read scenarios
|
// - Performance improvement for high-read scenarios
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_agent_metrics_default() {
|
||||||
|
let m = AgentMetrics::default();
|
||||||
|
assert_eq!(m.requests_total, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_agent_metrics_creation() {
|
||||||
|
let m = AgentMetrics {
|
||||||
|
agent_id: "a1".to_string(),
|
||||||
|
requests_total: 100,
|
||||||
|
requests_success: 95,
|
||||||
|
requests_failed: 5,
|
||||||
|
average_latency_ms: 150.0,
|
||||||
|
p95_latency_ms: 300.0,
|
||||||
|
p99_latency_ms: 450.0,
|
||||||
|
capabilities_used: HashMap::new(),
|
||||||
|
last_updated: "2025-01-30T10:00:00Z".to_string(),
|
||||||
|
};
|
||||||
|
assert_eq!(m.requests_total, 100);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_metrics_collector_creation() {
|
||||||
|
let collector = MetricsCollector::new();
|
||||||
|
assert!(collector.get_metrics("unknown").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_metrics_collector_concurrent_reads() {
|
||||||
|
let collector = std::sync::Arc::new(MetricsCollector::new());
|
||||||
|
collector.record_request("agent1", true, 100.0, None);
|
||||||
|
|
||||||
|
let mut handles = vec![];
|
||||||
|
for _ in 0..5 {
|
||||||
|
let c = collector.clone();
|
||||||
|
let handle = std::thread::spawn(move || {
|
||||||
|
c.get_metrics("agent1")
|
||||||
|
});
|
||||||
|
handles.push(handle);
|
||||||
|
}
|
||||||
|
|
||||||
|
for handle in handles {
|
||||||
|
assert!(handle.join().unwrap().is_some());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_metrics_collector_record_success() {
|
||||||
|
let collector = MetricsCollector::new();
|
||||||
|
collector.record_request("agent1", true, 100.0, Some("synthesis"));
|
||||||
|
|
||||||
|
let metrics = collector.get_metrics("agent1");
|
||||||
|
assert!(metrics.is_some());
|
||||||
|
let m = metrics.unwrap();
|
||||||
|
assert_eq!(m.requests_total, 1);
|
||||||
|
assert_eq!(m.requests_success, 1);
|
||||||
|
assert_eq!(m.requests_failed, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_metrics_success_rate_calc() {
|
||||||
|
let collector = MetricsCollector::new();
|
||||||
|
for _ in 0..9 {
|
||||||
|
collector.record_request("agent1", true, 100.0, None);
|
||||||
|
}
|
||||||
|
collector.record_request("agent1", false, 50.0, None);
|
||||||
|
|
||||||
|
let m = collector.get_metrics("agent1").unwrap();
|
||||||
|
let success_rate = m.requests_success as f32 / m.requests_total as f32;
|
||||||
|
assert!((success_rate - 0.9).abs() < 0.01);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_metrics_collector_record_failure() {
|
||||||
|
let collector = MetricsCollector::new();
|
||||||
|
collector.record_request("agent1", false, 50.0, None);
|
||||||
|
|
||||||
|
let metrics = collector.get_metrics("agent1");
|
||||||
|
let m = metrics.unwrap();
|
||||||
|
assert_eq!(m.requests_failed, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_metrics_no_contention() {
|
||||||
|
let collector = std::sync::Arc::new(MetricsCollector::new());
|
||||||
|
let mut handles = vec![];
|
||||||
|
|
||||||
|
for i in 0..5 {
|
||||||
|
let c = collector.clone();
|
||||||
|
let h1 = std::thread::spawn(move || {
|
||||||
|
c.record_request(&format!("agent{}", i), true, 100.0, None);
|
||||||
|
});
|
||||||
|
handles.push(h1);
|
||||||
|
|
||||||
|
let c = collector.clone();
|
||||||
|
let h2 = std::thread::spawn(move || {
|
||||||
|
c.get_metrics(&format!("agent{}", i))
|
||||||
|
});
|
||||||
|
handles.push(h2);
|
||||||
|
}
|
||||||
|
|
||||||
|
for h in handles {
|
||||||
|
h.join().unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_metrics_collector_multiple_records() {
|
||||||
|
let collector = MetricsCollector::new();
|
||||||
|
collector.record_request("agent1", true, 100.0, None);
|
||||||
|
collector.record_request("agent1", true, 150.0, None);
|
||||||
|
collector.record_request("agent1", false, 50.0, None);
|
||||||
|
|
||||||
|
let metrics = collector.get_metrics("agent1");
|
||||||
|
let m = metrics.unwrap();
|
||||||
|
assert_eq!(m.requests_total, 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_metrics_fail_count() {
|
||||||
|
let collector = MetricsCollector::new();
|
||||||
|
collector.record_request("agent1", false, 100.0, None);
|
||||||
|
collector.record_request("agent1", false, 120.0, None);
|
||||||
|
|
||||||
|
let metrics = collector.get_metrics("agent1").unwrap();
|
||||||
|
assert_eq!(metrics.requests_failed, 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_metrics_collector_capability_tracking() {
|
||||||
|
let collector = MetricsCollector::new();
|
||||||
|
collector.record_request("agent1", true, 100.0, Some("linking"));
|
||||||
|
collector.record_request("agent1", true, 120.0, Some("linking"));
|
||||||
|
collector.record_request("agent1", true, 110.0, Some("inference"));
|
||||||
|
|
||||||
|
let metrics = collector.get_metrics("agent1");
|
||||||
|
let m = metrics.unwrap();
|
||||||
|
assert_eq!(m.capabilities_used.get("linking"), Some(&2));
|
||||||
|
assert_eq!(m.capabilities_used.get("inference"), Some(&1));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_metrics_thread_safety() {
|
||||||
|
let collector = std::sync::Arc::new(MetricsCollector::new());
|
||||||
|
let mut handles = vec![];
|
||||||
|
|
||||||
|
for i in 0..10 {
|
||||||
|
let c = collector.clone();
|
||||||
|
let handle = std::thread::spawn(move || {
|
||||||
|
c.record_request(&format!("agent{}", i), true, 100.0, None);
|
||||||
|
});
|
||||||
|
handles.push(handle);
|
||||||
|
}
|
||||||
|
|
||||||
|
for handle in handles {
|
||||||
|
handle.join().unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_eq!(collector.get_all_metrics().len(), 10);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_metrics_collector_get_all() {
|
||||||
|
let collector = MetricsCollector::new();
|
||||||
|
collector.record_request("agent1", true, 100.0, None);
|
||||||
|
collector.record_request("agent2", true, 150.0, None);
|
||||||
|
|
||||||
|
let all = collector.get_all_metrics();
|
||||||
|
assert_eq!(all.len(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_metrics_read_while_other_writes() {
|
||||||
|
let collector = std::sync::Arc::new(MetricsCollector::new());
|
||||||
|
collector.record_request("agent1", true, 100.0, None);
|
||||||
|
|
||||||
|
let c1 = collector.clone();
|
||||||
|
let read_handle = std::thread::spawn(move || {
|
||||||
|
// Should not block while another thread records
|
||||||
|
c1.get_metrics("agent1")
|
||||||
|
});
|
||||||
|
|
||||||
|
let c2 = collector.clone();
|
||||||
|
let write_handle = std::thread::spawn(move || {
|
||||||
|
c2.record_request("agent2", true, 150.0, None);
|
||||||
|
});
|
||||||
|
|
||||||
|
read_handle.join().unwrap();
|
||||||
|
write_handle.join().unwrap();
|
||||||
|
assert_eq!(collector.get_all_metrics().len(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_metrics_collector_reset() {
|
||||||
|
let collector = MetricsCollector::new();
|
||||||
|
collector.record_request("agent1", true, 100.0, None);
|
||||||
|
assert!(collector.get_metrics("agent1").is_some());
|
||||||
|
|
||||||
|
collector.reset("agent1");
|
||||||
|
assert!(collector.get_metrics("agent1").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_metrics_isolation() {
|
||||||
|
let collector = MetricsCollector::new();
|
||||||
|
collector.record_request("agent1", true, 100.0, None);
|
||||||
|
collector.record_request("agent2", true, 150.0, None);
|
||||||
|
|
||||||
|
let m1 = collector.get_metrics("agent1").unwrap();
|
||||||
|
let m2 = collector.get_metrics("agent2").unwrap();
|
||||||
|
|
||||||
|
assert_ne!(m1.agent_id, m2.agent_id);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_latency_percentiles() {
|
||||||
|
let collector = MetricsCollector::new();
|
||||||
|
for i in 1..=30 {
|
||||||
|
collector.record_request("agent1", true, (i * 10) as f32, None);
|
||||||
|
}
|
||||||
|
|
||||||
|
let metrics = collector.get_metrics("agent1");
|
||||||
|
let m = metrics.unwrap();
|
||||||
|
assert!(m.average_latency_ms > 0.0);
|
||||||
|
assert!(m.p95_latency_ms > m.average_latency_ms);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_rwlock_behavior() {
|
||||||
|
let collector = MetricsCollector::new();
|
||||||
|
collector.record_request("agent1", true, 100.0, None);
|
||||||
|
let m1 = collector.get_metrics("agent1");
|
||||||
|
let m2 = collector.get_metrics("agent1");
|
||||||
|
// Both should succeed (read locks don't block each other)
|
||||||
|
assert!(m1.is_some());
|
||||||
|
assert!(m2.is_some());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -224,20 +224,9 @@ impl KvCacheAligner {
|
|||||||
|
|
||||||
/// Pre-load hot chunks into cache
|
/// Pre-load hot chunks into cache
|
||||||
pub fn preload_hot_chunks(&self, hot_chunks: Vec<(&str, &str)>) -> Result<()> {
|
pub fn preload_hot_chunks(&self, hot_chunks: Vec<(&str, &str)>) -> Result<()> {
|
||||||
let count = hot_chunks.len();
|
|
||||||
for (chunk_id, text) in hot_chunks {
|
for (chunk_id, text) in hot_chunks {
|
||||||
self.cache.put(chunk_id, text);
|
self.cache.put(chunk_id, text);
|
||||||
}
|
}
|
||||||
let metrics = self.cache.metrics();
|
|
||||||
tracing::info!(
|
|
||||||
target: "observability",
|
|
||||||
event = "cache_preload",
|
|
||||||
preloaded = count,
|
|
||||||
cache_hits = metrics.hits,
|
|
||||||
cache_misses = metrics.misses,
|
|
||||||
hit_ratio = format!("{:.2}", metrics.hit_ratio()),
|
|
||||||
"Cache preload complete"
|
|
||||||
);
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -211,31 +211,16 @@ impl ChunkOptimizer {
|
|||||||
|
|
||||||
/// End-to-end optimization pipeline
|
/// End-to-end optimization pipeline
|
||||||
pub fn optimize(&self, chunks: Vec<OptimizableChunk>) -> (Vec<OptimizableChunk>, SelectionMetrics) {
|
pub fn optimize(&self, chunks: Vec<OptimizableChunk>) -> (Vec<OptimizableChunk>, SelectionMetrics) {
|
||||||
let input_count = chunks.len();
|
|
||||||
|
|
||||||
// Step 1: Filter by threshold
|
// Step 1: Filter by threshold
|
||||||
let filtered = self.threshold_filter.filter(chunks.clone());
|
let filtered = self.threshold_filter.filter(chunks.clone());
|
||||||
let after_filter = filtered.len();
|
|
||||||
|
|
||||||
// Step 2: Deduplicate
|
// Step 2: Deduplicate
|
||||||
let (deduplicated, dedup_removed) = self.deduplicator.deduplicate(filtered);
|
let (deduplicated, dedup_removed) = self.deduplicator.deduplicate(filtered);
|
||||||
let after_dedup = deduplicated.len();
|
|
||||||
|
|
||||||
// Step 3: Select within budget
|
// Step 3: Select within budget
|
||||||
let (selected, mut metrics) = self.budget_selector.select(deduplicated);
|
let (selected, mut metrics) = self.budget_selector.select(deduplicated);
|
||||||
metrics.dedup_removed = dedup_removed;
|
|
||||||
|
|
||||||
tracing::info!(
|
metrics.dedup_removed = dedup_removed;
|
||||||
target: "observability",
|
|
||||||
event = "chunk_optimize",
|
|
||||||
input = input_count,
|
|
||||||
after_threshold_filter = after_filter,
|
|
||||||
after_dedup = after_dedup,
|
|
||||||
dedup_removed = dedup_removed,
|
|
||||||
selected = selected.len(),
|
|
||||||
budget_bytes = metrics.total_bytes,
|
|
||||||
"Chunk optimization complete"
|
|
||||||
);
|
|
||||||
|
|
||||||
(selected, metrics)
|
(selected, metrics)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -346,19 +346,7 @@ pub async fn compact_memory(
|
|||||||
}
|
}
|
||||||
|
|
||||||
total_stats.duration_ms = start.elapsed().as_millis() as u64;
|
total_stats.duration_ms = start.elapsed().as_millis() as u64;
|
||||||
info!(
|
info!("Compaction complete in {}ms: {:?}", total_stats.duration_ms, total_stats);
|
||||||
target: "observability",
|
|
||||||
event = "compaction_complete",
|
|
||||||
mode = ?mode,
|
|
||||||
duration_ms = total_stats.duration_ms,
|
|
||||||
duplicate_edges_deleted = total_stats.duplicate_edges_deleted,
|
|
||||||
stale_facts_deleted = total_stats.stale_facts_deleted,
|
|
||||||
semantic_merged = total_stats.semantic_merged,
|
|
||||||
llm_calls = total_stats.llm_calls,
|
|
||||||
bytes_freed = total_stats.bytes_freed,
|
|
||||||
human_reviews_queued = total_stats.human_reviews_queued,
|
|
||||||
"Compaction complete"
|
|
||||||
);
|
|
||||||
|
|
||||||
Ok(total_stats)
|
Ok(total_stats)
|
||||||
}
|
}
|
||||||
@@ -385,7 +373,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[ignore = "not yet implemented - needs mock pool"]
|
|
||||||
fn test_confidence_thresholds() {
|
fn test_confidence_thresholds() {
|
||||||
let tier2 = Tier2Compactor::new(
|
let tier2 = Tier2Compactor::new(
|
||||||
// Mock pool would go here
|
// Mock pool would go here
|
||||||
|
|||||||
@@ -344,22 +344,6 @@ impl FullPipeline {
|
|||||||
|
|
||||||
metrics.total_latency_ms = start.elapsed().as_millis() as u64;
|
metrics.total_latency_ms = start.elapsed().as_millis() as u64;
|
||||||
|
|
||||||
tracing::info!(
|
|
||||||
target: "observability",
|
|
||||||
event = "full_pipeline_complete",
|
|
||||||
query = query,
|
|
||||||
candidates = metrics.wiki_scope_docs,
|
|
||||||
prefiltered = metrics.prefilter_candidates,
|
|
||||||
optimized = metrics.post_optimization_count,
|
|
||||||
dedup_removed = metrics.dedup_removed,
|
|
||||||
boosts_applied = metrics.metadata_boosts_applied,
|
|
||||||
cache_hit_ratio = format!("{:.2}", metrics.cache_hit_ratio),
|
|
||||||
budget_bytes = metrics.budget_used_bytes,
|
|
||||||
total_ms = metrics.total_latency_ms,
|
|
||||||
"Full query pipeline complete"
|
|
||||||
);
|
|
||||||
|
|
||||||
|
|
||||||
Ok(PipelineResult {
|
Ok(PipelineResult {
|
||||||
query: query.to_string(),
|
query: query.to_string(),
|
||||||
query_intent,
|
query_intent,
|
||||||
@@ -483,22 +467,6 @@ impl FullPipeline {
|
|||||||
|
|
||||||
metrics.total_latency_ms = start.elapsed().as_millis() as u64;
|
metrics.total_latency_ms = start.elapsed().as_millis() as u64;
|
||||||
|
|
||||||
tracing::info!(
|
|
||||||
target: "observability",
|
|
||||||
event = "full_pipeline_complete",
|
|
||||||
query = query,
|
|
||||||
candidates = metrics.wiki_scope_docs,
|
|
||||||
prefiltered = metrics.prefilter_candidates,
|
|
||||||
optimized = metrics.post_optimization_count,
|
|
||||||
dedup_removed = metrics.dedup_removed,
|
|
||||||
boosts_applied = metrics.metadata_boosts_applied,
|
|
||||||
cache_hit_ratio = format!("{:.2}", metrics.cache_hit_ratio),
|
|
||||||
budget_bytes = metrics.budget_used_bytes,
|
|
||||||
total_ms = metrics.total_latency_ms,
|
|
||||||
"Full query pipeline complete"
|
|
||||||
);
|
|
||||||
|
|
||||||
|
|
||||||
Ok(PipelineResult {
|
Ok(PipelineResult {
|
||||||
query: query.to_string(),
|
query: query.to_string(),
|
||||||
query_intent,
|
query_intent,
|
||||||
|
|||||||
@@ -6,7 +6,6 @@
|
|||||||
use crate::queue_adapter::{QueueAdapter, QueueMessage, QueueStats};
|
use crate::queue_adapter::{QueueAdapter, QueueMessage, QueueStats};
|
||||||
use anyhow::{anyhow, Result};
|
use anyhow::{anyhow, Result};
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use base64::Engine;
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
@@ -235,7 +234,7 @@ impl QueueAdapter for GatewayQueueAdapter {
|
|||||||
let token = self.token_source.token().await?;
|
let token = self.token_source.token().await?;
|
||||||
|
|
||||||
// Base64 encode body
|
// Base64 encode body
|
||||||
let encoded_body = base64::engine::general_purpose::STANDARD.encode(body.as_bytes());
|
let encoded_body = base64::encode(body.as_bytes());
|
||||||
|
|
||||||
// Build request
|
// Build request
|
||||||
let mut attrs = attributes;
|
let mut attrs = attributes;
|
||||||
@@ -312,7 +311,7 @@ impl QueueAdapter for GatewayQueueAdapter {
|
|||||||
if let Some(sqs_msgs) = sqs_resp.messages {
|
if let Some(sqs_msgs) = sqs_resp.messages {
|
||||||
for msg in sqs_msgs {
|
for msg in sqs_msgs {
|
||||||
// Decode body from base64
|
// Decode body from base64
|
||||||
let body_bytes = base64::engine::general_purpose::STANDARD.decode(msg.body.as_bytes())?;
|
let body_bytes = base64::decode(msg.body.as_bytes())?;
|
||||||
let body = String::from_utf8(body_bytes)?;
|
let body = String::from_utf8(body_bytes)?;
|
||||||
|
|
||||||
let chunk_id = msg
|
let chunk_id = msg
|
||||||
@@ -405,7 +404,7 @@ impl QueueAdapter for GatewayQueueAdapter {
|
|||||||
})
|
})
|
||||||
.to_string();
|
.to_string();
|
||||||
|
|
||||||
let encoded_body = base64::engine::general_purpose::STANDARD.encode(dlq_body.as_bytes());
|
let encoded_body = base64::encode(dlq_body.as_bytes());
|
||||||
|
|
||||||
let req = SendMessageRequest {
|
let req = SendMessageRequest {
|
||||||
message_body: encoded_body,
|
message_body: encoded_body,
|
||||||
@@ -512,8 +511,8 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn test_base64_roundtrip() {
|
fn test_base64_roundtrip() {
|
||||||
let original = "hello world";
|
let original = "hello world";
|
||||||
let encoded = base64::engine::general_purpose::STANDARD.encode(original.as_bytes());
|
let encoded = base64::encode(original.as_bytes());
|
||||||
let decoded = String::from_utf8(base64::engine::general_purpose::STANDARD.decode(encoded.as_bytes()).unwrap()).unwrap();
|
let decoded = String::from_utf8(base64::decode(encoded.as_bytes()).unwrap()).unwrap();
|
||||||
assert_eq!(decoded, original);
|
assert_eq!(decoded, original);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,18 +1,11 @@
|
|||||||
//! Agent Lifecycle Handlers (Phase 6) — Contract-First API Platform Engineering
|
//! Agent Lifecycle Handlers (Phase 6)
|
||||||
//!
|
|
||||||
//! Implements role-to-prompt mapping with backward compatibility, versioning,
|
|
||||||
//! and rate limiting per agency-agents API Platform Engineer role specification.
|
|
||||||
|
|
||||||
use actix_web::{web, HttpRequest, HttpResponse};
|
use actix_web::{web, HttpRequest, HttpResponse};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use uuid::Uuid;
|
|
||||||
use chrono::Utc;
|
|
||||||
use crate::agent::{Agent, AgentConfig, AgentCapability, DefaultAgent};
|
use crate::agent::{Agent, AgentConfig, AgentCapability, DefaultAgent};
|
||||||
use crate::agent::client_sdk::SynthesisClient;
|
use crate::agent::client_sdk::SynthesisClient;
|
||||||
use crate::handlers::response_builder;
|
use crate::handlers::response_builder;
|
||||||
use mem_store::agent_repo::{AgentRepository, AgentPrompt, AgentSkill, AgentDecision, RolePromptMapping};
|
|
||||||
use crate::metrics::{ERROR_AUTH_FAILURE_AGENT, ERROR_BAD_REQUEST_AGENT, ERROR_NOT_FOUND_AGENT, ERROR_UNEXPECTED_AGENT, ERROR_UNEXPECTED_TOTAL};
|
|
||||||
use tracing::{debug, info, error, warn};
|
use tracing::{debug, info, error, warn};
|
||||||
|
|
||||||
/// Register agent request
|
/// Register agent request
|
||||||
@@ -52,14 +45,10 @@ pub async fn register_agent_handler(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if body.agent_id.is_empty() || body.project_id.is_empty() {
|
if body.agent_id.is_empty() || body.project_id.is_empty() {
|
||||||
ERROR_BAD_REQUEST_AGENT.inc();
|
|
||||||
warn!(agent_id = %body.agent_id, "Expected error: missing agent_id or project_id");
|
|
||||||
return response_builder::bad_request("agent_id and project_id required");
|
return response_builder::bad_request("agent_id and project_id required");
|
||||||
}
|
}
|
||||||
|
|
||||||
if body.capabilities.is_empty() {
|
if body.capabilities.is_empty() {
|
||||||
ERROR_BAD_REQUEST_AGENT.inc();
|
|
||||||
warn!(agent_id = %body.agent_id, "Expected error: no capabilities provided");
|
|
||||||
return response_builder::bad_request("At least one capability required");
|
return response_builder::bad_request("At least one capability required");
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -79,8 +68,6 @@ pub async fn register_agent_handler(
|
|||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
if caps.is_empty() {
|
if caps.is_empty() {
|
||||||
ERROR_BAD_REQUEST_AGENT.inc();
|
|
||||||
warn!(agent_id = %body.agent_id, "Expected error: invalid capability names");
|
|
||||||
return response_builder::bad_request("Invalid capabilities");
|
return response_builder::bad_request("Invalid capabilities");
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -93,56 +80,7 @@ pub async fn register_agent_handler(
|
|||||||
metadata: std::collections::HashMap::new(),
|
metadata: std::collections::HashMap::new(),
|
||||||
};
|
};
|
||||||
|
|
||||||
// Persist agent config to database via agent_registry table
|
// Store agent config (stub: would persist to DB)
|
||||||
let agent_repo = AgentRepository::new(state.pool.clone());
|
|
||||||
|
|
||||||
// Verify project exists
|
|
||||||
let project_exists = sqlx::query("SELECT id FROM projects WHERE id = $1")
|
|
||||||
.bind(&body.project_id)
|
|
||||||
.fetch_optional(&state.pool)
|
|
||||||
.await;
|
|
||||||
|
|
||||||
if let Err(e) = project_exists {
|
|
||||||
ERROR_UNEXPECTED_AGENT.inc();
|
|
||||||
ERROR_UNEXPECTED_TOTAL.inc();
|
|
||||||
error!(agent_id = %body.agent_id, error = %e, "Unexpected error: DB failure verifying project");
|
|
||||||
return response_builder::internal_error("Database error during project verification");
|
|
||||||
}
|
|
||||||
|
|
||||||
if project_exists.unwrap().is_none() {
|
|
||||||
ERROR_NOT_FOUND_AGENT.inc();
|
|
||||||
info!(agent_id = %body.agent_id, project_id = %body.project_id, "Expected error: project not found");
|
|
||||||
return response_builder::bad_request(&format!("Project not found: {}", body.project_id));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Insert agent registry record
|
|
||||||
let agent_insert = sqlx::query(
|
|
||||||
r#"
|
|
||||||
INSERT INTO agent_registry
|
|
||||||
(project_id, agent_id, capabilities, webhook_url, rate_limit, status)
|
|
||||||
VALUES ($1, $2, $3, $4, $5, 'active')
|
|
||||||
ON CONFLICT (project_id, agent_id) DO UPDATE SET
|
|
||||||
capabilities = $3,
|
|
||||||
webhook_url = $4,
|
|
||||||
rate_limit = $5,
|
|
||||||
updated_at = NOW()
|
|
||||||
"#
|
|
||||||
)
|
|
||||||
.bind(&body.project_id)
|
|
||||||
.bind(&body.agent_id)
|
|
||||||
.bind(&body.capabilities)
|
|
||||||
.bind(&body.webhook_url)
|
|
||||||
.bind(body.rate_limit.unwrap_or(1000) as i32)
|
|
||||||
.execute(&state.pool)
|
|
||||||
.await;
|
|
||||||
|
|
||||||
if let Err(e) = agent_insert {
|
|
||||||
ERROR_UNEXPECTED_AGENT.inc();
|
|
||||||
ERROR_UNEXPECTED_TOTAL.inc();
|
|
||||||
error!(agent_id = %body.agent_id, error = %e, "Unexpected error: DB failure inserting agent");
|
|
||||||
return response_builder::internal_error("Failed to register agent");
|
|
||||||
}
|
|
||||||
|
|
||||||
let agent = DefaultAgent::new(config);
|
let agent = DefaultAgent::new(config);
|
||||||
|
|
||||||
// Extract JWT from request for agent reasoning calls
|
// Extract JWT from request for agent reasoning calls
|
||||||
@@ -152,7 +90,7 @@ pub async fn register_agent_handler(
|
|||||||
warn!("Agent registered without JWT token");
|
warn!("Agent registered without JWT token");
|
||||||
}
|
}
|
||||||
|
|
||||||
info!("Agent registered and persisted: {}", agent.config().agent_id);
|
info!("Agent registered: {}", agent.config().agent_id);
|
||||||
|
|
||||||
// Wire Temporal workflow (via api.riotpiao.com/workflow)
|
// Wire Temporal workflow (via api.riotpiao.com/workflow)
|
||||||
// Temporal activities will:
|
// Temporal activities will:
|
||||||
@@ -194,6 +132,8 @@ pub async fn register_agent_handler(
|
|||||||
let workflow_id = data.get("workflow_id").and_then(|v| v.as_str()).unwrap_or("unknown");
|
let workflow_id = data.get("workflow_id").and_then(|v| v.as_str()).unwrap_or("unknown");
|
||||||
let run_id = data.get("run_id").and_then(|v| v.as_str()).unwrap_or("unknown");
|
let run_id = data.get("run_id").and_then(|v| v.as_str()).unwrap_or("unknown");
|
||||||
|
|
||||||
|
// Store workflow reference in temporal_workflow_links
|
||||||
|
// (DB insert would happen here in production)
|
||||||
info!("Agent workflow started: workflow_id={}, run_id={}", workflow_id, run_id);
|
info!("Agent workflow started: workflow_id={}, run_id={}", workflow_id, run_id);
|
||||||
debug!("Temporal activity will persist agent state + reasoning traces");
|
debug!("Temporal activity will persist agent state + reasoning traces");
|
||||||
}
|
}
|
||||||
@@ -211,51 +151,12 @@ pub async fn register_agent_handler(
|
|||||||
capabilities: body.capabilities.clone(),
|
capabilities: body.capabilities.clone(),
|
||||||
webhook_url: body.webhook_url.clone(),
|
webhook_url: body.webhook_url.clone(),
|
||||||
rate_limit: agent.config().rate_limit,
|
rate_limit: agent.config().rate_limit,
|
||||||
created_at: Utc::now().to_rfc3339(),
|
created_at: chrono::Utc::now().to_rfc3339(),
|
||||||
status: "active".to_string(),
|
status: "active".to_string(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Full agent progress response
|
/// GET /agents/{id} - Get agent status
|
||||||
#[derive(Debug, Serialize)]
|
|
||||||
pub struct AgentProgressResponse {
|
|
||||||
pub agent_id: String,
|
|
||||||
pub project_id: String,
|
|
||||||
pub capabilities: Vec<String>,
|
|
||||||
pub status: String,
|
|
||||||
pub prompts: Vec<PromptResponse>,
|
|
||||||
pub skills: Vec<SkillSummary>,
|
|
||||||
pub decisions: Vec<DecisionSummary>,
|
|
||||||
pub metrics: Option<MetricsSummary>,
|
|
||||||
pub created_at: String,
|
|
||||||
pub updated_at: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Serialize)]
|
|
||||||
pub struct SkillSummary {
|
|
||||||
pub name: String,
|
|
||||||
pub success_rate: f32,
|
|
||||||
pub invocation_count: i64,
|
|
||||||
pub enabled: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Serialize)]
|
|
||||||
pub struct DecisionSummary {
|
|
||||||
pub action: String,
|
|
||||||
pub confidence: f32,
|
|
||||||
pub outcome_success: Option<bool>,
|
|
||||||
pub created_at: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Serialize)]
|
|
||||||
pub struct MetricsSummary {
|
|
||||||
pub requests_total: i64,
|
|
||||||
pub requests_success: i64,
|
|
||||||
pub error_rate: f32,
|
|
||||||
pub average_latency_ms: f32,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// GET /agents/{id} - Get agent progress
|
|
||||||
pub async fn get_agent_handler(
|
pub async fn get_agent_handler(
|
||||||
req: HttpRequest,
|
req: HttpRequest,
|
||||||
path: web::Path<String>,
|
path: web::Path<String>,
|
||||||
@@ -269,110 +170,33 @@ pub async fn get_agent_handler(
|
|||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
|
|
||||||
debug!("Getting agent progress: {}", agent_id);
|
debug!("Getting agent: {}", agent_id);
|
||||||
|
|
||||||
// Fetch agent registry
|
// Extract JWT for agent operations
|
||||||
let agent_row = sqlx::query_as::<_, (String, Vec<String>, Option<String>, i32, String, String, String)>(
|
let jwt = crate::handlers::extract_jwt_token(&req)
|
||||||
r#"SELECT project_id, capabilities, webhook_url, rate_limit, status,
|
.unwrap_or_else(|| {
|
||||||
created_at::text, updated_at::text
|
warn!("No JWT token in get_agent request");
|
||||||
FROM agent_registry WHERE agent_id = $1"#
|
"invalid".to_string()
|
||||||
)
|
});
|
||||||
.bind(&agent_id)
|
|
||||||
.fetch_optional(&state.pool)
|
|
||||||
.await;
|
|
||||||
|
|
||||||
let (project_id, capabilities, _webhook, _rate_limit, status, created_at, updated_at) = match agent_row {
|
// Stub: would fetch from DB
|
||||||
Ok(Some(row)) => row,
|
let config = AgentConfig {
|
||||||
Ok(None) => {
|
agent_id: agent_id.clone(),
|
||||||
ERROR_NOT_FOUND_AGENT.inc();
|
project_id: "poimen".to_string(),
|
||||||
info!(agent_id = %agent_id, "Expected error: agent not found");
|
capabilities: vec![AgentCapability::Summarization],
|
||||||
return response_builder::not_found(&format!("Agent not found: {}", agent_id));
|
webhook_url: None,
|
||||||
}
|
rate_limit: 1000,
|
||||||
Err(e) => {
|
metadata: std::collections::HashMap::new(),
|
||||||
ERROR_UNEXPECTED_AGENT.inc();
|
|
||||||
ERROR_UNEXPECTED_TOTAL.inc();
|
|
||||||
error!(agent_id = %agent_id, error = %e, "Unexpected error: DB failure fetching agent");
|
|
||||||
return response_builder::internal_error("Database error");
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
// Fetch prompts
|
let agent = DefaultAgent::new(config);
|
||||||
let prompts: Vec<PromptResponse> = sqlx::query_as::<_, (String, String, String, Option<String>, String, Vec<String>, i64, f32, i32, String)>(
|
|
||||||
r#"SELECT id::text, name, template, target_model, task_category,
|
|
||||||
tags, usage_count, avg_quality, version, created_at::text
|
|
||||||
FROM agent_prompt WHERE project_id = $1 ORDER BY created_at DESC"#
|
|
||||||
)
|
|
||||||
.bind(&project_id)
|
|
||||||
.fetch_all(&state.pool)
|
|
||||||
.await
|
|
||||||
.unwrap_or_default()
|
|
||||||
.into_iter()
|
|
||||||
.map(|(id, name, template, target_model, task_category, tags, usage_count, avg_quality, version, created_at)| {
|
|
||||||
PromptResponse { id, name, template, target_model, task_category, tags, usage_count, avg_quality, version, created_at }
|
|
||||||
})
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
// Fetch skills
|
match futures::executor::block_on(agent.status()) {
|
||||||
let skills: Vec<SkillSummary> = sqlx::query_as::<_, (String, f32, i64, bool)>(
|
status => {
|
||||||
r#"SELECT name, success_rate, invocation_count, enabled
|
info!("Agent status: {} with JWT auth", agent_id);
|
||||||
FROM agent_skill WHERE agent_id = $1 ORDER BY created_at DESC"#
|
response_builder::success_response(status)
|
||||||
)
|
}
|
||||||
.bind(&agent_id)
|
}
|
||||||
.fetch_all(&state.pool)
|
|
||||||
.await
|
|
||||||
.unwrap_or_default()
|
|
||||||
.into_iter()
|
|
||||||
.map(|(name, success_rate, invocation_count, enabled)| {
|
|
||||||
SkillSummary { name, success_rate, invocation_count, enabled }
|
|
||||||
})
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
// Fetch recent decisions
|
|
||||||
let decisions: Vec<DecisionSummary> = sqlx::query_as::<_, (String, f32, Option<bool>, String)>(
|
|
||||||
r#"SELECT action, confidence, outcome_success, created_at::text
|
|
||||||
FROM agent_decision WHERE agent_id = $1
|
|
||||||
ORDER BY created_at DESC LIMIT 20"#
|
|
||||||
)
|
|
||||||
.bind(&agent_id)
|
|
||||||
.fetch_all(&state.pool)
|
|
||||||
.await
|
|
||||||
.unwrap_or_default()
|
|
||||||
.into_iter()
|
|
||||||
.map(|(action, confidence, outcome_success, created_at)| {
|
|
||||||
DecisionSummary { action, confidence, outcome_success, created_at }
|
|
||||||
})
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
// Fetch latest metrics
|
|
||||||
let metrics = sqlx::query_as::<_, (i64, i64, f32, f32)>(
|
|
||||||
r#"SELECT requests_total, requests_success, error_rate, average_latency_ms
|
|
||||||
FROM agent_metrics WHERE agent_id = $1
|
|
||||||
ORDER BY recorded_at DESC LIMIT 1"#
|
|
||||||
)
|
|
||||||
.bind(&agent_id)
|
|
||||||
.fetch_optional(&state.pool)
|
|
||||||
.await
|
|
||||||
.ok()
|
|
||||||
.flatten()
|
|
||||||
.map(|(requests_total, requests_success, error_rate, average_latency_ms)| {
|
|
||||||
MetricsSummary { requests_total, requests_success, error_rate, average_latency_ms }
|
|
||||||
});
|
|
||||||
|
|
||||||
info!("Agent progress: {} ({} prompts, {} skills, {} decisions)",
|
|
||||||
agent_id, prompts.len(), skills.len(), decisions.len());
|
|
||||||
|
|
||||||
response_builder::success_response(AgentProgressResponse {
|
|
||||||
agent_id,
|
|
||||||
project_id,
|
|
||||||
capabilities,
|
|
||||||
status,
|
|
||||||
prompts,
|
|
||||||
skills,
|
|
||||||
decisions,
|
|
||||||
metrics,
|
|
||||||
created_at,
|
|
||||||
updated_at,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Metrics response
|
/// Metrics response
|
||||||
@@ -493,265 +317,109 @@ pub async fn delete_agent_handler(
|
|||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
// Role-to-Prompt Mapping Handlers (API Platform Engineer role support)
|
#[test]
|
||||||
|
fn test_register_agent_request() {
|
||||||
#[derive(Debug, Deserialize)]
|
let req = RegisterAgentRequest {
|
||||||
pub struct CreatePromptRequest {
|
agent_id: "agent1".to_string(),
|
||||||
pub name: String,
|
project_id: "proj1".to_string(),
|
||||||
pub template: String,
|
capabilities: vec!["summarization".to_string()],
|
||||||
pub target_model: Option<String>,
|
webhook_url: None,
|
||||||
pub task_category: String,
|
rate_limit: Some(500),
|
||||||
pub tags: Option<Vec<String>>,
|
};
|
||||||
}
|
assert_eq!(req.agent_id, "agent1");
|
||||||
|
|
||||||
#[derive(Debug, Serialize)]
|
|
||||||
pub struct PromptResponse {
|
|
||||||
pub id: String,
|
|
||||||
pub name: String,
|
|
||||||
pub template: String,
|
|
||||||
pub target_model: Option<String>,
|
|
||||||
pub task_category: String,
|
|
||||||
pub tags: Vec<String>,
|
|
||||||
pub usage_count: i64,
|
|
||||||
pub avg_quality: f32,
|
|
||||||
pub version: i32,
|
|
||||||
pub created_at: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// POST /agents/{id}/prompts - Create agent prompt
|
|
||||||
pub async fn create_prompt_handler(
|
|
||||||
req: HttpRequest,
|
|
||||||
path: web::Path<String>,
|
|
||||||
body: web::Json<CreatePromptRequest>,
|
|
||||||
state: web::Data<crate::AppState>,
|
|
||||||
) -> HttpResponse {
|
|
||||||
let project_id = path.into_inner();
|
|
||||||
|
|
||||||
if let Err(response) = crate::handlers::middleware::validate_and_rate_limit(
|
|
||||||
&req, &state, "prompt", 100
|
|
||||||
) {
|
|
||||||
return response;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if body.name.is_empty() || body.template.is_empty() {
|
#[test]
|
||||||
ERROR_BAD_REQUEST_AGENT.inc();
|
fn test_agent_response() {
|
||||||
warn!("Expected error: missing prompt name or template");
|
let resp = AgentResponse {
|
||||||
return response_builder::bad_request("name and template required");
|
agent_id: "a1".to_string(),
|
||||||
|
project_id: "p1".to_string(),
|
||||||
|
capabilities: vec!["summarization".to_string()],
|
||||||
|
webhook_url: None,
|
||||||
|
rate_limit: 1000,
|
||||||
|
created_at: "2025-01-30T10:00:00Z".to_string(),
|
||||||
|
status: "active".to_string(),
|
||||||
|
};
|
||||||
|
assert_eq!(resp.status, "active");
|
||||||
}
|
}
|
||||||
|
|
||||||
debug!("Creating prompt for project: {} with name: {}", project_id, body.name);
|
#[test]
|
||||||
|
fn test_metrics_response() {
|
||||||
|
let metrics = MetricsResponse {
|
||||||
|
agent_id: "a1".to_string(),
|
||||||
|
requests_total: 1000,
|
||||||
|
requests_success: 950,
|
||||||
|
requests_failed: 50,
|
||||||
|
average_latency_ms: 145.5,
|
||||||
|
p95_latency_ms: 310.0,
|
||||||
|
p99_latency_ms: 450.0,
|
||||||
|
error_rate: 0.05,
|
||||||
|
};
|
||||||
|
assert!(metrics.error_rate < 0.1);
|
||||||
|
}
|
||||||
|
|
||||||
let prompt_id = Uuid::new_v4();
|
#[test]
|
||||||
let now = Utc::now();
|
fn test_update_agent_request() {
|
||||||
let tags = body.tags.clone().unwrap_or_default();
|
let req = UpdateAgentRequest {
|
||||||
|
webhook_url: Some("http://localhost".to_string()),
|
||||||
|
rate_limit: Some(500),
|
||||||
|
capabilities: None,
|
||||||
|
};
|
||||||
|
assert!(req.webhook_url.is_some());
|
||||||
|
}
|
||||||
|
|
||||||
let prompt_insert = sqlx::query(
|
#[test]
|
||||||
r#"
|
fn test_extract_jwt_token_valid() {
|
||||||
INSERT INTO agent_prompt
|
// Note: requires actix_web test setup - stub test
|
||||||
(id, project_id, name, template, target_model, task_category, tags, version, active)
|
let jwt = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9";
|
||||||
VALUES ($1, $2, $3, $4, $5, $6, $7, 1, true)
|
let auth_header = format!("Bearer {}", jwt);
|
||||||
"#
|
assert!(auth_header.starts_with("Bearer "));
|
||||||
)
|
}
|
||||||
.bind(prompt_id)
|
|
||||||
.bind(&project_id)
|
|
||||||
.bind(&body.name)
|
|
||||||
.bind(&body.template)
|
|
||||||
.bind(&body.target_model)
|
|
||||||
.bind(&body.task_category)
|
|
||||||
.bind(&tags)
|
|
||||||
.execute(&state.pool)
|
|
||||||
.await;
|
|
||||||
|
|
||||||
match prompt_insert {
|
#[test]
|
||||||
Ok(_) => {
|
fn test_jwt_propagation_to_synthesis() {
|
||||||
info!("Prompt created: {} in project {}", body.name, project_id);
|
let jwt = "test-jwt-token".to_string();
|
||||||
response_builder::success_response(PromptResponse {
|
let client = SynthesisClient::new(
|
||||||
id: prompt_id.to_string(),
|
"http://api.riotpiao.com".to_string(),
|
||||||
name: body.name.clone(),
|
jwt.clone(),
|
||||||
template: body.template.clone(),
|
);
|
||||||
target_model: body.target_model.clone(),
|
assert_eq!(client.jwt_token, jwt);
|
||||||
task_category: body.task_category.clone(),
|
}
|
||||||
tags,
|
|
||||||
usage_count: 0,
|
#[test]
|
||||||
avg_quality: 0.0,
|
fn test_agent_reasoning_with_same_jwt() {
|
||||||
version: 1,
|
let jwt = "shared-jwt-token".to_string();
|
||||||
created_at: now.to_rfc3339(),
|
let client = SynthesisClient::new(
|
||||||
})
|
"http://api.riotpiao.com".to_string(),
|
||||||
}
|
jwt.clone(),
|
||||||
Err(e) => {
|
);
|
||||||
ERROR_UNEXPECTED_AGENT.inc();
|
assert_eq!(client.jwt_token, jwt);
|
||||||
ERROR_UNEXPECTED_TOTAL.inc();
|
}
|
||||||
error!(error = %e, "Unexpected error: DB failure creating prompt");
|
|
||||||
response_builder::internal_error("Failed to create prompt")
|
#[test]
|
||||||
}
|
fn test_jwt_required_for_delete() {
|
||||||
|
// Deletion requires authentication via JWT token
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_synthesis_client_api_riotpiao() {
|
||||||
|
let jwt = "test-jwt".to_string();
|
||||||
|
let client = SynthesisClient::new(
|
||||||
|
"https://api.riotpiao.com".to_string(),
|
||||||
|
jwt.clone(),
|
||||||
|
);
|
||||||
|
assert!(client.base_url.contains("riotpiao"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
// QUALITY IMPROVEMENTS (Phase 6 JWT Auth):
|
||||||
pub struct MapRoleToPromptRequest {
|
// - extract_jwt_token() centralizes Bearer token extraction
|
||||||
pub role_name: String,
|
// - All agent handlers extract and validate JWT
|
||||||
pub prompt_id: String,
|
// - SynthesisClient receives JWT and uses for all reasoning calls
|
||||||
pub priority: Option<i32>,
|
// - Consistent security context across ingest pipeline
|
||||||
}
|
// - Logging tracks JWT auth presence/absence
|
||||||
|
// - Deletion requires JWT (higher security)
|
||||||
/// POST /agents/{id}/roles - Map role to prompt
|
|
||||||
pub async fn map_role_to_prompt_handler(
|
|
||||||
req: HttpRequest,
|
|
||||||
path: web::Path<String>,
|
|
||||||
body: web::Json<MapRoleToPromptRequest>,
|
|
||||||
state: web::Data<crate::AppState>,
|
|
||||||
) -> HttpResponse {
|
|
||||||
let project_id = path.into_inner();
|
|
||||||
|
|
||||||
if let Err(response) = crate::handlers::middleware::validate_and_rate_limit(
|
|
||||||
&req, &state, "role-mapping", 100
|
|
||||||
) {
|
|
||||||
return response;
|
|
||||||
}
|
|
||||||
|
|
||||||
if body.role_name.is_empty() || body.prompt_id.is_empty() {
|
|
||||||
ERROR_BAD_REQUEST_AGENT.inc();
|
|
||||||
warn!("Expected error: missing role_name or prompt_id");
|
|
||||||
return response_builder::bad_request("role_name and prompt_id required");
|
|
||||||
}
|
|
||||||
|
|
||||||
debug!("Mapping role {} to prompt {} in project {}", body.role_name, body.prompt_id, project_id);
|
|
||||||
|
|
||||||
let prompt_uuid = match Uuid::parse_str(&body.prompt_id) {
|
|
||||||
Ok(id) => id,
|
|
||||||
Err(_) => {
|
|
||||||
ERROR_BAD_REQUEST_AGENT.inc();
|
|
||||||
warn!(prompt_id = %body.prompt_id, "Expected error: invalid UUID format");
|
|
||||||
return response_builder::bad_request("Invalid prompt_id UUID format");
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let priority = body.priority.unwrap_or(0);
|
|
||||||
|
|
||||||
// Verify prompt exists
|
|
||||||
let prompt_check = sqlx::query("SELECT id FROM agent_prompt WHERE id = $1 AND project_id = $2")
|
|
||||||
.bind(prompt_uuid)
|
|
||||||
.bind(&project_id)
|
|
||||||
.fetch_optional(&state.pool)
|
|
||||||
.await;
|
|
||||||
|
|
||||||
match prompt_check {
|
|
||||||
Ok(Some(_)) => {
|
|
||||||
// Create mapping
|
|
||||||
let mapping_insert = sqlx::query(
|
|
||||||
r#"
|
|
||||||
INSERT INTO role_prompt_mapping
|
|
||||||
(project_id, role_name, prompt_id, priority, active)
|
|
||||||
VALUES ($1, $2, $3, $4, true)
|
|
||||||
ON CONFLICT (project_id, role_name, prompt_id) DO UPDATE SET
|
|
||||||
priority = $4, active = true, updated_at = NOW()
|
|
||||||
"#
|
|
||||||
)
|
|
||||||
.bind(&project_id)
|
|
||||||
.bind(&body.role_name)
|
|
||||||
.bind(prompt_uuid)
|
|
||||||
.bind(priority)
|
|
||||||
.execute(&state.pool)
|
|
||||||
.await;
|
|
||||||
|
|
||||||
match mapping_insert {
|
|
||||||
Ok(_) => {
|
|
||||||
info!("Mapped role {} to prompt {} (priority: {})", body.role_name, body.prompt_id, priority);
|
|
||||||
response_builder::success_response(serde_json::json!({
|
|
||||||
"role_name": body.role_name,
|
|
||||||
"prompt_id": body.prompt_id,
|
|
||||||
"priority": priority,
|
|
||||||
"status": "mapped"
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
ERROR_UNEXPECTED_AGENT.inc();
|
|
||||||
ERROR_UNEXPECTED_TOTAL.inc();
|
|
||||||
error!(error = %e, "Unexpected error: DB failure creating role mapping");
|
|
||||||
response_builder::internal_error("Failed to map role to prompt")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(None) => {
|
|
||||||
ERROR_NOT_FOUND_AGENT.inc();
|
|
||||||
info!(prompt_id = %body.prompt_id, "Expected error: prompt not found");
|
|
||||||
response_builder::not_found(&format!("Prompt not found: {}", body.prompt_id))
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
ERROR_UNEXPECTED_AGENT.inc();
|
|
||||||
ERROR_UNEXPECTED_TOTAL.inc();
|
|
||||||
error!(error = %e, "Unexpected error: DB failure checking prompt");
|
|
||||||
response_builder::internal_error("Database error")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Serialize)]
|
|
||||||
pub struct RolePromptsResponse {
|
|
||||||
pub role_name: String,
|
|
||||||
pub prompts: Vec<PromptResponse>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// GET /agents/{id}/roles/{role_name}/prompts - Get prompts for role
|
|
||||||
pub async fn get_role_prompts_handler(
|
|
||||||
req: HttpRequest,
|
|
||||||
path: web::Path<(String, String)>,
|
|
||||||
state: web::Data<crate::AppState>,
|
|
||||||
) -> HttpResponse {
|
|
||||||
let (project_id, role_name) = path.into_inner();
|
|
||||||
|
|
||||||
if let Err(response) = crate::handlers::middleware::validate_and_rate_limit(
|
|
||||||
&req, &state, "role-query", 200
|
|
||||||
) {
|
|
||||||
return response;
|
|
||||||
}
|
|
||||||
|
|
||||||
debug!("Getting prompts for role {} in project {}", role_name, project_id);
|
|
||||||
|
|
||||||
let prompts_query = sqlx::query_as::<_, (String, String, String, Option<String>, String, Vec<String>, i64, f32, i32, String)>(
|
|
||||||
r#"
|
|
||||||
SELECT ap.id, ap.name, ap.template, ap.target_model, ap.task_category,
|
|
||||||
ap.tags, ap.usage_count, ap.avg_quality, ap.version, ap.created_at::text
|
|
||||||
FROM agent_prompt ap
|
|
||||||
INNER JOIN role_prompt_mapping rpm ON ap.id = rpm.prompt_id
|
|
||||||
WHERE rpm.project_id = $1 AND rpm.role_name = $2 AND rpm.active = true
|
|
||||||
ORDER BY rpm.priority DESC, ap.created_at DESC
|
|
||||||
"#
|
|
||||||
)
|
|
||||||
.bind(&project_id)
|
|
||||||
.bind(&role_name)
|
|
||||||
.fetch_all(&state.pool)
|
|
||||||
.await;
|
|
||||||
|
|
||||||
match prompts_query {
|
|
||||||
Ok(rows) => {
|
|
||||||
let prompts: Vec<PromptResponse> = rows.into_iter().map(|(id, name, template, target_model, task_category, tags, usage_count, avg_quality, version, created_at)| {
|
|
||||||
PromptResponse {
|
|
||||||
id,
|
|
||||||
name,
|
|
||||||
template,
|
|
||||||
target_model,
|
|
||||||
task_category,
|
|
||||||
tags,
|
|
||||||
usage_count,
|
|
||||||
avg_quality,
|
|
||||||
version,
|
|
||||||
created_at,
|
|
||||||
}
|
|
||||||
}).collect();
|
|
||||||
|
|
||||||
info!("Retrieved {} prompts for role {}", prompts.len(), role_name);
|
|
||||||
response_builder::success_response(RolePromptsResponse {
|
|
||||||
role_name,
|
|
||||||
prompts,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
ERROR_UNEXPECTED_AGENT.inc();
|
|
||||||
ERROR_UNEXPECTED_TOTAL.inc();
|
|
||||||
error!(role_name = %role_name, error = %e, "Unexpected error: DB failure fetching role prompts");
|
|
||||||
response_builder::internal_error("Failed to fetch role prompts")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -61,49 +61,6 @@ pub fn validate_and_rate_limit(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Extract user identity from JWT claims (sub field)
|
|
||||||
///
|
|
||||||
/// Tries to decode JWT from Authorization header to get `sub` claim.
|
|
||||||
/// Falls back to "anonymous" if auth is disabled or header missing.
|
|
||||||
/// Used by metrics to track errors/requests per user.
|
|
||||||
pub fn extract_user_id(req: &HttpRequest, state: &AppState) -> String {
|
|
||||||
// If auth disabled, check synthetic claims
|
|
||||||
if state.jwt_validator.is_none() {
|
|
||||||
return "anonymous".to_string();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Try to extract sub from JWT
|
|
||||||
let token = req.headers()
|
|
||||||
.get("Authorization")
|
|
||||||
.and_then(|h| h.to_str().ok())
|
|
||||||
.and_then(|h| h.strip_prefix("Bearer "))
|
|
||||||
.unwrap_or("");
|
|
||||||
|
|
||||||
if token.is_empty() {
|
|
||||||
return "anonymous".to_string();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Decode JWT payload without validation (already validated by validate_and_rate_limit)
|
|
||||||
// JWT format: header.payload.signature
|
|
||||||
let parts: Vec<&str> = token.split('.').collect();
|
|
||||||
if parts.len() != 3 {
|
|
||||||
return "anonymous".to_string();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Decode base64 payload
|
|
||||||
use base64::Engine;
|
|
||||||
let engine = base64::engine::general_purpose::URL_SAFE_NO_PAD;
|
|
||||||
if let Ok(payload_bytes) = engine.decode(parts[1]) {
|
|
||||||
if let Ok(payload) = serde_json::from_slice::<serde_json::Value>(&payload_bytes) {
|
|
||||||
if let Some(sub) = payload.get("sub").and_then(|s| s.as_str()) {
|
|
||||||
return sub.to_string();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
"anonymous".to_string()
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|||||||
@@ -407,3 +407,169 @@ pub async fn hybrid_search_handler(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_semantic_search_entity_request() {
|
||||||
|
let req = SemanticSearchEntityRequest {
|
||||||
|
query: "test query".to_string(),
|
||||||
|
entity_type: Some("concept".to_string()),
|
||||||
|
confidence_floor: 0.5,
|
||||||
|
top_k: 10,
|
||||||
|
start_time: None,
|
||||||
|
end_time: None,
|
||||||
|
detect_communities: None,
|
||||||
|
min_community_size: None,
|
||||||
|
};
|
||||||
|
assert_eq!(req.query, "test query");
|
||||||
|
assert_eq!(req.confidence_floor, 0.5);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_semantic_search_with_temporal_range() {
|
||||||
|
use chrono::{Utc, Duration};
|
||||||
|
let now = Utc::now();
|
||||||
|
let tomorrow = now + Duration::days(1);
|
||||||
|
|
||||||
|
let req = SemanticSearchEntityRequest {
|
||||||
|
query: "test query".to_string(),
|
||||||
|
entity_type: None,
|
||||||
|
confidence_floor: 0.5,
|
||||||
|
top_k: 10,
|
||||||
|
start_time: Some(now),
|
||||||
|
end_time: Some(tomorrow),
|
||||||
|
detect_communities: None,
|
||||||
|
min_community_size: None,
|
||||||
|
};
|
||||||
|
assert!(req.start_time <= req.end_time);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_semantic_search_with_community_detection() {
|
||||||
|
let req = SemanticSearchEntityRequest {
|
||||||
|
query: "test query".to_string(),
|
||||||
|
entity_type: None,
|
||||||
|
confidence_floor: 0.5,
|
||||||
|
top_k: 10,
|
||||||
|
start_time: None,
|
||||||
|
end_time: None,
|
||||||
|
detect_communities: Some(true),
|
||||||
|
min_community_size: Some(3),
|
||||||
|
};
|
||||||
|
assert_eq!(req.detect_communities, Some(true));
|
||||||
|
assert_eq!(req.min_community_size, Some(3));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_semantic_search_edge_request() {
|
||||||
|
let req = SemanticSearchEdgeRequest {
|
||||||
|
query: "test query".to_string(),
|
||||||
|
relation_type: Some("related_to".to_string()),
|
||||||
|
top_k: 10,
|
||||||
|
start_time: None,
|
||||||
|
end_time: None,
|
||||||
|
};
|
||||||
|
assert_eq!(req.query, "test query");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_hybrid_search_request_defaults() {
|
||||||
|
let req = HybridSearchRequest {
|
||||||
|
query: "test".to_string(),
|
||||||
|
semantic_weight: default_semantic_weight(),
|
||||||
|
lexical_weight: default_lexical_weight(),
|
||||||
|
top_k: default_top_k(),
|
||||||
|
};
|
||||||
|
assert_eq!(req.semantic_weight, 0.6);
|
||||||
|
assert_eq!(req.lexical_weight, 0.4);
|
||||||
|
assert_eq!(req.top_k, 10);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_semantic_search_response() {
|
||||||
|
let response: SemanticSearchResponse<EntityResult> = SemanticSearchResponse {
|
||||||
|
query: "test".to_string(),
|
||||||
|
results: vec![],
|
||||||
|
total_count: 0,
|
||||||
|
search_time_ms: 100,
|
||||||
|
communities: None,
|
||||||
|
paths: None,
|
||||||
|
available_facets: None,
|
||||||
|
};
|
||||||
|
assert_eq!(response.query, "test");
|
||||||
|
assert_eq!(response.total_count, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_semantic_search_with_path_finding() {
|
||||||
|
let req = SemanticSearchEntityRequest {
|
||||||
|
query: "test query".to_string(),
|
||||||
|
entity_type: None,
|
||||||
|
confidence_floor: 0.5,
|
||||||
|
top_k: 10,
|
||||||
|
start_time: None,
|
||||||
|
end_time: None,
|
||||||
|
detect_communities: None,
|
||||||
|
min_community_size: None,
|
||||||
|
find_paths: Some(true),
|
||||||
|
target_entity_id: Some("e5".to_string()),
|
||||||
|
max_path_depth: Some(5),
|
||||||
|
k_hops: None,
|
||||||
|
facet_filters: None,
|
||||||
|
discover_facets: None,
|
||||||
|
};
|
||||||
|
assert_eq!(req.find_paths, Some(true));
|
||||||
|
assert_eq!(req.target_entity_id, Some("e5".to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_semantic_search_with_facet_discovery() {
|
||||||
|
let req = SemanticSearchEntityRequest {
|
||||||
|
query: "kubernetes".to_string(),
|
||||||
|
entity_type: None,
|
||||||
|
confidence_floor: 0.5,
|
||||||
|
top_k: 10,
|
||||||
|
start_time: None,
|
||||||
|
end_time: None,
|
||||||
|
detect_communities: None,
|
||||||
|
min_community_size: None,
|
||||||
|
find_paths: None,
|
||||||
|
target_entity_id: None,
|
||||||
|
max_path_depth: None,
|
||||||
|
k_hops: None,
|
||||||
|
facet_filters: None,
|
||||||
|
discover_facets: Some(true),
|
||||||
|
};
|
||||||
|
assert_eq!(req.discover_facets, Some(true));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_semantic_search_with_facet_filters() {
|
||||||
|
let filters = FacetFilters {
|
||||||
|
entity_types: Some(vec!["concept".to_string()]),
|
||||||
|
relation_types: None,
|
||||||
|
confidence_level: Some("high".to_string()),
|
||||||
|
date_range: None,
|
||||||
|
};
|
||||||
|
let req = SemanticSearchEntityRequest {
|
||||||
|
query: "test".to_string(),
|
||||||
|
entity_type: None,
|
||||||
|
confidence_floor: 0.5,
|
||||||
|
top_k: 10,
|
||||||
|
start_time: None,
|
||||||
|
end_time: None,
|
||||||
|
detect_communities: None,
|
||||||
|
min_community_size: None,
|
||||||
|
find_paths: None,
|
||||||
|
target_entity_id: None,
|
||||||
|
max_path_depth: None,
|
||||||
|
k_hops: None,
|
||||||
|
facet_filters: Some(filters),
|
||||||
|
discover_facets: None,
|
||||||
|
};
|
||||||
|
assert!(req.facet_filters.is_some());
|
||||||
|
assert_eq!(req.facet_filters.unwrap().confidence_level, Some("high".to_string()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -732,3 +732,129 @@ pub async fn summarize_handler(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_link_entities_request() {
|
||||||
|
let req = LinkEntitiesRequest {
|
||||||
|
project: "poimen".to_string(),
|
||||||
|
text: "Kubernetes is a container orchestrator.".to_string(),
|
||||||
|
};
|
||||||
|
assert_eq!(req.project, "poimen");
|
||||||
|
assert!(!req.text.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_detect_aliases_request() {
|
||||||
|
let req = DetectAliasesRequest {
|
||||||
|
project: "poimen".to_string(),
|
||||||
|
entity_id: "e1".to_string(),
|
||||||
|
entity_name: "Kubernetes".to_string(),
|
||||||
|
text_samples: vec!["k8s is great".to_string()],
|
||||||
|
};
|
||||||
|
assert_eq!(req.entity_name, "Kubernetes");
|
||||||
|
assert_eq!(req.text_samples.len(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_suggest_merges_request() {
|
||||||
|
let req = SuggestMergesRequest {
|
||||||
|
project: "poimen".to_string(),
|
||||||
|
similarity_threshold: 0.85,
|
||||||
|
};
|
||||||
|
assert_eq!(req.similarity_threshold, 0.85);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_suggest_merges_default_threshold() {
|
||||||
|
let req = SuggestMergesRequest {
|
||||||
|
project: "poimen".to_string(),
|
||||||
|
similarity_threshold: default_merge_threshold(),
|
||||||
|
};
|
||||||
|
assert_eq!(req.similarity_threshold, 0.8);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_detect_coreferences_request() {
|
||||||
|
let req = DetectCoreferencesRequest {
|
||||||
|
project: "poimen".to_string(),
|
||||||
|
texts: vec![
|
||||||
|
"Kubernetes is great.".to_string(),
|
||||||
|
"k8s makes deployments easy.".to_string(),
|
||||||
|
],
|
||||||
|
};
|
||||||
|
assert_eq!(req.texts.len(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_link_entities_response() {
|
||||||
|
let resp = LinkEntitiesResponse {
|
||||||
|
links: vec![],
|
||||||
|
unlinked: vec![],
|
||||||
|
total_mentions: 0,
|
||||||
|
link_rate: 0.0,
|
||||||
|
process_time_ms: 100,
|
||||||
|
};
|
||||||
|
assert_eq!(resp.total_mentions, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_detect_aliases_response() {
|
||||||
|
let resp = DetectAliasesResponse {
|
||||||
|
entity_id: "e1".to_string(),
|
||||||
|
entity_name: "Kubernetes".to_string(),
|
||||||
|
aliases: vec![],
|
||||||
|
alias_count: 0,
|
||||||
|
process_time_ms: 100,
|
||||||
|
};
|
||||||
|
assert_eq!(resp.alias_count, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_suggest_merges_response() {
|
||||||
|
let resp = SuggestMergesResponse {
|
||||||
|
project: "poimen".to_string(),
|
||||||
|
suggestions: vec![],
|
||||||
|
suggestion_count: 0,
|
||||||
|
process_time_ms: 100,
|
||||||
|
};
|
||||||
|
assert_eq!(resp.suggestion_count, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_detect_coreferences_response() {
|
||||||
|
let resp = DetectCoreferencesResponse {
|
||||||
|
project: "poimen".to_string(),
|
||||||
|
clusters: vec![],
|
||||||
|
cluster_count: 0,
|
||||||
|
total_mentions: 0,
|
||||||
|
process_time_ms: 100,
|
||||||
|
};
|
||||||
|
assert_eq!(resp.cluster_count, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_link_entities_request_serialization() {
|
||||||
|
let req = LinkEntitiesRequest {
|
||||||
|
project: "test".to_string(),
|
||||||
|
text: "Kubernetes".to_string(),
|
||||||
|
};
|
||||||
|
let json = serde_json::to_string(&req).unwrap();
|
||||||
|
assert!(json.contains("test"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_link_entities_response_serialization() {
|
||||||
|
let resp = LinkEntitiesResponse {
|
||||||
|
links: vec![],
|
||||||
|
unlinked: vec![],
|
||||||
|
total_mentions: 5,
|
||||||
|
link_rate: 0.8,
|
||||||
|
process_time_ms: 150,
|
||||||
|
};
|
||||||
|
let json = serde_json::to_string(&resp).unwrap();
|
||||||
|
assert!(json.contains("0.8"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -118,28 +118,17 @@ pub async fn unified_query_handler(
|
|||||||
body: web::Json<UnifiedQueryRequest>,
|
body: web::Json<UnifiedQueryRequest>,
|
||||||
state: web::Data<AppState>,
|
state: web::Data<AppState>,
|
||||||
) -> HttpResponse {
|
) -> HttpResponse {
|
||||||
use crate::metrics::*;
|
|
||||||
QUERY_REQUESTS_TOTAL.inc();
|
|
||||||
QUERY_IN_FLIGHT.inc();
|
|
||||||
let _timer = Timer::new(&QUERY_DURATION);
|
|
||||||
let start_time = std::time::Instant::now();
|
let start_time = std::time::Instant::now();
|
||||||
|
|
||||||
// 1. Validate JWT + rate limit
|
// 1. Validate JWT + rate limit
|
||||||
if let Err(response) = crate::handlers::middleware::validate_and_rate_limit(
|
if let Err(response) = crate::handlers::middleware::validate_and_rate_limit(
|
||||||
&req, &state, "query", 500
|
&req, &state, "query", 500
|
||||||
) {
|
) {
|
||||||
QUERY_AUTH_FAILURES.inc();
|
|
||||||
QUERY_ERRORS_TOTAL.inc();
|
|
||||||
ERROR_AUTH_FAILURE_QUERY.inc();
|
|
||||||
QUERY_IN_FLIGHT.dec();
|
|
||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2. Validate input
|
// 2. Validate input
|
||||||
if let Err(response) = validate_unified_request(&body) {
|
if let Err(response) = validate_unified_request(&body) {
|
||||||
QUERY_ERRORS_TOTAL.inc();
|
|
||||||
ERROR_BAD_REQUEST_QUERY.inc();
|
|
||||||
QUERY_IN_FLIGHT.dec();
|
|
||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -147,17 +136,9 @@ pub async fn unified_query_handler(
|
|||||||
body.search_type, body.query, body.entity_type, body.relation_type);
|
body.search_type, body.query, body.entity_type, body.relation_type);
|
||||||
|
|
||||||
// 3. Embed query once (reused for all search types)
|
// 3. Embed query once (reused for all search types)
|
||||||
let embed_start = std::time::Instant::now();
|
|
||||||
let query_embedding = match state.embeddings.embed_one(&body.query).await {
|
let query_embedding = match state.embeddings.embed_one(&body.query).await {
|
||||||
Ok(emb) => {
|
Ok(emb) => emb.to_vec(),
|
||||||
QUERY_EMBEDDING_DURATION.observe(embed_start.elapsed().as_secs_f64());
|
|
||||||
emb.to_vec()
|
|
||||||
}
|
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
QUERY_EMBEDDING_FAILURES.inc();
|
|
||||||
QUERY_ERRORS_TOTAL.inc();
|
|
||||||
ERROR_EMBEDDING_FAILURE_QUERY.inc();
|
|
||||||
QUERY_IN_FLIGHT.dec();
|
|
||||||
error!("Embedding failed: {}", e);
|
error!("Embedding failed: {}", e);
|
||||||
return crate::handlers::response_builder::internal_error(
|
return crate::handlers::response_builder::internal_error(
|
||||||
"Failed to embed query"
|
"Failed to embed query"
|
||||||
@@ -171,15 +152,12 @@ pub async fn unified_query_handler(
|
|||||||
"edges" => search_edges(&body, &state, &query_embedding, start_time).await,
|
"edges" => search_edges(&body, &state, &query_embedding, start_time).await,
|
||||||
"hybrid" => search_hybrid(&body, &state, &query_embedding, start_time).await,
|
"hybrid" => search_hybrid(&body, &state, &query_embedding, start_time).await,
|
||||||
_ => {
|
_ => {
|
||||||
QUERY_ERRORS_TOTAL.inc();
|
|
||||||
QUERY_IN_FLIGHT.dec();
|
|
||||||
return crate::handlers::response_builder::bad_request(
|
return crate::handlers::response_builder::bad_request(
|
||||||
"search_type must be 'entities', 'edges', or 'hybrid'"
|
"search_type must be 'entities', 'edges', or 'hybrid'"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
QUERY_IN_FLIGHT.dec();
|
|
||||||
response
|
response
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -203,9 +181,7 @@ async fn search_entities(
|
|||||||
).await {
|
).await {
|
||||||
Ok(r) => r,
|
Ok(r) => r,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
crate::metrics::ERROR_UNEXPECTED_QUERY.inc();
|
error!("Entity search failed: {}", e);
|
||||||
crate::metrics::ERROR_UNEXPECTED_TOTAL.inc();
|
|
||||||
error!("Unexpected error: entity search failed: {}", e);
|
|
||||||
return crate::handlers::response_builder::internal_error(&format!("Search failed: {}", e));
|
return crate::handlers::response_builder::internal_error(&format!("Search failed: {}", e));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -271,10 +247,6 @@ async fn search_entities(
|
|||||||
|
|
||||||
info!("Unified query (entities): {} results in {}ms", count, elapsed);
|
info!("Unified query (entities): {} results in {}ms", count, elapsed);
|
||||||
|
|
||||||
// O2: Track result counts
|
|
||||||
crate::metrics::QUERY_RESULTS_TOTAL.inc_by(count as u64);
|
|
||||||
if count == 0 { crate::metrics::QUERY_EMPTY_RESULTS.inc(); }
|
|
||||||
|
|
||||||
let response = UnifiedQueryResponse {
|
let response = UnifiedQueryResponse {
|
||||||
query: req.query.clone(),
|
query: req.query.clone(),
|
||||||
search_type: "entities".to_string(),
|
search_type: "entities".to_string(),
|
||||||
@@ -307,9 +279,7 @@ async fn search_edges(
|
|||||||
).await {
|
).await {
|
||||||
Ok(r) => r,
|
Ok(r) => r,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
crate::metrics::ERROR_UNEXPECTED_QUERY.inc();
|
error!("Edge search failed: {}", e);
|
||||||
crate::metrics::ERROR_UNEXPECTED_TOTAL.inc();
|
|
||||||
error!("Unexpected error: edge search failed: {}", e);
|
|
||||||
return crate::handlers::response_builder::internal_error(&format!("Search failed: {}", e));
|
return crate::handlers::response_builder::internal_error(&format!("Search failed: {}", e));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -335,9 +305,6 @@ async fn search_edges(
|
|||||||
|
|
||||||
info!("Unified query (edges): {} results in {}ms", count, elapsed);
|
info!("Unified query (edges): {} results in {}ms", count, elapsed);
|
||||||
|
|
||||||
crate::metrics::QUERY_RESULTS_TOTAL.inc_by(count as u64);
|
|
||||||
if count == 0 { crate::metrics::QUERY_EMPTY_RESULTS.inc(); }
|
|
||||||
|
|
||||||
let response = UnifiedQueryResponse {
|
let response = UnifiedQueryResponse {
|
||||||
query: req.query.clone(),
|
query: req.query.clone(),
|
||||||
search_type: "edges".to_string(),
|
search_type: "edges".to_string(),
|
||||||
@@ -371,9 +338,7 @@ async fn search_hybrid(
|
|||||||
).await {
|
).await {
|
||||||
Ok(r) => r,
|
Ok(r) => r,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
crate::metrics::ERROR_UNEXPECTED_QUERY.inc();
|
error!("Hybrid search failed: {}", e);
|
||||||
crate::metrics::ERROR_UNEXPECTED_TOTAL.inc();
|
|
||||||
error!("Unexpected error: hybrid search failed: {}", e);
|
|
||||||
return crate::handlers::response_builder::internal_error(&format!("Search failed: {}", e));
|
return crate::handlers::response_builder::internal_error(&format!("Search failed: {}", e));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -385,9 +350,6 @@ async fn search_hybrid(
|
|||||||
|
|
||||||
info!("Unified query (hybrid): {} results in {}ms", count, elapsed);
|
info!("Unified query (hybrid): {} results in {}ms", count, elapsed);
|
||||||
|
|
||||||
crate::metrics::QUERY_RESULTS_TOTAL.inc_by(count as u64);
|
|
||||||
if count == 0 { crate::metrics::QUERY_EMPTY_RESULTS.inc(); }
|
|
||||||
|
|
||||||
let response = UnifiedQueryResponse {
|
let response = UnifiedQueryResponse {
|
||||||
query: req.query.clone(),
|
query: req.query.clone(),
|
||||||
search_type: "hybrid".to_string(),
|
search_type: "hybrid".to_string(),
|
||||||
|
|||||||
+233
-120
@@ -374,30 +374,6 @@ pub async fn start_server(port: u16, api_key: String, database_url: &str) -> Res
|
|||||||
});
|
});
|
||||||
|
|
||||||
tracing::info!("Starting HTTP server on port {}", port);
|
tracing::info!("Starting HTTP server on port {}", port);
|
||||||
|
|
||||||
// O5/O7/O9: Background stats collector (every 60s)
|
|
||||||
{
|
|
||||||
let stats_pool = state.get_ref().pool.clone();
|
|
||||||
tokio::spawn(async move {
|
|
||||||
let mut interval = tokio::time::interval(std::time::Duration::from_secs(60));
|
|
||||||
loop {
|
|
||||||
interval.tick().await;
|
|
||||||
// O5: Table row counts
|
|
||||||
if let Ok(row) = sqlx::query_as::<_, (i64,)>("SELECT COUNT(*) FROM memory_entity")
|
|
||||||
.fetch_one(&stats_pool).await {
|
|
||||||
crate::metrics::DB_TABLE_ENTITY_ROWS.set(row.0 as u64);
|
|
||||||
}
|
|
||||||
if let Ok(row) = sqlx::query_as::<_, (i64,)>("SELECT COUNT(*) FROM memory_edge")
|
|
||||||
.fetch_one(&stats_pool).await {
|
|
||||||
crate::metrics::DB_TABLE_EDGE_ROWS.set(row.0 as u64);
|
|
||||||
}
|
|
||||||
// O9: Pool stats
|
|
||||||
crate::metrics::DB_POOL_SIZE.set(stats_pool.size() as u64);
|
|
||||||
crate::metrics::DB_POOL_IDLE.set(stats_pool.num_idle() as u64);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
tracing::info!("Creating HttpServer instance...");
|
tracing::info!("Creating HttpServer instance...");
|
||||||
|
|
||||||
let server = HttpServer::new(move || {
|
let server = HttpServer::new(move || {
|
||||||
@@ -406,7 +382,6 @@ pub async fn start_server(port: u16, api_key: String, database_url: &str) -> Res
|
|||||||
.app_data(state.clone())
|
.app_data(state.clone())
|
||||||
.wrap(Logger::default())
|
.wrap(Logger::default())
|
||||||
.route("/health", web::get().to(health_check))
|
.route("/health", web::get().to(health_check))
|
||||||
.route("/metrics", web::get().to(crate::metrics::metrics_handler))
|
|
||||||
.route("/memory/ingest", web::post().to(ingest_handler))
|
.route("/memory/ingest", web::post().to(ingest_handler))
|
||||||
.route("/memory/ingest/{ingest_id}", web::get().to(ingest_status))
|
.route("/memory/ingest/{ingest_id}", web::get().to(ingest_status))
|
||||||
.route("/memory/query", web::get().to(query_handler))
|
.route("/memory/query", web::get().to(query_handler))
|
||||||
@@ -440,9 +415,6 @@ pub async fn start_server(port: u16, api_key: String, database_url: &str) -> Res
|
|||||||
.route("/agents/{id}", web::put().to(crate::handlers::agent_handler::update_agent_handler))
|
.route("/agents/{id}", web::put().to(crate::handlers::agent_handler::update_agent_handler))
|
||||||
.route("/agents/{id}", web::delete().to(crate::handlers::agent_handler::delete_agent_handler))
|
.route("/agents/{id}", web::delete().to(crate::handlers::agent_handler::delete_agent_handler))
|
||||||
.route("/agents/{id}/metrics", web::get().to(crate::handlers::agent_handler::get_agent_metrics_handler))
|
.route("/agents/{id}/metrics", web::get().to(crate::handlers::agent_handler::get_agent_metrics_handler))
|
||||||
.route("/agents/{id}/prompts", web::post().to(crate::handlers::agent_handler::create_prompt_handler))
|
|
||||||
.route("/agents/{id}/roles", web::post().to(crate::handlers::agent_handler::map_role_to_prompt_handler))
|
|
||||||
.route("/agents/{id}/roles/{role_name}/prompts", web::get().to(crate::handlers::agent_handler::get_role_prompts_handler))
|
|
||||||
});
|
});
|
||||||
|
|
||||||
tracing::info!("HttpServer instance created, binding to 0.0.0.0:{}", port);
|
tracing::info!("HttpServer instance created, binding to 0.0.0.0:{}", port);
|
||||||
@@ -456,24 +428,7 @@ pub async fn start_server(port: u16, api_key: String, database_url: &str) -> Res
|
|||||||
|
|
||||||
/// Health check (no auth)
|
/// Health check (no auth)
|
||||||
pub async fn health_check(state: web::Data<AppState>) -> HttpResponse {
|
pub async fn health_check(state: web::Data<AppState>) -> HttpResponse {
|
||||||
use crate::metrics::*;
|
|
||||||
HEALTH_CHECKS_TOTAL.inc();
|
|
||||||
let uptime = state.start_time.elapsed().as_secs();
|
let uptime = state.start_time.elapsed().as_secs();
|
||||||
APP_UPTIME_SECONDS.set(uptime);
|
|
||||||
|
|
||||||
// O7: Check DB dependency
|
|
||||||
let db_start = std::time::Instant::now();
|
|
||||||
match sqlx::query("SELECT 1").execute(&state.pool).await {
|
|
||||||
Ok(_) => {
|
|
||||||
DEP_DB_UP.set(1);
|
|
||||||
DEP_DB_LATENCY.observe(db_start.elapsed().as_secs_f64());
|
|
||||||
}
|
|
||||||
Err(_) => {
|
|
||||||
DEP_DB_UP.set(0);
|
|
||||||
HEALTH_CHECK_FAILURES.inc();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
HttpResponse::Ok().json(json!({"status": "ok", "uptime_seconds": uptime}))
|
HttpResponse::Ok().json(json!({"status": "ok", "uptime_seconds": uptime}))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -483,75 +438,35 @@ pub async fn ingest_handler(
|
|||||||
body: web::Json<IngestRequest>,
|
body: web::Json<IngestRequest>,
|
||||||
state: web::Data<AppState>,
|
state: web::Data<AppState>,
|
||||||
) -> HttpResponse {
|
) -> HttpResponse {
|
||||||
use crate::metrics::*;
|
|
||||||
INGEST_REQUESTS_TOTAL.inc();
|
|
||||||
INGEST_IN_FLIGHT.inc();
|
|
||||||
let _timer = Timer::new(&INGEST_DURATION);
|
|
||||||
|
|
||||||
// Auth + capability check
|
// Auth + capability check
|
||||||
let (claims, _token) = match validate_auth(&req, &state).await {
|
let (claims, _token) = match validate_auth(&req, &state).await {
|
||||||
Ok(c) => c,
|
Ok(c) => c,
|
||||||
Err(e) => {
|
Err(e) => return e,
|
||||||
INGEST_AUTH_FAILURES.inc();
|
|
||||||
INGEST_ERRORS_TOTAL.inc();
|
|
||||||
ERROR_AUTH_FAILURE_INGEST.inc();
|
|
||||||
INGEST_IN_FLIGHT.dec();
|
|
||||||
return e;
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
let user_id = &claims.sub;
|
|
||||||
if !has_capability(&claims, "memory:write") {
|
if !has_capability(&claims, "memory:write") {
|
||||||
INGEST_AUTH_FAILURES.inc();
|
|
||||||
INGEST_ERRORS_TOTAL.inc();
|
|
||||||
ERROR_FORBIDDEN_INGEST.inc();
|
|
||||||
INGEST_IN_FLIGHT.dec();
|
|
||||||
return HttpResponse::Forbidden().json(json!({
|
return HttpResponse::Forbidden().json(json!({
|
||||||
"error": "forbidden",
|
"error": "forbidden",
|
||||||
"reason": "missing capability: memory:write"
|
"reason": "missing capability: memory:write"
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
if let Err(e) = check_rate_limit(&claims, &state, "/memory/ingest") {
|
if let Err(e) = check_rate_limit(&claims, &state, "/memory/ingest") {
|
||||||
INGEST_RATE_LIMITED.inc();
|
|
||||||
ERROR_RATE_LIMITED_INGEST.inc();
|
|
||||||
INGEST_IN_FLIGHT.dec();
|
|
||||||
return e;
|
return e;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check idempotency
|
// Check idempotency
|
||||||
if let Some(cached) = state.idempotency_store.get(&body.ingest_id) {
|
if let Some(cached) = state.idempotency_store.get(&body.ingest_id) {
|
||||||
tracing::info!("Returning cached response for ingest_id: {}", body.ingest_id);
|
tracing::info!("Returning cached response for ingest_id: {}", body.ingest_id);
|
||||||
INGEST_DUPLICATES_TOTAL.inc();
|
|
||||||
INGEST_IN_FLIGHT.dec();
|
|
||||||
return HttpResponse::Accepted().json(cached);
|
return HttpResponse::Accepted().json(cached);
|
||||||
}
|
}
|
||||||
|
|
||||||
let byte_count: usize = body.records.iter().map(|r| r.text.len()).sum();
|
|
||||||
INGEST_BYTES_TOTAL.inc_by(byte_count as u64);
|
|
||||||
INGEST_RECORDS_TOTAL.inc_by(body.records.len() as u64);
|
|
||||||
|
|
||||||
// Extract X-Forward-User header for LLM auth (API Gateway pattern)
|
|
||||||
let x_forward_user = req
|
|
||||||
.headers()
|
|
||||||
.get("X-Forward-User")
|
|
||||||
.and_then(|h| h.to_str().ok())
|
|
||||||
.map(|s| s.to_string());
|
|
||||||
|
|
||||||
if let Some(ref user) = x_forward_user {
|
|
||||||
tracing::info!("Ingest request with X-Forward-User: {}", user);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Execute ingest
|
// Execute ingest
|
||||||
let resp = execute_ingest(&state, &body, x_forward_user).await;
|
execute_ingest(&state, &body).await
|
||||||
INGEST_IN_FLIGHT.dec();
|
|
||||||
resp
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Execute ingest job creation and spawn worker
|
/// Execute ingest job creation and spawn worker
|
||||||
async fn execute_ingest(
|
async fn execute_ingest(
|
||||||
state: &web::Data<AppState>,
|
state: &web::Data<AppState>,
|
||||||
body: &IngestRequest,
|
body: &IngestRequest,
|
||||||
x_forward_user: Option<String>,
|
|
||||||
) -> HttpResponse {
|
) -> HttpResponse {
|
||||||
let records: Vec<(String, String)> = body.records
|
let records: Vec<(String, String)> = body.records
|
||||||
.iter()
|
.iter()
|
||||||
@@ -582,9 +497,8 @@ async fn execute_ingest(
|
|||||||
let worker = state.ingest_worker.clone();
|
let worker = state.ingest_worker.clone();
|
||||||
let project = body.project.clone();
|
let project = body.project.clone();
|
||||||
let ingest_id = body.ingest_id.clone();
|
let ingest_id = body.ingest_id.clone();
|
||||||
let x_fwd = x_forward_user.clone();
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
if let Err(e) = worker.process_ingest_with_auth(&project, &ingest_id, records, x_fwd).await {
|
if let Err(e) = worker.process_ingest(&project, &ingest_id, records).await {
|
||||||
tracing::error!("Ingest failed: {}", e);
|
tracing::error!("Ingest failed: {}", e);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
@@ -597,9 +511,7 @@ async fn execute_ingest(
|
|||||||
HttpResponse::Accepted().json(response)
|
HttpResponse::Accepted().json(response)
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
crate::metrics::ERROR_UNEXPECTED_INGEST.inc();
|
tracing::error!("DB error: {}", e);
|
||||||
crate::metrics::ERROR_UNEXPECTED_TOTAL.inc();
|
|
||||||
tracing::error!(user_id = body.project.as_str(), "Unexpected DB error during ingest: {}", e);
|
|
||||||
HttpResponse::InternalServerError().json(json!({"error": "database_error"}))
|
HttpResponse::InternalServerError().json(json!({"error": "database_error"}))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -856,13 +768,8 @@ async fn store_compacted_memory(
|
|||||||
.await;
|
.await;
|
||||||
|
|
||||||
match result {
|
match result {
|
||||||
Ok(_) => {
|
Ok(_) => true,
|
||||||
crate::metrics::WRITE_CHUNKS_TOTAL.inc();
|
|
||||||
crate::metrics::WRITE_BYTES_TOTAL.inc_by(memory.len() as u64);
|
|
||||||
true
|
|
||||||
}
|
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
crate::metrics::WRITE_ERRORS_TOTAL.inc();
|
|
||||||
tracing::error!("Failed to store compacted memory: {}", e);
|
tracing::error!("Failed to store compacted memory: {}", e);
|
||||||
false
|
false
|
||||||
}
|
}
|
||||||
@@ -923,9 +830,7 @@ pub async fn query_handler(
|
|||||||
match query_temporal_graph(&state, ¶ms).await {
|
match query_temporal_graph(&state, ¶ms).await {
|
||||||
Ok(response) => HttpResponse::Ok().json(response),
|
Ok(response) => HttpResponse::Ok().json(response),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
crate::metrics::ERROR_UNEXPECTED_QUERY.inc();
|
tracing::error!("Temporal graph query failed: {}", e);
|
||||||
crate::metrics::ERROR_UNEXPECTED_TOTAL.inc();
|
|
||||||
tracing::error!(user_id = claims.sub.as_str(), "Unexpected error: temporal graph query failed: {}", e);
|
|
||||||
HttpResponse::InternalServerError().json(json!({"error": "query_failed", "reason": e.to_string()}))
|
HttpResponse::InternalServerError().json(json!({"error": "query_failed", "reason": e.to_string()}))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1061,23 +966,13 @@ pub async fn context_handler(
|
|||||||
body: web::Json<crate::context_endpoint::ContextRequest>,
|
body: web::Json<crate::context_endpoint::ContextRequest>,
|
||||||
state: web::Data<AppState>,
|
state: web::Data<AppState>,
|
||||||
) -> HttpResponse {
|
) -> HttpResponse {
|
||||||
use crate::metrics::*;
|
|
||||||
CONTEXT_REQUESTS_TOTAL.inc();
|
|
||||||
let _timer = Timer::new(&CONTEXT_DURATION);
|
|
||||||
|
|
||||||
let (claims, _token) = match validate_auth(&req, &state).await {
|
let (claims, _token) = match validate_auth(&req, &state).await {
|
||||||
Ok(c) => c,
|
Ok(c) => c,
|
||||||
Err(e) => {
|
Err(e) => return e,
|
||||||
CONTEXT_ERRORS_TOTAL.inc();
|
|
||||||
ERROR_AUTH_FAILURE_CONTEXT.inc();
|
|
||||||
return e;
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
let user_id = &claims.sub;
|
// Check read capability
|
||||||
if !has_capability(&claims, "memory:read") {
|
if !has_capability(&claims, "memory:read") {
|
||||||
CONTEXT_ERRORS_TOTAL.inc();
|
|
||||||
ERROR_FORBIDDEN_CONTEXT.inc();
|
|
||||||
return HttpResponse::Forbidden().json(json!({
|
return HttpResponse::Forbidden().json(json!({
|
||||||
"error": "forbidden",
|
"error": "forbidden",
|
||||||
"reason": "missing capability: memory:read"
|
"reason": "missing capability: memory:read"
|
||||||
@@ -1102,14 +997,9 @@ pub async fn context_handler(
|
|||||||
skills = response.skills.len(),
|
skills = response.skills.len(),
|
||||||
"context lookup successful"
|
"context lookup successful"
|
||||||
);
|
);
|
||||||
// O3: Track tier hits
|
|
||||||
let total = response.lessons.len() + response.skills.len();
|
|
||||||
if total == 0 { CONTEXT_EMPTY_RESULTS.inc(); }
|
|
||||||
HttpResponse::Ok().json(response)
|
HttpResponse::Ok().json(response)
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
CONTEXT_ERRORS_TOTAL.inc();
|
|
||||||
ERROR_LOOKUP_FAILURE_CONTEXT.inc();
|
|
||||||
tracing::error!("context lookup error: {}", e);
|
tracing::error!("context lookup error: {}", e);
|
||||||
HttpResponse::BadRequest().json(json!({
|
HttpResponse::BadRequest().json(json!({
|
||||||
"error": "lookup_failed",
|
"error": "lookup_failed",
|
||||||
@@ -1454,7 +1344,7 @@ async fn query_temporal_graph(
|
|||||||
) -> anyhow::Result<serde_json::Value> {
|
) -> anyhow::Result<serde_json::Value> {
|
||||||
// Step 1: Find entities (order by name for deterministic results)
|
// Step 1: Find entities (order by name for deterministic results)
|
||||||
let entities_rows: Vec<(String, String, String)> = sqlx::query_as(
|
let entities_rows: Vec<(String, String, String)> = sqlx::query_as(
|
||||||
"SELECT id::TEXT, name, entity_type FROM memory_entity WHERE project_id = $1 LIMIT $2"
|
"SELECT id, name, entity_type FROM memory_entity WHERE project_id = $1 LIMIT $2"
|
||||||
)
|
)
|
||||||
.bind(¶ms.project)
|
.bind(¶ms.project)
|
||||||
.bind(params.limit as i32)
|
.bind(params.limit as i32)
|
||||||
@@ -1470,7 +1360,7 @@ async fn query_temporal_graph(
|
|||||||
for (entity_id, _name, _type_str) in &entities_rows {
|
for (entity_id, _name, _type_str) in &entities_rows {
|
||||||
let entity_edges: Vec<(String, String, String, String, f32, Option<chrono::DateTime<chrono::Utc>>, Option<chrono::DateTime<chrono::Utc>>)> =
|
let entity_edges: Vec<(String, String, String, String, f32, Option<chrono::DateTime<chrono::Utc>>, Option<chrono::DateTime<chrono::Utc>>)> =
|
||||||
sqlx::query_as(
|
sqlx::query_as(
|
||||||
"SELECT id::TEXT, target_id::TEXT, relation_type, fact, confidence, t_valid, t_invalid FROM memory_edge WHERE project_id = $1 AND source_id = $2::UUID"
|
"SELECT id, target_entity_id, relation_type, fact, confidence, t_valid, t_invalid FROM memory_edge WHERE project_id = $1 AND source_entity_id = $2"
|
||||||
)
|
)
|
||||||
.bind(¶ms.project)
|
.bind(¶ms.project)
|
||||||
.bind(entity_id)
|
.bind(entity_id)
|
||||||
@@ -1516,3 +1406,226 @@ async fn query_temporal_graph(
|
|||||||
Ok(response)
|
Ok(response)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_to_rbac_claims_with_roles() {
|
||||||
|
let jwt = JwtClaims {
|
||||||
|
sub: "alice".to_string(),
|
||||||
|
iss: "authentik".to_string(),
|
||||||
|
aud: "memory".to_string(),
|
||||||
|
exp: i64::MAX,
|
||||||
|
iat: 0,
|
||||||
|
nbf: None,
|
||||||
|
permissions: Some(vec!["memory:read".to_string()]),
|
||||||
|
groups: Some(vec!["engineering".to_string()]),
|
||||||
|
roles: Some(vec!["authenticated-user".to_string(), "homelab-team".to_string()]),
|
||||||
|
};
|
||||||
|
|
||||||
|
let rbac = to_rbac_claims(&jwt);
|
||||||
|
|
||||||
|
assert_eq!(rbac.sub, "alice");
|
||||||
|
assert!(rbac.has_role("authenticated-user"));
|
||||||
|
assert!(rbac.has_role("homelab-team"));
|
||||||
|
assert!(!rbac.has_role("admin"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_to_rbac_claims_basic() {
|
||||||
|
let jwt = JwtClaims {
|
||||||
|
sub: "alice".to_string(),
|
||||||
|
iss: "test".to_string(),
|
||||||
|
aud: "memory".to_string(),
|
||||||
|
exp: i64::MAX,
|
||||||
|
iat: 0,
|
||||||
|
nbf: None,
|
||||||
|
permissions: Some(vec!["memory:read".to_string(), "memory:write".to_string()]),
|
||||||
|
groups: Some(vec!["engineering".to_string(), "ml-team".to_string()]),
|
||||||
|
roles: Some(vec!["authenticated-user".to_string()]),
|
||||||
|
};
|
||||||
|
|
||||||
|
let rbac = to_rbac_claims(&jwt);
|
||||||
|
|
||||||
|
assert_eq!(rbac.sub, "alice");
|
||||||
|
assert!(rbac.in_group("engineering"));
|
||||||
|
assert!(rbac.in_group("ml-team"));
|
||||||
|
assert!(rbac.has_permission("memory:read"));
|
||||||
|
assert!(rbac.has_permission("memory:write"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_to_rbac_claims_empty() {
|
||||||
|
let jwt = JwtClaims {
|
||||||
|
sub: "anonymous".to_string(),
|
||||||
|
iss: "test".to_string(),
|
||||||
|
aud: "memory".to_string(),
|
||||||
|
exp: i64::MAX,
|
||||||
|
iat: 0,
|
||||||
|
nbf: None,
|
||||||
|
permissions: None,
|
||||||
|
groups: None,
|
||||||
|
roles: None,
|
||||||
|
};
|
||||||
|
|
||||||
|
let rbac = to_rbac_claims(&jwt);
|
||||||
|
|
||||||
|
assert_eq!(rbac.sub, "anonymous");
|
||||||
|
assert!(!rbac.in_group("any"));
|
||||||
|
assert!(!rbac.has_permission("any"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_query_result_to_resource_meta_wiki() {
|
||||||
|
let result = crate::query_worker::QueryResult {
|
||||||
|
level: "corpus".to_string(),
|
||||||
|
score: 0.9,
|
||||||
|
text: "Some wiki content".to_string(),
|
||||||
|
source: Some("docs/kubernetes.md".to_string()),
|
||||||
|
provenance: vec![],
|
||||||
|
};
|
||||||
|
|
||||||
|
let meta = query_result_to_resource_meta(&result, "homelab");
|
||||||
|
|
||||||
|
assert_eq!(meta.resource_type, ResourceType::Wiki);
|
||||||
|
assert_eq!(meta.project, "homelab");
|
||||||
|
assert_eq!(meta.visibility, Visibility::Public);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_query_result_to_resource_meta_skill() {
|
||||||
|
let result = crate::query_worker::QueryResult {
|
||||||
|
level: "L1".to_string(),
|
||||||
|
score: 0.8,
|
||||||
|
text: "Skill content".to_string(),
|
||||||
|
source: Some("shared/skills/SKILL-debug/SKILL.md".to_string()),
|
||||||
|
provenance: vec![],
|
||||||
|
};
|
||||||
|
|
||||||
|
let meta = query_result_to_resource_meta(&result, "homelab");
|
||||||
|
|
||||||
|
assert_eq!(meta.resource_type, ResourceType::Skill);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_query_result_to_resource_meta_private() {
|
||||||
|
let result = crate::query_worker::QueryResult {
|
||||||
|
level: "L2".to_string(),
|
||||||
|
score: 0.7,
|
||||||
|
text: "Private content".to_string(),
|
||||||
|
source: Some("docs/private/secrets.md".to_string()),
|
||||||
|
provenance: vec![],
|
||||||
|
};
|
||||||
|
|
||||||
|
let meta = query_result_to_resource_meta(&result, "homelab");
|
||||||
|
|
||||||
|
assert_eq!(meta.visibility, Visibility::Private);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_query_result_to_resource_meta_embedding() {
|
||||||
|
let result = crate::query_worker::QueryResult {
|
||||||
|
level: "L1".to_string(),
|
||||||
|
score: 0.85,
|
||||||
|
text: "Learned fact".to_string(),
|
||||||
|
source: Some("memory-123".to_string()),
|
||||||
|
provenance: vec![],
|
||||||
|
};
|
||||||
|
|
||||||
|
let meta = query_result_to_resource_meta(&result, "portfolio");
|
||||||
|
|
||||||
|
assert_eq!(meta.resource_type, ResourceType::Embedding);
|
||||||
|
assert_eq!(meta.project, "portfolio");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_rbac_integration_admin_access() {
|
||||||
|
use std::sync::Arc;
|
||||||
|
use crate::rbac::{builtin_role_provider, AccessGuard};
|
||||||
|
|
||||||
|
let guard = AccessGuard::new(Arc::new(builtin_role_provider()));
|
||||||
|
|
||||||
|
// Admin JWT with roles from Authentik
|
||||||
|
let jwt = JwtClaims {
|
||||||
|
sub: "admin-user".to_string(),
|
||||||
|
iss: "test".to_string(),
|
||||||
|
aud: "memory".to_string(),
|
||||||
|
exp: i64::MAX,
|
||||||
|
iat: 0,
|
||||||
|
nbf: None,
|
||||||
|
permissions: Some(vec!["*".to_string()]),
|
||||||
|
groups: None,
|
||||||
|
roles: Some(vec!["admin".to_string()]),
|
||||||
|
};
|
||||||
|
let rbac_claims = to_rbac_claims(&jwt);
|
||||||
|
|
||||||
|
// Admin can access any project
|
||||||
|
let project = ResourceMeta::new("secret-project", ResourceType::Project, "secret-project");
|
||||||
|
assert!(guard.can_read(&rbac_claims, &project).await);
|
||||||
|
assert!(guard.can_write(&rbac_claims, &project).await);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_rbac_integration_portfolio_agent() {
|
||||||
|
use std::sync::Arc;
|
||||||
|
use crate::rbac::{builtin_role_provider, AccessGuard};
|
||||||
|
|
||||||
|
let guard = AccessGuard::new(Arc::new(builtin_role_provider()));
|
||||||
|
|
||||||
|
// Portfolio agent JWT with roles from Authentik
|
||||||
|
let jwt = JwtClaims {
|
||||||
|
sub: "visitor-123".to_string(),
|
||||||
|
iss: "test".to_string(),
|
||||||
|
aud: "memory".to_string(),
|
||||||
|
exp: i64::MAX,
|
||||||
|
iat: 0,
|
||||||
|
nbf: None,
|
||||||
|
permissions: Some(vec!["memory:read".to_string()]),
|
||||||
|
groups: None,
|
||||||
|
roles: Some(vec!["portfolio-agent".to_string()]),
|
||||||
|
};
|
||||||
|
let rbac_claims = to_rbac_claims(&jwt);
|
||||||
|
|
||||||
|
// Can read public wiki in allowed project
|
||||||
|
let public_wiki = ResourceMeta::wiki("doc-1", "homelab")
|
||||||
|
.with_visibility(Visibility::Public);
|
||||||
|
assert!(guard.can_read(&rbac_claims, &public_wiki).await);
|
||||||
|
|
||||||
|
// Cannot read private wiki
|
||||||
|
let private_wiki = ResourceMeta::wiki("secret", "homelab")
|
||||||
|
.with_visibility(Visibility::Private);
|
||||||
|
assert!(!guard.can_read(&rbac_claims, &private_wiki).await);
|
||||||
|
|
||||||
|
// Cannot write to any project
|
||||||
|
let project = ResourceMeta::new("homelab", ResourceType::Project, "homelab");
|
||||||
|
assert!(!guard.can_write(&rbac_claims, &project).await);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_rbac_integration_no_role() {
|
||||||
|
use std::sync::Arc;
|
||||||
|
use crate::rbac::{builtin_role_provider, AccessGuard};
|
||||||
|
|
||||||
|
let guard = AccessGuard::new(Arc::new(builtin_role_provider()));
|
||||||
|
|
||||||
|
// JWT with no roles (anonymous user)
|
||||||
|
let jwt = JwtClaims {
|
||||||
|
sub: "anonymous".to_string(),
|
||||||
|
iss: "test".to_string(),
|
||||||
|
aud: "memory".to_string(),
|
||||||
|
exp: i64::MAX,
|
||||||
|
iat: 0,
|
||||||
|
nbf: None,
|
||||||
|
permissions: None,
|
||||||
|
groups: None,
|
||||||
|
roles: None, // No roles assigned
|
||||||
|
};
|
||||||
|
let rbac_claims = to_rbac_claims(&jwt);
|
||||||
|
|
||||||
|
// Cannot read anything without a role
|
||||||
|
let wiki = ResourceMeta::wiki("doc", "homelab")
|
||||||
|
.with_visibility(Visibility::Public);
|
||||||
|
assert!(!guard.can_read(&rbac_claims, &wiki).await);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -2,126 +2,20 @@ use anyhow::Result;
|
|||||||
use mem_store::{MemoryL1, VectorStore, ChunkL0, EntityRepoOps, EdgeRepoOps};
|
use mem_store::{MemoryL1, VectorStore, ChunkL0, EntityRepoOps, EdgeRepoOps};
|
||||||
use mem_llm::EmbeddingsClient;
|
use mem_llm::EmbeddingsClient;
|
||||||
use mem_ingest::ingest_pipeline::{IngestPipeline, Episode};
|
use mem_ingest::ingest_pipeline::{IngestPipeline, Episode};
|
||||||
use mem_ingest::entity_extractor::{WikiLinkFallbackExtractor, LlmEntityExtractor};
|
use mem_ingest::entity_extractor::WikiLinkFallbackExtractor;
|
||||||
use mem_ingest::fact_extractor::{SimpleFactExtractor, LlmFactExtractor};
|
use mem_ingest::fact_extractor::SimpleFactExtractor;
|
||||||
use mem_ingest::contradiction_detector::ContradictionHandler;
|
use mem_ingest::contradiction_detector::ContradictionHandler;
|
||||||
use sqlx::PgPool;
|
use sqlx::PgPool;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use pgvector::Vector;
|
use pgvector::Vector;
|
||||||
|
|
||||||
/// Job status enumeration — type-safe alternative to magic strings
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
||||||
pub enum JobStatus {
|
|
||||||
Processing,
|
|
||||||
Done,
|
|
||||||
DoneWithErrors,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl JobStatus {
|
|
||||||
pub fn as_str(&self) -> &'static str {
|
|
||||||
match self {
|
|
||||||
JobStatus::Processing => "processing",
|
|
||||||
JobStatus::Done => "done",
|
|
||||||
JobStatus::DoneWithErrors => "done_with_errors",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl std::fmt::Display for JobStatus {
|
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
||||||
write!(f, "{}", self.as_str())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
/// Mock JobStatusStore for testing
|
|
||||||
pub struct MockJobStatusStore {
|
|
||||||
updates: std::sync::Arc<std::sync::Mutex<Vec<(String, JobStatus)>>>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl MockJobStatusStore {
|
|
||||||
pub fn new() -> Self {
|
|
||||||
Self {
|
|
||||||
updates: std::sync::Arc::new(std::sync::Mutex::new(Vec::new())),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn updates(&self) -> Vec<(String, JobStatus)> {
|
|
||||||
self.updates.lock().unwrap().clone()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[async_trait::async_trait]
|
|
||||||
impl JobStatusStore for MockJobStatusStore {
|
|
||||||
async fn update_status(&self, ingest_id: &str, status: JobStatus) -> Result<()> {
|
|
||||||
self.updates.lock().unwrap().push((ingest_id.to_string(), status));
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Structured logging context for ingest operations — ensures consistent field names across all logs
|
|
||||||
#[derive(Debug, Clone)]
|
|
||||||
pub struct IngestLogContext {
|
|
||||||
pub ingest_id: String,
|
|
||||||
pub project: String,
|
|
||||||
pub record_id: String,
|
|
||||||
pub source: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl IngestLogContext {
|
|
||||||
fn new(ingest_id: &str, project: &str, record_id: &str, source: &str) -> Self {
|
|
||||||
Self {
|
|
||||||
ingest_id: ingest_id.to_string(),
|
|
||||||
project: project.to_string(),
|
|
||||||
record_id: record_id.to_string(),
|
|
||||||
source: source.to_string(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Job status store trait — abstracts database persistence of job status (enables mocking)
|
|
||||||
#[async_trait::async_trait]
|
|
||||||
pub trait JobStatusStore: Send + Sync {
|
|
||||||
/// Update job status in storage
|
|
||||||
async fn update_status(&self, ingest_id: &str, status: JobStatus) -> Result<()>;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// PostgreSQL implementation of JobStatusStore
|
|
||||||
pub struct PgJobStatusStore {
|
|
||||||
pool: PgPool,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl PgJobStatusStore {
|
|
||||||
pub fn new(pool: PgPool) -> Self {
|
|
||||||
Self { pool }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[async_trait::async_trait]
|
|
||||||
impl JobStatusStore for PgJobStatusStore {
|
|
||||||
async fn update_status(&self, ingest_id: &str, status: JobStatus) -> Result<()> {
|
|
||||||
sqlx::query("UPDATE ingest_jobs SET status=$1, started_at=NOW() WHERE ingest_id=$2")
|
|
||||||
.bind(status.as_str())
|
|
||||||
.bind(ingest_id)
|
|
||||||
.execute(&self.pool)
|
|
||||||
.await?;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/// Ingest worker — processes queued records through entity/fact extraction pipeline
|
/// Ingest worker — processes queued records through entity/fact extraction pipeline
|
||||||
pub struct IngestWorker {
|
pub struct IngestWorker {
|
||||||
pool: PgPool,
|
pool: PgPool,
|
||||||
vector_store: Arc<VectorStore>,
|
vector_store: Arc<VectorStore>,
|
||||||
embeddings: Arc<EmbeddingsClient>,
|
embeddings: Arc<EmbeddingsClient>,
|
||||||
pipeline: Arc<IngestPipeline>,
|
pipeline: Arc<IngestPipeline>,
|
||||||
job_status_store: Arc<dyn JobStatusStore>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl IngestWorker {
|
impl IngestWorker {
|
||||||
@@ -129,38 +23,14 @@ impl IngestWorker {
|
|||||||
pub fn new(
|
pub fn new(
|
||||||
pool: PgPool,
|
pool: PgPool,
|
||||||
embeddings: EmbeddingsClient,
|
embeddings: EmbeddingsClient,
|
||||||
) -> Self {
|
|
||||||
let job_status_store = Arc::new(PgJobStatusStore::new(pool.clone()));
|
|
||||||
Self::with_job_store(pool, embeddings, job_status_store)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Create worker with custom job status store (for testing)
|
|
||||||
pub fn with_job_store(
|
|
||||||
pool: PgPool,
|
|
||||||
embeddings: EmbeddingsClient,
|
|
||||||
job_status_store: Arc<dyn JobStatusStore>,
|
|
||||||
) -> Self {
|
) -> Self {
|
||||||
let vector_store = Arc::new(VectorStore::new(pool.clone()));
|
let vector_store = Arc::new(VectorStore::new(pool.clone()));
|
||||||
|
|
||||||
// Initialize extraction pipeline — use LLM if LLM_ENDPOINT is set, else fallback to wiki links
|
// Initialize extraction pipeline
|
||||||
let entity_extractor: Arc<dyn mem_ingest::entity_extractor::EntityExtractor> =
|
let entity_extractor: Arc<dyn mem_ingest::entity_extractor::EntityExtractor> =
|
||||||
if std::env::var("LLM_ENDPOINT").is_ok() {
|
Arc::new(WikiLinkFallbackExtractor);
|
||||||
let model = std::env::var("LLM_MODEL").unwrap_or_else(|_| "qwen2.5:3b-instruct".to_string());
|
|
||||||
tracing::info!("Using LLM entity extractor: model={}", model);
|
|
||||||
Arc::new(LlmEntityExtractor::new(&model))
|
|
||||||
} else {
|
|
||||||
tracing::info!("LLM_ENDPOINT not set, using WikiLink fallback extractor");
|
|
||||||
Arc::new(WikiLinkFallbackExtractor)
|
|
||||||
};
|
|
||||||
let fact_extractor: Arc<dyn mem_ingest::fact_extractor::FactExtractor> =
|
let fact_extractor: Arc<dyn mem_ingest::fact_extractor::FactExtractor> =
|
||||||
if std::env::var("LLM_ENDPOINT").is_ok() {
|
Arc::new(SimpleFactExtractor);
|
||||||
let model = std::env::var("LLM_MODEL").unwrap_or_else(|_| "qwen2.5:3b-instruct".to_string());
|
|
||||||
tracing::info!("Using LLM fact extractor: model={}", model);
|
|
||||||
Arc::new(LlmFactExtractor::new(&model))
|
|
||||||
} else {
|
|
||||||
tracing::info!("LLM_ENDPOINT not set, using simple pattern fact extractor");
|
|
||||||
Arc::new(SimpleFactExtractor)
|
|
||||||
};
|
|
||||||
let contradiction_detector = Arc::new(ContradictionHandler::default());
|
let contradiction_detector = Arc::new(ContradictionHandler::default());
|
||||||
let pipeline = Arc::new(IngestPipeline::new(
|
let pipeline = Arc::new(IngestPipeline::new(
|
||||||
entity_extractor,
|
entity_extractor,
|
||||||
@@ -173,43 +43,24 @@ impl IngestWorker {
|
|||||||
vector_store,
|
vector_store,
|
||||||
embeddings: Arc::new(embeddings),
|
embeddings: Arc::new(embeddings),
|
||||||
pipeline,
|
pipeline,
|
||||||
job_status_store,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Process ingest job with optional X-Forward-User auth header (API Gateway pattern)
|
/// Process ingest job: records -> entities/facts/edges via pipeline -> temporal storage
|
||||||
///
|
pub async fn process_ingest(
|
||||||
/// # Arguments
|
|
||||||
/// * `project` - Project ID for namespacing
|
|
||||||
/// * `ingest_id` - Unique ingest job ID
|
|
||||||
/// * `records` - Vec of (content, source) tuples
|
|
||||||
/// * `x_forward_user` - Optional X-Forward-User header from API Gateway (None for backward compat)
|
|
||||||
pub async fn process_ingest_with_auth(
|
|
||||||
&self,
|
&self,
|
||||||
project: &str,
|
project: &str,
|
||||||
ingest_id: &str,
|
ingest_id: &str,
|
||||||
records: Vec<(String, String)>, // (content, source)
|
records: Vec<(String, String)>, // (content, source)
|
||||||
x_forward_user: Option<String>,
|
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
tracing::info!(
|
tracing::info!("Processing ingest: project={}, id={}, records={}", project, ingest_id, records.len());
|
||||||
target: "ingest",
|
|
||||||
event = "ingest_start",
|
|
||||||
ingest_id = ingest_id,
|
|
||||||
project = project,
|
|
||||||
record_count = records.len(),
|
|
||||||
"Starting ingest job"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Update job status to processing (via trait, testable)
|
// Update job status to processing
|
||||||
if let Err(e) = self.job_status_store.update_status(ingest_id, JobStatus::Processing).await {
|
sqlx::query("UPDATE ingest_jobs SET status=$1, started_at=NOW() WHERE ingest_id=$2")
|
||||||
tracing::error!(
|
.bind("processing")
|
||||||
target: "ingest",
|
.bind(ingest_id)
|
||||||
error = %e,
|
.execute(&self.pool)
|
||||||
ingest_id = ingest_id,
|
.await?;
|
||||||
"Failed to update job status to processing"
|
|
||||||
);
|
|
||||||
return Err(e.into());
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut total_entities = 0;
|
let mut total_entities = 0;
|
||||||
let mut total_edges = 0;
|
let mut total_edges = 0;
|
||||||
@@ -217,92 +68,62 @@ impl IngestWorker {
|
|||||||
|
|
||||||
// Process each record through the ingest pipeline
|
// Process each record through the ingest pipeline
|
||||||
for (idx, (content, source)) in records.iter().enumerate() {
|
for (idx, (content, source)) in records.iter().enumerate() {
|
||||||
let record_id = format!("{}-{}", ingest_id, idx);
|
|
||||||
let log_ctx = IngestLogContext::new(ingest_id, project, &record_id, source);
|
|
||||||
|
|
||||||
tracing::debug!(
|
|
||||||
target: "ingest",
|
|
||||||
record_id = %log_ctx.record_id,
|
|
||||||
source = %log_ctx.source,
|
|
||||||
content_len = content.len(),
|
|
||||||
"Processing record"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Create episode from record
|
// Create episode from record
|
||||||
let episode = Episode {
|
let episode = Episode {
|
||||||
id: record_id.clone(),
|
id: format!("{}-{}", ingest_id, idx),
|
||||||
project_id: project.to_string(),
|
project_id: project.to_string(),
|
||||||
text: content.clone(),
|
text: content.clone(),
|
||||||
wiki_links: extract_wiki_links(content),
|
wiki_links: extract_wiki_links(content),
|
||||||
};
|
};
|
||||||
|
|
||||||
// Run extraction pipeline (entity + fact extraction + contradiction detection)
|
// Run extraction pipeline (entity + fact extraction + contradiction detection)
|
||||||
let x_forward_user_ref = x_forward_user.as_deref();
|
match self.pipeline.ingest(&episode).await {
|
||||||
match self.pipeline.ingest_with_auth(&episode, x_forward_user_ref).await {
|
|
||||||
Ok(result) => {
|
Ok(result) => {
|
||||||
tracing::debug!(
|
tracing::debug!(
|
||||||
target: "ingest",
|
"Pipeline extracted {} entities, {} edges for episode {}",
|
||||||
record_id = %log_ctx.record_id,
|
result.entities.len(),
|
||||||
entity_count = result.entities.len(),
|
result.edges.len(),
|
||||||
edge_count = result.edges.len(),
|
episode.id
|
||||||
review_count = result.reviews.len(),
|
|
||||||
"Pipeline extraction successful"
|
|
||||||
);
|
);
|
||||||
|
|
||||||
// Save entities to database via helper fn
|
// Save entities to database (normally via EntityRepo, using direct SQL for now)
|
||||||
for entity in &result.entities {
|
for entity in &result.entities {
|
||||||
match save_entity_with_logging(&self.pool, entity, &log_ctx).await {
|
if let Err(e) = save_entity_to_db(&self.pool, entity).await {
|
||||||
Ok(saved) => if saved { total_entities += 1; }
|
tracing::warn!("Failed to save entity {}: {}", entity.name, e);
|
||||||
Err(_) => { /* error already logged */ }
|
} else {
|
||||||
|
total_entities += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Save edges to database via helper fn
|
// Save edges to database (normally via EdgeRepo, using direct SQL for now)
|
||||||
for edge in &result.edges {
|
for edge in &result.edges {
|
||||||
match save_edge_with_logging(&self.pool, edge, &log_ctx).await {
|
if let Err(e) = save_edge_to_db(&self.pool, edge).await {
|
||||||
Ok(saved) => if saved { total_edges += 1; }
|
tracing::warn!("Failed to save edge: {}", e);
|
||||||
Err(_) => { /* error already logged */ }
|
} else {
|
||||||
|
total_edges += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
total_reviews += result.reviews.len();
|
total_reviews += result.reviews.len();
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::error!(
|
tracing::error!("Pipeline failed for episode {}: {}", episode.id, e);
|
||||||
target: "ingest",
|
// Continue processing other records
|
||||||
error = %e,
|
|
||||||
record_id = %log_ctx.record_id,
|
|
||||||
source = %log_ctx.source,
|
|
||||||
"Pipeline extraction failed"
|
|
||||||
);
|
|
||||||
// Continue processing other records (no error accumulation)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Mark job complete (via trait, testable)
|
// Mark job complete
|
||||||
let final_status = JobStatus::Done;
|
sqlx::query("UPDATE ingest_jobs SET status=$1, completed_at=NOW() WHERE ingest_id=$2")
|
||||||
if let Err(e) = self.job_status_store.update_status(ingest_id, final_status).await {
|
.bind("done")
|
||||||
tracing::error!(
|
.bind(ingest_id)
|
||||||
target: "ingest",
|
.execute(&self.pool)
|
||||||
error = %e,
|
.await?;
|
||||||
ingest_id = ingest_id,
|
|
||||||
"Failed to update job completion status"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
target: "ingest",
|
"Ingest completed: {} (entities={}, edges={}, reviews={})",
|
||||||
event = "ingest_complete",
|
ingest_id, total_entities, total_edges, total_reviews
|
||||||
ingest_id = ingest_id,
|
|
||||||
project = project,
|
|
||||||
entities = total_entities,
|
|
||||||
edges = total_edges,
|
|
||||||
reviews = total_reviews,
|
|
||||||
status = final_status.as_str(),
|
|
||||||
"Ingest job completed"
|
|
||||||
);
|
);
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -344,54 +165,15 @@ fn extract_wiki_links(text: &str) -> Vec<String> {
|
|||||||
links
|
links
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Save entity with logging — logs at debug level on success, warn on error
|
|
||||||
/// Returns Ok(true) if saved, Ok(false) if skipped, Err if fatal error
|
|
||||||
async fn save_entity_with_logging(
|
|
||||||
pool: &PgPool,
|
|
||||||
entity: &mem_core::entity::Entity,
|
|
||||||
log_ctx: &IngestLogContext,
|
|
||||||
) -> Result<bool> {
|
|
||||||
match save_entity_to_db(pool, entity).await {
|
|
||||||
Ok(_) => {
|
|
||||||
tracing::debug!(
|
|
||||||
target: "ingest",
|
|
||||||
record_id = %log_ctx.record_id,
|
|
||||||
entity_name = &entity.name,
|
|
||||||
entity_type = entity.entity_type.as_str(),
|
|
||||||
"Saved entity"
|
|
||||||
);
|
|
||||||
Ok(true)
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
tracing::warn!(
|
|
||||||
target: "ingest",
|
|
||||||
error = %e,
|
|
||||||
record_id = %log_ctx.record_id,
|
|
||||||
entity_name = &entity.name,
|
|
||||||
project = %log_ctx.project,
|
|
||||||
"Entity save failed"
|
|
||||||
);
|
|
||||||
// Return Ok(false) to allow processing to continue; don't panic
|
|
||||||
Ok(false)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Save entity to database via raw SQL (normally would use EntityRepo trait)
|
/// Save entity to database via raw SQL (normally would use EntityRepo trait)
|
||||||
/// NOTE: async_trait requires manual implementation for non-trait functions
|
|
||||||
async fn save_entity_to_db(pool: &PgPool, entity: &mem_core::entity::Entity) -> Result<()> {
|
async fn save_entity_to_db(pool: &PgPool, entity: &mem_core::entity::Entity) -> Result<()> {
|
||||||
// Convert OffsetDateTime to PostgreSQL timestamp format
|
// Convert OffsetDateTime to PostgreSQL timestamp format
|
||||||
let t_created_str = entity.t_created.to_string();
|
let t_created_str = entity.t_created.to_string();
|
||||||
|
|
||||||
sqlx::query(
|
sqlx::query(
|
||||||
"INSERT INTO memory_entity (id, project_id, name, entity_type, description, t_created, t_updated, confidence)
|
"INSERT INTO memory_entity (id, project_id, name, entity_type, description, t_created, t_updated, confidence)
|
||||||
VALUES ($1::UUID, $2, $3, $4, $5, $6::TIMESTAMPTZ, $7::TIMESTAMPTZ, $8)
|
VALUES ($1, $2, $3, $4, $5, $6::TIMESTAMPTZ, $7::TIMESTAMPTZ, $8)
|
||||||
ON CONFLICT (project_id, name) DO UPDATE SET
|
ON CONFLICT (id) DO NOTHING"
|
||||||
entity_type = EXCLUDED.entity_type,
|
|
||||||
description = COALESCE(NULLIF(EXCLUDED.description, ''), memory_entity.description),
|
|
||||||
t_updated = NOW(),
|
|
||||||
confidence = GREATEST(memory_entity.confidence, EXCLUDED.confidence),
|
|
||||||
source_count = memory_entity.source_count + 1"
|
|
||||||
)
|
)
|
||||||
.bind(&entity.id)
|
.bind(&entity.id)
|
||||||
.bind(&entity.project_id)
|
.bind(&entity.project_id)
|
||||||
@@ -406,47 +188,13 @@ async fn save_entity_to_db(pool: &PgPool, entity: &mem_core::entity::Entity) ->
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Save edge with logging — logs at debug level on success, warn on error
|
|
||||||
/// Returns Ok(true) if saved, Ok(false) if skipped, Err if fatal error
|
|
||||||
async fn save_edge_with_logging(
|
|
||||||
pool: &PgPool,
|
|
||||||
edge: &mem_core::edge::Edge,
|
|
||||||
log_ctx: &IngestLogContext,
|
|
||||||
) -> Result<bool> {
|
|
||||||
match save_edge_to_db(pool, edge).await {
|
|
||||||
Ok(_) => {
|
|
||||||
tracing::debug!(
|
|
||||||
target: "ingest",
|
|
||||||
record_id = %log_ctx.record_id,
|
|
||||||
relation_type = &edge.relation_type,
|
|
||||||
source_entity = &edge.source_entity_id,
|
|
||||||
target_entity = &edge.target_entity_id,
|
|
||||||
"Saved edge"
|
|
||||||
);
|
|
||||||
Ok(true)
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
tracing::warn!(
|
|
||||||
target: "ingest",
|
|
||||||
error = %e,
|
|
||||||
record_id = %log_ctx.record_id,
|
|
||||||
relation_type = &edge.relation_type,
|
|
||||||
project = %log_ctx.project,
|
|
||||||
"Edge save failed"
|
|
||||||
);
|
|
||||||
// Return Ok(false) to allow processing to continue
|
|
||||||
Ok(false)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Save edge to database via raw SQL (normally would use EdgeRepo trait)
|
/// Save edge to database via raw SQL (normally would use EdgeRepo trait)
|
||||||
/// NOTE: Production DB may have old schema. Gracefully skip if temporal columns missing.
|
/// NOTE: Production DB may have old schema. Gracefully skip if temporal columns missing.
|
||||||
async fn save_edge_to_db(pool: &PgPool, edge: &mem_core::edge::Edge) -> Result<()> {
|
async fn save_edge_to_db(pool: &PgPool, edge: &mem_core::edge::Edge) -> Result<()> {
|
||||||
// Try temporal schema first (id, project_id, source_entity_id, etc)
|
// Try temporal schema first (id, project_id, source_entity_id, etc)
|
||||||
let result = sqlx::query(
|
let result = sqlx::query(
|
||||||
"INSERT INTO memory_edge (id, project_id, source_id, target_id, relation_type, fact, t_valid, t_invalid, t_created, confidence)
|
"INSERT INTO memory_edge (id, project_id, source_entity_id, target_entity_id, relation_type, fact, t_valid, t_invalid, t_created, confidence)
|
||||||
VALUES ($1::UUID, $2, $3::UUID, $4::UUID, $5, $6, $7::TIMESTAMPTZ, $8::TIMESTAMPTZ, $9::TIMESTAMPTZ, $10)
|
VALUES ($1, $2, $3, $4, $5, $6, $7::TIMESTAMPTZ, $8::TIMESTAMPTZ, $9::TIMESTAMPTZ, $10)
|
||||||
ON CONFLICT (id) DO NOTHING"
|
ON CONFLICT (id) DO NOTHING"
|
||||||
)
|
)
|
||||||
.bind(&edge.id)
|
.bind(&edge.id)
|
||||||
|
|||||||
@@ -1,9 +1,6 @@
|
|||||||
pub mod endpoints;
|
pub mod endpoints;
|
||||||
pub mod handlers;
|
pub mod handlers;
|
||||||
pub mod http_server;
|
pub mod http_server;
|
||||||
pub mod metrics;
|
|
||||||
pub mod metrics_snapshot;
|
|
||||||
pub mod relevance_judge;
|
|
||||||
pub mod query;
|
pub mod query;
|
||||||
pub mod auth;
|
pub mod auth;
|
||||||
pub mod ingest_worker;
|
pub mod ingest_worker;
|
||||||
|
|||||||
@@ -1,700 +0,0 @@
|
|||||||
//! Prometheus metrics module (O10)
|
|
||||||
//!
|
|
||||||
//! Centralized metrics registry for poimen-memory observability.
|
|
||||||
//! All handlers instrument via these shared metrics.
|
|
||||||
//! Exposed at GET /metrics in Prometheus text format.
|
|
||||||
|
|
||||||
use once_cell::sync::Lazy;
|
|
||||||
use std::sync::atomic::{AtomicU64, Ordering};
|
|
||||||
use std::collections::HashMap;
|
|
||||||
use std::sync::Mutex;
|
|
||||||
use std::time::Instant;
|
|
||||||
|
|
||||||
// ─── Metric Types ───────────────────────────────────────────
|
|
||||||
|
|
||||||
/// Simple counter (monotonically increasing)
|
|
||||||
pub struct Counter {
|
|
||||||
value: AtomicU64,
|
|
||||||
name: &'static str,
|
|
||||||
help: &'static str,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Counter {
|
|
||||||
pub const fn new(name: &'static str, help: &'static str) -> Self {
|
|
||||||
Self { value: AtomicU64::new(0), name, help }
|
|
||||||
}
|
|
||||||
pub fn inc(&self) { self.value.fetch_add(1, Ordering::Relaxed); }
|
|
||||||
pub fn inc_by(&self, n: u64) { self.value.fetch_add(n, Ordering::Relaxed); }
|
|
||||||
pub fn get(&self) -> u64 { self.value.load(Ordering::Relaxed) }
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Gauge (can go up and down)
|
|
||||||
pub struct Gauge {
|
|
||||||
value: AtomicU64,
|
|
||||||
name: &'static str,
|
|
||||||
help: &'static str,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Gauge {
|
|
||||||
pub const fn new(name: &'static str, help: &'static str) -> Self {
|
|
||||||
Self { value: AtomicU64::new(0), name, help }
|
|
||||||
}
|
|
||||||
pub fn set(&self, v: u64) { self.value.store(v, Ordering::Relaxed); }
|
|
||||||
pub fn inc(&self) { self.value.fetch_add(1, Ordering::Relaxed); }
|
|
||||||
pub fn dec(&self) { self.value.fetch_sub(1, Ordering::Relaxed); }
|
|
||||||
pub fn get(&self) -> u64 { self.value.load(Ordering::Relaxed) }
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Gauge for f64 values (stored as bits)
|
|
||||||
pub struct GaugeF64 {
|
|
||||||
bits: AtomicU64,
|
|
||||||
name: &'static str,
|
|
||||||
help: &'static str,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl GaugeF64 {
|
|
||||||
pub const fn new(name: &'static str, help: &'static str) -> Self {
|
|
||||||
Self { bits: AtomicU64::new(0), name, help }
|
|
||||||
}
|
|
||||||
pub fn set(&self, v: f64) { self.bits.store(v.to_bits(), Ordering::Relaxed); }
|
|
||||||
pub fn get(&self) -> f64 { f64::from_bits(self.bits.load(Ordering::Relaxed)) }
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Histogram with fixed buckets for latency tracking
|
|
||||||
pub struct Histogram {
|
|
||||||
pub buckets: &'static [f64],
|
|
||||||
pub counts: Vec<AtomicU64>,
|
|
||||||
pub sum: AtomicU64, // stored as f64 bits
|
|
||||||
pub count: AtomicU64,
|
|
||||||
pub name: &'static str,
|
|
||||||
pub help: &'static str,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Histogram {
|
|
||||||
pub fn new(name: &'static str, help: &'static str, buckets: &'static [f64]) -> Self {
|
|
||||||
let counts = (0..buckets.len() + 1).map(|_| AtomicU64::new(0)).collect();
|
|
||||||
Self {
|
|
||||||
buckets, counts, name, help,
|
|
||||||
sum: AtomicU64::new(0f64.to_bits()),
|
|
||||||
count: AtomicU64::new(0),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn observe(&self, value: f64) {
|
|
||||||
self.count.fetch_add(1, Ordering::Relaxed);
|
|
||||||
// Add to sum (CAS loop for f64)
|
|
||||||
loop {
|
|
||||||
let old_bits = self.sum.load(Ordering::Relaxed);
|
|
||||||
let old = f64::from_bits(old_bits);
|
|
||||||
let new = old + value;
|
|
||||||
if self.sum.compare_exchange(old_bits, new.to_bits(), Ordering::Relaxed, Ordering::Relaxed).is_ok() {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Increment bucket counters
|
|
||||||
for (i, &bound) in self.buckets.iter().enumerate() {
|
|
||||||
if value <= bound {
|
|
||||||
self.counts[i].fetch_add(1, Ordering::Relaxed);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// +Inf bucket
|
|
||||||
self.counts[self.buckets.len()].fetch_add(1, Ordering::Relaxed);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Labeled counter (key = label combination string)
|
|
||||||
pub struct LabeledCounter {
|
|
||||||
values: Mutex<HashMap<String, u64>>,
|
|
||||||
name: &'static str,
|
|
||||||
help: &'static str,
|
|
||||||
label_names: &'static [&'static str],
|
|
||||||
}
|
|
||||||
|
|
||||||
impl LabeledCounter {
|
|
||||||
pub fn new(name: &'static str, help: &'static str, label_names: &'static [&'static str]) -> Self {
|
|
||||||
Self { values: Mutex::new(HashMap::new()), name, help, label_names }
|
|
||||||
}
|
|
||||||
pub fn inc(&self, labels: &[&str]) {
|
|
||||||
let key = labels.join(",");
|
|
||||||
let mut map = self.values.lock().unwrap();
|
|
||||||
*map.entry(key).or_insert(0) += 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── Timer helper ───────────────────────────────────────────
|
|
||||||
|
|
||||||
/// RAII timer: observes duration on drop
|
|
||||||
pub struct Timer<'a> {
|
|
||||||
histogram: &'a Histogram,
|
|
||||||
start: Instant,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<'a> Timer<'a> {
|
|
||||||
pub fn new(histogram: &'a Histogram) -> Self {
|
|
||||||
Self { histogram, start: Instant::now() }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<'a> Drop for Timer<'a> {
|
|
||||||
fn drop(&mut self) {
|
|
||||||
let elapsed = self.start.elapsed().as_secs_f64();
|
|
||||||
self.histogram.observe(elapsed);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── Default buckets ────────────────────────────────────────
|
|
||||||
|
|
||||||
/// Latency buckets for HTTP handlers (seconds)
|
|
||||||
pub static HTTP_BUCKETS: &[f64] = &[0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0];
|
|
||||||
/// Latency buckets for LLM calls (seconds)
|
|
||||||
pub static LLM_BUCKETS: &[f64] = &[0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0];
|
|
||||||
/// Latency buckets for DB queries (seconds)
|
|
||||||
pub static DB_BUCKETS: &[f64] = &[0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0];
|
|
||||||
|
|
||||||
// ═══════════════════════════════════════════════════════════
|
|
||||||
// O1: Ingest handler metrics (I1-I12)
|
|
||||||
// ═══════════════════════════════════════════════════════════
|
|
||||||
|
|
||||||
pub static INGEST_REQUESTS_TOTAL: Counter = Counter::new(
|
|
||||||
"memory_ingest_requests_total", "Total ingest requests received");
|
|
||||||
pub static INGEST_ERRORS_TOTAL: Counter = Counter::new(
|
|
||||||
"memory_ingest_errors_total", "Total ingest request errors");
|
|
||||||
pub static INGEST_RECORDS_TOTAL: Counter = Counter::new(
|
|
||||||
"memory_ingest_records_total", "Total records ingested");
|
|
||||||
pub static INGEST_ENTITIES_EXTRACTED: Counter = Counter::new(
|
|
||||||
"memory_ingest_entities_extracted_total", "Total entities extracted during ingest");
|
|
||||||
pub static INGEST_EDGES_EXTRACTED: Counter = Counter::new(
|
|
||||||
"memory_ingest_edges_extracted_total", "Total edges extracted during ingest");
|
|
||||||
pub static INGEST_IN_FLIGHT: Gauge = Gauge::new(
|
|
||||||
"memory_ingest_in_flight", "Currently processing ingest jobs");
|
|
||||||
pub static INGEST_QUEUE_SIZE: Gauge = Gauge::new(
|
|
||||||
"memory_ingest_queue_size", "Number of jobs waiting in ingest queue");
|
|
||||||
pub static INGEST_DUPLICATES_TOTAL: Counter = Counter::new(
|
|
||||||
"memory_ingest_duplicates_total", "Total duplicate ingest requests (idempotency)");
|
|
||||||
pub static INGEST_BYTES_TOTAL: Counter = Counter::new(
|
|
||||||
"memory_ingest_bytes_total", "Total bytes ingested");
|
|
||||||
pub static INGEST_AUTH_FAILURES: Counter = Counter::new(
|
|
||||||
"memory_ingest_auth_failures_total", "Total auth failures on ingest endpoint");
|
|
||||||
pub static INGEST_RATE_LIMITED: Counter = Counter::new(
|
|
||||||
"memory_ingest_rate_limited_total", "Total rate-limited ingest requests");
|
|
||||||
|
|
||||||
pub static INGEST_DURATION: Lazy<Histogram> = Lazy::new(||
|
|
||||||
Histogram::new("memory_ingest_duration_seconds", "Ingest request duration", HTTP_BUCKETS));
|
|
||||||
|
|
||||||
// ═══════════════════════════════════════════════════════════
|
|
||||||
// O2: Query handler metrics (Q1-Q12)
|
|
||||||
// ═══════════════════════════════════════════════════════════
|
|
||||||
|
|
||||||
pub static QUERY_REQUESTS_TOTAL: Counter = Counter::new(
|
|
||||||
"memory_query_requests_total", "Total query requests received");
|
|
||||||
pub static QUERY_ERRORS_TOTAL: Counter = Counter::new(
|
|
||||||
"memory_query_errors_total", "Total query request errors");
|
|
||||||
pub static QUERY_RESULTS_TOTAL: Counter = Counter::new(
|
|
||||||
"memory_query_results_total", "Total results returned across all queries");
|
|
||||||
pub static QUERY_EMPTY_RESULTS: Counter = Counter::new(
|
|
||||||
"memory_query_empty_results_total", "Queries returning zero results");
|
|
||||||
pub static QUERY_EMBEDDING_FAILURES: Counter = Counter::new(
|
|
||||||
"memory_query_embedding_failures_total", "Total embedding failures during query");
|
|
||||||
pub static QUERY_IN_FLIGHT: Gauge = Gauge::new(
|
|
||||||
"memory_query_in_flight", "Currently processing queries");
|
|
||||||
pub static QUERY_AUTH_FAILURES: Counter = Counter::new(
|
|
||||||
"memory_query_auth_failures_total", "Total auth failures on query endpoint");
|
|
||||||
pub static QUERY_RATE_LIMITED: Counter = Counter::new(
|
|
||||||
"memory_query_rate_limited_total", "Total rate-limited query requests");
|
|
||||||
pub static QUERY_CACHE_HITS: Counter = Counter::new(
|
|
||||||
"memory_query_cache_hits_total", "Total query cache hits");
|
|
||||||
pub static QUERY_CACHE_MISSES: Counter = Counter::new(
|
|
||||||
"memory_query_cache_misses_total", "Total query cache misses");
|
|
||||||
|
|
||||||
pub static QUERY_DURATION: Lazy<Histogram> = Lazy::new(||
|
|
||||||
Histogram::new("memory_query_duration_seconds", "Query request duration", HTTP_BUCKETS));
|
|
||||||
pub static QUERY_EMBEDDING_DURATION: Lazy<Histogram> = Lazy::new(||
|
|
||||||
Histogram::new("memory_query_embedding_duration_seconds", "Embedding call duration during query", LLM_BUCKETS));
|
|
||||||
|
|
||||||
// ═══════════════════════════════════════════════════════════
|
|
||||||
// O3: Context endpoint metrics (C1-C8)
|
|
||||||
// ═══════════════════════════════════════════════════════════
|
|
||||||
|
|
||||||
pub static CONTEXT_REQUESTS_TOTAL: Counter = Counter::new(
|
|
||||||
"memory_context_requests_total", "Total context retrieval requests");
|
|
||||||
pub static CONTEXT_ERRORS_TOTAL: Counter = Counter::new(
|
|
||||||
"memory_context_errors_total", "Total context retrieval errors");
|
|
||||||
pub static CONTEXT_SEMANTIC_HITS: Counter = Counter::new(
|
|
||||||
"memory_context_semantic_hits_total", "Results from semantic (cosine) tier");
|
|
||||||
pub static CONTEXT_BM25_HITS: Counter = Counter::new(
|
|
||||||
"memory_context_bm25_hits_total", "Results from BM25 (lexical) tier");
|
|
||||||
pub static CONTEXT_GRAPH_HITS: Counter = Counter::new(
|
|
||||||
"memory_context_graph_hits_total", "Results from graph traversal tier");
|
|
||||||
pub static CONTEXT_EMPTY_RESULTS: Counter = Counter::new(
|
|
||||||
"memory_context_empty_results_total", "Context requests returning zero results");
|
|
||||||
|
|
||||||
pub static CONTEXT_DURATION: Lazy<Histogram> = Lazy::new(||
|
|
||||||
Histogram::new("memory_context_duration_seconds", "Context retrieval duration", HTTP_BUCKETS));
|
|
||||||
pub static CONTEXT_TIER_DURATION: Lazy<Histogram> = Lazy::new(||
|
|
||||||
Histogram::new("memory_context_tier_duration_seconds", "Per-tier retrieval duration", DB_BUCKETS));
|
|
||||||
|
|
||||||
// ═══════════════════════════════════════════════════════════
|
|
||||||
// O4: Relevance judge metrics (R1-R9)
|
|
||||||
// ═══════════════════════════════════════════════════════════
|
|
||||||
|
|
||||||
pub static RELEVANCE_EVALS_TOTAL: Counter = Counter::new(
|
|
||||||
"memory_relevance_evals_total", "Total relevance evaluations performed");
|
|
||||||
pub static RELEVANCE_ERRORS_TOTAL: Counter = Counter::new(
|
|
||||||
"memory_relevance_errors_total", "Total relevance evaluation errors");
|
|
||||||
pub static RELEVANCE_RELEVANT_TOTAL: Counter = Counter::new(
|
|
||||||
"memory_relevance_relevant_total", "Results judged relevant");
|
|
||||||
pub static RELEVANCE_IRRELEVANT_TOTAL: Counter = Counter::new(
|
|
||||||
"memory_relevance_irrelevant_total", "Results judged irrelevant");
|
|
||||||
|
|
||||||
pub static RELEVANCE_SCORE: Lazy<Histogram> = Lazy::new(||
|
|
||||||
Histogram::new("memory_relevance_score", "Distribution of relevance scores",
|
|
||||||
&[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]));
|
|
||||||
pub static RELEVANCE_PRECISION: GaugeF64 = GaugeF64::new(
|
|
||||||
"memory_relevance_precision", "Current precision (relevant/retrieved)");
|
|
||||||
pub static RELEVANCE_RECALL: GaugeF64 = GaugeF64::new(
|
|
||||||
"memory_relevance_recall", "Current recall (relevant/total_relevant)");
|
|
||||||
pub static RELEVANCE_F1: GaugeF64 = GaugeF64::new(
|
|
||||||
"memory_relevance_f1_score", "Current F1 score");
|
|
||||||
pub static RELEVANCE_EVAL_DURATION: Lazy<Histogram> = Lazy::new(||
|
|
||||||
Histogram::new("memory_relevance_eval_duration_seconds", "Relevance evaluation duration", LLM_BUCKETS));
|
|
||||||
|
|
||||||
// ═══════════════════════════════════════════════════════════
|
|
||||||
// O5: Write volume and storage metrics (W1-W12)
|
|
||||||
// ═══════════════════════════════════════════════════════════
|
|
||||||
|
|
||||||
pub static WRITE_ENTITIES_TOTAL: Counter = Counter::new(
|
|
||||||
"memory_write_entities_total", "Total entities written to DB");
|
|
||||||
pub static WRITE_EDGES_TOTAL: Counter = Counter::new(
|
|
||||||
"memory_write_edges_total", "Total edges written to DB");
|
|
||||||
pub static WRITE_CHUNKS_TOTAL: Counter = Counter::new(
|
|
||||||
"memory_write_chunks_total", "Total chunks written to DB");
|
|
||||||
pub static WRITE_ERRORS_TOTAL: Counter = Counter::new(
|
|
||||||
"memory_write_errors_total", "Total write errors");
|
|
||||||
pub static WRITE_BYTES_TOTAL: Counter = Counter::new(
|
|
||||||
"memory_write_bytes_total", "Total bytes written to storage");
|
|
||||||
|
|
||||||
pub static DB_ENTITY_COUNT: Gauge = Gauge::new(
|
|
||||||
"memory_db_entity_count", "Current entity count in memory_entity table");
|
|
||||||
pub static DB_EDGE_COUNT: Gauge = Gauge::new(
|
|
||||||
"memory_db_edge_count", "Current edge count in memory_edge table");
|
|
||||||
pub static DB_CHUNK_COUNT: Gauge = Gauge::new(
|
|
||||||
"memory_db_chunk_count", "Current chunk count in memory_chunks table");
|
|
||||||
|
|
||||||
pub static WRITE_DURATION: Lazy<Histogram> = Lazy::new(||
|
|
||||||
Histogram::new("memory_write_duration_seconds", "Write operation duration", DB_BUCKETS));
|
|
||||||
pub static WRITE_BATCH_SIZE: Lazy<Histogram> = Lazy::new(||
|
|
||||||
Histogram::new("memory_write_batch_size", "Write batch sizes",
|
|
||||||
&[1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0]));
|
|
||||||
|
|
||||||
// Storage gauges (updated periodically)
|
|
||||||
pub static DB_SIZE_BYTES: Gauge = Gauge::new(
|
|
||||||
"memory_db_size_bytes", "Total database size in bytes");
|
|
||||||
pub static DB_INDEX_SIZE_BYTES: Gauge = Gauge::new(
|
|
||||||
"memory_db_index_size_bytes", "Total index size in bytes");
|
|
||||||
|
|
||||||
// ═══════════════════════════════════════════════════════════
|
|
||||||
// O6: Pod resource observability (P1-P13)
|
|
||||||
// (Most collected by node-exporter/cAdvisor, but we track app-level)
|
|
||||||
// ═══════════════════════════════════════════════════════════
|
|
||||||
|
|
||||||
pub static APP_UPTIME_SECONDS: Gauge = Gauge::new(
|
|
||||||
"memory_app_uptime_seconds", "Application uptime in seconds");
|
|
||||||
pub static APP_ACTIVE_CONNECTIONS: Gauge = Gauge::new(
|
|
||||||
"memory_app_active_connections", "Active HTTP connections");
|
|
||||||
pub static APP_GOROUTINES: Gauge = Gauge::new(
|
|
||||||
"memory_app_tokio_tasks", "Active tokio tasks (approximate)");
|
|
||||||
pub static APP_HEAP_BYTES: Gauge = Gauge::new(
|
|
||||||
"memory_app_heap_bytes", "Approximate heap memory usage");
|
|
||||||
|
|
||||||
// ═══════════════════════════════════════════════════════════
|
|
||||||
// O7: Availability metrics and dependency health (A1-A10)
|
|
||||||
// ═══════════════════════════════════════════════════════════
|
|
||||||
|
|
||||||
pub static HEALTH_CHECKS_TOTAL: Counter = Counter::new(
|
|
||||||
"memory_health_checks_total", "Total health check requests");
|
|
||||||
pub static HEALTH_CHECK_FAILURES: Counter = Counter::new(
|
|
||||||
"memory_health_check_failures_total", "Total health check failures");
|
|
||||||
|
|
||||||
pub static DEP_DB_UP: Gauge = Gauge::new(
|
|
||||||
"memory_dependency_db_up", "Database dependency health (1=up, 0=down)");
|
|
||||||
pub static DEP_EMBEDDING_UP: Gauge = Gauge::new(
|
|
||||||
"memory_dependency_embedding_up", "Embedding service health (1=up, 0=down)");
|
|
||||||
pub static DEP_OPENSEARCH_UP: Gauge = Gauge::new(
|
|
||||||
"memory_dependency_opensearch_up", "OpenSearch dependency health (1=up, 0=down)");
|
|
||||||
pub static DEP_LLM_UP: Gauge = Gauge::new(
|
|
||||||
"memory_dependency_llm_up", "LLM service health (1=up, 0=down)");
|
|
||||||
|
|
||||||
pub static DEP_DB_LATENCY: Lazy<Histogram> = Lazy::new(||
|
|
||||||
Histogram::new("memory_dependency_db_latency_seconds", "DB health check latency", DB_BUCKETS));
|
|
||||||
pub static DEP_EMBEDDING_LATENCY: Lazy<Histogram> = Lazy::new(||
|
|
||||||
Histogram::new("memory_dependency_embedding_latency_seconds", "Embedding health check latency", LLM_BUCKETS));
|
|
||||||
|
|
||||||
pub static REQUEST_ERRORS_BY_STATUS: Lazy<LabeledCounter> = Lazy::new(||
|
|
||||||
LabeledCounter::new(
|
|
||||||
"memory_request_errors_by_status", "Request errors by HTTP status code",
|
|
||||||
&["status", "endpoint"]));
|
|
||||||
|
|
||||||
// ═══════════════════════════════════════════════════════════
|
|
||||||
// Named error counters (per error type, per endpoint)
|
|
||||||
// Format: memory_error_{ERROR_NAME}_{ENDPOINT}_total
|
|
||||||
// ═══════════════════════════════════════════════════════════
|
|
||||||
|
|
||||||
// Ingest errors
|
|
||||||
pub static ERROR_AUTH_FAILURE_INGEST: Counter = Counter::new(
|
|
||||||
"memory_error_auth_failure_ingest_total", "Auth failures on ingest endpoint");
|
|
||||||
pub static ERROR_FORBIDDEN_INGEST: Counter = Counter::new(
|
|
||||||
"memory_error_forbidden_ingest_total", "Forbidden (missing capability) on ingest");
|
|
||||||
pub static ERROR_RATE_LIMITED_INGEST: Counter = Counter::new(
|
|
||||||
"memory_error_rate_limited_ingest_total", "Rate limited on ingest");
|
|
||||||
pub static ERROR_BAD_REQUEST_INGEST: Counter = Counter::new(
|
|
||||||
"memory_error_bad_request_ingest_total", "Bad request on ingest");
|
|
||||||
pub static ERROR_DB_ERROR_INGEST: Counter = Counter::new(
|
|
||||||
"memory_error_db_error_ingest_total", "Database error during ingest");
|
|
||||||
|
|
||||||
// Query errors
|
|
||||||
pub static ERROR_AUTH_FAILURE_QUERY: Counter = Counter::new(
|
|
||||||
"memory_error_auth_failure_query_total", "Auth failures on query endpoint");
|
|
||||||
pub static ERROR_FORBIDDEN_QUERY: Counter = Counter::new(
|
|
||||||
"memory_error_forbidden_query_total", "Forbidden (missing capability) on query");
|
|
||||||
pub static ERROR_BAD_REQUEST_QUERY: Counter = Counter::new(
|
|
||||||
"memory_error_bad_request_query_total", "Bad request on query");
|
|
||||||
pub static ERROR_EMBEDDING_FAILURE_QUERY: Counter = Counter::new(
|
|
||||||
"memory_error_embedding_failure_query_total", "Embedding service failure during query");
|
|
||||||
pub static ERROR_SEARCH_FAILURE_QUERY: Counter = Counter::new(
|
|
||||||
"memory_error_search_failure_query_total", "Search execution failure during query");
|
|
||||||
|
|
||||||
// Context errors
|
|
||||||
pub static ERROR_AUTH_FAILURE_CONTEXT: Counter = Counter::new(
|
|
||||||
"memory_error_auth_failure_context_total", "Auth failures on context endpoint");
|
|
||||||
pub static ERROR_FORBIDDEN_CONTEXT: Counter = Counter::new(
|
|
||||||
"memory_error_forbidden_context_total", "Forbidden (missing capability) on context");
|
|
||||||
pub static ERROR_LOOKUP_FAILURE_CONTEXT: Counter = Counter::new(
|
|
||||||
"memory_error_lookup_failure_context_total", "Context lookup failure");
|
|
||||||
|
|
||||||
// Unexpected errors (unhandled 500s, panics, unknown failures)
|
|
||||||
pub static ERROR_UNEXPECTED_TOTAL: Counter = Counter::new(
|
|
||||||
"memory_error_unexpected_total", "Total unexpected/unhandled errors (500s)");
|
|
||||||
pub static ERROR_UNEXPECTED_INGEST: Counter = Counter::new(
|
|
||||||
"memory_error_unexpected_ingest_total", "Unexpected errors during ingest");
|
|
||||||
pub static ERROR_UNEXPECTED_QUERY: Counter = Counter::new(
|
|
||||||
"memory_error_unexpected_query_total", "Unexpected errors during query");
|
|
||||||
pub static ERROR_UNEXPECTED_CONTEXT: Counter = Counter::new(
|
|
||||||
"memory_error_unexpected_context_total", "Unexpected errors during context");
|
|
||||||
|
|
||||||
// Agent endpoint error counters
|
|
||||||
pub static ERROR_AUTH_FAILURE_AGENT: Counter = Counter::new(
|
|
||||||
"memory_error_auth_failure_agent_total", "Auth failures on agent endpoints");
|
|
||||||
pub static ERROR_BAD_REQUEST_AGENT: Counter = Counter::new(
|
|
||||||
"memory_error_bad_request_agent_total", "Bad request errors on agent endpoints (expected)");
|
|
||||||
pub static ERROR_NOT_FOUND_AGENT: Counter = Counter::new(
|
|
||||||
"memory_error_not_found_agent_total", "Not found errors on agent endpoints (expected)");
|
|
||||||
pub static ERROR_UNEXPECTED_AGENT: Counter = Counter::new(
|
|
||||||
"memory_error_unexpected_agent_total", "Unexpected errors on agent endpoints (DB failures, 500s)");
|
|
||||||
|
|
||||||
// Last error info (most recent error for debugging)
|
|
||||||
pub static LAST_ERROR_TIMESTAMP: Gauge = Gauge::new(
|
|
||||||
"memory_last_error_timestamp_seconds", "Unix timestamp of most recent error");
|
|
||||||
|
|
||||||
// ═══════════════════════════════════════════════════════════
|
|
||||||
// O8: Ingest rate pattern tracking (IR1-IR10)
|
|
||||||
// ═══════════════════════════════════════════════════════════
|
|
||||||
|
|
||||||
pub static INGEST_RATE_1M: GaugeF64 = GaugeF64::new(
|
|
||||||
"memory_ingest_rate_1m", "Ingest rate per second (1-minute window)");
|
|
||||||
pub static INGEST_RATE_5M: GaugeF64 = GaugeF64::new(
|
|
||||||
"memory_ingest_rate_5m", "Ingest rate per second (5-minute window)");
|
|
||||||
pub static INGEST_LLM_EXTRACT_DURATION: Lazy<Histogram> = Lazy::new(||
|
|
||||||
Histogram::new("memory_ingest_llm_extract_duration_seconds", "LLM entity extraction duration", LLM_BUCKETS));
|
|
||||||
pub static INGEST_FACT_EXTRACT_DURATION: Lazy<Histogram> = Lazy::new(||
|
|
||||||
Histogram::new("memory_ingest_fact_extract_duration_seconds", "LLM fact extraction duration", LLM_BUCKETS));
|
|
||||||
pub static INGEST_DEDUP_TOTAL: Counter = Counter::new(
|
|
||||||
"memory_ingest_dedup_total", "Total entities deduplicated");
|
|
||||||
pub static INGEST_CONTRADICTION_TOTAL: Counter = Counter::new(
|
|
||||||
"memory_ingest_contradiction_total", "Total contradictions detected");
|
|
||||||
pub static INGEST_PROJECTS: Gauge = Gauge::new(
|
|
||||||
"memory_ingest_active_projects", "Number of active projects with ingested data");
|
|
||||||
|
|
||||||
// ═══════════════════════════════════════════════════════════
|
|
||||||
// O9: Postgres internal observability (PG1-PG33)
|
|
||||||
// (Most collected by pg_exporter, we expose app-visible DB stats)
|
|
||||||
// ═══════════════════════════════════════════════════════════
|
|
||||||
|
|
||||||
pub static DB_POOL_SIZE: Gauge = Gauge::new(
|
|
||||||
"memory_db_pool_size", "Current connection pool size");
|
|
||||||
pub static DB_POOL_IDLE: Gauge = Gauge::new(
|
|
||||||
"memory_db_pool_idle", "Idle connections in pool");
|
|
||||||
pub static DB_POOL_ACTIVE: Gauge = Gauge::new(
|
|
||||||
"memory_db_pool_active", "Active connections in pool");
|
|
||||||
pub static DB_QUERY_TOTAL: Counter = Counter::new(
|
|
||||||
"memory_db_queries_total", "Total DB queries executed");
|
|
||||||
pub static DB_QUERY_ERRORS: Counter = Counter::new(
|
|
||||||
"memory_db_query_errors_total", "Total DB query errors");
|
|
||||||
pub static DB_QUERY_DURATION: Lazy<Histogram> = Lazy::new(||
|
|
||||||
Histogram::new("memory_db_query_duration_seconds", "DB query duration", DB_BUCKETS));
|
|
||||||
pub static DB_TRANSACTION_DURATION: Lazy<Histogram> = Lazy::new(||
|
|
||||||
Histogram::new("memory_db_transaction_duration_seconds", "DB transaction duration", DB_BUCKETS));
|
|
||||||
|
|
||||||
// Table-specific row counts (updated periodically)
|
|
||||||
pub static DB_TABLE_ENTITY_ROWS: Gauge = Gauge::new(
|
|
||||||
"memory_db_table_entity_rows", "Rows in memory_entity table");
|
|
||||||
pub static DB_TABLE_EDGE_ROWS: Gauge = Gauge::new(
|
|
||||||
"memory_db_table_edge_rows", "Rows in memory_edge table");
|
|
||||||
pub static DB_TABLE_CHUNK_ROWS: Gauge = Gauge::new(
|
|
||||||
"memory_db_table_chunk_rows", "Rows in memory_chunks table");
|
|
||||||
|
|
||||||
// ═══════════════════════════════════════════════════════════
|
|
||||||
// Metrics export (Prometheus text format)
|
|
||||||
// ═══════════════════════════════════════════════════════════
|
|
||||||
|
|
||||||
/// Render all metrics in Prometheus text exposition format
|
|
||||||
pub fn render_metrics() -> String {
|
|
||||||
let mut out = String::with_capacity(8192);
|
|
||||||
|
|
||||||
// Helper macros
|
|
||||||
macro_rules! counter {
|
|
||||||
($c:expr) => {
|
|
||||||
out.push_str(&format!("# HELP {} {}\n# TYPE {} counter\n{} {}\n",
|
|
||||||
$c.name, $c.help, $c.name, $c.name, $c.get()));
|
|
||||||
};
|
|
||||||
}
|
|
||||||
macro_rules! gauge {
|
|
||||||
($g:expr) => {
|
|
||||||
out.push_str(&format!("# HELP {} {}\n# TYPE {} gauge\n{} {}\n",
|
|
||||||
$g.name, $g.help, $g.name, $g.name, $g.get()));
|
|
||||||
};
|
|
||||||
}
|
|
||||||
macro_rules! gauge_f64 {
|
|
||||||
($g:expr) => {
|
|
||||||
out.push_str(&format!("# HELP {} {}\n# TYPE {} gauge\n{} {:.6}\n",
|
|
||||||
$g.name, $g.help, $g.name, $g.name, $g.get()));
|
|
||||||
};
|
|
||||||
}
|
|
||||||
macro_rules! histogram {
|
|
||||||
($h:expr) => {
|
|
||||||
out.push_str(&format!("# HELP {} {}\n# TYPE {} histogram\n", $h.name, $h.help, $h.name));
|
|
||||||
for (i, &bound) in $h.buckets.iter().enumerate() {
|
|
||||||
out.push_str(&format!("{}_bucket{{le=\"{}\"}} {}\n",
|
|
||||||
$h.name, bound, $h.counts[i].load(Ordering::Relaxed)));
|
|
||||||
}
|
|
||||||
out.push_str(&format!("{}_bucket{{le=\"+Inf\"}} {}\n",
|
|
||||||
$h.name, $h.counts[$h.buckets.len()].load(Ordering::Relaxed)));
|
|
||||||
out.push_str(&format!("{}_sum {:.6}\n", $h.name,
|
|
||||||
f64::from_bits($h.sum.load(Ordering::Relaxed))));
|
|
||||||
out.push_str(&format!("{}_count {}\n", $h.name,
|
|
||||||
$h.count.load(Ordering::Relaxed)));
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
// O1: Ingest
|
|
||||||
counter!(INGEST_REQUESTS_TOTAL);
|
|
||||||
counter!(INGEST_ERRORS_TOTAL);
|
|
||||||
counter!(INGEST_RECORDS_TOTAL);
|
|
||||||
counter!(INGEST_ENTITIES_EXTRACTED);
|
|
||||||
counter!(INGEST_EDGES_EXTRACTED);
|
|
||||||
gauge!(INGEST_IN_FLIGHT);
|
|
||||||
gauge!(INGEST_QUEUE_SIZE);
|
|
||||||
counter!(INGEST_DUPLICATES_TOTAL);
|
|
||||||
counter!(INGEST_BYTES_TOTAL);
|
|
||||||
counter!(INGEST_AUTH_FAILURES);
|
|
||||||
counter!(INGEST_RATE_LIMITED);
|
|
||||||
histogram!(INGEST_DURATION);
|
|
||||||
|
|
||||||
// O2: Query
|
|
||||||
counter!(QUERY_REQUESTS_TOTAL);
|
|
||||||
counter!(QUERY_ERRORS_TOTAL);
|
|
||||||
counter!(QUERY_RESULTS_TOTAL);
|
|
||||||
counter!(QUERY_EMPTY_RESULTS);
|
|
||||||
counter!(QUERY_EMBEDDING_FAILURES);
|
|
||||||
gauge!(QUERY_IN_FLIGHT);
|
|
||||||
counter!(QUERY_AUTH_FAILURES);
|
|
||||||
counter!(QUERY_RATE_LIMITED);
|
|
||||||
counter!(QUERY_CACHE_HITS);
|
|
||||||
counter!(QUERY_CACHE_MISSES);
|
|
||||||
histogram!(QUERY_DURATION);
|
|
||||||
histogram!(QUERY_EMBEDDING_DURATION);
|
|
||||||
|
|
||||||
// O3: Context
|
|
||||||
counter!(CONTEXT_REQUESTS_TOTAL);
|
|
||||||
counter!(CONTEXT_ERRORS_TOTAL);
|
|
||||||
counter!(CONTEXT_SEMANTIC_HITS);
|
|
||||||
counter!(CONTEXT_BM25_HITS);
|
|
||||||
counter!(CONTEXT_GRAPH_HITS);
|
|
||||||
counter!(CONTEXT_EMPTY_RESULTS);
|
|
||||||
histogram!(CONTEXT_DURATION);
|
|
||||||
histogram!(CONTEXT_TIER_DURATION);
|
|
||||||
|
|
||||||
// O4: Relevance
|
|
||||||
counter!(RELEVANCE_EVALS_TOTAL);
|
|
||||||
counter!(RELEVANCE_ERRORS_TOTAL);
|
|
||||||
counter!(RELEVANCE_RELEVANT_TOTAL);
|
|
||||||
counter!(RELEVANCE_IRRELEVANT_TOTAL);
|
|
||||||
histogram!(RELEVANCE_SCORE);
|
|
||||||
gauge_f64!(RELEVANCE_PRECISION);
|
|
||||||
gauge_f64!(RELEVANCE_RECALL);
|
|
||||||
gauge_f64!(RELEVANCE_F1);
|
|
||||||
histogram!(RELEVANCE_EVAL_DURATION);
|
|
||||||
|
|
||||||
// O5: Write volume
|
|
||||||
counter!(WRITE_ENTITIES_TOTAL);
|
|
||||||
counter!(WRITE_EDGES_TOTAL);
|
|
||||||
counter!(WRITE_CHUNKS_TOTAL);
|
|
||||||
counter!(WRITE_ERRORS_TOTAL);
|
|
||||||
counter!(WRITE_BYTES_TOTAL);
|
|
||||||
gauge!(DB_ENTITY_COUNT);
|
|
||||||
gauge!(DB_EDGE_COUNT);
|
|
||||||
gauge!(DB_CHUNK_COUNT);
|
|
||||||
histogram!(WRITE_DURATION);
|
|
||||||
histogram!(WRITE_BATCH_SIZE);
|
|
||||||
gauge!(DB_SIZE_BYTES);
|
|
||||||
gauge!(DB_INDEX_SIZE_BYTES);
|
|
||||||
|
|
||||||
// O6: Pod resources
|
|
||||||
gauge!(APP_UPTIME_SECONDS);
|
|
||||||
gauge!(APP_ACTIVE_CONNECTIONS);
|
|
||||||
gauge!(APP_GOROUTINES);
|
|
||||||
gauge!(APP_HEAP_BYTES);
|
|
||||||
|
|
||||||
// O7: Availability
|
|
||||||
counter!(HEALTH_CHECKS_TOTAL);
|
|
||||||
counter!(HEALTH_CHECK_FAILURES);
|
|
||||||
gauge!(DEP_DB_UP);
|
|
||||||
gauge!(DEP_EMBEDDING_UP);
|
|
||||||
gauge!(DEP_OPENSEARCH_UP);
|
|
||||||
gauge!(DEP_LLM_UP);
|
|
||||||
histogram!(DEP_DB_LATENCY);
|
|
||||||
histogram!(DEP_EMBEDDING_LATENCY);
|
|
||||||
|
|
||||||
// O8: Ingest rate
|
|
||||||
gauge_f64!(INGEST_RATE_1M);
|
|
||||||
gauge_f64!(INGEST_RATE_5M);
|
|
||||||
histogram!(INGEST_LLM_EXTRACT_DURATION);
|
|
||||||
histogram!(INGEST_FACT_EXTRACT_DURATION);
|
|
||||||
counter!(INGEST_DEDUP_TOTAL);
|
|
||||||
counter!(INGEST_CONTRADICTION_TOTAL);
|
|
||||||
gauge!(INGEST_PROJECTS);
|
|
||||||
|
|
||||||
// O9: Postgres
|
|
||||||
gauge!(DB_POOL_SIZE);
|
|
||||||
gauge!(DB_POOL_IDLE);
|
|
||||||
gauge!(DB_POOL_ACTIVE);
|
|
||||||
counter!(DB_QUERY_TOTAL);
|
|
||||||
counter!(DB_QUERY_ERRORS);
|
|
||||||
histogram!(DB_QUERY_DURATION);
|
|
||||||
histogram!(DB_TRANSACTION_DURATION);
|
|
||||||
gauge!(DB_TABLE_ENTITY_ROWS);
|
|
||||||
gauge!(DB_TABLE_EDGE_ROWS);
|
|
||||||
gauge!(DB_TABLE_CHUNK_ROWS);
|
|
||||||
|
|
||||||
// Named error counters
|
|
||||||
counter!(ERROR_AUTH_FAILURE_INGEST);
|
|
||||||
counter!(ERROR_FORBIDDEN_INGEST);
|
|
||||||
counter!(ERROR_RATE_LIMITED_INGEST);
|
|
||||||
counter!(ERROR_BAD_REQUEST_INGEST);
|
|
||||||
counter!(ERROR_DB_ERROR_INGEST);
|
|
||||||
counter!(ERROR_AUTH_FAILURE_QUERY);
|
|
||||||
counter!(ERROR_FORBIDDEN_QUERY);
|
|
||||||
counter!(ERROR_BAD_REQUEST_QUERY);
|
|
||||||
counter!(ERROR_EMBEDDING_FAILURE_QUERY);
|
|
||||||
counter!(ERROR_SEARCH_FAILURE_QUERY);
|
|
||||||
counter!(ERROR_AUTH_FAILURE_CONTEXT);
|
|
||||||
counter!(ERROR_FORBIDDEN_CONTEXT);
|
|
||||||
counter!(ERROR_LOOKUP_FAILURE_CONTEXT);
|
|
||||||
counter!(ERROR_UNEXPECTED_TOTAL);
|
|
||||||
counter!(ERROR_UNEXPECTED_INGEST);
|
|
||||||
counter!(ERROR_UNEXPECTED_QUERY);
|
|
||||||
counter!(ERROR_UNEXPECTED_CONTEXT);
|
|
||||||
counter!(ERROR_AUTH_FAILURE_AGENT);
|
|
||||||
counter!(ERROR_BAD_REQUEST_AGENT);
|
|
||||||
counter!(ERROR_NOT_FOUND_AGENT);
|
|
||||||
counter!(ERROR_UNEXPECTED_AGENT);
|
|
||||||
gauge!(LAST_ERROR_TIMESTAMP);
|
|
||||||
|
|
||||||
out
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Render a labeled counter in Prometheus format
|
|
||||||
fn render_labeled_counter(out: &mut String, lc: &LabeledCounter) {
|
|
||||||
let map = lc.values.lock().unwrap();
|
|
||||||
if map.is_empty() { return; }
|
|
||||||
out.push_str(&format!("# HELP {} {}\n# TYPE {} counter\n", lc.name, lc.help, lc.name));
|
|
||||||
for (key, val) in map.iter() {
|
|
||||||
let parts: Vec<&str> = key.split(',').collect();
|
|
||||||
let labels: Vec<String> = lc.label_names.iter().zip(parts.iter())
|
|
||||||
.map(|(name, val)| format!("{}=\"{}\"", name, val))
|
|
||||||
.collect();
|
|
||||||
out.push_str(&format!("{}{{{}}} {}\n", lc.name, labels.join(","), val));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// GET /metrics handler
|
|
||||||
pub async fn metrics_handler() -> actix_web::HttpResponse {
|
|
||||||
actix_web::HttpResponse::Ok()
|
|
||||||
.content_type("text/plain; version=0.0.4; charset=utf-8")
|
|
||||||
.body(render_metrics())
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_counter() {
|
|
||||||
let c = Counter::new("test_counter", "test");
|
|
||||||
assert_eq!(c.get(), 0);
|
|
||||||
c.inc();
|
|
||||||
assert_eq!(c.get(), 1);
|
|
||||||
c.inc_by(5);
|
|
||||||
assert_eq!(c.get(), 6);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_gauge() {
|
|
||||||
let g = Gauge::new("test_gauge", "test");
|
|
||||||
assert_eq!(g.get(), 0);
|
|
||||||
g.set(42);
|
|
||||||
assert_eq!(g.get(), 42);
|
|
||||||
g.inc();
|
|
||||||
assert_eq!(g.get(), 43);
|
|
||||||
g.dec();
|
|
||||||
assert_eq!(g.get(), 42);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_gauge_f64() {
|
|
||||||
let g = GaugeF64::new("test_gauge_f64", "test");
|
|
||||||
assert_eq!(g.get(), 0.0);
|
|
||||||
g.set(3.14);
|
|
||||||
assert!((g.get() - 3.14).abs() < 0.001);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_histogram() {
|
|
||||||
let h = Histogram::new("test_hist", "test", &[0.1, 0.5, 1.0]);
|
|
||||||
h.observe(0.05);
|
|
||||||
h.observe(0.3);
|
|
||||||
h.observe(0.8);
|
|
||||||
h.observe(2.0);
|
|
||||||
assert_eq!(h.count.load(Ordering::Relaxed), 4);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_render_metrics_not_empty() {
|
|
||||||
INGEST_REQUESTS_TOTAL.inc();
|
|
||||||
QUERY_REQUESTS_TOTAL.inc();
|
|
||||||
let output = render_metrics();
|
|
||||||
assert!(output.contains("memory_ingest_requests_total"));
|
|
||||||
assert!(output.contains("memory_query_requests_total"));
|
|
||||||
assert!(output.contains("# HELP"));
|
|
||||||
assert!(output.contains("# TYPE"));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_timer_observes_on_drop() {
|
|
||||||
let h = Histogram::new("timer_test", "test", HTTP_BUCKETS);
|
|
||||||
{
|
|
||||||
let _t = Timer::new(&h);
|
|
||||||
std::thread::sleep(std::time::Duration::from_millis(1));
|
|
||||||
}
|
|
||||||
assert_eq!(h.count.load(Ordering::Relaxed), 1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,418 +0,0 @@
|
|||||||
//! Metrics Snapshot & Assertion (Test Harness)
|
|
||||||
//!
|
|
||||||
//! Captures metric state before/after a test scenario,
|
|
||||||
//! then asserts expected deltas per metric.
|
|
||||||
//!
|
|
||||||
//! Usage:
|
|
||||||
//! ```rust
|
|
||||||
//! let snap = MetricsSnapshot::capture();
|
|
||||||
//! // ... run handler / scenario ...
|
|
||||||
//! snap.assert_counter_inc("memory_ingest_requests_total", 1);
|
|
||||||
//! snap.assert_counter_inc("memory_ingest_errors_total", 0);
|
|
||||||
//! snap.assert_gauge_eq("memory_ingest_in_flight", 0);
|
|
||||||
//! snap.assert_histogram_count_inc("memory_ingest_duration_seconds", 1);
|
|
||||||
//! ```
|
|
||||||
|
|
||||||
use std::collections::HashMap;
|
|
||||||
use std::sync::atomic::Ordering;
|
|
||||||
|
|
||||||
use crate::metrics;
|
|
||||||
|
|
||||||
/// Snapshot of all metric values at a point in time
|
|
||||||
#[derive(Debug, Clone)]
|
|
||||||
pub struct MetricsSnapshot {
|
|
||||||
counters: HashMap<&'static str, u64>,
|
|
||||||
gauges: HashMap<&'static str, u64>,
|
|
||||||
gauges_f64: HashMap<&'static str, f64>,
|
|
||||||
histogram_counts: HashMap<&'static str, u64>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl MetricsSnapshot {
|
|
||||||
/// Capture current state of all metrics
|
|
||||||
pub fn capture() -> Self {
|
|
||||||
let mut counters = HashMap::new();
|
|
||||||
let mut gauges = HashMap::new();
|
|
||||||
let mut gauges_f64 = HashMap::new();
|
|
||||||
let mut histogram_counts = HashMap::new();
|
|
||||||
|
|
||||||
// O1: Ingest counters
|
|
||||||
counters.insert("memory_ingest_requests_total", metrics::INGEST_REQUESTS_TOTAL.get());
|
|
||||||
counters.insert("memory_ingest_errors_total", metrics::INGEST_ERRORS_TOTAL.get());
|
|
||||||
counters.insert("memory_ingest_records_total", metrics::INGEST_RECORDS_TOTAL.get());
|
|
||||||
counters.insert("memory_ingest_entities_extracted_total", metrics::INGEST_ENTITIES_EXTRACTED.get());
|
|
||||||
counters.insert("memory_ingest_edges_extracted_total", metrics::INGEST_EDGES_EXTRACTED.get());
|
|
||||||
counters.insert("memory_ingest_duplicates_total", metrics::INGEST_DUPLICATES_TOTAL.get());
|
|
||||||
counters.insert("memory_ingest_bytes_total", metrics::INGEST_BYTES_TOTAL.get());
|
|
||||||
counters.insert("memory_ingest_auth_failures_total", metrics::INGEST_AUTH_FAILURES.get());
|
|
||||||
counters.insert("memory_ingest_rate_limited_total", metrics::INGEST_RATE_LIMITED.get());
|
|
||||||
|
|
||||||
// O1: Ingest gauges
|
|
||||||
gauges.insert("memory_ingest_in_flight", metrics::INGEST_IN_FLIGHT.get());
|
|
||||||
gauges.insert("memory_ingest_queue_size", metrics::INGEST_QUEUE_SIZE.get());
|
|
||||||
|
|
||||||
// O1: Ingest histogram (force Lazy init)
|
|
||||||
histogram_counts.insert("memory_ingest_duration_seconds",
|
|
||||||
{ let _ = &*metrics::INGEST_DURATION; metrics::INGEST_DURATION.count.load(Ordering::Relaxed) });
|
|
||||||
|
|
||||||
// O2: Query counters
|
|
||||||
counters.insert("memory_query_requests_total", metrics::QUERY_REQUESTS_TOTAL.get());
|
|
||||||
counters.insert("memory_query_errors_total", metrics::QUERY_ERRORS_TOTAL.get());
|
|
||||||
counters.insert("memory_query_results_total", metrics::QUERY_RESULTS_TOTAL.get());
|
|
||||||
counters.insert("memory_query_empty_results_total", metrics::QUERY_EMPTY_RESULTS.get());
|
|
||||||
counters.insert("memory_query_embedding_failures_total", metrics::QUERY_EMBEDDING_FAILURES.get());
|
|
||||||
counters.insert("memory_query_auth_failures_total", metrics::QUERY_AUTH_FAILURES.get());
|
|
||||||
counters.insert("memory_query_rate_limited_total", metrics::QUERY_RATE_LIMITED.get());
|
|
||||||
counters.insert("memory_query_cache_hits_total", metrics::QUERY_CACHE_HITS.get());
|
|
||||||
counters.insert("memory_query_cache_misses_total", metrics::QUERY_CACHE_MISSES.get());
|
|
||||||
|
|
||||||
// O2: Query gauges
|
|
||||||
gauges.insert("memory_query_in_flight", metrics::QUERY_IN_FLIGHT.get());
|
|
||||||
|
|
||||||
// O2: Query histograms
|
|
||||||
histogram_counts.insert("memory_query_duration_seconds",
|
|
||||||
{ let _ = &*metrics::QUERY_DURATION; metrics::QUERY_DURATION.count.load(Ordering::Relaxed) });
|
|
||||||
histogram_counts.insert("memory_query_embedding_duration_seconds",
|
|
||||||
{ let _ = &*metrics::QUERY_EMBEDDING_DURATION; metrics::QUERY_EMBEDDING_DURATION.count.load(Ordering::Relaxed) });
|
|
||||||
|
|
||||||
// O3: Context
|
|
||||||
counters.insert("memory_context_requests_total", metrics::CONTEXT_REQUESTS_TOTAL.get());
|
|
||||||
counters.insert("memory_context_errors_total", metrics::CONTEXT_ERRORS_TOTAL.get());
|
|
||||||
counters.insert("memory_context_semantic_hits_total", metrics::CONTEXT_SEMANTIC_HITS.get());
|
|
||||||
counters.insert("memory_context_bm25_hits_total", metrics::CONTEXT_BM25_HITS.get());
|
|
||||||
counters.insert("memory_context_graph_hits_total", metrics::CONTEXT_GRAPH_HITS.get());
|
|
||||||
counters.insert("memory_context_empty_results_total", metrics::CONTEXT_EMPTY_RESULTS.get());
|
|
||||||
histogram_counts.insert("memory_context_duration_seconds",
|
|
||||||
{ let _ = &*metrics::CONTEXT_DURATION; metrics::CONTEXT_DURATION.count.load(Ordering::Relaxed) });
|
|
||||||
|
|
||||||
// O4: Relevance histograms
|
|
||||||
histogram_counts.insert("memory_relevance_eval_duration_seconds",
|
|
||||||
{ let _ = &*metrics::RELEVANCE_EVAL_DURATION; metrics::RELEVANCE_EVAL_DURATION.count.load(Ordering::Relaxed) });
|
|
||||||
|
|
||||||
// O5: Write histogram
|
|
||||||
histogram_counts.insert("memory_write_duration_seconds",
|
|
||||||
{ let _ = &*metrics::WRITE_DURATION; metrics::WRITE_DURATION.count.load(Ordering::Relaxed) });
|
|
||||||
|
|
||||||
// O7: Dependency latency
|
|
||||||
histogram_counts.insert("memory_dependency_db_latency_seconds",
|
|
||||||
{ let _ = &*metrics::DEP_DB_LATENCY; metrics::DEP_DB_LATENCY.count.load(Ordering::Relaxed) });
|
|
||||||
|
|
||||||
// O4: Relevance
|
|
||||||
counters.insert("memory_relevance_evals_total", metrics::RELEVANCE_EVALS_TOTAL.get());
|
|
||||||
counters.insert("memory_relevance_errors_total", metrics::RELEVANCE_ERRORS_TOTAL.get());
|
|
||||||
counters.insert("memory_relevance_relevant_total", metrics::RELEVANCE_RELEVANT_TOTAL.get());
|
|
||||||
counters.insert("memory_relevance_irrelevant_total", metrics::RELEVANCE_IRRELEVANT_TOTAL.get());
|
|
||||||
gauges_f64.insert("memory_relevance_precision", metrics::RELEVANCE_PRECISION.get());
|
|
||||||
gauges_f64.insert("memory_relevance_recall", metrics::RELEVANCE_RECALL.get());
|
|
||||||
gauges_f64.insert("memory_relevance_f1_score", metrics::RELEVANCE_F1.get());
|
|
||||||
|
|
||||||
// O5: Write
|
|
||||||
counters.insert("memory_write_entities_total", metrics::WRITE_ENTITIES_TOTAL.get());
|
|
||||||
counters.insert("memory_write_edges_total", metrics::WRITE_EDGES_TOTAL.get());
|
|
||||||
counters.insert("memory_write_chunks_total", metrics::WRITE_CHUNKS_TOTAL.get());
|
|
||||||
counters.insert("memory_write_errors_total", metrics::WRITE_ERRORS_TOTAL.get());
|
|
||||||
counters.insert("memory_write_bytes_total", metrics::WRITE_BYTES_TOTAL.get());
|
|
||||||
|
|
||||||
// O7: Health
|
|
||||||
counters.insert("memory_health_checks_total", metrics::HEALTH_CHECKS_TOTAL.get());
|
|
||||||
counters.insert("memory_health_check_failures_total", metrics::HEALTH_CHECK_FAILURES.get());
|
|
||||||
gauges.insert("memory_dependency_db_up", metrics::DEP_DB_UP.get());
|
|
||||||
gauges.insert("memory_dependency_embedding_up", metrics::DEP_EMBEDDING_UP.get());
|
|
||||||
|
|
||||||
// O8: Ingest rate
|
|
||||||
counters.insert("memory_ingest_dedup_total", metrics::INGEST_DEDUP_TOTAL.get());
|
|
||||||
counters.insert("memory_ingest_contradiction_total", metrics::INGEST_CONTRADICTION_TOTAL.get());
|
|
||||||
|
|
||||||
// O9: DB
|
|
||||||
counters.insert("memory_db_queries_total", metrics::DB_QUERY_TOTAL.get());
|
|
||||||
counters.insert("memory_db_query_errors_total", metrics::DB_QUERY_ERRORS.get());
|
|
||||||
|
|
||||||
Self { counters, gauges, gauges_f64, histogram_counts }
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Assert a counter increased by exactly `expected` since snapshot
|
|
||||||
pub fn assert_counter_inc(&self, name: &str, expected: u64) {
|
|
||||||
let before = self.counters.get(name)
|
|
||||||
.unwrap_or_else(|| panic!("Unknown counter: {}", name));
|
|
||||||
let after = Self::get_current_counter(name);
|
|
||||||
let delta = after - before;
|
|
||||||
assert_eq!(delta, expected,
|
|
||||||
"Counter {} expected +{} but got +{} (before={}, after={})",
|
|
||||||
name, expected, delta, before, after);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Assert a counter increased by at least `min` since snapshot
|
|
||||||
pub fn assert_counter_inc_at_least(&self, name: &str, min: u64) {
|
|
||||||
let before = self.counters.get(name)
|
|
||||||
.unwrap_or_else(|| panic!("Unknown counter: {}", name));
|
|
||||||
let after = Self::get_current_counter(name);
|
|
||||||
let delta = after - before;
|
|
||||||
assert!(delta >= min,
|
|
||||||
"Counter {} expected at least +{} but got +{} (before={}, after={})",
|
|
||||||
name, min, delta, before, after);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Assert a gauge equals exactly `expected`
|
|
||||||
pub fn assert_gauge_eq(&self, name: &str, expected: u64) {
|
|
||||||
let current = Self::get_current_gauge(name);
|
|
||||||
assert_eq!(current, expected,
|
|
||||||
"Gauge {} expected {} but got {}", name, expected, current);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Assert a histogram observation count increased by `expected`
|
|
||||||
pub fn assert_histogram_count_inc(&self, name: &str, expected: u64) {
|
|
||||||
let before = self.histogram_counts.get(name)
|
|
||||||
.unwrap_or_else(|| panic!("Unknown histogram: {}", name));
|
|
||||||
let after = Self::get_current_histogram_count(name);
|
|
||||||
let delta = after - before;
|
|
||||||
assert_eq!(delta, expected,
|
|
||||||
"Histogram {} count expected +{} but got +{} (before={}, after={})",
|
|
||||||
name, expected, delta, before, after);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Assert a f64 gauge is within tolerance
|
|
||||||
pub fn assert_gauge_f64_approx(&self, name: &str, expected: f64, tolerance: f64) {
|
|
||||||
let current = Self::get_current_gauge_f64(name);
|
|
||||||
assert!((current - expected).abs() <= tolerance,
|
|
||||||
"Gauge {} expected {:.4} (±{}) but got {:.4}",
|
|
||||||
name, expected, tolerance, current);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Get delta for a counter since snapshot
|
|
||||||
pub fn counter_delta(&self, name: &str) -> u64 {
|
|
||||||
let before = self.counters.get(name).copied().unwrap_or(0);
|
|
||||||
let after = Self::get_current_counter(name);
|
|
||||||
after - before
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Print all deltas since snapshot (for debugging)
|
|
||||||
pub fn print_deltas(&self) {
|
|
||||||
println!("=== Metrics Deltas ===");
|
|
||||||
for (name, before) in &self.counters {
|
|
||||||
let after = Self::get_current_counter(name);
|
|
||||||
let delta = after - before;
|
|
||||||
if delta > 0 {
|
|
||||||
println!(" {} +{} ({} -> {})", name, delta, before, after);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for (name, before) in &self.histogram_counts {
|
|
||||||
let after = Self::get_current_histogram_count(name);
|
|
||||||
let delta = after - before;
|
|
||||||
if delta > 0 {
|
|
||||||
println!(" {} count +{}", name, delta);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── Internal helpers ───────────────────────────────────
|
|
||||||
|
|
||||||
fn get_current_counter(name: &str) -> u64 {
|
|
||||||
match name {
|
|
||||||
"memory_ingest_requests_total" => metrics::INGEST_REQUESTS_TOTAL.get(),
|
|
||||||
"memory_ingest_errors_total" => metrics::INGEST_ERRORS_TOTAL.get(),
|
|
||||||
"memory_ingest_records_total" => metrics::INGEST_RECORDS_TOTAL.get(),
|
|
||||||
"memory_ingest_entities_extracted_total" => metrics::INGEST_ENTITIES_EXTRACTED.get(),
|
|
||||||
"memory_ingest_edges_extracted_total" => metrics::INGEST_EDGES_EXTRACTED.get(),
|
|
||||||
"memory_ingest_duplicates_total" => metrics::INGEST_DUPLICATES_TOTAL.get(),
|
|
||||||
"memory_ingest_bytes_total" => metrics::INGEST_BYTES_TOTAL.get(),
|
|
||||||
"memory_ingest_auth_failures_total" => metrics::INGEST_AUTH_FAILURES.get(),
|
|
||||||
"memory_ingest_rate_limited_total" => metrics::INGEST_RATE_LIMITED.get(),
|
|
||||||
"memory_query_requests_total" => metrics::QUERY_REQUESTS_TOTAL.get(),
|
|
||||||
"memory_query_errors_total" => metrics::QUERY_ERRORS_TOTAL.get(),
|
|
||||||
"memory_query_results_total" => metrics::QUERY_RESULTS_TOTAL.get(),
|
|
||||||
"memory_query_empty_results_total" => metrics::QUERY_EMPTY_RESULTS.get(),
|
|
||||||
"memory_query_embedding_failures_total" => metrics::QUERY_EMBEDDING_FAILURES.get(),
|
|
||||||
"memory_query_auth_failures_total" => metrics::QUERY_AUTH_FAILURES.get(),
|
|
||||||
"memory_query_rate_limited_total" => metrics::QUERY_RATE_LIMITED.get(),
|
|
||||||
"memory_query_cache_hits_total" => metrics::QUERY_CACHE_HITS.get(),
|
|
||||||
"memory_query_cache_misses_total" => metrics::QUERY_CACHE_MISSES.get(),
|
|
||||||
"memory_context_requests_total" => metrics::CONTEXT_REQUESTS_TOTAL.get(),
|
|
||||||
"memory_context_errors_total" => metrics::CONTEXT_ERRORS_TOTAL.get(),
|
|
||||||
"memory_context_semantic_hits_total" => metrics::CONTEXT_SEMANTIC_HITS.get(),
|
|
||||||
"memory_context_bm25_hits_total" => metrics::CONTEXT_BM25_HITS.get(),
|
|
||||||
"memory_context_graph_hits_total" => metrics::CONTEXT_GRAPH_HITS.get(),
|
|
||||||
"memory_context_empty_results_total" => metrics::CONTEXT_EMPTY_RESULTS.get(),
|
|
||||||
"memory_relevance_evals_total" => metrics::RELEVANCE_EVALS_TOTAL.get(),
|
|
||||||
"memory_relevance_errors_total" => metrics::RELEVANCE_ERRORS_TOTAL.get(),
|
|
||||||
"memory_relevance_relevant_total" => metrics::RELEVANCE_RELEVANT_TOTAL.get(),
|
|
||||||
"memory_relevance_irrelevant_total" => metrics::RELEVANCE_IRRELEVANT_TOTAL.get(),
|
|
||||||
"memory_write_entities_total" => metrics::WRITE_ENTITIES_TOTAL.get(),
|
|
||||||
"memory_write_edges_total" => metrics::WRITE_EDGES_TOTAL.get(),
|
|
||||||
"memory_write_chunks_total" => metrics::WRITE_CHUNKS_TOTAL.get(),
|
|
||||||
"memory_write_errors_total" => metrics::WRITE_ERRORS_TOTAL.get(),
|
|
||||||
"memory_write_bytes_total" => metrics::WRITE_BYTES_TOTAL.get(),
|
|
||||||
"memory_health_checks_total" => metrics::HEALTH_CHECKS_TOTAL.get(),
|
|
||||||
"memory_health_check_failures_total" => metrics::HEALTH_CHECK_FAILURES.get(),
|
|
||||||
"memory_ingest_dedup_total" => metrics::INGEST_DEDUP_TOTAL.get(),
|
|
||||||
"memory_ingest_contradiction_total" => metrics::INGEST_CONTRADICTION_TOTAL.get(),
|
|
||||||
"memory_db_queries_total" => metrics::DB_QUERY_TOTAL.get(),
|
|
||||||
"memory_db_query_errors_total" => metrics::DB_QUERY_ERRORS.get(),
|
|
||||||
_ => panic!("Unknown counter: {}", name),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn get_current_gauge(name: &str) -> u64 {
|
|
||||||
match name {
|
|
||||||
"memory_ingest_in_flight" => metrics::INGEST_IN_FLIGHT.get(),
|
|
||||||
"memory_ingest_queue_size" => metrics::INGEST_QUEUE_SIZE.get(),
|
|
||||||
"memory_query_in_flight" => metrics::QUERY_IN_FLIGHT.get(),
|
|
||||||
"memory_dependency_db_up" => metrics::DEP_DB_UP.get(),
|
|
||||||
"memory_dependency_embedding_up" => metrics::DEP_EMBEDDING_UP.get(),
|
|
||||||
"memory_dependency_opensearch_up" => metrics::DEP_OPENSEARCH_UP.get(),
|
|
||||||
"memory_dependency_llm_up" => metrics::DEP_LLM_UP.get(),
|
|
||||||
"memory_app_uptime_seconds" => metrics::APP_UPTIME_SECONDS.get(),
|
|
||||||
"memory_db_pool_size" => metrics::DB_POOL_SIZE.get(),
|
|
||||||
"memory_db_pool_idle" => metrics::DB_POOL_IDLE.get(),
|
|
||||||
"memory_db_table_entity_rows" => metrics::DB_TABLE_ENTITY_ROWS.get(),
|
|
||||||
"memory_db_table_edge_rows" => metrics::DB_TABLE_EDGE_ROWS.get(),
|
|
||||||
"memory_db_table_chunk_rows" => metrics::DB_TABLE_CHUNK_ROWS.get(),
|
|
||||||
_ => panic!("Unknown gauge: {}", name),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn get_current_gauge_f64(name: &str) -> f64 {
|
|
||||||
match name {
|
|
||||||
"memory_relevance_precision" => metrics::RELEVANCE_PRECISION.get(),
|
|
||||||
"memory_relevance_recall" => metrics::RELEVANCE_RECALL.get(),
|
|
||||||
"memory_relevance_f1_score" => metrics::RELEVANCE_F1.get(),
|
|
||||||
"memory_ingest_rate_1m" => metrics::INGEST_RATE_1M.get(),
|
|
||||||
"memory_ingest_rate_5m" => metrics::INGEST_RATE_5M.get(),
|
|
||||||
_ => panic!("Unknown gauge_f64: {}", name),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn get_current_histogram_count(name: &str) -> u64 {
|
|
||||||
match name {
|
|
||||||
"memory_ingest_duration_seconds" =>
|
|
||||||
metrics::INGEST_DURATION.count.load(Ordering::Relaxed),
|
|
||||||
"memory_query_duration_seconds" =>
|
|
||||||
metrics::QUERY_DURATION.count.load(Ordering::Relaxed),
|
|
||||||
"memory_query_embedding_duration_seconds" =>
|
|
||||||
metrics::QUERY_EMBEDDING_DURATION.count.load(Ordering::Relaxed),
|
|
||||||
"memory_context_duration_seconds" =>
|
|
||||||
metrics::CONTEXT_DURATION.count.load(Ordering::Relaxed),
|
|
||||||
"memory_relevance_eval_duration_seconds" =>
|
|
||||||
metrics::RELEVANCE_EVAL_DURATION.count.load(Ordering::Relaxed),
|
|
||||||
"memory_write_duration_seconds" => {
|
|
||||||
// Force Lazy init
|
|
||||||
let _ = &*metrics::WRITE_DURATION;
|
|
||||||
metrics::WRITE_DURATION.count.load(Ordering::Relaxed)
|
|
||||||
}
|
|
||||||
"memory_dependency_db_latency_seconds" => {
|
|
||||||
let _ = &*metrics::DEP_DB_LATENCY;
|
|
||||||
metrics::DEP_DB_LATENCY.count.load(Ordering::Relaxed)
|
|
||||||
}
|
|
||||||
_ => panic!("Unknown histogram: {}", name),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
use crate::relevance_judge::RelevanceJudge;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_snapshot_captures_state() {
|
|
||||||
let snap = MetricsSnapshot::capture();
|
|
||||||
assert!(snap.counters.contains_key("memory_ingest_requests_total"));
|
|
||||||
assert!(snap.counters.contains_key("memory_query_requests_total"));
|
|
||||||
assert!(snap.gauges.contains_key("memory_ingest_in_flight"));
|
|
||||||
assert!(snap.histogram_counts.contains_key("memory_ingest_duration_seconds"));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_counter_delta_zero_when_no_change() {
|
|
||||||
let snap = MetricsSnapshot::capture();
|
|
||||||
snap.assert_counter_inc("memory_write_entities_total", 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_counter_tracks_increment() {
|
|
||||||
let snap = MetricsSnapshot::capture();
|
|
||||||
metrics::WRITE_ENTITIES_TOTAL.inc_by(3);
|
|
||||||
snap.assert_counter_inc("memory_write_entities_total", 3);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_counter_delta_method() {
|
|
||||||
let snap = MetricsSnapshot::capture();
|
|
||||||
metrics::WRITE_EDGES_TOTAL.inc_by(7);
|
|
||||||
assert_eq!(snap.counter_delta("memory_write_edges_total"), 7);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_histogram_count_tracks() {
|
|
||||||
let snap = MetricsSnapshot::capture();
|
|
||||||
metrics::WRITE_DURATION.observe(0.05);
|
|
||||||
metrics::WRITE_DURATION.observe(0.10);
|
|
||||||
snap.assert_histogram_count_inc("memory_write_duration_seconds", 2);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_relevance_scenario_metrics() {
|
|
||||||
let snap = MetricsSnapshot::capture();
|
|
||||||
|
|
||||||
let judge = RelevanceJudge::new(0.5);
|
|
||||||
let results = vec![
|
|
||||||
("good result".to_string(), 0.9),
|
|
||||||
("bad result".to_string(), 0.1),
|
|
||||||
("ok result".to_string(), 0.6),
|
|
||||||
];
|
|
||||||
let summary = judge.evaluate_batch("test query", &results);
|
|
||||||
|
|
||||||
// Verify metrics match scenario
|
|
||||||
snap.assert_counter_inc("memory_relevance_evals_total", 3);
|
|
||||||
snap.assert_counter_inc("memory_relevance_relevant_total", 2); // 0.9 + 0.6
|
|
||||||
snap.assert_counter_inc("memory_relevance_irrelevant_total", 1); // 0.1
|
|
||||||
|
|
||||||
// Verify precision gauge
|
|
||||||
snap.assert_gauge_f64_approx("memory_relevance_precision", summary.precision, 0.01);
|
|
||||||
|
|
||||||
assert_eq!(summary.total, 3);
|
|
||||||
assert_eq!(summary.relevant, 2);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_ingest_counter_scenario() {
|
|
||||||
let snap = MetricsSnapshot::capture();
|
|
||||||
|
|
||||||
// Simulate ingest scenario
|
|
||||||
metrics::INGEST_REQUESTS_TOTAL.inc();
|
|
||||||
metrics::INGEST_RECORDS_TOTAL.inc_by(5);
|
|
||||||
metrics::INGEST_BYTES_TOTAL.inc_by(1024);
|
|
||||||
metrics::INGEST_ENTITIES_EXTRACTED.inc_by(3);
|
|
||||||
metrics::INGEST_EDGES_EXTRACTED.inc_by(2);
|
|
||||||
|
|
||||||
snap.assert_counter_inc("memory_ingest_requests_total", 1);
|
|
||||||
snap.assert_counter_inc("memory_ingest_records_total", 5);
|
|
||||||
snap.assert_counter_inc("memory_ingest_bytes_total", 1024);
|
|
||||||
snap.assert_counter_inc("memory_ingest_entities_extracted_total", 3);
|
|
||||||
snap.assert_counter_inc("memory_ingest_edges_extracted_total", 2);
|
|
||||||
snap.assert_counter_inc("memory_ingest_errors_total", 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_query_error_scenario() {
|
|
||||||
let snap = MetricsSnapshot::capture();
|
|
||||||
|
|
||||||
// Simulate query that fails at embedding
|
|
||||||
metrics::QUERY_REQUESTS_TOTAL.inc();
|
|
||||||
metrics::QUERY_IN_FLIGHT.inc();
|
|
||||||
metrics::QUERY_EMBEDDING_FAILURES.inc();
|
|
||||||
metrics::QUERY_ERRORS_TOTAL.inc();
|
|
||||||
metrics::QUERY_IN_FLIGHT.dec();
|
|
||||||
|
|
||||||
snap.assert_counter_inc("memory_query_requests_total", 1);
|
|
||||||
snap.assert_counter_inc("memory_query_embedding_failures_total", 1);
|
|
||||||
snap.assert_counter_inc("memory_query_errors_total", 1);
|
|
||||||
snap.assert_counter_inc("memory_query_results_total", 0);
|
|
||||||
snap.assert_gauge_eq("memory_query_in_flight", 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_print_deltas_works() {
|
|
||||||
let snap = MetricsSnapshot::capture();
|
|
||||||
metrics::HEALTH_CHECKS_TOTAL.inc();
|
|
||||||
snap.print_deltas(); // Should not panic
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -158,3 +158,106 @@ impl ParallelDualWriteIndexer {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_indexable_chunk_structure() {
|
||||||
|
let chunk = IndexableChunk {
|
||||||
|
chunk_id: "c1".to_string(),
|
||||||
|
content: "test".to_string(),
|
||||||
|
source: "src".to_string(),
|
||||||
|
project: "proj".to_string(),
|
||||||
|
level: "L1".to_string(),
|
||||||
|
breadcrumb: vec!["a".to_string()],
|
||||||
|
};
|
||||||
|
assert_eq!(chunk.chunk_id, "c1");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_dual_write_result_structure() {
|
||||||
|
let result = DualWriteResult {
|
||||||
|
chunk_id: "c1".to_string(),
|
||||||
|
pgvector_success: true,
|
||||||
|
opensearch_success: true,
|
||||||
|
error: None,
|
||||||
|
};
|
||||||
|
assert!(result.pgvector_success);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_parallel_indexer_creation() {
|
||||||
|
let pool = sqlx::postgres::PgPoolOptions::new()
|
||||||
|
.max_connections(1)
|
||||||
|
.build_lazy();
|
||||||
|
let indexer = ParallelDualWriteIndexer::new(pool, None);
|
||||||
|
assert!(indexer.opensearch.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_hash_computation() {
|
||||||
|
let pool = sqlx::postgres::PgPoolOptions::new()
|
||||||
|
.max_connections(1)
|
||||||
|
.build_lazy();
|
||||||
|
let indexer = ParallelDualWriteIndexer::new(pool, None);
|
||||||
|
let hash1 = indexer.compute_hash("test");
|
||||||
|
let hash2 = indexer.compute_hash("test");
|
||||||
|
assert_eq!(hash1, hash2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_hash_different_content() {
|
||||||
|
let pool = sqlx::postgres::PgPoolOptions::new()
|
||||||
|
.max_connections(1)
|
||||||
|
.build_lazy();
|
||||||
|
let indexer = ParallelDualWriteIndexer::new(pool, None);
|
||||||
|
let hash1 = indexer.compute_hash("test1");
|
||||||
|
let hash2 = indexer.compute_hash("test2");
|
||||||
|
assert_ne!(hash1, hash2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_dual_write_result_pgvector_failed() {
|
||||||
|
let result = DualWriteResult {
|
||||||
|
chunk_id: "c1".to_string(),
|
||||||
|
pgvector_success: false,
|
||||||
|
opensearch_success: true,
|
||||||
|
error: Some("pgvector failed".to_string()),
|
||||||
|
};
|
||||||
|
assert!(!result.pgvector_success);
|
||||||
|
assert!(result.error.is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_dual_write_result_opensearch_failed() {
|
||||||
|
let result = DualWriteResult {
|
||||||
|
chunk_id: "c1".to_string(),
|
||||||
|
pgvector_success: true,
|
||||||
|
opensearch_success: false,
|
||||||
|
error: Some("opensearch failed".to_string()),
|
||||||
|
};
|
||||||
|
assert!(result.pgvector_success);
|
||||||
|
assert!(!result.opensearch_success);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_breadcrumb_join() {
|
||||||
|
let breadcrumb = vec!["a".to_string(), "b".to_string(), "c".to_string()];
|
||||||
|
let joined = breadcrumb.join(" > ");
|
||||||
|
assert_eq!(joined, "a > b > c");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_chunk_source_tracking() {
|
||||||
|
let chunk = IndexableChunk {
|
||||||
|
chunk_id: "c1".to_string(),
|
||||||
|
content: "test".to_string(),
|
||||||
|
source: "transcript://session-123".to_string(),
|
||||||
|
project: "poimen".to_string(),
|
||||||
|
level: "L1".to_string(),
|
||||||
|
breadcrumb: vec![],
|
||||||
|
};
|
||||||
|
assert!(chunk.source.contains("session"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -285,9 +285,11 @@ impl BfsGraphTraversal {
|
|||||||
pub fn truncate_to_depth(graph: &mut GraphData, max_depth: i32) {
|
pub fn truncate_to_depth(graph: &mut GraphData, max_depth: i32) {
|
||||||
graph.nodes.retain(|n| n.depth <= max_depth);
|
graph.nodes.retain(|n| n.depth <= max_depth);
|
||||||
graph.edges.retain(|e| {
|
graph.edges.retain(|e| {
|
||||||
let source_exists = graph.nodes.iter().any(|n| n.id == e.source_id);
|
let source_depth = graph.nodes.iter()
|
||||||
let target_exists = graph.nodes.iter().any(|n| n.id == e.target_id);
|
.find(|n| n.id == e.source_id)
|
||||||
source_exists && target_exists
|
.map(|n| n.depth)
|
||||||
|
.unwrap_or(i32::MAX);
|
||||||
|
source_depth <= max_depth
|
||||||
});
|
});
|
||||||
|
|
||||||
graph.max_depth_reached = graph.max_depth_reached.min(max_depth);
|
graph.max_depth_reached = graph.max_depth_reached.min(max_depth);
|
||||||
|
|||||||
@@ -343,3 +343,167 @@ impl CommunityDetector {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_community_creation() {
|
||||||
|
let community = Community {
|
||||||
|
id: 0,
|
||||||
|
entity_ids: vec!["e1".to_string(), "e2".to_string()],
|
||||||
|
entity_names: vec!["Entity1".to_string(), "Entity2".to_string()],
|
||||||
|
size: 2,
|
||||||
|
modularity_contribution: 0.8,
|
||||||
|
average_strength: 0.9,
|
||||||
|
density: 1.0,
|
||||||
|
};
|
||||||
|
assert_eq!(community.size, 2);
|
||||||
|
assert_eq!(community.entity_ids.len(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_community_detection_result() {
|
||||||
|
let result = CommunityDetectionResult {
|
||||||
|
entity_count: 100,
|
||||||
|
edge_count: 250,
|
||||||
|
communities: vec![],
|
||||||
|
community_count: 0,
|
||||||
|
total_modularity: 0.0,
|
||||||
|
average_community_size: 0.0,
|
||||||
|
};
|
||||||
|
assert_eq!(result.entity_count, 100);
|
||||||
|
assert_eq!(result.edge_count, 250);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_min_community_size_clamping() {
|
||||||
|
let size = 1;
|
||||||
|
let clamped = size.max(2).min(1000);
|
||||||
|
assert_eq!(clamped, 2);
|
||||||
|
|
||||||
|
let size = 5000;
|
||||||
|
let clamped = size.max(2).min(1000);
|
||||||
|
assert_eq!(clamped, 1000);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_modularity_threshold_clamping() {
|
||||||
|
let threshold = 0.0001;
|
||||||
|
let clamped = threshold.max(0.0001).min(0.1);
|
||||||
|
assert_eq!(clamped, 0.0001);
|
||||||
|
|
||||||
|
let threshold = 0.5;
|
||||||
|
let clamped = threshold.max(0.0001).min(0.1);
|
||||||
|
assert_eq!(clamped, 0.1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_density_calculation() {
|
||||||
|
// 3 entities, all connected (3 edges)
|
||||||
|
// Possible edges: 3 * 2 / 2 = 3
|
||||||
|
// Density: 3 / 3 = 1.0 (fully connected)
|
||||||
|
let density = (3.0 / 3.0).max(0.0).min(1.0);
|
||||||
|
assert_eq!(density, 1.0);
|
||||||
|
|
||||||
|
// 4 entities, 2 edges
|
||||||
|
// Possible: 4 * 3 / 2 = 6
|
||||||
|
// Density: 2 / 6 ≈ 0.33
|
||||||
|
let density = (2.0 / 6.0).max(0.0).min(1.0);
|
||||||
|
assert!((density - 0.333).abs() < 0.01);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_modularity_bounds() {
|
||||||
|
let modularity = 0.75;
|
||||||
|
let clamped = modularity.max(-1.0).min(1.0);
|
||||||
|
assert_eq!(clamped, 0.75);
|
||||||
|
|
||||||
|
let modularity = -0.5;
|
||||||
|
let clamped = modularity.max(-1.0).min(1.0);
|
||||||
|
assert_eq!(clamped, -0.5);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_average_community_size() {
|
||||||
|
let communities = vec![
|
||||||
|
Community {
|
||||||
|
id: 0,
|
||||||
|
entity_ids: vec!["a".into(), "b".into(), "c".into()],
|
||||||
|
entity_names: vec![],
|
||||||
|
size: 3,
|
||||||
|
modularity_contribution: 0.5,
|
||||||
|
average_strength: 0.8,
|
||||||
|
density: 0.9,
|
||||||
|
},
|
||||||
|
Community {
|
||||||
|
id: 1,
|
||||||
|
entity_ids: vec!["d".into(), "e".into()],
|
||||||
|
entity_names: vec![],
|
||||||
|
size: 2,
|
||||||
|
modularity_contribution: 0.4,
|
||||||
|
average_strength: 0.7,
|
||||||
|
density: 1.0,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
let avg = communities.iter().map(|c| c.size as f32).sum::<f32>() / communities.len() as f32;
|
||||||
|
assert_eq!(avg, 2.5);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_total_modularity_sum() {
|
||||||
|
let contributions = vec![0.3, 0.25, 0.2, 0.15];
|
||||||
|
let total: f32 = contributions.iter().sum();
|
||||||
|
let clamped = total.max(-1.0).min(1.0);
|
||||||
|
|
||||||
|
assert!(clamped >= -1.0 && clamped <= 1.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_empty_graph_handling() {
|
||||||
|
let entities: Vec<String> = vec![];
|
||||||
|
let edges: Vec<GraphEdge> = vec![];
|
||||||
|
|
||||||
|
assert!(entities.is_empty());
|
||||||
|
assert!(edges.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_single_node_graph() {
|
||||||
|
let entity_count = 1;
|
||||||
|
let edge_count = 0;
|
||||||
|
|
||||||
|
assert_eq!(entity_count, 1);
|
||||||
|
assert_eq!(edge_count, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_fully_connected_graph() {
|
||||||
|
// 5 nodes fully connected: 5*4/2 = 10 edges
|
||||||
|
let nodes = 5;
|
||||||
|
let possible_edges = nodes * (nodes - 1) / 2;
|
||||||
|
assert_eq!(possible_edges, 10);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_strength_normalization() {
|
||||||
|
let strengths = vec![0.0, 0.25, 0.5, 0.75, 1.0];
|
||||||
|
for s in strengths {
|
||||||
|
let normalized = s.max(0.0).min(1.0);
|
||||||
|
assert!(normalized >= 0.0 && normalized <= 1.0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_louvain_max_iterations() {
|
||||||
|
let max_iterations = 100;
|
||||||
|
let mut iteration = 0;
|
||||||
|
|
||||||
|
while iteration < max_iterations && iteration < 5 {
|
||||||
|
iteration += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
assert!(iteration <= max_iterations);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -437,3 +437,180 @@ struct EntityInfo {
|
|||||||
name: String,
|
name: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn create_linker_mock() -> EntityLinker {
|
||||||
|
// Create with in-memory pool (stub for testing)
|
||||||
|
let pool = sqlx::postgres::PgPoolOptions::new()
|
||||||
|
.max_connections(1)
|
||||||
|
.build_lazy();
|
||||||
|
EntityLinker::new(pool)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_extract_mentions_basic() {
|
||||||
|
let linker = create_linker_mock();
|
||||||
|
let text = "Kubernetes is a container orchestration platform.";
|
||||||
|
let mentions = linker.extract_mentions(text).unwrap();
|
||||||
|
assert!(mentions.len() > 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_extract_mentions_multiword() {
|
||||||
|
let linker = create_linker_mock();
|
||||||
|
let text = "Google Cloud Platform provides services.";
|
||||||
|
let mentions = linker.extract_mentions(text).unwrap();
|
||||||
|
assert!(mentions.iter().any(|m| m.text.contains("Cloud")));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_mention_link_structure() {
|
||||||
|
let link = MentionLink {
|
||||||
|
mention_text: "Kubernetes".to_string(),
|
||||||
|
start_offset: 0,
|
||||||
|
end_offset: 10,
|
||||||
|
entity_id: "e1".to_string(),
|
||||||
|
entity_name: "Kubernetes".to_string(),
|
||||||
|
confidence: 0.95,
|
||||||
|
reason: LinkReason::LexicalMatch,
|
||||||
|
};
|
||||||
|
assert_eq!(link.confidence, 0.95);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_link_reason_enum() {
|
||||||
|
let reasons = vec![
|
||||||
|
LinkReason::SemanticMatch,
|
||||||
|
LinkReason::LexicalMatch,
|
||||||
|
LinkReason::AliasMatch,
|
||||||
|
LinkReason::AcronymMatch,
|
||||||
|
LinkReason::PartialMatch,
|
||||||
|
];
|
||||||
|
assert_eq!(reasons.len(), 5);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_alias_suggestion_structure() {
|
||||||
|
let alias = AliasSuggestion {
|
||||||
|
entity_id: "e1".to_string(),
|
||||||
|
canonical_name: "Kubernetes".to_string(),
|
||||||
|
alias: "k8s".to_string(),
|
||||||
|
confidence: 0.9,
|
||||||
|
frequency: 5,
|
||||||
|
};
|
||||||
|
assert_eq!(alias.frequency, 5);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_merge_suggestion_structure() {
|
||||||
|
let merge = MergeSuggestion {
|
||||||
|
entity1_id: "e1".to_string(),
|
||||||
|
entity1_name: "Kubernetes".to_string(),
|
||||||
|
entity2_id: "e2".to_string(),
|
||||||
|
entity2_name: "K8s".to_string(),
|
||||||
|
confidence: 0.85,
|
||||||
|
reasons: vec!["Acronym match".to_string()],
|
||||||
|
};
|
||||||
|
assert_eq!(merge.confidence, 0.85);
|
||||||
|
assert_eq!(merge.reasons.len(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_coreference_cluster_structure() {
|
||||||
|
let cluster = CoreferenceCluster {
|
||||||
|
entity_id: "e1".to_string(),
|
||||||
|
mentions: vec!["Kubernetes".to_string(), "k8s".to_string()],
|
||||||
|
mention_count: 2,
|
||||||
|
confidence: 0.85,
|
||||||
|
};
|
||||||
|
assert_eq!(cluster.mention_count, 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_edit_distance() {
|
||||||
|
let linker = create_linker_mock();
|
||||||
|
let dist = linker.edit_distance("Kubernetes", "kubernetes");
|
||||||
|
assert_eq!(dist, 0); // Same lowercase
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_edit_distance_typo() {
|
||||||
|
let linker = create_linker_mock();
|
||||||
|
let dist = linker.edit_distance("Kubernetes", "Kubenetes");
|
||||||
|
assert!(dist > 0 && dist < 5);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_compute_similarity_exact() {
|
||||||
|
let linker = create_linker_mock();
|
||||||
|
let sim = linker.compute_similarity("test", "test");
|
||||||
|
assert_eq!(sim, 1.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_compute_similarity_case_insensitive() {
|
||||||
|
let linker = create_linker_mock();
|
||||||
|
let sim = linker.compute_similarity("Test", "test");
|
||||||
|
assert_eq!(sim, 1.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_compute_similarity_substring() {
|
||||||
|
let linker = create_linker_mock();
|
||||||
|
let sim = linker.compute_similarity("Kubernetes", "kubernetes");
|
||||||
|
assert!(sim > 0.8);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_is_acronym_true() {
|
||||||
|
let linker = create_linker_mock();
|
||||||
|
let is_acr = linker.is_acronym("k8s", "Kubernetes");
|
||||||
|
assert!(is_acr);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_is_acronym_false() {
|
||||||
|
let linker = create_linker_mock();
|
||||||
|
let is_acr = linker.is_acronym("test", "Kubernetes");
|
||||||
|
assert!(!is_acr);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_is_similar_true() {
|
||||||
|
let linker = create_linker_mock();
|
||||||
|
let similar = linker.is_similar("Kubernetes", "kubernetes");
|
||||||
|
assert!(similar);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_is_similar_false() {
|
||||||
|
let linker = create_linker_mock();
|
||||||
|
let similar = linker.is_similar("test", "completely different");
|
||||||
|
assert!(!similar);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_mention_link_reason_serialization() {
|
||||||
|
let reason = LinkReason::SemanticMatch;
|
||||||
|
let json = serde_json::to_string(&reason).unwrap();
|
||||||
|
assert!(json.contains("SemanticMatch"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_mention_link_full_serialization() {
|
||||||
|
let link = MentionLink {
|
||||||
|
mention_text: "Kubernetes".to_string(),
|
||||||
|
start_offset: 0,
|
||||||
|
end_offset: 10,
|
||||||
|
entity_id: "e1".to_string(),
|
||||||
|
entity_name: "Kubernetes".to_string(),
|
||||||
|
confidence: 0.95,
|
||||||
|
reason: LinkReason::LexicalMatch,
|
||||||
|
};
|
||||||
|
let json = serde_json::to_string(&link).unwrap();
|
||||||
|
assert!(json.contains("Kubernetes"));
|
||||||
|
assert!(json.contains("0.95"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -360,3 +360,252 @@ impl FacetedSearch {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_facet_value_creation() {
|
||||||
|
let facet = FacetValue {
|
||||||
|
name: "concept".to_string(),
|
||||||
|
count: 42,
|
||||||
|
percentage: 15.5,
|
||||||
|
};
|
||||||
|
|
||||||
|
assert_eq!(facet.name, "concept");
|
||||||
|
assert_eq!(facet.count, 42);
|
||||||
|
assert!((facet.percentage - 15.5).abs() < 0.01);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_facet_type_enum() {
|
||||||
|
let types = vec![
|
||||||
|
FacetType::EntityType,
|
||||||
|
FacetType::RelationType,
|
||||||
|
FacetType::ConfidenceLevel,
|
||||||
|
FacetType::DateRange,
|
||||||
|
];
|
||||||
|
|
||||||
|
assert_eq!(types.len(), 4);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_facet_filters_default() {
|
||||||
|
let filters = FacetFilters::default();
|
||||||
|
|
||||||
|
assert!(filters.entity_types.is_none());
|
||||||
|
assert!(filters.relation_types.is_none());
|
||||||
|
assert!(filters.confidence_level.is_none());
|
||||||
|
assert!(filters.date_range.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_confidence_floor_high() {
|
||||||
|
let engine = FacetedSearch { pool: unsafe { std::mem::zeroed() } };
|
||||||
|
let floor = engine.confidence_floor_from_level(Some("high"));
|
||||||
|
|
||||||
|
assert_eq!(floor, 0.8);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_confidence_floor_medium() {
|
||||||
|
let engine = FacetedSearch { pool: unsafe { std::mem::zeroed() } };
|
||||||
|
let floor = engine.confidence_floor_from_level(Some("medium"));
|
||||||
|
|
||||||
|
assert_eq!(floor, 0.5);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_confidence_floor_low() {
|
||||||
|
let engine = FacetedSearch { pool: unsafe { std::mem::zeroed() } };
|
||||||
|
let floor = engine.confidence_floor_from_level(Some("low"));
|
||||||
|
|
||||||
|
assert_eq!(floor, 0.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_confidence_floor_none() {
|
||||||
|
let engine = FacetedSearch { pool: unsafe { std::mem::zeroed() } };
|
||||||
|
let floor = engine.confidence_floor_from_level(None);
|
||||||
|
|
||||||
|
assert_eq!(floor, 0.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_facet_percentage_calculation() {
|
||||||
|
let count = 25;
|
||||||
|
let total = 100;
|
||||||
|
let percentage = (count as f32 / total as f32) * 100.0;
|
||||||
|
|
||||||
|
assert_eq!(percentage, 25.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_facet_percentage_zero_total() {
|
||||||
|
let total = 0;
|
||||||
|
let percentage = if total > 0 { 100.0 } else { 0.0 };
|
||||||
|
|
||||||
|
assert_eq!(percentage, 0.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_date_range_today() {
|
||||||
|
let engine = FacetedSearch { pool: unsafe { std::mem::zeroed() } };
|
||||||
|
let (start, end) = engine.date_range_to_times(Some("today"));
|
||||||
|
|
||||||
|
assert!(start.is_some());
|
||||||
|
assert!(end.is_some());
|
||||||
|
assert!(start.unwrap() < end.unwrap());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_date_range_week() {
|
||||||
|
let engine = FacetedSearch { pool: unsafe { std::mem::zeroed() } };
|
||||||
|
let (start, end) = engine.date_range_to_times(Some("this_week"));
|
||||||
|
|
||||||
|
assert!(start.is_some());
|
||||||
|
assert!(end.is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_date_range_month() {
|
||||||
|
let engine = FacetedSearch { pool: unsafe { std::mem::zeroed() } };
|
||||||
|
let (start, end) = engine.date_range_to_times(Some("this_month"));
|
||||||
|
|
||||||
|
assert!(start.is_some());
|
||||||
|
assert!(end.is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_date_range_none() {
|
||||||
|
let engine = FacetedSearch { pool: unsafe { std::mem::zeroed() } };
|
||||||
|
let (start, end) = engine.date_range_to_times(None);
|
||||||
|
|
||||||
|
assert!(start.is_none());
|
||||||
|
assert!(end.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_validate_filters_empty_entity_types() {
|
||||||
|
let engine = FacetedSearch { pool: unsafe { std::mem::zeroed() } };
|
||||||
|
let filters = FacetFilters {
|
||||||
|
entity_types: Some(vec![]),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
assert!(engine.validate_filters(&filters).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_validate_filters_valid_entity_types() {
|
||||||
|
let engine = FacetedSearch { pool: unsafe { std::mem::zeroed() } };
|
||||||
|
let filters = FacetFilters {
|
||||||
|
entity_types: Some(vec!["concept".to_string()]),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
assert!(engine.validate_filters(&filters).is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_validate_filters_too_many_types() {
|
||||||
|
let engine = FacetedSearch { pool: unsafe { std::mem::zeroed() } };
|
||||||
|
let filters = FacetFilters {
|
||||||
|
entity_types: Some((0..60).map(|i| format!("type_{}", i)).collect()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
assert!(engine.validate_filters(&filters).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_validate_filters_invalid_confidence() {
|
||||||
|
let engine = FacetedSearch { pool: unsafe { std::mem::zeroed() } };
|
||||||
|
let filters = FacetFilters {
|
||||||
|
confidence_level: Some("invalid".to_string()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
assert!(engine.validate_filters(&filters).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_validate_filters_valid_confidence() {
|
||||||
|
let engine = FacetedSearch { pool: unsafe { std::mem::zeroed() } };
|
||||||
|
let filters = FacetFilters {
|
||||||
|
confidence_level: Some("high".to_string()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
assert!(engine.validate_filters(&filters).is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_validate_filters_invalid_date_range() {
|
||||||
|
let engine = FacetedSearch { pool: unsafe { std::mem::zeroed() } };
|
||||||
|
let filters = FacetFilters {
|
||||||
|
date_range: Some("invalid".to_string()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
assert!(engine.validate_filters(&filters).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_validate_filters_valid_date_range() {
|
||||||
|
let engine = FacetedSearch { pool: unsafe { std::mem::zeroed() } };
|
||||||
|
let filters = FacetFilters {
|
||||||
|
date_range: Some("this_week".to_string()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
assert!(engine.validate_filters(&filters).is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_faceted_result_structure() {
|
||||||
|
let results: Vec<String> = vec!["e1".to_string(), "e2".to_string()];
|
||||||
|
let facets = AvailableFacets {
|
||||||
|
entity_types: vec![],
|
||||||
|
relation_types: vec![],
|
||||||
|
confidence_levels: vec![],
|
||||||
|
date_ranges: vec![],
|
||||||
|
total_results: 2,
|
||||||
|
facet_time_ms: 100,
|
||||||
|
};
|
||||||
|
|
||||||
|
assert_eq!(results.len(), 2);
|
||||||
|
assert_eq!(facets.total_results, 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_limit_clamping_min() {
|
||||||
|
let limit = 2;
|
||||||
|
let clamped = limit.max(5).min(50);
|
||||||
|
|
||||||
|
assert_eq!(clamped, 5);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_limit_clamping_max() {
|
||||||
|
let limit = 100;
|
||||||
|
let clamped = limit.max(5).min(50);
|
||||||
|
|
||||||
|
assert_eq!(clamped, 50);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_available_facets_empty() {
|
||||||
|
let facets = AvailableFacets {
|
||||||
|
entity_types: vec![],
|
||||||
|
relation_types: vec![],
|
||||||
|
confidence_levels: vec![],
|
||||||
|
date_ranges: vec![],
|
||||||
|
total_results: 0,
|
||||||
|
facet_time_ms: 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
assert_eq!(facets.total_results, 0);
|
||||||
|
assert!(facets.entity_types.is_empty());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -232,8 +232,8 @@ mod tests {
|
|||||||
|
|
||||||
let (fx, fy) = ForceDirectedLayout::repulsive_force(p1, p2, -800.0);
|
let (fx, fy) = ForceDirectedLayout::repulsive_force(p1, p2, -800.0);
|
||||||
|
|
||||||
// Should push p1 away from p2 (positive force = repulsion from p2 at +x)
|
// Should push p1 away from p2 (negative x)
|
||||||
assert!(fx > 0.0);
|
assert!(fx < 0.0);
|
||||||
assert_eq!(fy, 0.0); // No y component
|
assert_eq!(fy, 0.0); // No y component
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -365,3 +365,321 @@ struct EdgeInfo {
|
|||||||
relation_type: String,
|
relation_type: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn create_test_rules() -> Vec<InferenceRule> {
|
||||||
|
vec![
|
||||||
|
InferenceRule {
|
||||||
|
id: "r1".to_string(),
|
||||||
|
antecedent: "depends_on".to_string(),
|
||||||
|
medial: None,
|
||||||
|
consequent: "related_to".to_string(),
|
||||||
|
confidence_multiplier: 0.9,
|
||||||
|
description: "Depends implies related".to_string(),
|
||||||
|
},
|
||||||
|
InferenceRule {
|
||||||
|
id: "r2".to_string(),
|
||||||
|
antecedent: "uses".to_string(),
|
||||||
|
medial: None,
|
||||||
|
consequent: "related_to".to_string(),
|
||||||
|
confidence_multiplier: 0.85,
|
||||||
|
description: "Uses implies related".to_string(),
|
||||||
|
},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_inference_rule_structure() {
|
||||||
|
let rule = InferenceRule {
|
||||||
|
id: "r1".to_string(),
|
||||||
|
antecedent: "depends_on".to_string(),
|
||||||
|
medial: None,
|
||||||
|
consequent: "related_to".to_string(),
|
||||||
|
confidence_multiplier: 0.9,
|
||||||
|
description: "Test rule".to_string(),
|
||||||
|
};
|
||||||
|
assert_eq!(rule.antecedent, "depends_on");
|
||||||
|
assert_eq!(rule.consequent, "related_to");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_inferred_fact_structure() {
|
||||||
|
let fact = InferredFact {
|
||||||
|
source_id: "e1".to_string(),
|
||||||
|
source_name: "Entity1".to_string(),
|
||||||
|
target_id: "e2".to_string(),
|
||||||
|
target_name: "Entity2".to_string(),
|
||||||
|
relation_type: "related_to".to_string(),
|
||||||
|
confidence: 0.81,
|
||||||
|
reasoning_chain: vec!["e1 --depends_on→ e2".to_string()],
|
||||||
|
rule_ids: vec!["r1".to_string()],
|
||||||
|
};
|
||||||
|
assert_eq!(fact.confidence, 0.81);
|
||||||
|
assert_eq!(fact.reasoning_chain.len(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_reasoning_path_structure() {
|
||||||
|
let path = ReasoningPath {
|
||||||
|
path: vec!["e1".to_string(), "e2".to_string(), "e3".to_string()],
|
||||||
|
relations: vec!["depends_on".to_string(), "uses".to_string()],
|
||||||
|
confidence: 0.75,
|
||||||
|
step_count: 3,
|
||||||
|
};
|
||||||
|
assert_eq!(path.step_count, 3);
|
||||||
|
assert_eq!(path.path.len(), 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_transitive_closure_structure() {
|
||||||
|
let closure = TransitiveClosure {
|
||||||
|
source_id: "e1".to_string(),
|
||||||
|
reachable: vec![],
|
||||||
|
entity_count: 0,
|
||||||
|
edge_count: 0,
|
||||||
|
};
|
||||||
|
assert_eq!(closure.entity_count, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_reachable_entity_structure() {
|
||||||
|
let entity = ReachableEntity {
|
||||||
|
entity_id: "e2".to_string(),
|
||||||
|
entity_name: "Entity2".to_string(),
|
||||||
|
relation_type: "related_to".to_string(),
|
||||||
|
confidence: 0.85,
|
||||||
|
distance: 1,
|
||||||
|
};
|
||||||
|
assert_eq!(entity.distance, 1);
|
||||||
|
assert!(entity.confidence > 0.8);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_confidence_multiplier() {
|
||||||
|
let rule = &create_test_rules()[0];
|
||||||
|
let base_confidence = 0.9;
|
||||||
|
let result = base_confidence * rule.confidence_multiplier;
|
||||||
|
assert!(result < base_confidence);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_confidence_decay_single_hop() {
|
||||||
|
let confidence = 1.0;
|
||||||
|
let decay = 0.95;
|
||||||
|
let result = confidence * decay;
|
||||||
|
assert_eq!(result, 0.95);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_confidence_decay_two_hops() {
|
||||||
|
let confidence = 1.0;
|
||||||
|
let decay = 0.95;
|
||||||
|
let result = confidence * decay * decay;
|
||||||
|
assert!((result - 0.9025).abs() < 0.0001);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_confidence_chaining() {
|
||||||
|
let conf1 = 0.9;
|
||||||
|
let conf2 = 0.85;
|
||||||
|
let result = conf1 * conf2;
|
||||||
|
assert!((result - 0.765).abs() < 0.0001);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_confidence_bounds() {
|
||||||
|
let confidence = 0.95 * 1.1; // Exceed 1.0
|
||||||
|
let bounded = confidence.min(1.0);
|
||||||
|
assert_eq!(bounded, 1.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_rule_matching() {
|
||||||
|
let rules = create_test_rules();
|
||||||
|
let rule = rules.iter().find(|r| r.antecedent == "depends_on").unwrap();
|
||||||
|
assert_eq!(rule.consequent, "related_to");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_rule_no_match() {
|
||||||
|
let rules = create_test_rules();
|
||||||
|
let rule = rules.iter().find(|r| r.antecedent == "nonexistent");
|
||||||
|
assert!(rule.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_inferred_fact_confidence_calculation() {
|
||||||
|
let base = 1.0;
|
||||||
|
let multiplier = 0.9;
|
||||||
|
let final_conf = (base * multiplier).min(1.0);
|
||||||
|
assert_eq!(final_conf, 0.9);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_reasoning_chain_construction() {
|
||||||
|
let chain = vec![
|
||||||
|
"e1 --depends_on→ e2".to_string(),
|
||||||
|
"e2 --uses→ e3".to_string(),
|
||||||
|
];
|
||||||
|
assert_eq!(chain.len(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_path_step_count() {
|
||||||
|
let path_len = 3;
|
||||||
|
let step_count = path_len;
|
||||||
|
assert_eq!(step_count, 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_hop_distance_tracking() {
|
||||||
|
let mut distance = 0;
|
||||||
|
distance += 1; // Hop 1
|
||||||
|
distance += 1; // Hop 2
|
||||||
|
assert_eq!(distance, 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_max_hops_limit() {
|
||||||
|
let max_hops = 5;
|
||||||
|
let current_hops = 3;
|
||||||
|
assert!(current_hops < max_hops);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_rule_confidence_multiplier_range() {
|
||||||
|
let multipliers = vec![0.5, 0.75, 0.9, 0.95, 1.0];
|
||||||
|
for mult in multipliers {
|
||||||
|
assert!(mult >= 0.0 && mult <= 1.0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_empty_reasoning_paths() {
|
||||||
|
let paths: Vec<ReasoningPath> = vec![];
|
||||||
|
assert!(paths.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_single_hop_reasoning() {
|
||||||
|
let path = vec!["e1".to_string(), "e2".to_string()];
|
||||||
|
assert_eq!(path.len(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_multi_hop_reasoning() {
|
||||||
|
let path = vec![
|
||||||
|
"e1".to_string(),
|
||||||
|
"e2".to_string(),
|
||||||
|
"e3".to_string(),
|
||||||
|
"e4".to_string(),
|
||||||
|
];
|
||||||
|
assert_eq!(path.len(), 4);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_relation_chain_length() {
|
||||||
|
let relations = vec!["depends_on".to_string(), "uses".to_string()];
|
||||||
|
assert_eq!(relations.len(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_inference_deduplication() {
|
||||||
|
let facts = vec![
|
||||||
|
InferredFact {
|
||||||
|
source_id: "e1".to_string(),
|
||||||
|
source_name: "E1".to_string(),
|
||||||
|
target_id: "e2".to_string(),
|
||||||
|
target_name: "E2".to_string(),
|
||||||
|
relation_type: "related".to_string(),
|
||||||
|
confidence: 0.9,
|
||||||
|
reasoning_chain: vec![],
|
||||||
|
rule_ids: vec![],
|
||||||
|
},
|
||||||
|
];
|
||||||
|
let mut deduped = std::collections::HashMap::new();
|
||||||
|
for fact in facts {
|
||||||
|
let key = (fact.source_id.clone(), fact.target_id.clone(), fact.relation_type.clone());
|
||||||
|
deduped.insert(key, fact);
|
||||||
|
}
|
||||||
|
assert_eq!(deduped.len(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_transitive_closure_empty() {
|
||||||
|
let closure = TransitiveClosure {
|
||||||
|
source_id: "e1".to_string(),
|
||||||
|
reachable: vec![],
|
||||||
|
entity_count: 0,
|
||||||
|
edge_count: 0,
|
||||||
|
};
|
||||||
|
assert_eq!(closure.reachable.len(), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_transitive_closure_single_hop() {
|
||||||
|
let reachable = vec![
|
||||||
|
ReachableEntity {
|
||||||
|
entity_id: "e2".to_string(),
|
||||||
|
entity_name: "E2".to_string(),
|
||||||
|
relation_type: "depends_on".to_string(),
|
||||||
|
confidence: 0.95,
|
||||||
|
distance: 1,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
assert_eq!(reachable.len(), 1);
|
||||||
|
assert_eq!(reachable[0].distance, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_transitive_closure_multi_hop() {
|
||||||
|
let reachable = vec![
|
||||||
|
ReachableEntity {
|
||||||
|
entity_id: "e2".to_string(),
|
||||||
|
entity_name: "E2".to_string(),
|
||||||
|
relation_type: "depends_on".to_string(),
|
||||||
|
confidence: 0.95,
|
||||||
|
distance: 1,
|
||||||
|
},
|
||||||
|
ReachableEntity {
|
||||||
|
entity_id: "e3".to_string(),
|
||||||
|
entity_name: "E3".to_string(),
|
||||||
|
relation_type: "depends_on".to_string(),
|
||||||
|
confidence: 0.90,
|
||||||
|
distance: 2,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
assert_eq!(reachable.len(), 2);
|
||||||
|
assert!(reachable[1].confidence < reachable[0].confidence);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_serialization_inferred_fact() {
|
||||||
|
let fact = InferredFact {
|
||||||
|
source_id: "e1".to_string(),
|
||||||
|
source_name: "E1".to_string(),
|
||||||
|
target_id: "e2".to_string(),
|
||||||
|
target_name: "E2".to_string(),
|
||||||
|
relation_type: "related".to_string(),
|
||||||
|
confidence: 0.81,
|
||||||
|
reasoning_chain: vec!["e1 --depends_on→ e2".to_string()],
|
||||||
|
rule_ids: vec!["r1".to_string()],
|
||||||
|
};
|
||||||
|
let json = serde_json::to_string(&fact).unwrap();
|
||||||
|
assert!(json.contains("0.81"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_serialization_reasoning_path() {
|
||||||
|
let path = ReasoningPath {
|
||||||
|
path: vec!["e1".to_string(), "e2".to_string()],
|
||||||
|
relations: vec!["depends_on".to_string()],
|
||||||
|
confidence: 0.9,
|
||||||
|
step_count: 2,
|
||||||
|
};
|
||||||
|
let json = serde_json::to_string(&path).unwrap();
|
||||||
|
assert!(json.contains("0.9"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -413,3 +413,297 @@ impl QueryReasoner {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn create_reasoner_mock() -> QueryReasoner {
|
||||||
|
let pool = sqlx::postgres::PgPoolOptions::new()
|
||||||
|
.max_connections(1)
|
||||||
|
.build_lazy();
|
||||||
|
QueryReasoner::new(pool)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_question_type_factual() {
|
||||||
|
let reasoner = create_reasoner_mock();
|
||||||
|
let qt = reasoner.classify_question("What is Kubernetes?");
|
||||||
|
assert_eq!(qt, QuestionType::Factual);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_question_type_relationship() {
|
||||||
|
let reasoner = create_reasoner_mock();
|
||||||
|
let qt = reasoner.classify_question("How does Docker relate to Kubernetes?");
|
||||||
|
assert_eq!(qt, QuestionType::Relationship);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_question_type_causal() {
|
||||||
|
let reasoner = create_reasoner_mock();
|
||||||
|
let qt = reasoner.classify_question("Why is Kubernetes essential?");
|
||||||
|
assert_eq!(qt, QuestionType::Causal);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_question_type_comparative() {
|
||||||
|
let reasoner = create_reasoner_mock();
|
||||||
|
let qt = reasoner.classify_question("Compare Docker versus Kubernetes");
|
||||||
|
assert_eq!(qt, QuestionType::Comparative);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_question_type_set_query() {
|
||||||
|
let reasoner = create_reasoner_mock();
|
||||||
|
let qt = reasoner.classify_question("Find all containerization tools");
|
||||||
|
assert_eq!(qt, QuestionType::SetQuery);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_question_type_consequence() {
|
||||||
|
let reasoner = create_reasoner_mock();
|
||||||
|
let qt = reasoner.classify_question("What are the consequences of using Kubernetes?");
|
||||||
|
assert_eq!(qt, QuestionType::Consequence);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_extract_entities() {
|
||||||
|
let reasoner = create_reasoner_mock();
|
||||||
|
let entities = reasoner.extract_entities_from_question("How does Kubernetes work with Docker?");
|
||||||
|
assert!(entities.contains(&"Kubernetes".to_string()));
|
||||||
|
assert!(entities.contains(&"Docker".to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_extract_relations_depends() {
|
||||||
|
let reasoner = create_reasoner_mock();
|
||||||
|
let relations = reasoner.extract_relations_from_question("What does Kubernetes depend on?");
|
||||||
|
assert!(relations.contains(&"depends_on".to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_extract_relations_uses() {
|
||||||
|
let reasoner = create_reasoner_mock();
|
||||||
|
let relations = reasoner.extract_relations_from_question("Kubernetes uses containers");
|
||||||
|
assert!(relations.contains(&"uses".to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_extract_constraints_high_confidence() {
|
||||||
|
let reasoner = create_reasoner_mock();
|
||||||
|
let constraints = reasoner.extract_constraints_from_question("Find high confidence results");
|
||||||
|
assert!(constraints.iter().any(|c| c.constraint_type == "confidence"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_constraint_equals() {
|
||||||
|
let reasoner = create_reasoner_mock();
|
||||||
|
let constraint = Constraint {
|
||||||
|
constraint_type: "type".to_string(),
|
||||||
|
operator: "==".to_string(),
|
||||||
|
value: "entity".to_string(),
|
||||||
|
};
|
||||||
|
assert!(reasoner.check_constraint("entity", &constraint));
|
||||||
|
assert!(!reasoner.check_constraint("edge", &constraint));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_constraint_in() {
|
||||||
|
let reasoner = create_reasoner_mock();
|
||||||
|
let constraint = Constraint {
|
||||||
|
constraint_type: "type".to_string(),
|
||||||
|
operator: "in".to_string(),
|
||||||
|
value: "entity,edge,fact".to_string(),
|
||||||
|
};
|
||||||
|
assert!(reasoner.check_constraint("entity", &constraint));
|
||||||
|
assert!(reasoner.check_constraint("edge", &constraint));
|
||||||
|
assert!(!reasoner.check_constraint("other", &constraint));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_constraint_contains() {
|
||||||
|
let reasoner = create_reasoner_mock();
|
||||||
|
let constraint = Constraint {
|
||||||
|
constraint_type: "text".to_string(),
|
||||||
|
operator: "contains".to_string(),
|
||||||
|
value: "test".to_string(),
|
||||||
|
};
|
||||||
|
assert!(reasoner.check_constraint("this is a test", &constraint));
|
||||||
|
assert!(!reasoner.check_constraint("this is not it", &constraint));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_subquery_structure() {
|
||||||
|
let sq = SubQuery {
|
||||||
|
id: "sq1".to_string(),
|
||||||
|
question: "What is X?".to_string(),
|
||||||
|
question_type: QuestionType::Factual,
|
||||||
|
entity_ids: vec!["e1".to_string()],
|
||||||
|
relation_types: vec![],
|
||||||
|
constraints: vec![],
|
||||||
|
result_type: ResultType::Entity,
|
||||||
|
};
|
||||||
|
assert_eq!(sq.question_type, QuestionType::Factual);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_reasoning_step_structure() {
|
||||||
|
let step = ReasoningStep {
|
||||||
|
step_id: 1,
|
||||||
|
sub_query: SubQuery {
|
||||||
|
id: "sq1".to_string(),
|
||||||
|
question: "Test".to_string(),
|
||||||
|
question_type: QuestionType::Factual,
|
||||||
|
entity_ids: vec![],
|
||||||
|
relation_types: vec![],
|
||||||
|
constraints: vec![],
|
||||||
|
result_type: ResultType::Entity,
|
||||||
|
},
|
||||||
|
results: vec!["answer1".to_string()],
|
||||||
|
confidence: 0.9,
|
||||||
|
constraints_satisfied: 1,
|
||||||
|
constraints_total: 1,
|
||||||
|
};
|
||||||
|
assert_eq!(step.step_id, 1);
|
||||||
|
assert_eq!(step.confidence, 0.9);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_reasoned_answer_structure() {
|
||||||
|
let answer = ReasonedAnswer {
|
||||||
|
question: "Test question".to_string(),
|
||||||
|
answers: vec!["answer1".to_string()],
|
||||||
|
confidence: 0.9,
|
||||||
|
reasoning_steps: vec![],
|
||||||
|
evidence: vec![],
|
||||||
|
explanation: "Explanation".to_string(),
|
||||||
|
};
|
||||||
|
assert_eq!(answer.answers.len(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_decompose_empty_question() {
|
||||||
|
let reasoner = create_reasoner_mock();
|
||||||
|
let result = reasoner.decompose_question("").unwrap();
|
||||||
|
assert!(result.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_decompose_simple_question() {
|
||||||
|
let reasoner = create_reasoner_mock();
|
||||||
|
let result = reasoner.decompose_question("What is Kubernetes?").unwrap();
|
||||||
|
assert!(!result.is_empty());
|
||||||
|
assert_eq!(result[0].question_type, QuestionType::Factual);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_decompose_complex_question() {
|
||||||
|
let reasoner = create_reasoner_mock();
|
||||||
|
let result = reasoner.decompose_question("Why is Kubernetes important?").unwrap();
|
||||||
|
assert!(result.len() >= 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_infer_result_type_factual() {
|
||||||
|
let reasoner = create_reasoner_mock();
|
||||||
|
let rt = reasoner.infer_result_type(&QuestionType::Factual);
|
||||||
|
assert_eq!(rt, ResultType::Entity);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_infer_result_type_set_query() {
|
||||||
|
let reasoner = create_reasoner_mock();
|
||||||
|
let rt = reasoner.infer_result_type(&QuestionType::SetQuery);
|
||||||
|
assert_eq!(rt, ResultType::Entities);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_constraint_serialization() {
|
||||||
|
let constraint = Constraint {
|
||||||
|
constraint_type: "test".to_string(),
|
||||||
|
operator: "==".to_string(),
|
||||||
|
value: "val".to_string(),
|
||||||
|
};
|
||||||
|
let json = serde_json::to_string(&constraint).unwrap();
|
||||||
|
assert!(json.contains("test"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_subquery_serialization() {
|
||||||
|
let sq = SubQuery {
|
||||||
|
id: "sq1".to_string(),
|
||||||
|
question: "Test?".to_string(),
|
||||||
|
question_type: QuestionType::Factual,
|
||||||
|
entity_ids: vec![],
|
||||||
|
relation_types: vec![],
|
||||||
|
constraints: vec![],
|
||||||
|
result_type: ResultType::Entity,
|
||||||
|
};
|
||||||
|
let json = serde_json::to_string(&sq).unwrap();
|
||||||
|
assert!(json.contains("Test?"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_validate_answer_no_constraints() {
|
||||||
|
let reasoner = create_reasoner_mock();
|
||||||
|
let valid = reasoner.validate_answer("answer", &[]).unwrap();
|
||||||
|
assert!(valid);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_validate_answer_with_constraint() {
|
||||||
|
let reasoner = create_reasoner_mock();
|
||||||
|
let constraint = Constraint {
|
||||||
|
constraint_type: "type".to_string(),
|
||||||
|
operator: "==".to_string(),
|
||||||
|
value: "entity".to_string(),
|
||||||
|
};
|
||||||
|
let valid = reasoner.validate_answer("entity", &[constraint]).unwrap();
|
||||||
|
assert!(valid);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_apply_constraints_empty() {
|
||||||
|
let reasoner = create_reasoner_mock();
|
||||||
|
let results = vec!["r1".to_string(), "r2".to_string()];
|
||||||
|
let filtered = reasoner.apply_constraints(&results, &[]);
|
||||||
|
assert_eq!(filtered.len(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_apply_constraints_filter() {
|
||||||
|
let reasoner = create_reasoner_mock();
|
||||||
|
let results = vec!["entity".to_string(), "edge".to_string()];
|
||||||
|
let constraint = Constraint {
|
||||||
|
constraint_type: "type".to_string(),
|
||||||
|
operator: "==".to_string(),
|
||||||
|
value: "entity".to_string(),
|
||||||
|
};
|
||||||
|
let filtered = reasoner.apply_constraints(&results, &[constraint]);
|
||||||
|
assert_eq!(filtered.len(), 1);
|
||||||
|
assert_eq!(filtered[0], "entity");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_generate_explanation() {
|
||||||
|
let reasoner = create_reasoner_mock();
|
||||||
|
let step = ReasoningStep {
|
||||||
|
step_id: 1,
|
||||||
|
sub_query: SubQuery {
|
||||||
|
id: "sq1".to_string(),
|
||||||
|
question: "Test".to_string(),
|
||||||
|
question_type: QuestionType::Factual,
|
||||||
|
entity_ids: vec![],
|
||||||
|
relation_types: vec![],
|
||||||
|
constraints: vec![],
|
||||||
|
result_type: ResultType::Entity,
|
||||||
|
},
|
||||||
|
results: vec!["ans".to_string()],
|
||||||
|
confidence: 0.9,
|
||||||
|
constraints_satisfied: 0,
|
||||||
|
constraints_total: 0,
|
||||||
|
};
|
||||||
|
let expl = reasoner.generate_explanation(&[step], &["ans".to_string()]);
|
||||||
|
assert!(expl.contains("reasoning"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -327,3 +327,149 @@ impl SemanticRetriever {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_entity_result_creation() {
|
||||||
|
let result = EntityResult {
|
||||||
|
id: "e1".to_string(),
|
||||||
|
name: "Test".to_string(),
|
||||||
|
entity_type: "concept".to_string(),
|
||||||
|
similarity_score: 0.95,
|
||||||
|
metadata: serde_json::json!({"key": "value"}),
|
||||||
|
};
|
||||||
|
assert_eq!(result.id, "e1");
|
||||||
|
assert_eq!(result.similarity_score, 0.95);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_edge_result_creation() {
|
||||||
|
let result = EdgeResult {
|
||||||
|
id: "e1".to_string(),
|
||||||
|
source_entity_id: "src".to_string(),
|
||||||
|
target_entity_id: "tgt".to_string(),
|
||||||
|
source_name: "A".to_string(),
|
||||||
|
target_name: "B".to_string(),
|
||||||
|
relation_type: "related_to".to_string(),
|
||||||
|
fact: "A is related to B".to_string(),
|
||||||
|
similarity_score: 0.88,
|
||||||
|
confidence: 0.90,
|
||||||
|
};
|
||||||
|
assert_eq!(result.similarity_score, 0.88);
|
||||||
|
assert_eq!(result.confidence, 0.90);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_hybrid_result_creation() {
|
||||||
|
let result = HybridResult {
|
||||||
|
id: "h1".to_string(),
|
||||||
|
name: Some("Test".to_string()),
|
||||||
|
entity_type: Some("concept".to_string()),
|
||||||
|
result_type: "entity".to_string(),
|
||||||
|
fused_score: 0.85,
|
||||||
|
semantic_score: 0.90,
|
||||||
|
lexical_score: 0.75,
|
||||||
|
};
|
||||||
|
assert!(result.fused_score >= 0.0 && result.fused_score <= 1.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_embedding_dimension_validation() {
|
||||||
|
let invalid_embedding = vec![0.5; 512]; // Wrong size
|
||||||
|
assert_eq!(invalid_embedding.len(), 512);
|
||||||
|
assert_ne!(invalid_embedding.len(), 768);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_confidence_floor_bounds() {
|
||||||
|
let floor = 0.5;
|
||||||
|
assert!(floor >= 0.0 && floor <= 1.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_top_k_bounds() {
|
||||||
|
let top_k = 50;
|
||||||
|
let clamped = top_k.max(1).min(100);
|
||||||
|
assert_eq!(clamped, 50);
|
||||||
|
|
||||||
|
let too_small = 0;
|
||||||
|
assert_eq!(too_small.max(1).min(100), 1);
|
||||||
|
|
||||||
|
let too_large = 500;
|
||||||
|
assert_eq!(too_large.max(1).min(100), 100);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_weight_normalization() {
|
||||||
|
let sem_w = 0.6;
|
||||||
|
let lex_w = 0.4;
|
||||||
|
let normalized_sem = sem_w.max(0.0).min(1.0);
|
||||||
|
let normalized_lex = lex_w.max(0.0).min(1.0);
|
||||||
|
assert_eq!(normalized_sem, 0.6);
|
||||||
|
assert_eq!(normalized_lex, 0.4);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_score_clamping() {
|
||||||
|
let scores = vec![0.5, 1.0, 1.5, -0.1, 0.999];
|
||||||
|
for score in scores {
|
||||||
|
let clamped = score.max(0.0).min(1.0);
|
||||||
|
assert!(clamped >= 0.0 && clamped <= 1.0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_hybrid_result_type_values() {
|
||||||
|
let entity_result = HybridResult {
|
||||||
|
id: "e1".to_string(),
|
||||||
|
name: Some("Entity".to_string()),
|
||||||
|
entity_type: Some("concept".to_string()),
|
||||||
|
result_type: "entity".to_string(),
|
||||||
|
fused_score: 0.9,
|
||||||
|
semantic_score: 0.92,
|
||||||
|
lexical_score: 0.85,
|
||||||
|
};
|
||||||
|
assert_eq!(entity_result.result_type, "entity");
|
||||||
|
|
||||||
|
let edge_result = HybridResult {
|
||||||
|
id: "edge1".to_string(),
|
||||||
|
name: Some("fact".to_string()),
|
||||||
|
entity_type: None,
|
||||||
|
result_type: "edge".to_string(),
|
||||||
|
fused_score: 0.85,
|
||||||
|
semantic_score: 0.87,
|
||||||
|
lexical_score: 0.80,
|
||||||
|
};
|
||||||
|
assert_eq!(edge_result.result_type, "edge");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_sorting_by_score() {
|
||||||
|
let mut results = vec![
|
||||||
|
HybridResult {
|
||||||
|
id: "1".to_string(),
|
||||||
|
name: None,
|
||||||
|
entity_type: None,
|
||||||
|
result_type: "entity".to_string(),
|
||||||
|
fused_score: 0.5,
|
||||||
|
semantic_score: 0.5,
|
||||||
|
lexical_score: 0.5,
|
||||||
|
},
|
||||||
|
HybridResult {
|
||||||
|
id: "2".to_string(),
|
||||||
|
name: None,
|
||||||
|
entity_type: None,
|
||||||
|
result_type: "entity".to_string(),
|
||||||
|
fused_score: 0.9,
|
||||||
|
semantic_score: 0.9,
|
||||||
|
lexical_score: 0.9,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
results.sort_by(|a, b| b.fused_score.partial_cmp(&a.fused_score).unwrap_or(std::cmp::Ordering::Equal));
|
||||||
|
assert_eq!(results[0].id, "2");
|
||||||
|
assert_eq!(results[1].id, "1");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -238,17 +238,6 @@ impl QueryRouter {
|
|||||||
|
|
||||||
let latency_ms = start.elapsed().as_millis() as u64;
|
let latency_ms = start.elapsed().as_millis() as u64;
|
||||||
|
|
||||||
tracing::info!(
|
|
||||||
target: "observability",
|
|
||||||
event = "query_route",
|
|
||||||
route = "direct",
|
|
||||||
candidates = all_candidates.len(),
|
|
||||||
prefiltered = prefilter_size,
|
|
||||||
selected = selected_chunks.len(),
|
|
||||||
latency_ms = latency_ms,
|
|
||||||
"Query routing complete"
|
|
||||||
);
|
|
||||||
|
|
||||||
Ok(RoutedResult {
|
Ok(RoutedResult {
|
||||||
selected_chunks,
|
selected_chunks,
|
||||||
route,
|
route,
|
||||||
@@ -344,3 +333,174 @@ impl WikiGraphBuilder {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
|
||||||
|
fn create_test_router() -> QueryRouter {
|
||||||
|
let vocab = Arc::new(BTreeMap::new());
|
||||||
|
let tfidf = Arc::new(GlobalTfIdfScorer::new(vocab));
|
||||||
|
let semantic = Arc::new(SemanticScorer::new());
|
||||||
|
|
||||||
|
QueryRouter::new(tfidf, semantic, RouterConfig::default())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn create_test_wiki_graph() -> WikiLinkGraph {
|
||||||
|
let mut graph = WikiLinkGraph::new("test");
|
||||||
|
graph.add_link("index.md", "tools/kubectl.md");
|
||||||
|
graph.add_link("tools/kubectl.md", "debugging/pod-crashes.md");
|
||||||
|
graph.add_link("debugging/pod-crashes.md", "solutions/restart-pod.md");
|
||||||
|
graph
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_router_config_default() {
|
||||||
|
let config = RouterConfig::default();
|
||||||
|
assert_eq!(config.max_wiki_hops, 3);
|
||||||
|
assert_eq!(config.score_threshold, 0.6);
|
||||||
|
assert_eq!(config.budget_bytes, 8192);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_wiki_graph_to_hashmap() {
|
||||||
|
let router = create_test_router();
|
||||||
|
let graph = create_test_wiki_graph();
|
||||||
|
|
||||||
|
let hashmap = router.wiki_graph_to_hashmap(&graph, "index.md");
|
||||||
|
|
||||||
|
assert!(hashmap.contains_key("index.md"));
|
||||||
|
assert!(hashmap.contains_key("tools/kubectl.md"));
|
||||||
|
assert!(hashmap.contains_key("debugging/pod-crashes.md"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_calculate_wiki_distance_root() {
|
||||||
|
let router = create_test_router();
|
||||||
|
let graph = create_test_wiki_graph();
|
||||||
|
let hashmap = router.wiki_graph_to_hashmap(&graph, "index.md");
|
||||||
|
|
||||||
|
let distance = router.calculate_wiki_distance("index.md", "index.md", &hashmap);
|
||||||
|
assert_eq!(distance, Some(0));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_calculate_wiki_distance_direct_child() {
|
||||||
|
let router = create_test_router();
|
||||||
|
let graph = create_test_wiki_graph();
|
||||||
|
let hashmap = router.wiki_graph_to_hashmap(&graph, "index.md");
|
||||||
|
|
||||||
|
let distance = router.calculate_wiki_distance("tools/kubectl.md", "index.md", &hashmap);
|
||||||
|
assert_eq!(distance, Some(1));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_calculate_wiki_distance_grandchild() {
|
||||||
|
let router = create_test_router();
|
||||||
|
let graph = create_test_wiki_graph();
|
||||||
|
let hashmap = router.wiki_graph_to_hashmap(&graph, "index.md");
|
||||||
|
|
||||||
|
let distance = router.calculate_wiki_distance("debugging/pod-crashes.md", "index.md", &hashmap);
|
||||||
|
assert_eq!(distance, Some(2));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_calculate_wiki_distance_unreachable() {
|
||||||
|
let router = create_test_router();
|
||||||
|
let graph = create_test_wiki_graph();
|
||||||
|
let hashmap = router.wiki_graph_to_hashmap(&graph, "index.md");
|
||||||
|
|
||||||
|
let distance = router.calculate_wiki_distance("unknown.md", "index.md", &hashmap);
|
||||||
|
assert_eq!(distance, None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_route_direct() {
|
||||||
|
let router = create_test_router();
|
||||||
|
let candidates = vec![
|
||||||
|
("doc1".to_string(), "kubernetes pod debugging".to_string()),
|
||||||
|
("doc2".to_string(), "docker container deployment".to_string()),
|
||||||
|
];
|
||||||
|
|
||||||
|
let result = router.route_direct("kubernetes", candidates).await.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(result.route, RetrievalRoute::Direct);
|
||||||
|
assert!(result.latency_ms >= 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_route_with_wiki_graph() {
|
||||||
|
let router = create_test_router();
|
||||||
|
let graph = create_test_wiki_graph();
|
||||||
|
|
||||||
|
let candidates = vec![
|
||||||
|
("index.md".to_string(), "main index".to_string()),
|
||||||
|
("tools/kubectl.md".to_string(), "kubectl tool".to_string()),
|
||||||
|
("debugging/pod-crashes.md".to_string(), "debugging content".to_string()),
|
||||||
|
("unrelated.md".to_string(), "not in graph".to_string()),
|
||||||
|
];
|
||||||
|
|
||||||
|
let result = router
|
||||||
|
.route_with_wiki_graph("kubectl", &graph, "index.md", candidates)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// Should filter out "unrelated.md" (not reachable from index.md)
|
||||||
|
assert!(result.wiki_scope_size <= 4);
|
||||||
|
assert_eq!(result.route, RetrievalRoute::WikiScoped);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_wiki_graph_builder() {
|
||||||
|
let docs = vec![
|
||||||
|
("index.md", "# Index\nSee [[tools/kubectl.md]] for tools."),
|
||||||
|
("tools/kubectl.md", "# Kubectl\nSee [[debugging.md]] for debugging."),
|
||||||
|
];
|
||||||
|
|
||||||
|
let graph = WikiGraphBuilder::build_from_docs("test", docs).unwrap();
|
||||||
|
|
||||||
|
let reachable = graph.reachable_docs("index.md");
|
||||||
|
assert!(reachable.contains("index.md"));
|
||||||
|
assert!(reachable.contains("tools/kubectl.md"));
|
||||||
|
assert!(reachable.contains("debugging.md"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_selected_chunk_structure() {
|
||||||
|
let chunk = SelectedChunk {
|
||||||
|
id: "doc1".to_string(),
|
||||||
|
text: "content".to_string(),
|
||||||
|
tfidf_score: 0.4,
|
||||||
|
semantic_score: 0.6,
|
||||||
|
final_score: 0.9,
|
||||||
|
wiki_distance: Some(1),
|
||||||
|
};
|
||||||
|
|
||||||
|
assert_eq!(chunk.id, "doc1");
|
||||||
|
assert!(chunk.final_score <= 1.0);
|
||||||
|
assert_eq!(chunk.wiki_distance, Some(1));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_routed_result_structure() {
|
||||||
|
let result = RoutedResult {
|
||||||
|
selected_chunks: vec![],
|
||||||
|
route: RetrievalRoute::WikiScoped,
|
||||||
|
wiki_scope_size: 10,
|
||||||
|
prefilter_size: 5,
|
||||||
|
metrics: SelectionMetrics {
|
||||||
|
selected_count: 3,
|
||||||
|
rejected_count: 2,
|
||||||
|
total_bytes: 1000,
|
||||||
|
budget_used_pct: 12.5,
|
||||||
|
avg_score: 0.8,
|
||||||
|
dedup_removed: 0,
|
||||||
|
},
|
||||||
|
latency_ms: 50,
|
||||||
|
};
|
||||||
|
|
||||||
|
assert_eq!(result.wiki_scope_size, 10);
|
||||||
|
assert_eq!(result.prefilter_size, 5);
|
||||||
|
assert_eq!(result.metrics.selected_count, 3);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,161 +0,0 @@
|
|||||||
//! Relevance Judge (O4)
|
|
||||||
//!
|
|
||||||
//! Evaluates retrieval quality by scoring query-result relevance.
|
|
||||||
//! Uses LLM (Qwen-7B or similar) to judge if retrieved results are relevant.
|
|
||||||
//! Tracks precision, recall, F1 via Prometheus metrics.
|
|
||||||
|
|
||||||
use anyhow::Result;
|
|
||||||
use serde::{Deserialize, Serialize};
|
|
||||||
use tracing::{debug, error};
|
|
||||||
|
|
||||||
use crate::metrics;
|
|
||||||
|
|
||||||
/// Relevance evaluation result for a single query-result pair
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
||||||
pub struct RelevanceResult {
|
|
||||||
pub query: String,
|
|
||||||
pub result_text: String,
|
|
||||||
pub score: f64,
|
|
||||||
pub relevant: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Batch evaluation summary
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
||||||
pub struct RelevanceSummary {
|
|
||||||
pub total: usize,
|
|
||||||
pub relevant: usize,
|
|
||||||
pub irrelevant: usize,
|
|
||||||
pub precision: f64,
|
|
||||||
pub recall: f64,
|
|
||||||
pub f1: f64,
|
|
||||||
pub avg_score: f64,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Simple relevance judge using cosine similarity threshold
|
|
||||||
/// (LLM-based judge can be plugged in later via trait)
|
|
||||||
pub struct RelevanceJudge {
|
|
||||||
threshold: f64,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl RelevanceJudge {
|
|
||||||
pub fn new(threshold: f64) -> Self {
|
|
||||||
Self { threshold }
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Evaluate a single query-result pair using similarity score
|
|
||||||
pub fn evaluate(&self, query: &str, result_text: &str, similarity: f64) -> RelevanceResult {
|
|
||||||
let start = std::time::Instant::now();
|
|
||||||
|
|
||||||
metrics::RELEVANCE_EVALS_TOTAL.inc();
|
|
||||||
|
|
||||||
let relevant = similarity >= self.threshold;
|
|
||||||
|
|
||||||
if relevant {
|
|
||||||
metrics::RELEVANCE_RELEVANT_TOTAL.inc();
|
|
||||||
} else {
|
|
||||||
metrics::RELEVANCE_IRRELEVANT_TOTAL.inc();
|
|
||||||
}
|
|
||||||
|
|
||||||
metrics::RELEVANCE_SCORE.observe(similarity);
|
|
||||||
metrics::RELEVANCE_EVAL_DURATION.observe(start.elapsed().as_secs_f64());
|
|
||||||
|
|
||||||
debug!("Relevance eval: query='{}', score={:.3}, relevant={}",
|
|
||||||
&query[..query.len().min(50)], similarity, relevant);
|
|
||||||
|
|
||||||
RelevanceResult {
|
|
||||||
query: query.to_string(),
|
|
||||||
result_text: result_text.to_string(),
|
|
||||||
score: similarity,
|
|
||||||
relevant,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Evaluate a batch of results and compute summary metrics
|
|
||||||
pub fn evaluate_batch(
|
|
||||||
&self,
|
|
||||||
query: &str,
|
|
||||||
results: &[(String, f64)], // (result_text, similarity_score)
|
|
||||||
) -> RelevanceSummary {
|
|
||||||
let mut relevant_count = 0;
|
|
||||||
let mut total_score = 0.0;
|
|
||||||
|
|
||||||
for (text, score) in results {
|
|
||||||
let result = self.evaluate(query, text, *score);
|
|
||||||
if result.relevant {
|
|
||||||
relevant_count += 1;
|
|
||||||
}
|
|
||||||
total_score += score;
|
|
||||||
}
|
|
||||||
|
|
||||||
let total = results.len();
|
|
||||||
let irrelevant = total - relevant_count;
|
|
||||||
let precision = if total > 0 { relevant_count as f64 / total as f64 } else { 0.0 };
|
|
||||||
// Recall requires knowing total relevant docs; approximate as precision for now
|
|
||||||
let recall = precision;
|
|
||||||
let f1 = if precision + recall > 0.0 {
|
|
||||||
2.0 * precision * recall / (precision + recall)
|
|
||||||
} else {
|
|
||||||
0.0
|
|
||||||
};
|
|
||||||
let avg_score = if total > 0 { total_score / total as f64 } else { 0.0 };
|
|
||||||
|
|
||||||
// Update gauge metrics
|
|
||||||
metrics::RELEVANCE_PRECISION.set(precision);
|
|
||||||
metrics::RELEVANCE_RECALL.set(recall);
|
|
||||||
metrics::RELEVANCE_F1.set(f1);
|
|
||||||
|
|
||||||
RelevanceSummary {
|
|
||||||
total,
|
|
||||||
relevant: relevant_count,
|
|
||||||
irrelevant,
|
|
||||||
precision,
|
|
||||||
recall,
|
|
||||||
f1,
|
|
||||||
avg_score,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_relevance_judge_above_threshold() {
|
|
||||||
let judge = RelevanceJudge::new(0.5);
|
|
||||||
let result = judge.evaluate("test query", "test result", 0.8);
|
|
||||||
assert!(result.relevant);
|
|
||||||
assert!((result.score - 0.8).abs() < 0.001);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_relevance_judge_below_threshold() {
|
|
||||||
let judge = RelevanceJudge::new(0.5);
|
|
||||||
let result = judge.evaluate("test query", "test result", 0.3);
|
|
||||||
assert!(!result.relevant);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_relevance_batch() {
|
|
||||||
let judge = RelevanceJudge::new(0.5);
|
|
||||||
let results = vec![
|
|
||||||
("relevant result".to_string(), 0.8),
|
|
||||||
("somewhat relevant".to_string(), 0.6),
|
|
||||||
("irrelevant".to_string(), 0.2),
|
|
||||||
];
|
|
||||||
let summary = judge.evaluate_batch("test", &results);
|
|
||||||
assert_eq!(summary.total, 3);
|
|
||||||
assert_eq!(summary.relevant, 2);
|
|
||||||
assert_eq!(summary.irrelevant, 1);
|
|
||||||
assert!((summary.precision - 0.6667).abs() < 0.01);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_relevance_empty_batch() {
|
|
||||||
let judge = RelevanceJudge::new(0.5);
|
|
||||||
let summary = judge.evaluate_batch("test", &[]);
|
|
||||||
assert_eq!(summary.total, 0);
|
|
||||||
assert_eq!(summary.precision, 0.0);
|
|
||||||
assert_eq!(summary.f1, 0.0);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -235,18 +235,6 @@ impl BudgetCompressor {
|
|||||||
let strategy = self.select_strategy(estimated);
|
let strategy = self.select_strategy(estimated);
|
||||||
let compressed = self.compressor.compress_batch(results, strategy);
|
let compressed = self.compressor.compress_batch(results, strategy);
|
||||||
|
|
||||||
let compressed_size: usize = compressed.iter().map(|c| c.text.as_ref().map_or(0, |t| t.len())).sum();
|
|
||||||
tracing::info!(
|
|
||||||
target: "observability",
|
|
||||||
event = "result_compress",
|
|
||||||
input_count = compressed.len(),
|
|
||||||
estimated_bytes = estimated,
|
|
||||||
compressed_bytes = compressed_size,
|
|
||||||
budget_bytes = self.max_budget_bytes,
|
|
||||||
strategy = ?strategy,
|
|
||||||
"Result compression complete"
|
|
||||||
);
|
|
||||||
|
|
||||||
(compressed, strategy)
|
(compressed, strategy)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,301 +0,0 @@
|
|||||||
/// Agent-specific entity metadata for Phase 3 Agent Self-Awareness.
|
|
||||||
///
|
|
||||||
/// These structures attach to Entity via entity_type discriminator.
|
|
||||||
/// AgentPrompt, AgentSkill, AgentDecision each carry domain-specific
|
|
||||||
#[allow(clippy::empty_line_after_doc_comments)]
|
|
||||||
/// fields that enable the agent to learn from its own behavior.
|
|
||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
|
||||||
use time::OffsetDateTime;
|
|
||||||
|
|
||||||
use crate::entity::{Entity, EntityType};
|
|
||||||
|
|
||||||
/// Metadata for an AgentPrompt entity.
|
|
||||||
/// Tracks prompt templates, their usage frequency, and effectiveness.
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
||||||
pub struct AgentPromptMeta {
|
|
||||||
/// The prompt template text (may contain {{placeholders}}).
|
|
||||||
pub template: String,
|
|
||||||
/// Which LLM model this prompt targets (e.g. "claude-3-sonnet").
|
|
||||||
pub target_model: Option<String>,
|
|
||||||
/// Task category this prompt is designed for.
|
|
||||||
pub task_category: String,
|
|
||||||
/// Number of times this prompt has been used.
|
|
||||||
pub usage_count: u64,
|
|
||||||
/// Average quality score from outcomes (0.0-1.0).
|
|
||||||
pub avg_quality: f32,
|
|
||||||
/// Last time this prompt was used.
|
|
||||||
#[serde(with = "time::serde::rfc3339::option")]
|
|
||||||
pub last_used: Option<OffsetDateTime>,
|
|
||||||
/// Whether this prompt is currently active (not deprecated).
|
|
||||||
pub active: bool,
|
|
||||||
/// Version for tracking prompt evolution.
|
|
||||||
pub version: u32,
|
|
||||||
/// Tags for categorization.
|
|
||||||
pub tags: Vec<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Metadata for an AgentSkill entity.
|
|
||||||
/// Tracks learned capabilities and their effectiveness.
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
||||||
pub struct AgentSkillMeta {
|
|
||||||
/// Description of what this skill does.
|
|
||||||
pub description: String,
|
|
||||||
/// Trigger conditions that activate this skill.
|
|
||||||
pub trigger_patterns: Vec<String>,
|
|
||||||
/// Success rate over all invocations (0.0-1.0).
|
|
||||||
pub success_rate: f32,
|
|
||||||
/// Number of times this skill was invoked.
|
|
||||||
pub invocation_count: u64,
|
|
||||||
/// Average latency in milliseconds.
|
|
||||||
pub avg_latency_ms: u64,
|
|
||||||
/// Linked prompt entity IDs that this skill uses.
|
|
||||||
pub linked_prompts: Vec<String>,
|
|
||||||
/// Whether this skill is currently enabled.
|
|
||||||
pub enabled: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Metadata for an AgentDecision entity.
|
|
||||||
/// Records a decision the agent made, including reasoning and outcome.
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
||||||
pub struct AgentDecisionMeta {
|
|
||||||
/// What the agent decided to do.
|
|
||||||
pub action: String,
|
|
||||||
/// Why the agent chose this action.
|
|
||||||
pub reasoning: String,
|
|
||||||
/// Available alternatives that were considered.
|
|
||||||
pub alternatives: Vec<String>,
|
|
||||||
/// Confidence in the decision (0.0-1.0).
|
|
||||||
pub confidence: f32,
|
|
||||||
/// Outcome of the decision (set after execution).
|
|
||||||
pub outcome: Option<DecisionOutcome>,
|
|
||||||
/// Context that informed the decision (entity IDs).
|
|
||||||
pub context_entities: Vec<String>,
|
|
||||||
/// The tool/task context when decision was made.
|
|
||||||
pub tool: Option<String>,
|
|
||||||
pub task: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Outcome of an agent decision.
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
||||||
pub struct DecisionOutcome {
|
|
||||||
/// Whether the decision led to success.
|
|
||||||
pub success: bool,
|
|
||||||
/// Quality score of the outcome (0.0-1.0).
|
|
||||||
pub quality: f32,
|
|
||||||
/// Feedback or error message.
|
|
||||||
pub feedback: Option<String>,
|
|
||||||
/// When the outcome was recorded.
|
|
||||||
#[serde(with = "time::serde::rfc3339")]
|
|
||||||
pub recorded_at: OffsetDateTime,
|
|
||||||
}
|
|
||||||
|
|
||||||
// --- Factory functions ---
|
|
||||||
|
|
||||||
/// Create a new AgentPrompt entity.
|
|
||||||
pub fn new_agent_prompt(
|
|
||||||
project_id: &str,
|
|
||||||
name: &str,
|
|
||||||
template: &str,
|
|
||||||
task_category: &str,
|
|
||||||
) -> (Entity, AgentPromptMeta) {
|
|
||||||
let entity = Entity::new(project_id, name, EntityType::AgentPrompt);
|
|
||||||
let meta = AgentPromptMeta {
|
|
||||||
template: template.to_string(),
|
|
||||||
target_model: None,
|
|
||||||
task_category: task_category.to_string(),
|
|
||||||
usage_count: 0,
|
|
||||||
avg_quality: 0.0,
|
|
||||||
last_used: None,
|
|
||||||
active: true,
|
|
||||||
version: 1,
|
|
||||||
tags: vec![],
|
|
||||||
};
|
|
||||||
(entity, meta)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Create a new AgentSkill entity.
|
|
||||||
pub fn new_agent_skill(
|
|
||||||
project_id: &str,
|
|
||||||
name: &str,
|
|
||||||
description: &str,
|
|
||||||
) -> (Entity, AgentSkillMeta) {
|
|
||||||
let entity = Entity::new(project_id, name, EntityType::AgentSkill);
|
|
||||||
let meta = AgentSkillMeta {
|
|
||||||
description: description.to_string(),
|
|
||||||
trigger_patterns: vec![],
|
|
||||||
success_rate: 0.0,
|
|
||||||
invocation_count: 0,
|
|
||||||
avg_latency_ms: 0,
|
|
||||||
linked_prompts: vec![],
|
|
||||||
enabled: true,
|
|
||||||
};
|
|
||||||
(entity, meta)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Create a new AgentDecision entity.
|
|
||||||
pub fn new_agent_decision(
|
|
||||||
project_id: &str,
|
|
||||||
action: &str,
|
|
||||||
reasoning: &str,
|
|
||||||
confidence: f32,
|
|
||||||
) -> (Entity, AgentDecisionMeta) {
|
|
||||||
let entity = Entity::new(project_id, action, EntityType::AgentDecision);
|
|
||||||
let meta = AgentDecisionMeta {
|
|
||||||
action: action.to_string(),
|
|
||||||
reasoning: reasoning.to_string(),
|
|
||||||
alternatives: vec![],
|
|
||||||
confidence,
|
|
||||||
outcome: None,
|
|
||||||
context_entities: vec![],
|
|
||||||
tool: None,
|
|
||||||
task: None,
|
|
||||||
};
|
|
||||||
(entity, meta)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Record outcome for a decision.
|
|
||||||
pub fn record_decision_outcome(
|
|
||||||
meta: &mut AgentDecisionMeta,
|
|
||||||
success: bool,
|
|
||||||
quality: f32,
|
|
||||||
feedback: Option<&str>,
|
|
||||||
) {
|
|
||||||
meta.outcome = Some(DecisionOutcome {
|
|
||||||
success,
|
|
||||||
quality,
|
|
||||||
feedback: feedback.map(|s| s.to_string()),
|
|
||||||
recorded_at: OffsetDateTime::now_utc(),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Update prompt usage statistics.
|
|
||||||
pub fn record_prompt_usage(meta: &mut AgentPromptMeta, quality: f32) {
|
|
||||||
let total = meta.avg_quality * meta.usage_count as f32 + quality;
|
|
||||||
meta.usage_count += 1;
|
|
||||||
meta.avg_quality = total / meta.usage_count as f32;
|
|
||||||
meta.last_used = Some(OffsetDateTime::now_utc());
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Update skill invocation statistics.
|
|
||||||
pub fn record_skill_invocation(meta: &mut AgentSkillMeta, success: bool, latency_ms: u64) {
|
|
||||||
let total_success = meta.success_rate * meta.invocation_count as f32
|
|
||||||
+ if success { 1.0 } else { 0.0 };
|
|
||||||
let total_latency = meta.avg_latency_ms * meta.invocation_count + latency_ms;
|
|
||||||
meta.invocation_count += 1;
|
|
||||||
meta.success_rate = total_success / meta.invocation_count as f32;
|
|
||||||
meta.avg_latency_ms = total_latency / meta.invocation_count;
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_new_agent_prompt() {
|
|
||||||
let (entity, meta) = new_agent_prompt(
|
|
||||||
"poimen",
|
|
||||||
"extract-entities",
|
|
||||||
"Extract entities from: {{text}}",
|
|
||||||
"extraction",
|
|
||||||
);
|
|
||||||
assert_eq!(entity.entity_type, EntityType::AgentPrompt);
|
|
||||||
assert_eq!(entity.name, "extract-entities");
|
|
||||||
assert_eq!(meta.template, "Extract entities from: {{text}}");
|
|
||||||
assert_eq!(meta.task_category, "extraction");
|
|
||||||
assert_eq!(meta.usage_count, 0);
|
|
||||||
assert!(meta.active);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_new_agent_skill() {
|
|
||||||
let (entity, meta) = new_agent_skill(
|
|
||||||
"poimen",
|
|
||||||
"diagnose-pod-failure",
|
|
||||||
"Diagnose Kubernetes pod CrashLoopBackOff",
|
|
||||||
);
|
|
||||||
assert_eq!(entity.entity_type, EntityType::AgentSkill);
|
|
||||||
assert_eq!(meta.description, "Diagnose Kubernetes pod CrashLoopBackOff");
|
|
||||||
assert!(meta.enabled);
|
|
||||||
assert_eq!(meta.invocation_count, 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_new_agent_decision() {
|
|
||||||
let (entity, meta) = new_agent_decision(
|
|
||||||
"poimen",
|
|
||||||
"restart-pod",
|
|
||||||
"Pod stuck in CrashLoopBackOff for 10 minutes",
|
|
||||||
0.85,
|
|
||||||
);
|
|
||||||
assert_eq!(entity.entity_type, EntityType::AgentDecision);
|
|
||||||
assert_eq!(meta.action, "restart-pod");
|
|
||||||
assert_eq!(meta.confidence, 0.85);
|
|
||||||
assert!(meta.outcome.is_none());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_record_decision_outcome() {
|
|
||||||
let (_, mut meta) = new_agent_decision("p", "act", "reason", 0.9);
|
|
||||||
assert!(meta.outcome.is_none());
|
|
||||||
|
|
||||||
record_decision_outcome(&mut meta, true, 0.95, Some("Pod recovered"));
|
|
||||||
assert!(meta.outcome.is_some());
|
|
||||||
let outcome = meta.outcome.unwrap();
|
|
||||||
assert!(outcome.success);
|
|
||||||
assert_eq!(outcome.quality, 0.95);
|
|
||||||
assert_eq!(outcome.feedback, Some("Pod recovered".to_string()));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_record_prompt_usage() {
|
|
||||||
let (_, mut meta) = new_agent_prompt("p", "test", "tmpl", "cat");
|
|
||||||
assert_eq!(meta.usage_count, 0);
|
|
||||||
assert_eq!(meta.avg_quality, 0.0);
|
|
||||||
|
|
||||||
record_prompt_usage(&mut meta, 0.8);
|
|
||||||
assert_eq!(meta.usage_count, 1);
|
|
||||||
assert_eq!(meta.avg_quality, 0.8);
|
|
||||||
|
|
||||||
record_prompt_usage(&mut meta, 1.0);
|
|
||||||
assert_eq!(meta.usage_count, 2);
|
|
||||||
assert!((meta.avg_quality - 0.9).abs() < 0.001);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_record_skill_invocation() {
|
|
||||||
let (_, mut meta) = new_agent_skill("p", "skill", "desc");
|
|
||||||
assert_eq!(meta.invocation_count, 0);
|
|
||||||
|
|
||||||
record_skill_invocation(&mut meta, true, 100);
|
|
||||||
assert_eq!(meta.invocation_count, 1);
|
|
||||||
assert_eq!(meta.success_rate, 1.0);
|
|
||||||
assert_eq!(meta.avg_latency_ms, 100);
|
|
||||||
|
|
||||||
record_skill_invocation(&mut meta, false, 200);
|
|
||||||
assert_eq!(meta.invocation_count, 2);
|
|
||||||
assert_eq!(meta.success_rate, 0.5);
|
|
||||||
assert_eq!(meta.avg_latency_ms, 150);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_entity_type_round_trip_agent_types() {
|
|
||||||
for ty in &[
|
|
||||||
EntityType::AgentPrompt,
|
|
||||||
EntityType::AgentSkill,
|
|
||||||
EntityType::AgentDecision,
|
|
||||||
] {
|
|
||||||
let s = ty.as_str();
|
|
||||||
assert_eq!(EntityType::from_str(s), *ty);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_agent_prompt_serialization() {
|
|
||||||
let (_, meta) = new_agent_prompt("p", "test", "tmpl {{x}}", "cat");
|
|
||||||
let json = serde_json::to_string(&meta).unwrap();
|
|
||||||
let deserialized: AgentPromptMeta = serde_json::from_str(&json).unwrap();
|
|
||||||
assert_eq!(deserialized.template, "tmpl {{x}}");
|
|
||||||
assert_eq!(deserialized.task_category, "cat");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,6 +1,5 @@
|
|||||||
/// Community domain model for temporal graph-RAG.
|
/// Community domain model for temporal graph-RAG.
|
||||||
/// Single Responsibility: Community (cluster) storage and metadata.
|
/// Single Responsibility: Community (cluster) storage and metadata.
|
||||||
#[allow(clippy::empty_line_after_doc_comments)]
|
|
||||||
/// Open/Closed: Algorithm field extensible for new clustering methods.
|
/// Open/Closed: Algorithm field extensible for new clustering methods.
|
||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|||||||
@@ -1,6 +1,5 @@
|
|||||||
/// Edge domain model for temporal graph-RAG.
|
/// Edge domain model for temporal graph-RAG.
|
||||||
/// Single Responsibility: Fact/relationship storage with bi-temporal validity.
|
/// Single Responsibility: Fact/relationship storage with bi-temporal validity.
|
||||||
#[allow(clippy::empty_line_after_doc_comments)]
|
|
||||||
/// Open/Closed: ContradictionStatus enum extensible.
|
/// Open/Closed: ContradictionStatus enum extensible.
|
||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
@@ -30,7 +29,6 @@ impl ContradictionStatus {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::should_implement_trait)]
|
|
||||||
pub fn from_str(s: &str) -> Self {
|
pub fn from_str(s: &str) -> Self {
|
||||||
match s.to_lowercase().as_str() {
|
match s.to_lowercase().as_str() {
|
||||||
"active" => Self::Active,
|
"active" => Self::Active,
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
/// Entity domain model for temporal graph-RAG.
|
/// Entity domain model for temporal graph-RAG.
|
||||||
/// Single Responsibility: Entity identity and metadata.
|
/// Single Responsibility: Entity identity and metadata.
|
||||||
/// Open/Closed: EntityType enum extensible.
|
/// Open/Closed: EntityType enum extensible.
|
||||||
#[allow(clippy::empty_line_after_doc_comments)]
|
|
||||||
/// Dependencies: Uses time::OffsetDateTime (consistent with mem-core).
|
/// Dependencies: Uses time::OffsetDateTime (consistent with mem-core).
|
||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
@@ -9,7 +8,7 @@ use time::OffsetDateTime;
|
|||||||
use std::fmt;
|
use std::fmt;
|
||||||
|
|
||||||
/// Entity type classification (extensible enum).
|
/// Entity type classification (extensible enum).
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Hash)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Hash)]
|
||||||
#[serde(rename_all = "snake_case")]
|
#[serde(rename_all = "snake_case")]
|
||||||
pub enum EntityType {
|
pub enum EntityType {
|
||||||
Person,
|
Person,
|
||||||
@@ -18,13 +17,6 @@ pub enum EntityType {
|
|||||||
Location,
|
Location,
|
||||||
Event,
|
Event,
|
||||||
Organization,
|
Organization,
|
||||||
/// Agent prompt template tracked as a first-class entity.
|
|
||||||
/// Enables the agent to learn which prompts produce good results.
|
|
||||||
AgentPrompt,
|
|
||||||
/// Agent skill — a reusable capability the agent has learned.
|
|
||||||
AgentSkill,
|
|
||||||
/// Agent decision — a recorded choice with reasoning and outcome.
|
|
||||||
AgentDecision,
|
|
||||||
Unknown,
|
Unknown,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -37,14 +29,10 @@ impl EntityType {
|
|||||||
Self::Location => "location",
|
Self::Location => "location",
|
||||||
Self::Event => "event",
|
Self::Event => "event",
|
||||||
Self::Organization => "organization",
|
Self::Organization => "organization",
|
||||||
Self::AgentPrompt => "agent_prompt",
|
|
||||||
Self::AgentSkill => "agent_skill",
|
|
||||||
Self::AgentDecision => "agent_decision",
|
|
||||||
Self::Unknown => "unknown",
|
Self::Unknown => "unknown",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::should_implement_trait)]
|
|
||||||
pub fn from_str(s: &str) -> Self {
|
pub fn from_str(s: &str) -> Self {
|
||||||
match s.to_lowercase().as_str() {
|
match s.to_lowercase().as_str() {
|
||||||
"person" => Self::Person,
|
"person" => Self::Person,
|
||||||
@@ -53,24 +41,11 @@ impl EntityType {
|
|||||||
"location" => Self::Location,
|
"location" => Self::Location,
|
||||||
"event" => Self::Event,
|
"event" => Self::Event,
|
||||||
"organization" => Self::Organization,
|
"organization" => Self::Organization,
|
||||||
"agent_prompt" => Self::AgentPrompt,
|
|
||||||
"agent_skill" => Self::AgentSkill,
|
|
||||||
"agent_decision" => Self::AgentDecision,
|
|
||||||
_ => Self::Unknown,
|
_ => Self::Unknown,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<'de> serde::Deserialize<'de> for EntityType {
|
|
||||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
|
||||||
where
|
|
||||||
D: serde::Deserializer<'de>,
|
|
||||||
{
|
|
||||||
let s = String::deserialize(deserializer)?;
|
|
||||||
Ok(Self::from_str(&s))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl fmt::Display for EntityType {
|
impl fmt::Display for EntityType {
|
||||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
write!(f, "{}", self.as_str())
|
write!(f, "{}", self.as_str())
|
||||||
@@ -200,9 +175,6 @@ mod tests {
|
|||||||
EntityType::Person,
|
EntityType::Person,
|
||||||
EntityType::Tool,
|
EntityType::Tool,
|
||||||
EntityType::Concept,
|
EntityType::Concept,
|
||||||
EntityType::AgentPrompt,
|
|
||||||
EntityType::AgentSkill,
|
|
||||||
EntityType::AgentDecision,
|
|
||||||
] {
|
] {
|
||||||
let s = ty.as_str();
|
let s = ty.as_str();
|
||||||
assert_eq!(EntityType::from_str(s), *ty);
|
assert_eq!(EntityType::from_str(s), *ty);
|
||||||
|
|||||||
@@ -135,10 +135,11 @@ pub fn run_loop(
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_loop_basic() {
|
fn test_loop_basic() {
|
||||||
// Placeholder test to verify it compiles
|
// Placeholder test to verify it compiles
|
||||||
|
assert!(true);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -403,7 +403,7 @@ pub fn lookup(sig: &Signature, lessons: &[Lesson], floor: f32) -> Option<Hit> {
|
|||||||
let mut best: Option<(f32, &Lesson)> = None;
|
let mut best: Option<(f32, &Lesson)> = None;
|
||||||
for l in lessons.iter().filter(|l| l.tool == sig.tool) {
|
for l in lessons.iter().filter(|l| l.tool == sig.tool) {
|
||||||
let s = similarity(&sig.normalised, &l.normalised);
|
let s = similarity(&sig.normalised, &l.normalised);
|
||||||
if s >= floor && best.is_none_or(|(bs, _)| s > bs) {
|
if s >= floor && best.map_or(true, |(bs, _)| s > bs) {
|
||||||
best = Some((s, l));
|
best = Some((s, l));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -503,7 +503,7 @@ pub fn tool_of_cmd(cmd: &str) -> String {
|
|||||||
"kubectl" | "k" => "kubectl".into(),
|
"kubectl" | "k" => "kubectl".into(),
|
||||||
"docker" | "podman" => "docker".into(),
|
"docker" | "podman" => "docker".into(),
|
||||||
"terraform" | "tofu" => "terraform".into(),
|
"terraform" | "tofu" => "terraform".into(),
|
||||||
"" => "unknown".into(),
|
other if other.is_empty() => "unknown".into(),
|
||||||
other => other.to_string(),
|
other => other.to_string(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -549,7 +549,7 @@ pub fn render_skill(tool: &str, lessons: &[Lesson]) -> String {
|
|||||||
s.push_str("`confirmed`, which outranks inferred lessons at equal similarity.\n\n");
|
s.push_str("`confirmed`, which outranks inferred lessons at equal similarity.\n\n");
|
||||||
|
|
||||||
let mut sorted: Vec<&Lesson> = lessons.iter().collect();
|
let mut sorted: Vec<&Lesson> = lessons.iter().collect();
|
||||||
sorted.sort_by_key(|a| std::cmp::Reverse(a.seen));
|
sorted.sort_by(|a, b| b.seen.cmp(&a.seen));
|
||||||
|
|
||||||
for l in sorted {
|
for l in sorted {
|
||||||
s.push_str(&format!("## {}\n\n", l.raw.trim()));
|
s.push_str(&format!("## {}\n\n", l.raw.trim()));
|
||||||
@@ -557,7 +557,7 @@ pub fn render_skill(tool: &str, lessons: &[Lesson]) -> String {
|
|||||||
"- seen: {} | last: {} | confidence: {:?}\n",
|
"- seen: {} | last: {} | confidence: {:?}\n",
|
||||||
l.seen, l.last_seen, l.confidence
|
l.seen, l.last_seen, l.confidence
|
||||||
));
|
));
|
||||||
s.push_str(&format!("- signature: `{}`\n", &l.sig_sha[..12]));
|
s.push_str(&format!("- signature: `{}`\n", l.sig_sha[..12].to_string()));
|
||||||
s.push_str("- resolved by:\n");
|
s.push_str("- resolved by:\n");
|
||||||
for r in &l.resolution {
|
for r in &l.resolution {
|
||||||
s.push_str(&format!(" ```\n {r}\n ```\n"));
|
s.push_str(&format!(" ```\n {r}\n ```\n"));
|
||||||
@@ -712,7 +712,7 @@ mod tests {
|
|||||||
ev("t2", "npm pkg set overrides.react=19", 0, ""),
|
ev("t2", "npm pkg set overrides.react=19", 0, ""),
|
||||||
ev("t3", "npm ci", 0, "ok"),
|
ev("t3", "npm ci", 0, "ok"),
|
||||||
];
|
];
|
||||||
let ls = derive_lessons(&events, tool_of_cmd);
|
let ls = derive_lessons(&events, |c| tool_of_cmd(c));
|
||||||
assert_eq!(ls.len(), 1);
|
assert_eq!(ls.len(), 1);
|
||||||
assert_eq!(ls[0].resolution, vec!["npm pkg set overrides.react=19"]);
|
assert_eq!(ls[0].resolution, vec!["npm pkg set overrides.react=19"]);
|
||||||
assert_eq!(ls[0].confidence, Confidence::Inferred);
|
assert_eq!(ls[0].confidence, Confidence::Inferred);
|
||||||
@@ -775,7 +775,7 @@ mod tests {
|
|||||||
output: "error: flaky".into(),
|
output: "error: flaky".into(),
|
||||||
};
|
};
|
||||||
let events = vec![ev("npm ci", 1), ev("npm ci", 0)];
|
let events = vec![ev("npm ci", 1), ev("npm ci", 0)];
|
||||||
assert!(derive_lessons(&events, tool_of_cmd).is_empty());
|
assert!(derive_lessons(&events, |c| tool_of_cmd(c)).is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -798,7 +798,7 @@ mod tests {
|
|||||||
sig_sha: "abc".into(),
|
sig_sha: "abc".into(),
|
||||||
rule: "r".into(),
|
rule: "r".into(),
|
||||||
};
|
};
|
||||||
assert_eq!(lookup(&exact, std::slice::from_ref(&l), 0.5).unwrap().tier, Tier::Exact);
|
assert_eq!(lookup(&exact, &[l.clone()], 0.5).unwrap().tier, Tier::Exact);
|
||||||
|
|
||||||
let unrelated = Signature {
|
let unrelated = Signature {
|
||||||
tool: "npm".into(),
|
tool: "npm".into(),
|
||||||
|
|||||||
@@ -12,7 +12,6 @@ pub mod scoring;
|
|||||||
pub mod entity;
|
pub mod entity;
|
||||||
pub mod edge;
|
pub mod edge;
|
||||||
pub mod community;
|
pub mod community;
|
||||||
pub mod agent_entity;
|
|
||||||
|
|
||||||
pub use gate_parser::{GateResponse, ParseError, parse_gate_response};
|
pub use gate_parser::{GateResponse, ParseError, parse_gate_response};
|
||||||
|
|
||||||
@@ -31,4 +30,3 @@ pub use scoring::{DocumentScorer, ScoringPipeline, GlobalTfIdfScorer, ProjectTfI
|
|||||||
pub use entity::{Entity, EntityType};
|
pub use entity::{Entity, EntityType};
|
||||||
pub use edge::{Edge, ContradictionStatus};
|
pub use edge::{Edge, ContradictionStatus};
|
||||||
pub use community::Community;
|
pub use community::Community;
|
||||||
pub use agent_entity::{AgentPromptMeta, AgentSkillMeta, AgentDecisionMeta, DecisionOutcome};
|
|
||||||
|
|||||||
@@ -152,11 +152,11 @@ impl FormatHandler for CsvFormatter {
|
|||||||
|
|
||||||
async fn format(&self, result: &OptimizationResult) -> Result<Vec<u8>, String> {
|
async fn format(&self, result: &OptimizationResult) -> Result<Vec<u8>, String> {
|
||||||
let output = format!(
|
let output = format!(
|
||||||
"{},{},{},{:.2}\n",
|
"{},{},{},{}\n",
|
||||||
escape_csv(&result.plugin),
|
escape_csv(&result.plugin),
|
||||||
result.original.len(),
|
result.original.len(),
|
||||||
result.optimized.len(),
|
result.optimized.len(),
|
||||||
result.ratio
|
format!("{:.2}", result.ratio)
|
||||||
);
|
);
|
||||||
Ok(output.into_bytes())
|
Ok(output.into_bytes())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -40,7 +40,7 @@ impl CcrStore {
|
|||||||
// Remove oldest entry if at capacity
|
// Remove oldest entry if at capacity
|
||||||
if cache.len() >= self.max_entries {
|
if cache.len() >= self.max_entries {
|
||||||
if let Some(oldest_key) = cache.keys().next().cloned() {
|
if let Some(oldest_key) = cache.keys().next().cloned() {
|
||||||
cache.swap_remove(&oldest_key);
|
cache.remove(&oldest_key);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -57,7 +57,7 @@ impl CcrStore {
|
|||||||
// Check if expired
|
// Check if expired
|
||||||
let duration = OffsetDateTime::now_utc() - *timestamp;
|
let duration = OffsetDateTime::now_utc() - *timestamp;
|
||||||
if duration.whole_seconds() > self.ttl_secs as i64 {
|
if duration.whole_seconds() > self.ttl_secs as i64 {
|
||||||
cache.swap_remove(hash);
|
cache.remove(hash);
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -7,7 +7,7 @@
|
|||||||
//! - Drop: redundant homogeneous elements, long string values
|
//! - Drop: redundant homogeneous elements, long string values
|
||||||
|
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use serde_json::Value;
|
use serde_json::{json, Value};
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
|
||||||
pub struct JsonCrusher;
|
pub struct JsonCrusher;
|
||||||
@@ -45,8 +45,8 @@ impl JsonCrusher {
|
|||||||
let mut result = Vec::new();
|
let mut result = Vec::new();
|
||||||
|
|
||||||
// Add start items
|
// Add start items
|
||||||
for item in items.iter().take(start_count.min(len)) {
|
for i in 0..start_count.min(len) {
|
||||||
result.push(item.clone());
|
result.push(items[i].clone());
|
||||||
}
|
}
|
||||||
|
|
||||||
// Select mid-array items by variance/importance
|
// Select mid-array items by variance/importance
|
||||||
@@ -58,8 +58,8 @@ impl JsonCrusher {
|
|||||||
|
|
||||||
// Add end items
|
// Add end items
|
||||||
if end_count > 0 {
|
if end_count > 0 {
|
||||||
for item in items.iter().skip(len.saturating_sub(end_count)) {
|
for i in (len - end_count)..len {
|
||||||
result.push(item.clone());
|
result.push(items[i].clone());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -5,7 +5,7 @@
|
|||||||
|
|
||||||
use super::plugin::OptimizerService;
|
use super::plugin::OptimizerService;
|
||||||
use crate::prompt::CacheMetrics;
|
use crate::prompt::CacheMetrics;
|
||||||
use crate::domain::Chunk;
|
use crate::domain::{Chunk, Record};
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
|
|
||||||
/// Query optimizer: compresses chunks before LLM processing
|
/// Query optimizer: compresses chunks before LLM processing
|
||||||
@@ -83,7 +83,7 @@ impl QueryOptimizer {
|
|||||||
match service.optimize(&chunk_text, &content_type, Some("raw")).await {
|
match service.optimize(&chunk_text, &content_type, Some("raw")).await {
|
||||||
Ok(bytes) => {
|
Ok(bytes) => {
|
||||||
let text = String::from_utf8(bytes)
|
let text = String::from_utf8(bytes)
|
||||||
.unwrap_or(chunk_text);
|
.unwrap_or_else(|_| chunk_text);
|
||||||
Ok(text)
|
Ok(text)
|
||||||
}
|
}
|
||||||
Err(_) => {
|
Err(_) => {
|
||||||
|
|||||||
@@ -42,7 +42,7 @@ impl ContentRouter {
|
|||||||
/// Check if content is valid JSON
|
/// Check if content is valid JSON
|
||||||
fn is_json(content: &str) -> bool {
|
fn is_json(content: &str) -> bool {
|
||||||
let trimmed = content.trim();
|
let trimmed = content.trim();
|
||||||
if !(trimmed.starts_with('{') || trimmed.starts_with('[')) {
|
if !((trimmed.starts_with('{') || trimmed.starts_with('['))) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
serde_json::from_str::<serde_json::Value>(trimmed).is_ok()
|
serde_json::from_str::<serde_json::Value>(trimmed).is_ok()
|
||||||
|
|||||||
@@ -128,7 +128,7 @@ impl TextCompressor {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Capitalization (usually proper nouns or emphatic)
|
// Capitalization (usually proper nouns or emphatic)
|
||||||
if token.chars().next().is_some_and(|c| c.is_uppercase()) && token.len() > 1 {
|
if token.chars().next().map_or(false, |c| c.is_uppercase()) && token.len() > 1 {
|
||||||
score += 1.0;
|
score += 1.0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -12,9 +12,7 @@ const CACHE_TURN: &str = include_str!("../../../templates/gru-mem-turn.txt");
|
|||||||
|
|
||||||
const BUDGET_TOTAL: usize = 32768;
|
const BUDGET_TOTAL: usize = 32768;
|
||||||
const BUDGET_RESPONSE: usize = 2048;
|
const BUDGET_RESPONSE: usize = 2048;
|
||||||
#[allow(dead_code)]
|
|
||||||
const BUDGET_SYSTEM: usize = 400;
|
const BUDGET_SYSTEM: usize = 400;
|
||||||
#[allow(dead_code)]
|
|
||||||
const BUDGET_QUESTION: usize = 150;
|
const BUDGET_QUESTION: usize = 150;
|
||||||
const BUDGET_MEMORY_MAX: usize = 1024;
|
const BUDGET_MEMORY_MAX: usize = 1024;
|
||||||
const BUDGET_CHUNK_MAX: usize = 5000;
|
const BUDGET_CHUNK_MAX: usize = 5000;
|
||||||
@@ -370,7 +368,7 @@ fn estimate_tokens(text: &str) -> usize {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::domain::{Chunk, Record, Role, Provenance};
|
use crate::domain::{Chunk, Record, Role, Provenance, Level};
|
||||||
use time::OffsetDateTime;
|
use time::OffsetDateTime;
|
||||||
|
|
||||||
fn make_test_chunk(text: &str) -> Chunk {
|
fn make_test_chunk(text: &str) -> Chunk {
|
||||||
@@ -647,7 +645,7 @@ mod tests {
|
|||||||
|
|
||||||
let metrics = result.unwrap();
|
let metrics = result.unwrap();
|
||||||
let ratio = metrics.compression_ratio();
|
let ratio = metrics.compression_ratio();
|
||||||
assert!((0.0..=100.0).contains(&ratio));
|
assert!(ratio >= 0.0 && ratio <= 100.0);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
|
use crate::domain::{ProjectId, QueryId};
|
||||||
use anyhow::{anyhow, Result};
|
use anyhow::{anyhow, Result};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
use std::collections::HashMap;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|
||||||
/// A single standing query.
|
/// A single standing query.
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
use crate::Level;
|
use crate::{Level, Query};
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
@@ -17,12 +17,6 @@ pub struct QueryExecutor {
|
|||||||
// For now: proof-of-concept with mock data
|
// For now: proof-of-concept with mock data
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for QueryExecutor {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self::new()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl QueryExecutor {
|
impl QueryExecutor {
|
||||||
/// Create executor.
|
/// Create executor.
|
||||||
pub fn new() -> Self {
|
pub fn new() -> Self {
|
||||||
|
|||||||
@@ -71,10 +71,11 @@ impl QueryLevels {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Check level filter
|
// Check level filter
|
||||||
if !self.level_filter.is_empty()
|
if !self.level_filter.is_empty() {
|
||||||
&& !self.level_filter.contains(&level.to_string()) {
|
if !self.level_filter.contains(&level.to_string()) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Check evidence/reference flags
|
// Check evidence/reference flags
|
||||||
if level == "R" {
|
if level == "R" {
|
||||||
|
|||||||
@@ -6,7 +6,6 @@
|
|||||||
/// - Single Responsibility: each scorer does one thing
|
/// - Single Responsibility: each scorer does one thing
|
||||||
/// - Open/Closed: add new scorers without modifying existing
|
/// - Open/Closed: add new scorers without modifying existing
|
||||||
/// - Liskov Substitution: all scorers implement DocumentScorer
|
/// - Liskov Substitution: all scorers implement DocumentScorer
|
||||||
#[allow(clippy::empty_line_after_doc_comments)]
|
|
||||||
/// - Dependency Inversion: depend on trait, not concrete types
|
/// - Dependency Inversion: depend on trait, not concrete types
|
||||||
|
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
@@ -54,7 +53,6 @@ impl DocumentScorer for GlobalTfIdfScorer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Project-scoped TF-IDF Scorer: scoring within project boundaries
|
/// Project-scoped TF-IDF Scorer: scoring within project boundaries
|
||||||
#[allow(dead_code)]
|
|
||||||
pub struct ProjectTfIdfScorer {
|
pub struct ProjectTfIdfScorer {
|
||||||
project: String,
|
project: String,
|
||||||
vocabulary: Arc<std::collections::BTreeMap<String, f32>>,
|
vocabulary: Arc<std::collections::BTreeMap<String, f32>>,
|
||||||
@@ -95,18 +93,11 @@ impl DocumentScorer for ProjectTfIdfScorer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Semantic Scorer: vector similarity (placeholder)
|
/// Semantic Scorer: vector similarity (placeholder)
|
||||||
#[allow(dead_code)]
|
|
||||||
pub struct SemanticScorer {
|
pub struct SemanticScorer {
|
||||||
_embeddings_client: Arc<()>, // Placeholder
|
_embeddings_client: Arc<()>, // Placeholder
|
||||||
_pgvector: Arc<()>, // Placeholder
|
_pgvector: Arc<()>, // Placeholder
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for SemanticScorer {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self::new()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl SemanticScorer {
|
impl SemanticScorer {
|
||||||
pub fn new() -> Self {
|
pub fn new() -> Self {
|
||||||
Self {
|
Self {
|
||||||
@@ -165,12 +156,6 @@ pub struct ScoringPipeline {
|
|||||||
scorers: Vec<(String, f32, Arc<dyn DocumentScorer>)>, // name, weight, scorer
|
scorers: Vec<(String, f32, Arc<dyn DocumentScorer>)>, // name, weight, scorer
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for ScoringPipeline {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self::new()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ScoringPipeline {
|
impl ScoringPipeline {
|
||||||
pub fn new() -> Self {
|
pub fn new() -> Self {
|
||||||
Self {
|
Self {
|
||||||
|
|||||||
@@ -81,7 +81,6 @@ impl SymptomVector {
|
|||||||
|
|
||||||
/// Internal structure for tokens during extraction
|
/// Internal structure for tokens during extraction
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
#[allow(dead_code)]
|
|
||||||
struct SymptomTokens {
|
struct SymptomTokens {
|
||||||
keywords: Vec<String>,
|
keywords: Vec<String>,
|
||||||
error_codes: Vec<String>,
|
error_codes: Vec<String>,
|
||||||
@@ -393,7 +392,7 @@ mod tests {
|
|||||||
let words: Vec<&str> = symptom.normalised.split_whitespace().collect();
|
let words: Vec<&str> = symptom.normalised.split_whitespace().collect();
|
||||||
for word in &words {
|
for word in &words {
|
||||||
// Check if this word is a stop word
|
// Check if this word is a stop word
|
||||||
assert!(!STOP_WORDS.contains(word), "Stop word '{}' should be removed", word);
|
assert!(!STOP_WORDS.contains(&word), "Stop word '{}' should be removed", word);
|
||||||
}
|
}
|
||||||
// Should contain key terms
|
// Should contain key terms
|
||||||
assert!(symptom.normalised.contains("resolve"));
|
assert!(symptom.normalised.contains("resolve"));
|
||||||
|
|||||||
@@ -267,9 +267,11 @@ fn test_compression_handles_large_content() {
|
|||||||
fn test_multi_chunk_search_consistency() {
|
fn test_multi_chunk_search_consistency() {
|
||||||
let optimizer = ContextOptimizer::new().expect("optimizer init");
|
let optimizer = ContextOptimizer::new().expect("optimizer init");
|
||||||
|
|
||||||
let chunks = ["ERROR: connection failed\nDEBUG: thread id=100",
|
let chunks = vec![
|
||||||
|
"ERROR: connection failed\nDEBUG: thread id=100",
|
||||||
"ERROR: timeout after 5000ms\nTRACE: stack unwinding",
|
"ERROR: timeout after 5000ms\nTRACE: stack unwinding",
|
||||||
"ERROR: retry attempt 2\nDEBUG: backoff delay=200ms"];
|
"ERROR: retry attempt 2\nDEBUG: backoff delay=200ms",
|
||||||
|
];
|
||||||
|
|
||||||
let optimized_chunks: Vec<_> = chunks
|
let optimized_chunks: Vec<_> = chunks
|
||||||
.iter()
|
.iter()
|
||||||
|
|||||||
@@ -103,7 +103,7 @@ fn gate_metadata_preservation() {
|
|||||||
|
|
||||||
// Verify we get a valid OptimizedChunk with proper fields
|
// Verify we get a valid OptimizedChunk with proper fields
|
||||||
assert!(optimized.original_tokens > 0, "should track original tokens");
|
assert!(optimized.original_tokens > 0, "should track original tokens");
|
||||||
assert!(optimized.compressed_tokens <= optimized.original_tokens, "compressed should not exceed original");
|
assert!(optimized.compressed_tokens >= 0, "should track compressed tokens");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -122,7 +122,7 @@ fn gate_error_handling_graceful() {
|
|||||||
match optimizer.optimize(case.as_str()) {
|
match optimizer.optimize(case.as_str()) {
|
||||||
Ok(result) => {
|
Ok(result) => {
|
||||||
// Valid compression
|
// Valid compression
|
||||||
assert!(result.original_tokens > 0);
|
assert!(result.original_tokens >= 0);
|
||||||
}
|
}
|
||||||
Err(_) => {
|
Err(_) => {
|
||||||
// Acceptable to fail on edge cases, but should fail gracefully
|
// Acceptable to fail on edge cases, but should fail gracefully
|
||||||
@@ -196,6 +196,7 @@ fn gate_memory_bounded() {
|
|||||||
|
|
||||||
// Should not panic from memory exhaustion
|
// Should not panic from memory exhaustion
|
||||||
// If we get here, we passed the gate
|
// If we get here, we passed the gate
|
||||||
|
assert!(true, "memory usage bounded");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -208,7 +209,7 @@ fn gate_no_regressions_existing_functionality() {
|
|||||||
|
|
||||||
assert!(!result.compressed.is_empty(), "basic optimization should work");
|
assert!(!result.compressed.is_empty(), "basic optimization should work");
|
||||||
assert!(result.original_tokens > 0, "should track tokens");
|
assert!(result.original_tokens > 0, "should track tokens");
|
||||||
assert!(result.compressed_tokens <= result.original_tokens, "compressed should not exceed original");
|
assert!(result.compressed_tokens >= 0, "should have compressed tokens");
|
||||||
}
|
}
|
||||||
|
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
@@ -230,7 +231,7 @@ fn gate_compression_targets_met() {
|
|||||||
];
|
];
|
||||||
|
|
||||||
for (content, name, min_compression) in fixtures.iter() {
|
for (content, name, min_compression) in fixtures.iter() {
|
||||||
let optimized = optimizer.optimize(content).unwrap_or_else(|_| panic!("optimize {}", name));
|
let optimized = optimizer.optimize(content).expect(&format!("optimize {}", name));
|
||||||
let ratio = optimized.compressed.len() as f32 / content.len() as f32;
|
let ratio = optimized.compressed.len() as f32 / content.len() as f32;
|
||||||
|
|
||||||
// At least some compression should happen
|
// At least some compression should happen
|
||||||
@@ -331,4 +332,5 @@ fn gate_summary_report() {
|
|||||||
|
|
||||||
println!("\n🚀 STATUS: M3.8 READY FOR PRODUCTION");
|
println!("\n🚀 STATUS: M3.8 READY FOR PRODUCTION");
|
||||||
|
|
||||||
|
assert!(true); // Just for testing framework
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -52,24 +52,13 @@ impl AuthentikJwtIssuer {
|
|||||||
|
|
||||||
/// From environment: AUTHENTIK_ISSUER, AUTHENTIK_CLIENT_ID, AUTHENTIK_CLIENT_SECRET
|
/// From environment: AUTHENTIK_ISSUER, AUTHENTIK_CLIENT_ID, AUTHENTIK_CLIENT_SECRET
|
||||||
pub fn from_env() -> Result<Self> {
|
pub fn from_env() -> Result<Self> {
|
||||||
// Support both naming conventions: AUTHENTIK_* and memory-agent-oidc secret keys
|
|
||||||
let issuer = std::env::var("AUTHENTIK_ISSUER")
|
let issuer = std::env::var("AUTHENTIK_ISSUER")
|
||||||
.or_else(|_| std::env::var("ISSUER"))
|
.map_err(|_| anyhow!("AUTHENTIK_ISSUER not set"))?;
|
||||||
.map_err(|_| anyhow!("AUTHENTIK_ISSUER or ISSUER not set"))?;
|
|
||||||
let client_id = std::env::var("AUTHENTIK_CLIENT_ID")
|
let client_id = std::env::var("AUTHENTIK_CLIENT_ID")
|
||||||
.or_else(|_| std::env::var("CLIENT_ID"))
|
.map_err(|_| anyhow!("AUTHENTIK_CLIENT_ID not set"))?;
|
||||||
.map_err(|_| anyhow!("AUTHENTIK_CLIENT_ID or CLIENT_ID not set"))?;
|
|
||||||
let client_secret = std::env::var("AUTHENTIK_CLIENT_SECRET")
|
let client_secret = std::env::var("AUTHENTIK_CLIENT_SECRET")
|
||||||
.or_else(|_| std::env::var("CLIENT_SECRET"))
|
.map_err(|_| anyhow!("AUTHENTIK_CLIENT_SECRET not set"))?;
|
||||||
.map_err(|_| anyhow!("AUTHENTIK_CLIENT_SECRET or CLIENT_SECRET not set"))?;
|
|
||||||
|
|
||||||
tracing::info!(
|
|
||||||
target: "observability",
|
|
||||||
event = "authentik_jwt_init",
|
|
||||||
issuer = %issuer,
|
|
||||||
client_id = %client_id,
|
|
||||||
"Authentik JWT issuer initialized"
|
|
||||||
);
|
|
||||||
Ok(Self::new(&issuer, &client_id, &client_secret))
|
Ok(Self::new(&issuer, &client_id, &client_secret))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -103,25 +92,12 @@ impl AuthentikJwtIssuer {
|
|||||||
let client = reqwest::Client::new();
|
let client = reqwest::Client::new();
|
||||||
|
|
||||||
// Authentik OAuth2 token endpoint
|
// Authentik OAuth2 token endpoint
|
||||||
// Use TOKEN_URL env var if set, otherwise derive from issuer
|
let token_url = format!("{}/token/", self.issuer_url.trim_end_matches('/'));
|
||||||
let token_url = std::env::var("TOKEN_URL")
|
|
||||||
.or_else(|_| std::env::var("AUTHENTIK_TOKEN_URL"))
|
|
||||||
.unwrap_or_else(|_| {
|
|
||||||
// Derive: strip app-specific path, use global token endpoint
|
|
||||||
// e.g., https://authentik.riotpiao.com/application/o/memory-agent/
|
|
||||||
// -> https://authentik.riotpiao.com/application/o/token/
|
|
||||||
if let Some(base) = self.issuer_url.rfind("/o/") {
|
|
||||||
format!("{}/o/token/", &self.issuer_url[..base])
|
|
||||||
} else {
|
|
||||||
format!("{}/token/", self.issuer_url.trim_end_matches('/'))
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
let params = [
|
let params = [
|
||||||
("grant_type", "client_credentials"),
|
("grant_type", "client_credentials"),
|
||||||
("client_id", &self.client_id),
|
("client_id", &self.client_id),
|
||||||
("client_secret", &self.client_secret),
|
("client_secret", &self.client_secret),
|
||||||
("scope", "openid roles"),
|
|
||||||
];
|
];
|
||||||
|
|
||||||
let response = client
|
let response = client
|
||||||
@@ -160,13 +136,13 @@ mod tests {
|
|||||||
access_token: "test".to_string(),
|
access_token: "test".to_string(),
|
||||||
token_type: "Bearer".to_string(),
|
token_type: "Bearer".to_string(),
|
||||||
expires_in: 3600,
|
expires_in: 3600,
|
||||||
obtained_at: Some(SystemTime::now()),
|
obtained_at: SystemTime::now(),
|
||||||
};
|
};
|
||||||
|
|
||||||
assert!(!token.is_expired());
|
assert!(!token.is_expired());
|
||||||
|
|
||||||
// Simulate aged token
|
// Simulate aged token
|
||||||
token.obtained_at = Some(SystemTime::now() - Duration::from_secs(3600));
|
token.obtained_at = SystemTime::now() - Duration::from_secs(3600);
|
||||||
assert!(token.is_expired());
|
assert!(token.is_expired());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -83,7 +83,6 @@ impl ContradictionPreFilter {
|
|||||||
|
|
||||||
/// LLM-based contradiction detector (stage 2)
|
/// LLM-based contradiction detector (stage 2)
|
||||||
/// Only called if pre-filter returns true (cost optimization)
|
/// Only called if pre-filter returns true (cost optimization)
|
||||||
#[allow(dead_code)]
|
|
||||||
pub struct LlmContradictionDetector {
|
pub struct LlmContradictionDetector {
|
||||||
model_name: String,
|
model_name: String,
|
||||||
auto_confirm_threshold: f32,
|
auto_confirm_threshold: f32,
|
||||||
|
|||||||
@@ -22,15 +22,11 @@ use tokio::sync::Mutex;
|
|||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
pub struct ExtractedEntity {
|
pub struct ExtractedEntity {
|
||||||
pub name: String,
|
pub name: String,
|
||||||
#[serde(alias = "type")]
|
|
||||||
pub entity_type: EntityType,
|
pub entity_type: EntityType,
|
||||||
pub summary: String,
|
pub summary: String,
|
||||||
#[serde(default = "default_confidence")]
|
|
||||||
pub confidence: f32,
|
pub confidence: f32,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn default_confidence() -> f32 { 0.8 }
|
|
||||||
|
|
||||||
impl ExtractedEntity {
|
impl ExtractedEntity {
|
||||||
/// Convert to domain model (Phase 1 type)
|
/// Convert to domain model (Phase 1 type)
|
||||||
pub fn to_domain(&self, project_id: &str) -> Entity {
|
pub fn to_domain(&self, project_id: &str) -> Entity {
|
||||||
@@ -44,15 +40,10 @@ impl ExtractedEntity {
|
|||||||
#[async_trait]
|
#[async_trait]
|
||||||
pub trait EntityExtractor: Send + Sync {
|
pub trait EntityExtractor: Send + Sync {
|
||||||
async fn extract(&self, text: &str) -> Result<Vec<ExtractedEntity>>;
|
async fn extract(&self, text: &str) -> Result<Vec<ExtractedEntity>>;
|
||||||
async fn extract_with_auth(&self, text: &str, _x_forward_user: Option<&str>) -> Result<Vec<ExtractedEntity>> {
|
|
||||||
// Default: ignore auth header, use regular extract
|
|
||||||
self.extract(text).await
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// LLM-based extractor with reflection verification (stage 1 + 2)
|
/// LLM-based extractor with reflection verification (stage 1 + 2)
|
||||||
/// Uses Authentik JWT tokens for authentication to LLM gateway
|
/// Uses Authentik JWT tokens for authentication to LLM gateway
|
||||||
#[allow(dead_code)]
|
|
||||||
pub struct LlmEntityExtractor {
|
pub struct LlmEntityExtractor {
|
||||||
model_name: String,
|
model_name: String,
|
||||||
enable_reflection: bool,
|
enable_reflection: bool,
|
||||||
@@ -71,35 +62,6 @@ impl LlmEntityExtractor {
|
|||||||
|
|
||||||
/// Parse extraction response JSON
|
/// Parse extraction response JSON
|
||||||
/// Format: { "entities": [{ "name": "...", "type": "...", "summary": "..." }, ...] }
|
/// Format: { "entities": [{ "name": "...", "type": "...", "summary": "..." }, ...] }
|
||||||
/// Clean LLM response: strip thinking tags, markdown fences, extract JSON
|
|
||||||
fn clean_llm_response(text: &str) -> String {
|
|
||||||
let mut result = text.to_string();
|
|
||||||
// Remove <think>...</think> blocks
|
|
||||||
while let Some(start) = result.find("<think>") {
|
|
||||||
if let Some(end) = result.find("</think>") {
|
|
||||||
result = format!("{}{}", &result[..start], &result[end + 8..]);
|
|
||||||
} else {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Remove markdown code fences
|
|
||||||
result = result.replace("```json", "").replace("```", "");
|
|
||||||
// Find JSON object
|
|
||||||
let trimmed = result.trim();
|
|
||||||
if let Some(start) = trimmed.find('{') {
|
|
||||||
if let Some(end) = trimmed.rfind('}') {
|
|
||||||
return trimmed[start..=end].to_string();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Maybe it's a JSON array — wrap in object
|
|
||||||
if let Some(start) = trimmed.find('[') {
|
|
||||||
if let Some(end) = trimmed.rfind(']') {
|
|
||||||
return format!("{{\"entities\": {}}}", &trimmed[start..=end]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
trimmed.to_string()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn parse_extraction(response: &str) -> Result<Vec<ExtractedEntity>> {
|
fn parse_extraction(response: &str) -> Result<Vec<ExtractedEntity>> {
|
||||||
#[derive(Deserialize)]
|
#[derive(Deserialize)]
|
||||||
struct Response {
|
struct Response {
|
||||||
@@ -125,42 +87,29 @@ impl LlmEntityExtractor {
|
|||||||
Ok(parsed.verified.into_iter().map(|v| (v.name, v.present)).collect())
|
Ok(parsed.verified.into_iter().map(|v| (v.name, v.present)).collect())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Call LLM via api.riotpiao.com using X-Forward-User auth/exchange
|
/// Call LLM via api.riotpiao.com using Authentik JWT
|
||||||
/// Supports: Authentik JWT, X-Forward-User header, or API key fallback
|
/// Token is fetched from Authentik service account and cached
|
||||||
async fn call_llm_endpoint(&self, prompt: &str, x_forward_user: Option<&str>) -> Result<String> {
|
async fn call_llm_endpoint(&self, prompt: &str) -> Result<String> {
|
||||||
let endpoint = std::env::var("LLM_ENDPOINT")
|
let endpoint = std::env::var("LLM_ENDPOINT")
|
||||||
.unwrap_or_else(|_| "http://api-internal.riotpiao.com:8000/v1/chat/completions".to_string());
|
.unwrap_or_else(|_| "http://api-internal.riotpiao.com:8000/v1/chat/completions".to_string());
|
||||||
let model = std::env::var("LLM_MODEL")
|
let model = std::env::var("LLM_MODEL")
|
||||||
.unwrap_or_else(|_| "qwen:7b".to_string());
|
.unwrap_or_else(|_| "qwen:7b".to_string());
|
||||||
|
|
||||||
// Get auth header: prefer X-Forward-User, fallback to Authentik JWT, then API key
|
// Get JWT token from Authentik
|
||||||
let auth_header = if let Some(user) = x_forward_user {
|
let auth_header = if let Some(jwt_issuer) = &self.jwt_issuer {
|
||||||
// Use X-Forward-User directly (API Gateway pattern)
|
|
||||||
tracing::info!("Using X-Forward-User for LLM auth: {}", user);
|
|
||||||
format!("X-Forward-User: {}", user)
|
|
||||||
} else if let Some(jwt_issuer) = &self.jwt_issuer {
|
|
||||||
let issuer = jwt_issuer.lock().await;
|
let issuer = jwt_issuer.lock().await;
|
||||||
match issuer.get_access_token().await {
|
match issuer.get_access_token().await {
|
||||||
Ok(token) => {
|
Ok(token) => format!("Bearer {}", token),
|
||||||
tracing::info!("Using Authentik JWT for LLM auth");
|
|
||||||
format!("Bearer {}", token)
|
|
||||||
},
|
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::warn!("Failed to get Authentik JWT: {}", e);
|
tracing::warn!("Failed to get Authentik JWT: {}", e);
|
||||||
// Fallback to env var
|
return Err(e);
|
||||||
let api_key = std::env::var("LLM_API_KEY")
|
|
||||||
.or_else(|_| std::env::var("MEM_API_KEY"))
|
|
||||||
.unwrap_or_else(|_| "test-key".to_string());
|
|
||||||
tracing::info!("Falling back to LLM_API_KEY");
|
|
||||||
format!("Bearer {}", api_key)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Fallback to env var if Authentik not configured
|
// Fallback to env var if Authentik not configured
|
||||||
let api_key = std::env::var("LLM_API_KEY")
|
let api_key = std::env::var("LLM_API_KEY")
|
||||||
.or_else(|_| std::env::var("MEM_API_KEY"))
|
.or_else(|_| std::env::var("MEM_API_KEY"))
|
||||||
.unwrap_or_else(|_| "test-key".to_string());
|
.unwrap_or_else(|_| "default-key".to_string());
|
||||||
tracing::info!("Using LLM_API_KEY for LLM auth");
|
|
||||||
format!("Bearer {}", api_key)
|
format!("Bearer {}", api_key)
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -174,61 +123,35 @@ impl LlmEntityExtractor {
|
|||||||
{"role": "user", "content": prompt}
|
{"role": "user", "content": prompt}
|
||||||
],
|
],
|
||||||
"temperature": 0.3,
|
"temperature": 0.3,
|
||||||
"max_tokens": 12000
|
"max_tokens": 500
|
||||||
});
|
});
|
||||||
|
|
||||||
let mut request = client
|
let response = client
|
||||||
.post(&endpoint)
|
.post(&endpoint)
|
||||||
.header("Content-Type", "application/json");
|
.header("Authorization", auth_header)
|
||||||
|
.header("Content-Type", "application/json")
|
||||||
// Set auth header (varies by auth method)
|
|
||||||
if auth_header.starts_with("X-Forward-User") {
|
|
||||||
request = request.header("X-Forward-User", auth_header.split(": ").nth(1).unwrap_or("unknown"));
|
|
||||||
} else {
|
|
||||||
request = request.header("Authorization", auth_header);
|
|
||||||
}
|
|
||||||
|
|
||||||
let response = request
|
|
||||||
.json(&payload)
|
.json(&payload)
|
||||||
.timeout(std::time::Duration::from_secs(90))
|
.timeout(std::time::Duration::from_secs(30))
|
||||||
.send()
|
.send()
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
let status = response.status();
|
if !response.status().is_success() {
|
||||||
if !status.is_success() {
|
tracing::warn!(
|
||||||
let error_text = response.text().await.unwrap_or_default();
|
|
||||||
tracing::error!(
|
|
||||||
"LLM API error: {} - {}",
|
"LLM API error: {} - {}",
|
||||||
status,
|
response.status(),
|
||||||
error_text
|
response.text().await.unwrap_or_default()
|
||||||
);
|
);
|
||||||
// Return error instead of silently returning empty array
|
// Fallback to mock response on error
|
||||||
return Err(anyhow::anyhow!("LLM API failed with status {}: {}", status, error_text));
|
return Ok(r#"{"entities": []}"#.to_string());
|
||||||
}
|
}
|
||||||
|
|
||||||
let data: serde_json::Value = response.json().await?;
|
let data: serde_json::Value = response.json().await?;
|
||||||
// Extract content — some models put JSON in "content", others in "reasoning"
|
let content = data["choices"][0]["message"]["content"]
|
||||||
let msg = &data["choices"][0]["message"];
|
.as_str()
|
||||||
let raw_content = msg["content"].as_str().unwrap_or("").to_string();
|
.unwrap_or("{}")
|
||||||
let raw_reasoning = msg["reasoning"].as_str().unwrap_or("").to_string();
|
.to_string();
|
||||||
|
|
||||||
// Use content if non-empty, otherwise try reasoning field
|
tracing::debug!("LLM response (via Authentik JWT): {}", content);
|
||||||
let raw = if !raw_content.trim().is_empty() { &raw_content } else { &raw_reasoning };
|
|
||||||
let content = Self::clean_llm_response(raw);
|
|
||||||
|
|
||||||
let tokens = &data["usage"];
|
|
||||||
tracing::info!(
|
|
||||||
target: "observability",
|
|
||||||
event = "llm_entity_call",
|
|
||||||
model = %model,
|
|
||||||
endpoint = %endpoint,
|
|
||||||
raw_len = raw.len(),
|
|
||||||
cleaned_len = content.len(),
|
|
||||||
prompt_tokens = %tokens["prompt_tokens"],
|
|
||||||
completion_tokens = %tokens["completion_tokens"],
|
|
||||||
has_reasoning = !raw_reasoning.is_empty(),
|
|
||||||
"LLM entity extraction call complete"
|
|
||||||
);
|
|
||||||
Ok(content)
|
Ok(content)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -285,10 +208,7 @@ Respond in JSON:
|
|||||||
|
|
||||||
// Try real LLM first, fallback to mock if not configured
|
// Try real LLM first, fallback to mock if not configured
|
||||||
let extraction_response = if std::env::var("LLM_ENDPOINT").is_ok() {
|
let extraction_response = if std::env::var("LLM_ENDPOINT").is_ok() {
|
||||||
self.call_llm_endpoint(&prompt, None).await.unwrap_or_else(|e| {
|
self.call_llm_endpoint(&prompt).await.unwrap_or_else(|_| self.simulate_llm(&prompt).unwrap_or_default())
|
||||||
tracing::error!("LLM entity extraction failed: {}, using mock", e);
|
|
||||||
self.simulate_llm(&prompt).unwrap_or_default()
|
|
||||||
})
|
|
||||||
} else {
|
} else {
|
||||||
self.simulate_llm(&prompt)?
|
self.simulate_llm(&prompt)?
|
||||||
};
|
};
|
||||||
@@ -313,27 +233,14 @@ Respond in JSON:
|
|||||||
);
|
);
|
||||||
|
|
||||||
let reflection = if std::env::var("LLM_ENDPOINT").is_ok() {
|
let reflection = if std::env::var("LLM_ENDPOINT").is_ok() {
|
||||||
self.call_llm_endpoint(&reflection_prompt, None).await.unwrap_or_else(|e| {
|
self.call_llm_endpoint(&reflection_prompt).await.unwrap_or_else(|_| self.simulate_llm(&reflection_prompt).unwrap_or_default())
|
||||||
tracing::warn!("Reflection LLM call failed: {}, skipping verification", e);
|
|
||||||
String::new()
|
|
||||||
})
|
|
||||||
} else {
|
} else {
|
||||||
self.simulate_llm(&reflection_prompt)?
|
self.simulate_llm(&reflection_prompt)?
|
||||||
};
|
};
|
||||||
|
let verified = Self::parse_reflection(&reflection)?;
|
||||||
|
|
||||||
// If reflection succeeded, filter entities; otherwise keep all
|
// Filter: keep only entities marked present
|
||||||
if !reflection.is_empty() {
|
entities.retain(|e| verified.iter().any(|(name, present)| name == &e.name && *present));
|
||||||
match Self::parse_reflection(&reflection) {
|
|
||||||
Ok(verified) => {
|
|
||||||
entities.retain(|e| verified.iter().any(|(name, present)| name == &e.name && *present));
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
tracing::warn!("Reflection parse failed: {}, keeping all entities", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
tracing::info!("Reflection skipped, keeping {} unverified entities", entities.len());
|
|
||||||
}
|
|
||||||
|
|
||||||
// Adjust confidence for reflected entities (slight penalty for needing verification)
|
// Adjust confidence for reflected entities (slight penalty for needing verification)
|
||||||
for entity in &mut entities {
|
for entity in &mut entities {
|
||||||
@@ -343,85 +250,6 @@ Respond in JSON:
|
|||||||
|
|
||||||
Ok(entities)
|
Ok(entities)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Extract with X-Forward-User auth header (API Gateway pattern)
|
|
||||||
async fn extract_with_auth(&self, text: &str, x_forward_user: Option<&str>) -> Result<Vec<ExtractedEntity>> {
|
|
||||||
let mut entities = vec![];
|
|
||||||
|
|
||||||
// Extract speaker if available
|
|
||||||
use crate::speaker_extractor::{HeuristicSpeakerExtractor, SpeakerConfig};
|
|
||||||
if let Ok(speaker_extractor) = HeuristicSpeakerExtractor::new(SpeakerConfig::default()) {
|
|
||||||
if let Ok(Some(speaker)) = speaker_extractor.extract_speaker(text).await {
|
|
||||||
entities.push(ExtractedEntity {
|
|
||||||
name: speaker.name,
|
|
||||||
entity_type: mem_core::entity::EntityType::Person,
|
|
||||||
summary: "Speaker in this episode".to_string(),
|
|
||||||
confidence: speaker.confidence,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Extract entities with auth header
|
|
||||||
let prompt = format!(
|
|
||||||
r#"Extract named entities from this text.
|
|
||||||
|
|
||||||
For each entity provide:
|
|
||||||
- name: Canonical name (proper capitalization)
|
|
||||||
- type: One of [person, tool, concept, location, event, organization]
|
|
||||||
- summary: One sentence
|
|
||||||
|
|
||||||
CRITICAL: Only extract entities EXPLICITLY mentioned. No inference.
|
|
||||||
|
|
||||||
Text:
|
|
||||||
"{}"
|
|
||||||
|
|
||||||
Respond in JSON:
|
|
||||||
{{"entities": [{{"name": "...", "type": "...", "summary": "..."}}, ...]}}
|
|
||||||
"#,
|
|
||||||
text
|
|
||||||
);
|
|
||||||
|
|
||||||
// Use provided X-Forward-User for auth
|
|
||||||
let extraction_response = if std::env::var("LLM_ENDPOINT").is_ok() {
|
|
||||||
self.call_llm_endpoint(&prompt, x_forward_user).await.unwrap_or_else(|e| {
|
|
||||||
tracing::error!("LLM entity extraction with auth failed: {}", e);
|
|
||||||
self.simulate_llm(&prompt).unwrap_or_default()
|
|
||||||
})
|
|
||||||
} else {
|
|
||||||
self.simulate_llm(&prompt)?
|
|
||||||
};
|
|
||||||
|
|
||||||
let extracted = Self::parse_extraction(&extraction_response)?;
|
|
||||||
entities.extend(extracted);
|
|
||||||
|
|
||||||
// Optional: reflection verification with auth
|
|
||||||
if self.enable_reflection && std::env::var("LLM_ENDPOINT").is_ok() {
|
|
||||||
let reflection_prompt = format!(
|
|
||||||
r#"Verify these entities are explicitly in the text:
|
|
||||||
|
|
||||||
Text:
|
|
||||||
"{}"
|
|
||||||
|
|
||||||
Entities:
|
|
||||||
{:?}
|
|
||||||
|
|
||||||
Respond in JSON:
|
|
||||||
{{"verified": [{{"name": "...", "present": true/false}}, ...]}}
|
|
||||||
"#,
|
|
||||||
text, entities
|
|
||||||
);
|
|
||||||
|
|
||||||
if let Ok(reflection) = self.call_llm_endpoint(&reflection_prompt, x_forward_user).await {
|
|
||||||
if !reflection.is_empty() {
|
|
||||||
if let Ok(verified) = Self::parse_reflection(&reflection) {
|
|
||||||
entities.retain(|e| verified.iter().any(|(name, present)| name == &e.name && *present));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(entities)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Fallback extractor: Use wiki_links if LLM fails (stage 3)
|
/// Fallback extractor: Use wiki_links if LLM fails (stage 3)
|
||||||
@@ -440,7 +268,7 @@ impl EntityExtractor for WikiLinkFallbackExtractor {
|
|||||||
entities.push(ExtractedEntity {
|
entities.push(ExtractedEntity {
|
||||||
name: name_str.to_string(),
|
name: name_str.to_string(),
|
||||||
entity_type: EntityType::Unknown,
|
entity_type: EntityType::Unknown,
|
||||||
summary: "Mentioned in episode".to_string(),
|
summary: format!("Mentioned in episode"),
|
||||||
confidence: 0.7, // Lower confidence for fallback
|
confidence: 0.7, // Lower confidence for fallback
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -528,6 +356,6 @@ mod tests {
|
|||||||
let text = "[[Entity1]] and [[Entity2]]";
|
let text = "[[Entity1]] and [[Entity2]]";
|
||||||
|
|
||||||
let entities = composite.extract(text).await.unwrap();
|
let entities = composite.extract(text).await.unwrap();
|
||||||
assert!(!entities.is_empty());
|
assert!(entities.len() > 0);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,12 +1,12 @@
|
|||||||
//! Fact extraction: Identify relationships between entities
|
//! Fact extraction: Identify relationships between entities
|
||||||
//!
|
//!
|
||||||
//! Three implementations:
|
//! Two implementations:
|
||||||
//! 1. SimpleFactExtractor: Pattern-based (verbs + wiki links)
|
//! 1. SimpleFactExtractor: Pattern-based (verbs + wiki links)
|
||||||
//! 2. LlmFactExtractor: LLM-based extraction with entity context
|
//! 2. LlmFactExtractor: LLM-based (placeholder for production)
|
||||||
//! 3. Fallback chain: LLM → Simple pattern matching
|
|
||||||
//!
|
//!
|
||||||
//! Aligned with Zep paper §2.2.2: Facts as edges between entity pairs,
|
//! CRAP: 12 (Simple pattern matching + LLM placeholder)
|
||||||
//! with temporal extraction and dedup against existing edges.
|
//! SOLID: Trait-based (Open/Closed)
|
||||||
|
//! DRY: Reuses EntityExtractor pattern
|
||||||
|
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
@@ -27,18 +27,20 @@ pub struct ExtractedFact {
|
|||||||
pub trait FactExtractor: Send + Sync {
|
pub trait FactExtractor: Send + Sync {
|
||||||
async fn extract(&self, text: &str) -> Result<Vec<ExtractedFact>>;
|
async fn extract(&self, text: &str) -> Result<Vec<ExtractedFact>>;
|
||||||
|
|
||||||
/// Extract facts with entity context (Zep §2.2.2: facts between known entities)
|
/// Extract facts with GRM context (optional, defaults to extract())
|
||||||
async fn extract_with_context(
|
async fn extract_with_context(
|
||||||
&self,
|
&self,
|
||||||
text: &str,
|
text: &str,
|
||||||
_entity_contexts: &[crate::grm_retriever::EntityContext],
|
_entity_contexts: &[crate::grm_retriever::EntityContext],
|
||||||
) -> Result<Vec<ExtractedFact>> {
|
) -> Result<Vec<ExtractedFact>> {
|
||||||
|
// Default: ignore context, use plain extraction
|
||||||
self.extract(text).await
|
self.extract(text).await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Simple fact extractor based on verb patterns
|
/// Simple fact extractor based on verb patterns
|
||||||
/// Pattern: [[Entity1]] verb [[Entity2]]
|
/// Pattern: [[Entity1]] verb [[Entity2]]
|
||||||
|
/// Common verbs: uses, manages, runs, deployed_to, works_with
|
||||||
pub struct SimpleFactExtractor;
|
pub struct SimpleFactExtractor;
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
@@ -46,15 +48,17 @@ impl FactExtractor for SimpleFactExtractor {
|
|||||||
async fn extract(&self, text: &str) -> Result<Vec<ExtractedFact>> {
|
async fn extract(&self, text: &str) -> Result<Vec<ExtractedFact>> {
|
||||||
let mut facts = vec![];
|
let mut facts = vec![];
|
||||||
|
|
||||||
|
// Extract [[Entity]] patterns
|
||||||
let entity_pattern = Regex::new(r"\[\[([^\]]+)\]\]")?;
|
let entity_pattern = Regex::new(r"\[\[([^\]]+)\]\]")?;
|
||||||
let _entities: Vec<String> = entity_pattern
|
let entities: Vec<String> = entity_pattern
|
||||||
.captures_iter(text)
|
.captures_iter(text)
|
||||||
.filter_map(|cap| cap.get(1).map(|m| m.as_str().to_string()))
|
.filter_map(|cap| cap.get(1).map(|m| m.as_str().to_string()))
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
let verbs = ["uses", "manages", "runs", "deployed_to", "works_with",
|
// Common relationship verbs
|
||||||
"depends_on", "contains", "extends", "implements", "connects_to"];
|
let verbs = ["uses", "manages", "runs", "deployed_to", "works_with"];
|
||||||
|
|
||||||
|
// Simple heuristic: if two entities appear close together with a verb between them
|
||||||
for verb in &verbs {
|
for verb in &verbs {
|
||||||
let pattern = format!(
|
let pattern = format!(
|
||||||
r"\[\[([^\]]+)\]\].*?{}.*?\[\[([^\]]+)\]\]",
|
r"\[\[([^\]]+)\]\].*?{}.*?\[\[([^\]]+)\]\]",
|
||||||
@@ -67,7 +71,12 @@ impl FactExtractor for SimpleFactExtractor {
|
|||||||
source_entity_id: src.as_str().to_string(),
|
source_entity_id: src.as_str().to_string(),
|
||||||
target_entity_id: tgt.as_str().to_string(),
|
target_entity_id: tgt.as_str().to_string(),
|
||||||
relation_type: verb.to_uppercase(),
|
relation_type: verb.to_uppercase(),
|
||||||
fact: format!("{} {} {}", src.as_str(), verb, tgt.as_str()),
|
fact: format!(
|
||||||
|
"{} {} {}",
|
||||||
|
src.as_str(),
|
||||||
|
verb,
|
||||||
|
tgt.as_str()
|
||||||
|
),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -78,251 +87,18 @@ impl FactExtractor for SimpleFactExtractor {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// LLM-based fact extractor (Zep §2.2.2 alignment)
|
/// LLM-based fact extractor (placeholder for production)
|
||||||
/// Extracts relationships between entity pairs using LLM
|
/// TODO (Phase 2.6): Implement with real LLM API
|
||||||
pub struct LlmFactExtractor {
|
/// TODO (Phase 2.6): Support complex relationships (3-way, temporal, conditional)
|
||||||
model_name: String,
|
pub struct LlmFactExtractor;
|
||||||
jwt_issuer: Option<std::sync::Arc<tokio::sync::Mutex<crate::authentik_jwt::AuthentikJwtIssuer>>>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl LlmFactExtractor {
|
|
||||||
pub fn new(model_name: &str) -> Self {
|
|
||||||
let jwt_issuer = crate::authentik_jwt::AuthentikJwtIssuer::from_env().ok();
|
|
||||||
Self {
|
|
||||||
model_name: model_name.to_string(),
|
|
||||||
jwt_issuer: jwt_issuer.map(|iss| std::sync::Arc::new(tokio::sync::Mutex::new(iss))),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Clean LLM response: strip thinking tags, markdown fences, extract JSON
|
|
||||||
fn clean_llm_response(text: &str) -> String {
|
|
||||||
let mut result = text.to_string();
|
|
||||||
while let Some(start) = result.find("<think>") {
|
|
||||||
if let Some(end) = result.find("</think>") {
|
|
||||||
result = format!("{}{}", &result[..start], &result[end + 8..]);
|
|
||||||
} else { break; }
|
|
||||||
}
|
|
||||||
result = result.replace("```json", "").replace("```", "");
|
|
||||||
let trimmed = result.trim();
|
|
||||||
if let Some(start) = trimmed.find('{') {
|
|
||||||
if let Some(end) = trimmed.rfind('}') {
|
|
||||||
return trimmed[start..=end].to_string();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if let Some(start) = trimmed.find('[') {
|
|
||||||
if let Some(end) = trimmed.rfind(']') {
|
|
||||||
return format!("{{\"facts\": {}}}", &trimmed[start..=end]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
trimmed.to_string()
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn call_llm(&self, prompt: &str) -> Result<String> {
|
|
||||||
let endpoint = std::env::var("LLM_ENDPOINT")
|
|
||||||
.unwrap_or_else(|_| "http://localhost:11434/v1/chat/completions".to_string());
|
|
||||||
|
|
||||||
// Get auth header: Authentik JWT if configured, else API key
|
|
||||||
let auth_header = if let Some(jwt_issuer) = &self.jwt_issuer {
|
|
||||||
let issuer = jwt_issuer.lock().await;
|
|
||||||
match issuer.get_access_token().await {
|
|
||||||
Ok(token) => format!("Bearer {}", token),
|
|
||||||
Err(e) => {
|
|
||||||
tracing::warn!(target: "observability", event = "fact_jwt_fallback", error = %e, "JWT failed, using API key");
|
|
||||||
let key = std::env::var("LLM_API_KEY").unwrap_or_else(|_| "default-key".to_string());
|
|
||||||
format!("Bearer {}", key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
let key = std::env::var("LLM_API_KEY")
|
|
||||||
.or_else(|_| std::env::var("MEM_API_KEY"))
|
|
||||||
.unwrap_or_else(|_| "default-key".to_string());
|
|
||||||
format!("Bearer {}", key)
|
|
||||||
};
|
|
||||||
|
|
||||||
let start = std::time::Instant::now();
|
|
||||||
let client = reqwest::Client::new();
|
|
||||||
let payload = serde_json::json!({
|
|
||||||
"model": self.model_name,
|
|
||||||
"messages": [
|
|
||||||
{"role": "system", "content": "You are a fact extraction specialist. Extract relationships between entities from text. Output ONLY valid JSON."},
|
|
||||||
{"role": "user", "content": prompt}
|
|
||||||
],
|
|
||||||
"max_tokens": 12000,
|
|
||||||
"temperature": 0.1
|
|
||||||
});
|
|
||||||
|
|
||||||
let response = client
|
|
||||||
.post(&endpoint)
|
|
||||||
.header("Authorization", &auth_header)
|
|
||||||
.header("Content-Type", "application/json")
|
|
||||||
.json(&payload)
|
|
||||||
.timeout(std::time::Duration::from_secs(120))
|
|
||||||
.send()
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
let status = response.status();
|
|
||||||
if !status.is_success() {
|
|
||||||
let body = response.text().await.unwrap_or_default();
|
|
||||||
tracing::warn!(target: "observability", event = "fact_llm_error", status = %status, body = %body, "Fact LLM call failed");
|
|
||||||
return Err(anyhow::anyhow!("LLM API error: {}", status));
|
|
||||||
}
|
|
||||||
|
|
||||||
let elapsed = start.elapsed();
|
|
||||||
let data: serde_json::Value = response.json().await?;
|
|
||||||
|
|
||||||
// Handle both content and reasoning fields (ornith uses reasoning)
|
|
||||||
let msg = &data["choices"][0]["message"];
|
|
||||||
let raw_content = msg["content"].as_str().unwrap_or("").to_string();
|
|
||||||
let raw_reasoning = msg["reasoning"].as_str().unwrap_or("").to_string();
|
|
||||||
let raw = if !raw_content.trim().is_empty() { &raw_content } else { &raw_reasoning };
|
|
||||||
let cleaned = Self::clean_llm_response(raw);
|
|
||||||
|
|
||||||
let tokens = &data["usage"];
|
|
||||||
tracing::info!(
|
|
||||||
target: "observability",
|
|
||||||
event = "llm_fact_call",
|
|
||||||
model = %self.model_name,
|
|
||||||
endpoint = %endpoint,
|
|
||||||
raw_len = raw.len(),
|
|
||||||
cleaned_len = cleaned.len(),
|
|
||||||
prompt_tokens = %tokens["prompt_tokens"],
|
|
||||||
completion_tokens = %tokens["completion_tokens"],
|
|
||||||
duration_ms = elapsed.as_millis() as u64,
|
|
||||||
has_reasoning = !raw_reasoning.is_empty(),
|
|
||||||
"LLM fact extraction call complete"
|
|
||||||
);
|
|
||||||
Ok(cleaned)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl FactExtractor for LlmFactExtractor {
|
impl FactExtractor for LlmFactExtractor {
|
||||||
async fn extract(&self, text: &str) -> Result<Vec<ExtractedFact>> {
|
async fn extract(&self, _text: &str) -> Result<Vec<ExtractedFact>> {
|
||||||
self.extract_with_context(text, &[]).await
|
// TODO (Phase 2.6): Implement LLM-based extraction
|
||||||
}
|
// Pattern: Send text to api.riotpiao.com with prompt
|
||||||
|
// Parse response for [source, relation, target] tuples
|
||||||
async fn extract_with_context(
|
Ok(vec![])
|
||||||
&self,
|
|
||||||
text: &str,
|
|
||||||
entity_contexts: &[crate::grm_retriever::EntityContext],
|
|
||||||
) -> Result<Vec<ExtractedFact>> {
|
|
||||||
// Build entity list for prompt
|
|
||||||
let entity_names: Vec<&str> = entity_contexts
|
|
||||||
.iter()
|
|
||||||
.map(|e| e.entity_name.as_str())
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
if entity_names.is_empty() {
|
|
||||||
tracing::debug!("No entities provided, skipping fact extraction");
|
|
||||||
return Ok(vec![]);
|
|
||||||
}
|
|
||||||
|
|
||||||
let prompt = format!(
|
|
||||||
r#"Extract relationships (facts) between these entities from the text.
|
|
||||||
|
|
||||||
Entities: {:?}
|
|
||||||
|
|
||||||
Text:
|
|
||||||
"{}"
|
|
||||||
|
|
||||||
For each relationship provide:
|
|
||||||
- source: Entity name (must be from the list above)
|
|
||||||
- target: Entity name (must be from the list above)
|
|
||||||
- relation: Verb/predicate describing the relationship (e.g., "uses", "manages", "is_part_of", "deployed_on")
|
|
||||||
- fact: One-sentence natural language description
|
|
||||||
|
|
||||||
CRITICAL: Only extract relationships EXPLICITLY stated or strongly implied. Source and target must both be from the entity list.
|
|
||||||
|
|
||||||
Respond in JSON:
|
|
||||||
{{"facts": [{{"source": "...", "target": "...", "relation": "...", "fact": "..."}}, ...]}}
|
|
||||||
"#,
|
|
||||||
entity_names, text
|
|
||||||
);
|
|
||||||
|
|
||||||
let llm_ok = std::env::var("LLM_ENDPOINT").is_ok();
|
|
||||||
let response = if llm_ok {
|
|
||||||
match self.call_llm(&prompt).await {
|
|
||||||
Ok(r) => r,
|
|
||||||
Err(e) => {
|
|
||||||
tracing::warn!("Fact extraction LLM failed: {}, returning empty", e);
|
|
||||||
return Ok(vec![]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
tracing::debug!("LLM_ENDPOINT not set, skipping LLM fact extraction");
|
|
||||||
return Ok(vec![]);
|
|
||||||
};
|
|
||||||
|
|
||||||
// Parse response
|
|
||||||
#[derive(Deserialize)]
|
|
||||||
struct FactResponse {
|
|
||||||
facts: Vec<RawFact>,
|
|
||||||
}
|
|
||||||
#[derive(Deserialize)]
|
|
||||||
struct RawFact {
|
|
||||||
source: String,
|
|
||||||
target: String,
|
|
||||||
relation: String,
|
|
||||||
fact: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
// Try parsing, if trailing chars error try trimming to valid JSON
|
|
||||||
let parsed = match serde_json::from_str::<FactResponse>(&response) {
|
|
||||||
Ok(r) => Ok(r),
|
|
||||||
Err(e) if e.to_string().contains("trailing") => {
|
|
||||||
// Find the closing of the top-level object and retry
|
|
||||||
let mut depth = 0i32;
|
|
||||||
let mut end = 0;
|
|
||||||
for (i, c) in response.char_indices() {
|
|
||||||
match c {
|
|
||||||
'{' | '[' => depth += 1,
|
|
||||||
'}' | ']' => { depth -= 1; if depth == 0 { end = i + 1; break; } },
|
|
||||||
_ => {}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if end > 0 {
|
|
||||||
serde_json::from_str::<FactResponse>(&response[..end])
|
|
||||||
} else {
|
|
||||||
Err(e)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(e) => Err(e),
|
|
||||||
};
|
|
||||||
match parsed {
|
|
||||||
Ok(parsed) => {
|
|
||||||
let facts: Vec<ExtractedFact> = parsed.facts
|
|
||||||
.into_iter()
|
|
||||||
.filter(|f| {
|
|
||||||
// Validate source and target are known entities
|
|
||||||
let src_ok = entity_names.iter().any(|e| e.eq_ignore_ascii_case(&f.source));
|
|
||||||
let tgt_ok = entity_names.iter().any(|e| e.eq_ignore_ascii_case(&f.target));
|
|
||||||
if !src_ok || !tgt_ok {
|
|
||||||
tracing::debug!(
|
|
||||||
"Dropping fact with unknown entity: {} -> {}",
|
|
||||||
f.source, f.target
|
|
||||||
);
|
|
||||||
}
|
|
||||||
src_ok && tgt_ok && f.source != f.target
|
|
||||||
})
|
|
||||||
.map(|f| ExtractedFact {
|
|
||||||
source_entity_id: f.source,
|
|
||||||
target_entity_id: f.target,
|
|
||||||
relation_type: f.relation.to_uppercase(),
|
|
||||||
fact: f.fact,
|
|
||||||
})
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
tracing::info!(
|
|
||||||
"LLM fact extraction: {} facts from {} entities",
|
|
||||||
facts.len(), entity_names.len()
|
|
||||||
);
|
|
||||||
Ok(facts)
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
tracing::warn!("Fact extraction JSON parse failed: {}", e);
|
|
||||||
Ok(vec![])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -334,38 +110,9 @@ mod tests {
|
|||||||
async fn test_simple_fact_extraction() {
|
async fn test_simple_fact_extraction() {
|
||||||
let extractor = SimpleFactExtractor;
|
let extractor = SimpleFactExtractor;
|
||||||
let text = "[[Rock]] uses [[Kubernetes]] and [[ArgoCD]]";
|
let text = "[[Rock]] uses [[Kubernetes]] and [[ArgoCD]]";
|
||||||
|
|
||||||
let facts = extractor.extract(text).await.unwrap();
|
let facts = extractor.extract(text).await.unwrap();
|
||||||
assert!(!facts.is_empty());
|
assert!(facts.len() > 0);
|
||||||
assert!(facts.iter().any(|f| f.relation_type == "USES"));
|
assert!(facts.iter().any(|f| f.relation_type == "USES"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn test_simple_no_wiki_links() {
|
|
||||||
let extractor = SimpleFactExtractor;
|
|
||||||
let text = "Kubernetes uses etcd for storage";
|
|
||||||
let facts = extractor.extract(text).await.unwrap();
|
|
||||||
assert!(facts.is_empty()); // No [[wiki links]]
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_clean_llm_response() {
|
|
||||||
let input = r#"<think>reasoning here</think>{"facts": [{"source": "A", "target": "B", "relation": "uses", "fact": "A uses B"}]}"#;
|
|
||||||
let cleaned = LlmFactExtractor::clean_llm_response(input);
|
|
||||||
assert!(cleaned.starts_with("{"));
|
|
||||||
assert!(cleaned.contains("facts"));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_strip_thinking_no_tags() {
|
|
||||||
let input = r#"{"facts": []}"#;
|
|
||||||
let cleaned = LlmFactExtractor::clean_llm_response(input);
|
|
||||||
assert_eq!(cleaned, input);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn test_llm_fact_no_entities_returns_empty() {
|
|
||||||
let extractor = LlmFactExtractor::new("test");
|
|
||||||
let facts = extractor.extract_with_context("some text", &[]).await.unwrap();
|
|
||||||
assert!(facts.is_empty());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,7 +10,10 @@
|
|||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use tracing::debug;
|
use std::collections::HashMap;
|
||||||
|
use tracing::{debug, info};
|
||||||
|
use mem_core::entity::Entity;
|
||||||
|
use mem_core::edge::Edge;
|
||||||
|
|
||||||
/// Memorability decision for entity or fact
|
/// Memorability decision for entity or fact
|
||||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq)]
|
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq)]
|
||||||
|
|||||||
@@ -59,15 +59,10 @@ impl IngestPipeline {
|
|||||||
/// Execute extraction pipeline for episode
|
/// Execute extraction pipeline for episode
|
||||||
/// CRAP: 14 (Low: orchestration only, delegates to stages)
|
/// CRAP: 14 (Low: orchestration only, delegates to stages)
|
||||||
pub async fn ingest(&self, episode: &Episode) -> Result<ExtractionResult> {
|
pub async fn ingest(&self, episode: &Episode) -> Result<ExtractionResult> {
|
||||||
self.ingest_with_auth(episode, None).await
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Ingest with optional X-Forward-User auth header
|
|
||||||
pub async fn ingest_with_auth(&self, episode: &Episode, x_forward_user: Option<&str>) -> Result<ExtractionResult> {
|
|
||||||
debug!("Starting ingest for episode: {}", episode.id);
|
debug!("Starting ingest for episode: {}", episode.id);
|
||||||
|
|
||||||
// Stage 1: Extract entities (with optional auth header)
|
// Stage 1: Extract entities
|
||||||
let extracted_entities = self.entity_extractor.extract_with_auth(&episode.text, x_forward_user).await?;
|
let extracted_entities = self.entity_extractor.extract(&episode.text).await?;
|
||||||
debug!("Extracted {} entities", extracted_entities.len());
|
debug!("Extracted {} entities", extracted_entities.len());
|
||||||
|
|
||||||
// Convert to domain entities
|
// Convert to domain entities
|
||||||
@@ -149,7 +144,6 @@ impl IngestPipeline {
|
|||||||
|
|
||||||
/// Async queue worker: Process episodes from queue
|
/// Async queue worker: Process episodes from queue
|
||||||
/// CRAP: 12 (Async loop, straightforward)
|
/// CRAP: 12 (Async loop, straightforward)
|
||||||
#[allow(dead_code)]
|
|
||||||
pub struct QueueWorker {
|
pub struct QueueWorker {
|
||||||
pipeline: Arc<IngestPipeline>,
|
pipeline: Arc<IngestPipeline>,
|
||||||
batch_size: usize,
|
batch_size: usize,
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ use tracing::{debug, info};
|
|||||||
use crate::grm_retriever::{
|
use crate::grm_retriever::{
|
||||||
EntityContext, FactContext, GraphContextRetriever, MemorabilityDecision, GrmConfig, MockGrmRetriever,
|
EntityContext, FactContext, GraphContextRetriever, MemorabilityDecision, GrmConfig, MockGrmRetriever,
|
||||||
};
|
};
|
||||||
use mem_core::entity::Entity;
|
use mem_core::entity::{Entity, EntityType};
|
||||||
use mem_core::edge::Edge;
|
use mem_core::edge::Edge;
|
||||||
|
|
||||||
/// Entity filtering result
|
/// Entity filtering result
|
||||||
@@ -88,7 +88,7 @@ impl MemorabilityGate {
|
|||||||
let (filtered, reason) = match context.decision {
|
let (filtered, reason) = match context.decision {
|
||||||
MemorabilityDecision::Keep => {
|
MemorabilityDecision::Keep => {
|
||||||
if context.matched_entity_id.is_some() {
|
if context.matched_entity_id.is_some() {
|
||||||
(true, "Existing entity (merge required)".to_string())
|
(true, format!("Existing entity (merge required)"))
|
||||||
} else {
|
} else {
|
||||||
(false, format!("New entity (score: {:.2})", context.memorability_score))
|
(false, format!("New entity (score: {:.2})", context.memorability_score))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,7 +20,6 @@ pub struct RefMetadata {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Obsidian REST API client
|
/// Obsidian REST API client
|
||||||
#[allow(dead_code)]
|
|
||||||
pub struct ObsidianClient {
|
pub struct ObsidianClient {
|
||||||
base_url: String,
|
base_url: String,
|
||||||
}
|
}
|
||||||
@@ -48,7 +47,6 @@ impl ObsidianClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// ObsidianRefSource: Fetches & chunks reference documents from Obsidian vault
|
/// ObsidianRefSource: Fetches & chunks reference documents from Obsidian vault
|
||||||
#[allow(dead_code)]
|
|
||||||
pub struct ObsidianRefSource {
|
pub struct ObsidianRefSource {
|
||||||
client: ObsidianClient,
|
client: ObsidianClient,
|
||||||
project: String,
|
project: String,
|
||||||
@@ -70,13 +68,11 @@ impl ObsidianRefSource {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Check if a file path is allowed (matches configured prefixes)
|
/// Check if a file path is allowed (matches configured prefixes)
|
||||||
#[allow(dead_code)]
|
|
||||||
fn is_allowed_path(&self, path: &str) -> bool {
|
fn is_allowed_path(&self, path: &str) -> bool {
|
||||||
self.allowed_paths.iter().any(|prefix| path.starts_with(prefix))
|
self.allowed_paths.iter().any(|prefix| path.starts_with(prefix))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Chunk reference document via heading-boundary logic
|
/// Chunk reference document via heading-boundary logic
|
||||||
#[allow(dead_code)]
|
|
||||||
fn chunk_document(&self, path: &str, content: &str) -> Vec<Record> {
|
fn chunk_document(&self, path: &str, content: &str) -> Vec<Record> {
|
||||||
// M3.6.1 heading-boundary chunking
|
// M3.6.1 heading-boundary chunking
|
||||||
// - Split by headings
|
// - Split by headings
|
||||||
@@ -207,7 +203,7 @@ mod tests {
|
|||||||
let chunks = source.chunk_document("docs/test.md", content);
|
let chunks = source.chunk_document("docs/test.md", content);
|
||||||
|
|
||||||
// Should split by headings
|
// Should split by headings
|
||||||
assert!(!chunks.is_empty());
|
assert!(chunks.len() > 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -60,7 +60,8 @@ impl MetricsCollector {
|
|||||||
self.by_project
|
self.by_project
|
||||||
.lock()
|
.lock()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.get(project).cloned()
|
.get(project)
|
||||||
|
.map(|m| m.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get all project metrics.
|
/// Get all project metrics.
|
||||||
|
|||||||
@@ -306,7 +306,7 @@ impl QueryMetricsRepository {
|
|||||||
let mut repo = self.metrics.lock().unwrap();
|
let mut repo = self.metrics.lock().unwrap();
|
||||||
repo.get_mut(query_id)
|
repo.get_mut(query_id)
|
||||||
.ok_or_else(|| format!("Query {} not found", query_id))
|
.ok_or_else(|| format!("Query {} not found", query_id))
|
||||||
.map(f)
|
.map(|metrics| f(metrics))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get progress for a query
|
/// Get progress for a query
|
||||||
|
|||||||
@@ -4,10 +4,9 @@
|
|||||||
///
|
///
|
||||||
/// Used to scope queries to project namespaces and enable graph traversal.
|
/// Used to scope queries to project namespaces and enable graph traversal.
|
||||||
/// For example: poimen/tools/kubectl.md [[debugging.md]] creates an edge
|
/// For example: poimen/tools/kubectl.md [[debugging.md]] creates an edge
|
||||||
#[allow(clippy::empty_line_after_doc_comments)]
|
|
||||||
/// from tools/kubectl to debugging (within same project).
|
/// from tools/kubectl to debugging (within same project).
|
||||||
|
|
||||||
use anyhow::Result;
|
use anyhow::{anyhow, Result};
|
||||||
use regex::Regex;
|
use regex::Regex;
|
||||||
use std::collections::{HashMap, HashSet};
|
use std::collections::{HashMap, HashSet};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
@@ -80,7 +79,6 @@ impl WikiLinkParser {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Graph Index: Stores and queries wiki-link relationships
|
/// Graph Index: Stores and queries wiki-link relationships
|
||||||
#[allow(dead_code)]
|
|
||||||
pub struct WikiLinkGraph {
|
pub struct WikiLinkGraph {
|
||||||
/// Forward links: source -> [targets]
|
/// Forward links: source -> [targets]
|
||||||
forward_links: HashMap<String, Vec<String>>,
|
forward_links: HashMap<String, Vec<String>>,
|
||||||
@@ -102,11 +100,11 @@ impl WikiLinkGraph {
|
|||||||
/// Add a wiki-link edge
|
/// Add a wiki-link edge
|
||||||
pub fn add_link(&mut self, source: &str, target: &str) {
|
pub fn add_link(&mut self, source: &str, target: &str) {
|
||||||
self.forward_links.entry(source.to_string())
|
self.forward_links.entry(source.to_string())
|
||||||
.or_default()
|
.or_insert_with(Vec::new)
|
||||||
.push(target.to_string());
|
.push(target.to_string());
|
||||||
|
|
||||||
self.backward_links.entry(target.to_string())
|
self.backward_links.entry(target.to_string())
|
||||||
.or_default()
|
.or_insert_with(Vec::new)
|
||||||
.push(source.to_string());
|
.push(source.to_string());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ pub enum AuthMode {
|
|||||||
|
|
||||||
impl AuthMode {
|
impl AuthMode {
|
||||||
/// Detect from base URL or explicit env var.
|
/// Detect from base URL or explicit env var.
|
||||||
pub fn detect(_base_url: &str, api_key: &str) -> Self {
|
pub fn detect(base_url: &str, api_key: &str) -> Self {
|
||||||
if api_key.is_empty() {
|
if api_key.is_empty() {
|
||||||
return Self::None;
|
return Self::None;
|
||||||
}
|
}
|
||||||
@@ -87,7 +87,6 @@ struct Choice {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize)]
|
||||||
#[allow(dead_code)]
|
|
||||||
struct MessageResponse {
|
struct MessageResponse {
|
||||||
role: String,
|
role: String,
|
||||||
content: String,
|
content: String,
|
||||||
@@ -209,11 +208,12 @@ impl ChatClient {
|
|||||||
Ok(r) => r,
|
Ok(r) => r,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
last_error = Some(anyhow!("Request failed: {}", e));
|
last_error = Some(anyhow!("Request failed: {}", e));
|
||||||
if (e.is_timeout() || e.is_status())
|
if e.is_timeout() || e.is_status() {
|
||||||
&& attempt < self.max_retries - 1 {
|
if attempt < self.max_retries - 1 {
|
||||||
tokio::time::sleep(Duration::from_millis(100 * 2_u64.pow(attempt))).await;
|
tokio::time::sleep(Duration::from_millis(100 * 2_u64.pow(attempt))).await;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
return Err(last_error.unwrap());
|
return Err(last_error.unwrap());
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -27,7 +27,6 @@ struct EmbeddingRequest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize)]
|
||||||
#[allow(dead_code)]
|
|
||||||
#[serde(untagged)]
|
#[serde(untagged)]
|
||||||
enum EmbeddingResponse {
|
enum EmbeddingResponse {
|
||||||
Success {
|
Success {
|
||||||
@@ -43,7 +42,6 @@ enum EmbeddingResponse {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize)]
|
||||||
#[allow(dead_code)]
|
|
||||||
struct EmbeddingData {
|
struct EmbeddingData {
|
||||||
embedding: Vec<f32>,
|
embedding: Vec<f32>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
@@ -64,15 +62,9 @@ impl EmbeddingsClient {
|
|||||||
/// - `LLM_API_BASE`: Gateway endpoint (default: https://api.riotpiao.com)
|
/// - `LLM_API_BASE`: Gateway endpoint (default: https://api.riotpiao.com)
|
||||||
/// - `LLM_API_KEY`: API key (optional)
|
/// - `LLM_API_KEY`: API key (optional)
|
||||||
pub fn from_env() -> Result<Self> {
|
pub fn from_env() -> Result<Self> {
|
||||||
let mut base_url = env::var("LLM_API_BASE")
|
let base_url = env::var("LLM_API_BASE")
|
||||||
.unwrap_or_else(|_| "https://api.riotpiao.com".to_string());
|
.unwrap_or_else(|_| "https://api.riotpiao.com".to_string());
|
||||||
|
|
||||||
// Strip trailing /v1 to avoid double /v1/v1/embeddings
|
|
||||||
base_url = base_url.trim_end_matches('/').to_string();
|
|
||||||
if base_url.ends_with("/v1") {
|
|
||||||
base_url = base_url[..base_url.len() - 3].to_string();
|
|
||||||
}
|
|
||||||
|
|
||||||
let model = env::var("EMBEDDINGS_MODEL")
|
let model = env::var("EMBEDDINGS_MODEL")
|
||||||
.unwrap_or_else(|_| "nomic-ai/nomic-embed-text-v2-moe".to_string());
|
.unwrap_or_else(|_| "nomic-ai/nomic-embed-text-v2-moe".to_string());
|
||||||
|
|
||||||
@@ -128,10 +120,10 @@ impl EmbeddingsClient {
|
|||||||
/// Embed a single text string, returning a 768-dim vector
|
/// Embed a single text string, returning a 768-dim vector
|
||||||
pub async fn embed_one(&self, text: &str) -> Result<Vector> {
|
pub async fn embed_one(&self, text: &str) -> Result<Vector> {
|
||||||
let embeddings = self.embed(&[text.to_string()]).await?;
|
let embeddings = self.embed(&[text.to_string()]).await?;
|
||||||
embeddings
|
Ok(embeddings
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.next()
|
.next()
|
||||||
.ok_or_else(|| anyhow!("empty embedding response"))
|
.ok_or_else(|| anyhow!("empty embedding response"))?)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Embed multiple texts, batched at ≤32 per request, preserving input order
|
/// Embed multiple texts, batched at ≤32 per request, preserving input order
|
||||||
@@ -167,24 +159,15 @@ impl EmbeddingsClient {
|
|||||||
let url = format!("{}/v1/embeddings", self.base_url);
|
let url = format!("{}/v1/embeddings", self.base_url);
|
||||||
let mut builder = self.http.post(&url);
|
let mut builder = self.http.post(&url);
|
||||||
|
|
||||||
// Send as Bearer token (gateway expects Authorization: Bearer <key>)
|
// Send apikey header even though route currently doesn't require auth
|
||||||
|
// This future-proofs for when the route's auth plugin gets enabled
|
||||||
if !self.api_key.is_empty() {
|
if !self.api_key.is_empty() {
|
||||||
builder = builder.header("Authorization", format!("Bearer {}", &self.api_key));
|
builder = builder.header("apikey", &self.api_key);
|
||||||
}
|
}
|
||||||
|
|
||||||
let resp = builder.json(&req).send().await?;
|
let resp = builder.json(&req).send().await?;
|
||||||
let status = resp.status();
|
let _status = resp.status();
|
||||||
let raw_body = resp.text().await?;
|
let body: EmbeddingResponse = resp.json().await?;
|
||||||
|
|
||||||
if !status.is_success() {
|
|
||||||
tracing::error!("Embedding API returned {}: {}", status, &raw_body[..raw_body.len().min(500)]);
|
|
||||||
return Err(anyhow!("Embedding API returned {}: {}", status, &raw_body[..raw_body.len().min(200)]));
|
|
||||||
}
|
|
||||||
|
|
||||||
let body: EmbeddingResponse = serde_json::from_str(&raw_body).map_err(|e| {
|
|
||||||
tracing::error!("Failed to parse embedding response: {}. Raw body: {}", e, &raw_body[..raw_body.len().min(500)]);
|
|
||||||
anyhow!("Failed to parse embedding response: {}. Raw: {}", e, &raw_body[..raw_body.len().min(200)])
|
|
||||||
})?;
|
|
||||||
|
|
||||||
match body {
|
match body {
|
||||||
EmbeddingResponse::Error { error } => {
|
EmbeddingResponse::Error { error } => {
|
||||||
@@ -219,95 +202,4 @@ mod tests {
|
|||||||
assert_eq!(BATCH_SIZE, 32);
|
assert_eq!(BATCH_SIZE, 32);
|
||||||
assert_eq!(EMBEDDINGS_DIM, 768);
|
assert_eq!(EMBEDDINGS_DIM, 768);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_strip_trailing_v1() {
|
|
||||||
// Simulates LLM_API_BASE=https://api.riotpiao.com/v1
|
|
||||||
let mut base = "https://api.riotpiao.com/v1".to_string();
|
|
||||||
base = base.trim_end_matches('/').to_string();
|
|
||||||
if base.ends_with("/v1") {
|
|
||||||
base = base[..base.len() - 3].to_string();
|
|
||||||
}
|
|
||||||
assert_eq!(base, "https://api.riotpiao.com");
|
|
||||||
assert_eq!(format!("{}/v1/embeddings", base), "https://api.riotpiao.com/v1/embeddings");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_no_strip_when_no_v1() {
|
|
||||||
let mut base = "https://api.riotpiao.com".to_string();
|
|
||||||
base = base.trim_end_matches('/').to_string();
|
|
||||||
if base.ends_with("/v1") {
|
|
||||||
base = base[..base.len() - 3].to_string();
|
|
||||||
}
|
|
||||||
assert_eq!(base, "https://api.riotpiao.com");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_parse_real_embedding_response() {
|
|
||||||
// Exact format returned by embeddings-predictor service
|
|
||||||
let raw = r#"{"object":"list","data":[{"object":"embedding","embedding":[0.1,0.2,0.3],"index":0}],"model":"nomic-ai/nomic-embed-text-v2-moe","usage":{"prompt_tokens":3,"total_tokens":3}}"#;
|
|
||||||
let parsed: EmbeddingResponse = serde_json::from_str(raw).expect("should parse");
|
|
||||||
match parsed {
|
|
||||||
EmbeddingResponse::Success { data, .. } => {
|
|
||||||
assert_eq!(data.len(), 1);
|
|
||||||
assert_eq!(data[0].embedding.len(), 3);
|
|
||||||
assert_eq!(data[0].index, 0);
|
|
||||||
}
|
|
||||||
EmbeddingResponse::Error { error } => panic!("parsed as error: {:?}", error),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_parse_embedding_error_response() {
|
|
||||||
let raw = r#"{"error":"model not found"}"#;
|
|
||||||
let parsed: EmbeddingResponse = serde_json::from_str(raw).expect("should parse");
|
|
||||||
match parsed {
|
|
||||||
EmbeddingResponse::Error { error } => {
|
|
||||||
assert_eq!(error.as_str().unwrap(), "model not found");
|
|
||||||
}
|
|
||||||
EmbeddingResponse::Success { .. } => panic!("should be error"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_parse_768_dim_response() {
|
|
||||||
// 768 floats
|
|
||||||
let embedding: Vec<f32> = (0..768).map(|i| i as f32 * 0.001).collect();
|
|
||||||
let raw = format!(
|
|
||||||
r#"{{"object":"list","data":[{{"object":"embedding","embedding":{},"index":0}}],"model":"test","usage":{{}}}}"#,
|
|
||||||
serde_json::to_string(&embedding).unwrap()
|
|
||||||
);
|
|
||||||
let parsed: EmbeddingResponse = serde_json::from_str(&raw).expect("should parse 768-dim");
|
|
||||||
match parsed {
|
|
||||||
EmbeddingResponse::Success { data, .. } => {
|
|
||||||
assert_eq!(data[0].embedding.len(), 768);
|
|
||||||
}
|
|
||||||
_ => panic!("should be success"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_parse_html_fails_gracefully() {
|
|
||||||
// Simulates gateway returning HTML error page
|
|
||||||
let raw = "<html><body>502 Bad Gateway</body></html>";
|
|
||||||
let result: Result<EmbeddingResponse, _> = serde_json::from_str(raw);
|
|
||||||
assert!(result.is_err(), "HTML should fail to parse as JSON");
|
|
||||||
let err_msg = result.unwrap_err().to_string();
|
|
||||||
assert!(err_msg.contains("expected"), "Error should mention parsing: {}", err_msg);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_parse_multi_input_response() {
|
|
||||||
// Array input returns multiple embeddings
|
|
||||||
let raw = r#"{"object":"list","data":[{"object":"embedding","embedding":[0.1,0.2,0.3],"index":0},{"object":"embedding","embedding":[0.4,0.5,0.6],"index":1}],"model":"test","usage":{}}"#;
|
|
||||||
let parsed: EmbeddingResponse = serde_json::from_str(raw).expect("should parse");
|
|
||||||
match parsed {
|
|
||||||
EmbeddingResponse::Success { data, .. } => {
|
|
||||||
assert_eq!(data.len(), 2);
|
|
||||||
assert_eq!(data[0].index, 0);
|
|
||||||
assert_eq!(data[1].index, 1);
|
|
||||||
}
|
|
||||||
_ => panic!("should be success"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
Generated
+52
@@ -0,0 +1,52 @@
|
|||||||
|
{
|
||||||
|
"db_name": "PostgreSQL",
|
||||||
|
"query": "\n SELECT \n version_num,\n operation,\n snapshot,\n changed_at,\n changed_by,\n COALESCE(fields_changed, '{}') as \"fields_changed!\"\n FROM memory_entity_version\n WHERE entity_id = $1\n ORDER BY version_num DESC\n ",
|
||||||
|
"describe": {
|
||||||
|
"columns": [
|
||||||
|
{
|
||||||
|
"ordinal": 0,
|
||||||
|
"name": "version_num",
|
||||||
|
"type_info": "Int4"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 1,
|
||||||
|
"name": "operation",
|
||||||
|
"type_info": "Varchar"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 2,
|
||||||
|
"name": "snapshot",
|
||||||
|
"type_info": "Jsonb"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 3,
|
||||||
|
"name": "changed_at",
|
||||||
|
"type_info": "Timestamptz"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 4,
|
||||||
|
"name": "changed_by",
|
||||||
|
"type_info": "Varchar"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 5,
|
||||||
|
"name": "fields_changed!",
|
||||||
|
"type_info": "TextArray"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"parameters": {
|
||||||
|
"Left": [
|
||||||
|
"Text"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"nullable": [
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
null
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"hash": "1e81bb729531ca33e4cef21623bcfe4fafb0c1bd435353b205f582bfda8873bc"
|
||||||
|
}
|
||||||
Generated
+52
@@ -0,0 +1,52 @@
|
|||||||
|
{
|
||||||
|
"db_name": "PostgreSQL",
|
||||||
|
"query": "\n SELECT \n version_num,\n operation,\n snapshot,\n changed_at,\n changed_by,\n COALESCE(fields_changed, '{}') as \"fields_changed!\"\n FROM memory_edge_version\n WHERE edge_id = $1\n ORDER BY version_num DESC\n ",
|
||||||
|
"describe": {
|
||||||
|
"columns": [
|
||||||
|
{
|
||||||
|
"ordinal": 0,
|
||||||
|
"name": "version_num",
|
||||||
|
"type_info": "Int4"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 1,
|
||||||
|
"name": "operation",
|
||||||
|
"type_info": "Varchar"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 2,
|
||||||
|
"name": "snapshot",
|
||||||
|
"type_info": "Jsonb"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 3,
|
||||||
|
"name": "changed_at",
|
||||||
|
"type_info": "Timestamptz"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 4,
|
||||||
|
"name": "changed_by",
|
||||||
|
"type_info": "Varchar"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 5,
|
||||||
|
"name": "fields_changed!",
|
||||||
|
"type_info": "TextArray"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"parameters": {
|
||||||
|
"Left": [
|
||||||
|
"Uuid"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"nullable": [
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
null
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"hash": "62d65d4afc4d292b37de8e5cb59fbd51c602bdc1b437988f54e6c7fe268b9816"
|
||||||
|
}
|
||||||
Generated
+53
@@ -0,0 +1,53 @@
|
|||||||
|
{
|
||||||
|
"db_name": "PostgreSQL",
|
||||||
|
"query": "\n SELECT \n version_num,\n operation,\n snapshot,\n changed_at,\n changed_by,\n COALESCE(fields_changed, '{}') as \"fields_changed!\"\n FROM memory_entity_version\n WHERE entity_id = $1 AND changed_at <= $2\n ORDER BY version_num DESC\n LIMIT 1\n ",
|
||||||
|
"describe": {
|
||||||
|
"columns": [
|
||||||
|
{
|
||||||
|
"ordinal": 0,
|
||||||
|
"name": "version_num",
|
||||||
|
"type_info": "Int4"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 1,
|
||||||
|
"name": "operation",
|
||||||
|
"type_info": "Varchar"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 2,
|
||||||
|
"name": "snapshot",
|
||||||
|
"type_info": "Jsonb"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 3,
|
||||||
|
"name": "changed_at",
|
||||||
|
"type_info": "Timestamptz"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 4,
|
||||||
|
"name": "changed_by",
|
||||||
|
"type_info": "Varchar"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 5,
|
||||||
|
"name": "fields_changed!",
|
||||||
|
"type_info": "TextArray"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"parameters": {
|
||||||
|
"Left": [
|
||||||
|
"Text",
|
||||||
|
"Timestamptz"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"nullable": [
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
null
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"hash": "aee5900f5e3d7cbba23729bbf2dd033dcc4cb41f6c851bf447a9238810684d18"
|
||||||
|
}
|
||||||
Generated
+53
@@ -0,0 +1,53 @@
|
|||||||
|
{
|
||||||
|
"db_name": "PostgreSQL",
|
||||||
|
"query": "\n SELECT \n version_num,\n operation,\n snapshot,\n changed_at,\n changed_by,\n COALESCE(fields_changed, '{}') as \"fields_changed!\"\n FROM memory_entity_version\n WHERE entity_id = $1 AND version_num = $2\n ",
|
||||||
|
"describe": {
|
||||||
|
"columns": [
|
||||||
|
{
|
||||||
|
"ordinal": 0,
|
||||||
|
"name": "version_num",
|
||||||
|
"type_info": "Int4"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 1,
|
||||||
|
"name": "operation",
|
||||||
|
"type_info": "Varchar"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 2,
|
||||||
|
"name": "snapshot",
|
||||||
|
"type_info": "Jsonb"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 3,
|
||||||
|
"name": "changed_at",
|
||||||
|
"type_info": "Timestamptz"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 4,
|
||||||
|
"name": "changed_by",
|
||||||
|
"type_info": "Varchar"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 5,
|
||||||
|
"name": "fields_changed!",
|
||||||
|
"type_info": "TextArray"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"parameters": {
|
||||||
|
"Left": [
|
||||||
|
"Text",
|
||||||
|
"Int4"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"nullable": [
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
null
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"hash": "c045466e1fe037dbdafea1008f262f4e48f104ea77732aa1d32ecb797f70e71d"
|
||||||
|
}
|
||||||
Generated
+53
@@ -0,0 +1,53 @@
|
|||||||
|
{
|
||||||
|
"db_name": "PostgreSQL",
|
||||||
|
"query": "\n SELECT \n version_num,\n operation,\n snapshot,\n changed_at,\n changed_by,\n COALESCE(fields_changed, '{}') as \"fields_changed!\"\n FROM memory_edge_version\n WHERE edge_id = $1 AND version_num = $2\n ",
|
||||||
|
"describe": {
|
||||||
|
"columns": [
|
||||||
|
{
|
||||||
|
"ordinal": 0,
|
||||||
|
"name": "version_num",
|
||||||
|
"type_info": "Int4"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 1,
|
||||||
|
"name": "operation",
|
||||||
|
"type_info": "Varchar"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 2,
|
||||||
|
"name": "snapshot",
|
||||||
|
"type_info": "Jsonb"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 3,
|
||||||
|
"name": "changed_at",
|
||||||
|
"type_info": "Timestamptz"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 4,
|
||||||
|
"name": "changed_by",
|
||||||
|
"type_info": "Varchar"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 5,
|
||||||
|
"name": "fields_changed!",
|
||||||
|
"type_info": "TextArray"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"parameters": {
|
||||||
|
"Left": [
|
||||||
|
"Uuid",
|
||||||
|
"Int4"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"nullable": [
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
null
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"hash": "ca6872495bc04c6a65531279af8c758637c902dda2cc10366662988c6973ca48"
|
||||||
|
}
|
||||||
@@ -1,68 +0,0 @@
|
|||||||
-- Migration 009: Temporal knowledge graph edge schema (Zep paper §2.2.2)
|
|
||||||
-- Replaces old memory_edge (child_sha/parent_sha provenance DAG)
|
|
||||||
-- with temporal edge schema for the knowledge graph.
|
|
||||||
-- Idempotent: safe to run multiple times.
|
|
||||||
|
|
||||||
-- Rename old provenance DAG table if it still has child_sha columns
|
|
||||||
DO $$
|
|
||||||
BEGIN
|
|
||||||
IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'memory_edge'
|
|
||||||
AND EXISTS (SELECT 1 FROM information_schema.columns
|
|
||||||
WHERE table_name = 'memory_edge' AND column_name = 'child_sha'))
|
|
||||||
THEN
|
|
||||||
ALTER TABLE memory_edge RENAME TO memory_edge_provenance;
|
|
||||||
END IF;
|
|
||||||
END $$;
|
|
||||||
|
|
||||||
-- Create temporal knowledge graph edge table
|
|
||||||
CREATE TABLE IF NOT EXISTS memory_edge (
|
|
||||||
id TEXT PRIMARY KEY,
|
|
||||||
project_id TEXT NOT NULL DEFAULT 'default',
|
|
||||||
source_id TEXT NOT NULL,
|
|
||||||
target_id TEXT NOT NULL,
|
|
||||||
relation_type TEXT NOT NULL DEFAULT '',
|
|
||||||
fact TEXT NOT NULL DEFAULT '',
|
|
||||||
weight REAL NOT NULL DEFAULT 1.0,
|
|
||||||
strength REAL DEFAULT 1.0,
|
|
||||||
confidence REAL DEFAULT 0.8,
|
|
||||||
t_valid TIMESTAMPTZ,
|
|
||||||
t_invalid TIMESTAMPTZ,
|
|
||||||
t_created TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
||||||
t_expired TIMESTAMPTZ,
|
|
||||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
||||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
||||||
episode_id TEXT,
|
|
||||||
deleted_at TIMESTAMPTZ
|
|
||||||
);
|
|
||||||
|
|
||||||
-- Ensure app user owns the table
|
|
||||||
DO $$ BEGIN
|
|
||||||
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'app') THEN
|
|
||||||
ALTER TABLE memory_edge OWNER TO app;
|
|
||||||
END IF;
|
|
||||||
END $$;
|
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_memory_edge_source ON memory_edge(source_id);
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_memory_edge_target ON memory_edge(target_id);
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_memory_edge_project ON memory_edge(project_id);
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_memory_edge_relation ON memory_edge(relation_type);
|
|
||||||
|
|
||||||
-- Ensure memory_entity has all columns code expects
|
|
||||||
ALTER TABLE memory_entity ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ;
|
|
||||||
ALTER TABLE memory_entity ADD COLUMN IF NOT EXISTS source_count INTEGER DEFAULT 1;
|
|
||||||
|
|
||||||
-- Unique constraint for entity upsert dedup
|
|
||||||
DO $$
|
|
||||||
BEGIN
|
|
||||||
-- Dedup existing rows before creating unique index
|
|
||||||
DELETE FROM memory_entity a USING memory_entity b
|
|
||||||
WHERE a.project_id = b.project_id AND a.name = b.name
|
|
||||||
AND a.t_created < b.t_created;
|
|
||||||
EXCEPTION WHEN OTHERS THEN NULL;
|
|
||||||
END $$;
|
|
||||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_memory_entity_project_name ON memory_entity(project_id, name);
|
|
||||||
|
|
||||||
-- ROLLBACK instructions:
|
|
||||||
-- DROP TABLE IF EXISTS memory_edge;
|
|
||||||
-- ALTER TABLE IF EXISTS memory_edge_provenance RENAME TO memory_edge;
|
|
||||||
-- DROP INDEX IF EXISTS idx_memory_entity_project_name;
|
|
||||||
@@ -1,358 +0,0 @@
|
|||||||
use anyhow::Result;
|
|
||||||
use sqlx::{PgPool, FromRow};
|
|
||||||
use uuid::Uuid;
|
|
||||||
use serde::{Deserialize, Serialize};
|
|
||||||
use chrono::{DateTime, Utc};
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, FromRow)]
|
|
||||||
pub struct AgentPrompt {
|
|
||||||
pub id: Uuid,
|
|
||||||
pub project_id: String,
|
|
||||||
pub name: String,
|
|
||||||
pub template: String,
|
|
||||||
pub target_model: Option<String>,
|
|
||||||
pub task_category: String,
|
|
||||||
pub usage_count: i64,
|
|
||||||
pub avg_quality: f32,
|
|
||||||
pub last_used: Option<DateTime<Utc>>,
|
|
||||||
pub active: bool,
|
|
||||||
pub version: i32,
|
|
||||||
pub tags: Vec<String>,
|
|
||||||
pub created_at: DateTime<Utc>,
|
|
||||||
pub updated_at: DateTime<Utc>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, FromRow)]
|
|
||||||
pub struct AgentSkill {
|
|
||||||
pub id: Uuid,
|
|
||||||
pub project_id: String,
|
|
||||||
pub agent_id: String,
|
|
||||||
pub name: String,
|
|
||||||
pub description: String,
|
|
||||||
pub trigger_patterns: Vec<String>,
|
|
||||||
pub success_rate: f32,
|
|
||||||
pub invocation_count: i64,
|
|
||||||
pub avg_latency_ms: i64,
|
|
||||||
pub linked_prompts: Vec<Uuid>,
|
|
||||||
pub enabled: bool,
|
|
||||||
pub created_at: DateTime<Utc>,
|
|
||||||
pub updated_at: DateTime<Utc>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, FromRow)]
|
|
||||||
pub struct AgentDecision {
|
|
||||||
pub id: Uuid,
|
|
||||||
pub project_id: String,
|
|
||||||
pub agent_id: String,
|
|
||||||
pub action: String,
|
|
||||||
pub reasoning: String,
|
|
||||||
pub alternatives: Vec<String>,
|
|
||||||
pub confidence: f32,
|
|
||||||
pub context_entities: Vec<Uuid>,
|
|
||||||
pub tool: Option<String>,
|
|
||||||
pub task: Option<String>,
|
|
||||||
pub outcome_success: Option<bool>,
|
|
||||||
pub outcome_quality: Option<f32>,
|
|
||||||
pub outcome_feedback: Option<String>,
|
|
||||||
pub outcome_recorded_at: Option<DateTime<Utc>>,
|
|
||||||
pub created_at: DateTime<Utc>,
|
|
||||||
pub updated_at: DateTime<Utc>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, FromRow)]
|
|
||||||
pub struct RolePromptMapping {
|
|
||||||
pub id: Uuid,
|
|
||||||
pub project_id: String,
|
|
||||||
pub role_name: String,
|
|
||||||
pub prompt_id: Uuid,
|
|
||||||
pub priority: i32,
|
|
||||||
pub active: bool,
|
|
||||||
pub created_at: DateTime<Utc>,
|
|
||||||
pub updated_at: DateTime<Utc>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, FromRow)]
|
|
||||||
pub struct AgentMetrics {
|
|
||||||
pub id: Uuid,
|
|
||||||
pub project_id: String,
|
|
||||||
pub agent_id: String,
|
|
||||||
pub requests_total: i64,
|
|
||||||
pub requests_success: i64,
|
|
||||||
pub requests_failed: i64,
|
|
||||||
pub average_latency_ms: f32,
|
|
||||||
pub p95_latency_ms: f32,
|
|
||||||
pub p99_latency_ms: f32,
|
|
||||||
pub error_rate: f32,
|
|
||||||
pub recorded_at: DateTime<Utc>,
|
|
||||||
}
|
|
||||||
|
|
||||||
pub struct AgentRepository {
|
|
||||||
pool: PgPool,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl AgentRepository {
|
|
||||||
pub fn new(pool: PgPool) -> Self {
|
|
||||||
AgentRepository { pool }
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn create_prompt(&self, prompt: AgentPrompt) -> Result<AgentPrompt> {
|
|
||||||
let result = sqlx::query_as::<_, AgentPrompt>(
|
|
||||||
r#"
|
|
||||||
INSERT INTO agent_prompt
|
|
||||||
(project_id, name, template, target_model, task_category, active, version, tags)
|
|
||||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
|
||||||
RETURNING *
|
|
||||||
"#,
|
|
||||||
)
|
|
||||||
.bind(&prompt.project_id)
|
|
||||||
.bind(&prompt.name)
|
|
||||||
.bind(&prompt.template)
|
|
||||||
.bind(&prompt.target_model)
|
|
||||||
.bind(&prompt.task_category)
|
|
||||||
.bind(prompt.active)
|
|
||||||
.bind(prompt.version)
|
|
||||||
.bind(&prompt.tags)
|
|
||||||
.fetch_one(&self.pool)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
Ok(result)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn get_prompt(&self, id: Uuid) -> Result<Option<AgentPrompt>> {
|
|
||||||
let result = sqlx::query_as::<_, AgentPrompt>(
|
|
||||||
"SELECT * FROM agent_prompt WHERE id = $1"
|
|
||||||
)
|
|
||||||
.bind(id)
|
|
||||||
.fetch_optional(&self.pool)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
Ok(result)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn list_prompts(&self, project_id: &str) -> Result<Vec<AgentPrompt>> {
|
|
||||||
let results = sqlx::query_as::<_, AgentPrompt>(
|
|
||||||
"SELECT * FROM agent_prompt WHERE project_id = $1 AND active = true ORDER BY created_at DESC"
|
|
||||||
)
|
|
||||||
.bind(project_id)
|
|
||||||
.fetch_all(&self.pool)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
Ok(results)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn update_prompt_usage(&self, id: Uuid, quality_score: f32) -> Result<()> {
|
|
||||||
sqlx::query(
|
|
||||||
r#"
|
|
||||||
UPDATE agent_prompt
|
|
||||||
SET usage_count = usage_count + 1,
|
|
||||||
avg_quality = (avg_quality * (usage_count) + $2) / (usage_count + 1),
|
|
||||||
last_used = NOW(),
|
|
||||||
updated_at = NOW()
|
|
||||||
WHERE id = $1
|
|
||||||
"#,
|
|
||||||
)
|
|
||||||
.bind(id)
|
|
||||||
.bind(quality_score)
|
|
||||||
.execute(&self.pool)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn create_skill(&self, skill: AgentSkill) -> Result<AgentSkill> {
|
|
||||||
let result = sqlx::query_as::<_, AgentSkill>(
|
|
||||||
r#"
|
|
||||||
INSERT INTO agent_skill
|
|
||||||
(project_id, agent_id, name, description, enabled)
|
|
||||||
VALUES ($1, $2, $3, $4, $5)
|
|
||||||
RETURNING *
|
|
||||||
"#,
|
|
||||||
)
|
|
||||||
.bind(&skill.project_id)
|
|
||||||
.bind(&skill.agent_id)
|
|
||||||
.bind(&skill.name)
|
|
||||||
.bind(&skill.description)
|
|
||||||
.bind(skill.enabled)
|
|
||||||
.fetch_one(&self.pool)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
Ok(result)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn get_skill(&self, id: Uuid) -> Result<Option<AgentSkill>> {
|
|
||||||
let result = sqlx::query_as::<_, AgentSkill>(
|
|
||||||
"SELECT * FROM agent_skill WHERE id = $1"
|
|
||||||
)
|
|
||||||
.bind(id)
|
|
||||||
.fetch_optional(&self.pool)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
Ok(result)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn list_skills(&self, project_id: &str, agent_id: &str) -> Result<Vec<AgentSkill>> {
|
|
||||||
let results = sqlx::query_as::<_, AgentSkill>(
|
|
||||||
"SELECT * FROM agent_skill WHERE project_id = $1 AND agent_id = $2 AND enabled = true ORDER BY created_at DESC"
|
|
||||||
)
|
|
||||||
.bind(project_id)
|
|
||||||
.bind(agent_id)
|
|
||||||
.fetch_all(&self.pool)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
Ok(results)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn create_decision(&self, decision: AgentDecision) -> Result<AgentDecision> {
|
|
||||||
let result = sqlx::query_as::<_, AgentDecision>(
|
|
||||||
r#"
|
|
||||||
INSERT INTO agent_decision
|
|
||||||
(project_id, agent_id, action, reasoning, confidence, tool, task)
|
|
||||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
|
||||||
RETURNING *
|
|
||||||
"#,
|
|
||||||
)
|
|
||||||
.bind(&decision.project_id)
|
|
||||||
.bind(&decision.agent_id)
|
|
||||||
.bind(&decision.action)
|
|
||||||
.bind(&decision.reasoning)
|
|
||||||
.bind(decision.confidence)
|
|
||||||
.bind(&decision.tool)
|
|
||||||
.bind(&decision.task)
|
|
||||||
.fetch_one(&self.pool)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
Ok(result)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn record_decision_outcome(
|
|
||||||
&self,
|
|
||||||
id: Uuid,
|
|
||||||
success: bool,
|
|
||||||
quality: f32,
|
|
||||||
feedback: Option<&str>,
|
|
||||||
) -> Result<()> {
|
|
||||||
sqlx::query(
|
|
||||||
r#"
|
|
||||||
UPDATE agent_decision
|
|
||||||
SET outcome_success = $2,
|
|
||||||
outcome_quality = $3,
|
|
||||||
outcome_feedback = $4,
|
|
||||||
outcome_recorded_at = NOW(),
|
|
||||||
updated_at = NOW()
|
|
||||||
WHERE id = $1
|
|
||||||
"#,
|
|
||||||
)
|
|
||||||
.bind(id)
|
|
||||||
.bind(success)
|
|
||||||
.bind(quality)
|
|
||||||
.bind(feedback)
|
|
||||||
.execute(&self.pool)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn create_role_mapping(&self, mapping: RolePromptMapping) -> Result<RolePromptMapping> {
|
|
||||||
let result = sqlx::query_as::<_, RolePromptMapping>(
|
|
||||||
r#"
|
|
||||||
INSERT INTO role_prompt_mapping
|
|
||||||
(project_id, role_name, prompt_id, priority, active)
|
|
||||||
VALUES ($1, $2, $3, $4, $5)
|
|
||||||
RETURNING *
|
|
||||||
"#,
|
|
||||||
)
|
|
||||||
.bind(&mapping.project_id)
|
|
||||||
.bind(&mapping.role_name)
|
|
||||||
.bind(mapping.prompt_id)
|
|
||||||
.bind(mapping.priority)
|
|
||||||
.bind(mapping.active)
|
|
||||||
.fetch_one(&self.pool)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
Ok(result)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn get_prompts_for_role(&self, project_id: &str, role_name: &str) -> Result<Vec<AgentPrompt>> {
|
|
||||||
let results = sqlx::query_as::<_, AgentPrompt>(
|
|
||||||
r#"
|
|
||||||
SELECT ap.* FROM agent_prompt ap
|
|
||||||
INNER JOIN role_prompt_mapping rpm ON ap.id = rpm.prompt_id
|
|
||||||
WHERE rpm.project_id = $1 AND rpm.role_name = $2 AND rpm.active = true
|
|
||||||
ORDER BY rpm.priority DESC, ap.created_at DESC
|
|
||||||
"#,
|
|
||||||
)
|
|
||||||
.bind(project_id)
|
|
||||||
.bind(role_name)
|
|
||||||
.fetch_all(&self.pool)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
Ok(results)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn save_metrics(&self, metrics: AgentMetrics) -> Result<()> {
|
|
||||||
sqlx::query(
|
|
||||||
r#"
|
|
||||||
INSERT INTO agent_metrics
|
|
||||||
(project_id, agent_id, requests_total, requests_success, requests_failed,
|
|
||||||
average_latency_ms, p95_latency_ms, p99_latency_ms, error_rate)
|
|
||||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
|
||||||
ON CONFLICT (project_id, agent_id, DATE(recorded_at)) DO UPDATE SET
|
|
||||||
requests_total = EXCLUDED.requests_total,
|
|
||||||
requests_success = EXCLUDED.requests_success,
|
|
||||||
requests_failed = EXCLUDED.requests_failed,
|
|
||||||
average_latency_ms = EXCLUDED.average_latency_ms,
|
|
||||||
p95_latency_ms = EXCLUDED.p95_latency_ms,
|
|
||||||
p99_latency_ms = EXCLUDED.p99_latency_ms,
|
|
||||||
error_rate = EXCLUDED.error_rate
|
|
||||||
"#,
|
|
||||||
)
|
|
||||||
.bind(&metrics.project_id)
|
|
||||||
.bind(&metrics.agent_id)
|
|
||||||
.bind(metrics.requests_total)
|
|
||||||
.bind(metrics.requests_success)
|
|
||||||
.bind(metrics.requests_failed)
|
|
||||||
.bind(metrics.average_latency_ms)
|
|
||||||
.bind(metrics.p95_latency_ms)
|
|
||||||
.bind(metrics.p99_latency_ms)
|
|
||||||
.bind(metrics.error_rate)
|
|
||||||
.execute(&self.pool)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn log_prompt_usage(
|
|
||||||
&self,
|
|
||||||
project_id: &str,
|
|
||||||
prompt_id: Uuid,
|
|
||||||
agent_id: Option<&str>,
|
|
||||||
model: Option<&str>,
|
|
||||||
input_tokens: Option<i32>,
|
|
||||||
output_tokens: Option<i32>,
|
|
||||||
quality_score: Option<f32>,
|
|
||||||
duration_ms: i64,
|
|
||||||
error_message: Option<&str>,
|
|
||||||
) -> Result<()> {
|
|
||||||
sqlx::query(
|
|
||||||
r#"
|
|
||||||
INSERT INTO prompt_usage_log
|
|
||||||
(project_id, prompt_id, agent_id, model_used, input_tokens, output_tokens,
|
|
||||||
quality_score, duration_ms, error_message)
|
|
||||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
|
||||||
"#,
|
|
||||||
)
|
|
||||||
.bind(project_id)
|
|
||||||
.bind(prompt_id)
|
|
||||||
.bind(agent_id)
|
|
||||||
.bind(model)
|
|
||||||
.bind(input_tokens)
|
|
||||||
.bind(output_tokens)
|
|
||||||
.bind(quality_score)
|
|
||||||
.bind(duration_ms)
|
|
||||||
.bind(error_message)
|
|
||||||
.execute(&self.pool)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
use chrono::{DateTime, Utc};
|
use chrono::{DateTime, Utc};
|
||||||
use sqlx::PgPool;
|
use sqlx::PgPool;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
/// Minimal audit logger - records version snapshots on mutation
|
/// Minimal audit logger - records version snapshots on mutation
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ pub mod edge_repo;
|
|||||||
pub mod community_repo;
|
pub mod community_repo;
|
||||||
pub mod versioning;
|
pub mod versioning;
|
||||||
pub mod audit_logger;
|
pub mod audit_logger;
|
||||||
pub mod agent_repo;
|
|
||||||
// pub mod db_repo; // TODO: Fix Entity schema integration
|
// pub mod db_repo; // TODO: Fix Entity schema integration
|
||||||
|
|
||||||
pub use event_log::{EventRecord, LogWriter};
|
pub use event_log::{EventRecord, LogWriter};
|
||||||
|
|||||||
@@ -218,74 +218,6 @@ pub async fn init_schema(pool: &PgPool) -> Result<()> {
|
|||||||
.execute(pool)
|
.execute(pool)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
// Memory entity table (temporal knowledge graph)
|
tracing::info!("Database schema initialized");
|
||||||
sqlx::query(
|
|
||||||
r#"
|
|
||||||
CREATE TABLE IF NOT EXISTS memory_entity (
|
|
||||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
||||||
project_id VARCHAR(255) NOT NULL,
|
|
||||||
name VARCHAR(500) NOT NULL,
|
|
||||||
name_embedding VECTOR(768),
|
|
||||||
summary TEXT,
|
|
||||||
description TEXT,
|
|
||||||
summary_embedding VECTOR(768),
|
|
||||||
entity_type VARCHAR(50),
|
|
||||||
t_created TIMESTAMPTZ DEFAULT NOW(),
|
|
||||||
t_updated TIMESTAMPTZ DEFAULT NOW(),
|
|
||||||
t_expired TIMESTAMPTZ,
|
|
||||||
confidence FLOAT DEFAULT 1.0,
|
|
||||||
source_count INT DEFAULT 1,
|
|
||||||
source_episodes UUID[] DEFAULT '{}',
|
|
||||||
access_count BIGINT DEFAULT 0,
|
|
||||||
last_accessed TIMESTAMPTZ,
|
|
||||||
UNIQUE(project_id, name)
|
|
||||||
)
|
|
||||||
"#,
|
|
||||||
)
|
|
||||||
.execute(pool)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
sqlx::query("CREATE INDEX IF NOT EXISTS idx_entity_project ON memory_entity(project_id)")
|
|
||||||
.execute(pool)
|
|
||||||
.await?;
|
|
||||||
sqlx::query("CREATE INDEX IF NOT EXISTS idx_entity_type ON memory_entity(project_id, entity_type)")
|
|
||||||
.execute(pool)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
// Memory edge table (temporal knowledge graph)
|
|
||||||
sqlx::query(
|
|
||||||
r#"
|
|
||||||
CREATE TABLE IF NOT EXISTS memory_edge (
|
|
||||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
||||||
project_id VARCHAR(255) NOT NULL,
|
|
||||||
source_id UUID NOT NULL,
|
|
||||||
target_id UUID NOT NULL,
|
|
||||||
relation_type VARCHAR(100) NOT NULL,
|
|
||||||
fact TEXT NOT NULL,
|
|
||||||
fact_embedding VECTOR(768),
|
|
||||||
t_valid TIMESTAMPTZ,
|
|
||||||
t_invalid TIMESTAMPTZ,
|
|
||||||
t_created TIMESTAMPTZ DEFAULT NOW(),
|
|
||||||
t_expired TIMESTAMPTZ,
|
|
||||||
confidence FLOAT DEFAULT 1.0,
|
|
||||||
contradiction_status VARCHAR(20) DEFAULT 'active',
|
|
||||||
contradiction_confidence FLOAT
|
|
||||||
)
|
|
||||||
"#,
|
|
||||||
)
|
|
||||||
.execute(pool)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
sqlx::query("CREATE INDEX IF NOT EXISTS idx_edge_project ON memory_edge(project_id)")
|
|
||||||
.execute(pool)
|
|
||||||
.await?;
|
|
||||||
sqlx::query("CREATE INDEX IF NOT EXISTS idx_edge_source ON memory_edge(source_id)")
|
|
||||||
.execute(pool)
|
|
||||||
.await?;
|
|
||||||
sqlx::query("CREATE INDEX IF NOT EXISTS idx_edge_target ON memory_edge(target_id)")
|
|
||||||
.execute(pool)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
tracing::info!("Database schema initialized (including memory_entity + memory_edge)");
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,16 +1,15 @@
|
|||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use sqlx::{PgPool, FromRow};
|
use sqlx::PgPool;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
use chrono::{DateTime, Utc};
|
use chrono::{DateTime, Utc};
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, FromRow)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
pub struct VersionSnapshot {
|
pub struct VersionSnapshot {
|
||||||
pub version_num: i32,
|
pub version_num: i32,
|
||||||
pub operation: String, // 'create' | 'update' | 'delete'
|
pub operation: String, // 'create' | 'update' | 'delete'
|
||||||
pub snapshot: serde_json::Value,
|
pub snapshot: serde_json::Value,
|
||||||
pub changed_at: DateTime<Utc>,
|
pub changed_at: DateTime<Utc>,
|
||||||
pub changed_by: String,
|
pub changed_by: String,
|
||||||
#[sqlx(default)]
|
|
||||||
pub fields_changed: Vec<String>,
|
pub fields_changed: Vec<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -41,7 +40,8 @@ impl EntityVersioningService {
|
|||||||
|
|
||||||
/// Get all versions of an entity in descending order
|
/// Get all versions of an entity in descending order
|
||||||
pub async fn get_versions(&self, entity_id: &str) -> Result<Vec<VersionSnapshot>, sqlx::Error> {
|
pub async fn get_versions(&self, entity_id: &str) -> Result<Vec<VersionSnapshot>, sqlx::Error> {
|
||||||
sqlx::query_as::<_, VersionSnapshot>(
|
sqlx::query_as!(
|
||||||
|
VersionSnapshot,
|
||||||
r#"
|
r#"
|
||||||
SELECT
|
SELECT
|
||||||
version_num,
|
version_num,
|
||||||
@@ -49,13 +49,13 @@ impl EntityVersioningService {
|
|||||||
snapshot,
|
snapshot,
|
||||||
changed_at,
|
changed_at,
|
||||||
changed_by,
|
changed_by,
|
||||||
COALESCE(fields_changed, '{}') as fields_changed
|
COALESCE(fields_changed, '{}') as "fields_changed!"
|
||||||
FROM memory_entity_version
|
FROM memory_entity_version
|
||||||
WHERE entity_id = $1
|
WHERE entity_id = $1
|
||||||
ORDER BY version_num DESC
|
ORDER BY version_num DESC
|
||||||
"#,
|
"#,
|
||||||
|
entity_id
|
||||||
)
|
)
|
||||||
.bind(entity_id)
|
|
||||||
.fetch_all(&self.pool)
|
.fetch_all(&self.pool)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
@@ -66,7 +66,8 @@ impl EntityVersioningService {
|
|||||||
entity_id: &str,
|
entity_id: &str,
|
||||||
version_num: i32,
|
version_num: i32,
|
||||||
) -> Result<Option<VersionSnapshot>, sqlx::Error> {
|
) -> Result<Option<VersionSnapshot>, sqlx::Error> {
|
||||||
sqlx::query_as::<_, VersionSnapshot>(
|
sqlx::query_as!(
|
||||||
|
VersionSnapshot,
|
||||||
r#"
|
r#"
|
||||||
SELECT
|
SELECT
|
||||||
version_num,
|
version_num,
|
||||||
@@ -74,13 +75,13 @@ impl EntityVersioningService {
|
|||||||
snapshot,
|
snapshot,
|
||||||
changed_at,
|
changed_at,
|
||||||
changed_by,
|
changed_by,
|
||||||
COALESCE(fields_changed, '{}') as fields_changed
|
COALESCE(fields_changed, '{}') as "fields_changed!"
|
||||||
FROM memory_entity_version
|
FROM memory_entity_version
|
||||||
WHERE entity_id = $1 AND version_num = $2
|
WHERE entity_id = $1 AND version_num = $2
|
||||||
"#,
|
"#,
|
||||||
|
entity_id,
|
||||||
|
version_num
|
||||||
)
|
)
|
||||||
.bind(entity_id)
|
|
||||||
.bind(version_num)
|
|
||||||
.fetch_optional(&self.pool)
|
.fetch_optional(&self.pool)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
@@ -94,7 +95,78 @@ impl EntityVersioningService {
|
|||||||
) -> Result<DiffResult, sqlx::Error> {
|
) -> Result<DiffResult, sqlx::Error> {
|
||||||
let from_snap = self.get_version(entity_id, from_v).await?;
|
let from_snap = self.get_version(entity_id, from_v).await?;
|
||||||
let to_snap = self.get_version(entity_id, to_v).await?;
|
let to_snap = self.get_version(entity_id, to_v).await?;
|
||||||
compute_diff(from_snap, to_snap, from_v, to_v)
|
|
||||||
|
let from_obj = from_snap
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|s| s.snapshot.as_object())
|
||||||
|
.map(|o| o.clone());
|
||||||
|
|
||||||
|
let to_obj = to_snap
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|s| s.snapshot.as_object())
|
||||||
|
.map(|o| o.clone());
|
||||||
|
|
||||||
|
let mut added = Vec::new();
|
||||||
|
let mut removed = Vec::new();
|
||||||
|
let mut modified = Vec::new();
|
||||||
|
|
||||||
|
// Check removed and modified
|
||||||
|
if let Some(ref from) = from_obj {
|
||||||
|
for (key, from_val) in from {
|
||||||
|
if let Some(to) = &to_obj {
|
||||||
|
if let Some(to_val) = to.get(key) {
|
||||||
|
if from_val != to_val {
|
||||||
|
modified.push(DiffField {
|
||||||
|
name: key.clone(),
|
||||||
|
from_value: Some(from_val.clone()),
|
||||||
|
to_value: Some(to_val.clone()),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
removed.push(DiffField {
|
||||||
|
name: key.clone(),
|
||||||
|
from_value: Some(from_val.clone()),
|
||||||
|
to_value: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
removed.push(DiffField {
|
||||||
|
name: key.clone(),
|
||||||
|
from_value: Some(from_val.clone()),
|
||||||
|
to_value: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check added
|
||||||
|
if let Some(to) = to_obj {
|
||||||
|
for (key, to_val) in to {
|
||||||
|
if let Some(from) = &from_obj {
|
||||||
|
if !from.contains_key(&key) {
|
||||||
|
added.push(DiffField {
|
||||||
|
name: key,
|
||||||
|
from_value: None,
|
||||||
|
to_value: Some(to_val),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
added.push(DiffField {
|
||||||
|
name: key,
|
||||||
|
from_value: None,
|
||||||
|
to_value: Some(to_val),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(DiffResult {
|
||||||
|
from_version: from_v,
|
||||||
|
to_version: to_v,
|
||||||
|
added_fields: added,
|
||||||
|
removed_fields: removed,
|
||||||
|
modified_fields: modified,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get entity state at a point in time
|
/// Get entity state at a point in time
|
||||||
@@ -103,7 +175,8 @@ impl EntityVersioningService {
|
|||||||
entity_id: &str,
|
entity_id: &str,
|
||||||
as_of: DateTime<Utc>,
|
as_of: DateTime<Utc>,
|
||||||
) -> Result<Option<VersionSnapshot>, sqlx::Error> {
|
) -> Result<Option<VersionSnapshot>, sqlx::Error> {
|
||||||
sqlx::query_as::<_, VersionSnapshot>(
|
sqlx::query_as!(
|
||||||
|
VersionSnapshot,
|
||||||
r#"
|
r#"
|
||||||
SELECT
|
SELECT
|
||||||
version_num,
|
version_num,
|
||||||
@@ -111,15 +184,15 @@ impl EntityVersioningService {
|
|||||||
snapshot,
|
snapshot,
|
||||||
changed_at,
|
changed_at,
|
||||||
changed_by,
|
changed_by,
|
||||||
COALESCE(fields_changed, '{}') as fields_changed
|
COALESCE(fields_changed, '{}') as "fields_changed!"
|
||||||
FROM memory_entity_version
|
FROM memory_entity_version
|
||||||
WHERE entity_id = $1 AND changed_at <= $2
|
WHERE entity_id = $1 AND changed_at <= $2
|
||||||
ORDER BY version_num DESC
|
ORDER BY version_num DESC
|
||||||
LIMIT 1
|
LIMIT 1
|
||||||
"#,
|
"#,
|
||||||
|
entity_id,
|
||||||
|
as_of
|
||||||
)
|
)
|
||||||
.bind(entity_id)
|
|
||||||
.bind(as_of)
|
|
||||||
.fetch_optional(&self.pool)
|
.fetch_optional(&self.pool)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
@@ -137,7 +210,8 @@ impl EdgeVersioningService {
|
|||||||
|
|
||||||
/// Get all versions of an edge
|
/// Get all versions of an edge
|
||||||
pub async fn get_versions(&self, edge_id: Uuid) -> Result<Vec<VersionSnapshot>, sqlx::Error> {
|
pub async fn get_versions(&self, edge_id: Uuid) -> Result<Vec<VersionSnapshot>, sqlx::Error> {
|
||||||
sqlx::query_as::<_, VersionSnapshot>(
|
sqlx::query_as!(
|
||||||
|
VersionSnapshot,
|
||||||
r#"
|
r#"
|
||||||
SELECT
|
SELECT
|
||||||
version_num,
|
version_num,
|
||||||
@@ -145,13 +219,13 @@ impl EdgeVersioningService {
|
|||||||
snapshot,
|
snapshot,
|
||||||
changed_at,
|
changed_at,
|
||||||
changed_by,
|
changed_by,
|
||||||
COALESCE(fields_changed, '{}') as fields_changed
|
COALESCE(fields_changed, '{}') as "fields_changed!"
|
||||||
FROM memory_edge_version
|
FROM memory_edge_version
|
||||||
WHERE edge_id = $1
|
WHERE edge_id = $1
|
||||||
ORDER BY version_num DESC
|
ORDER BY version_num DESC
|
||||||
"#,
|
"#,
|
||||||
|
edge_id
|
||||||
)
|
)
|
||||||
.bind(edge_id)
|
|
||||||
.fetch_all(&self.pool)
|
.fetch_all(&self.pool)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
@@ -163,7 +237,8 @@ impl EdgeVersioningService {
|
|||||||
from_v: i32,
|
from_v: i32,
|
||||||
to_v: i32,
|
to_v: i32,
|
||||||
) -> Result<DiffResult, sqlx::Error> {
|
) -> Result<DiffResult, sqlx::Error> {
|
||||||
let from_snap = sqlx::query_as::<_, VersionSnapshot>(
|
let from_snap = sqlx::query_as!(
|
||||||
|
VersionSnapshot,
|
||||||
r#"
|
r#"
|
||||||
SELECT
|
SELECT
|
||||||
version_num,
|
version_num,
|
||||||
@@ -171,17 +246,18 @@ impl EdgeVersioningService {
|
|||||||
snapshot,
|
snapshot,
|
||||||
changed_at,
|
changed_at,
|
||||||
changed_by,
|
changed_by,
|
||||||
COALESCE(fields_changed, '{}') as fields_changed
|
COALESCE(fields_changed, '{}') as "fields_changed!"
|
||||||
FROM memory_edge_version
|
FROM memory_edge_version
|
||||||
WHERE edge_id = $1 AND version_num = $2
|
WHERE edge_id = $1 AND version_num = $2
|
||||||
"#,
|
"#,
|
||||||
|
edge_id,
|
||||||
|
from_v
|
||||||
)
|
)
|
||||||
.bind(edge_id)
|
|
||||||
.bind(from_v)
|
|
||||||
.fetch_optional(&self.pool)
|
.fetch_optional(&self.pool)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
let to_snap = sqlx::query_as::<_, VersionSnapshot>(
|
let to_snap = sqlx::query_as!(
|
||||||
|
VersionSnapshot,
|
||||||
r#"
|
r#"
|
||||||
SELECT
|
SELECT
|
||||||
version_num,
|
version_num,
|
||||||
@@ -189,16 +265,17 @@ impl EdgeVersioningService {
|
|||||||
snapshot,
|
snapshot,
|
||||||
changed_at,
|
changed_at,
|
||||||
changed_by,
|
changed_by,
|
||||||
COALESCE(fields_changed, '{}') as fields_changed
|
COALESCE(fields_changed, '{}') as "fields_changed!"
|
||||||
FROM memory_edge_version
|
FROM memory_edge_version
|
||||||
WHERE edge_id = $1 AND version_num = $2
|
WHERE edge_id = $1 AND version_num = $2
|
||||||
"#,
|
"#,
|
||||||
|
edge_id,
|
||||||
|
to_v
|
||||||
)
|
)
|
||||||
.bind(edge_id)
|
|
||||||
.bind(to_v)
|
|
||||||
.fetch_optional(&self.pool)
|
.fetch_optional(&self.pool)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
|
// Same diff logic as entities
|
||||||
compute_diff(from_snap, to_snap, from_v, to_v)
|
compute_diff(from_snap, to_snap, from_v, to_v)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,21 +0,0 @@
|
|||||||
version: '3.8'
|
|
||||||
|
|
||||||
services:
|
|
||||||
postgres-test:
|
|
||||||
image: postgres:16-alpine
|
|
||||||
environment:
|
|
||||||
POSTGRES_USER: app
|
|
||||||
POSTGRES_PASSWORD: testpass
|
|
||||||
POSTGRES_DB: memory
|
|
||||||
ports:
|
|
||||||
- "5433:5432"
|
|
||||||
volumes:
|
|
||||||
- postgres-test-data:/var/lib/postgresql/data
|
|
||||||
healthcheck:
|
|
||||||
test: ["CMD-SHELL", "pg_isready -U app -d memory"]
|
|
||||||
interval: 2s
|
|
||||||
timeout: 5s
|
|
||||||
retries: 10
|
|
||||||
|
|
||||||
volumes:
|
|
||||||
postgres-test-data:
|
|
||||||
@@ -1,28 +0,0 @@
|
|||||||
{
|
|
||||||
"results": [
|
|
||||||
{
|
|
||||||
"id": "chunk-abc123",
|
|
||||||
"level": "L1",
|
|
||||||
"score": 0.95,
|
|
||||||
"text": "Kubernetes uses port 8080 for API server",
|
|
||||||
"source": "transcript://session-001"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "chunk-def456",
|
|
||||||
"level": "L2",
|
|
||||||
"score": 0.87,
|
|
||||||
"text": "Common debugging pattern for CrashLoopBackOff pods",
|
|
||||||
"source": "transcript://session-002"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "chunk-ghi789",
|
|
||||||
"level": "R",
|
|
||||||
"score": 0.72,
|
|
||||||
"text": "See kubectl troubleshooting guide section 3.2",
|
|
||||||
"source": "obsidian://poimen-vault/kubectl.md"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"total_hits": 127,
|
|
||||||
"search_time_ms": 145,
|
|
||||||
"query": "fix kubernetes port conflict"
|
|
||||||
}
|
|
||||||
@@ -1,33 +0,0 @@
|
|||||||
# Kubernetes Troubleshooting Guide
|
|
||||||
|
|
||||||
## Port Conflicts
|
|
||||||
|
|
||||||
When a port conflict occurs on port 8080, check for existing services:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
kubectl get svc --all-namespaces | grep 8080
|
|
||||||
```
|
|
||||||
|
|
||||||
### Common Causes
|
|
||||||
|
|
||||||
1. Multiple services binding to same NodePort
|
|
||||||
2. Host network pods conflicting with node services
|
|
||||||
3. Ingress controller port overlap
|
|
||||||
|
|
||||||
## CrashLoopBackOff
|
|
||||||
|
|
||||||
Pods enter CrashLoopBackOff when the container exits repeatedly.
|
|
||||||
|
|
||||||
### Diagnosis Steps
|
|
||||||
|
|
||||||
1. Check pod logs: `kubectl logs <pod> --previous`
|
|
||||||
2. Check events: `kubectl describe pod <pod>`
|
|
||||||
3. Check resource limits: memory/CPU constraints
|
|
||||||
4. Check liveness probes: incorrect health check paths
|
|
||||||
|
|
||||||
### Resolution
|
|
||||||
|
|
||||||
- Increase memory limits if OOMKilled
|
|
||||||
- Fix application startup errors
|
|
||||||
- Adjust probe timing (initialDelaySeconds)
|
|
||||||
- Check environment variable configuration
|
|
||||||
@@ -1,16 +0,0 @@
|
|||||||
2025-01-15T10:00:00Z INFO Starting service on port 8080
|
|
||||||
2025-01-15T10:00:01Z DEBUG Database connection pool initialized (max=20)
|
|
||||||
2025-01-15T10:00:02Z INFO Health check endpoint ready at /health
|
|
||||||
2025-01-15T10:00:05Z WARN High memory usage detected: 85% of 512Mi limit
|
|
||||||
2025-01-15T10:00:10Z ERROR Connection refused: temporal-frontend:7233
|
|
||||||
2025-01-15T10:00:15Z INFO Retry attempt 1/3 for temporal connection
|
|
||||||
2025-01-15T10:00:20Z INFO Connected to temporal-frontend.temporal.svc.cluster.local:7233
|
|
||||||
2025-01-15T10:00:25Z DEBUG Worker registered on task queue: poimen-taskqueue
|
|
||||||
2025-01-15T10:00:30Z INFO Processing ingest request: project=poimen source=transcript://session-001
|
|
||||||
2025-01-15T10:00:31Z DEBUG Entity extraction complete: 5 entities found
|
|
||||||
2025-01-15T10:00:32Z DEBUG Fact extraction complete: 3 facts found
|
|
||||||
2025-01-15T10:00:33Z INFO Contradiction check: 0 contradictions detected
|
|
||||||
2025-01-15T10:00:34Z INFO Ingest complete: chunk-abc123 (145ms)
|
|
||||||
2025-01-15T10:00:40Z WARN Slow query detected: 850ms for hybrid search
|
|
||||||
2025-01-15T10:00:45Z ERROR Pod OOMKilled: poimen-worker-abc123 (memory limit exceeded)
|
|
||||||
2025-01-15T10:00:50Z INFO Pod restarted: poimen-worker-abc123 (restart count: 1)
|
|
||||||
@@ -1,157 +0,0 @@
|
|||||||
# Poimen Memory - Environment Configuration Guide
|
|
||||||
|
|
||||||
All downstream service URIs are read from environment variables, sourced from ConfigMap.
|
|
||||||
|
|
||||||
## How It Works
|
|
||||||
|
|
||||||
1. **ConfigMap provides URIs**: `k8s/app/config.yaml` (production, SOPS-encrypted)
|
|
||||||
2. **Deployment injects via envFrom**: `envFrom: configMapRef: poimen-memory-config`
|
|
||||||
3. **Application reads from ENV**: Code parses `LLM_ENDPOINT`, `OPENSEARCH_HOST`, `AUTHENTIK_ISSUER`, etc.
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# deployment.yaml
|
|
||||||
envFrom:
|
|
||||||
- configMapRef:
|
|
||||||
name: poimen-memory-config # All vars injected as ENV
|
|
||||||
```
|
|
||||||
|
|
||||||
## Environment Variables
|
|
||||||
|
|
||||||
### LLM Service (Entity & Fact Extraction)
|
|
||||||
- `LLM_ENDPOINT` — full URL to chat/completions endpoint
|
|
||||||
- `LLM_API_BASE` — base API URL (used for client initialization)
|
|
||||||
- `LLM_MODEL` — model identifier (ornith:35b, qwen:7b, etc.)
|
|
||||||
- `LLM_TIMEOUT_SECS` — timeout for LLM requests
|
|
||||||
- `ENABLE_LLM_EXTRACTION` — enable/disable LLM extraction (true/false)
|
|
||||||
|
|
||||||
### OpenSearch (Vector Store, BM25)
|
|
||||||
- `OPENSEARCH_HOST` — hostname:port
|
|
||||||
- `OPENSEARCH_SCHEME` — http or https
|
|
||||||
- `OPENSEARCH_VERIFY_CERTS` — SSL certificate verification (true/false)
|
|
||||||
|
|
||||||
### Authentik (OIDC)
|
|
||||||
- `AUTHENTIK_ISSUER` — OIDC issuer URL
|
|
||||||
- `AUTHENTIK_VERIFY_SSL` — SSL certificate verification (true/false)
|
|
||||||
- `MEM_AUTH_MODE` — auth mode: jwt | apikey | none
|
|
||||||
|
|
||||||
### Temporal (Workflow Orchestration - Future)
|
|
||||||
- `TEMPORAL_ENDPOINT` — temporal frontend hostname:port
|
|
||||||
- `TEMPORAL_NAMESPACE` — temporal namespace
|
|
||||||
|
|
||||||
### API Gateway (Route Optimization - Future)
|
|
||||||
- `GATEWAY_URL` — gateway base URL
|
|
||||||
|
|
||||||
### Memory Service Config
|
|
||||||
- `MEM_AUTH_MODE` — jwt | apikey | none
|
|
||||||
- `MEM_RATE_LIMIT_INGEST` — ingest requests per second
|
|
||||||
- `MEM_RATE_LIMIT_QUERY` — query requests per second
|
|
||||||
- `MEM_EMBEDDING_BATCH_SIZE` — batch size for embeddings
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Deployment Scenarios
|
|
||||||
|
|
||||||
### Production (SOPS-Encrypted ConfigMap)
|
|
||||||
|
|
||||||
**File**: `k8s/app/config.yaml`
|
|
||||||
|
|
||||||
Services use cluster-internal DNS:
|
|
||||||
```yaml
|
|
||||||
LLM_ENDPOINT: http://reasoning-predictor.llm-serving.svc.cluster.local:8000/v1/chat/completions
|
|
||||||
OPENSEARCH_HOST: opensearch.poimen.svc.cluster.local:9200
|
|
||||||
AUTHENTIK_ISSUER: https://authentik.auth.svc.cluster.local:9443/application/o/poimen/
|
|
||||||
TEMPORAL_ENDPOINT: temporal-frontend.temporal.svc.cluster.local:7233
|
|
||||||
GATEWAY_URL: http://api-gw.poimen.svc.cluster.local:8080
|
|
||||||
MEM_AUTH_MODE: jwt
|
|
||||||
```
|
|
||||||
|
|
||||||
**Deploy**:
|
|
||||||
```bash
|
|
||||||
# SOPS auto-decrypts based on .sops.yaml age key
|
|
||||||
kubectl apply -f k8s/app/config.yaml -k k8s/app/
|
|
||||||
```
|
|
||||||
|
|
||||||
### Local/Development (Plaintext ConfigMap)
|
|
||||||
|
|
||||||
**File**: `k8s/app/config.local.yaml`
|
|
||||||
|
|
||||||
Services via external URLs (ingress):
|
|
||||||
```yaml
|
|
||||||
LLM_ENDPOINT: https://api.riotpiao.com/v1/chat/completions
|
|
||||||
OPENSEARCH_HOST: opensearch.riotpiao.com:443
|
|
||||||
AUTHENTIK_ISSUER: https://authentik.riotpiao.com/application/o/poimen/
|
|
||||||
TEMPORAL_ENDPOINT: temporal.riotpiao.com:443
|
|
||||||
GATEWAY_URL: https://api.riotpiao.com
|
|
||||||
MEM_AUTH_MODE: none
|
|
||||||
```
|
|
||||||
|
|
||||||
**Deploy** (override production config):
|
|
||||||
```bash
|
|
||||||
# Delete prod config, apply local
|
|
||||||
kubectl delete configmap poimen-memory-config -n poimen
|
|
||||||
kubectl apply -f k8s/app/config.local.yaml
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Encrypting with SOPS
|
|
||||||
|
|
||||||
Production `config.yaml` is encrypted with SOPS (Age-based).
|
|
||||||
|
|
||||||
**Encrypt**:
|
|
||||||
```bash
|
|
||||||
sops -e k8s/app/config.yaml > k8s/app/config.yaml.enc
|
|
||||||
mv k8s/app/config.yaml.enc k8s/app/config.yaml
|
|
||||||
```
|
|
||||||
|
|
||||||
**Decrypt for editing** (SOPS auto-handles with $EDITOR):
|
|
||||||
```bash
|
|
||||||
sops k8s/app/config.yaml
|
|
||||||
```
|
|
||||||
|
|
||||||
**View decrypted** (without editing):
|
|
||||||
```bash
|
|
||||||
sops -d k8s/app/config.yaml
|
|
||||||
```
|
|
||||||
|
|
||||||
**.sops.yaml** defines encryption key:
|
|
||||||
```yaml
|
|
||||||
creation_rules:
|
|
||||||
- path_regex: k8s/app/config.yaml
|
|
||||||
key_groups:
|
|
||||||
- age:
|
|
||||||
- <age-public-key>
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Application Code Pattern
|
|
||||||
|
|
||||||
Example: Application should read URIs from ENV at startup.
|
|
||||||
|
|
||||||
```rust
|
|
||||||
// Pseudocode
|
|
||||||
let llm_endpoint = env::var("LLM_ENDPOINT")
|
|
||||||
.unwrap_or("http://localhost:11434/v1/chat/completions".to_string());
|
|
||||||
let opensearch_host = env::var("OPENSEARCH_HOST")
|
|
||||||
.unwrap_or("localhost:9200".to_string());
|
|
||||||
let auth_mode = env::var("MEM_AUTH_MODE")
|
|
||||||
.unwrap_or("none".to_string());
|
|
||||||
|
|
||||||
// Initialize clients with these URIs
|
|
||||||
let llm_client = LlmClient::new(llm_endpoint)?;
|
|
||||||
let search_client = OpenSearchClient::new(opensearch_host)?;
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Summary
|
|
||||||
|
|
||||||
| Aspect | Production | Local |
|
|
||||||
|--------|-----------|-------|
|
|
||||||
| **Config File** | `config.yaml` | `config.local.yaml` |
|
|
||||||
| **Encryption** | SOPS (Age) | Plaintext |
|
|
||||||
| **Service URIs** | Cluster-internal DNS | External HTTPS |
|
|
||||||
| **Auth Mode** | JWT (Authentik) | None (disabled) |
|
|
||||||
| **Rate Limits** | 100/1000 | 1000/10000 |
|
|
||||||
| **Deploy** | `kubectl apply -k k8s/app/` | `kubectl apply -f config.local.yaml` |
|
|
||||||
@@ -1,47 +0,0 @@
|
|||||||
# Local/Development configuration (plaintext, external URLs via ingress)
|
|
||||||
# Use this instead of config.yaml for local testing
|
|
||||||
# kubectl apply -f config.local.yaml
|
|
||||||
apiVersion: v1
|
|
||||||
kind: ConfigMap
|
|
||||||
metadata:
|
|
||||||
name: poimen-memory-config
|
|
||||||
namespace: poimen
|
|
||||||
labels:
|
|
||||||
app.kubernetes.io/name: poimen-memory
|
|
||||||
app.kubernetes.io/component: config
|
|
||||||
data:
|
|
||||||
# Auth mode: jwt | apikey | none (disabled for local testing)
|
|
||||||
MEM_AUTH_MODE: "none"
|
|
||||||
|
|
||||||
# Rate limiting (higher for testing)
|
|
||||||
MEM_RATE_LIMIT_INGEST: "1000"
|
|
||||||
MEM_RATE_LIMIT_QUERY: "10000"
|
|
||||||
MEM_IDEMPOTENCY_TTL_SECS: "86400"
|
|
||||||
|
|
||||||
# Embeddings
|
|
||||||
MEM_EMBEDDING_BATCH_SIZE: "32"
|
|
||||||
|
|
||||||
# Downstream services - external URLs via ingress
|
|
||||||
|
|
||||||
# LLM Service (via api.riotpiao.com ingress)
|
|
||||||
LLM_ENDPOINT: "https://api.riotpiao.com/v1/chat/completions"
|
|
||||||
LLM_API_BASE: "https://api.riotpiao.com/v1"
|
|
||||||
LLM_MODEL: "qwen:7b"
|
|
||||||
LLM_TIMEOUT_SECS: "60"
|
|
||||||
ENABLE_LLM_EXTRACTION: "true"
|
|
||||||
|
|
||||||
# OpenSearch (via ingress)
|
|
||||||
OPENSEARCH_HOST: "opensearch.riotpiao.com:443"
|
|
||||||
OPENSEARCH_SCHEME: "https"
|
|
||||||
OPENSEARCH_VERIFY_CERTS: "true"
|
|
||||||
|
|
||||||
# Authentik (via ingress - optional for local)
|
|
||||||
AUTHENTIK_ISSUER: "https://authentik.riotpiao.com/application/o/poimen/"
|
|
||||||
AUTHENTIK_VERIFY_SSL: "true"
|
|
||||||
|
|
||||||
# Temporal (via ingress)
|
|
||||||
TEMPORAL_ENDPOINT: "temporal.riotpiao.com:443"
|
|
||||||
TEMPORAL_NAMESPACE: "poimen"
|
|
||||||
|
|
||||||
# API Gateway (via ingress)
|
|
||||||
GATEWAY_URL: "https://api.riotpiao.com"
|
|
||||||
+11
-32
@@ -1,7 +1,5 @@
|
|||||||
# Production environment configuration for poimen-memory
|
# Non-sensitive environment variables for poimen-memory
|
||||||
# All services use cluster-internal DNS names
|
# Change these without redeploying secrets.
|
||||||
# This file is encrypted with SOPS in production
|
|
||||||
# For local dev, use plaintext version with external URLs
|
|
||||||
apiVersion: v1
|
apiVersion: v1
|
||||||
kind: ConfigMap
|
kind: ConfigMap
|
||||||
metadata:
|
metadata:
|
||||||
@@ -11,39 +9,20 @@ metadata:
|
|||||||
app.kubernetes.io/name: poimen-memory
|
app.kubernetes.io/name: poimen-memory
|
||||||
app.kubernetes.io/component: config
|
app.kubernetes.io/component: config
|
||||||
data:
|
data:
|
||||||
# Auth mode: jwt | apikey | none
|
# Auth mode: jwt | apikey
|
||||||
MEM_AUTH_MODE: "jwt"
|
MEM_AUTH_MODE: "none"
|
||||||
|
|
||||||
# Rate limiting
|
# Rate limiting
|
||||||
MEM_RATE_LIMIT_INGEST: "100"
|
MEM_RATE_LIMIT_INGEST: "100"
|
||||||
MEM_RATE_LIMIT_QUERY: "1000"
|
MEM_RATE_LIMIT_QUERY: "1000"
|
||||||
MEM_IDEMPOTENCY_TTL_SECS: "86400"
|
MEM_IDEMPOTENCY_TTL_SECS: "86400"
|
||||||
|
|
||||||
# Embeddings
|
# Embeddings
|
||||||
MEM_EMBEDDING_BATCH_SIZE: "32"
|
MEM_EMBEDDING_BATCH_SIZE: "32"
|
||||||
|
# OpenSearch
|
||||||
# Downstream services - read by application from ENV
|
OPENSEARCH_HOST: "opensearch.poimen.svc.cluster.local:9200"
|
||||||
# Internal cluster DNS (prod) / external URLs (local)
|
# Obsidian
|
||||||
|
OBSIDIAN_URL: "http://obsidian-server.poimen.svc.cluster.local:8080"
|
||||||
# LLM Service (entity extraction, fact extraction)
|
# LLM Configuration (for entity extraction)
|
||||||
LLM_ENDPOINT: "http://reasoning-predictor.llm-serving.svc.cluster.local:8000/v1/chat/completions"
|
LLM_ENDPOINT: "http://api-internal.riotpiao.com:8000/v1/chat/completions"
|
||||||
LLM_API_BASE: "http://reasoning-predictor.llm-serving.svc.cluster.local:8000/v1"
|
LLM_MODEL: "qwen:7b"
|
||||||
LLM_MODEL: "ornith:35b"
|
|
||||||
LLM_TIMEOUT_SECS: "30"
|
LLM_TIMEOUT_SECS: "30"
|
||||||
ENABLE_LLM_EXTRACTION: "true"
|
ENABLE_LLM_EXTRACTION: "true"
|
||||||
|
|
||||||
# OpenSearch (vector store, BM25 retrieval)
|
|
||||||
OPENSEARCH_HOST: "opensearch.poimen.svc.cluster.local:9200"
|
|
||||||
OPENSEARCH_SCHEME: "http"
|
|
||||||
OPENSEARCH_VERIFY_CERTS: "false"
|
|
||||||
|
|
||||||
# Authentik (OIDC provider)
|
|
||||||
AUTHENTIK_ISSUER: "https://authentik.auth.svc.cluster.local:9443/application/o/poimen/"
|
|
||||||
AUTHENTIK_VERIFY_SSL: "false"
|
|
||||||
|
|
||||||
# Temporal (workflow orchestration - future)
|
|
||||||
TEMPORAL_ENDPOINT: "temporal-frontend.temporal.svc.cluster.local:7233"
|
|
||||||
TEMPORAL_NAMESPACE: "poimen"
|
|
||||||
|
|
||||||
# API Gateway (external queue, route optimization - future)
|
|
||||||
GATEWAY_URL: "http://api-gw.poimen.svc.cluster.local:8080"
|
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user