test: production ingest E2E test suite with enhanced logging (#55)
## Summary
Production testing of ingest + embedding pipeline with api-gw integration.
## Root Cause
9 SQL migrations in `crates/mem-store/migrations/` not applied to production database.
Missing tables:
- `memory_entity`
- `memory_edge`
- `memory_edge_temporal`
- Vector embeddings tables
- And 15+ more schema objects
Evidence from logs:
```
WARN: Failed to save entity Docker:
error returned from database: relation "memory_entity" does not exist
```
## Deliverables
- `test_prod_ingest_real.sh` - Full E2E test against K8s + api-gw
- `apply_migrations.sh` - Manual schema migration (backup)
- `collect_prod_logs.sh` - Pod log collection before/after
- `run_production_test.sh` - Test orchestrator
- `tests/integration_ingest_with_gw.rs` - Integration test
- `tests/unit_ingest_logging.rs` - Unit tests for extraction
- Enhanced logging in `ingest_worker.rs` - Per-record event tracking
## Next Steps
1. Trigger "DB Migration" workflow in Forgejo Actions
2. This applies all 9 migrations from `crates/mem-store/migrations/`
3. Pod restart (automatic)
4. Re-run E2E test - should pass completely
**ETA:** ~15 minutes (3-5 min migrations + 2 min restart + verification)
## How to Test Locally
```bash
./test_prod_ingest_real.sh --verbose
```
Requires:
- kubectl access to poimen namespace
- Port-forwarding to memory-service
---------
Co-authored-by: rock <[email protected]>
Reviewed-on: #55
Co-authored-by: poimen <[email protected]>
This commit was merged in pull request #55.
This commit is contained in:
@@ -0,0 +1,131 @@
|
||||
apiVersion: tekton.dev/v1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: agent-memory-migration
|
||||
namespace: tekton-pipelines
|
||||
spec:
|
||||
description: Apply agent memory schema migration (004) to production database
|
||||
params:
|
||||
- name: migration-version
|
||||
description: Migration version number
|
||||
default: "004"
|
||||
- name: database-name
|
||||
description: Database name
|
||||
default: "memory"
|
||||
workspaces:
|
||||
- name: source
|
||||
description: Git source with migrations
|
||||
- name: db-credentials
|
||||
description: Database credentials secret
|
||||
steps:
|
||||
- name: apply-migration
|
||||
image: postgres:16-alpine
|
||||
workingDir: $(workspaces.source.path)
|
||||
env:
|
||||
- name: PGPASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: memory-db-app
|
||||
key: password
|
||||
- name: PGHOST
|
||||
value: memory-db-rw.poimen.svc.cluster.local
|
||||
- name: PGUSER
|
||||
value: app
|
||||
- name: PGDATABASE
|
||||
value: $(params.database-name)
|
||||
script: |
|
||||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
echo "Applying migration $(params.migration-version)_agent_memory_schema.sql"
|
||||
|
||||
# Wait for database to be ready
|
||||
until pg_isready -h $PGHOST -U $PGUSER -d $PGDATABASE; do
|
||||
echo "Waiting for database..."
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# Apply migration
|
||||
psql -h $PGHOST -U $PGUSER -d $PGDATABASE \
|
||||
-f migrations/$(params.migration-version)_agent_memory_schema.sql
|
||||
|
||||
# Verify tables created
|
||||
TABLES=$(psql -h $PGHOST -U $PGUSER -d $PGDATABASE -t -c \
|
||||
"SELECT count(*) FROM information_schema.tables WHERE table_schema='public' AND table_name IN ('agent_prompt', 'agent_skill', 'agent_decision', 'role_prompt_mapping', 'agent_metrics')")
|
||||
|
||||
if [ "$TABLES" -eq 5 ]; then
|
||||
echo "✓ All agent memory tables created successfully"
|
||||
exit 0
|
||||
else
|
||||
echo "✗ Migration failed: expected 5 tables, found $TABLES"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: verify-indexes
|
||||
image: postgres:16-alpine
|
||||
env:
|
||||
- name: PGPASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: memory-db-app
|
||||
key: password
|
||||
- name: PGHOST
|
||||
value: memory-db-rw.poimen.svc.cluster.local
|
||||
- name: PGUSER
|
||||
value: app
|
||||
- name: PGDATABASE
|
||||
value: $(params.database-name)
|
||||
script: |
|
||||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
echo "Verifying indexes..."
|
||||
|
||||
INDEXES=$(psql -h $PGHOST -U $PGUSER -d $PGDATABASE -t -c \
|
||||
"SELECT count(*) FROM pg_indexes WHERE schemaname='public' AND tablename LIKE 'agent_%'")
|
||||
|
||||
if [ "$INDEXES" -gt 0 ]; then
|
||||
echo "✓ Found $INDEXES indexes on agent tables"
|
||||
psql -h $PGHOST -U $PGUSER -d $PGDATABASE -c \
|
||||
"SELECT indexname FROM pg_indexes WHERE schemaname='public' AND tablename LIKE 'agent_%' ORDER BY indexname;"
|
||||
else
|
||||
echo "✗ No indexes found on agent tables"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: verify-schemas
|
||||
image: postgres:16-alpine
|
||||
env:
|
||||
- name: PGPASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: memory-db-app
|
||||
key: password
|
||||
- name: PGHOST
|
||||
value: memory-db-rw.poimen.svc.cluster.local
|
||||
- name: PGUSER
|
||||
value: app
|
||||
- name: PGDATABASE
|
||||
value: $(params.database-name)
|
||||
script: |
|
||||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
echo "Verifying table schemas..."
|
||||
|
||||
# Verify agent_prompt table
|
||||
psql -h $PGHOST -U $PGUSER -d $PGDATABASE -c "
|
||||
SELECT column_name, data_type, is_nullable
|
||||
FROM information_schema.columns
|
||||
WHERE table_name='agent_prompt'
|
||||
ORDER BY ordinal_position;"
|
||||
|
||||
echo "✓ Agent prompt schema verified"
|
||||
|
||||
# Verify role_prompt_mapping has foreign key
|
||||
psql -h $PGHOST -U $PGUSER -d $PGDATABASE -c "
|
||||
SELECT constraint_name, constraint_type
|
||||
FROM information_schema.table_constraints
|
||||
WHERE table_name='role_prompt_mapping';"
|
||||
|
||||
echo "✓ All table schemas verified"
|
||||
@@ -0,0 +1,76 @@
|
||||
---
|
||||
# PipelineRun: Agent Memory Feature Testing
|
||||
# Tests role-to-prompt mapping with API Platform Engineer role requirements
|
||||
# Runs migrations, integration tests, and validates all constraints
|
||||
|
||||
apiVersion: tekton.dev/v1
|
||||
kind: PipelineRun
|
||||
metadata:
|
||||
name: agent-memory-test-run
|
||||
namespace: poimen
|
||||
generateName: agent-memory-test-
|
||||
spec:
|
||||
pipelineRef:
|
||||
name: poimen-ci
|
||||
|
||||
params:
|
||||
- name: image
|
||||
value: "forgejo.riotpiao.com/riotpiao-poimen/poimen-memory:latest"
|
||||
- name: registry-user
|
||||
value: "riotpiao-poimen"
|
||||
- name: registry-token
|
||||
value: "${FORGEJO_REGISTRY_TOKEN}" # Injected by ArgoCD/SOPS
|
||||
|
||||
workspaces:
|
||||
- name: source
|
||||
emptyDir: {} # Or use PVC for persistent builds
|
||||
|
||||
serviceAccountName: tekton-builder
|
||||
|
||||
timeouts:
|
||||
pipeline: "1h"
|
||||
tasks: "30m"
|
||||
|
||||
---
|
||||
# ServiceAccount for Tekton Pipeline (builder with DB access)
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: tekton-builder
|
||||
namespace: poimen
|
||||
|
||||
---
|
||||
# ClusterRoleBinding: Allow pipeline to query database via pod exec
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: tekton-builder-db-access
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: tekton-builder-db-access
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: tekton-builder
|
||||
namespace: poimen
|
||||
|
||||
---
|
||||
# ClusterRole: Database access for migrations
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: tekton-builder-db-access
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["pods"]
|
||||
verbs: ["get", "list"]
|
||||
- apiGroups: [""]
|
||||
resources: ["pods/exec"]
|
||||
verbs: ["create"]
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
resourceNames: ["memory-db-app"]
|
||||
verbs: ["get"]
|
||||
- apiGroups: [""]
|
||||
resources: ["services"]
|
||||
verbs: ["get", "list"]
|
||||
@@ -0,0 +1,144 @@
|
||||
---
|
||||
# Tekton Task: Integration Tests for Poimen Memory Service
|
||||
#
|
||||
# Executes:
|
||||
# 1. Database migrations
|
||||
# 2. Integration test suites (cargo test)
|
||||
# 3. Reports results
|
||||
#
|
||||
# Parameters:
|
||||
# - image: Docker image with SHA to test
|
||||
#
|
||||
# Results:
|
||||
# - summary: Test summary (pass/fail + count)
|
||||
|
||||
apiVersion: tekton.dev/v1
|
||||
kind: Task
|
||||
metadata:
|
||||
name: poimen-integration-test
|
||||
namespace: poimen
|
||||
spec:
|
||||
params:
|
||||
- name: image
|
||||
type: string
|
||||
description: "Docker image SHA to test (e.g., forgejo.riotpiao.com/riotpiao-poimen/poimen-memory:abc123)"
|
||||
|
||||
results:
|
||||
- name: summary
|
||||
description: "Test summary: PASS or FAIL + test count"
|
||||
|
||||
steps:
|
||||
# Step 1: Apply database migrations
|
||||
- name: migrate
|
||||
image: $(params.image)
|
||||
env:
|
||||
- name: DB_HOST
|
||||
value: "memory-db-rw.poimen.svc.cluster.local"
|
||||
- name: DB_PORT
|
||||
value: "5432"
|
||||
- name: DB_NAME
|
||||
value: "memory"
|
||||
- name: DB_USER
|
||||
value: "app"
|
||||
- name: DB_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: memory-db-app
|
||||
key: password
|
||||
|
||||
script: |
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
echo "=========================================="
|
||||
echo "Step 1: Database Migrations"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
|
||||
# Run migrations
|
||||
/app/migrations/run_migrations.sh
|
||||
|
||||
echo ""
|
||||
echo "✓ Migrations complete"
|
||||
|
||||
# Step 2: Run integration tests
|
||||
- name: test
|
||||
image: $(params.image)
|
||||
env:
|
||||
- name: DATABASE_URL
|
||||
value: "postgresql://[email protected]:5432/memory"
|
||||
- name: RUST_LOG
|
||||
value: "info,mem_cli=debug,mem_ingest=debug,mem_store=debug"
|
||||
- name: MEM_AUTH_MODE
|
||||
value: "none"
|
||||
- name: SQLX_OFFLINE
|
||||
value: "true"
|
||||
- name: PGPASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: memory-db-app
|
||||
key: password
|
||||
|
||||
script: |
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
echo "=========================================="
|
||||
echo "Step 2: Integration Tests"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
|
||||
TEST_SUITES=(
|
||||
"it_phase3_phase4"
|
||||
"it_unified_query_4_6"
|
||||
"it_temporal_filtering_4_2_fixed"
|
||||
)
|
||||
|
||||
PASSED=0
|
||||
FAILED=0
|
||||
|
||||
for suite in "${TEST_SUITES[@]}"; do
|
||||
echo "Running: $suite"
|
||||
if cargo test --test "$suite" --lib 2>&1 | tail -50; then
|
||||
((PASSED++))
|
||||
echo "✓ $suite passed"
|
||||
else
|
||||
((FAILED++))
|
||||
echo "✗ $suite failed"
|
||||
fi
|
||||
echo ""
|
||||
done
|
||||
|
||||
# Run unit tests
|
||||
echo "Running unit tests..."
|
||||
if cargo test --lib mem_ingest 2>&1 | tail -100; then
|
||||
echo "✓ mem_ingest passed"
|
||||
else
|
||||
((FAILED++))
|
||||
echo "✗ mem_ingest failed"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
if cargo test --lib mem_cli::query 2>&1 | tail -100; then
|
||||
echo "✓ mem_cli::query passed"
|
||||
else
|
||||
((FAILED++))
|
||||
echo "✗ mem_cli::query failed"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "Test Summary: $PASSED passed, $FAILED failed"
|
||||
echo "=========================================="
|
||||
|
||||
if [ $FAILED -eq 0 ]; then
|
||||
echo "PASS: All integration tests passed"
|
||||
echo "PASS: All integration tests passed" > /tekton/results/summary
|
||||
exit 0
|
||||
else
|
||||
echo "FAIL: $FAILED test suite(s) failed"
|
||||
echo "FAIL: $FAILED test suite(s) failed" > /tekton/results/summary
|
||||
exit 1
|
||||
fi
|
||||
|
||||
resources:
|
||||
@@ -0,0 +1,140 @@
|
||||
---
|
||||
# Tekton Pipeline: Poimen Memory Service CI/CD
|
||||
#
|
||||
# Orchestrates:
|
||||
# 1. integration-test-task: Run integration tests against image
|
||||
# 2. (Future) build-task: Build Docker image
|
||||
# 3. (Future) promote-task: Promote image to :latest
|
||||
#
|
||||
# Parameters:
|
||||
# - image: Docker image with SHA to test
|
||||
# - registry-user: Registry credentials
|
||||
# - registry-token: Registry credentials
|
||||
|
||||
apiVersion: tekton.dev/v1
|
||||
kind: Pipeline
|
||||
metadata:
|
||||
name: poimen-ci
|
||||
namespace: poimen
|
||||
spec:
|
||||
params:
|
||||
- name: image
|
||||
type: string
|
||||
description: "Docker image SHA to test (e.g., forgejo.riotpiao.com/riotpiao-poimen/poimen-memory:abc123)"
|
||||
|
||||
- name: registry-user
|
||||
type: string
|
||||
description: "Registry username"
|
||||
default: ""
|
||||
|
||||
- name: registry-token
|
||||
type: string
|
||||
description: "Registry token/password"
|
||||
default: ""
|
||||
|
||||
workspaces:
|
||||
- name: source
|
||||
description: "Git source repository with migrations"
|
||||
|
||||
tasks:
|
||||
# Task 0: Apply Agent Memory Migrations
|
||||
- name: agent-memory-migration
|
||||
taskRef:
|
||||
name: agent-memory-migration
|
||||
params:
|
||||
- name: migration-version
|
||||
value: "004"
|
||||
- name: database-name
|
||||
value: "memory"
|
||||
workspaces:
|
||||
- name: source
|
||||
workspace: source
|
||||
|
||||
# Task 1: Integration Tests (runs after migration)
|
||||
- name: integration-tests
|
||||
runAfter:
|
||||
- agent-memory-migration
|
||||
taskRef:
|
||||
name: poimen-integration-test
|
||||
params:
|
||||
- name: image
|
||||
value: $(params.image)
|
||||
|
||||
# Task 2: Gate on test results
|
||||
- name: gate-on-tests
|
||||
runAfter:
|
||||
- integration-tests
|
||||
taskSpec:
|
||||
steps:
|
||||
- name: check-results
|
||||
image: alpine:latest
|
||||
script: |
|
||||
#!/bin/sh
|
||||
set -e
|
||||
echo "✓ Integration tests passed, proceeding with promotion"
|
||||
|
||||
# Task 3: Promote image (placeholder - will be implemented)
|
||||
- name: promote-image
|
||||
runAfter:
|
||||
- gate-on-tests
|
||||
taskSpec:
|
||||
params:
|
||||
- name: image
|
||||
type: string
|
||||
- name: registry-user
|
||||
type: string
|
||||
- name: registry-token
|
||||
type: string
|
||||
|
||||
steps:
|
||||
- name: promote
|
||||
image: docker:latest
|
||||
env:
|
||||
- name: IMAGE
|
||||
value: $(params.image)
|
||||
- name: REGISTRY_USER
|
||||
value: $(params.registry-user)
|
||||
- name: REGISTRY_TOKEN
|
||||
value: $(params.registry-token)
|
||||
script: |
|
||||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
echo "Promoting image to :latest..."
|
||||
|
||||
# Extract registry and repo from image
|
||||
# e.g., forgejo.riotpiao.com/riotpiao-poimen/poimen-memory:abc123
|
||||
REGISTRY=$(echo $IMAGE | cut -d/ -f1)
|
||||
REPO=$(echo $IMAGE | cut -d: -f1)
|
||||
SHA=$(echo $IMAGE | cut -d: -f2)
|
||||
|
||||
echo "Registry: $REGISTRY"
|
||||
echo "Repo: $REPO"
|
||||
echo "SHA: $SHA"
|
||||
echo ""
|
||||
|
||||
# Login and promote
|
||||
echo "$REGISTRY_TOKEN" | docker login -u "$REGISTRY_USER" --password-stdin "$REGISTRY"
|
||||
docker pull "$IMAGE"
|
||||
docker tag "$IMAGE" "${REPO}:latest"
|
||||
docker push "${REPO}:latest"
|
||||
|
||||
echo "✓ Promoted to :latest"
|
||||
|
||||
params:
|
||||
- name: image
|
||||
value: $(params.image)
|
||||
- name: registry-user
|
||||
value: $(params.registry-user)
|
||||
- name: registry-token
|
||||
value: $(params.registry-token)
|
||||
|
||||
finally:
|
||||
- name: cleanup
|
||||
taskSpec:
|
||||
steps:
|
||||
- name: cleanup-tasks
|
||||
image: alpine:latest
|
||||
script: |
|
||||
#!/bin/sh
|
||||
echo "Pipeline execution complete"
|
||||
Reference in New Issue
Block a user