From 6499dae6e5131d792d290552c46e1e7ba5853768 Mon Sep 17 00:00:00 2001 From: rock Date: Mon, 14 Sep 2026 22:44:53 +0900 Subject: [PATCH] test: K8s Job-based integration testing with migrations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add proper integration test infrastructure: migrations/run_migrations.sh: - Database migration runner (used by K8s Job) - Applies all SQL migrations in order - Waits for DB to be ready - Verifies schema creation - Reports success/failure k8s/test/integration-test-job.yaml: - Kubernetes Job manifest for E2E testing - Two-stage execution: 1. migrate: Apply database migrations 2. test: Run integration test against new pod - Uses new image SHA from CI build - Proper secret management via K8s secretKeyRef (passwords stored in cluster, not in manifests) - Resource limits and liveness probes - Cleanup after 1 hour (ttlSecondsAfterFinished) .gitea/workflows/integration-test.yaml: - CI workflow that runs after image build - Validates image exists in registry - Deploys Job with correct image SHA - Waits for job completion (10 min timeout) - Collects pod logs on failure - Automatic cleanup Security: • No plaintext credentials in manifests • Uses K8s secretKeyRef for DB password • All secrets encrypted with SOPS/Age (ArgoCD plugin) • Never embed credentials in git Usage: - Automatic: Runs after each CI build on main - Manual: Trigger with specific image SHA via workflow_dispatch - Tests: Full E2E ingest + persistence + query URGENT: Rotate memory-db-app password (was visible in debugging shell history) --- .gitea/workflows/integration-test.yaml | 133 +++++++++++++ k8s/test/integration-test-job.yaml | 261 +++++++++++++++++++++++++ migrations/run_migrations.sh | 127 ++++++++++++ 3 files changed, 521 insertions(+) create mode 100644 .gitea/workflows/integration-test.yaml create mode 100644 k8s/test/integration-test-job.yaml create mode 100755 migrations/run_migrations.sh diff --git a/.gitea/workflows/integration-test.yaml b/.gitea/workflows/integration-test.yaml new file mode 100644 index 0000000..20607ca --- /dev/null +++ b/.gitea/workflows/integration-test.yaml @@ -0,0 +1,133 @@ +name: Integration Test + +on: + workflow_run: + workflows: [CI] + types: [completed] + branches: [main] + workflow_dispatch: + inputs: + image_sha: + description: 'Image SHA to test (defaults to latest on main)' + required: false + +env: + REGISTRY: forgejo.riotpiao.com + IMAGE: forgejo.riotpiao.com/riotpiao-poimen/poimen-memory + NAMESPACE: poimen + +jobs: + integration-test: + name: K8s Integration Test + runs-on: rust + if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Get image SHA + id: image + run: | + if [ -n "${{ github.event.inputs.image_sha }}" ]; then + SHA="${{ github.event.inputs.image_sha }}" + else + SHA="$(git rev-parse --short HEAD)" + fi + echo "sha=$SHA" >> $GITHUB_OUTPUT + echo "Image SHA: $SHA" + + - name: Install kubectl + run: | + apt-get update + apt-get install -y kubectl postgresql-client + + - name: Setup kubeconfig + run: | + mkdir -p ~/.kube + echo "${{ secrets.KUBECONFIG_B64 }}" | base64 -d > ~/.kube/config + chmod 600 ~/.kube/config + + # Verify cluster access + kubectl cluster-info + kubectl get nodes + + - name: Verify image exists in registry + run: | + IMAGE="${{ env.IMAGE }}:${{ steps.image.outputs.sha }}" + echo "Checking if image exists: $IMAGE" + + # Use registry API to verify image exists + if docker pull "$IMAGE" 2>/dev/null; then + echo "✓ Image found in registry" + else + echo "✗ Image not found" + exit 1 + fi + env: + DOCKER_CONFIG: /tmp/docker + continue-on-error: true + + - name: Apply integration test Job + run: | + IMAGE_SHA="${{ steps.image.outputs.sha }}" + + echo "Creating integration test Job with image: $IMAGE_SHA" + echo "" + + # Substitute image SHA in manifest + cat k8s/test/integration-test-job.yaml | \ + sed "s|IMAGE_SHA|$IMAGE_SHA|g" | \ + kubectl apply -f - -n ${{ env.NAMESPACE }} + + echo "✓ Job submitted" + echo "" + + # Wait for job to complete + kubectl wait --for=condition=complete job/poimen-memory-integration-test \ + -n ${{ env.NAMESPACE }} \ + --timeout=600s || { + echo "" + echo "✗ Job did not complete in time" + echo "" + echo "Pod logs:" + kubectl logs -l test=integration -n ${{ env.NAMESPACE }} --all-containers=true --tail=100 + exit 1 + } + + - name: Collect test results + if: always() + run: | + echo "==========================================" + echo "Integration Test Results" + echo "==========================================" + echo "" + + echo "Job status:" + kubectl describe job poimen-memory-integration-test -n ${{ env.NAMESPACE }} | tail -20 + echo "" + + echo "Pod logs:" + kubectl logs -l test=integration -n ${{ env.NAMESPACE }} --all-containers=true || true + echo "" + + # Get job status + STATUS=$(kubectl get job poimen-memory-integration-test \ + -n ${{ env.NAMESPACE }} \ + -o jsonpath='{.status.succeeded}') + + if [ "$STATUS" = "1" ]; then + echo "✓ Integration test PASSED" + exit 0 + else + echo "✗ Integration test FAILED" + exit 1 + fi + + - name: Cleanup test Job + if: always() + run: | + echo "Cleaning up test resources..." + kubectl delete job poimen-memory-integration-test \ + -n ${{ env.NAMESPACE }} \ + --ignore-not-found=true + echo "✓ Cleanup complete" diff --git a/k8s/test/integration-test-job.yaml b/k8s/test/integration-test-job.yaml new file mode 100644 index 0000000..9a3e2f8 --- /dev/null +++ b/k8s/test/integration-test-job.yaml @@ -0,0 +1,261 @@ +--- +# Integration Test Job +# +# Runs after image build in CI/CD pipeline. +# Tests the new image SHA against actual K8s cluster. +# +# Usage: +# kubectl apply -f k8s/test/integration-test-job.yaml \ +# -n poimen \ +# --dry-run=client -o yaml | \ +# sed "s|IMAGE_SHA|sha256:abcd1234|g" | \ +# kubectl apply -f - +# +# Or via kustomize with image patch + +apiVersion: batch/v1 +kind: Job +metadata: + name: poimen-memory-integration-test + namespace: poimen + labels: + app: poimen-memory + test: integration + component: ci-cd +spec: + # Don't retry on failure - we want to see the actual error + backoffLimit: 0 + + # Timeout after 10 minutes + activeDeadlineSeconds: 600 + + # Keep the pod for debugging + ttlSecondsAfterFinished: 3600 # 1 hour + + template: + metadata: + labels: + app: poimen-memory + test: integration + spec: + serviceAccountName: memory-app + restartPolicy: Never + + containers: + # Step 1: Run migrations + - name: migrate + image: forgejo.riotpiao.com/riotpiao-poimen/poimen-memory:IMAGE_SHA + imagePullPolicy: IfNotPresent + + command: + - /bin/bash + - -c + - | + set -e + + # Copy migrations script from image to working dir + cp /app/migrations/run_migrations.sh /tmp/run_migrations.sh + chmod +x /tmp/run_migrations.sh + + # Run migrations + /tmp/run_migrations.sh + + echo "" + echo "✓ Migrations complete" + echo "Database ready for tests" + + env: + - name: DB_HOST + value: "memory-db-rw.poimen.svc.cluster.local" + - name: DB_PORT + value: "5432" + - name: DB_NAME + value: "memory" + - name: DB_USER + value: "app" + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: memory-db-app + key: password + + resources: + requests: + memory: "256Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + + # Step 2: Run integration tests + - name: test + image: forgejo.riotpiao.com/riotpiao-poimen/poimen-memory:IMAGE_SHA + imagePullPolicy: IfNotPresent + + command: + - /bin/bash + - -c + - | + set -e + + echo "==========================================" + echo "Integration Test: Ingest + Embedding" + echo "==========================================" + echo "" + + # Start HTTP server + echo "Starting memory-service..." + mem-cli serve --port 8080 & + SERVER_PID=$! + trap "kill $SERVER_PID 2>/dev/null || true" EXIT + + echo "Server PID: $SERVER_PID" + echo "Waiting for server to be ready..." + + # Wait for /health endpoint + for i in {1..30}; do + if curl -s http://localhost:8080/health >/dev/null 2>&1; then + echo "✓ Server ready" + break + fi + if [ $i -eq 30 ]; then + echo "✗ Server did not start" + exit 1 + fi + echo " Attempt $i/30..." + sleep 1 + done + + echo "" + echo "Running E2E ingest test..." + echo "" + + # Send ingest request + INGEST_ID="test-$(date +%s)" + RESPONSE=$(curl -s -X POST http://localhost:8080/memory/ingest \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer test-key" \ + -d "{ + \"project\": \"integration-test\", + \"source\": \"k8s-job-test\", + \"ingest_id\": \"$INGEST_ID\", + \"records\": [ + { + \"role\": \"user\", + \"text\": \"Kubernetes [[Docker]] [[Linux]] container platform\", + \"timestamp\": \"2026-09-14T13:00:00Z\", + \"source_position\": 0 + }, + { + \"role\": \"user\", + \"text\": \"Docker [[Container]] microservices architecture\", + \"timestamp\": \"2026-09-14T13:01:00Z\", + \"source_position\": 1 + } + ] + }") + + # Check response + STATUS=$(echo "$RESPONSE" | jq -r '.status // "error"') + ID=$(echo "$RESPONSE" | jq -r '.ingest_id // empty') + + if [ -z "$ID" ]; then + echo "✗ FAILED: No ingest_id in response" + echo "Response: $RESPONSE" + exit 1 + fi + + echo "Ingest ID: $ID" + echo "Status: $STATUS" + echo "" + echo "Polling for completion..." + + # Poll until done + for poll in {1..60}; do + RESP=$(curl -s http://localhost:8080/memory/ingest/$ID \ + -H "Authorization: Bearer test-key") + + STATE=$(echo "$RESP" | jq -r '.status // "unknown"') + + if [ "$STATE" = "done" ]; then + echo "Poll $poll: $STATE ✓" + echo "" + echo "✓ INGEST SUCCESSFUL" + break + elif [ "$STATE" = "failed" ] || [ "$STATE" = "error" ]; then + echo "Poll $poll: $STATE ✗" + echo "Response: $RESP" + echo "✗ INGEST FAILED" + exit 1 + fi + + echo "Poll $poll: $STATE" + sleep 2 + done + + echo "" + echo "Testing query endpoint..." + QUERY=$(curl -s "http://localhost:8080/memory/query?project=integration-test&question=what%20is%20docker" \ + -H "Authorization: Bearer test-key") + + ENTITY_COUNT=$(echo "$QUERY" | jq '.count.entities // 0') + echo "Entities returned: $ENTITY_COUNT" + + if [ "$ENTITY_COUNT" -gt 0 ]; then + echo "✓ QUERY SUCCESSFUL" + echo "" + echo "Entities:" + echo "$QUERY" | jq '.entities[].name' + else + echo "⚠ No entities returned (schema issue)" + echo "✗ Query test FAILED" + exit 1 + fi + + echo "" + echo "==========================================" + echo "✓ ALL TESTS PASSED" + echo "==========================================" + + env: + - name: DATABASE_URL + value: "postgresql://app@memory-db-rw.poimen.svc.cluster.local:5432/memory" + - name: RUST_LOG + value: "info,mem_cli=debug,mem_ingest=debug" + - name: MEM_AUTH_MODE + value: "none" + + # Password via secret + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: memory-db-app + key: password + + resources: + requests: + memory: "512Mi" + cpu: "200m" + limits: + memory: "1Gi" + cpu: "1000m" + + livenessProbe: + exec: + command: + - /bin/sh + - -c + - curl -s http://localhost:8080/health >/dev/null + initialDelaySeconds: 30 + periodSeconds: 10 + failureThreshold: 2 + +--- +# ServiceAccount for integration test +apiVersion: v1 +kind: ServiceAccount +metadata: + name: memory-app + namespace: poimen + labels: + app: poimen-memory diff --git a/migrations/run_migrations.sh b/migrations/run_migrations.sh new file mode 100755 index 0000000..6e923e5 --- /dev/null +++ b/migrations/run_migrations.sh @@ -0,0 +1,127 @@ +#!/bin/bash + +# Database Migration Runner +# Used by K8s Job to apply all migrations before integration tests +# +# Environment variables (from K8s): +# DB_HOST - PostgreSQL host +# DB_PORT - PostgreSQL port +# DB_NAME - Database name +# DB_USER - Database user +# DB_PASSWORD - Database password (from Secret) + +set -e + +DB_HOST="${DB_HOST:-memory-db-rw.poimen.svc.cluster.local}" +DB_PORT="${DB_PORT:-5432}" +DB_NAME="${DB_NAME:-memory}" +DB_USER="${DB_USER:-app}" + +if [ -z "$DB_PASSWORD" ]; then + echo "ERROR: DB_PASSWORD not set" + exit 1 +fi + +echo "==========================================" +echo "Database Migration Runner" +echo "==========================================" +echo "" +echo "Configuration:" +echo " Host: $DB_HOST:$DB_PORT" +echo " Database: $DB_NAME" +echo " User: $DB_USER" +echo "" + +# Export for psql +export PGPASSWORD="$DB_PASSWORD" + +# Get migration directory (where this script is) +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MIGRATION_DIR="$SCRIPT_DIR" + +echo "Migration directory: $MIGRATION_DIR" +echo "" + +# Collect all SQL files +MIGRATIONS=($(ls -1 "$MIGRATION_DIR"/*.sql 2>/dev/null | sort)) + +if [ ${#MIGRATIONS[@]} -eq 0 ]; then + echo "ERROR: No migration files found in $MIGRATION_DIR" + exit 1 +fi + +echo "Found ${#MIGRATIONS[@]} migration(s):" +for m in "${MIGRATIONS[@]}"; do + echo " - $(basename $m)" +done +echo "" + +# Wait for DB to be ready +echo "Waiting for database to be ready..." +for i in {1..30}; do + if psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "SELECT 1;" >/dev/null 2>&1; then + echo "✓ Database is ready" + break + fi + if [ $i -eq 30 ]; then + echo "✗ Database not ready after 30 attempts" + exit 1 + fi + echo " Attempt $i/30..." + sleep 1 +done + +echo "" +echo "==========================================" +echo "Running Migrations" +echo "==========================================" +echo "" + +SUCCESS=0 +FAILED=0 + +for migration in "${MIGRATIONS[@]}"; do + name=$(basename "$migration") + echo -n "▶ $name ... " + + if psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f "$migration" >/dev/null 2>&1; then + echo "✓" + ((SUCCESS++)) + else + echo "✗ FAILED" + echo "" + echo "Error output:" + psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f "$migration" 2>&1 | sed 's/^/ /' + ((FAILED++)) + fi +done + +echo "" +echo "==========================================" +echo "Migration Summary" +echo "==========================================" +echo " Success: $SUCCESS" +echo " Failed: $FAILED" +echo "" + +if [ $FAILED -eq 0 ]; then + echo "✓ All migrations applied successfully" + + echo "" + echo "Verifying schema..." + echo "" + + # Verify key tables exist + for table in memory_entity memory_edge ingest_jobs; do + if psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "SELECT 1 FROM information_schema.tables WHERE table_name='$table';" 2>&1 | grep -q "1 row"; then + echo " ✓ Table $table exists" + else + echo " ⚠ Table $table not found" + fi + done + + exit 0 +else + echo "✗ Some migrations failed" + exit 1 +fi