test: K8s Job-based integration testing with migrations

Add proper integration test infrastructure: migrations/run_migrations.sh: - Database migration runner (used by K8s Job) - Applies all SQL migrations in order - Waits for DB to be ready - Verifies schema creation - Reports success/failure k8s/test/integration-test-job.yaml: - Kubernetes Job manifest for E2E testing - Two-stage execution: 1. migrate: Apply database migrations 2. test: Run integration test against new pod - Uses new image SHA from CI build - Proper secret management via K8s secretKeyRef (passwords stored in cluster, not in manifests) - Resource limits and liveness probes - Cleanup after 1 hour (ttlSecondsAfterFinished) .gitea/workflows/integration-test.yaml: - CI workflow that runs after image build - Validates image exists in registry - Deploys Job with correct image SHA - Waits for job completion (10 min timeout) - Collects pod logs on failure - Automatic cleanup Security: • No plaintext credentials in manifests • Uses K8s secretKeyRef for DB password • All secrets encrypted with SOPS/Age (ArgoCD plugin) • Never embed credentials in git Usage: - Automatic: Runs after each CI build on main - Manual: Trigger with specific image SHA via workflow_dispatch - Tests: Full E2E ingest + persistence + query URGENT: Rotate memory-db-app password (was visible in debugging shell history)
2026-09-14 22:44:53 +09:00
parent 6915dc2462
commit 6499dae6e5
3 changed files with 521 additions and 0 deletions
@@ -0,0 +1,133 @@
+name: Integration Test
+
+on:
+  workflow_run:
+    workflows: [CI]
+    types: [completed]
+    branches: [main]
+  workflow_dispatch:
+    inputs:
+      image_sha:
+        description: 'Image SHA to test (defaults to latest on main)'
+        required: false
+
+env:
+  REGISTRY: forgejo.riotpiao.com
+  IMAGE: forgejo.riotpiao.com/riotpiao-poimen/poimen-memory
+  NAMESPACE: poimen
+
+jobs:
+  integration-test:
+    name: K8s Integration Test
+    runs-on: rust
+    if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success'
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Get image SHA
+        id: image
+        run: |
+          if [ -n "${{ github.event.inputs.image_sha }}" ]; then
+            SHA="${{ github.event.inputs.image_sha }}"
+          else
+            SHA="$(git rev-parse --short HEAD)"
+          fi
+          echo "sha=$SHA" >> $GITHUB_OUTPUT
+          echo "Image SHA: $SHA"
+
+      - name: Install kubectl
+        run: |
+          apt-get update
+          apt-get install -y kubectl postgresql-client
+
+      - name: Setup kubeconfig
+        run: |
+          mkdir -p ~/.kube
+          echo "${{ secrets.KUBECONFIG_B64 }}" | base64 -d > ~/.kube/config
+          chmod 600 ~/.kube/config
+          
+          # Verify cluster access
+          kubectl cluster-info
+          kubectl get nodes
+
+      - name: Verify image exists in registry
+        run: |
+          IMAGE="${{ env.IMAGE }}:${{ steps.image.outputs.sha }}"
+          echo "Checking if image exists: $IMAGE"
+          
+          # Use registry API to verify image exists
+          if docker pull "$IMAGE" 2>/dev/null; then
+            echo "✓ Image found in registry"
+          else
+            echo "✗ Image not found"
+            exit 1
+          fi
+        env:
+          DOCKER_CONFIG: /tmp/docker
+        continue-on-error: true
+
+      - name: Apply integration test Job
+        run: |
+          IMAGE_SHA="${{ steps.image.outputs.sha }}"
+          
+          echo "Creating integration test Job with image: $IMAGE_SHA"
+          echo ""
+          
+          # Substitute image SHA in manifest
+          cat k8s/test/integration-test-job.yaml | \
+            sed "s|IMAGE_SHA|$IMAGE_SHA|g" | \
+            kubectl apply -f - -n ${{ env.NAMESPACE }}
+          
+          echo "✓ Job submitted"
+          echo ""
+          
+          # Wait for job to complete
+          kubectl wait --for=condition=complete job/poimen-memory-integration-test \
+            -n ${{ env.NAMESPACE }} \
+            --timeout=600s || {
+              echo ""
+              echo "✗ Job did not complete in time"
+              echo ""
+              echo "Pod logs:"
+              kubectl logs -l test=integration -n ${{ env.NAMESPACE }} --all-containers=true --tail=100
+              exit 1
+            }
+
+      - name: Collect test results
+        if: always()
+        run: |
+          echo "=========================================="
+          echo "Integration Test Results"
+          echo "=========================================="
+          echo ""
+          
+          echo "Job status:"
+          kubectl describe job poimen-memory-integration-test -n ${{ env.NAMESPACE }} | tail -20
+          echo ""
+          
+          echo "Pod logs:"
+          kubectl logs -l test=integration -n ${{ env.NAMESPACE }} --all-containers=true || true
+          echo ""
+          
+          # Get job status
+          STATUS=$(kubectl get job poimen-memory-integration-test \
+            -n ${{ env.NAMESPACE }} \
+            -o jsonpath='{.status.succeeded}')
+          
+          if [ "$STATUS" = "1" ]; then
+            echo "✓ Integration test PASSED"
+            exit 0
+          else
+            echo "✗ Integration test FAILED"
+            exit 1
+          fi
+
+      - name: Cleanup test Job
+        if: always()
+        run: |
+          echo "Cleaning up test resources..."
+          kubectl delete job poimen-memory-integration-test \
+            -n ${{ env.NAMESPACE }} \
+            --ignore-not-found=true
+          echo "✓ Cleanup complete"
@@ -0,0 +1,261 @@
+---
+# Integration Test Job
+# 
+# Runs after image build in CI/CD pipeline.
+# Tests the new image SHA against actual K8s cluster.
+#
+# Usage:
+#   kubectl apply -f k8s/test/integration-test-job.yaml \
+#     -n poimen \
+#     --dry-run=client -o yaml | \
+#     sed "s|IMAGE_SHA|sha256:abcd1234|g" | \
+#     kubectl apply -f -
+#
+# Or via kustomize with image patch
+
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: poimen-memory-integration-test
+  namespace: poimen
+  labels:
+    app: poimen-memory
+    test: integration
+    component: ci-cd
+spec:
+  # Don't retry on failure - we want to see the actual error
+  backoffLimit: 0
+  
+  # Timeout after 10 minutes
+  activeDeadlineSeconds: 600
+  
+  # Keep the pod for debugging
+  ttlSecondsAfterFinished: 3600  # 1 hour
+  
+  template:
+    metadata:
+      labels:
+        app: poimen-memory
+        test: integration
+    spec:
+      serviceAccountName: memory-app
+      restartPolicy: Never
+      
+      containers:
+      # Step 1: Run migrations
+      - name: migrate
+        image: forgejo.riotpiao.com/riotpiao-poimen/poimen-memory:IMAGE_SHA
+        imagePullPolicy: IfNotPresent
+        
+        command:
+          - /bin/bash
+          - -c
+          - |
+            set -e
+            
+            # Copy migrations script from image to working dir
+            cp /app/migrations/run_migrations.sh /tmp/run_migrations.sh
+            chmod +x /tmp/run_migrations.sh
+            
+            # Run migrations
+            /tmp/run_migrations.sh
+            
+            echo ""
+            echo "✓ Migrations complete"
+            echo "Database ready for tests"
+        
+        env:
+        - name: DB_HOST
+          value: "memory-db-rw.poimen.svc.cluster.local"
+        - name: DB_PORT
+          value: "5432"
+        - name: DB_NAME
+          value: "memory"
+        - name: DB_USER
+          value: "app"
+        - name: DB_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: memory-db-app
+              key: password
+        
+        resources:
+          requests:
+            memory: "256Mi"
+            cpu: "100m"
+          limits:
+            memory: "512Mi"
+            cpu: "500m"
+      
+      # Step 2: Run integration tests
+      - name: test
+        image: forgejo.riotpiao.com/riotpiao-poimen/poimen-memory:IMAGE_SHA
+        imagePullPolicy: IfNotPresent
+        
+        command:
+          - /bin/bash
+          - -c
+          - |
+            set -e
+            
+            echo "=========================================="
+            echo "Integration Test: Ingest + Embedding"
+            echo "=========================================="
+            echo ""
+            
+            # Start HTTP server
+            echo "Starting memory-service..."
+            mem-cli serve --port 8080 &
+            SERVER_PID=$!
+            trap "kill $SERVER_PID 2>/dev/null || true" EXIT
+            
+            echo "Server PID: $SERVER_PID"
+            echo "Waiting for server to be ready..."
+            
+            # Wait for /health endpoint
+            for i in {1..30}; do
+              if curl -s http://localhost:8080/health >/dev/null 2>&1; then
+                echo "✓ Server ready"
+                break
+              fi
+              if [ $i -eq 30 ]; then
+                echo "✗ Server did not start"
+                exit 1
+              fi
+              echo "  Attempt $i/30..."
+              sleep 1
+            done
+            
+            echo ""
+            echo "Running E2E ingest test..."
+            echo ""
+            
+            # Send ingest request
+            INGEST_ID="test-$(date +%s)"
+            RESPONSE=$(curl -s -X POST http://localhost:8080/memory/ingest \
+              -H "Content-Type: application/json" \
+              -H "Authorization: Bearer test-key" \
+              -d "{
+                \"project\": \"integration-test\",
+                \"source\": \"k8s-job-test\",
+                \"ingest_id\": \"$INGEST_ID\",
+                \"records\": [
+                  {
+                    \"role\": \"user\",
+                    \"text\": \"Kubernetes [[Docker]] [[Linux]] container platform\",
+                    \"timestamp\": \"2026-09-14T13:00:00Z\",
+                    \"source_position\": 0
+                  },
+                  {
+                    \"role\": \"user\",
+                    \"text\": \"Docker [[Container]] microservices architecture\",
+                    \"timestamp\": \"2026-09-14T13:01:00Z\",
+                    \"source_position\": 1
+                  }
+                ]
+              }")
+            
+            # Check response
+            STATUS=$(echo "$RESPONSE" | jq -r '.status // "error"')
+            ID=$(echo "$RESPONSE" | jq -r '.ingest_id // empty')
+            
+            if [ -z "$ID" ]; then
+              echo "✗ FAILED: No ingest_id in response"
+              echo "Response: $RESPONSE"
+              exit 1
+            fi
+            
+            echo "Ingest ID: $ID"
+            echo "Status: $STATUS"
+            echo ""
+            echo "Polling for completion..."
+            
+            # Poll until done
+            for poll in {1..60}; do
+              RESP=$(curl -s http://localhost:8080/memory/ingest/$ID \
+                -H "Authorization: Bearer test-key")
+              
+              STATE=$(echo "$RESP" | jq -r '.status // "unknown"')
+              
+              if [ "$STATE" = "done" ]; then
+                echo "Poll $poll: $STATE ✓"
+                echo ""
+                echo "✓ INGEST SUCCESSFUL"
+                break
+              elif [ "$STATE" = "failed" ] || [ "$STATE" = "error" ]; then
+                echo "Poll $poll: $STATE ✗"
+                echo "Response: $RESP"
+                echo "✗ INGEST FAILED"
+                exit 1
+              fi
+              
+              echo "Poll $poll: $STATE"
+              sleep 2
+            done
+            
+            echo ""
+            echo "Testing query endpoint..."
+            QUERY=$(curl -s "http://localhost:8080/memory/query?project=integration-test&question=what%20is%20docker" \
+              -H "Authorization: Bearer test-key")
+            
+            ENTITY_COUNT=$(echo "$QUERY" | jq '.count.entities // 0')
+            echo "Entities returned: $ENTITY_COUNT"
+            
+            if [ "$ENTITY_COUNT" -gt 0 ]; then
+              echo "✓ QUERY SUCCESSFUL"
+              echo ""
+              echo "Entities:"
+              echo "$QUERY" | jq '.entities[].name'
+            else
+              echo "⚠ No entities returned (schema issue)"
+              echo "✗ Query test FAILED"
+              exit 1
+            fi
+            
+            echo ""
+            echo "=========================================="
+            echo "✓ ALL TESTS PASSED"
+            echo "=========================================="
+        
+        env:
+        - name: DATABASE_URL
+          value: "postgresql://[email protected]:5432/memory"
+        - name: RUST_LOG
+          value: "info,mem_cli=debug,mem_ingest=debug"
+        - name: MEM_AUTH_MODE
+          value: "none"
+        
+        # Password via secret
+        - name: PGPASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: memory-db-app
+              key: password
+        
+        resources:
+          requests:
+            memory: "512Mi"
+            cpu: "200m"
+          limits:
+            memory: "1Gi"
+            cpu: "1000m"
+        
+        livenessProbe:
+          exec:
+            command:
+            - /bin/sh
+            - -c
+            - curl -s http://localhost:8080/health >/dev/null
+          initialDelaySeconds: 30
+          periodSeconds: 10
+          failureThreshold: 2
+
+---
+# ServiceAccount for integration test
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: memory-app
+  namespace: poimen
+  labels:
+    app: poimen-memory
@@ -0,0 +1,127 @@
+#!/bin/bash
+
+# Database Migration Runner
+# Used by K8s Job to apply all migrations before integration tests
+# 
+# Environment variables (from K8s):
+#   DB_HOST     - PostgreSQL host
+#   DB_PORT     - PostgreSQL port
+#   DB_NAME     - Database name
+#   DB_USER     - Database user
+#   DB_PASSWORD - Database password (from Secret)
+
+set -e
+
+DB_HOST="${DB_HOST:-memory-db-rw.poimen.svc.cluster.local}"
+DB_PORT="${DB_PORT:-5432}"
+DB_NAME="${DB_NAME:-memory}"
+DB_USER="${DB_USER:-app}"
+
+if [ -z "$DB_PASSWORD" ]; then
+  echo "ERROR: DB_PASSWORD not set"
+  exit 1
+fi
+
+echo "=========================================="
+echo "Database Migration Runner"
+echo "=========================================="
+echo ""
+echo "Configuration:"
+echo "  Host: $DB_HOST:$DB_PORT"
+echo "  Database: $DB_NAME"
+echo "  User: $DB_USER"
+echo ""
+
+# Export for psql
+export PGPASSWORD="$DB_PASSWORD"
+
+# Get migration directory (where this script is)
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+MIGRATION_DIR="$SCRIPT_DIR"
+
+echo "Migration directory: $MIGRATION_DIR"
+echo ""
+
+# Collect all SQL files
+MIGRATIONS=($(ls -1 "$MIGRATION_DIR"/*.sql 2>/dev/null | sort))
+
+if [ ${#MIGRATIONS[@]} -eq 0 ]; then
+  echo "ERROR: No migration files found in $MIGRATION_DIR"
+  exit 1
+fi
+
+echo "Found ${#MIGRATIONS[@]} migration(s):"
+for m in "${MIGRATIONS[@]}"; do
+  echo "  - $(basename $m)"
+done
+echo ""
+
+# Wait for DB to be ready
+echo "Waiting for database to be ready..."
+for i in {1..30}; do
+  if psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "SELECT 1;" >/dev/null 2>&1; then
+    echo "✓ Database is ready"
+    break
+  fi
+  if [ $i -eq 30 ]; then
+    echo "✗ Database not ready after 30 attempts"
+    exit 1
+  fi
+  echo "  Attempt $i/30..."
+  sleep 1
+done
+
+echo ""
+echo "=========================================="
+echo "Running Migrations"
+echo "=========================================="
+echo ""
+
+SUCCESS=0
+FAILED=0
+
+for migration in "${MIGRATIONS[@]}"; do
+  name=$(basename "$migration")
+  echo -n "▶ $name ... "
+  
+  if psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f "$migration" >/dev/null 2>&1; then
+    echo "✓"
+    ((SUCCESS++))
+  else
+    echo "✗ FAILED"
+    echo ""
+    echo "Error output:"
+    psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f "$migration" 2>&1 | sed 's/^/  /'
+    ((FAILED++))
+  fi
+done
+
+echo ""
+echo "=========================================="
+echo "Migration Summary"
+echo "=========================================="
+echo "  Success: $SUCCESS"
+echo "  Failed:  $FAILED"
+echo ""
+
+if [ $FAILED -eq 0 ]; then
+  echo "✓ All migrations applied successfully"
+  
+  echo ""
+  echo "Verifying schema..."
+  echo ""
+  
+  # Verify key tables exist
+  for table in memory_entity memory_edge ingest_jobs; do
+    if psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "SELECT 1 FROM information_schema.tables WHERE table_name='$table';" 2>&1 | grep -q "1 row"; then
+      echo "  ✓ Table $table exists"
+    else
+      echo "  ⚠ Table $table not found"
+    fi
+  done
+  
+  exit 0
+else
+  echo "✗ Some migrations failed"
+  exit 1
+fi