test: production ingest E2E test suite with enhanced logging (#55)

## Summary
Production testing of ingest + embedding pipeline with api-gw integration.

## Root Cause
9 SQL migrations in `crates/mem-store/migrations/` not applied to production database.

Missing tables:
- `memory_entity`
- `memory_edge`
- `memory_edge_temporal`
- Vector embeddings tables
- And 15+ more schema objects

Evidence from logs:
```
WARN: Failed to save entity Docker:
      error returned from database: relation "memory_entity" does not exist
```

## Deliverables
- `test_prod_ingest_real.sh` - Full E2E test against K8s + api-gw
- `apply_migrations.sh` - Manual schema migration (backup)
- `collect_prod_logs.sh` - Pod log collection before/after
- `run_production_test.sh` - Test orchestrator
- `tests/integration_ingest_with_gw.rs` - Integration test
- `tests/unit_ingest_logging.rs` - Unit tests for extraction
- Enhanced logging in `ingest_worker.rs` - Per-record event tracking

## Next Steps
1. Trigger "DB Migration" workflow in Forgejo Actions
2. This applies all 9 migrations from `crates/mem-store/migrations/`
3. Pod restart (automatic)
4. Re-run E2E test - should pass completely

**ETA:** ~15 minutes (3-5 min migrations + 2 min restart + verification)

## How to Test Locally
```bash
./test_prod_ingest_real.sh --verbose
```

Requires:
- kubectl access to poimen namespace
- Port-forwarding to memory-service

---------

Co-authored-by: rock <[email protected]>
Reviewed-on: #55
Co-authored-by: poimen <[email protected]>

.gitea/workflows/build.yaml

@@ -35,11 +35,9 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Cargo build, test, clippy (single compile pass)
+      - name: Cargo test (lib only, no full build)
         run: |
-          cargo build --all --verbose
           cargo test --all --lib --verbose 2>&1 | tail -150 || true
-          cargo clippy --all --all-targets -- -D warnings 2>&1 | tail -50 || true
 
       - name: Get short SHA
         id: sha
@@ -60,7 +58,8 @@ jobs:
       - name: Clean cargo before Docker build
         run: |
           cargo clean || true
-          rm -rf ~/.cargo/registry/cache ~/.cargo/registry/index ~/.cargo/git || true
+          rm -rf target/ || true
+          rm -rf ~/.cargo/registry/cache || true
           df -h /
 
       - name: Build and push Docker image (SHA tag only)
@@ -71,7 +70,114 @@ jobs:
           docker push "${IMAGE}:${{ steps.sha.outputs.short_sha }}"
           echo "Pushed: ${IMAGE}:${{ steps.sha.outputs.short_sha }}"
 
-      - name: Prune unused images and cleanup
+      - name: Install kubectl
+        run: |
+          apt-get update
+          apt-get install -y curl
+          curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+          chmod +x kubectl
+          mv kubectl /usr/local/bin/
+
+      - name: Setup kubeconfig for Tekton
+        run: |
+          mkdir -p ~/.kube
+          echo "${KUBECONFIG_B64}" | base64 -d > ~/.kube/config
+          chmod 600 ~/.kube/config
+          kubectl cluster-info 2>&1 | head -3
+          echo "✓ kubeconfig ready"
+        env:
+          KUBECONFIG_B64: ${{ secrets.KUBECONFIG_B64 }}
+
+      - name: Trigger Tekton PipelineRun (CI/CD)
+        id: tekton
+        run: |
+          SHA="${{ steps.sha.outputs.short_sha }}"
+          RUN_NAME="poimen-ci-${SHA}"
+          NAMESPACE="poimen"
+          IMAGE="${REGISTRY}/riotpiao-poimen/poimen-memory:${SHA}"
+          REGISTRY_USER="${{ secrets.FORGEJO_REGISTRY_USER }}"
+          REGISTRY_TOKEN="${{ secrets.FORGEJO_REGISTRY_TOKEN }}"
+
+          echo "Triggering Tekton PipelineRun: ${RUN_NAME}"
+          echo "Image: ${IMAGE}"
+          echo ""
+
+          # Create PipelineRun
+          cat <<YAML | kubectl create -f -
+          apiVersion: tekton.dev/v1
+          kind: PipelineRun
+          metadata:
+            name: ${RUN_NAME}
+            namespace: ${NAMESPACE}
+            labels:
+              commit-sha: "${SHA}"
+          spec:
+            pipelineRef:
+              name: poimen-ci
+            params:
+            - name: image
+              value: "${IMAGE}"
+            - name: registry-user
+              value: "${REGISTRY_USER}"
+            - name: registry-token
+              value: "${REGISTRY_TOKEN}"
+          YAML
+
+          echo "✓ PipelineRun created"
+          echo ""
+          echo "Waiting for completion (timeout 10m)..."
+
+          # Wait for PipelineRun to complete
+          if kubectl wait pipelinerun/${RUN_NAME} -n ${NAMESPACE} \
+               --for=condition=Succeeded --timeout=600s 2>/dev/null; then
+            echo "result=pass" >> $GITHUB_OUTPUT
+            echo "✓ Pipeline passed"
+          else
+            echo "result=fail" >> $GITHUB_OUTPUT
+            echo "✗ Pipeline failed or timed out"
+          fi
+
+          # Print pipeline summary
+          echo ""
+          echo "=== PipelineRun Status ==="
+          kubectl describe pipelinerun ${RUN_NAME} -n ${NAMESPACE} | tail -30
+
+          # Print task results
+          echo ""
+          echo "=== Task Results ==="
+          SUMMARY=$(kubectl get pipelinerun ${RUN_NAME} -n ${NAMESPACE} \
+            -o jsonpath='{.status.taskRuns[*].status.taskResults[?(@.name=="summary")].value}')
+          echo "Summary: ${SUMMARY}"
+
+          # Print logs from integration-tests task
+          echo ""
+          echo "=== Integration Test Logs ==="
+          POD=$(kubectl get pod -n ${NAMESPACE} \
+            -l tekton.dev/pipelineRun=${RUN_NAME} -l tekton.dev/pipelineTask=integration-tests \
+            -o name | head -1)
+          if [ -n "$POD" ]; then
+            kubectl logs -n ${NAMESPACE} "${POD}" -c step-test 2>/dev/null | tail -200 || true
+          fi
+
+      - name: Gate on test result
+        if: steps.tekton.outputs.result != 'pass'
+        run: |
+          echo "✗ Integration tests FAILED"
+          echo "Image NOT promoted to :latest"
+          exit 1
+
+      - name: Promote image to latest
+        run: |
+          docker login -u "${REGISTRY_USER}" -p "${REGISTRY_TOKEN}" "${REGISTRY}"
+          docker tag "${IMAGE}:${{ steps.sha.outputs.short_sha }}" "${IMAGE}:latest"
+          docker push "${IMAGE}:latest"
+          echo "✓ Promoted to :latest"
+        env:
+          REGISTRY_USER: ${{ secrets.FORGEJO_REGISTRY_USER }}
+          REGISTRY_TOKEN: ${{ secrets.FORGEJO_REGISTRY_TOKEN }}
+
+      - name: Cleanup
+        if: always()
         run: |
           docker image prune -a --force 2>&1 | tail -3 || true
           cargo clean || true