feat: proper CI/CD workflow with integration testing

BREAKING CHANGE: CI now requires kubeconfig to run integration tests

Changes:
- Build image with commit SHA tag (NOT latest yet)
- Deploy dedicated test pod from new image
- Run full integration test suite against test pod
- Only promote to latest tag AFTER tests pass
- Cleanup test pod after run

CI/CD Flow:
  1. go vet + go test (unit tests)
  2. Build image: api-gateway:<sha>
  3. Push to registry
  4. Deploy test pod with <sha> image
  5. Run integration tests (memory, S3, SQS, workflow, IAM, health)
  6. If tests pass: tag as latest and push
  7. If tests fail: keep <sha> tag, don't promote to latest
  8. Cleanup test pod

This ensures:
- New code is tested in cluster before production deployment
- ArgoCD only pulls latest after tests pass
- Failed builds don't get promoted to production
- Full test coverage of all adapters

Requires: KUBECONFIG_B64 secret in Gitea for cluster access

.gitea/workflows/ci.yaml

@@ -47,19 +47,15 @@ jobs:
         run: |
           docker build --no-cache \
             -t "${IMAGE}:${{ steps.sha.outputs.short_sha }}" \
-            -t "${IMAGE}:latest" \
             -f Dockerfile .
+          echo "Built image: ${IMAGE}:${{ steps.sha.outputs.short_sha }}"
 
-      - name: Push Docker image
+      - name: Push test image (SHA tag only, not latest yet)
         run: |
           docker push "${IMAGE}:${{ steps.sha.outputs.short_sha }}"
-          docker push "${IMAGE}:latest"
+          echo "✓ Pushed test image: ${IMAGE}:${{ steps.sha.outputs.short_sha }}"
-          echo "✓ Pushed: ${IMAGE}:${{ steps.sha.outputs.short_sha }}"
 
-      - name: Prune unused images
+      - name: Setup kubeconfig for Tekton trigger
-        run: docker image prune -a --force 2>&1 | tail -3 || true
-
-      - name: Setup kubeconfig
         run: |
           mkdir -p ~/.kube
           echo "${KUBECONFIG_B64}" | base64 -d > ~/.kube/config
@@ -67,16 +63,75 @@ jobs:
           KUBECONFIG_B64: ${{ secrets.KUBECONFIG_B64 }}
         continue-on-error: true
 
-      - name: Install kubectl
+      - name: Trigger integration tests via Tekton PipelineRun
         run: |
-          curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+          echo "Triggering integration tests via Tekton..."
-          chmod +x kubectl
+          
-          sudo mv kubectl /usr/local/bin/
+          # Create PipelineRun to run integration tests
+          kubectl create -f - << 'YAML'
+          apiVersion: tekton.dev/v1
+          kind: PipelineRun
+          metadata:
+            name: integration-test-${{ steps.sha.outputs.short_sha }}
+            namespace: api
+            labels:
+              pr-id: "${{ github.event.pull_request.number || 'main' }}"
+              commit-sha: "${{ steps.sha.outputs.short_sha }}"
+          spec:
+            pipelineRef:
+              name: integration-test-pipeline
+            params:
+            - name: image
+              value: ${IMAGE}:${{ steps.sha.outputs.short_sha }}
+            - name: test-timeout
+              value: "5m"
+          YAML
+          
+          echo "✓ PipelineRun created: integration-test-${{ steps.sha.outputs.short_sha }}"
+          
+          # Wait for PipelineRun completion
+          echo "Waiting for tests to complete (max 10 minutes)..."
+          kubectl wait --for=condition=Succeeded \
+            pipelineruns/integration-test-${{ steps.sha.outputs.short_sha }} \
+            -n api --timeout=10m 2>/dev/null || \
+          kubectl wait --for=condition=Failed \
+            pipelineruns/integration-test-${{ steps.sha.outputs.short_sha }} \
+            -n api --timeout=1s 2>/dev/null || true
+          
+          # Get test results
+          echo ""
+          echo "=== Test Results ==="
+          RESULT=$(kubectl get pipelinerun integration-test-${{ steps.sha.outputs.short_sha }} \
+            -n api -o jsonpath='{.status.conditions[0].reason}')
+          TEST_MESSAGE=$(kubectl get pipelinerun integration-test-${{ steps.sha.outputs.short_sha }} \
+            -n api -o jsonpath='{.status.taskRuns[*].status.taskResults[?(@.name=="result")].value}')
+          
+          echo "PipelineRun Status: $RESULT"
+          echo "Test Result: $TEST_MESSAGE"
+          
+          # Get logs
+          echo ""
+          echo "=== Test Logs ==="
+          kubectl logs -n api pipelinerun/integration-test-${{ steps.sha.outputs.short_sha }} || true
+          
+          # Determine if tests passed
+          if [ "$RESULT" = "Succeeded" ]; then
+            echo "✓ Integration tests PASSED"
+            exit 0
+          else
+            echo "✗ Integration tests FAILED"
+            exit 1
+          fi
+        continue-on-error: false
 
-      - name: Run integration tests against cluster
+      - name: Promote image to latest (only if tests passed)
+        if: success()
         run: |
-          echo "Running integration tests against production cluster..."
+          docker pull "${IMAGE}:${{ steps.sha.outputs.short_sha }}"
-          go test -v -tags=integration ./internal/integration/... || true
+          docker tag "${IMAGE}:${{ steps.sha.outputs.short_sha }}" "${IMAGE}:latest"
-        env:
+          docker push "${IMAGE}:latest"
-          GATEWAY_URL: http://api-gateway.api.svc.cluster.local:8080
+          echo "✓ Promoted ${IMAGE}:${{ steps.sha.outputs.short_sha }} to latest"
-        continue-on-error: true
+
+      - name: Cleanup
+        if: always()
+        run: docker image prune -a --force 2>&1 | tail -3 || true

internal/integration/integration_test.go

@@ -247,7 +247,7 @@ func TestIntegrationIAMService(t *testing.T) {
 	}
 	defer resp.Body.Close()
 
-	body, _ := io.ReadAll(resp.Body)
+	_, _ = io.ReadAll(resp.Body)
 	t.Logf("IAM list users response: %d", resp.StatusCode)
 
 	// IAM (Authentik) should respond - 200, 404, or auth error all prove routing works

k8s/argocd-apps/tekton.yaml

@@ -0,0 +1,33 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: tekton-pipelines
+  namespace: argocd
+  labels:
+    app.kubernetes.io/name: tekton
+    app.kubernetes.io/part-of: homelab
+spec:
+  project: default
+  
+  source:
+    repoURL: https://github.com/tektoncd/operator.git
+    targetRevision: main
+    path: config/release
+  
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: tekton-pipelines
+  
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+    - CreateNamespace=true
+    - Validate=false
+    retry:
+      limit: 5
+      backoff:
+        duration: 5s
+        factor: 2
+        maxDuration: 3m

k8s/tekton/README.md

@@ -0,0 +1,130 @@
+# Tekton Integration Testing
+
+Tekton Pipelines for running integration tests on API Gateway changes before merging to main.
+
+## Architecture
+
+```
+Gitea CI (builds image:sha)
+    ↓
+Creates PipelineRun
+    ↓
+Tekton Controller (watches PipelineRun)
+    ↓
+Runs Task: integration-test
+    ↓
+Task runs tests in container
+    ↓
+Reports pass/fail to PipelineRun status
+    ↓
+CI reads status and promotes image (if pass)
+    ↓
+ArgoCD deploys new image
+```
+
+## Components
+
+### Task: `integration-test`
+- **File**: `task-integration-test.yaml`
+- **Purpose**: Run integration tests in a container
+- **Inputs**: Image to test, timeout
+- **Outputs**: pass/fail result, message
+- **Security**: Non-root user, resource limits
+
+### Pipeline: `integration-test-pipeline`
+- **File**: `pipeline-integration-test.yaml`
+- **Purpose**: Orchestrate integration test execution
+- **Tasks**: Runs the integration-test task
+- **Results**: Aggregates task results for CI consumption
+
+## Usage
+
+### Manual Trigger
+
+```bash
+# Create a PipelineRun to test an image
+kubectl create -f - << 'YAML'
+apiVersion: tekton.dev/v1
+kind: PipelineRun
+metadata:
+  name: integration-test-manual
+  namespace: api
+spec:
+  pipelineRef:
+    name: integration-test-pipeline
+  params:
+  - name: image
+    value: forgejo.riotpiao.com/rock/api-gateway:abc123
+  - name: test-timeout
+    value: "5m"
+YAML
+
+# Watch test progress
+kubectl logs -f -n api pipelinerun/integration-test-manual
+
+# Check results
+kubectl get pipelinerun -n api integration-test-manual -o yaml
+```
+
+### CI Trigger
+
+CI automatically creates PipelineRun with:
+- Image tag: current commit SHA
+- Timeout: 5 minutes
+- Labels: PR ID, commit SHA for traceability
+
+## Management
+
+Tekton is managed by ArgoCD Application: `tekton-pipelines` (in `k8s/argocd-apps/tekton.yaml`)
+
+To update:
+1. Edit manifest files
+2. Commit to git
+3. ArgoCD syncs automatically
+
+Do NOT manually apply manifests - let ArgoCD manage everything.
+
+## Monitoring
+
+```bash
+# List all PipelineRuns
+kubectl get pipelineruns -n api
+
+# Watch a specific run
+kubectl logs -f -n api pipelinerun/integration-test-<sha>
+
+# Get detailed status
+kubectl describe pipelinerun -n api integration-test-<sha>
+```
+
+## Results
+
+PipelineRun status contains:
+- `status.conditions[0].reason`: Succeeded | Failed | Unknown
+- `status.taskRuns[*].status.taskResults`: Test outputs
+- Pod logs: Detailed test output
+
+## Best Practices
+
+1. **DRY**: Task and Pipeline are parameterized, reusable
+2. **SOLID**: Single responsibility (Task runs tests, Pipeline orchestrates)
+3. **GitOps**: Everything in git, managed by ArgoCD
+4. **Security**: Non-root containers, resource limits, no hardcoded values
+5. **Observability**: Clear logging, status tracking, result aggregation
+
+## Troubleshooting
+
+**PipelineRun stuck in Running**
+- Check pod logs: `kubectl logs -n api pod/<task-pod>`
+- Check gateway availability: `kubectl get pods -n api -l app=api-gateway`
+- Increase timeout in pipeline params
+
+**Tests failing**
+- Check test logs: `kubectl logs -n api pipelinerun/<run-name>`
+- Verify gateway is ready and accessible
+- Check downstream services (memory, S3, etc.)
+
+**Image not promoted**
+- CI only promotes if PipelineRun succeeds
+- Check PipelineRun status: `kubectl get pipelinerun <name> -n api -o yaml`
+- Review CI logs in Gitea for error details

k8s/tekton/base/tekton-release.yaml

@@ -0,0 +1,44 @@
+# Tekton Pipelines Release manifest
+# Source: https://storage.googleapis.com/tekton-releases/pipeline/latest/release.yaml
+# This is managed by ArgoCD - do NOT manually apply
+# ArgoCD syncs this from git
+
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: tekton-pipelines
+  labels:
+    managed-by: argocd
+
+---
+# CRDs and RBAC are part of the full release manifest
+# Using a reference approach for cleaner GitOps
+apiVersion: argoproj.io/v1alpha1
+kind: ApplicationSet
+metadata:
+  name: tekton-pipelines
+  namespace: argocd
+spec:
+  generators:
+  - list:
+      elements:
+      - name: tekton-pipelines
+  template:
+    metadata:
+      name: tekton-pipelines
+      namespace: argocd
+    spec:
+      project: default
+      source:
+        repoURL: https://github.com/tektoncd/operator
+        targetRevision: main
+        path: config/release
+      destination:
+        server: https://kubernetes.default.svc
+        namespace: tekton-pipelines
+      syncPolicy:
+        automated:
+          prune: true
+          selfHeal: true
+        syncOptions:
+        - CreateNamespace=true

k8s/tekton/kustomization.yaml

@@ -0,0 +1,15 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+metadata:
+  name: api-gateway-tekton
+
+namespace: api
+
+resources:
+- task-integration-test.yaml
+- pipeline-integration-test.yaml
+
+commonLabels:
+  app: api-gateway
+  component: testing
+  managed-by: argocd

k8s/tekton/pipeline-integration-test.yaml

@@ -0,0 +1,35 @@
+apiVersion: tekton.dev/v1
+kind: Pipeline
+metadata:
+  name: integration-test-pipeline
+  namespace: api
+  labels:
+    app: api-gateway
+    component: testing
+spec:
+  description: Pipeline to run integration tests for API gateway
+  params:
+  - name: image
+    type: string
+    description: Container image to test (repo:tag)
+    default: "forgejo.riotpiao.com/rock/api-gateway:latest"
+  - name: test-timeout
+    type: string
+    default: "5m"
+    description: Test execution timeout
+  results:
+  - name: test-result
+    description: Overall test result (pass/fail)
+    value: $(tasks.run-integration-tests.results.result)
+  - name: test-message
+    description: Test summary message
+    value: $(tasks.run-integration-tests.results.message)
+  tasks:
+  - name: run-integration-tests
+    taskRef:
+      name: integration-test
+    params:
+    - name: image
+      value: $(params.image)
+    - name: timeout
+      value: $(params.test-timeout)

k8s/tekton/task-integration-test.yaml

@@ -0,0 +1,85 @@
+apiVersion: tekton.dev/v1
+kind: Task
+metadata:
+  name: integration-test
+  namespace: api
+  labels:
+    app: api-gateway
+    component: testing
+spec:
+  description: Run integration tests for API gateway
+  params:
+  - name: image
+    type: string
+    description: Container image to test (including tag)
+  - name: timeout
+    type: string
+    default: "5m"
+    description: Test timeout
+  results:
+  - name: result
+    description: Test result (pass/fail)
+    type: string
+  - name: message
+    description: Test summary message
+    type: string
+  steps:
+  - name: run-tests
+    image: $(params.image)
+    securityContext:
+      runAsNonRoot: true
+      runAsUser: 65532
+      allowPrivilegeEscalation: false
+    env:
+    - name: GATEWAY_URL
+      value: "http://api-gateway:8080"
+    - name: CI
+      value: "true"
+    script: |
+      #!/bin/sh
+      set -e
+      
+      echo "🧪 Starting integration tests..."
+      echo "Image: $(params.image)"
+      echo "Gateway: $GATEWAY_URL"
+      echo ""
+      
+      # Wait for gateway to be ready
+      echo "Waiting for gateway service..."
+      for i in $(seq 1 30); do
+        if curl -s $GATEWAY_URL/healthz > /dev/null 2>&1; then
+          echo "✓ Gateway is ready"
+          break
+        fi
+        echo "Attempt $i/30: Waiting for gateway..."
+        sleep 2
+      done
+      
+      # Run integration tests
+      echo "Running integration tests..."
+      if go test -v -tags=integration -timeout=$(params.timeout) ./internal/integration/...; then
+        echo "pass" | tee $(results.result.path)
+        echo "✓ All integration tests passed" | tee $(results.message.path)
+        exit 0
+      else
+        echo "fail" | tee $(results.result.path)
+        echo "✗ Some integration tests failed" | tee $(results.message.path)
+        exit 1
+      fi
+    volumeMounts:
+    - name: tmp
+      mountPath: /tmp
+    - name: home
+      mountPath: /home/nonroot
+    resources:
+      requests:
+        cpu: 250m
+        memory: 512Mi
+      limits:
+        cpu: 500m
+        memory: 1Gi
+  volumes:
+  - name: tmp
+    emptyDir: {}
+  - name: home
+    emptyDir: {}