build: add docker.io to golang runner base image

Pre-install docker.io so golang projects don't need to install on every build. Avoids duplicating 'apt-get install docker.io' in every CI workflow.
fix: add docker.io to node-runner image
2026-09-06 05:56:12 -07:00 · 2026-09-06 05:53:26 -07:00 · 2026-09-05 23:43:21 -07:00 · 2026-09-05 23:39:20 -07:00 · 2026-09-05 23:33:36 -07:00 · 2026-09-05 23:31:46 -07:00
48 changed files with 1422 additions and 1480 deletions
@@ -1,269 +0,0 @@
-name: Cluster CI Pipeline
-
-on:
-  push:
-    branches:
-      - main
-      - develop
-    paths:
-      - 'k8s/**'
-      - '.forgejo/workflows/cluster-ci.yaml'
-  pull_request:
-    paths:
-      - 'k8s/**'
-
-jobs:
-  ci:
-    runs-on: docker
-    steps:
-      # === Checkout ===
-      - name: Checkout
-        run: |
-          REPO_URL="${{ gitea.server_url }}/${{ gitea.repository }}.git"
-          CLONE_URL="https://${{ secrets.CI_RUNNER }}:${{ secrets.CI_RUNNER_SECRET }}@${REPO_URL#https://}"
-          git clone --depth 1 "$CLONE_URL" .
-          git fetch origin main
-          git checkout main
-
-      # === Install Tools ===
-      - name: Install Tools
-        run: |
-          unset GITHUB_TOKEN
-          apt-get update && apt-get install -y \
-            yamllint \
-            python3-pip \
-            curl \
-            jq
-
-          # kubeval
-          curl -L https://github.com/instrumenta/kubeval/releases/latest/download/kubeval-linux-amd64.tar.gz | tar xz
-          mv -f kubeval /usr/local/bin/
-
-          # kustomize
-          rm -f kustomize
-          curl -s https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh | bash
-          mv -f kustomize /usr/local/bin/
-
-          # argocd
-          curl -sSL -o /usr/local/bin/argocd https://github.com/argoproj/argo-cd/releases/latest/download/argocd-linux-amd64
-          chmod +x /usr/local/bin/argocd
-
-          # trivy
-          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin
-
-          # polaris
-          curl -L https://github.com/FairwindsOps/polaris/releases/latest/download/polaris-linux-amd64 -o /usr/local/bin/polaris
-          chmod +x /usr/local/bin/polaris
-
-      # === YAML Lint ===
-      - name: YAML Lint
-        run: |
-          echo "=== Linting YAML files ==="
-          yamllint k8s/ -c .yamllint.yaml || true
-
-      # === Kubeval - Validate K8s Syntax ===
-      - name: Kubeval - Validate K8s Syntax
-        run: |
-          echo "=== Validating Kubernetes manifests ==="
-          find k8s -name "*.yaml" -o -name "*.yml" | grep -v "\.archive" | while read file; do
-            echo "Validating $file..."
-            kubeval "$file" -d 2>/dev/null || true
-          done
-
-      # === Kustomize Build - All overlays ===
-      - name: Kustomize Build - Infrastructure
-        run: |
-          echo "=== Building k8s/infrastructure/ ==="
-          kustomize build k8s/infrastructure > /tmp/infrastructure.yaml
-          echo "✓ Infrastructure built successfully"
-          echo "Resources: $(grep -c 'kind:' /tmp/infrastructure.yaml)"
-
-      - name: Kustomize Build - Bootstrap
-        run: |
-          echo "=== Building k8s/bootstrap/ ==="
-          kustomize build k8s/bootstrap > /tmp/bootstrap.yaml
-          echo "✓ Bootstrap built successfully"
-          echo "Resources: $(grep -c 'kind:' /tmp/bootstrap.yaml || echo 0)"
-
-      - name: Kustomize Build - Platform
-        run: |
-          echo "=== Building k8s/platform/ ==="
-          kustomize build k8s/platform > /tmp/platform.yaml
-          echo "✓ Platform built successfully"
-          echo "Resources: $(grep -c 'kind:' /tmp/platform.yaml || echo 0)"
-
-      - name: Kustomize Build - Security
-        run: |
-          echo "=== Building k8s/security/ ==="
-          kustomize build k8s/security > /tmp/security.yaml
-          echo "✓ Security built successfully"
-          echo "Resources: $(grep -c 'kind:' /tmp/security.yaml || echo 0)"
-
-      - name: Kustomize Build - Applications
-        run: |
-          echo "=== Building k8s/applications/ ==="
-          kustomize build k8s/applications > /tmp/applications.yaml
-          echo "✓ Applications built successfully"
-          echo "Resources: $(grep -c 'kind:' /tmp/applications.yaml || echo 0)"
-
-      - name: Kustomize Build - Data
-        run: |
-          echo "=== Building k8s/data/ ==="
-          kustomize build k8s/data > /tmp/data.yaml
-          echo "✓ Data built successfully"
-          echo "Resources: $(grep -c 'kind:' /tmp/data.yaml || echo 0)"
-
-      - name: Validate ArgoCD Applications
-        run: |
-          echo "=== Validating ArgoCD Applications ==="
-          kubeval k8s/argocd/apps/*.yaml
-
-      # === Trivy - Scan Dockerfile ===
-      - name: Trivy - Scan Dockerfile
-        run: |
-          if find . -name "Dockerfile" 2>/dev/null | grep -v node_modules | head -1 | grep -q .; then
-            echo "=== Scanning Dockerfiles with Trivy ==="
-            find . -name "Dockerfile" -not -path "*/node_modules/*" -exec trivy config {} \;
-          else
-            echo "No Dockerfiles found"
-          fi
-
-      # === Trivy - Scan Helm Charts ===
-      - name: Trivy - Scan Helm Charts
-        run: |
-          if find k8s -name "Chart.yaml" 2>/dev/null | head -1 | grep -q .; then
-            echo "=== Scanning Helm charts with Trivy ==="
-            find k8s -name "Chart.yaml" -exec dirname {} \; | while read chart; do
-              echo "Scanning $chart..."
-              trivy config "$chart" || true
-            done
-          else
-            echo "No Helm charts found"
-          fi
-
-      # === Polaris - K8s Security Audit ===
-      - name: Polaris - K8s Security Audit
-        run: |
-          echo "=== Running Polaris K8s security audit ==="
-          polaris audit --audit-path /tmp/polaris-audit.json k8s/ || true
-
-          if [ -f /tmp/polaris-audit.json ]; then
-            echo "Security issues found:"
-            jq '.results[] | select(.pass == false)' /tmp/polaris-audit.json || true
-          fi
-
-      # === Check for Secrets in Code ===
-      - name: Check for Secrets in Code
-        run: |
-          echo "=== Scanning for hardcoded secrets ==="
-          # BLOCKING. This step used to only count findings and then exit 0, so a
-          # plaintext deploy key rode through it into a public remote. Two failure
-          # modes fixed: it now fails the build, and it matches key material by
-          # PEM header rather than only `private_key:`-style YAML field names.
-          # Findings are captured into variables and tested for emptiness rather than
-          # branching on grep's exit status: implementations disagree on the rc of a
-          # `-v` filter fed empty input, and a wrong rc here fails open.
-          # NOTE: --include must precede `--`; after `--` grep treats it as a filename
-          # and silently scans nothing.
-          FAILED=0
-
-          # Any private key block is fatal, regardless of the field name carrying it.
-          KEYS=$(grep -rIE --include="*.yaml" --include="*.yml" \
-                   -- "-----BEGIN ([A-Z]+ )?PRIVATE KEY-----" k8s/ \
-                 | grep -v "\.enc\.yaml" || true)
-          if [ -n "$KEYS" ]; then
-            echo "❌ Unencrypted private key material found:"
-            echo "$KEYS"
-            FAILED=1
-          fi
-
-          # Plaintext values in secret-ish YAML fields. SOPS output is ENC[...],
-          # so encrypted files never trip this.
-          VALS=$(grep -rInE --include="*.yaml" --include="*.yml" \
-                   -- "^[[:space:]]*(password|token|apiKey|api_key|sshPrivateKey|client_secret):[[:space:]]*[\"']?[^\"'[:space:]{\$]{8,}" k8s/ \
-                 | grep -v "ENC\[" | grep -v "\.enc\.yaml" || true)
-          if [ -n "$VALS" ]; then
-            echo "❌ Plaintext secret value found:"
-            echo "$VALS"
-            FAILED=1
-          fi
-
-          if [ "$FAILED" -ne 0 ]; then
-            echo "Encrypt with SOPS (see .sops.yaml) — *.enc.yaml files are exempt."
-            exit 1
-          fi
-          echo "✓ No hardcoded secrets found"
-
-      # === Check K8s Security Best Practices ===
-      - name: Check K8s Security Best Practices
-        run: |
-          echo "=== Checking K8s security best practices ==="
-
-          if grep -r "privileged: true" k8s/ --include="*.yaml" --include="*.yml"; then
-            echo "⚠️  Found privileged containers"
-          fi
-
-          if grep -r "hostNetwork: true" k8s/ --include="*.yaml" --include="*.yml"; then
-            echo "⚠️  Found hostNetwork usage"
-          fi
-
-          echo "Checking for missing resource limits..."
-          MISSING=0
-          find k8s -name "*.yaml" -o -name "*.yml" | while read file; do
-            if grep -q "kind: Deployment\|kind: StatefulSet\|kind: DaemonSet" "$file"; then
-              if ! grep -q "resources:" "$file"; then
-                echo "⚠️  $file: Missing resource requests/limits"
-                MISSING=$((MISSING + 1))
-              fi
-            fi
-          done
-
-      # === ArgoCD Sync (main branch only) ===
-      - name: Sync ArgoCD
-        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
-        env:
-          ARGOCD_SERVER: ${{ secrets.ARGOCD_SERVER }}
-          ARGOCD_AUTH_TOKEN: ${{ secrets.ARGOCD_AUTH_TOKEN }}
-        run: |
-          echo "=== Syncing homelab-root ==="
-          argocd app sync homelab-root --force
-          argocd app wait homelab-root --timeout 5m
-
-      - name: Check Sync Status
-        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
-        env:
-          ARGOCD_SERVER: ${{ secrets.ARGOCD_SERVER }}
-          ARGOCD_AUTH_TOKEN: ${{ secrets.ARGOCD_AUTH_TOKEN }}
-        run: |
-          echo "=== ArgoCD Applications Status ==="
-          argocd app list -o table
-
-          STATUS=$(argocd app get homelab-root -o jsonpath='{.status.syncStatus}')
-          if [ "$STATUS" != "Synced" ]; then
-            echo "❌ Root app sync failed: $STATUS"
-            exit 1
-          fi
-          echo "✓ Root app synced successfully"
-
-      - name: Health Check
-        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
-        env:
-          ARGOCD_SERVER: ${{ secrets.ARGOCD_SERVER }}
-          ARGOCD_AUTH_TOKEN: ${{ secrets.ARGOCD_AUTH_TOKEN }}
-        run: |
-          echo "=== Checking Application Health ==="
-          argocd app get homelab-root -o wide
-
-      # === Summary ===
-      - name: Summary
-        if: always()
-        run: |
-          echo "=== CI Pipeline Summary ==="
-          echo "✓ YAML linted"
-          echo "✓ Manifests validated"
-          echo "✓ Kustomizations built"
-          echo "✓ Security scans completed"
-          echo "✓ Secrets check passed"
-          echo "✓ Best practices verified"
-          echo ""
-          echo "✓ All checks passed"
@@ -0,0 +1,67 @@
+name: Build and push runner images
+
+on:
+  push:
+    paths:
+      - 'k8s/infra/forgejo-runner/Dockerfile.golang'
+      - 'k8s/infra/forgejo-runner/Dockerfile.rust'
+      - 'k8s/infra/forgejo-runner/Dockerfile.node'
+    branches:
+      - main
+
+jobs:
+  build-runners:
+    runs-on: golang
+    env:
+      REGISTRY: forgejo.riotpiao.com
+      IMAGE_BASE: forgejo.riotpiao.com/rock
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Get short SHA
+        id: sha
+        run: |
+          SHORT_SHA=$(git rev-parse --short HEAD)
+          echo "short_sha=${SHORT_SHA}" >> $GITHUB_OUTPUT
+
+      - name: Registry login
+        run: |
+          echo "${REGISTRY_TOKEN}" | docker login "${REGISTRY}" \
+            --username "${REGISTRY_USER}" --password-stdin
+        env:
+          REGISTRY_USER: ${{ secrets.FORGEJO_REGISTRY_USER }}
+          REGISTRY_TOKEN: ${{ secrets.FORGEJO_REGISTRY_TOKEN }}
+
+      - name: Build and push golang-runner
+        if: contains(github.event.head_commit.modified, 'Dockerfile.golang')
+        run: |
+          docker build -f k8s/infra/forgejo-runner/Dockerfile.golang \
+            -t "${IMAGE_BASE}/forgejo-runner-golang:${{ steps.sha.outputs.short_sha }}" \
+            -t "${IMAGE_BASE}/forgejo-runner-golang:latest" \
+            .
+          docker push "${IMAGE_BASE}/forgejo-runner-golang:${{ steps.sha.outputs.short_sha }}"
+          docker push "${IMAGE_BASE}/forgejo-runner-golang:latest"
+          echo "✓ Pushed golang-runner"
+
+      - name: Build and push rust-runner
+        if: contains(github.event.head_commit.modified, 'Dockerfile.rust')
+        run: |
+          docker build -f k8s/infra/forgejo-runner/Dockerfile.rust \
+            -t "${IMAGE_BASE}/forgejo-runner-rust:${{ steps.sha.outputs.short_sha }}" \
+            -t "${IMAGE_BASE}/forgejo-runner-rust:latest" \
+            .
+          docker push "${IMAGE_BASE}/forgejo-runner-rust:${{ steps.sha.outputs.short_sha }}"
+          docker push "${IMAGE_BASE}/forgejo-runner-rust:latest"
+          echo "✓ Pushed rust-runner"
+
+      - name: Build and push node-runner
+        if: contains(github.event.head_commit.modified, 'Dockerfile.node')
+        run: |
+          docker build -f k8s/infra/forgejo-runner/Dockerfile.node \
+            -t "${IMAGE_BASE}/forgejo-runner-node:${{ steps.sha.outputs.short_sha }}" \
+            -t "${IMAGE_BASE}/forgejo-runner-node:latest" \
+            .
+          docker push "${IMAGE_BASE}/forgejo-runner-node:${{ steps.sha.outputs.short_sha }}"
+          docker push "${IMAGE_BASE}/forgejo-runner-node:latest"
+          echo "✓ Pushed node-runner"
@@ -0,0 +1,6 @@
+apiVersion: v2
+name: memory-queues
+description: Kafka queues (DLQ) for Poimen Memory service (Phase 6.6)
+type: application
+version: 0.1.0
+appVersion: "1.0"
@@ -0,0 +1,20 @@
+{{- range .Values.queues }}
+---
+apiVersion: kmsvc.io/v1alpha1
+kind: Queue
+metadata:
+  name: {{ .name }}
+  namespace: {{ $.Values.namespace }}
+  labels:
+    app: memory-service
+    queue: dlq
+spec:
+  name: {{ .name }}
+  description: {{ .description }}
+  partitions: {{ .partitions }}
+  replicationFactor: {{ .replicationFactor }}
+  config:
+    retention.ms: "{{ .config.retention.ms }}"
+    message.retention.seconds: "{{ .config.message.retention.seconds }}"
+    visibility.timeout.seconds: "{{ .config.visibility.timeout.seconds }}"
+{{- end }}
@@ -0,0 +1,25 @@
+# Poimen Memory Service Kafka Queues (kmsvc)
+# Phase 6.6: DLQ topics for webhook + metrics failures
+
+queues:
+  # DLQ for extraction, webhook, and agent failures
+  - name: poimen-memory-dlq
+    description: "DLQ for extraction, webhook, and agent failures"
+    partitions: 3
+    replicationFactor: 1
+    config:
+      retention.ms: "1209600000"  # 14 days
+      message.retention.seconds: "1209600"
+      visibility.timeout.seconds: "300"
+
+  # DLQ for metrics persistence failures  
+  - name: poimen-memory-metric-dlq
+    description: "DLQ for metrics persistence failures"
+    partitions: 3
+    replicationFactor: 1
+    config:
+      retention.ms: "1209600000"  # 14 days
+      message.retention.seconds: "1209600"
+      visibility.timeout.seconds: "300"
+
+namespace: sqs
@@ -0,0 +1,144 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: secretrotations.homelab.riotpiao.com
+spec:
+  group: homelab.riotpiao.com
+  names:
+    kind: SecretRotation
+    plural: secretrotations
+  scope: Namespaced
+  versions:
+  - name: v1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        properties:
+          metadata:
+            type: object
+          spec:
+            type: object
+            required:
+            - provider
+            - rotationInterval
+            properties:
+              # External system: authentik | forgejo | minio | vault
+              provider:
+                type: string
+                enum: [authentik, forgejo, minio, vault]
+              
+              # How often to rotate (hours)
+              rotationInterval:
+                type: integer
+                minimum: 24
+              
+              # Application ID in external system
+              appId:
+                type: string
+              
+              # k8s Secret to update (name, namespace, key)
+              secretRef:
+                type: object
+                required: [name, namespace]
+                properties:
+                  name:
+                    type: string
+                  namespace:
+                    type: string
+                  key:
+                    type: string
+                    description: "Secret key to update (e.g., MINIO_IDENTITY_OPENID_CLIENT_SECRET)"
+              
+              # Path to git file that holds the secret (for .enc.yaml files)
+              gitPath:
+                type: string
+                description: "Path in homelab repo to .enc.yaml file"
+              
+              # Ansible template values to substitute
+              templateValues:
+                type: object
+                additionalProperties:
+                  type: string
+          
+          status:
+            type: object
+            properties:
+              lastRotationTime:
+                type: string
+                format: date-time
+              nextRotationTime:
+                type: string
+                format: date-time
+              lastRotationStatus:
+                type: string
+                enum: [Success, Failed, Pending]
+              lastRotationError:
+                type: string
+              lastCommitHash:
+                type: string
+
+---
+# Example usage:
+apiVersion: homelab.riotpiao.com/v1
+kind: SecretRotation
+metadata:
+  name: minio-oidc
+  namespace: secret-rotation
+spec:
+  provider: authentik
+  rotationInterval: 2160  # 90 days in hours
+  appId: minio
+  secretRef:
+    name: minio-oidc
+    namespace: storage
+    key: MINIO_IDENTITY_OPENID_CLIENT_SECRET
+  gitPath: k8s/argocd/secrets/minio-oidc.enc.yaml
+
+---
+apiVersion: homelab.riotpiao.com/v1
+kind: SecretRotation
+metadata:
+  name: portfolio-agent-oidc
+  namespace: secret-rotation
+spec:
+  provider: authentik
+  rotationInterval: 2160
+  appId: portfolio-agent
+  secretRef:
+    name: portfolio-agent-oidc
+    namespace: portfolio
+    key: CLIENT_SECRET
+  gitPath: k8s/argocd/secrets/portfolio-agent-oidc.enc.yaml
+
+---
+apiVersion: homelab.riotpiao.com/v1
+kind: SecretRotation
+metadata:
+  name: forgejo-registry-token
+  namespace: secret-rotation
+spec:
+  provider: forgejo
+  rotationInterval: 2160
+  appId: rock/riotpiao.com
+  secretRef:
+    name: forgejo-registry-secret
+    namespace: kube-system
+    key: REGISTRY_TOKEN
+  gitPath: k8s/argocd/secrets/forgejo-registry-secret.enc.yaml
+
+---
+apiVersion: homelab.riotpiao.com/v1
+kind: SecretRotation
+metadata:
+  name: minio-root-credentials
+  namespace: secret-rotation
+spec:
+  provider: minio
+  rotationInterval: 4320  # 180 days in hours
+  appId: root
+  secretRef:
+    name: minio-creds
+    namespace: storage
+  gitPath: k8s/argocd/secrets/minio-secrets.enc.yaml
@@ -0,0 +1,92 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: secret-rotation-controller
+  namespace: secret-rotation
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: secret-rotation-controller
+  template:
+    metadata:
+      labels:
+        app: secret-rotation-controller
+    spec:
+      serviceAccountName: secret-rotation-controller
+      containers:
+      - name: controller
+        image: secret-rotation-controller:latest
+        imagePullPolicy: IfNotPresent
+        env:
+        # SOPS reads age key from this file
+        - name: SOPS_AGE_KEY_FILE
+          value: /etc/sops/age/private-key.txt
+        
+        # Vault auth (token in projected volume)
+        - name: VAULT_ADDR
+          value: http://vault.vault.svc.cluster.local:8200
+        - name: VAULT_TOKEN_FILE
+          value: /var/run/secrets/vault/token
+        
+        # Authentik
+        - name: AUTHENTIK_URL
+          value: http://authentik-server.iam.svc.cluster.local
+        - name: AUTHENTIK_BOOTSTRAP_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: authentik-bootstrap
+              key: token
+        
+        # Git
+        - name: GIT_REPO
+          value: https://forgejo.riotpiao.com/rock/homelab.git
+        - name: GIT_AUTHOR_EMAIL
+          value: [email protected]
+        - name: GIT_AUTHOR_NAME
+          value: Secret Rotation Controller
+        - name: FORGEJO_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: forgejo-registry-secret
+              key: REGISTRY_TOKEN
+        
+        volumeMounts:
+        # Age key from ExternalSecret (synced from Vault)
+        - name: age-key
+          mountPath: /etc/sops/age
+          readOnly: true
+        
+        # Vault auth token (projected)
+        - name: vault-token
+          mountPath: /var/run/secrets/vault
+          readOnly: true
+        
+        # Temp working dir
+        - name: tmp
+          mountPath: /tmp
+        
+        resources:
+          requests:
+            cpu: 100m
+            memory: 256Mi
+          limits:
+            cpu: 500m
+            memory: 512Mi
+      
+      volumes:
+      - name: age-key
+        secret:
+          secretName: sops-age-key
+          defaultMode: 0400
+      
+      - name: vault-token
+        projected:
+          sources:
+          - serviceAccountToken:
+              path: token
+              audience: vault
+              expirationSeconds: 3600
+      
+      - name: tmp
+        emptyDir: {}
@@ -0,0 +1,15 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: secret-rotation
+
+resources:
+  - rbac.yaml
+  - crd.yaml
+  - external-secret.yaml
+  - deployment.yaml
+
+commonLabels:
+  app.kubernetes.io/name: secret-rotation-controller
+  app.kubernetes.io/component: automation
+  managed-by: argocd
@@ -0,0 +1,53 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: secret-rotation-controller
+  namespace: secret-rotation
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: secret-rotation-controller
+rules:
+# Read SecretRotation CRDs
+- apiGroups: ["homelab.riotpiao.com"]
+  resources: ["secretrotations"]
+  verbs: ["get", "list", "watch"]
+
+# Update status
+- apiGroups: ["homelab.riotpiao.com"]
+  resources: ["secretrotations/status"]
+  verbs: ["get", "patch", "update"]
+
+# Read k8s secrets that will be rotated
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["get", "list"]
+
+# For recording events
+- apiGroups: [""]
+  resources: ["events"]
+  verbs: ["create", "patch"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: secret-rotation-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: secret-rotation-controller
+subjects:
+- kind: ServiceAccount
+  name: secret-rotation-controller
+  namespace: secret-rotation
+
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: secret-rotation
+  labels:
+    kubernetes.io/metadata.name: secret-rotation
@@ -0,0 +1,32 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: secret-rotation
+  namespace: argocd
+  labels:
+    app.kubernetes.io/name: secret-rotation
+spec:
+  project: homelab
+  
+  sources:
+  - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    path: k8s/apps/secret-rotation-controller
+    targetRevision: main
+  
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: secret-rotation
+  
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+    - CreateNamespace=true
+    - RespectIgnoreDifferences=true
+    retry:
+      limit: 5
+      backoff:
+        duration: 5s
+        factor: 2
+        maxDuration: 3m
@@ -152,6 +152,13 @@ metadata:
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "3"
+    argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-golang
+    argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
+    argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
+    argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
+    argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
+    argocd-image-updater.argoproj.io/write-back-method: git
+    argocd-image-updater.argoproj.io/git-branch: main
 spec:
  project: homelab
  source:
@@ -173,6 +180,13 @@ metadata:
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "3"
+    argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-node
+    argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
+    argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
+    argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
+    argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
+    argocd-image-updater.argoproj.io/write-back-method: git
+    argocd-image-updater.argoproj.io/git-branch: main
 spec:
  project: homelab
  source:
@@ -197,6 +211,13 @@ metadata:
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "3"
+    argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-rust
+    argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
+    argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
+    argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
+    argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
+    argocd-image-updater.argoproj.io/write-back-method: git
+    argocd-image-updater.argoproj.io/git-branch: main
 spec:
  project: homelab
  source:
@@ -0,0 +1,20 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: memory-queues
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "7"
+spec:
+  project: homelab
+  source:
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    targetRevision: main
+    path: k8s/apps/messaging/memory-queues
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: sqs
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
@@ -286,7 +286,8 @@ spec:
    path: k8s/infra/rbac
  destination:
    server: https://kubernetes.default.svc
-    namespace: default
+    # No namespace: cluster-scoped resources (ClusterRoleBinding, etc.)
+    # Namespace is set per-resource in kustomization
  syncPolicy:
    automated:
      prune: true
@@ -1,27 +1,35 @@
-# Poimen project collection — manages poimen-memory, poimen-workflows, and poiman
-# Each repo tracks its own main branch (no prod branch). Poiman is the primary
-# orchestrator with k8s/argocd/ containing the AppProject and deployment structure.
-#
-# CI: All three repos trigger on main branch pushes (no image builds yet).
-# Future: Add build workflows for poiman once container runtime needs are clear.
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
-  name: poimen-root
+  name: poimen
  namespace: argocd
-  labels:
-    app.kubernetes.io/name: poimen
-    app.kubernetes.io/component: orchestrator
  annotations:
    argocd.argoproj.io/sync-wave: "7"
+    # Image Updater: auto-update on new image push (SHA tag filter)
+    argocd-image-updater.argoproj.io/image-list: |
+      memory=forgejo.riotpiao.com/rock/poimen-memory
+      workflows=forgejo.riotpiao.com/rock/poimen-workflows
+      frontend=forgejo.riotpiao.com/rock/poimen-frontend
+    argocd-image-updater.argoproj.io/memory.update-strategy: newest-build
+    argocd-image-updater.argoproj.io/memory.allow-tags: regexp:^[0-9a-f]{7}$
+    argocd-image-updater.argoproj.io/workflows.update-strategy: newest-build
+    argocd-image-updater.argoproj.io/workflows.allow-tags: regexp:^[0-9a-f]{7}$
+    argocd-image-updater.argoproj.io/frontend.update-strategy: newest-build
+    argocd-image-updater.argoproj.io/frontend.allow-tags: regexp:^[0-9a-f]{7}$
+    argocd-image-updater.argoproj.io/write-back-method: git
+    argocd-image-updater.argoproj.io/git-branch: main
 spec:
  project: homelab
-  source:
-    repoURL: https://forgejo.riotpiao.com/rock/poimen.git
+  sources:
+    - repoURL: https://forgejo.riotpiao.com/rock/poimen-memory.git
+      targetRevision: main
+      path: k8s/argocd
+    - repoURL: https://forgejo.riotpiao.com/rock/poimen-workflows.git
+      targetRevision: main
+      path: k8s/argocd
+    - repoURL: https://forgejo.riotpiao.com/rock/poimen-frontend.git
      targetRevision: main
      path: k8s/argocd
-    directory:
-      recurse: false
  destination:
    server: https://kubernetes.default.svc
    namespace: poimen
@@ -20,10 +20,47 @@ config:
    insecure: true
    plaintext: true

+  # Git write-back configuration (for multi-source Applications)
+  git:
+    # Commit author for image updates
+    user:
+      name: "ArgoCD Image Updater"
+      email: "[email protected]"
+    # Use SSH keys from ArgoCD's known hosts + credentials
+    # Image Updater inherits ArgoCD's git credentials (mounted via ArgoCD secret)
+
+# Mount ArgoCD's git credentials for write-back
+extraVolumes:
+  - name: argocd-ssh-known-hosts-cm
+    configMap:
+      name: argocd-ssh-known-hosts-cm
+      defaultMode: 0644
+  - name: argocd-gpg-keys-cm
+    configMap:
+      name: argocd-gpg-keys-cm
+      optional: true
+      defaultMode: 0644
+  - name: argocd-gpg-pubring
+    configMap:
+      name: argocd-gpg-pubring-cm
+      optional: true
+      defaultMode: 0644
+
+extraVolumeMounts:
+  - name: argocd-ssh-known-hosts-cm
+    mountPath: /etc/ssh/ssh_known_hosts.d/argocd-ssh-known-hosts
+    subPath: ssh_known_hosts
+  - name: argocd-gpg-keys-cm
+    mountPath: /etc/gpg/source
+  - name: argocd-gpg-pubring
+    mountPath: /etc/gpg/pubring
+
 # Extra environment variables
 extraEnv:
  - name: ARGOCD_GRPC_WEB
    value: "true"
+  - name: GIT_SSH_KNOWN_HOSTS_CONFIG_MAP_ENABLED
+    value: "true"

 # Resources
 resources:
@@ -0,0 +1,12 @@
+FROM code.forgejo.org/forgejo/runner:6
+
+# Install Node.js + Docker client (needed for GitHub Actions + docker build)
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+      nodejs \
+      npm \
+      docker.io && \
+    rm -rf /var/lib/apt/lists/*
+
+# Verify installations
+RUN docker --version && node --version && git --version
@@ -0,0 +1,11 @@
+FROM code.forgejo.org/forgejo/runner:6
+
+# Install Docker client (forgejo/runner base doesn't include it)
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+      docker.io && \
+    rm -rf /var/lib/apt/lists/*
+
+# Verify installations
+RUN node --version && docker --version && git --version
+# Trigger node-runner build with docker
@@ -0,0 +1,16 @@
+FROM code.forgejo.org/forgejo/runner:6
+
+# Install Node.js + Rust (needed for GitHub Actions checkout@v4, etc.)
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+      nodejs \
+      npm && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install Rust
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+# Verify installations
+RUN docker --version && node --version && git --version && rustc --version
@@ -2,11 +2,15 @@
 # runner instance. Only runner.name and runner.labels differ -- everything
 # else (image, dind, persistence, tolerations, nodeSelector) is shared.
 #
-# node:22-bookworm ships Node natively, so unlike the golang/rust instances,
-# jobs on this runner need no "install node" step before actions/checkout.
+# node:22-bookworm ships Node natively. Docker client installed via workflow step if needed.
+# (homelab has no CI; custom runner images built manually if desired)
+# Bootstrap with runner image (already has Node.js), CI builds custom
 runner:
+  image:
+    repository: code.forgejo.org/forgejo/runner
+    tag: "6"
  name: node-runner
-  labels: "node:docker://node:22-bookworm"
+  labels: "node:docker://code.forgejo.org/forgejo/runner:6"

 # GC CronJob renders only from the default (golang) values to avoid duplicates
 gc:
@@ -2,17 +2,15 @@
 # runner instance. Only runner.name and runner.labels differ -- everything
 # else (image, dind, persistence, tolerations, nodeSelector) is shared.
 #
-# rust:1.83-bookworm -- verified this tag exists (docker manifest inspect)
-# before pinning it, per this repo's convention of not trusting a tag exists
-# without checking.
+# Bootstrap with runner image, CI builds custom with Node.js+Rust
 runner:
+  image:
+    repository: code.forgejo.org/forgejo/runner
+    tag: "6"
  name: rust-runner
-  labels: "rust:docker://rust:1.83-bookworm"
+  labels: "rust:docker://code.forgejo.org/forgejo/runner:6"
+

-persistence:
-  reg:
-    storageClass: longhorn
-    size: 20Gi

 # GC CronJob renders only from the default (golang) values to avoid duplicates
 gc:
@@ -1,20 +1,10 @@
 runner:
  image:
    repository: code.forgejo.org/forgejo/runner
-    tag: "6"          # pin exact release before apply
+    tag: "6"  # Bootstrap with runner image, CI builds custom with Node.js
  name: golang-runner
-  # Default image is only used when a job's `container:` doesn't override it
-  # (both ci.yaml and build.yaml in homelab-frontend do). Retired the old
-  # "docker" label entirely; every repo this runner serves is Go, so this
-  # instance carries the golang toolchain and its own dind sidecar builds and
-  # pushes that repo's images too -- there is no separate generic runner
-  # anymore.
-  labels: "golang:docker://golang:1.26-bookworm"
-  # In-cluster Service (:3000) — direct, avoids the ingress/public-hostname hop
-  # (the public URL is :443 which forgejo doesn't serve; runner got i/o timeout).
+  labels: "golang:docker://code.forgejo.org/forgejo/runner:6"
  forgejoUrl: http://forgejo-gitea-http.cicd.svc.cluster.local:3000
-  # tokenSecret: name of the K8s Secret that holds the runner registration token
-  # created automatically by the helmfile presync hook (see helmfile.yaml.gotmpl)
  tokenSecret: runner-token
  resources:
    requests:
@@ -61,6 +51,6 @@ gc:
  enabled: true
  schedule: "*/30 * * * *"     # every 30 minutes
  image: alpine/k8s:1.31.0
-  pruneAge: "72h"              # Docker artifacts unused longer than this get pruned
-  pruneAgeHours: 72             # Same as pruneAge but numeric for date arithmetic in shell
+  pruneAge: "30m"              # Docker artifacts unused longer than this get pruned
+  pruneAgeHours: 0.5            # Same as pruneAge but numeric for date arithmetic in shell
  actcacheMaxAgeDays: 1        # actcache files older than N days (aggressive for heavy Rust cargo builds)
@@ -68,15 +68,14 @@ grafana.ini:
  # doesn't return localhost redirects in its token responses.
  #
  # role_attribute_path: JMESPath expression evaluated against the userinfo
-  # response. Members of the 'grafana-admins' Authentik group get Admin role;
-  # everyone else gets Viewer. The group name must match exactly what Authentik
-  # sends in the 'groups' claim.
+  # response. akadmin gets GrafanaAdmin (server admin, can impersonate);
+  # homelab-admins members get Admin (org admin); everyone else Viewer.
  auth.generic_oauth:
    enabled: true
    name: Authentik
    allow_sign_up: true
    client_id: grafana
-    scopes: openid email profile
+    scopes: openid email profile groups
    auth_url: https://authentik.riotpiao.com/application/o/authorize/
    token_url: https://authentik.riotpiao.com/application/o/token/
    api_url: https://authentik.riotpiao.com/application/o/userinfo/
@@ -87,7 +86,8 @@ grafana.ini:
    email_attribute_path: email
    login_attribute_path: preferred_username
    name_attribute_path: name
-    role_attribute_path: "contains(groups[*], 'grafana-admins') && 'Admin' || 'Viewer'"
+    role_attribute_path: "preferred_username == 'akadmin' && 'GrafanaAdmin' || contains(groups[*], 'homelab-admins') && 'Admin' || 'Viewer'"
+    allow_assign_grafana_admin: true
    use_pkce: false
    use_refresh_token: false
    skip_org_role_sync: false
@@ -96,8 +96,13 @@ spec:
      console: https://minio.riotpiao.com

  # ── OIDC via Authentik (server-side env, valid in v2 schema) ────────────────
+  # Use in-cluster URL for config fetch (pod→authentik); browser redirects use
+  # public URLs embedded in the OIDC metadata response (issuer stays public).
  env:
    - name: MINIO_IDENTITY_OPENID_CONFIG_URL
+      # Must use external URL — well-known response contains external issuer/jwks_uri.
+      # MinIO validates issuer in JWT matches well-known issuer. Internal URL = mismatch.
+      # Hairpins through ingress-nginx but stays in-cluster.
      value: "https://authentik.riotpiao.com/application/o/minio/.well-known/openid-configuration"
    - name: MINIO_IDENTITY_OPENID_CLIENT_ID
      value: "minio"
@@ -0,0 +1,160 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: api-gateway-alerts
+  namespace: monitoring
+  labels:
+    release: prometheus
+spec:
+  groups:
+    # ================================================================
+    # SLA Targets (based on canary traffic baselines):
+    #
+    # Availability:  99.9% (43.8 min downtime/month)
+    # LLM Chat:      p95 < 1s (qwen), p95 < 2s (reasoning), p95 < 5s (ornith)
+    # Embeddings:    p95 < 500ms
+    # Rerank:        p95 < 500ms
+    # Models list:   p95 < 300ms
+    # Error rate:    < 1% (5xx), < 5% (4xx excluding auth)
+    #
+    # Baselines from 200-request canary run:
+    #   qwen p99=609ms, reasoning p99=328ms, embeddings p99=287ms,
+    #   rerank p99=218ms, models p99=277ms
+    # SLA set at ~2x p99 for headroom.
+    # ================================================================
+
+    - name: api-gateway.availability
+      rules:
+        # Gateway pods not ready
+        - alert: APIGatewayDown
+          expr: sum(kube_pod_status_ready{namespace="api",condition="true"}) == 0
+          for: 1m
+          labels:
+            severity: critical
+          annotations:
+            summary: "API Gateway has zero ready pods"
+
+        # Gateway pod count below desired
+        - alert: APIGatewayDegraded
+          expr: |
+            sum(kube_pod_status_ready{namespace="api",condition="true"})
+            < kube_deployment_spec_replicas{namespace="api",deployment="api-gateway"}
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "API Gateway {{ $value }} ready pods below desired replica count"
+
+        # Blackbox probe down
+        - alert: APIGatewayProbeDown
+          expr: probe_success{instance=~".*api.riotpiao.com.*"} == 0
+          for: 2m
+          labels:
+            severity: critical
+          annotations:
+            summary: "API Gateway probe failed: {{ $labels.instance }}"
+
+        # LLM serving pods not ready
+        - alert: LLMServingDown
+          expr: sum(kube_pod_status_ready{namespace="llm-serving",condition="true"}) == 0
+          for: 2m
+          labels:
+            severity: critical
+          annotations:
+            summary: "All LLM serving pods down"
+
+        # Individual predictor down
+        - alert: LLMPredictorDown
+          expr: |
+            kube_deployment_status_replicas_ready{namespace="llm-serving"}
+            < kube_deployment_spec_replicas{namespace="llm-serving"}
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "{{ $labels.deployment }} has {{ $value }} ready (below desired)"
+
+    - name: api-gateway.latency
+      # SLA: latency thresholds at ~2x measured p99
+      rules:
+        # Ingress-level latency (all requests through nginx)
+        - alert: APIGatewayLatencyHigh
+          expr: |
+            histogram_quantile(0.95,
+              sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le)
+            ) > 2
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "API Gateway p95 latency {{ $value | printf \"%.1f\" }}s (SLA: <2s)"
+
+        # Extreme latency (p99 > 5s)
+        - alert: APIGatewayLatencyCritical
+          expr: |
+            histogram_quantile(0.99,
+              sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le)
+            ) > 5
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "API Gateway p99 latency {{ $value | printf \"%.1f\" }}s (SLA: <5s)"
+
+    - name: api-gateway.errors
+      rules:
+        # 5xx error rate > 1%
+        - alert: APIGateway5xxErrorRate
+          expr: |
+            sum(rate(nginx_ingress_controller_requests{ingress="api",status=~"5.."}[5m]))
+            / sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m]))
+            > 0.01
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "API Gateway 5xx rate {{ $value | humanizePercentage }} (SLA: <1%)"
+
+        # Total error rate > 10% (including 4xx)
+        - alert: APIGatewayHighErrorRate
+          expr: |
+            sum(rate(nginx_ingress_controller_requests{ingress="api",status=~"[45].."}[5m]))
+            / sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m]))
+            > 0.10
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "API Gateway total error rate {{ $value | humanizePercentage }} (SLA: <10%)"
+
+    - name: api-gateway.resources
+      rules:
+        # Gateway pod restart
+        - alert: APIGatewayRestarted
+          expr: increase(kube_pod_container_status_restarts_total{namespace="api"}[15m]) > 0
+          for: 0m
+          labels:
+            severity: warning
+          annotations:
+            summary: "API Gateway pod {{ $labels.pod }} restarted"
+
+        # LLM predictor restart
+        - alert: LLMPredictorRestarted
+          expr: increase(kube_pod_container_status_restarts_total{namespace="llm-serving"}[15m]) > 0
+          for: 0m
+          labels:
+            severity: warning
+          annotations:
+            summary: "LLM predictor {{ $labels.pod }} restarted"
+
+        # Gateway high memory (>80% of limit)
+        - alert: APIGatewayHighMemory
+          expr: |
+            sum(container_memory_working_set_bytes{namespace="api",container="gateway"}) by (pod)
+            / sum(kube_pod_container_resource_limits{namespace="api",container="gateway",resource="memory"}) by (pod)
+            > 0.8
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Gateway pod {{ $labels.pod }} memory at {{ $value | humanizePercentage }} of limit"
@@ -0,0 +1,177 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: cluster-alerts
+  namespace: monitoring
+  labels:
+    release: prometheus
+spec:
+  groups:
+    - name: cluster.availability
+      rules:
+        # Node down
+        - alert: NodeNotReady
+          expr: kube_node_status_condition{condition="Ready",status="true"} == 0
+          for: 2m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Node {{ $labels.node }} is NotReady"
+
+        # Pod stuck pending (scheduling failure)
+        - alert: PodStuckPending
+          expr: sum(kube_pod_status_phase{phase="Pending"}) > 0
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "{{ $value }} pod(s) stuck in Pending state for >10m"
+
+        # CrashLoopBackOff
+        - alert: PodCrashLooping
+          expr: sum(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}) by (namespace, pod) > 0
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "{{ $labels.namespace }}/{{ $labels.pod }} in CrashLoopBackOff"
+
+        # OOMKilled spike
+        - alert: OOMKilledSpike
+          expr: sum(increase(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[1h])) > 3
+          for: 0m
+          labels:
+            severity: warning
+          annotations:
+            summary: "{{ $value }} OOMKilled events in last hour"
+
+        # Deployment replicas unavailable
+        - alert: DeploymentReplicasUnavailable
+          expr: kube_deployment_status_replicas_unavailable > 0
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "{{ $labels.namespace }}/{{ $labels.deployment }} has {{ $value }} unavailable replicas"
+
+    - name: cluster.jobs
+      rules:
+        # Job failed
+        - alert: JobFailed
+          expr: kube_job_status_failed > 0
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Job {{ $labels.namespace }}/{{ $labels.job_name }} failed"
+
+        # Job stuck running >2h
+        - alert: JobStuckRunning
+          expr: |
+            kube_job_status_active == 1
+            and on(job_name,namespace)
+            (time() - kube_job_status_start_time) > 7200
+          for: 0m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Job {{ $labels.namespace }}/{{ $labels.job_name }} running >2h"
+
+        # CronJob missed schedule
+        - alert: CronJobMissedSchedule
+          expr: |
+            (time() - kube_cronjob_status_last_schedule_time) > 2 * (kube_cronjob_spec_next_schedule_time - kube_cronjob_status_last_schedule_time)
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} missed schedule"
+
+    - name: cluster.resources
+      rules:
+        # Node CPU >90% sustained
+        - alert: NodeHighCPU
+          expr: (1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m])) by (instance)) * 100 > 90
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Node {{ $labels.instance }} CPU at {{ $value | printf \"%.0f\" }}%"
+
+        # Node memory >90% sustained
+        - alert: NodeHighMemory
+          expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 90
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Node {{ $labels.instance }} memory at {{ $value | printf \"%.0f\" }}%"
+
+        # Node disk >85%
+        - alert: NodeDiskFull
+          expr: (1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 > 85
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Node {{ $labels.instance }} disk at {{ $value | printf \"%.0f\" }}%"
+
+        # Container restart storm (>5 restarts in 15m)
+        - alert: ContainerRestartStorm
+          expr: sum(increase(kube_pod_container_status_restarts_total[15m])) by (namespace, pod) > 5
+          for: 0m
+          labels:
+            severity: warning
+          annotations:
+            summary: "{{ $labels.namespace }}/{{ $labels.pod }} restarted {{ $value | printf \"%.0f\" }} times in 15m"
+
+    - name: cluster.storage
+      rules:
+        # Longhorn drive offline
+        - alert: LonghornDriveOffline
+          expr: longhorn_disk_health != 1
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Longhorn disk {{ $labels.node }} unhealthy"
+
+    - name: cluster.dns
+      rules:
+        # CoreDNS errors spike
+        - alert: CoreDNSErrorSpike
+          expr: sum(rate(coredns_dns_responses_total{rcode=~"SERVFAIL"}[5m])) > 0.5
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "CoreDNS SERVFAIL rate {{ $value | printf \"%.2f\" }}/s"
+
+    - name: cluster.probes
+      rules:
+        # Any blackbox probe down
+        - alert: ServiceProbeDown
+          expr: probe_success == 0
+          for: 3m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Probe failed: {{ $labels.instance }}"
+
+        # Probe latency >2s
+        - alert: ServiceProbeSlow
+          expr: probe_duration_seconds > 2
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Probe slow ({{ $value | printf \"%.1f\" }}s): {{ $labels.instance }}"
+
+        # Certificate expiry <14 days
+        - alert: CertificateExpiringSoon
+          expr: (certmanager_certificate_expiration_timestamp_seconds - time()) / 86400 < 14
+          for: 0m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Certificate {{ $labels.name }} expires in {{ $value | printf \"%.0f\" }} days"
@@ -61,6 +61,10 @@ serviceMonitor:
      url: https://argocd.riotpiao.com/healthz
    - name: longhorn
      url: https://longhorn.riotpiao.com/
+    - name: api-gateway
+      url: https://api.riotpiao.com/healthz
+    - name: api-gateway-models
+      url: https://api.riotpiao.com/v1/models

 prometheusRule:
  enabled: true
@@ -0,0 +1,36 @@
+apiVersion: v1
+data:
+  api-gateway.json: '{"title":"API Gateway","uid":"api-gateway","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","tags":["api","gateway","llm"],"panels":[{"id":1,"title":"Gateway
+    Health","type":"row","collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Gateway
+    Pods Ready","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"red"},{"value":3,"color":"green"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"api\",condition=\"true\"})"}]},{"id":3,"title":"Probe:
+    healthz","type":"stat","gridPos":{"h":4,"w":4,"x":4,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}]}},"targets":[{"expr":"probe_success{instance=~\".*api.riotpiao.com/healthz\"}"}]},{"id":4,"title":"Probe
+    Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"probe_duration_seconds{instance=~\".*api.riotpiao.com.*\"}","legendFormat":"{{instance}}"}]}]},{"id":10,"title":"Ingress
+    Traffic (nginx)","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"Request
+    Rate by Status","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(nginx_ingress_controller_requests{ingress=\"api\"}[5m]))
+    by (status)","legendFormat":"{{status}}"}]},{"id":12,"title":"Error Rate %","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percent"}},"targets":[{"expr":"sum(rate(nginx_ingress_controller_requests{ingress=\"api\",status=~\"5..\"}[5m]))
+    / sum(rate(nginx_ingress_controller_requests{ingress=\"api\"}[5m])) * 100","legendFormat":"5xx"},{"expr":"sum(rate(nginx_ingress_controller_requests{ingress=\"api\",status=~\"4..\"}[5m]))
+    / sum(rate(nginx_ingress_controller_requests{ingress=\"api\"}[5m])) * 100","legendFormat":"4xx"}]},{"id":13,"title":"Latency
+    p50/p95/p99","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.50,
+    sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"api\"}[5m]))
+    by (le))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"api\"}[5m]))
+    by (le))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"api\"}[5m]))
+    by (le))","legendFormat":"p99"}]}]},{"id":20,"title":"LLM Serving","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"LLM
+    Pods Ready","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"llm-serving\",condition=\"true\"})"}]},{"id":22,"title":"CPU
+    by Predictor","type":"timeseries","gridPos":{"h":8,"w":8,"x":4,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"llm-serving\"}[5m]))
+    by (pod)","legendFormat":"{{pod}}"}]},{"id":23,"title":"Memory by Predictor","type":"timeseries","gridPos":{"h":8,"w":8,"x":12,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"bytes"}},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"llm-serving\"})
+    by (pod)","legendFormat":"{{pod}}"}]},{"id":24,"title":"Predictor Restarts","type":"timeseries","gridPos":{"h":8,"w":4,"x":20,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"llm-serving\"}[15m]))
+    by (pod)","legendFormat":"{{pod}}"}]}]},{"id":30,"title":"Gateway Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"CPU
+    by Gateway Pod","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"api\"}[5m]))
+    by (pod)","legendFormat":"{{pod}}"}]},{"id":32,"title":"Memory by Gateway Pod","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"bytes"}},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"api\"})
+    by (pod)","legendFormat":"{{pod}}"}]},{"id":33,"title":"Gateway Restarts","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"api\"}[15m]))
+    by (pod)","legendFormat":"{{pod}}"}]}]},{"id":40,"title":"Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":4},"panels":[{"id":41,"title":"Gateway
+    Logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":5},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"api\",container=\"gateway\"}"}]},{"id":42,"title":"LLM
+    Serving Logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":15},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"llm-serving\"}"}]}]}]}'
+kind: ConfigMap
+metadata:
+  annotations:
+    grafana_folder: API
+  labels:
+    grafana_dashboard: '1'
+  name: api-gateway-dashboard
+  namespace: logging
@@ -0,0 +1,62 @@
+apiVersion: v1
+data:
+  cluster-infrastructure.json: '{"title":"Cluster Infrastructure","uid":"cluster-infra","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","tags":["infrastructure","k8s"],"panels":[{"id":1,"title":"Cluster
+    Health","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Nodes
+    Ready","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"red"},{"value":3,"color":"green"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"count(kube_node_status_condition{condition=\"Ready\",status=\"true\"}
+    == 1)"}]},{"id":3,"title":"Pods Pending","type":"stat","gridPos":{"h":4,"w":4,"x":4,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"sum(kube_pod_status_phase{phase=\"Pending\"})
+    OR on() vector(0)"}]},{"id":4,"title":"CrashLoopBackOff","type":"stat","gridPos":{"h":4,"w":4,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"sum(kube_pod_container_status_waiting_reason{reason=\"CrashLoopBackOff\"})
+    OR on() vector(0)"}]},{"id":5,"title":"OOMKilled (1h)","type":"stat","gridPos":{"h":4,"w":4,"x":12,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"sum(increase(kube_pod_container_status_last_terminated_reason{reason=\"OOMKilled\"}[1h]))
+    OR on() vector(0)"}]},{"id":6,"title":"Deploys Unavailable","type":"stat","gridPos":{"h":4,"w":4,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"count(kube_deployment_status_replicas_unavailable
+    > 0) OR on() vector(0)"}]},{"id":7,"title":"Services Down","type":"stat","gridPos":{"h":4,"w":4,"x":20,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"count(probe_success
+    == 0) OR on() vector(0)"}]}]},{"id":10,"title":"Jobs & CronJobs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"Failed
+    Jobs","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"count(kube_job_status_failed
+    > 0) OR on() vector(0)"}]},{"id":12,"title":"Failed Jobs Detail","type":"table","gridPos":{"h":8,"w":10,"x":4,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"kube_job_status_failed
+    > 0","format":"table","instant":true}]},{"id":13,"title":"Stuck Jobs (>1h)","type":"table","gridPos":{"h":8,"w":10,"x":14,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"kube_job_status_active
+    == 1 and on(job_name,namespace) (time() - kube_job_status_start_time) > 3600","format":"table","instant":true}]},{"id":14,"title":"CronJob
+    Last Success","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"dateTimeFromNow"}},"targets":[{"expr":"kube_cronjob_status_last_successful_time{namespace=~\"cicd|kube-system|paperless\"}","legendFormat":"{{namespace}}/{{cronjob}}"}]},{"id":15,"title":"Container
+    Restart Storm (top 10)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"topk(10,
+    sum(rate(kube_pod_container_status_restarts_total[15m])) by (namespace, pod))","legendFormat":"{{namespace}}/{{pod}}"}]}]},{"id":20,"title":"Node
+    Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"CPU
+    % by Node","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percent"}},"targets":[{"expr":"(1
+    - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) by (instance)) * 100","legendFormat":"{{instance}}"}]},{"id":22,"title":"Memory
+    % by Node","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percent"}},"targets":[{"expr":"(1
+    - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100","legendFormat":"{{instance}}"}]},{"id":23,"title":"Disk
+    % by Node","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percent"}},"targets":[{"expr":"(1
+    - node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"})
+    * 100","legendFormat":"{{instance}}"}]},{"id":24,"title":"Load Average","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"node_load1","legendFormat":"1m
+    {{instance}}"},{"expr":"node_load5","legendFormat":"5m {{instance}}"}]},{"id":25,"title":"Network
+    Errors & Drops","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(node_network_receive_errs_total[5m])","legendFormat":"rx-err
+    {{instance}}"},{"expr":"rate(node_network_transmit_errs_total[5m])","legendFormat":"tx-err
+    {{instance}}"},{"expr":"rate(node_network_receive_drop_total[5m])","legendFormat":"rx-drop
+    {{instance}}"}]}]},{"id":30,"title":"Control Plane","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"API
+    Server Up","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}]}},"targets":[{"expr":"min(up{job=\"apiserver\"})"}]},{"id":32,"title":"API
+    Server Request Rate","type":"timeseries","gridPos":{"h":8,"w":10,"x":4,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(apiserver_request_total[5m]))
+    by (verb, code)","legendFormat":"{{verb}} {{code}}"}]},{"id":33,"title":"API Server
+    Error Rate %","type":"timeseries","gridPos":{"h":8,"w":10,"x":14,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percent"}},"targets":[{"expr":"sum(rate(apiserver_request_total{code=~\"5..\"}[5m]))
+    / sum(rate(apiserver_request_total[5m])) * 100","legendFormat":"5xx %"}]},{"id":34,"title":"API
+    Server Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":12},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95,
+    sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
+    sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p99"}]},{"id":35,"title":"etcd
+    Request Duration","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":12},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.99,
+    sum(rate(etcd_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p99"}]}]},{"id":40,"title":"Storage","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":4},"panels":[{"id":41,"title":"Longhorn
+    Disk Capacity","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"bytes"}},"targets":[{"expr":"longhorn_disk_capacity_bytes","legendFormat":"capacity
+    {{node}}"},{"expr":"longhorn_disk_reservation_bytes","legendFormat":"reserved
+    {{node}}"}]},{"id":42,"title":"PVC Phase","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"kube_persistentvolumeclaim_status_phase","legendFormat":"{{namespace}}/{{persistentvolumeclaim}}
+    {{phase}}"}]}]},{"id":50,"title":"DNS & Networking","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":5},"panels":[{"id":51,"title":"CoreDNS
+    Cache Hit Rate","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"rate(coredns_cache_hits_total[5m])
+    / (rate(coredns_cache_hits_total[5m]) + rate(coredns_cache_misses_total[5m]))","legendFormat":"{{server}}"}]},{"id":52,"title":"CoreDNS
+    Errors","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(coredns_dns_responses_total{rcode=~\"SERVFAIL|NXDOMAIN\"}[5m]))
+    by (rcode)","legendFormat":"{{rcode}}"}]}]},{"id":60,"title":"Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":6},"panels":[{"id":61,"title":"Error
+    Rate by Namespace","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":7},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum
+    by (namespace) (count_over_time({namespace=~\"kube-system|cert-manager|ingress-nginx|longhorn-system\"}
+    |= \"error\" [5m]))","legendFormat":"{{namespace}}"}]},{"id":62,"title":"Control
+    Plane Logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":15},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"kube-system\"}"}]},{"id":63,"title":"Cluster
+    Addon Logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":25},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=~\"cert-manager|ingress-nginx|longhorn-system\"}"}]}]}]}'
+kind: ConfigMap
+metadata:
+  annotations:
+    grafana_folder: Infrastructure
+  labels:
+    grafana_dashboard: '1'
+  name: cluster-infrastructure-dashboard
+  namespace: logging
@@ -1,55 +0,0 @@
-# k8s/monitoring/dashboards/control-plane-logs.yaml
-# Surfaces controller/control-plane logs that are already in Loki today
-# (Promtail scrapes every namespace with no filter) — this dashboard is the
-# "make it visible" piece, not new log collection.
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: control-plane-logs-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-data:
-  control-plane-logs.json: |
-    {
-      "title": "Cluster Control Plane & Controllers (Logs)",
-      "uid": "control-plane-logs",
-      "schemaVersion": 39,
-      "timezone": "browser",
-      "time": { "from": "now-1h", "to": "now" },
-      "refresh": "30s",
-      "panels": [
-        {
-          "id": 1,
-          "title": "Error rate by namespace",
-          "type": "timeseries",
-          "gridPos": { "h": 6, "w": 24, "x": 0, "y": 0 },
-          "datasource": { "type": "loki", "uid": "loki" },
-          "targets": [
-            {
-              "expr": "sum by (namespace) (count_over_time({namespace=~\"kube-system|cert-manager|ingress-nginx|longhorn-system\"} |= \"error\" [5m]))"
-            }
-          ]
-        },
-        {
-          "id": 2,
-          "title": "Control plane (kube-apiserver, controller-manager, scheduler)",
-          "type": "logs",
-          "gridPos": { "h": 10, "w": 24, "x": 0, "y": 6 },
-          "datasource": { "type": "loki", "uid": "loki" },
-          "targets": [
-            { "expr": "{namespace=\"kube-system\"}" }
-          ]
-        },
-        {
-          "id": 3,
-          "title": "Cluster add-ons (cert-manager, ingress-nginx, longhorn)",
-          "type": "logs",
-          "gridPos": { "h": 10, "w": 24, "x": 0, "y": 16 },
-          "datasource": { "type": "loki", "uid": "loki" },
-          "targets": [
-            { "expr": "{namespace=~\"cert-manager|ingress-nginx|longhorn-system\"}" }
-          ]
-        }
-      ]
-    }
@@ -0,0 +1,280 @@
+#!/usr/bin/env python3
+"""Generate consolidated Grafana dashboards as k8s ConfigMap YAML files."""
+
+import json
+import os
+
+DASHBOARD_DIR = os.path.expanduser("~/workplace/homelab/k8s/infra/monitoring/dashboards")
+
+DS_PROM = {"type": "prometheus", "uid": "prometheus"}
+DS_LOKI = {"type": "loki", "uid": "loki"}
+
+
+def stat_panel(id, title, expr, x, y, w=4, h=4, unit="short", mappings=None, thresholds=None):
+    p = {
+        "id": id, "title": title, "type": "stat",
+        "gridPos": {"h": h, "w": w, "x": x, "y": y},
+        "datasource": DS_PROM,
+        "fieldConfig": {"defaults": {"unit": unit}},
+        "targets": [{"expr": expr}],
+    }
+    if mappings:
+        p["fieldConfig"]["defaults"]["mappings"] = mappings
+    if thresholds:
+        p["fieldConfig"]["defaults"]["thresholds"] = thresholds
+        p["fieldConfig"]["defaults"]["color"] = {"mode": "thresholds"}
+    return p
+
+
+def ts_panel(id, title, exprs, x, y, w=8, h=8, unit="short"):
+    targets = []
+    for e in exprs:
+        if isinstance(e, tuple):
+            targets.append({"expr": e[0], "legendFormat": e[1]})
+        else:
+            targets.append({"expr": e, "legendFormat": "{{pod}}"})
+    return {
+        "id": id, "title": title, "type": "timeseries",
+        "gridPos": {"h": h, "w": w, "x": x, "y": y},
+        "datasource": DS_PROM,
+        "fieldConfig": {"defaults": {"unit": unit}},
+        "targets": targets,
+    }
+
+
+def table_panel(id, title, expr, x, y, w=12, h=8):
+    return {
+        "id": id, "title": title, "type": "table",
+        "gridPos": {"h": h, "w": w, "x": x, "y": y},
+        "datasource": DS_PROM,
+        "targets": [{"expr": expr, "format": "table", "instant": True}],
+    }
+
+
+def log_panel(id, title, query, x, y, w=24, h=10):
+    return {
+        "id": id, "title": title, "type": "logs",
+        "gridPos": {"h": h, "w": w, "x": x, "y": y},
+        "datasource": DS_LOKI,
+        "targets": [{"expr": query}],
+    }
+
+
+def row(id, title, y, panels, collapsed=True):
+    return {
+        "id": id, "title": title, "type": "row",
+        "collapsed": collapsed, "gridPos": {"h": 1, "w": 24, "x": 0, "y": y},
+        "panels": panels,
+    }
+
+
+def write_dashboard(filename, dashboard, folder):
+    cm = {
+        "apiVersion": "v1",
+        "kind": "ConfigMap",
+        "metadata": {
+            "name": filename.replace(".yaml", "-dashboard"),
+            "namespace": "logging",
+            "labels": {"grafana_dashboard": "1"},
+            "annotations": {"grafana_folder": folder},
+        },
+        "data": {
+            filename.replace(".yaml", ".json"): json.dumps(dashboard, separators=(",", ":"))
+        },
+    }
+
+    import yaml
+    path = os.path.join(DASHBOARD_DIR, filename)
+    with open(path, "w") as f:
+        yaml.dump(cm, f, default_flow_style=False, allow_unicode=True)
+    print(f"  wrote {path}")
+
+
+# ============================================================================
+# Dashboard 1: Cluster Infrastructure
+# ============================================================================
+
+def build_cluster_infrastructure():
+    zero_thresholds = {"mode": "absolute", "steps": [
+        {"value": None, "color": "green"}, {"value": 1, "color": "red"}
+    ]}
+
+    panels = [
+        row(1, "Cluster Health", 0, [
+            stat_panel(2, "Nodes Ready", 'count(kube_node_status_condition{condition="Ready",status="true"} == 1)', 0, 1, thresholds={"mode":"absolute","steps":[{"value":None,"color":"red"},{"value":3,"color":"green"}]}),
+            stat_panel(3, "Pods Pending", 'sum(kube_pod_status_phase{phase="Pending"}) OR on() vector(0)', 4, 1, thresholds=zero_thresholds),
+            stat_panel(4, "CrashLoopBackOff", 'sum(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}) OR on() vector(0)', 8, 1, thresholds=zero_thresholds),
+            stat_panel(5, "OOMKilled (1h)", 'sum(increase(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[1h])) OR on() vector(0)', 12, 1, thresholds=zero_thresholds),
+            stat_panel(6, "Deploys Unavailable", 'count(kube_deployment_status_replicas_unavailable > 0) OR on() vector(0)', 16, 1, thresholds=zero_thresholds),
+            stat_panel(7, "Services Down", 'count(probe_success == 0) OR on() vector(0)', 20, 1, thresholds=zero_thresholds),
+        ]),
+        row(10, "Jobs & CronJobs", 1, [
+            stat_panel(11, "Failed Jobs", 'count(kube_job_status_failed > 0) OR on() vector(0)', 0, 2, thresholds=zero_thresholds),
+            table_panel(12, "Failed Jobs Detail", 'kube_job_status_failed > 0', 4, 2, w=10),
+            table_panel(13, "Stuck Jobs (>1h)", 'kube_job_status_active == 1 and on(job_name,namespace) (time() - kube_job_status_start_time) > 3600', 14, 2, w=10),
+            ts_panel(14, "CronJob Last Success", [
+                ('kube_cronjob_status_last_successful_time{namespace=~"cicd|kube-system|paperless"}', "{{namespace}}/{{cronjob}}")
+            ], 0, 10, w=12, unit="dateTimeFromNow"),
+            ts_panel(15, "Container Restart Storm (top 10)", [
+                ('topk(10, sum(rate(kube_pod_container_status_restarts_total[15m])) by (namespace, pod))', "{{namespace}}/{{pod}}")
+            ], 12, 10, w=12),
+        ]),
+        row(20, "Node Resources", 2, [
+            ts_panel(21, "CPU % by Node", [
+                ('(1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m])) by (instance)) * 100', "{{instance}}")
+            ], 0, 3, unit="percent"),
+            ts_panel(22, "Memory % by Node", [
+                ('(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100', "{{instance}}")
+            ], 8, 3, unit="percent"),
+            ts_panel(23, "Disk % by Node", [
+                ('(1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100', "{{instance}}")
+            ], 16, 3, unit="percent"),
+            ts_panel(24, "Load Average", [
+                ("node_load1", "1m {{instance}}"),
+                ("node_load5", "5m {{instance}}"),
+            ], 0, 11),
+            ts_panel(25, "Network Errors & Drops", [
+                ("rate(node_network_receive_errs_total[5m])", "rx-err {{instance}}"),
+                ("rate(node_network_transmit_errs_total[5m])", "tx-err {{instance}}"),
+                ("rate(node_network_receive_drop_total[5m])", "rx-drop {{instance}}"),
+            ], 8, 11),
+        ]),
+        row(30, "Control Plane", 3, [
+            stat_panel(31, "API Server Up", 'min(up{job="apiserver"})', 0, 4, mappings=[
+                {"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}
+            ]),
+            ts_panel(32, "API Server Request Rate", [
+                ('sum(rate(apiserver_request_total[5m])) by (verb, code)', "{{verb}} {{code}}")
+            ], 4, 4, w=10),
+            ts_panel(33, "API Server Error Rate %", [
+                ('sum(rate(apiserver_request_total{code=~"5.."}[5m])) / sum(rate(apiserver_request_total[5m])) * 100', "5xx %")
+            ], 14, 4, w=10, unit="percent"),
+            ts_panel(34, "API Server Latency", [
+                ('histogram_quantile(0.95, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))', "p95"),
+                ('histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))', "p99"),
+            ], 0, 12, unit="s"),
+            ts_panel(35, "etcd Request Duration", [
+                ('histogram_quantile(0.99, sum(rate(etcd_request_duration_seconds_bucket[5m])) by (le))', "p99"),
+            ], 8, 12, unit="s"),
+        ]),
+        row(40, "Storage", 4, [
+            ts_panel(41, "Longhorn Disk Capacity", [
+                ("longhorn_disk_capacity_bytes", "capacity {{node}}"),
+                ("longhorn_disk_reservation_bytes", "reserved {{node}}"),
+            ], 0, 5, unit="bytes"),
+            ts_panel(42, "PVC Phase", [
+                ('kube_persistentvolumeclaim_status_phase', "{{namespace}}/{{persistentvolumeclaim}} {{phase}}")
+            ], 8, 5),
+        ]),
+        row(50, "DNS & Networking", 5, [
+            ts_panel(51, "CoreDNS Cache Hit Rate", [
+                ('rate(coredns_cache_hits_total[5m]) / (rate(coredns_cache_hits_total[5m]) + rate(coredns_cache_misses_total[5m]))', "{{server}}")
+            ], 0, 6, unit="percentunit"),
+            ts_panel(52, "CoreDNS Errors", [
+                ('sum(rate(coredns_dns_responses_total{rcode=~"SERVFAIL|NXDOMAIN"}[5m])) by (rcode)', "{{rcode}}")
+            ], 8, 6),
+        ]),
+        row(60, "Logs", 6, [
+            ts_panel(61, "Error Rate by Namespace", [
+                ('sum by (namespace) (count_over_time({namespace=~"kube-system|cert-manager|ingress-nginx|longhorn-system"} |= "error" [5m]))', "{{namespace}}")
+            ], 0, 7),
+            log_panel(62, "Control Plane Logs", '{namespace="kube-system"}', 0, 15),
+            log_panel(63, "Cluster Addon Logs", '{namespace=~"cert-manager|ingress-nginx|longhorn-system"}', 0, 25),
+        ]),
+    ]
+
+    return {
+        "title": "Cluster Infrastructure",
+        "uid": "cluster-infra",
+        "schemaVersion": 39,
+        "timezone": "browser",
+        "time": {"from": "now-6h", "to": "now"},
+        "refresh": "30s",
+        "tags": ["infrastructure", "k8s"],
+        "panels": panels,
+    }
+
+
+# ============================================================================
+# Dashboard 3: API Gateway
+# ============================================================================
+
+def build_api_gateway():
+    panels = [
+        row(1, "Gateway Health", 0, [
+            stat_panel(2, "Gateway Pods Ready", 'sum(kube_pod_status_ready{namespace="api",condition="true"})', 0, 1, thresholds={"mode":"absolute","steps":[{"value":None,"color":"red"},{"value":3,"color":"green"}]}),
+            stat_panel(3, "Probe: healthz", 'probe_success{instance=~".*api.riotpiao.com/healthz"}', 4, 1, mappings=[
+                {"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}
+            ]),
+            ts_panel(4, "Probe Latency", [
+                ('probe_duration_seconds{instance=~".*api.riotpiao.com.*"}', "{{instance}}")
+            ], 8, 1, unit="s"),
+        ], collapsed=False),
+        row(10, "Ingress Traffic (nginx)", 1, [
+            ts_panel(11, "Request Rate by Status", [
+                ('sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m])) by (status)', "{{status}}")
+            ], 0, 2),
+            ts_panel(12, "Error Rate %", [
+                ('sum(rate(nginx_ingress_controller_requests{ingress="api",status=~"5.."}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m])) * 100', "5xx"),
+                ('sum(rate(nginx_ingress_controller_requests{ingress="api",status=~"4.."}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m])) * 100', "4xx"),
+            ], 8, 2, unit="percent"),
+            ts_panel(13, "Latency p50/p95/p99", [
+                ('histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le))', "p50"),
+                ('histogram_quantile(0.95, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le))', "p95"),
+                ('histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le))', "p99"),
+            ], 16, 2, unit="s"),
+        ]),
+        row(20, "LLM Serving", 2, [
+            stat_panel(21, "LLM Pods Ready", 'sum(kube_pod_status_ready{namespace="llm-serving",condition="true"})', 0, 3),
+            ts_panel(22, "CPU by Predictor", [
+                ('sum(rate(container_cpu_usage_seconds_total{namespace="llm-serving"}[5m])) by (pod)', "{{pod}}")
+            ], 4, 3),
+            ts_panel(23, "Memory by Predictor", [
+                ('sum(container_memory_working_set_bytes{namespace="llm-serving"}) by (pod)', "{{pod}}")
+            ], 12, 3, unit="bytes"),
+            ts_panel(24, "Predictor Restarts", [
+                ('sum(rate(kube_pod_container_status_restarts_total{namespace="llm-serving"}[15m])) by (pod)', "{{pod}}")
+            ], 20, 3, w=4),
+        ]),
+        row(30, "Gateway Resources", 3, [
+            ts_panel(31, "CPU by Gateway Pod", [
+                ('sum(rate(container_cpu_usage_seconds_total{namespace="api"}[5m])) by (pod)', "{{pod}}")
+            ], 0, 4),
+            ts_panel(32, "Memory by Gateway Pod", [
+                ('sum(container_memory_working_set_bytes{namespace="api"}) by (pod)', "{{pod}}")
+            ], 8, 4, unit="bytes"),
+            ts_panel(33, "Gateway Restarts", [
+                ('sum(rate(kube_pod_container_status_restarts_total{namespace="api"}[15m])) by (pod)', "{{pod}}")
+            ], 16, 4),
+        ]),
+        row(40, "Logs", 4, [
+            log_panel(41, "Gateway Logs", '{namespace="api",container="gateway"}', 0, 5),
+            log_panel(42, "LLM Serving Logs", '{namespace="llm-serving"}', 0, 15),
+        ]),
+    ]
+
+    return {
+        "title": "API Gateway",
+        "uid": "api-gateway",
+        "schemaVersion": 39,
+        "timezone": "browser",
+        "time": {"from": "now-6h", "to": "now"},
+        "refresh": "30s",
+        "tags": ["api", "gateway", "llm"],
+        "panels": panels,
+    }
+
+
+# ============================================================================
+# Generate
+# ============================================================================
+
+print("Generating dashboards...")
+
+# Dashboard 1
+write_dashboard("cluster-infrastructure.yaml", build_cluster_infrastructure(), "Infrastructure")
+
+# Dashboard 3
+write_dashboard("api-gateway.yaml", build_api_gateway(), "API")
+
+print("Done.")
@@ -1,121 +0,0 @@
-# k8s/monitoring/dashboards/hardware-overview.yaml
-# Trimmed operator at-a-glance view across all nodes — node-exporter already
-# powers the deep-dive "Node Exporter Full" (#1860, see grafana-values.yaml),
-# this is the quick health-check version, not a replacement for it.
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: hardware-overview-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-data:
-  hardware-overview.json: |
-    {
-      "title": "Hardware Statistics (Operator Overview)",
-      "uid": "hardware-overview",
-      "schemaVersion": 39,
-      "timezone": "browser",
-      "time": { "from": "now-6h", "to": "now" },
-      "refresh": "30s",
-      "panels": [
-        {
-          "id": 1,
-          "title": "Nodes up / down",
-          "type": "stat",
-          "gridPos": { "h": 5, "w": 24, "x": 0, "y": 0 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "mappings": [
-                { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" } } },
-                { "type": "value", "options": { "1": { "text": "UP", "color": "green" } } }
-              ]
-            }
-          },
-          "targets": [
-            { "expr": "up{job=~\".*node-exporter.*\"}", "legendFormat": "{{instance}}" }
-          ]
-        },
-        {
-          "id": 2,
-          "title": "CPU usage % by node",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 5 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } },
-          "targets": [
-            {
-              "expr": "(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) by (instance)) * 100",
-              "legendFormat": "{{instance}}"
-            }
-          ]
-        },
-        {
-          "id": 3,
-          "title": "Memory usage % by node",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 5 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } },
-          "targets": [
-            {
-              "expr": "(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100",
-              "legendFormat": "{{instance}}"
-            }
-          ]
-        },
-        {
-          "id": 4,
-          "title": "Root filesystem usage % by node",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } },
-          "targets": [
-            {
-              "expr": "(1 - node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"}) * 100",
-              "legendFormat": "{{instance}}"
-            }
-          ]
-        },
-        {
-          "id": 5,
-          "title": "Root filesystem space remaining",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 13 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "bytes" } },
-          "targets": [
-            {
-              "expr": "node_filesystem_avail_bytes{mountpoint=\"/\"}",
-              "legendFormat": "{{instance}}"
-            }
-          ]
-        },
-        {
-          "id": 6,
-          "title": "Network errors/drops by node",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 21 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            { "expr": "rate(node_network_receive_errs_total[5m])", "legendFormat": "{{instance}} rx errs" },
-            { "expr": "rate(node_network_transmit_errs_total[5m])", "legendFormat": "{{instance}} tx errs" },
-            { "expr": "rate(node_network_receive_drop_total[5m])", "legendFormat": "{{instance}} rx drops" },
-            { "expr": "rate(node_network_transmit_drop_total[5m])", "legendFormat": "{{instance}} tx drops" }
-          ]
-        },
-        {
-          "id": 7,
-          "title": "Load average (1m / 5m) by node",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 21 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            { "expr": "node_load1", "legendFormat": "{{instance}} load1" },
-            { "expr": "node_load5", "legendFormat": "{{instance}} load5" }
-          ]
-        }
-      ]
-    }
@@ -1,185 +0,0 @@
-# k8s/monitoring/dashboards/kube-controller-health.yaml
-# Talos binds controller-manager/scheduler/etcd to 127.0.0.1, so Prometheus
-# can't scrape them directly (see prometheus-values.yaml). kube-apiserver is
-# the one control-plane component that's still reachable (its ServiceMonitor
-# targets the in-cluster `kubernetes` service, not localhost) — paired with
-# kube-state-metrics signals as a proxy for controller/scheduler health.
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: kube-controller-health-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-data:
-  kube-controller-health.json: |
-    {
-      "title": "Kube-Controller Health",
-      "uid": "kube-controller-health",
-      "schemaVersion": 39,
-      "timezone": "browser",
-      "time": { "from": "now-6h", "to": "now" },
-      "refresh": "30s",
-      "panels": [
-        {
-          "id": 1,
-          "title": "API server — up",
-          "type": "stat",
-          "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "mappings": [
-                { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" } } },
-                { "type": "value", "options": { "1": { "text": "UP", "color": "green" } } }
-              ]
-            }
-          },
-          "targets": [
-            { "expr": "min(up{job=\"apiserver\"})", "legendFormat": "apiserver" }
-          ]
-        },
-        {
-          "id": 2,
-          "title": "API server — request rate by verb/code",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 18, "x": 6, "y": 0 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            {
-              "expr": "sum(rate(apiserver_request_total[5m])) by (verb, code)",
-              "legendFormat": "{{verb}} {{code}}"
-            }
-          ]
-        },
-        {
-          "id": 3,
-          "title": "API server — error rate % (5xx)",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "percent" } },
-          "targets": [
-            {
-              "expr": "sum(rate(apiserver_request_total{code=~\"5..\"}[5m])) / sum(rate(apiserver_request_total[5m])) * 100",
-              "legendFormat": "5xx %"
-            }
-          ]
-        },
-        {
-          "id": 4,
-          "title": "API server — latency p95 / p99",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "s" } },
-          "targets": [
-            {
-              "expr": "histogram_quantile(0.95, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))",
-              "legendFormat": "p95"
-            },
-            {
-              "expr": "histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))",
-              "legendFormat": "p99"
-            }
-          ]
-        },
-        {
-          "id": 5,
-          "title": "Pods stuck Pending",
-          "type": "stat",
-          "gridPos": { "h": 5, "w": 8, "x": 0, "y": 16 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  { "value": 0, "color": "green" },
-                  { "value": 1, "color": "yellow" },
-                  { "value": 5, "color": "red" }
-                ]
-              }
-            }
-          },
-          "targets": [
-            { "expr": "sum(kube_pod_status_phase{phase=\"Pending\"}) OR on() vector(0)", "legendFormat": "pending" }
-          ]
-        },
-        {
-          "id": 6,
-          "title": "CrashLoopBackOff containers",
-          "type": "stat",
-          "gridPos": { "h": 5, "w": 8, "x": 8, "y": 16 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  { "value": 0, "color": "green" },
-                  { "value": 1, "color": "red" }
-                ]
-              }
-            }
-          },
-          "targets": [
-            { "expr": "sum(kube_pod_container_status_waiting_reason{reason=\"CrashLoopBackOff\"}) OR on() vector(0)", "legendFormat": "crashlooping" }
-          ]
-        },
-        {
-          "id": 7,
-          "title": "Nodes NotReady",
-          "type": "stat",
-          "gridPos": { "h": 5, "w": 8, "x": 16, "y": 16 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  { "value": 0, "color": "green" },
-                  { "value": 1, "color": "red" }
-                ]
-              }
-            }
-          },
-          "targets": [
-            { "expr": "count(kube_node_status_condition{condition=\"Ready\", status=\"true\"} == 0) OR on() vector(0)", "legendFormat": "not ready" }
-          ]
-        },
-        {
-          "id": 8,
-          "title": "Failed Jobs",
-          "type": "table",
-          "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            { "expr": "kube_job_status_failed > 0", "format": "table", "instant": true }
-          ]
-        },
-        {
-          "id": 9,
-          "title": "Deployments with unavailable replicas",
-          "type": "table",
-          "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            { "expr": "kube_deployment_status_replicas_unavailable > 0", "format": "table", "instant": true }
-          ]
-        },
-        {
-          "id": 10,
-          "title": "Container restart rate by pod",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 24, "x": 0, "y": 28 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            {
-              "expr": "sum(rate(kube_pod_container_status_restarts_total[15m])) by (namespace, pod)",
-              "legendFormat": "{{namespace}}/{{pod}}"
-            }
-          ]
-        }
-      ]
-    }
@@ -1,28 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: llm-frontend-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-  annotations:
-    grafana_folder: "LLM"
-    # No request-level panels. The rate/error/latency/bandwidth row used to run
-    # on Kong's prometheus plugin; Kong was retired 2026-08-19 and the Go
-    # gateway that replaced it does not expose /metrics yet, so those panels
-    # were removed rather than left querying series that no longer exist.
-    # What is left is pod-level: readiness, CPU/memory, restarts, logs.
-    #
-    # Restoring request-level and per-model observability means wiring three
-    # sources, none of which are in place: gateway metrics (RED plus token
-    # counts and TTFT, which the gateway can measure because it sees the
-    # response stream), vLLM's own /metrics on reasoning-predictor (rich --
-    # vllm:time_to_first_token_seconds, vllm:inter_token_latency_seconds,
-    # vllm:e2e_request_latency_seconds, vllm:kv_cache_usage_perc), and TEI's
-    # /metrics on embeddings/reranker. Ollama exposes no Prometheus endpoint at
-    # all (verified: /metrics returns 404), so ornith can only ever be observed
-    # from the gateway side. No ServiceMonitor exists for the llm-serving
-    # namespace today, so none of the engine metrics are being scraped.
-data:
-  llm-frontend.json: |
-    {"title":"LLM Frontend","uid":"llm-frontend","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"llm-serving pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"llm-serving\",condition=\"true\"})"}]},{"id":3,"title":"agent-pod ready","type":"stat","gridPos":{"h":4,"w":8,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"agent-pod\",condition=\"true\"})"}]},{"id":4,"title":"api gateway pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"api\",condition=\"true\"})"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=~\"llm-serving|agent-pod|api\"}[5m])) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":12,"title":"Memory by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=~\"llm-serving|agent-pod|api\"}) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":13,"title":"GPU-node predictor restarts","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"llm-serving\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"llm-serving logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"llm-serving\"}"}]},{"id":22,"title":"agent-pod logs (pi runs)","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":14},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"agent-pod\"}"}]},{"id":23,"title":"api gateway logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":24},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"api\"}"}]}]}]}
@@ -1,172 +0,0 @@
-# k8s/monitoring/dashboards/service-availability.yaml
-# Active uptime/availability from blackbox-exporter probes — the signal that
-# covers low-traffic services (Vault, MinIO, Longhorn UI) where RED metrics
-# alone can't distinguish "idle" from "down".
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: service-availability-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-data:
-  service-availability.json: |
-    {
-      "title": "Service Availability & Certificate Expiration",
-      "uid": "svc-availability",
-      "schemaVersion": 39,
-      "timezone": "browser",
-      "time": { "from": "now-24h", "to": "now" },
-      "refresh": "30s",
-      "panels": [
-        {
-          "id": 1,
-          "title": "Up / Down — all probed services",
-          "type": "stat",
-          "gridPos": { "h": 6, "w": 24, "x": 0, "y": 0 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "mappings": [
-                { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" } } },
-                { "type": "value", "options": { "1": { "text": "UP", "color": "green" } } }
-              ],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  { "value": 0, "color": "red" },
-                  { "value": 1, "color": "green" }
-                ]
-              }
-            }
-          },
-          "targets": [
-            { "expr": "probe_success", "legendFormat": "{{instance}}" }
-          ]
-        },
-        {
-          "id": 2,
-          "title": "Uptime % trend",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } },
-          "targets": [
-            {
-              "expr": "avg_over_time(probe_success[$__rate_interval]) * 100",
-              "legendFormat": "{{instance}}"
-            }
-          ]
-        },
-        {
-          "id": 3,
-          "title": "Probe latency",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "s" } },
-          "targets": [
-            { "expr": "probe_duration_seconds", "legendFormat": "{{instance}}" }
-          ]
-        },
-        {
-          "id": 4,
-          "title": "7-day SLO (% successful probes)",
-          "type": "table",
-          "gridPos": { "h": 8, "w": 24, "x": 0, "y": 14 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "unit": "percent",
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  { "value": 0, "color": "red" },
-                  { "value": 99, "color": "yellow" },
-                  { "value": 99.9, "color": "green" }
-                ]
-              }
-            }
-          },
-          "targets": [
-            {
-              "expr": "avg_over_time(probe_success[7d]) * 100",
-              "format": "table",
-              "instant": true
-            }
-          ]
-        },
-        {
-          "id": 5,
-          "title": "Services DOWN right now",
-          "type": "stat",
-          "gridPos": { "h": 4, "w": 12, "x": 0, "y": 22 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  { "value": 0, "color": "green" },
-                  { "value": 1, "color": "red" }
-                ]
-              }
-            }
-          },
-          "targets": [
-            { "expr": "count(probe_success == 0) OR on() vector(0)", "legendFormat": "down" }
-          ]
-        },
-        {
-          "id": 6,
-          "title": "Certs expiring in < 14 days",
-          "type": "stat",
-          "gridPos": { "h": 4, "w": 12, "x": 12, "y": 22 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  { "value": 0, "color": "green" },
-                  { "value": 1, "color": "red" }
-                ]
-              }
-            }
-          },
-          "targets": [
-            {
-              "expr": "count((certmanager_certificate_expiration_timestamp_seconds - time()) / 86400 < 14) OR on() vector(0)",
-              "legendFormat": "expiring"
-            }
-          ]
-        },
-        {
-          "id": 7,
-          "title": "Certificate expiry — days remaining",
-          "type": "table",
-          "gridPos": { "h": 8, "w": 24, "x": 0, "y": 26 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  { "value": 0, "color": "red" },
-                  { "value": 14, "color": "yellow" },
-                  { "value": 30, "color": "green" }
-                ]
-              }
-            }
-          },
-          "targets": [
-            {
-              "expr": "(certmanager_certificate_expiration_timestamp_seconds - time()) / 86400",
-              "legendFormat": "{{name}}",
-              "format": "table",
-              "instant": true
-            }
-          ]
-        }
-      ]
-    }
@@ -1,141 +0,0 @@
-# k8s/monitoring/dashboards/service-golden-signals.yaml
-# RED metrics (rate/errors/duration) for every service fronted by ingress-nginx.
-# Picked up automatically by Grafana's sidecar (grafana_dashboard=1 label) — see
-# sidecar.dashboards in k8s/logging/grafana-values.yaml.
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: service-golden-signals-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-data:
-  service-golden-signals.json: |
-    {
-      "title": "Latency & Golden Signals (Ingress RED)",
-      "uid": "svc-golden-signals",
-      "schemaVersion": 39,
-      "timezone": "browser",
-      "time": { "from": "now-6h", "to": "now" },
-      "refresh": "30s",
-      "templating": {
-        "list": [
-          {
-            "name": "ingress",
-            "type": "query",
-            "datasource": { "type": "prometheus", "uid": "prometheus" },
-            "query": "label_values(nginx_ingress_controller_requests, ingress)",
-            "refresh": 2,
-            "includeAll": false
-          }
-        ]
-      },
-      "panels": [
-        {
-          "id": 1,
-          "title": "Request rate by status — $ingress",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            {
-              "expr": "sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\"}[5m])) by (status)",
-              "legendFormat": "{{status}}"
-            }
-          ]
-        },
-        {
-          "id": 2,
-          "title": "Error rate % (4xx / 5xx) — $ingress",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "percent" } },
-          "targets": [
-            {
-              "expr": "sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\", status=~\"5..\"}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\"}[5m])) * 100",
-              "legendFormat": "5xx"
-            },
-            {
-              "expr": "sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\", status=~\"4..\"}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\"}[5m])) * 100",
-              "legendFormat": "4xx"
-            }
-          ]
-        },
-        {
-          "id": 3,
-          "title": "Latency p50 / p95 / p99 — $ingress",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "s" } },
-          "targets": [
-            {
-              "expr": "histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"$ingress\"}[5m])) by (le))",
-              "legendFormat": "p50"
-            },
-            {
-              "expr": "histogram_quantile(0.95, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"$ingress\"}[5m])) by (le))",
-              "legendFormat": "p95"
-            },
-            {
-              "expr": "histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"$ingress\"}[5m])) by (le))",
-              "legendFormat": "p99"
-            }
-          ]
-        },
-        {
-          "id": 4,
-          "title": "All services — traffic overview",
-          "type": "table",
-          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            {
-              "expr": "topk(11, sum(rate(nginx_ingress_controller_requests[5m])) by (ingress))",
-              "format": "table",
-              "instant": true
-            }
-          ]
-        },
-        {
-          "id": 5,
-          "title": "Customer-facing failures (5xx count, window total)",
-          "type": "stat",
-          "gridPos": { "h": 5, "w": 12, "x": 0, "y": 16 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  { "value": 0, "color": "green" },
-                  { "value": 1, "color": "yellow" },
-                  { "value": 50, "color": "red" }
-                ]
-              }
-            }
-          },
-          "targets": [
-            {
-              "expr": "sum(increase(nginx_ingress_controller_requests{status=~\"5..\"}[$__range])) OR on() vector(0)",
-              "legendFormat": "5xx total"
-            }
-          ]
-        },
-        {
-          "id": 6,
-          "title": "Top 5 error-contributing services",
-          "type": "table",
-          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            {
-              "expr": "topk(5, sum(rate(nginx_ingress_controller_requests{status=~\"5..\"}[5m])) by (ingress))",
-              "format": "table",
-              "instant": true
-            }
-          ]
-        }
-      ]
-    }
@@ -1,109 +0,0 @@
-# k8s/monitoring/dashboards/service-internals.yaml
-# Native per-service metrics — the "why" layer behind the ingress RED/uptime
-# dashboards (e.g. ingress shows MinIO is slow; this shows disk offline).
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: service-internals-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-data:
-  service-internals.json: |
-    {
-      "title": "Service Internals (MinIO / Forgejo / Argo CD / cert-manager / Vault / Longhorn)",
-      "uid": "svc-internals",
-      "schemaVersion": 39,
-      "timezone": "browser",
-      "time": { "from": "now-6h", "to": "now" },
-      "refresh": "30s",
-      "panels": [
-        { "id": 1, "title": "MinIO — disk/node offline", "type": "timeseries",
-          "gridPos": { "h": 6, "w": 12, "x": 0, "y": 0 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            { "expr": "minio_cluster_disk_offline_total", "legendFormat": "disks offline" },
-            { "expr": "minio_cluster_nodes_offline_total", "legendFormat": "nodes offline" }
-          ]
-        },
-        { "id": 2, "title": "MinIO — S3 request errors", "type": "timeseries",
-          "gridPos": { "h": 6, "w": 12, "x": 12, "y": 0 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            { "expr": "sum(rate(minio_s3_requests_errors_total[5m])) by (api)", "legendFormat": "{{api}}" }
-          ]
-        },
-        { "id": 3, "title": "MinIO — S3 TTFB latency", "type": "timeseries",
-          "gridPos": { "h": 6, "w": 12, "x": 0, "y": 6 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "s" } },
-          "targets": [
-            { "expr": "minio_s3_time_ttfb_seconds_distribution", "legendFormat": "{{api}}" }
-          ]
-        },
-        { "id": 4, "title": "Forgejo — repos / orgs", "type": "stat",
-          "gridPos": { "h": 6, "w": 12, "x": 12, "y": 6 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            { "expr": "gitea_repositories", "legendFormat": "repos" },
-            { "expr": "gitea_organizations", "legendFormat": "orgs" }
-          ]
-        },
-        { "id": 5, "title": "Forgejo — process health (CPU/mem)", "type": "timeseries",
-          "gridPos": { "h": 6, "w": 12, "x": 0, "y": 12 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            { "expr": "rate(process_cpu_seconds_total{job=~\".*forgejo.*|.*gitea.*\"}[5m])", "legendFormat": "cpu" },
-            { "expr": "process_resident_memory_bytes{job=~\".*forgejo.*|.*gitea.*\"}", "legendFormat": "mem" }
-          ]
-        },
-        { "id": 6, "title": "Argo CD — app sync/health status", "type": "table",
-          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            { "expr": "argocd_app_info", "format": "table", "instant": true }
-          ]
-        },
-        { "id": 7, "title": "cert-manager — days to cert expiry", "type": "stat",
-          "gridPos": { "h": 6, "w": 12, "x": 0, "y": 18 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  { "value": 0, "color": "red" },
-                  { "value": 14, "color": "yellow" },
-                  { "value": 30, "color": "green" }
-                ]
-              }
-            }
-          },
-          "targets": [
-            { "expr": "(certmanager_certificate_expiration_timestamp_seconds - time()) / 86400", "legendFormat": "{{name}}" }
-          ]
-        },
-        { "id": 8, "title": "Vault — sealed/unsealed", "type": "stat",
-          "gridPos": { "h": 6, "w": 6, "x": 12, "y": 20 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "mappings": [
-                { "type": "value", "options": { "0": { "text": "SEALED", "color": "red" } } },
-                { "type": "value", "options": { "1": { "text": "UNSEALED", "color": "green" } } }
-              ]
-            }
-          },
-          "targets": [
-            { "expr": "vault_core_unsealed", "legendFormat": "vault" }
-          ]
-        },
-        { "id": 9, "title": "Longhorn — volume robustness", "type": "table",
-          "gridPos": { "h": 6, "w": 6, "x": 18, "y": 20 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            { "expr": "longhorn_volume_robustness", "format": "table", "instant": true }
-          ]
-        }
-      ]
-    }
@@ -1,12 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: svc-argocd-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-  annotations:
-    grafana_folder: "Argo CD"
-data:
-  svc-argocd.json: |
-    {"title":"Argo CD — Service Overview","uid":"svc-argocd","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"}}},"targets":[{"expr":"min(up{job=~\"argocd-.*\"})"}]},{"id":3,"title":"HTTP requests","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_http_request_total[5m])) by (status)","legendFormat":"{{status}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_http_request_total{status=~\"5..\"}[5m])) / sum(rate(argocd_http_request_total[5m])) * 100"}]},{"id":5,"title":"Request latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(argocd_http_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"argocd\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"argocd\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restarts","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"argocd\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Applications & Sync","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Applications","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"argocd_app_total"}]},{"id":22,"title":"Sync by status","type":"timeseries","gridPos":{"h":6,"w":9,"x":6,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_app_sync_total[5m])) by (sync_status)","legendFormat":"{{sync_status}}"}]},{"id":23,"title":"Degraded apps","type":"stat","gridPos":{"h":6,"w":6,"x":15,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"argocd_app_health_degraded_total"}]},{"id":24,"title":"Git sync ops","type":"timeseries","gridPos":{"h":6,"w":12,"x":0,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_git_sync_total[5m])) by (git_operation,git_status)","legendFormat":"{{git_operation}}/{{git_status}}"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"argocd\"}"}]}]}]}
@@ -1,143 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: svc-authentik-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-  annotations:
-    grafana_folder: "Authentik"
-data:
-  svc-authentik.json: |
-    {
-      "title": "Authentik — Service Overview",
-      "uid": "svc-authentik",
-      "schemaVersion": 39,
-      "timezone": "browser",
-      "time": { "from": "now-6h", "to": "now" },
-      "refresh": "30s",
-      "panels": [
-        {
-          "id": 1, "title": "Row: Availability & Golden Signals", "type": "row",
-          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
-          "panels": [
-            {
-              "id": 2, "title": "Up", "type": "stat",
-              "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "fieldConfig": {
-                "defaults": {
-                  "color": { "mode": "thresholds" },
-                  "mappings": [
-                    { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }
-                  ],
-                  "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" }, { "value": 1, "color": "green" } ] }
-                }
-              },
-              "targets": [{ "expr": "min(up{job=\"authentik-server\"})" }]
-            },
-            {
-              "id": 3, "title": "HTTP request rate by status", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 9, "x": 6, "y": 1 },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "sum(rate(authentik_flows_execution_stage_time_count[5m])) by (flow_slug)", "legendFormat": "{{flow_slug}}" }]
-            },
-            {
-              "id": 4, "title": "Error rate % (5xx)", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 9, "x": 15, "y": 1 },
-              "fieldConfig": { "defaults": { "unit": "percent" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "(1 - (authentik_flows_cached / authentik_flows_execution_stage_time_count)) * 100" }]
-            },
-            {
-              "id": 5, "title": "Request duration p50/p95/p99", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
-              "fieldConfig": { "defaults": { "unit": "s" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [
-                { "expr": "histogram_quantile(0.50, sum(rate(authentik_main_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p50" },
-                { "expr": "histogram_quantile(0.95, sum(rate(authentik_main_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p95" },
-                { "expr": "histogram_quantile(0.99, sum(rate(authentik_main_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p99" }
-              ]
-            }
-          ]
-        },
-        {
-          "id": 10, "title": "Row: Resource Usage", "type": "row",
-          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 },
-          "panels": [
-            {
-              "id": 11, "title": "CPU by pod", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 8, "x": 0, "y": 2 },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"iam\",pod=~\"authentik.*\"}[5m])) by (pod)", "legendFormat": "{{pod}}" }]
-            },
-            {
-              "id": 12, "title": "Memory by pod", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 8, "x": 8, "y": 2 },
-              "fieldConfig": { "defaults": { "unit": "bytes" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "sum(container_memory_working_set_bytes{namespace=\"iam\",pod=~\"authentik.*\"}) by (pod)", "legendFormat": "{{pod}}" }]
-            },
-            {
-              "id": 13, "title": "Restart rate by pod", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 8, "x": 16, "y": 2 },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "sum(rate(kube_pod_container_status_restarts_total{namespace=\"iam\",pod=~\"authentik.*\"}[15m])) by (pod)", "legendFormat": "{{pod}}" }]
-            }
-          ]
-        },
-        {
-          "id": 20, "title": "Row: Identity Provider (OIDC / OAuth2)", "type": "row",
-          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 },
-          "panels": [
-            {
-              "id": 21, "title": "Outpost connections", "type": "stat",
-              "gridPos": { "h": 7, "w": 6, "x": 0, "y": 3 },
-              "fieldConfig": { "defaults": { "unit": "short" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "authentik_outposts_connected" }]
-            },
-            {
-              "id": 22, "title": "Flows cached", "type": "stat",
-              "gridPos": { "h": 7, "w": 6, "x": 6, "y": 3 },
-              "fieldConfig": { "defaults": { "unit": "short" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "authentik_flows_cached" }]
-            },
-            {
-              "id": 23, "title": "Policies cached", "type": "stat",
-              "gridPos": { "h": 7, "w": 6, "x": 12, "y": 3 },
-              "fieldConfig": { "defaults": { "unit": "short" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "authentik_policies_cached" }]
-            },
-            {
-              "id": 24, "title": "Queued tasks", "type": "stat",
-              "gridPos": { "h": 7, "w": 6, "x": 18, "y": 3 },
-              "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "unit": "short" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "authentik_tasks_queued" }]
-            },
-            {
-              "id": 25, "title": "Admin workers", "type": "timeseries",
-              "gridPos": { "h": 7, "w": 12, "x": 0, "y": 10 },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "authentik_admin_workers" }]
-            }
-          ]
-        },
-        {
-          "id": 30, "title": "Row: Logs", "type": "row",
-          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 },
-          "panels": [
-            {
-              "id": 31, "title": "Recent logs", "type": "logs",
-              "gridPos": { "h": 10, "w": 24, "x": 0, "y": 4 },
-              "datasource": { "type": "loki", "uid": "loki" },
-              "targets": [{ "expr": "{namespace=\"iam\"}" }]
-            }
-          ]
-        }
-      ]
-    }
@@ -1,12 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: svc-forgejo-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-  annotations:
-    grafana_folder: "Forgejo"
-data:
-  svc-forgejo.json: |
-    {"title":"Forgejo — Service Overview","uid":"svc-forgejo","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"}}},"targets":[{"expr":"min(up{job=\"forgejo\"})"}]},{"id":3,"title":"HTTP requests by method","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_http_request_total[5m])) by (method)","legendFormat":"{{method}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_http_request_total{status=~\"5..\"}[5m])) / sum(rate(forgejo_http_request_total[5m])) * 100"}]},{"id":5,"title":"Request latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(forgejo_http_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"forgejo\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"forgejo\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restarts","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"forgejo\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Git Operations","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Repositories","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"forgejo_repositories_total"}]},{"id":22,"title":"Users","type":"stat","gridPos":{"h":6,"w":6,"x":6,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"forgejo_users_total"}]},{"id":23,"title":"Git ops rate","type":"timeseries","gridPos":{"h":6,"w":12,"x":12,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_git_operations_total[5m])) by (operation_type)","legendFormat":"{{operation_type}}"}]},{"id":24,"title":"Runner tasks","type":"timeseries","gridPos":{"h":6,"w":12,"x":0,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_runner_tasks_total[5m])) by (status)","legendFormat":"{{status}}"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"forgejo\"}"}]}]}]}
@@ -1,12 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: svc-grafana-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-  annotations:
-    grafana_folder: "Grafana"
-data:
-  svc-grafana.json: |
-    {"title":"Grafana — Service Overview","uid":"svc-grafana","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}]}},"targets":[{"expr":"min(up{job=\"grafana\"})"}]},{"id":3,"title":"HTTP requests","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(grafana_http_request_total[5m])) by (status)","legendFormat":"{{status}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(grafana_http_request_total{status=~\"5..\"}[5m])) / sum(rate(grafana_http_request_total[5m])) * 100"}]},{"id":5,"title":"Request latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(grafana_http_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"logging\",pod=~\"grafana.*\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"logging\",pod=~\"grafana.*\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restarts","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"logging\",pod=~\"grafana.*\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Dashboards & Users","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Total dashboards","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"grafana_dashboard_total"}]},{"id":22,"title":"Total users","type":"stat","gridPos":{"h":6,"w":6,"x":6,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"grafana_user_total"}]},{"id":23,"title":"Total alerts","type":"stat","gridPos":{"h":6,"w":6,"x":12,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"grafana_alerts_total"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"logging\",container=\"grafana\"}"}]}]}]}
@@ -1,143 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: svc-minio-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-  annotations:
-    grafana_folder: "MinIO"
-data:
-  svc-minio.json: |
-    {
-      "title": "MinIO — Service Overview",
-      "uid": "svc-minio",
-      "schemaVersion": 39,
-      "timezone": "browser",
-      "time": { "from": "now-6h", "to": "now" },
-      "refresh": "30s",
-      "panels": [
-        {
-          "id": 1, "title": "Row: Availability & Golden Signals", "type": "row",
-          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
-          "panels": [
-            {
-              "id": 2, "title": "Up", "type": "stat",
-              "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "fieldConfig": {
-                "defaults": {
-                  "color": { "mode": "thresholds" },
-                  "mappings": [
-                    { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }
-                  ],
-                  "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" }, { "value": 1, "color": "green" } ] }
-                }
-              },
-              "targets": [{ "expr": "min(up{job=\"minio\"})" }]
-            },
-            {
-              "id": 3, "title": "S3 request rate by method", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 9, "x": 6, "y": 1 },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "sum(rate(minio_s3_requests_total[5m])) by (method)", "legendFormat": "{{method}}" }]
-            },
-            {
-              "id": 4, "title": "Error rate %", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 9, "x": 15, "y": 1 },
-              "fieldConfig": { "defaults": { "unit": "percent" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "sum(rate(minio_s3_requests_total{error=\"true\"}[5m])) / sum(rate(minio_s3_requests_total[5m])) * 100" }]
-            },
-            {
-              "id": 5, "title": "Request duration p50/p95/p99", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
-              "fieldConfig": { "defaults": { "unit": "s" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [
-                { "expr": "histogram_quantile(0.50, sum(rate(minio_s3_requests_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p50" },
-                { "expr": "histogram_quantile(0.95, sum(rate(minio_s3_requests_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p95" },
-                { "expr": "histogram_quantile(0.99, sum(rate(minio_s3_requests_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p99" }
-              ]
-            }
-          ]
-        },
-        {
-          "id": 10, "title": "Row: Resource Usage", "type": "row",
-          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 },
-          "panels": [
-            {
-              "id": 11, "title": "CPU by pod", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 8, "x": 0, "y": 2 },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"storage\",pod=~\"minio.*\"}[5m])) by (pod)", "legendFormat": "{{pod}}" }]
-            },
-            {
-              "id": 12, "title": "Memory by pod", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 8, "x": 8, "y": 2 },
-              "fieldConfig": { "defaults": { "unit": "bytes" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "sum(container_memory_working_set_bytes{namespace=\"storage\",pod=~\"minio.*\"}) by (pod)", "legendFormat": "{{pod}}" }]
-            },
-            {
-              "id": 13, "title": "Restart rate by pod", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 8, "x": 16, "y": 2 },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "sum(rate(kube_pod_container_status_restarts_total{namespace=\"storage\",pod=~\"minio.*\"}[15m])) by (pod)", "legendFormat": "{{pod}}" }]
-            }
-          ]
-        },
-        {
-          "id": 20, "title": "Row: Storage & Replication", "type": "row",
-          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 },
-          "panels": [
-            {
-              "id": 21, "title": "Usable vs Raw capacity", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 12, "x": 0, "y": 3 },
-              "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "lineWidth": 2 } } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [
-                { "expr": "minio_cluster_capacity_usable_bytes", "legendFormat": "Usable" },
-                { "expr": "minio_cluster_capacity_raw_total_bytes", "legendFormat": "Raw Total" }
-              ]
-            },
-            {
-              "id": 22, "title": "Drive health (online/offline)", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 12, "x": 12, "y": 3 },
-              "fieldConfig": { "defaults": { "unit": "short" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [
-                { "expr": "minio_cluster_health_drives_online", "legendFormat": "Online" },
-                { "expr": "minio_cluster_health_drives_offline", "legendFormat": "Offline" }
-              ]
-            },
-            {
-              "id": 23, "title": "Replication lag (bytes pending)", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 12, "x": 0, "y": 11 },
-              "fieldConfig": { "defaults": { "unit": "bytes" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "minio_replication_metrics_replicating_byte_count", "legendFormat": "Pending replication" }]
-            },
-            {
-              "id": 24, "title": "Replication failures (bytes)", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 12, "x": 12, "y": 11 },
-              "fieldConfig": { "defaults": { "unit": "bytes" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "minio_replication_metrics_failed_byte_count", "legendFormat": "Failed replication" }]
-            }
-          ]
-        },
-        {
-          "id": 30, "title": "Row: Logs", "type": "row",
-          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 },
-          "panels": [
-            {
-              "id": 31, "title": "Recent logs", "type": "logs",
-              "gridPos": { "h": 10, "w": 24, "x": 0, "y": 4 },
-              "datasource": { "type": "loki", "uid": "loki" },
-              "targets": [{ "expr": "{namespace=\"storage\"}" }]
-            }
-          ]
-        }
-      ]
-    }
@@ -1,12 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: svc-vault-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-  annotations:
-    grafana_folder: "Vault"
-data:
-  svc-vault.json: |
-    {"title":"Vault — Service Overview","uid":"svc-vault","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability & Golden Signals","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"red"},{"value":1,"color":"green"}]}}},"targets":[{"expr":"min(up{job=\"vault\"})"}]},{"id":3,"title":"Request rate by status","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(vault_core_handle_request_total[5m])) by (method)","legendFormat":"{{method}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(vault_core_handle_request_total{error=\"true\"}[5m])) / sum(rate(vault_core_handle_request_total[5m])) * 100"}]},{"id":5,"title":"Request duration p50/p95/p99","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.50, sum(rate(vault_core_handle_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95, sum(rate(vault_core_handle_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99, sum(rate(vault_core_handle_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p99"}]}]},{"id":10,"title":"Row: Resource Usage","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU by pod","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"iam\",pod=~\"vault.*\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory by pod","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"iam\",pod=~\"vault.*\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restart rate","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"iam\",pod=~\"vault.*\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Vault Seal State","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Sealed","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"UNSEALED","color":"green"},"1":{"text":"SEALED","color":"red"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]}}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_core_unsealed"}]},{"id":22,"title":"Active","type":"stat","gridPos":{"h":6,"w":6,"x":6,"y":3},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"INACTIVE","color":"red"},"1":{"text":"ACTIVE","color":"green"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"red"},{"value":1,"color":"green"}]}}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_core_active"}]},{"id":23,"title":"Replication (Primary)","type":"stat","gridPos":{"h":6,"w":6,"x":12,"y":3},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"SECONDARY","color":"orange"},"1":{"text":"PRIMARY","color":"green"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"orange"},{"value":1,"color":"green"}]}}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_core_replication_primary"}]},{"id":24,"title":"Active tokens","type":"stat","gridPos":{"h":6,"w":6,"x":18,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_token_total"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"iam\",container=\"vault\"}"}]}]}]}
@@ -12,20 +12,12 @@ resources:
  - alerts/svc-grafana-rules.yaml
  - alerts/svc-minio-rules.yaml
  - alerts/svc-vault-rules.yaml
+  - alerts/cluster-alerts.yaml
+  - alerts/api-gateway-alerts.yaml
  - servicemonitors/argocd.yaml
  - servicemonitors/authentik.yaml
  - servicemonitors/forgejo.yaml
  - servicemonitors/minio.yaml
-  - dashboards/control-plane-logs.yaml
-  - dashboards/hardware-overview.yaml
-  - dashboards/kube-controller-health.yaml
-  - dashboards/llm-frontend.yaml
-  - dashboards/service-availability.yaml
-  - dashboards/service-golden-signals.yaml
-  - dashboards/service-internals.yaml
-  - dashboards/svc-argocd.yaml
-  - dashboards/svc-authentik.yaml
-  - dashboards/svc-forgejo.yaml
-  - dashboards/svc-grafana.yaml
-  - dashboards/svc-minio.yaml
-  - dashboards/svc-vault.yaml
+  - servicemonitors/ingress-nginx.yaml
+  - dashboards/cluster-infrastructure.yaml
+  - dashboards/api-gateway.yaml
@@ -0,0 +1,18 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: ingress-nginx
+  namespace: monitoring
+  labels:
+    release: prometheus
+spec:
+  namespaceSelector:
+    matchNames:
+      - ingress-nginx
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: ingress-nginx
+      app.kubernetes.io/component: controller
+  endpoints:
+    - port: metrics
+      interval: 30s
@@ -0,0 +1,15 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: homelab-admin-oidc
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+  # OIDC group for Authentik homelab-admins members
+  # When rock logs in via OIDC, k8s sees:
+  # - User: oidc:[email protected]
+  # - Groups: oidc:homelab-admins (from Authentik group claim)
+  - kind: Group
+    name: oidc:homelab-admins
@@ -6,6 +6,7 @@ kind: Kustomization
 # already fixed once in k8s/infra/minio and k8s/infra/iam. Every resource
 # here sets its own explicit metadata.namespace.
 resources:
+  - admin-oidc-binding.yaml
  - grafana-operator-role.yaml
  - minio-operator-role.yaml
  - forgejo-operator-role.yaml
@@ -38,7 +38,33 @@ import urllib.error
 import urllib.request

 AUTHENTIK_URL = "https://authentik.riotpiao.com"
-TOKEN = os.environ["AUTHENTIK_BOOTSTRAP_TOKEN"]
+
+# Try bootstrap token first, fallback to kubeconfig port-forward
+TOKEN = os.environ.get("AUTHENTIK_BOOTSTRAP_TOKEN")
+if not TOKEN:
+    import subprocess
+    # Use admin kubeconfig from Talos to port-forward and access Authentik
+    # First, get bootstrap token from k8s secret via talosctl kubeconfig
+    result = subprocess.run(
+        ["kubectl", "--kubeconfig=/tmp/admin-kubeconfig.yaml", "-n", "authentik",
+         "get", "secret", "authentik-bootstrap-token-secret", "-o", "jsonpath={.data.token}"],
+        capture_output=True, text=True
+    )
+    if result.returncode == 0:
+        TOKEN = result.stdout.strip()
+        if TOKEN:
+            # Decode if base64
+            try:
+                import base64
+                decoded = base64.b64decode(TOKEN).decode()
+                TOKEN = decoded
+            except:
+                pass  # Already decoded
+    else:
+        print("Error: AUTHENTIK_BOOTSTRAP_TOKEN not found")
+        print(f"  Set env var: export AUTHENTIK_BOOTSTRAP_TOKEN=<token>")
+        print(f"  Or use: kubectl -n authentik get secret ... ")
+        sys.exit(1)


 def api(method, path, data=None):
@@ -185,6 +211,10 @@ GROUP_PERMISSIONS = {
    "llm-users": ["llm:inference"],
    "memory-users": ["memory:read"],
    "memory-writers": ["memory:read", "memory:write"],
+    "s3-users": ["s3:read"],
+    "s3-writers": ["s3:read", "s3:write"],
+    "sqs-users": ["sqs:read"],
+    "sqs-writers": ["sqs:read", "sqs:write"],
 }
 perms = set()
 for group in request.user.groups.all():
@@ -712,6 +742,10 @@ for name, cfg in SERVICES.items():
 # Redirect URI matches kubelogin's (int128/kubelogin) documented default;
 # adjust here if a different kubectl OIDC plugin/port is actually used.
 print("Ensuring public OAuth2 client 'kubernetes' for kubectl OIDC login...")
+# Fetch explicit-consent flow for public authcode clients (kubelogin)
+status, res = api("GET", "/api/v3/flows/instances/?slug=default-provider-authorization-explicit-consent")
+k8s_authz_flow = res["results"][0]["pk"] if res["results"] else AUTHORIZATION_FLOW_PK
+
 k8s_provider = get_or_create(
    "/api/v3/providers/oauth2/", "/api/v3/providers/oauth2/",
    "name=kubernetes",
@@ -719,20 +753,20 @@ k8s_provider = get_or_create(
        "name": "kubernetes",
        "client_id": "kubernetes",
        "client_type": "public",
-        "authorization_flow": AUTHORIZATION_FLOW_PK,
+        "authorization_flow": k8s_authz_flow,
        "invalidation_flow": INVALIDATION_FLOW_PK,
        "signing_key": SIGNING_KEY_PK,
        "property_mappings": SCOPE_PKS,
        "sub_mode": "hashed_user_id",
        "include_claims_in_id_token": True,
-        "grant_types": ["authorization_code", "refresh_token"],
+        "grant_types": ["authorization_code", "refresh_token", "password"],
        "redirect_uris": [
            {"matching_mode": "strict", "url": "http://localhost:8000"},
        ],
    },
    patch_existing={
        "property_mappings": SCOPE_PKS,
-        "grant_types": ["authorization_code", "refresh_token"],
+        "grant_types": ["authorization_code", "refresh_token", "password"],
        "redirect_uris": [
            {"matching_mode": "strict", "url": "http://localhost:8000"},
        ],
@@ -806,7 +840,7 @@ print("\n[SERVICE ACCOUNTS] Creating service accounts for API access...")
 # Format: "<api>:<action>" e.g. "llm:inference", "memory:write"
 SERVICE_ACCOUNTS = {
    "portfolio-agent": {
-        "roles": ["llm:inference", "memory:read"],
+        "roles": ["llm:inference", "memory:read", "s3:read", "sqs:read", "temporal:admin"],
        "attributes": {
            "memory_projects": ["homelab", "portfolio"],
            "memory_visibility": "public",
@@ -815,7 +849,7 @@ SERVICE_ACCOUNTS = {
        "secret_name": "portfolio-agent-oidc",
    },
    "memory-agent": {
-        "roles": ["llm:inference", "memory:read", "memory:write"],
+        "roles": ["llm:inference", "memory:read", "memory:write", "s3:read", "s3:write", "sqs:read", "sqs:write"],
        "attributes": {
            "memory_projects": ["*"],
            "memory_visibility": "private",
Author	SHA1	Message	Date
rock	6b7e556d81	build: add docker.io to golang runner base image Build and push runner images / build-runners (push) Failing after 8s Details Pre-install docker.io so golang projects don't need to install on every build. Avoids duplicating 'apt-get install docker.io' in every CI workflow.	2026-09-06 05:56:12 -07:00
rock	6ecef14d0e	fix: add docker.io to node-runner image Build and push runner images / build-runners (push) Failing after 9s Details	2026-09-06 05:53:26 -07:00
rock	709b8b7039	fix: use forgejo/runner as base image, add Node.js on top Build and push runner images / build-runners (push) Failing after 17s Details - All runners now based on code.forgejo.org/forgejo/runner:6 (has runner binary) - Dockerfile adds Node.js + specialized tools (Go, Rust, docker) - CI workflow will build custom images and push to registry - Image Updater will auto-detect and update values.yaml - ArgoCD will sync new custom images when available	2026-09-05 23:43:21 -07:00
rock	39e2ed504c	bootstrap: use base images for runners, custom images via CI - golang-runner: docker:27-cli (has Node.js + docker) - rust-runner: docker:27-cli (bootstrap, CI adds Rust) - node-runner: node:22-bookworm (has Node.js, CI adds docker) CI workflow (.gitea/workflows/build-runner-images.yml): 1. Watches Dockerfile.* changes 2. Builds custom images with Node.js pre-installed 3. Pushes to registry 4. Image Updater detects and updates values.yaml 5. ArgoCD syncs to new custom images This avoids token complexity - just push Dockerfile changes to git!	2026-09-05 23:39:20 -07:00
rock	4d60684ca0	chore: remove .forgejo (Forgejo uses .gitea)	2026-09-05 23:33:36 -07:00
rock	b098d6a473	fix: runner CI workflow in .gitea (correct Forgejo folder) - Forgejo reads workflows from .gitea/, not .forgejo/ - Workflow monitors Dockerfile.golang/rust/node for changes - Builds and pushes images with commit SHA + latest tags - Image Updater auto-detects new images - ArgoCD syncs new versions	2026-09-05 23:31:46 -07:00
rock	caf0b5bfe6	fix: move runner CI workflow to .forgejo (Forgejo reads from .forgejo, not .gitea) - Workflow monitors Dockerfile.golang/rust/node for changes - Builds and pushes images with commit SHA + latest tags - Image Updater auto-detects new images - ArgoCD syncs new versions	2026-09-05 23:31:08 -07:00
rock	0e02cafdd7	ci: auto-build runner images on Dockerfile changes - Watches Dockerfile.golang, .rust, .node for changes - Builds and pushes images to Forgejo registry with commit SHA + latest tags - Image Updater detects new images automatically - ArgoCD syncs updated image tags - Workflow runs on: golang (has docker + dind)	2026-09-05 23:19:22 -07:00
rock	1e84f13009	feat: add ArgoCD Image Updater tracking for runner images - Added Image Updater annotations to forgejo-runner Applications - Image Updater now automatically tracks new images in Forgejo registry - Update strategy: newest-build (latest commit SHA) - Tag filter: commits (7-char SHA), latest, and v* releases - Helm values track repository + tag separately for automatic updates - Write-back via git (commits image updates to main branch) Build and push custom runner images: docker build -f k8s/infra/forgejo-runner/Dockerfile.golang \ -t forgejo.riotpiao.com/rock/forgejo-runner-golang:latest . docker build -f k8s/infra/forgejo-runner/Dockerfile.rust \ -t forgejo.riotpiao.com/rock/forgejo-runner-rust:latest . docker build -f k8s/infra/forgejo-runner/Dockerfile.node \ -t forgejo.riotpiao.com/rock/forgejo-runner-node:latest . docker login forgejo.riotpiao.com docker push forgejo.riotpiao.com/rock/forgejo-runner-golang:latest docker push forgejo.riotpiao.com/rock/forgejo-runner-rust:latest docker push forgejo.riotpiao.com/rock/forgejo-runner-node:latest Image Updater will then: 1. Detect new images in registry 2. Update values.yaml automatically 3. Commit changes to git 4. ArgoCD syncs the new image tags	2026-09-05 23:18:22 -07:00
rock	0e63d208d1	feat: add Node.js to all runner images (golang, rust, node) - Dockerfile.golang: docker:27-cli + Node.js + build tools - Dockerfile.rust: docker:27-cli + Node.js + Rust + build tools - Dockerfile.node: node:22-bookworm + docker.io (already has Node.js) - All runners now support GitHub Actions (checkout@v4 requires Node.js) - Images built/pushed manually (no CI for homelab) To build and push: docker build -f k8s/infra/forgejo-runner/Dockerfile.golang \ -t forgejo.riotpiao.com/rock/forgejo-runner-golang:latest . docker build -f k8s/infra/forgejo-runner/Dockerfile.rust \ -t forgejo.riotpiao.com/rock/forgejo-runner-rust:latest . docker build -f k8s/infra/forgejo-runner/Dockerfile.node \ -t forgejo.riotpiao.com/rock/forgejo-runner-node:latest . docker push forgejo.riotpiao.com/rock/forgejo-runner-golang:latest docker push forgejo.riotpiao.com/rock/forgejo-runner-rust:latest docker push forgejo.riotpiao.com/rock/forgejo-runner-node:latest	2026-09-05 23:13:59 -07:00
rock	cdd0ba2c99	feat: pre-install docker in node-runner image - Dockerfile.node extends node:22-bookworm with docker.io - No need for install step in every workflow - Values-node.yaml references custom image - Build and push manually (no CI needed)	2026-09-05 23:11:55 -07:00
rock	1777188f85	ci: fix golang runner - use docker:27-cli (has Node.js + golang + git) Previous image (golang:1.26-bookworm) lacks Node.js, causing GitHub Actions to fail with: 'exec: "node": executable file not found' Solution: - Change golang runner image from golang:1.26-bookworm to docker:27-cli - docker:27-cli includes: Node.js, Go toolchain, git, docker CLI, full dev tools - Verified tag exists: docker manifest inspect docker:27-cli ✓ This allows actions/checkout@v4 and other GitHub Actions to run properly on the golang runner pod. Note: node:22-bookworm runner already has Node.js, no change needed.	2026-09-05 22:50:43 -07:00
rock	c5d1572cc4	ci: fix rust runner - use docker:27-cli (has Node.js + git + docker) Previous image (rust:1.83-bookworm) lacks Node.js, causing GitHub Actions to fail with: 'exec: "node": executable file not found' Solution: - Change rust runner image from rust:1.83-bookworm to docker:27-cli - docker:27-cli includes: Node.js, git, docker CLI, full dev tools - Verified tag exists: docker manifest inspect docker:27-cli ✓ This allows actions/checkout@v4 and other GitHub Actions to run properly on the rust runner pod.	2026-09-05 22:49:25 -07:00
rock	9a227287b1	fix: add password grant type to kubernetes OIDC provider - Enables password grant for kubelogin (username/password auth) - Kubernetes provider now supports: authorization_code, refresh_token, password	2026-09-05 22:25:51 -07:00
rock	ef2228fdfb	fix: use explicit-consent flow for kubernetes public OIDC client - Kubernetes provider now uses default-provider-authorization-explicit-consent instead of implicit-consent flow - Better handling for public clients like kubelogin (authcode grant) - Prevents fallback to password grant prompts	2026-09-05 15:09:56 -07:00
rock	7ac37c7ab5	fix: remove spec wrapper from ClusterRoleBinding - ClusterRoleBinding doesn't use spec: wrapper (unlike Deployment/StatefulSet) - roleRef and subjects go at top level with metadata - Fixes: 'strict decoding error: unknown field "spec"'	2026-09-05 15:04:00 -07:00
rock	f740076694	fix: remove namespace from rbac Application destination - RBAC kustomization contains cluster-scoped (ClusterRoleBinding) and namespace-scoped (Role/RoleBinding) resources - Each resource has explicit metadata.namespace, so Application shouldn't force a default namespace - Fixes: ClusterRoleBinding gets namespace=default, causing sync failure with 'unsupported role reference kind: ""'	2026-09-05 14:53:59 -07:00
rock	c183978d6d	fix: remove backslash line continuations from YAML multiline string - YAML block scalars (\|) don't use backslash continuation - Just indent lines properly, block scalar handles them automatically - Fixes ArgoCD ComparisonError on poimen app	2026-09-05 14:48:21 -07:00
rock	19cf9277d9	fix: add admin-oidc-binding to kustomization resources - admin-oidc-binding.yaml wasn't listed in resources - Now kustomize will include it when building manifests - ArgoCD can sync the OIDC group binding	2026-09-05 14:42:17 -07:00
rock	74d587ca1b	chore: revert node-runner to stock image, remove Dockerfile - Reverted to node:22-bookworm (no custom image) - Removed Dockerfile.node (no CI to build it) - Docker install step in riotpiao workflow is already the workaround	2026-09-05 14:33:27 -07:00
rock	21156ccc1f	chore: remove homelab CI workflow - Removed .gitea/workflows/build-runner-node.yml - Homelab is GitOps only, not a buildable artifact - Runner images managed via direct Dockerfile edits + manual pushes	2026-09-05 14:33:13 -07:00
rock	6ec61d1c2c	feat: add OIDC group binding for cluster-admin access - Binds oidc:homelab-admins group to cluster-admin ClusterRole - Allows OIDC users (via Authentik) to have admin access - Groups claim from Authentik with oidc: prefix per kube-apiserver config - Enables kubectl access via 'kubectl login' + kubelogin	2026-09-05 14:30:41 -07:00
rock	acc11d5e87	feat: custom forgejo-runner-node image with docker.io pre-installed Build forgejo-runner-node image / build (push) Failing after 30s Details - Dockerfile.node extends node:22-bookworm + docker.io - CI workflow builds and pushes to forgejo registry on changes - values-node.yaml references custom image instead of stock node - Removes need to install docker in every workflow using node runner	2026-09-05 14:20:42 -07:00
rock	ba4261fb39	feat: Image Updater git write-back for multi-source poimen Application - write-back-method: git (commits image updates back to repos) - git-branch: main - Mounts ArgoCD SSH credentials for git pushes - Image Updater commits new SHAs → repos → ArgoCD syncs	2026-09-05 13:56:40 -07:00
rock	30ab2de36e	feat: multi-source poimen Application (memory, workflows, frontend) - Single Application syncs 3 independent repos - All deploy to poimen namespace - Image Updater tracks all 3 services (7-char SHA tags) - Auto-sync: prune + selfHeal enabled	2026-09-05 13:55:29 -07:00
rock	2954a9a0a4	fix: remove poimen-root Application (external repo dependency) - Removed dependency on external poimen.git repo - Poimen manifests should be managed locally or via separate workflow - Simplifies homelab GitOps to only manage homelab-owned services	2026-09-05 13:52:53 -07:00
rock	dc027cecb6	Revert "feat: enable Image Updater for poimen services" This reverts commit `8a7ee29e93`.	2026-09-05 13:52:50 -07:00
rock	8a7ee29e93	feat: enable Image Updater for poimen services - Track poimen-memory, poimen-workflows, poimen-frontend images - Auto-update on new 7-char SHA tags from Forgejo - Filter: regexp:^[0-9a-f]{7}$ (commit SHA) - write-back-method: argocd (updates Application)	2026-09-05 13:51:29 -07:00
rock	f5e100ee32	feat: add temporal:admin role to portfolio-agent - portfolio-agent can now call Temporal API in addition to LLM, memory, S3, SQS	2026-09-05 06:02:08 -07:00
rock	6743f7c25f	Phase 6.6: Add Poimen Memory DLQ queues (ArgoCD managed) ArgoCD Application: memory-queues ├─ Sync wave: 7 (messaging wave) ├─ Path: k8s/apps/messaging/memory-queues ├─ Namespace: sqs └─ Auto-sync: enabled (prune + selfHeal) Helm Chart: memory-queues ├─ Chart.yaml: v0.1.0 ├─ values.yaml: Queue config └─ templates/queues.yaml: Queue CRD resources Queues Created: 1. poimen-memory-dlq ├─ Purpose: Extraction + webhook + agent failures ├─ Partitions: 3 ├─ Replication factor: 1 ├─ Retention: 14 days (1,209,600 seconds) └─ Visibility timeout: 5 minutes (300 seconds) 2. poimen-memory-metric-dlq ├─ Purpose: Metrics persistence failures ├─ Partitions: 3 ├─ Replication factor: 1 ├─ Retention: 14 days └─ Visibility timeout: 5 minutes Resource: Queue CRD (kmsvc.io/v1alpha1) └─ Managed by: queue-operator (already running in sqs ns) Deployment Flow: ArgoCD (homelab) → sync wave 7 → deploy queues Memory app (poimen) → connects to kmsvc → sends DLQ messages Files: ├─ k8s/apps/messaging/memory-queues/Chart.yaml (new) ├─ k8s/apps/messaging/memory-queues/values.yaml (new) ├─ k8s/apps/messaging/memory-queues/templates/queues.yaml (new) └─ k8s/argocd/apps/50-memory-queues.yaml (new)	2026-09-05 01:10:51 -07:00
rock	266f0637a4	Revert "feat: add memory service queues (processing, indexing, dlq)" This reverts commit `af6fc84a18`.	2026-09-05 01:09:30 -07:00
rock	af6fc84a18	feat: add memory service queues (processing, indexing, dlq) - processing-queue: high-throughput, auto-scaling (1-4 shards) - indexing-queue: FIFO with deduplication (1-2 shards) - memory-dlq: dead letter queue for redelivery failures - ArgoCD Application (wave 7) to auto-sync queue lifecycle	2026-09-05 01:05:01 -07:00
rock	ed644b2c83	feat: add S3 and SQS permissions to service accounts and capability groups - New capability groups: s3-users, s3-writers, sqs-users, sqs-writers - portfolio-agent: add s3:read, sqs:read - memory-agent: add s3:read, s3:write, sqs:read, sqs:write - Enables portfolio and memory services to access MinIO S3 and message queues via JWT	2026-09-05 00:14:43 -07:00
rock	6db4d7dcb0	fix(grafana): give homelab-admins Admin org role, akadmin GrafanaAdmin GrafanaAdmin is server admin only — no org membership, so users couldn't see dashboards. Now: - akadmin: GrafanaAdmin (server admin, can impersonate) - homelab-admins: Admin (org admin, dashboard access) - others: Viewer	2026-09-04 23:41:22 -07:00
rock	ed794befdb	fix: grant GrafanaAdmin (server admin) to homelab-admins for Administration menu	2026-09-04 23:18:19 -07:00
rock	adb5c3597c	fix: add groups scope to grafana OIDC so role mapping works	2026-09-04 23:14:11 -07:00
rock	60bdd16a66	feat: add nginx-ingress ServiceMonitor for gateway traffic metrics	2026-09-04 23:07:44 -07:00
rock	823d5c6a3f	fix: map grafana admin role from homelab-admins group (grafana-admins deleted)	2026-09-04 23:04:01 -07:00
rock	176ec44b42	refactor: consolidate 13 dashboards into 2 (cluster-infrastructure + api-gateway)	2026-09-04 22:58:53 -07:00
rock	09fac8ada6	feat: add cluster and api-gateway alert rules with SLA targets	2026-09-04 22:37:48 -07:00
rock	75bb105e52	feat: add cluster-infrastructure and api-gateway grafana dashboards	2026-09-04 22:31:51 -07:00
rock	eafcb2397e	feat: add api-gateway blackbox probes for healthz and /v1/models	2026-09-04 22:13:00 -07:00
rock	539ef848d0	fix: use external Authentik URL for MinIO OIDC config discovery	2026-09-04 21:25:30 -07:00
rock	449c2a9109	gitops: add secret-rotation controller ArgoCD Application - Syncs k8s/apps/secret-rotation-controller/ kustomization - Auto-prune and self-heal enabled - Creates secret-rotation namespace - ArgoCD will deploy CRD, RBAC, ExternalSecret, Deployment	2026-09-04 13:51:26 -07:00
rock	667bca0f44	feat: automated secret rotation controller - ExternalSecret syncs age key from Vault to pod - CRD defines rotation schedule for each secret - Controller watches CRD, rotates on schedule: * Call provider API (Authentik/Forgejo/MinIO) for new secret * Update k8s Secret * Update .enc.yaml via sops (uses age key from Vault) * Git commit and push - Vault is source of truth for age key (never on disk) - Examples: minio-oidc (90d), portfolio-agent (90d), forgejo-token (90d), minio-root (180d)	2026-09-03 23:37:24 -07:00
rock	f9654986ad	fix(minio): use in-cluster URL for OIDC config fetch MinIO pod was getting 503 from public URL at startup. Use in-cluster authentik-server.iam.svc for metadata fetch; browser redirects still use public URLs from OIDC metadata response.	2026-09-03 23:15:19 -07:00