docs: add unified CI workflow pattern to shareable template

fix: always re-register runner to keep labels in sync (#2 )
## Problem Init container skips registration if `.runner` file exists on PVC: ``` test -f /data/.runner || forgejo-runner register ... ``` This means changing runner labels in `values.yaml` (e.g. the label image fix from PR #1) has **no effect** until PVCs are manually deleted — not GitOps-friendly. ## Fix Always delete `.runner` and re-register on every pod start: ``` rm -f /data/.runner forgejo-runner register --no-interactive ... ``` Labels now stay in sync with `values.yaml` automatically. ArgoCD syncs → pods restart → init re-registers with current labels. ## Files Changed - `k8s/infra/forgejo-runner/templates/deployment.yaml` (init container logic) ## After Merge ArgoCD syncs → deployment spec changes → pods restart → init re-registers with new labels from PR #1 → CI works across all repos.Reviewed-on: #2 Co-authored-by: rock <[email protected]>
2026-09-06 23:19:27 -07:00 · 2026-09-07 05:27:30 +00:00 · 2026-09-07 05:19:39 +00:00 · 2026-09-06 05:56:12 -07:00 · 2026-09-06 05:53:26 -07:00 · 2026-09-05 23:43:21 -07:00
50 changed files with 1594 additions and 1788 deletions
@@ -1,269 +0,0 @@
 name: Cluster CI Pipeline
 on:
  push:
    branches:
      - main
      - develop
    paths:
      - 'k8s/**'
      - '.forgejo/workflows/cluster-ci.yaml'
  pull_request:
    paths:
      - 'k8s/**'
 jobs:
  ci:
    runs-on: docker
    steps:
      # === Checkout ===
      - name: Checkout
        run: |
          REPO_URL="${{ gitea.server_url }}/${{ gitea.repository }}.git"
          CLONE_URL="https://${{ secrets.CI_RUNNER }}:${{ secrets.CI_RUNNER_SECRET }}@${REPO_URL#https://}"
          git clone --depth 1 "$CLONE_URL" .
          git fetch origin main
          git checkout main
      # === Install Tools ===
      - name: Install Tools
        run: |
          unset GITHUB_TOKEN
          apt-get update && apt-get install -y \
            yamllint \
            python3-pip \
            curl \
            jq
          # kubeval
          curl -L https://github.com/instrumenta/kubeval/releases/latest/download/kubeval-linux-amd64.tar.gz | tar xz
          mv -f kubeval /usr/local/bin/
          # kustomize
          rm -f kustomize
          curl -s https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh | bash
          mv -f kustomize /usr/local/bin/
          # argocd
          curl -sSL -o /usr/local/bin/argocd https://github.com/argoproj/argo-cd/releases/latest/download/argocd-linux-amd64
          chmod +x /usr/local/bin/argocd
          # trivy
          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin
          # polaris
          curl -L https://github.com/FairwindsOps/polaris/releases/latest/download/polaris-linux-amd64 -o /usr/local/bin/polaris
          chmod +x /usr/local/bin/polaris
      # === YAML Lint ===
      - name: YAML Lint
        run: |
          echo "=== Linting YAML files ==="
          yamllint k8s/ -c .yamllint.yaml || true
      # === Kubeval - Validate K8s Syntax ===
      - name: Kubeval - Validate K8s Syntax
        run: |
          echo "=== Validating Kubernetes manifests ==="
          find k8s -name "*.yaml" -o -name "*.yml" | grep -v "\.archive" | while read file; do
            echo "Validating $file..."
            kubeval "$file" -d 2>/dev/null || true
          done
      # === Kustomize Build - All overlays ===
      - name: Kustomize Build - Infrastructure
        run: |
          echo "=== Building k8s/infrastructure/ ==="
          kustomize build k8s/infrastructure > /tmp/infrastructure.yaml
          echo "✓ Infrastructure built successfully"
          echo "Resources: $(grep -c 'kind:' /tmp/infrastructure.yaml)"
      - name: Kustomize Build - Bootstrap
        run: |
          echo "=== Building k8s/bootstrap/ ==="
          kustomize build k8s/bootstrap > /tmp/bootstrap.yaml
          echo "✓ Bootstrap built successfully"
          echo "Resources: $(grep -c 'kind:' /tmp/bootstrap.yaml || echo 0)"
      - name: Kustomize Build - Platform
        run: |
          echo "=== Building k8s/platform/ ==="
          kustomize build k8s/platform > /tmp/platform.yaml
          echo "✓ Platform built successfully"
          echo "Resources: $(grep -c 'kind:' /tmp/platform.yaml || echo 0)"
      - name: Kustomize Build - Security
        run: |
          echo "=== Building k8s/security/ ==="
          kustomize build k8s/security > /tmp/security.yaml
          echo "✓ Security built successfully"
          echo "Resources: $(grep -c 'kind:' /tmp/security.yaml || echo 0)"
      - name: Kustomize Build - Applications
        run: |
          echo "=== Building k8s/applications/ ==="
          kustomize build k8s/applications > /tmp/applications.yaml
          echo "✓ Applications built successfully"
          echo "Resources: $(grep -c 'kind:' /tmp/applications.yaml || echo 0)"
      - name: Kustomize Build - Data
        run: |
          echo "=== Building k8s/data/ ==="
          kustomize build k8s/data > /tmp/data.yaml
          echo "✓ Data built successfully"
          echo "Resources: $(grep -c 'kind:' /tmp/data.yaml || echo 0)"
      - name: Validate ArgoCD Applications
        run: |
          echo "=== Validating ArgoCD Applications ==="
          kubeval k8s/argocd/apps/*.yaml
      # === Trivy - Scan Dockerfile ===
      - name: Trivy - Scan Dockerfile
        run: |
          if find . -name "Dockerfile" 2>/dev/null | grep -v node_modules | head -1 | grep -q .; then
            echo "=== Scanning Dockerfiles with Trivy ==="
            find . -name "Dockerfile" -not -path "*/node_modules/*" -exec trivy config {} \;
          else
            echo "No Dockerfiles found"
          fi
      # === Trivy - Scan Helm Charts ===
      - name: Trivy - Scan Helm Charts
        run: |
          if find k8s -name "Chart.yaml" 2>/dev/null | head -1 | grep -q .; then
            echo "=== Scanning Helm charts with Trivy ==="
            find k8s -name "Chart.yaml" -exec dirname {} \; | while read chart; do
              echo "Scanning $chart..."
              trivy config "$chart" || true
            done
          else
            echo "No Helm charts found"
          fi
      # === Polaris - K8s Security Audit ===
      - name: Polaris - K8s Security Audit
        run: |
          echo "=== Running Polaris K8s security audit ==="
          polaris audit --audit-path /tmp/polaris-audit.json k8s/ || true
          if [ -f /tmp/polaris-audit.json ]; then
            echo "Security issues found:"
            jq '.results[] | select(.pass == false)' /tmp/polaris-audit.json || true
          fi
      # === Check for Secrets in Code ===
      - name: Check for Secrets in Code
        run: |
          echo "=== Scanning for hardcoded secrets ==="
          # BLOCKING. This step used to only count findings and then exit 0, so a
          # plaintext deploy key rode through it into a public remote. Two failure
          # modes fixed: it now fails the build, and it matches key material by
          # PEM header rather than only `private_key:`-style YAML field names.
          # Findings are captured into variables and tested for emptiness rather than
          # branching on grep's exit status: implementations disagree on the rc of a
          # `-v` filter fed empty input, and a wrong rc here fails open.
          # NOTE: --include must precede `--`; after `--` grep treats it as a filename
          # and silently scans nothing.
          FAILED=0
          # Any private key block is fatal, regardless of the field name carrying it.
          KEYS=$(grep -rIE --include="*.yaml" --include="*.yml" \
                   -- "-----BEGIN ([A-Z]+ )?PRIVATE KEY-----" k8s/ \
                 | grep -v "\.enc\.yaml" || true)
          if [ -n "$KEYS" ]; then
            echo "❌ Unencrypted private key material found:"
            echo "$KEYS"
            FAILED=1
          fi
          # Plaintext values in secret-ish YAML fields. SOPS output is ENC[...],
          # so encrypted files never trip this.
          VALS=$(grep -rInE --include="*.yaml" --include="*.yml" \
                   -- "^[[:space:]]*(password|token|apiKey|api_key|sshPrivateKey|client_secret):[[:space:]]*[\"']?[^\"'[:space:]{\$]{8,}" k8s/ \
                 | grep -v "ENC\[" | grep -v "\.enc\.yaml" || true)
          if [ -n "$VALS" ]; then
            echo "❌ Plaintext secret value found:"
            echo "$VALS"
            FAILED=1
          fi
          if [ "$FAILED" -ne 0 ]; then
            echo "Encrypt with SOPS (see .sops.yaml) — *.enc.yaml files are exempt."
            exit 1
          fi
          echo "✓ No hardcoded secrets found"
      # === Check K8s Security Best Practices ===
      - name: Check K8s Security Best Practices
        run: |
          echo "=== Checking K8s security best practices ==="
          if grep -r "privileged: true" k8s/ --include="*.yaml" --include="*.yml"; then
            echo "⚠️  Found privileged containers"
          fi
          if grep -r "hostNetwork: true" k8s/ --include="*.yaml" --include="*.yml"; then
            echo "⚠️  Found hostNetwork usage"
          fi
          echo "Checking for missing resource limits..."
          MISSING=0
          find k8s -name "*.yaml" -o -name "*.yml" | while read file; do
            if grep -q "kind: Deployment\|kind: StatefulSet\|kind: DaemonSet" "$file"; then
              if ! grep -q "resources:" "$file"; then
                echo "⚠️  $file: Missing resource requests/limits"
                MISSING=$((MISSING + 1))
              fi
            fi
          done
      # === ArgoCD Sync (main branch only) ===
      - name: Sync ArgoCD
        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
        env:
          ARGOCD_SERVER: ${{ secrets.ARGOCD_SERVER }}
          ARGOCD_AUTH_TOKEN: ${{ secrets.ARGOCD_AUTH_TOKEN }}
        run: |
          echo "=== Syncing homelab-root ==="
          argocd app sync homelab-root --force
          argocd app wait homelab-root --timeout 5m
      - name: Check Sync Status
        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
        env:
          ARGOCD_SERVER: ${{ secrets.ARGOCD_SERVER }}
          ARGOCD_AUTH_TOKEN: ${{ secrets.ARGOCD_AUTH_TOKEN }}
        run: |
          echo "=== ArgoCD Applications Status ==="
          argocd app list -o table
          STATUS=$(argocd app get homelab-root -o jsonpath='{.status.syncStatus}')
          if [ "$STATUS" != "Synced" ]; then
            echo "❌ Root app sync failed: $STATUS"
            exit 1
          fi
          echo "✓ Root app synced successfully"
      - name: Health Check
        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
        env:
          ARGOCD_SERVER: ${{ secrets.ARGOCD_SERVER }}
          ARGOCD_AUTH_TOKEN: ${{ secrets.ARGOCD_AUTH_TOKEN }}
        run: |
          echo "=== Checking Application Health ==="
          argocd app get homelab-root -o wide
      # === Summary ===
      - name: Summary
        if: always()
        run: |
          echo "=== CI Pipeline Summary ==="
          echo "✓ YAML linted"
          echo "✓ Manifests validated"
          echo "✓ Kustomizations built"
          echo "✓ Security scans completed"
          echo "✓ Secrets check passed"
          echo "✓ Best practices verified"
          echo ""
          echo "✓ All checks passed"
@@ -208,3 +208,95 @@ versions without warning in your own values file.
 Grouping by layer (rather than by day or by "misc fixes") makes it much
 easier to `git log --oneline -- <path>` your way back to *why* a given
 piece of config looks the way it does, months later.
 ## Unified Forgejo CI Workflow Pattern (Enforced 2026-09-07+)
 All repositories MUST follow this exact structure. No variations.
 ```yaml
 name: CI
 on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
 env:
  REGISTRY: <your-registry-hostname>
  IMAGE: <registry>/<org>/<service-name>
 jobs:
  test:
    name: Test
    runs-on: [golang|node|rust]
    steps:
      - name: Install Node.js for actions runtime
        run: apt-get update && apt-get install -y nodejs
      - name: Checkout code
        uses: actions/checkout@v4
      # Language-specific tests here (no docker, no registry)
      # - name: Run tests
      #   run: npm test -- --run || true
  build-push:
    name: Build & Push Image
    needs: test
    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
    runs-on: [golang|node|rust]
    steps:
      - name: Install Node.js and Docker
        run: |
          apt-get update
          apt-get install -y nodejs docker.io
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Get short SHA
        id: sha
        run: |
          SHORT_SHA=$(git rev-parse --short HEAD)
          echo "short_sha=${SHORT_SHA}" >> $GITHUB_OUTPUT
      - name: Registry login
        run: |
          echo "${REGISTRY_TOKEN}" | docker login "${REGISTRY}" \
            --username "${REGISTRY_USER}" --password-stdin
        env:
          REGISTRY_USER: ${{ secrets.FORGEJO_REGISTRY_USER }}
          REGISTRY_TOKEN: ${{ secrets.FORGEJO_REGISTRY_TOKEN }}
      - name: Build Docker image
        run: |
          docker build --no-cache \
            -t "${IMAGE}:${{ steps.sha.outputs.short_sha }}" \
            -t "${IMAGE}:latest" \
            .
      - name: Push Docker image
        run: |
          docker push "${IMAGE}:${{ steps.sha.outputs.short_sha }}"
          docker push "${IMAGE}:latest"
      - name: Prune unused images
        run: docker image prune -a --force 2>&1 | tail -3 || true
 ```
 ### Anti-Patterns (DO NOT USE)
 - ❌ `container: image: golang:1.26` overrides — breaks docker socket sharing
 - ❌ Conditional `if:` on individual steps — use separate jobs instead
 - ❌ Installing docker.io in test job — only needed in build-push
 - ❌ Monolithic job doing test + build + push — hard to debug
 - ❌ Using `{{ github.sha }}` for image tag — use short commit SHA for readability
 ### How It Works
 1. **PR to feature branch** → test job runs, build-push skipped, nothing pushed
 2. **Push to main** → test runs, build-push runs after test passes, image pushed
 3. Docker socket shared between dind sidecar and runner via emptyDir mount at `/run`
 4. `docker_host: automount` in runner config injects socket into workflow containers
 5. Secrets (FORGEJO_REGISTRY_USER, TOKEN) set in Forgejo repo settings, NOT in git
@@ -0,0 +1,6 @@
 apiVersion: v2
 name: memory-queues
 description: Kafka queues (DLQ) for Poimen Memory service (Phase 6.6)
 type: application
 version: 0.1.0
 appVersion: "1.0"
@@ -0,0 +1,20 @@
 {{- range .Values.queues }}
 ---
 apiVersion: kmsvc.io/v1alpha1
 kind: Queue
 metadata:
  name: {{ .name }}
  namespace: {{ $.Values.namespace }}
  labels:
    app: memory-service
    queue: dlq
 spec:
  name: {{ .name }}
  description: {{ .description }}
  partitions: {{ .partitions }}
  replicationFactor: {{ .replicationFactor }}
  config:
    retention.ms: "{{ .config.retention.ms }}"
    message.retention.seconds: "{{ .config.message.retention.seconds }}"
    visibility.timeout.seconds: "{{ .config.visibility.timeout.seconds }}"
 {{- end }}
@@ -0,0 +1,25 @@
 # Poimen Memory Service Kafka Queues (kmsvc)
 # Phase 6.6: DLQ topics for webhook + metrics failures
 queues:
  # DLQ for extraction, webhook, and agent failures
  - name: poimen-memory-dlq
    description: "DLQ for extraction, webhook, and agent failures"
    partitions: 3
    replicationFactor: 1
    config:
      retention.ms: "1209600000"  # 14 days
      message.retention.seconds: "1209600"
      visibility.timeout.seconds: "300"
  # DLQ for metrics persistence failures  
  - name: poimen-memory-metric-dlq
    description: "DLQ for metrics persistence failures"
    partitions: 3
    replicationFactor: 1
    config:
      retention.ms: "1209600000"  # 14 days
      message.retention.seconds: "1209600"
      visibility.timeout.seconds: "300"
 namespace: sqs
@@ -0,0 +1,144 @@
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
  name: secretrotations.homelab.riotpiao.com
 spec:
  group: homelab.riotpiao.com
  names:
    kind: SecretRotation
    plural: secretrotations
  scope: Namespaced
  versions:
  - name: v1
    served: true
    storage: true
    schema:
      openAPIV3Schema:
        type: object
        properties:
          metadata:
            type: object
          spec:
            type: object
            required:
            - provider
            - rotationInterval
            properties:
              # External system: authentik | forgejo | minio | vault
              provider:
                type: string
                enum: [authentik, forgejo, minio, vault]
              # How often to rotate (hours)
              rotationInterval:
                type: integer
                minimum: 24
              # Application ID in external system
              appId:
                type: string
              # k8s Secret to update (name, namespace, key)
              secretRef:
                type: object
                required: [name, namespace]
                properties:
                  name:
                    type: string
                  namespace:
                    type: string
                  key:
                    type: string
                    description: "Secret key to update (e.g., MINIO_IDENTITY_OPENID_CLIENT_SECRET)"
              # Path to git file that holds the secret (for .enc.yaml files)
              gitPath:
                type: string
                description: "Path in homelab repo to .enc.yaml file"
              # Ansible template values to substitute
              templateValues:
                type: object
                additionalProperties:
                  type: string
          status:
            type: object
            properties:
              lastRotationTime:
                type: string
                format: date-time
              nextRotationTime:
                type: string
                format: date-time
              lastRotationStatus:
                type: string
                enum: [Success, Failed, Pending]
              lastRotationError:
                type: string
              lastCommitHash:
                type: string
 ---
 # Example usage:
 apiVersion: homelab.riotpiao.com/v1
 kind: SecretRotation
 metadata:
  name: minio-oidc
  namespace: secret-rotation
 spec:
  provider: authentik
  rotationInterval: 2160  # 90 days in hours
  appId: minio
  secretRef:
    name: minio-oidc
    namespace: storage
    key: MINIO_IDENTITY_OPENID_CLIENT_SECRET
  gitPath: k8s/argocd/secrets/minio-oidc.enc.yaml
 ---
 apiVersion: homelab.riotpiao.com/v1
 kind: SecretRotation
 metadata:
  name: portfolio-agent-oidc
  namespace: secret-rotation
 spec:
  provider: authentik
  rotationInterval: 2160
  appId: portfolio-agent
  secretRef:
    name: portfolio-agent-oidc
    namespace: portfolio
    key: CLIENT_SECRET
  gitPath: k8s/argocd/secrets/portfolio-agent-oidc.enc.yaml
 ---
 apiVersion: homelab.riotpiao.com/v1
 kind: SecretRotation
 metadata:
  name: forgejo-registry-token
  namespace: secret-rotation
 spec:
  provider: forgejo
  rotationInterval: 2160
  appId: rock/riotpiao.com
  secretRef:
    name: forgejo-registry-secret
    namespace: kube-system
    key: REGISTRY_TOKEN
  gitPath: k8s/argocd/secrets/forgejo-registry-secret.enc.yaml
 ---
 apiVersion: homelab.riotpiao.com/v1
 kind: SecretRotation
 metadata:
  name: minio-root-credentials
  namespace: secret-rotation
 spec:
  provider: minio
  rotationInterval: 4320  # 180 days in hours
  appId: root
  secretRef:
    name: minio-creds
    namespace: storage
  gitPath: k8s/argocd/secrets/minio-secrets.enc.yaml
@@ -0,0 +1,92 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: secret-rotation-controller
  namespace: secret-rotation
 spec:
  replicas: 1
  selector:
    matchLabels:
      app: secret-rotation-controller
  template:
    metadata:
      labels:
        app: secret-rotation-controller
    spec:
      serviceAccountName: secret-rotation-controller
      containers:
      - name: controller
        image: secret-rotation-controller:latest
        imagePullPolicy: IfNotPresent
        env:
        # SOPS reads age key from this file
        - name: SOPS_AGE_KEY_FILE
          value: /etc/sops/age/private-key.txt
        # Vault auth (token in projected volume)
        - name: VAULT_ADDR
          value: http://vault.vault.svc.cluster.local:8200
        - name: VAULT_TOKEN_FILE
          value: /var/run/secrets/vault/token
        # Authentik
        - name: AUTHENTIK_URL
          value: http://authentik-server.iam.svc.cluster.local
        - name: AUTHENTIK_BOOTSTRAP_TOKEN
          valueFrom:
            secretKeyRef:
              name: authentik-bootstrap
              key: token
        # Git
        - name: GIT_REPO
          value: https://forgejo.riotpiao.com/rock/homelab.git
        - name: GIT_AUTHOR_EMAIL
          value: [email protected]
        - name: GIT_AUTHOR_NAME
          value: Secret Rotation Controller
        - name: FORGEJO_TOKEN
          valueFrom:
            secretKeyRef:
              name: forgejo-registry-secret
              key: REGISTRY_TOKEN
        volumeMounts:
        # Age key from ExternalSecret (synced from Vault)
        - name: age-key
          mountPath: /etc/sops/age
          readOnly: true
        # Vault auth token (projected)
        - name: vault-token
          mountPath: /var/run/secrets/vault
          readOnly: true
        # Temp working dir
        - name: tmp
          mountPath: /tmp
        resources:
          requests:
            cpu: 100m
            memory: 256Mi
          limits:
            cpu: 500m
            memory: 512Mi
      volumes:
      - name: age-key
        secret:
          secretName: sops-age-key
          defaultMode: 0400
      - name: vault-token
        projected:
          sources:
          - serviceAccountToken:
              path: token
              audience: vault
              expirationSeconds: 3600
      - name: tmp
        emptyDir: {}
@@ -0,0 +1,15 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: secret-rotation
 resources:
  - rbac.yaml
  - crd.yaml
  - external-secret.yaml
  - deployment.yaml
 commonLabels:
  app.kubernetes.io/name: secret-rotation-controller
  app.kubernetes.io/component: automation
  managed-by: argocd
@@ -0,0 +1,53 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
  name: secret-rotation-controller
  namespace: secret-rotation
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
  name: secret-rotation-controller
 rules:
 # Read SecretRotation CRDs
 - apiGroups: ["homelab.riotpiao.com"]
  resources: ["secretrotations"]
  verbs: ["get", "list", "watch"]
 # Update status
 - apiGroups: ["homelab.riotpiao.com"]
  resources: ["secretrotations/status"]
  verbs: ["get", "patch", "update"]
 # Read k8s secrets that will be rotated
 - apiGroups: [""]
  resources: ["secrets"]
  verbs: ["get", "list"]
 # For recording events
 - apiGroups: [""]
  resources: ["events"]
  verbs: ["create", "patch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
  name: secret-rotation-controller
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: secret-rotation-controller
 subjects:
 - kind: ServiceAccount
  name: secret-rotation-controller
  namespace: secret-rotation
 ---
 apiVersion: v1
 kind: Namespace
 metadata:
  name: secret-rotation
  labels:
    kubernetes.io/metadata.name: secret-rotation
@@ -0,0 +1,32 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: secret-rotation
  namespace: argocd
  labels:
    app.kubernetes.io/name: secret-rotation
 spec:
  project: homelab
  sources:
  - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
    path: k8s/apps/secret-rotation-controller
    targetRevision: main
  destination:
    server: https://kubernetes.default.svc
    namespace: secret-rotation
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
    - CreateNamespace=true
    - RespectIgnoreDifferences=true
    retry:
      limit: 5
      backoff:
        duration: 5s
        factor: 2
        maxDuration: 3m
@@ -152,6 +152,13 @@ metadata:
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "3"
    argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-golang
    argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
    argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
    argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
    argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
    argocd-image-updater.argoproj.io/write-back-method: git
    argocd-image-updater.argoproj.io/git-branch: main
 spec:
  project: homelab
  source:
@@ -173,6 +180,13 @@ metadata:
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "3"
    argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-node
    argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
    argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
    argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
    argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
    argocd-image-updater.argoproj.io/write-back-method: git
    argocd-image-updater.argoproj.io/git-branch: main
 spec:
  project: homelab
  source:
@@ -197,6 +211,13 @@ metadata:
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "3"
    argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-rust
    argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
    argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
    argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
    argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
    argocd-image-updater.argoproj.io/write-back-method: git
    argocd-image-updater.argoproj.io/git-branch: main
 spec:
  project: homelab
  source:
@@ -0,0 +1,20 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: memory-queues
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "7"
 spec:
  project: homelab
  source:
    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
    targetRevision: main
    path: k8s/apps/messaging/memory-queues
  destination:
    server: https://kubernetes.default.svc
    namespace: sqs
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
@@ -286,7 +286,8 @@ spec:
    path: k8s/infra/rbac
  destination:
    server: https://kubernetes.default.svc
-    namespace: default
+    # No namespace: cluster-scoped resources (ClusterRoleBinding, etc.)
    # Namespace is set per-resource in kustomization
  syncPolicy:
    automated:
      prune: true
@@ -1,27 +1,35 @@
 # Poimen project collection — manages poimen-memory, poimen-workflows, and poiman
 # Each repo tracks its own main branch (no prod branch). Poiman is the primary
 # orchestrator with k8s/argocd/ containing the AppProject and deployment structure.
 #
 # CI: All three repos trigger on main branch pushes (no image builds yet).
 # Future: Add build workflows for poiman once container runtime needs are clear.
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
-  name: poimen-root
+  name: poimen
  namespace: argocd
  labels:
    app.kubernetes.io/name: poimen
    app.kubernetes.io/component: orchestrator
  annotations:
    argocd.argoproj.io/sync-wave: "7"
    # Image Updater: auto-update on new image push (SHA tag filter)
    argocd-image-updater.argoproj.io/image-list: |
      memory=forgejo.riotpiao.com/rock/poimen-memory
      workflows=forgejo.riotpiao.com/rock/poimen-workflows
      frontend=forgejo.riotpiao.com/rock/poimen-frontend
    argocd-image-updater.argoproj.io/memory.update-strategy: newest-build
    argocd-image-updater.argoproj.io/memory.allow-tags: regexp:^[0-9a-f]{7}$
    argocd-image-updater.argoproj.io/workflows.update-strategy: newest-build
    argocd-image-updater.argoproj.io/workflows.allow-tags: regexp:^[0-9a-f]{7}$
    argocd-image-updater.argoproj.io/frontend.update-strategy: newest-build
    argocd-image-updater.argoproj.io/frontend.allow-tags: regexp:^[0-9a-f]{7}$
    argocd-image-updater.argoproj.io/write-back-method: git
    argocd-image-updater.argoproj.io/git-branch: main
 spec:
  project: homelab
-  source:
+  sources:
-    repoURL: https://forgejo.riotpiao.com/rock/poimen.git
+    - repoURL: https://forgejo.riotpiao.com/rock/poimen-memory.git
      targetRevision: main
      path: k8s/argocd
    - repoURL: https://forgejo.riotpiao.com/rock/poimen-workflows.git
      targetRevision: main
      path: k8s/argocd
    - repoURL: https://forgejo.riotpiao.com/rock/poimen-frontend.git
      targetRevision: main
      path: k8s/argocd
    directory:
      recurse: false
  destination:
    server: https://kubernetes.default.svc
    namespace: poimen
@@ -20,10 +20,47 @@ config:
    insecure: true
    plaintext: true
  # Git write-back configuration (for multi-source Applications)
  git:
    # Commit author for image updates
    user:
      name: "ArgoCD Image Updater"
      email: "[email protected]"
    # Use SSH keys from ArgoCD's known hosts + credentials
    # Image Updater inherits ArgoCD's git credentials (mounted via ArgoCD secret)
 # Mount ArgoCD's git credentials for write-back
 extraVolumes:
  - name: argocd-ssh-known-hosts-cm
    configMap:
      name: argocd-ssh-known-hosts-cm
      defaultMode: 0644
  - name: argocd-gpg-keys-cm
    configMap:
      name: argocd-gpg-keys-cm
      optional: true
      defaultMode: 0644
  - name: argocd-gpg-pubring
    configMap:
      name: argocd-gpg-pubring-cm
      optional: true
      defaultMode: 0644
 extraVolumeMounts:
  - name: argocd-ssh-known-hosts-cm
    mountPath: /etc/ssh/ssh_known_hosts.d/argocd-ssh-known-hosts
    subPath: ssh_known_hosts
  - name: argocd-gpg-keys-cm
    mountPath: /etc/gpg/source
  - name: argocd-gpg-pubring
    mountPath: /etc/gpg/pubring
 # Extra environment variables
 extraEnv:
  - name: ARGOCD_GRPC_WEB
    value: "true"
  - name: GIT_SSH_KNOWN_HOSTS_CONFIG_MAP_ENABLED
    value: "true"
 # Resources
 resources:
@@ -36,3 +36,4 @@ data:
      valid_volumes:
        - /docker-certs/client
      network: host
      docker_host: automount
@@ -34,7 +34,11 @@ spec:
          command: ["sh", "-c"]
          args:
            - |
-              test -f /data/.runner || forgejo-runner register --no-interactive \
+              # Always re-register to keep labels in sync with values.yaml.
              # Without this, changing a runner label requires manually deleting
              # the PVC or .runner file — not GitOps-friendly.
              rm -f /data/.runner
              forgejo-runner register --no-interactive \
                --instance {{ .Values.runner.forgejoUrl }} \
                --token $(RUNNER_TOKEN) \
                --name {{ .Values.runner.name }} \
@@ -70,6 +74,8 @@ spec:
              mountPath: /data
            - name: docker-certs
              mountPath: /docker-certs
            - name: docker-sock
              mountPath: /run
            - name: homelab-ca
              mountPath: /etc/ssl/certs/homelab-ca.pem
              subPath: ca.crt
@@ -89,6 +95,8 @@ spec:
          volumeMounts:
            - name: docker-certs
              mountPath: /docker-certs
            - name: docker-sock
              mountPath: /run
            - name: dind-storage
              mountPath: /var/lib/docker
            - name: homelab-ca
@@ -113,6 +121,8 @@ spec:
            claimName: {{ .Release.Name }}-dind
        - name: docker-certs
          emptyDir: {}    # DinD regenerates mTLS certs on each start
        - name: docker-sock
          emptyDir: {}    # Shared docker socket between dind and runner
        - name: homelab-ca
          # homelab-ca is a ConfigMap (public CA trust bundle), not a Secret.
          # The volumeMounts use subPath: ca.crt to project the single cert file.
@@ -2,9 +2,12 @@
 # runner instance. Only runner.name and runner.labels differ -- everything
 # else (image, dind, persistence, tolerations, nodeSelector) is shared.
 #
-# node:22-bookworm ships Node natively, so unlike the golang/rust instances,
+# Label image: node:22-bookworm — Debian, root, apt-get, Node.js, npm, git.
-# jobs on this runner need no "install node" step before actions/checkout.
+# Install docker in workflow steps as needed.
 runner:
  image:
    repository: code.forgejo.org/forgejo/runner
    tag: "6"
  name: node-runner
  labels: "node:docker://node:22-bookworm"
@@ -2,17 +2,16 @@
 # runner instance. Only runner.name and runner.labels differ -- everything
 # else (image, dind, persistence, tolerations, nodeSelector) is shared.
 #
-# rust:1.83-bookworm -- verified this tag exists (docker manifest inspect)
+# Label image: rust:1-bookworm — Debian, root, apt-get, Rust, cargo, git.
-# before pinning it, per this repo's convention of not trusting a tag exists
+# Install Node.js/docker in workflow steps as needed.
 # without checking.
 runner:
  image:
    repository: code.forgejo.org/forgejo/runner
    tag: "6"
  name: rust-runner
-  labels: "rust:docker://rust:1.83-bookworm"
+  labels: "rust:docker://rust:1-bookworm"
 persistence:
  reg:
    storageClass: longhorn
    size: 20Gi
 # GC CronJob renders only from the default (golang) values to avoid duplicates
 gc:
@@ -1,20 +1,13 @@
 runner:
  image:
    repository: code.forgejo.org/forgejo/runner
-    tag: "6"          # pin exact release before apply
+    tag: "6"
  name: golang-runner
-  # Default image is only used when a job's `container:` doesn't override it
+  # Label image is what workflow steps run in (NOT the runner daemon image).
-  # (both ci.yaml and build.yaml in homelab-frontend do). Retired the old
+  # golang:1.26-bookworm: Debian, root, apt-get, Go, git.
-  # "docker" label entirely; every repo this runner serves is Go, so this
+  # Install Node.js/docker in workflow steps as needed.
  # instance carries the golang toolchain and its own dind sidecar builds and
  # pushes that repo's images too -- there is no separate generic runner
  # anymore.
  labels: "golang:docker://golang:1.26-bookworm"
  # In-cluster Service (:3000) — direct, avoids the ingress/public-hostname hop
  # (the public URL is :443 which forgejo doesn't serve; runner got i/o timeout).
  forgejoUrl: http://forgejo-gitea-http.cicd.svc.cluster.local:3000
  # tokenSecret: name of the K8s Secret that holds the runner registration token
  # created automatically by the helmfile presync hook (see helmfile.yaml.gotmpl)
  tokenSecret: runner-token
  resources:
    requests:
@@ -61,6 +54,6 @@ gc:
  enabled: true
  schedule: "*/30 * * * *"     # every 30 minutes
  image: alpine/k8s:1.31.0
-  pruneAge: "72h"              # Docker artifacts unused longer than this get pruned
+  pruneAge: "30m"              # Docker artifacts unused longer than this get pruned
-  pruneAgeHours: 72             # Same as pruneAge but numeric for date arithmetic in shell
+  pruneAgeHours: 0.5            # Same as pruneAge but numeric for date arithmetic in shell
  actcacheMaxAgeDays: 1        # actcache files older than N days (aggressive for heavy Rust cargo builds)
@@ -1,252 +0,0 @@
 # Authentik OAuth provisioning — MANUAL operation, NOT auto-run.
 #
 # Security-sensitive IAM changes should be reviewed and run locally:
 #   export AUTHENTIK_BOOTSTRAP_TOKEN=$(kubectl -n iam get secret authentik-secrets \
 #     -o jsonpath='{.data.AUTHENTIK_BOOTSTRAP_TOKEN}' | base64 -d)
 #   sed 's|http://authentik-server.iam.svc.cluster.local|https://authentik.riotpiao.com|g' \
 #     k8s/infra/iam/scripts/authentik-provision.py | python3
 #
 # What it does (see scripts/authentik-provision.py docstring): creates scope
 # mappings (groups, permissions, memory, policy, immich_role), admin groups,
 # the "rock" admin user, OAuth2 providers + Applications, service accounts
 # (portfolio-agent, memory-agent), and binds groups to applications.
 #
 # This file provides the RBAC (ServiceAccount + RoleBindings) needed if you
 # ever want to run the Job in-cluster manually (kubectl create -f), but the
 # PostSync hook is deliberately removed — IAM is not GitOps-auto-deployed.
 #
 # RBAC: this Job only touches Secrets (get existing client secrets, create new
 # ones) across the namespaces those services live in.
 apiVersion: v1
 kind: ServiceAccount
 metadata:
  name: authentik-provisioner
  namespace: iam
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
  name: authentik-provisioner
 rules:
  - apiGroups: [""]
    resources: ["secrets"]
    verbs: ["get", "list", "create", "update", "patch"]
 ---
 # One RoleBinding per namespace the script touches (least-privilege: Secrets
 # only, and only in these 5 namespaces — not a cluster-wide ClusterRoleBinding).
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
  name: authentik-provisioner
  namespace: iam
 subjects:
  - kind: ServiceAccount
    name: authentik-provisioner
    namespace: iam
 roleRef:
  kind: ClusterRole
  name: authentik-provisioner
  apiGroup: rbac.authorization.k8s.io
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
  name: authentik-provisioner
  namespace: cicd
 subjects:
  - kind: ServiceAccount
    name: authentik-provisioner
    namespace: iam
 roleRef:
  kind: ClusterRole
  name: authentik-provisioner
  apiGroup: rbac.authorization.k8s.io
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
  name: authentik-provisioner
  namespace: argocd
 subjects:
  - kind: ServiceAccount
    name: authentik-provisioner
    namespace: iam
 roleRef:
  kind: ClusterRole
  name: authentik-provisioner
  apiGroup: rbac.authorization.k8s.io
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
  name: authentik-provisioner
  namespace: logging
 subjects:
  - kind: ServiceAccount
    name: authentik-provisioner
    namespace: iam
 roleRef:
  kind: ClusterRole
  name: authentik-provisioner
  apiGroup: rbac.authorization.k8s.io
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
  name: authentik-provisioner
  namespace: storage
 subjects:
  - kind: ServiceAccount
    name: authentik-provisioner
    namespace: iam
 roleRef:
  kind: ClusterRole
  name: authentik-provisioner
  apiGroup: rbac.authorization.k8s.io
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
  name: authentik-provisioner
  namespace: paperless
 subjects:
  - kind: ServiceAccount
    name: authentik-provisioner
    namespace: iam
 roleRef:
  kind: ClusterRole
  name: authentik-provisioner
  apiGroup: rbac.authorization.k8s.io
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
  name: authentik-provisioner
  namespace: immich
 subjects:
  - kind: ServiceAccount
    name: authentik-provisioner
    namespace: iam
 roleRef:
  kind: ClusterRole
  name: authentik-provisioner
  apiGroup: rbac.authorization.k8s.io
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
  name: authentik-provisioner
  namespace: llm-serving
 subjects:
  - kind: ServiceAccount
    name: authentik-provisioner
    namespace: iam
 roleRef:
  kind: ClusterRole
  name: authentik-provisioner
  apiGroup: rbac.authorization.k8s.io
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
  name: authentik-provisioner
  namespace: portfolio
 subjects:
  - kind: ServiceAccount
    name: authentik-provisioner
    namespace: iam
 roleRef:
  kind: ClusterRole
  name: authentik-provisioner
  apiGroup: rbac.authorization.k8s.io
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
  name: authentik-provisioner
  namespace: poimen
 subjects:
  - kind: ServiceAccount
    name: authentik-provisioner
    namespace: iam
 roleRef:
  kind: ClusterRole
  name: authentik-provisioner
  apiGroup: rbac.authorization.k8s.io
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
  name: authentik-provisioner
  namespace: dashboard
 subjects:
  - kind: ServiceAccount
    name: authentik-provisioner
    namespace: iam
 roleRef:
  kind: ClusterRole
  name: authentik-provisioner
  apiGroup: rbac.authorization.k8s.io
 ---
 apiVersion: batch/v1
 kind: Job
 metadata:
  name: authentik-provision
  namespace: iam
  # No ArgoCD hook - run manually when IAM changes are needed
 spec:
  ttlSecondsAfterFinished: 600
  backoffLimit: 3
  template:
    spec:
      serviceAccountName: authentik-provisioner
      restartPolicy: Never
      securityContext:
        runAsNonRoot: true
        runAsUser: 1000
        seccompProfile:
          type: RuntimeDefault
      containers:
        - name: provision
          image: python:3.12-alpine
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop: ["ALL"]
          env:
            - name: AUTHENTIK_BOOTSTRAP_TOKEN
              valueFrom:
                secretKeyRef:
                  name: authentik-secrets
                  key: AUTHENTIK_BOOTSTRAP_TOKEN
          volumeMounts:
            - name: script
              mountPath: /script
          command:
            - /bin/sh
            - -c
            - |
              set -e
              echo "waiting for authentik-server..."
              until wget -q -O /dev/null http://authentik-server.iam.svc.cluster.local/-/health/ready/ 2>/dev/null; do
                sleep 5
              done
              echo "installing kubectl (via python urllib - no apk/curl: this"
              echo "container runs as non-root UID 1000 and can't write to"
              echo "apk's directories or /usr/local/bin, both root-owned in"
              echo "the python:3.12-alpine image; /tmp is world-writable)..."
              python3 -c "
              import urllib.request, os, stat
              kver = urllib.request.urlopen('https://dl.k8s.io/release/stable.txt').read().decode().strip()
              url = f'https://dl.k8s.io/release/{kver}/bin/linux/amd64/kubectl'
              urllib.request.urlretrieve(url, '/tmp/kubectl')
              st = os.stat('/tmp/kubectl')
              os.chmod('/tmp/kubectl', st.st_mode | stat.S_IEXEC)
              "
              export PATH="/tmp:$PATH"
              echo "running provisioning script..."
              python3 /script/authentik-provision.py
      volumes:
        - name: script
          configMap:
            name: authentik-provision-script
@@ -1,35 +1,12 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-# NOTE: no top-level `namespace:` transformer here (removed) - it used to
+
 # force-rewrite metadata.namespace to "iam" on every resource in this
 # kustomization, which was harmless while every manifest here only ever
 # targeted the iam namespace itself. authentik-provision-job.yaml's
 # RoleBindings deliberately target cicd/argocd/logging/storage (least-
 # privilege access for the authentik-provisioner ServiceAccount to touch
 # Secrets in those namespaces) - the namespace transformer would have
 # silently rewritten all of them back to iam, breaking the RBAC. Every
 # manifest in this directory already sets its own explicit
 # metadata.namespace, so dropping the transformer changes nothing for the
 # existing resources/.
 resources:
  - authentik-provision-job.yaml
  - rbac-dashboard-rolebinding.yaml
-# Provisioning/verification python lives in scripts/*.py (real files, linted +
+# IAM provisioning is manual-only (security-sensitive).
-# diff-friendly) and is generated into ConfigMaps here rather than embedded in
+# Script: scripts/iam/authentik-provision.py
-# the job YAML. disableNameSuffixHash keeps the names stable so the Jobs'
+# Run:
-# configMap volume refs and PostSync hook-delete semantics keep working; each
+#   export AUTHENTIK_BOOTSTRAP_TOKEN=$(kubectl -n iam get secret authentik-secrets \
-# hook Job is recreated per sync so it always mounts the latest script.
+#     -o jsonpath='{.data.AUTHENTIK_BOOTSTRAP_TOKEN}' | base64 -d)
-configMapGenerator:
+#   python3 scripts/iam/authentik-provision.py
  - name: authentik-provision-script
    namespace: iam
    files:
      - authentik-provision.py=scripts/authentik-provision.py
 generatorOptions:
  disableNameSuffixHash: true
 # authentik-migrations-job.yaml removed — redundant + broken. The authentik
 # `server` entrypoint runs migrations itself; this standalone job lacked the
 # authentik-secrets envFrom (Secret key missing) and always failed.
 # SOPS secrets (*.enc.yaml) handled by ArgoCD SOPS plugin at sync time
 # authentik/vault deployed via ArgoCD Helm source
@@ -68,15 +68,14 @@ grafana.ini:
  # doesn't return localhost redirects in its token responses.
  #
  # role_attribute_path: JMESPath expression evaluated against the userinfo
-  # response. Members of the 'grafana-admins' Authentik group get Admin role;
+  # response. akadmin gets GrafanaAdmin (server admin, can impersonate);
-  # everyone else gets Viewer. The group name must match exactly what Authentik
+  # homelab-admins members get Admin (org admin); everyone else Viewer.
  # sends in the 'groups' claim.
  auth.generic_oauth:
    enabled: true
    name: Authentik
    allow_sign_up: true
    client_id: grafana
-    scopes: openid email profile
+    scopes: openid email profile groups
    auth_url: https://authentik.riotpiao.com/application/o/authorize/
    token_url: https://authentik.riotpiao.com/application/o/token/
    api_url: https://authentik.riotpiao.com/application/o/userinfo/
@@ -87,7 +86,8 @@ grafana.ini:
    email_attribute_path: email
    login_attribute_path: preferred_username
    name_attribute_path: name
-    role_attribute_path: "contains(groups[*], 'grafana-admins') && 'Admin' || 'Viewer'"
+    role_attribute_path: "preferred_username == 'akadmin' && 'GrafanaAdmin' || contains(groups[*], 'homelab-admins') && 'Admin' || 'Viewer'"
    allow_assign_grafana_admin: true
    use_pkce: false
    use_refresh_token: false
    skip_org_role_sync: false
@@ -96,8 +96,13 @@ spec:
      console: https://minio.riotpiao.com
  # ── OIDC via Authentik (server-side env, valid in v2 schema) ────────────────
  # Use in-cluster URL for config fetch (pod→authentik); browser redirects use
  # public URLs embedded in the OIDC metadata response (issuer stays public).
  env:
    - name: MINIO_IDENTITY_OPENID_CONFIG_URL
      # Must use external URL — well-known response contains external issuer/jwks_uri.
      # MinIO validates issuer in JWT matches well-known issuer. Internal URL = mismatch.
      # Hairpins through ingress-nginx but stays in-cluster.
      value: "https://authentik.riotpiao.com/application/o/minio/.well-known/openid-configuration"
    - name: MINIO_IDENTITY_OPENID_CLIENT_ID
      value: "minio"
@@ -0,0 +1,160 @@
 apiVersion: monitoring.coreos.com/v1
 kind: PrometheusRule
 metadata:
  name: api-gateway-alerts
  namespace: monitoring
  labels:
    release: prometheus
 spec:
  groups:
    # ================================================================
    # SLA Targets (based on canary traffic baselines):
    #
    # Availability:  99.9% (43.8 min downtime/month)
    # LLM Chat:      p95 < 1s (qwen), p95 < 2s (reasoning), p95 < 5s (ornith)
    # Embeddings:    p95 < 500ms
    # Rerank:        p95 < 500ms
    # Models list:   p95 < 300ms
    # Error rate:    < 1% (5xx), < 5% (4xx excluding auth)
    #
    # Baselines from 200-request canary run:
    #   qwen p99=609ms, reasoning p99=328ms, embeddings p99=287ms,
    #   rerank p99=218ms, models p99=277ms
    # SLA set at ~2x p99 for headroom.
    # ================================================================
    - name: api-gateway.availability
      rules:
        # Gateway pods not ready
        - alert: APIGatewayDown
          expr: sum(kube_pod_status_ready{namespace="api",condition="true"}) == 0
          for: 1m
          labels:
            severity: critical
          annotations:
            summary: "API Gateway has zero ready pods"
        # Gateway pod count below desired
        - alert: APIGatewayDegraded
          expr: |
            sum(kube_pod_status_ready{namespace="api",condition="true"})
            < kube_deployment_spec_replicas{namespace="api",deployment="api-gateway"}
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "API Gateway {{ $value }} ready pods below desired replica count"
        # Blackbox probe down
        - alert: APIGatewayProbeDown
          expr: probe_success{instance=~".*api.riotpiao.com.*"} == 0
          for: 2m
          labels:
            severity: critical
          annotations:
            summary: "API Gateway probe failed: {{ $labels.instance }}"
        # LLM serving pods not ready
        - alert: LLMServingDown
          expr: sum(kube_pod_status_ready{namespace="llm-serving",condition="true"}) == 0
          for: 2m
          labels:
            severity: critical
          annotations:
            summary: "All LLM serving pods down"
        # Individual predictor down
        - alert: LLMPredictorDown
          expr: |
            kube_deployment_status_replicas_ready{namespace="llm-serving"}
            < kube_deployment_spec_replicas{namespace="llm-serving"}
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "{{ $labels.deployment }} has {{ $value }} ready (below desired)"
    - name: api-gateway.latency
      # SLA: latency thresholds at ~2x measured p99
      rules:
        # Ingress-level latency (all requests through nginx)
        - alert: APIGatewayLatencyHigh
          expr: |
            histogram_quantile(0.95,
              sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le)
            ) > 2
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "API Gateway p95 latency {{ $value | printf \"%.1f\" }}s (SLA: <2s)"
        # Extreme latency (p99 > 5s)
        - alert: APIGatewayLatencyCritical
          expr: |
            histogram_quantile(0.99,
              sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le)
            ) > 5
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "API Gateway p99 latency {{ $value | printf \"%.1f\" }}s (SLA: <5s)"
    - name: api-gateway.errors
      rules:
        # 5xx error rate > 1%
        - alert: APIGateway5xxErrorRate
          expr: |
            sum(rate(nginx_ingress_controller_requests{ingress="api",status=~"5.."}[5m]))
            / sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m]))
            > 0.01
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "API Gateway 5xx rate {{ $value | humanizePercentage }} (SLA: <1%)"
        # Total error rate > 10% (including 4xx)
        - alert: APIGatewayHighErrorRate
          expr: |
            sum(rate(nginx_ingress_controller_requests{ingress="api",status=~"[45].."}[5m]))
            / sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m]))
            > 0.10
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "API Gateway total error rate {{ $value | humanizePercentage }} (SLA: <10%)"
    - name: api-gateway.resources
      rules:
        # Gateway pod restart
        - alert: APIGatewayRestarted
          expr: increase(kube_pod_container_status_restarts_total{namespace="api"}[15m]) > 0
          for: 0m
          labels:
            severity: warning
          annotations:
            summary: "API Gateway pod {{ $labels.pod }} restarted"
        # LLM predictor restart
        - alert: LLMPredictorRestarted
          expr: increase(kube_pod_container_status_restarts_total{namespace="llm-serving"}[15m]) > 0
          for: 0m
          labels:
            severity: warning
          annotations:
            summary: "LLM predictor {{ $labels.pod }} restarted"
        # Gateway high memory (>80% of limit)
        - alert: APIGatewayHighMemory
          expr: |
            sum(container_memory_working_set_bytes{namespace="api",container="gateway"}) by (pod)
            / sum(kube_pod_container_resource_limits{namespace="api",container="gateway",resource="memory"}) by (pod)
            > 0.8
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "Gateway pod {{ $labels.pod }} memory at {{ $value | humanizePercentage }} of limit"
@@ -0,0 +1,177 @@
 apiVersion: monitoring.coreos.com/v1
 kind: PrometheusRule
 metadata:
  name: cluster-alerts
  namespace: monitoring
  labels:
    release: prometheus
 spec:
  groups:
    - name: cluster.availability
      rules:
        # Node down
        - alert: NodeNotReady
          expr: kube_node_status_condition{condition="Ready",status="true"} == 0
          for: 2m
          labels:
            severity: critical
          annotations:
            summary: "Node {{ $labels.node }} is NotReady"
        # Pod stuck pending (scheduling failure)
        - alert: PodStuckPending
          expr: sum(kube_pod_status_phase{phase="Pending"}) > 0
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "{{ $value }} pod(s) stuck in Pending state for >10m"
        # CrashLoopBackOff
        - alert: PodCrashLooping
          expr: sum(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}) by (namespace, pod) > 0
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "{{ $labels.namespace }}/{{ $labels.pod }} in CrashLoopBackOff"
        # OOMKilled spike
        - alert: OOMKilledSpike
          expr: sum(increase(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[1h])) > 3
          for: 0m
          labels:
            severity: warning
          annotations:
            summary: "{{ $value }} OOMKilled events in last hour"
        # Deployment replicas unavailable
        - alert: DeploymentReplicasUnavailable
          expr: kube_deployment_status_replicas_unavailable > 0
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "{{ $labels.namespace }}/{{ $labels.deployment }} has {{ $value }} unavailable replicas"
    - name: cluster.jobs
      rules:
        # Job failed
        - alert: JobFailed
          expr: kube_job_status_failed > 0
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Job {{ $labels.namespace }}/{{ $labels.job_name }} failed"
        # Job stuck running >2h
        - alert: JobStuckRunning
          expr: |
            kube_job_status_active == 1
            and on(job_name,namespace)
            (time() - kube_job_status_start_time) > 7200
          for: 0m
          labels:
            severity: warning
          annotations:
            summary: "Job {{ $labels.namespace }}/{{ $labels.job_name }} running >2h"
        # CronJob missed schedule
        - alert: CronJobMissedSchedule
          expr: |
            (time() - kube_cronjob_status_last_schedule_time) > 2 * (kube_cronjob_spec_next_schedule_time - kube_cronjob_status_last_schedule_time)
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} missed schedule"
    - name: cluster.resources
      rules:
        # Node CPU >90% sustained
        - alert: NodeHighCPU
          expr: (1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m])) by (instance)) * 100 > 90
          for: 15m
          labels:
            severity: warning
          annotations:
            summary: "Node {{ $labels.instance }} CPU at {{ $value | printf \"%.0f\" }}%"
        # Node memory >90% sustained
        - alert: NodeHighMemory
          expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 90
          for: 15m
          labels:
            severity: warning
          annotations:
            summary: "Node {{ $labels.instance }} memory at {{ $value | printf \"%.0f\" }}%"
        # Node disk >85%
        - alert: NodeDiskFull
          expr: (1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 > 85
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "Node {{ $labels.instance }} disk at {{ $value | printf \"%.0f\" }}%"
        # Container restart storm (>5 restarts in 15m)
        - alert: ContainerRestartStorm
          expr: sum(increase(kube_pod_container_status_restarts_total[15m])) by (namespace, pod) > 5
          for: 0m
          labels:
            severity: warning
          annotations:
            summary: "{{ $labels.namespace }}/{{ $labels.pod }} restarted {{ $value | printf \"%.0f\" }} times in 15m"
    - name: cluster.storage
      rules:
        # Longhorn drive offline
        - alert: LonghornDriveOffline
          expr: longhorn_disk_health != 1
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "Longhorn disk {{ $labels.node }} unhealthy"
    - name: cluster.dns
      rules:
        # CoreDNS errors spike
        - alert: CoreDNSErrorSpike
          expr: sum(rate(coredns_dns_responses_total{rcode=~"SERVFAIL"}[5m])) > 0.5
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "CoreDNS SERVFAIL rate {{ $value | printf \"%.2f\" }}/s"
    - name: cluster.probes
      rules:
        # Any blackbox probe down
        - alert: ServiceProbeDown
          expr: probe_success == 0
          for: 3m
          labels:
            severity: critical
          annotations:
            summary: "Probe failed: {{ $labels.instance }}"
        # Probe latency >2s
        - alert: ServiceProbeSlow
          expr: probe_duration_seconds > 2
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Probe slow ({{ $value | printf \"%.1f\" }}s): {{ $labels.instance }}"
        # Certificate expiry <14 days
        - alert: CertificateExpiringSoon
          expr: (certmanager_certificate_expiration_timestamp_seconds - time()) / 86400 < 14
          for: 0m
          labels:
            severity: warning
          annotations:
            summary: "Certificate {{ $labels.name }} expires in {{ $value | printf \"%.0f\" }} days"
@@ -61,6 +61,10 @@ serviceMonitor:
      url: https://argocd.riotpiao.com/healthz
    - name: longhorn
      url: https://longhorn.riotpiao.com/
    - name: api-gateway
      url: https://api.riotpiao.com/healthz
    - name: api-gateway-models
      url: https://api.riotpiao.com/v1/models
 prometheusRule:
  enabled: true
@@ -0,0 +1,36 @@
 apiVersion: v1
 data:
  api-gateway.json: '{"title":"API Gateway","uid":"api-gateway","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","tags":["api","gateway","llm"],"panels":[{"id":1,"title":"Gateway
    Health","type":"row","collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Gateway
    Pods Ready","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"red"},{"value":3,"color":"green"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"api\",condition=\"true\"})"}]},{"id":3,"title":"Probe:
    healthz","type":"stat","gridPos":{"h":4,"w":4,"x":4,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}]}},"targets":[{"expr":"probe_success{instance=~\".*api.riotpiao.com/healthz\"}"}]},{"id":4,"title":"Probe
    Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"probe_duration_seconds{instance=~\".*api.riotpiao.com.*\"}","legendFormat":"{{instance}}"}]}]},{"id":10,"title":"Ingress
    Traffic (nginx)","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"Request
    Rate by Status","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(nginx_ingress_controller_requests{ingress=\"api\"}[5m]))
    by (status)","legendFormat":"{{status}}"}]},{"id":12,"title":"Error Rate %","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percent"}},"targets":[{"expr":"sum(rate(nginx_ingress_controller_requests{ingress=\"api\",status=~\"5..\"}[5m]))
    / sum(rate(nginx_ingress_controller_requests{ingress=\"api\"}[5m])) * 100","legendFormat":"5xx"},{"expr":"sum(rate(nginx_ingress_controller_requests{ingress=\"api\",status=~\"4..\"}[5m]))
    / sum(rate(nginx_ingress_controller_requests{ingress=\"api\"}[5m])) * 100","legendFormat":"4xx"}]},{"id":13,"title":"Latency
    p50/p95/p99","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.50,
    sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"api\"}[5m]))
    by (le))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"api\"}[5m]))
    by (le))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"api\"}[5m]))
    by (le))","legendFormat":"p99"}]}]},{"id":20,"title":"LLM Serving","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"LLM
    Pods Ready","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"llm-serving\",condition=\"true\"})"}]},{"id":22,"title":"CPU
    by Predictor","type":"timeseries","gridPos":{"h":8,"w":8,"x":4,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"llm-serving\"}[5m]))
    by (pod)","legendFormat":"{{pod}}"}]},{"id":23,"title":"Memory by Predictor","type":"timeseries","gridPos":{"h":8,"w":8,"x":12,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"bytes"}},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"llm-serving\"})
    by (pod)","legendFormat":"{{pod}}"}]},{"id":24,"title":"Predictor Restarts","type":"timeseries","gridPos":{"h":8,"w":4,"x":20,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"llm-serving\"}[15m]))
    by (pod)","legendFormat":"{{pod}}"}]}]},{"id":30,"title":"Gateway Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"CPU
    by Gateway Pod","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"api\"}[5m]))
    by (pod)","legendFormat":"{{pod}}"}]},{"id":32,"title":"Memory by Gateway Pod","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"bytes"}},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"api\"})
    by (pod)","legendFormat":"{{pod}}"}]},{"id":33,"title":"Gateway Restarts","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"api\"}[15m]))
    by (pod)","legendFormat":"{{pod}}"}]}]},{"id":40,"title":"Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":4},"panels":[{"id":41,"title":"Gateway
    Logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":5},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"api\",container=\"gateway\"}"}]},{"id":42,"title":"LLM
    Serving Logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":15},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"llm-serving\"}"}]}]}]}'
 kind: ConfigMap
 metadata:
  annotations:
    grafana_folder: API
  labels:
    grafana_dashboard: '1'
  name: api-gateway-dashboard
  namespace: logging
@@ -0,0 +1,62 @@
 apiVersion: v1
 data:
  cluster-infrastructure.json: '{"title":"Cluster Infrastructure","uid":"cluster-infra","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","tags":["infrastructure","k8s"],"panels":[{"id":1,"title":"Cluster
    Health","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Nodes
    Ready","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"red"},{"value":3,"color":"green"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"count(kube_node_status_condition{condition=\"Ready\",status=\"true\"}
    == 1)"}]},{"id":3,"title":"Pods Pending","type":"stat","gridPos":{"h":4,"w":4,"x":4,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"sum(kube_pod_status_phase{phase=\"Pending\"})
    OR on() vector(0)"}]},{"id":4,"title":"CrashLoopBackOff","type":"stat","gridPos":{"h":4,"w":4,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"sum(kube_pod_container_status_waiting_reason{reason=\"CrashLoopBackOff\"})
    OR on() vector(0)"}]},{"id":5,"title":"OOMKilled (1h)","type":"stat","gridPos":{"h":4,"w":4,"x":12,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"sum(increase(kube_pod_container_status_last_terminated_reason{reason=\"OOMKilled\"}[1h]))
    OR on() vector(0)"}]},{"id":6,"title":"Deploys Unavailable","type":"stat","gridPos":{"h":4,"w":4,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"count(kube_deployment_status_replicas_unavailable
    > 0) OR on() vector(0)"}]},{"id":7,"title":"Services Down","type":"stat","gridPos":{"h":4,"w":4,"x":20,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"count(probe_success
    == 0) OR on() vector(0)"}]}]},{"id":10,"title":"Jobs & CronJobs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"Failed
    Jobs","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"count(kube_job_status_failed
    > 0) OR on() vector(0)"}]},{"id":12,"title":"Failed Jobs Detail","type":"table","gridPos":{"h":8,"w":10,"x":4,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"kube_job_status_failed
    > 0","format":"table","instant":true}]},{"id":13,"title":"Stuck Jobs (>1h)","type":"table","gridPos":{"h":8,"w":10,"x":14,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"kube_job_status_active
    == 1 and on(job_name,namespace) (time() - kube_job_status_start_time) > 3600","format":"table","instant":true}]},{"id":14,"title":"CronJob
    Last Success","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"dateTimeFromNow"}},"targets":[{"expr":"kube_cronjob_status_last_successful_time{namespace=~\"cicd|kube-system|paperless\"}","legendFormat":"{{namespace}}/{{cronjob}}"}]},{"id":15,"title":"Container
    Restart Storm (top 10)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"topk(10,
    sum(rate(kube_pod_container_status_restarts_total[15m])) by (namespace, pod))","legendFormat":"{{namespace}}/{{pod}}"}]}]},{"id":20,"title":"Node
    Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"CPU
    % by Node","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percent"}},"targets":[{"expr":"(1
    - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) by (instance)) * 100","legendFormat":"{{instance}}"}]},{"id":22,"title":"Memory
    % by Node","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percent"}},"targets":[{"expr":"(1
    - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100","legendFormat":"{{instance}}"}]},{"id":23,"title":"Disk
    % by Node","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percent"}},"targets":[{"expr":"(1
    - node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"})
    * 100","legendFormat":"{{instance}}"}]},{"id":24,"title":"Load Average","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"node_load1","legendFormat":"1m
    {{instance}}"},{"expr":"node_load5","legendFormat":"5m {{instance}}"}]},{"id":25,"title":"Network
    Errors & Drops","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(node_network_receive_errs_total[5m])","legendFormat":"rx-err
    {{instance}}"},{"expr":"rate(node_network_transmit_errs_total[5m])","legendFormat":"tx-err
    {{instance}}"},{"expr":"rate(node_network_receive_drop_total[5m])","legendFormat":"rx-drop
    {{instance}}"}]}]},{"id":30,"title":"Control Plane","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"API
    Server Up","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}]}},"targets":[{"expr":"min(up{job=\"apiserver\"})"}]},{"id":32,"title":"API
    Server Request Rate","type":"timeseries","gridPos":{"h":8,"w":10,"x":4,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(apiserver_request_total[5m]))
    by (verb, code)","legendFormat":"{{verb}} {{code}}"}]},{"id":33,"title":"API Server
    Error Rate %","type":"timeseries","gridPos":{"h":8,"w":10,"x":14,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percent"}},"targets":[{"expr":"sum(rate(apiserver_request_total{code=~\"5..\"}[5m]))
    / sum(rate(apiserver_request_total[5m])) * 100","legendFormat":"5xx %"}]},{"id":34,"title":"API
    Server Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":12},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95,
    sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
    sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p99"}]},{"id":35,"title":"etcd
    Request Duration","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":12},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.99,
    sum(rate(etcd_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p99"}]}]},{"id":40,"title":"Storage","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":4},"panels":[{"id":41,"title":"Longhorn
    Disk Capacity","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"bytes"}},"targets":[{"expr":"longhorn_disk_capacity_bytes","legendFormat":"capacity
    {{node}}"},{"expr":"longhorn_disk_reservation_bytes","legendFormat":"reserved
    {{node}}"}]},{"id":42,"title":"PVC Phase","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"kube_persistentvolumeclaim_status_phase","legendFormat":"{{namespace}}/{{persistentvolumeclaim}}
    {{phase}}"}]}]},{"id":50,"title":"DNS & Networking","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":5},"panels":[{"id":51,"title":"CoreDNS
    Cache Hit Rate","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"rate(coredns_cache_hits_total[5m])
    / (rate(coredns_cache_hits_total[5m]) + rate(coredns_cache_misses_total[5m]))","legendFormat":"{{server}}"}]},{"id":52,"title":"CoreDNS
    Errors","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(coredns_dns_responses_total{rcode=~\"SERVFAIL|NXDOMAIN\"}[5m]))
    by (rcode)","legendFormat":"{{rcode}}"}]}]},{"id":60,"title":"Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":6},"panels":[{"id":61,"title":"Error
    Rate by Namespace","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":7},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum
    by (namespace) (count_over_time({namespace=~\"kube-system|cert-manager|ingress-nginx|longhorn-system\"}
    |= \"error\" [5m]))","legendFormat":"{{namespace}}"}]},{"id":62,"title":"Control
    Plane Logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":15},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"kube-system\"}"}]},{"id":63,"title":"Cluster
    Addon Logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":25},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=~\"cert-manager|ingress-nginx|longhorn-system\"}"}]}]}]}'
 kind: ConfigMap
 metadata:
  annotations:
    grafana_folder: Infrastructure
  labels:
    grafana_dashboard: '1'
  name: cluster-infrastructure-dashboard
  namespace: logging
@@ -1,55 +0,0 @@
 # k8s/monitoring/dashboards/control-plane-logs.yaml
 # Surfaces controller/control-plane logs that are already in Loki today
 # (Promtail scrapes every namespace with no filter) — this dashboard is the
 # "make it visible" piece, not new log collection.
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: control-plane-logs-dashboard
  namespace: logging
  labels:
    grafana_dashboard: "1"
 data:
  control-plane-logs.json: |
    {
      "title": "Cluster Control Plane & Controllers (Logs)",
      "uid": "control-plane-logs",
      "schemaVersion": 39,
      "timezone": "browser",
      "time": { "from": "now-1h", "to": "now" },
      "refresh": "30s",
      "panels": [
        {
          "id": 1,
          "title": "Error rate by namespace",
          "type": "timeseries",
          "gridPos": { "h": 6, "w": 24, "x": 0, "y": 0 },
          "datasource": { "type": "loki", "uid": "loki" },
          "targets": [
            {
              "expr": "sum by (namespace) (count_over_time({namespace=~\"kube-system|cert-manager|ingress-nginx|longhorn-system\"} |= \"error\" [5m]))"
            }
          ]
        },
        {
          "id": 2,
          "title": "Control plane (kube-apiserver, controller-manager, scheduler)",
          "type": "logs",
          "gridPos": { "h": 10, "w": 24, "x": 0, "y": 6 },
          "datasource": { "type": "loki", "uid": "loki" },
          "targets": [
            { "expr": "{namespace=\"kube-system\"}" }
          ]
        },
        {
          "id": 3,
          "title": "Cluster add-ons (cert-manager, ingress-nginx, longhorn)",
          "type": "logs",
          "gridPos": { "h": 10, "w": 24, "x": 0, "y": 16 },
          "datasource": { "type": "loki", "uid": "loki" },
          "targets": [
            { "expr": "{namespace=~\"cert-manager|ingress-nginx|longhorn-system\"}" }
          ]
        }
      ]
    }
@@ -0,0 +1,280 @@
 #!/usr/bin/env python3
 """Generate consolidated Grafana dashboards as k8s ConfigMap YAML files."""
 import json
 import os
 DASHBOARD_DIR = os.path.expanduser("~/workplace/homelab/k8s/infra/monitoring/dashboards")
 DS_PROM = {"type": "prometheus", "uid": "prometheus"}
 DS_LOKI = {"type": "loki", "uid": "loki"}
 def stat_panel(id, title, expr, x, y, w=4, h=4, unit="short", mappings=None, thresholds=None):
    p = {
        "id": id, "title": title, "type": "stat",
        "gridPos": {"h": h, "w": w, "x": x, "y": y},
        "datasource": DS_PROM,
        "fieldConfig": {"defaults": {"unit": unit}},
        "targets": [{"expr": expr}],
    }
    if mappings:
        p["fieldConfig"]["defaults"]["mappings"] = mappings
    if thresholds:
        p["fieldConfig"]["defaults"]["thresholds"] = thresholds
        p["fieldConfig"]["defaults"]["color"] = {"mode": "thresholds"}
    return p
 def ts_panel(id, title, exprs, x, y, w=8, h=8, unit="short"):
    targets = []
    for e in exprs:
        if isinstance(e, tuple):
            targets.append({"expr": e[0], "legendFormat": e[1]})
        else:
            targets.append({"expr": e, "legendFormat": "{{pod}}"})
    return {
        "id": id, "title": title, "type": "timeseries",
        "gridPos": {"h": h, "w": w, "x": x, "y": y},
        "datasource": DS_PROM,
        "fieldConfig": {"defaults": {"unit": unit}},
        "targets": targets,
    }
 def table_panel(id, title, expr, x, y, w=12, h=8):
    return {
        "id": id, "title": title, "type": "table",
        "gridPos": {"h": h, "w": w, "x": x, "y": y},
        "datasource": DS_PROM,
        "targets": [{"expr": expr, "format": "table", "instant": True}],
    }
 def log_panel(id, title, query, x, y, w=24, h=10):
    return {
        "id": id, "title": title, "type": "logs",
        "gridPos": {"h": h, "w": w, "x": x, "y": y},
        "datasource": DS_LOKI,
        "targets": [{"expr": query}],
    }
 def row(id, title, y, panels, collapsed=True):
    return {
        "id": id, "title": title, "type": "row",
        "collapsed": collapsed, "gridPos": {"h": 1, "w": 24, "x": 0, "y": y},
        "panels": panels,
    }
 def write_dashboard(filename, dashboard, folder):
    cm = {
        "apiVersion": "v1",
        "kind": "ConfigMap",
        "metadata": {
            "name": filename.replace(".yaml", "-dashboard"),
            "namespace": "logging",
            "labels": {"grafana_dashboard": "1"},
            "annotations": {"grafana_folder": folder},
        },
        "data": {
            filename.replace(".yaml", ".json"): json.dumps(dashboard, separators=(",", ":"))
        },
    }
    import yaml
    path = os.path.join(DASHBOARD_DIR, filename)
    with open(path, "w") as f:
        yaml.dump(cm, f, default_flow_style=False, allow_unicode=True)
    print(f"  wrote {path}")
 # ============================================================================
 # Dashboard 1: Cluster Infrastructure
 # ============================================================================
 def build_cluster_infrastructure():
    zero_thresholds = {"mode": "absolute", "steps": [
        {"value": None, "color": "green"}, {"value": 1, "color": "red"}
    ]}
    panels = [
        row(1, "Cluster Health", 0, [
            stat_panel(2, "Nodes Ready", 'count(kube_node_status_condition{condition="Ready",status="true"} == 1)', 0, 1, thresholds={"mode":"absolute","steps":[{"value":None,"color":"red"},{"value":3,"color":"green"}]}),
            stat_panel(3, "Pods Pending", 'sum(kube_pod_status_phase{phase="Pending"}) OR on() vector(0)', 4, 1, thresholds=zero_thresholds),
            stat_panel(4, "CrashLoopBackOff", 'sum(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}) OR on() vector(0)', 8, 1, thresholds=zero_thresholds),
            stat_panel(5, "OOMKilled (1h)", 'sum(increase(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[1h])) OR on() vector(0)', 12, 1, thresholds=zero_thresholds),
            stat_panel(6, "Deploys Unavailable", 'count(kube_deployment_status_replicas_unavailable > 0) OR on() vector(0)', 16, 1, thresholds=zero_thresholds),
            stat_panel(7, "Services Down", 'count(probe_success == 0) OR on() vector(0)', 20, 1, thresholds=zero_thresholds),
        ]),
        row(10, "Jobs & CronJobs", 1, [
            stat_panel(11, "Failed Jobs", 'count(kube_job_status_failed > 0) OR on() vector(0)', 0, 2, thresholds=zero_thresholds),
            table_panel(12, "Failed Jobs Detail", 'kube_job_status_failed > 0', 4, 2, w=10),
            table_panel(13, "Stuck Jobs (>1h)", 'kube_job_status_active == 1 and on(job_name,namespace) (time() - kube_job_status_start_time) > 3600', 14, 2, w=10),
            ts_panel(14, "CronJob Last Success", [
                ('kube_cronjob_status_last_successful_time{namespace=~"cicd|kube-system|paperless"}', "{{namespace}}/{{cronjob}}")
            ], 0, 10, w=12, unit="dateTimeFromNow"),
            ts_panel(15, "Container Restart Storm (top 10)", [
                ('topk(10, sum(rate(kube_pod_container_status_restarts_total[15m])) by (namespace, pod))', "{{namespace}}/{{pod}}")
            ], 12, 10, w=12),
        ]),
        row(20, "Node Resources", 2, [
            ts_panel(21, "CPU % by Node", [
                ('(1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m])) by (instance)) * 100', "{{instance}}")
            ], 0, 3, unit="percent"),
            ts_panel(22, "Memory % by Node", [
                ('(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100', "{{instance}}")
            ], 8, 3, unit="percent"),
            ts_panel(23, "Disk % by Node", [
                ('(1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100', "{{instance}}")
            ], 16, 3, unit="percent"),
            ts_panel(24, "Load Average", [
                ("node_load1", "1m {{instance}}"),
                ("node_load5", "5m {{instance}}"),
            ], 0, 11),
            ts_panel(25, "Network Errors & Drops", [
                ("rate(node_network_receive_errs_total[5m])", "rx-err {{instance}}"),
                ("rate(node_network_transmit_errs_total[5m])", "tx-err {{instance}}"),
                ("rate(node_network_receive_drop_total[5m])", "rx-drop {{instance}}"),
            ], 8, 11),
        ]),
        row(30, "Control Plane", 3, [
            stat_panel(31, "API Server Up", 'min(up{job="apiserver"})', 0, 4, mappings=[
                {"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}
            ]),
            ts_panel(32, "API Server Request Rate", [
                ('sum(rate(apiserver_request_total[5m])) by (verb, code)', "{{verb}} {{code}}")
            ], 4, 4, w=10),
            ts_panel(33, "API Server Error Rate %", [
                ('sum(rate(apiserver_request_total{code=~"5.."}[5m])) / sum(rate(apiserver_request_total[5m])) * 100', "5xx %")
            ], 14, 4, w=10, unit="percent"),
            ts_panel(34, "API Server Latency", [
                ('histogram_quantile(0.95, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))', "p95"),
                ('histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))', "p99"),
            ], 0, 12, unit="s"),
            ts_panel(35, "etcd Request Duration", [
                ('histogram_quantile(0.99, sum(rate(etcd_request_duration_seconds_bucket[5m])) by (le))', "p99"),
            ], 8, 12, unit="s"),
        ]),
        row(40, "Storage", 4, [
            ts_panel(41, "Longhorn Disk Capacity", [
                ("longhorn_disk_capacity_bytes", "capacity {{node}}"),
                ("longhorn_disk_reservation_bytes", "reserved {{node}}"),
            ], 0, 5, unit="bytes"),
            ts_panel(42, "PVC Phase", [
                ('kube_persistentvolumeclaim_status_phase', "{{namespace}}/{{persistentvolumeclaim}} {{phase}}")
            ], 8, 5),
        ]),
        row(50, "DNS & Networking", 5, [
            ts_panel(51, "CoreDNS Cache Hit Rate", [
                ('rate(coredns_cache_hits_total[5m]) / (rate(coredns_cache_hits_total[5m]) + rate(coredns_cache_misses_total[5m]))', "{{server}}")
            ], 0, 6, unit="percentunit"),
            ts_panel(52, "CoreDNS Errors", [
                ('sum(rate(coredns_dns_responses_total{rcode=~"SERVFAIL|NXDOMAIN"}[5m])) by (rcode)', "{{rcode}}")
            ], 8, 6),
        ]),
        row(60, "Logs", 6, [
            ts_panel(61, "Error Rate by Namespace", [
                ('sum by (namespace) (count_over_time({namespace=~"kube-system|cert-manager|ingress-nginx|longhorn-system"} |= "error" [5m]))', "{{namespace}}")
            ], 0, 7),
            log_panel(62, "Control Plane Logs", '{namespace="kube-system"}', 0, 15),
            log_panel(63, "Cluster Addon Logs", '{namespace=~"cert-manager|ingress-nginx|longhorn-system"}', 0, 25),
        ]),
    ]
    return {
        "title": "Cluster Infrastructure",
        "uid": "cluster-infra",
        "schemaVersion": 39,
        "timezone": "browser",
        "time": {"from": "now-6h", "to": "now"},
        "refresh": "30s",
        "tags": ["infrastructure", "k8s"],
        "panels": panels,
    }
 # ============================================================================
 # Dashboard 3: API Gateway
 # ============================================================================
 def build_api_gateway():
    panels = [
        row(1, "Gateway Health", 0, [
            stat_panel(2, "Gateway Pods Ready", 'sum(kube_pod_status_ready{namespace="api",condition="true"})', 0, 1, thresholds={"mode":"absolute","steps":[{"value":None,"color":"red"},{"value":3,"color":"green"}]}),
            stat_panel(3, "Probe: healthz", 'probe_success{instance=~".*api.riotpiao.com/healthz"}', 4, 1, mappings=[
                {"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}
            ]),
            ts_panel(4, "Probe Latency", [
                ('probe_duration_seconds{instance=~".*api.riotpiao.com.*"}', "{{instance}}")
            ], 8, 1, unit="s"),
        ], collapsed=False),
        row(10, "Ingress Traffic (nginx)", 1, [
            ts_panel(11, "Request Rate by Status", [
                ('sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m])) by (status)', "{{status}}")
            ], 0, 2),
            ts_panel(12, "Error Rate %", [
                ('sum(rate(nginx_ingress_controller_requests{ingress="api",status=~"5.."}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m])) * 100', "5xx"),
                ('sum(rate(nginx_ingress_controller_requests{ingress="api",status=~"4.."}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m])) * 100', "4xx"),
            ], 8, 2, unit="percent"),
            ts_panel(13, "Latency p50/p95/p99", [
                ('histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le))', "p50"),
                ('histogram_quantile(0.95, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le))', "p95"),
                ('histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le))', "p99"),
            ], 16, 2, unit="s"),
        ]),
        row(20, "LLM Serving", 2, [
            stat_panel(21, "LLM Pods Ready", 'sum(kube_pod_status_ready{namespace="llm-serving",condition="true"})', 0, 3),
            ts_panel(22, "CPU by Predictor", [
                ('sum(rate(container_cpu_usage_seconds_total{namespace="llm-serving"}[5m])) by (pod)', "{{pod}}")
            ], 4, 3),
            ts_panel(23, "Memory by Predictor", [
                ('sum(container_memory_working_set_bytes{namespace="llm-serving"}) by (pod)', "{{pod}}")
            ], 12, 3, unit="bytes"),
            ts_panel(24, "Predictor Restarts", [
                ('sum(rate(kube_pod_container_status_restarts_total{namespace="llm-serving"}[15m])) by (pod)', "{{pod}}")
            ], 20, 3, w=4),
        ]),
        row(30, "Gateway Resources", 3, [
            ts_panel(31, "CPU by Gateway Pod", [
                ('sum(rate(container_cpu_usage_seconds_total{namespace="api"}[5m])) by (pod)', "{{pod}}")
            ], 0, 4),
            ts_panel(32, "Memory by Gateway Pod", [
                ('sum(container_memory_working_set_bytes{namespace="api"}) by (pod)', "{{pod}}")
            ], 8, 4, unit="bytes"),
            ts_panel(33, "Gateway Restarts", [
                ('sum(rate(kube_pod_container_status_restarts_total{namespace="api"}[15m])) by (pod)', "{{pod}}")
            ], 16, 4),
        ]),
        row(40, "Logs", 4, [
            log_panel(41, "Gateway Logs", '{namespace="api",container="gateway"}', 0, 5),
            log_panel(42, "LLM Serving Logs", '{namespace="llm-serving"}', 0, 15),
        ]),
    ]
    return {
        "title": "API Gateway",
        "uid": "api-gateway",
        "schemaVersion": 39,
        "timezone": "browser",
        "time": {"from": "now-6h", "to": "now"},
        "refresh": "30s",
        "tags": ["api", "gateway", "llm"],
        "panels": panels,
    }
 # ============================================================================
 # Generate
 # ============================================================================
 print("Generating dashboards...")
 # Dashboard 1
 write_dashboard("cluster-infrastructure.yaml", build_cluster_infrastructure(), "Infrastructure")
 # Dashboard 3
 write_dashboard("api-gateway.yaml", build_api_gateway(), "API")
 print("Done.")
@@ -1,121 +0,0 @@
 # k8s/monitoring/dashboards/hardware-overview.yaml
 # Trimmed operator at-a-glance view across all nodes — node-exporter already
 # powers the deep-dive "Node Exporter Full" (#1860, see grafana-values.yaml),
 # this is the quick health-check version, not a replacement for it.
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: hardware-overview-dashboard
  namespace: logging
  labels:
    grafana_dashboard: "1"
 data:
  hardware-overview.json: |
    {
      "title": "Hardware Statistics (Operator Overview)",
      "uid": "hardware-overview",
      "schemaVersion": 39,
      "timezone": "browser",
      "time": { "from": "now-6h", "to": "now" },
      "refresh": "30s",
      "panels": [
        {
          "id": 1,
          "title": "Nodes up / down",
          "type": "stat",
          "gridPos": { "h": 5, "w": 24, "x": 0, "y": 0 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "fieldConfig": {
            "defaults": {
              "mappings": [
                { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" } } },
                { "type": "value", "options": { "1": { "text": "UP", "color": "green" } } }
              ]
            }
          },
          "targets": [
            { "expr": "up{job=~\".*node-exporter.*\"}", "legendFormat": "{{instance}}" }
          ]
        },
        {
          "id": 2,
          "title": "CPU usage % by node",
          "type": "timeseries",
          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 5 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } },
          "targets": [
            {
              "expr": "(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) by (instance)) * 100",
              "legendFormat": "{{instance}}"
            }
          ]
        },
        {
          "id": 3,
          "title": "Memory usage % by node",
          "type": "timeseries",
          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 5 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } },
          "targets": [
            {
              "expr": "(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100",
              "legendFormat": "{{instance}}"
            }
          ]
        },
        {
          "id": 4,
          "title": "Root filesystem usage % by node",
          "type": "timeseries",
          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } },
          "targets": [
            {
              "expr": "(1 - node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"}) * 100",
              "legendFormat": "{{instance}}"
            }
          ]
        },
        {
          "id": 5,
          "title": "Root filesystem space remaining",
          "type": "timeseries",
          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 13 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "fieldConfig": { "defaults": { "unit": "bytes" } },
          "targets": [
            {
              "expr": "node_filesystem_avail_bytes{mountpoint=\"/\"}",
              "legendFormat": "{{instance}}"
            }
          ]
        },
        {
          "id": 6,
          "title": "Network errors/drops by node",
          "type": "timeseries",
          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 21 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "targets": [
            { "expr": "rate(node_network_receive_errs_total[5m])", "legendFormat": "{{instance}} rx errs" },
            { "expr": "rate(node_network_transmit_errs_total[5m])", "legendFormat": "{{instance}} tx errs" },
            { "expr": "rate(node_network_receive_drop_total[5m])", "legendFormat": "{{instance}} rx drops" },
            { "expr": "rate(node_network_transmit_drop_total[5m])", "legendFormat": "{{instance}} tx drops" }
          ]
        },
        {
          "id": 7,
          "title": "Load average (1m / 5m) by node",
          "type": "timeseries",
          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 21 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "targets": [
            { "expr": "node_load1", "legendFormat": "{{instance}} load1" },
            { "expr": "node_load5", "legendFormat": "{{instance}} load5" }
          ]
        }
      ]
    }
@@ -1,185 +0,0 @@
 # k8s/monitoring/dashboards/kube-controller-health.yaml
 # Talos binds controller-manager/scheduler/etcd to 127.0.0.1, so Prometheus
 # can't scrape them directly (see prometheus-values.yaml). kube-apiserver is
 # the one control-plane component that's still reachable (its ServiceMonitor
 # targets the in-cluster `kubernetes` service, not localhost) — paired with
 # kube-state-metrics signals as a proxy for controller/scheduler health.
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: kube-controller-health-dashboard
  namespace: logging
  labels:
    grafana_dashboard: "1"
 data:
  kube-controller-health.json: |
    {
      "title": "Kube-Controller Health",
      "uid": "kube-controller-health",
      "schemaVersion": 39,
      "timezone": "browser",
      "time": { "from": "now-6h", "to": "now" },
      "refresh": "30s",
      "panels": [
        {
          "id": 1,
          "title": "API server — up",
          "type": "stat",
          "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "fieldConfig": {
            "defaults": {
              "mappings": [
                { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" } } },
                { "type": "value", "options": { "1": { "text": "UP", "color": "green" } } }
              ]
            }
          },
          "targets": [
            { "expr": "min(up{job=\"apiserver\"})", "legendFormat": "apiserver" }
          ]
        },
        {
          "id": 2,
          "title": "API server — request rate by verb/code",
          "type": "timeseries",
          "gridPos": { "h": 8, "w": 18, "x": 6, "y": 0 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "targets": [
            {
              "expr": "sum(rate(apiserver_request_total[5m])) by (verb, code)",
              "legendFormat": "{{verb}} {{code}}"
            }
          ]
        },
        {
          "id": 3,
          "title": "API server — error rate % (5xx)",
          "type": "timeseries",
          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "fieldConfig": { "defaults": { "unit": "percent" } },
          "targets": [
            {
              "expr": "sum(rate(apiserver_request_total{code=~\"5..\"}[5m])) / sum(rate(apiserver_request_total[5m])) * 100",
              "legendFormat": "5xx %"
            }
          ]
        },
        {
          "id": 4,
          "title": "API server — latency p95 / p99",
          "type": "timeseries",
          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "fieldConfig": { "defaults": { "unit": "s" } },
          "targets": [
            {
              "expr": "histogram_quantile(0.95, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))",
              "legendFormat": "p95"
            },
            {
              "expr": "histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))",
              "legendFormat": "p99"
            }
          ]
        },
        {
          "id": 5,
          "title": "Pods stuck Pending",
          "type": "stat",
          "gridPos": { "h": 5, "w": 8, "x": 0, "y": 16 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "fieldConfig": {
            "defaults": {
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  { "value": 0, "color": "green" },
                  { "value": 1, "color": "yellow" },
                  { "value": 5, "color": "red" }
                ]
              }
            }
          },
          "targets": [
            { "expr": "sum(kube_pod_status_phase{phase=\"Pending\"}) OR on() vector(0)", "legendFormat": "pending" }
          ]
        },
        {
          "id": 6,
          "title": "CrashLoopBackOff containers",
          "type": "stat",
          "gridPos": { "h": 5, "w": 8, "x": 8, "y": 16 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "fieldConfig": {
            "defaults": {
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  { "value": 0, "color": "green" },
                  { "value": 1, "color": "red" }
                ]
              }
            }
          },
          "targets": [
            { "expr": "sum(kube_pod_container_status_waiting_reason{reason=\"CrashLoopBackOff\"}) OR on() vector(0)", "legendFormat": "crashlooping" }
          ]
        },
        {
          "id": 7,
          "title": "Nodes NotReady",
          "type": "stat",
          "gridPos": { "h": 5, "w": 8, "x": 16, "y": 16 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "fieldConfig": {
            "defaults": {
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  { "value": 0, "color": "green" },
                  { "value": 1, "color": "red" }
                ]
              }
            }
          },
          "targets": [
            { "expr": "count(kube_node_status_condition{condition=\"Ready\", status=\"true\"} == 0) OR on() vector(0)", "legendFormat": "not ready" }
          ]
        },
        {
          "id": 8,
          "title": "Failed Jobs",
          "type": "table",
          "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "targets": [
            { "expr": "kube_job_status_failed > 0", "format": "table", "instant": true }
          ]
        },
        {
          "id": 9,
          "title": "Deployments with unavailable replicas",
          "type": "table",
          "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "targets": [
            { "expr": "kube_deployment_status_replicas_unavailable > 0", "format": "table", "instant": true }
          ]
        },
        {
          "id": 10,
          "title": "Container restart rate by pod",
          "type": "timeseries",
          "gridPos": { "h": 8, "w": 24, "x": 0, "y": 28 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "targets": [
            {
              "expr": "sum(rate(kube_pod_container_status_restarts_total[15m])) by (namespace, pod)",
              "legendFormat": "{{namespace}}/{{pod}}"
            }
          ]
        }
      ]
    }
@@ -1,28 +0,0 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: llm-frontend-dashboard
  namespace: logging
  labels:
    grafana_dashboard: "1"
  annotations:
    grafana_folder: "LLM"
    # No request-level panels. The rate/error/latency/bandwidth row used to run
    # on Kong's prometheus plugin; Kong was retired 2026-08-19 and the Go
    # gateway that replaced it does not expose /metrics yet, so those panels
    # were removed rather than left querying series that no longer exist.
    # What is left is pod-level: readiness, CPU/memory, restarts, logs.
    #
    # Restoring request-level and per-model observability means wiring three
    # sources, none of which are in place: gateway metrics (RED plus token
    # counts and TTFT, which the gateway can measure because it sees the
    # response stream), vLLM's own /metrics on reasoning-predictor (rich --
    # vllm:time_to_first_token_seconds, vllm:inter_token_latency_seconds,
    # vllm:e2e_request_latency_seconds, vllm:kv_cache_usage_perc), and TEI's
    # /metrics on embeddings/reranker. Ollama exposes no Prometheus endpoint at
    # all (verified: /metrics returns 404), so ornith can only ever be observed
    # from the gateway side. No ServiceMonitor exists for the llm-serving
    # namespace today, so none of the engine metrics are being scraped.
 data:
  llm-frontend.json: |
    {"title":"LLM Frontend","uid":"llm-frontend","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"llm-serving pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"llm-serving\",condition=\"true\"})"}]},{"id":3,"title":"agent-pod ready","type":"stat","gridPos":{"h":4,"w":8,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"agent-pod\",condition=\"true\"})"}]},{"id":4,"title":"api gateway pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"api\",condition=\"true\"})"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=~\"llm-serving|agent-pod|api\"}[5m])) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":12,"title":"Memory by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=~\"llm-serving|agent-pod|api\"}) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":13,"title":"GPU-node predictor restarts","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"llm-serving\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"llm-serving logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"llm-serving\"}"}]},{"id":22,"title":"agent-pod logs (pi runs)","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":14},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"agent-pod\"}"}]},{"id":23,"title":"api gateway logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":24},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"api\"}"}]}]}]}
@@ -1,172 +0,0 @@
 # k8s/monitoring/dashboards/service-availability.yaml
 # Active uptime/availability from blackbox-exporter probes — the signal that
 # covers low-traffic services (Vault, MinIO, Longhorn UI) where RED metrics
 # alone can't distinguish "idle" from "down".
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: service-availability-dashboard
  namespace: logging
  labels:
    grafana_dashboard: "1"
 data:
  service-availability.json: |
    {
      "title": "Service Availability & Certificate Expiration",
      "uid": "svc-availability",
      "schemaVersion": 39,
      "timezone": "browser",
      "time": { "from": "now-24h", "to": "now" },
      "refresh": "30s",
      "panels": [
        {
          "id": 1,
          "title": "Up / Down — all probed services",
          "type": "stat",
          "gridPos": { "h": 6, "w": 24, "x": 0, "y": 0 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "fieldConfig": {
            "defaults": {
              "mappings": [
                { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" } } },
                { "type": "value", "options": { "1": { "text": "UP", "color": "green" } } }
              ],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  { "value": 0, "color": "red" },
                  { "value": 1, "color": "green" }
                ]
              }
            }
          },
          "targets": [
            { "expr": "probe_success", "legendFormat": "{{instance}}" }
          ]
        },
        {
          "id": 2,
          "title": "Uptime % trend",
          "type": "timeseries",
          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } },
          "targets": [
            {
              "expr": "avg_over_time(probe_success[$__rate_interval]) * 100",
              "legendFormat": "{{instance}}"
            }
          ]
        },
        {
          "id": 3,
          "title": "Probe latency",
          "type": "timeseries",
          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "fieldConfig": { "defaults": { "unit": "s" } },
          "targets": [
            { "expr": "probe_duration_seconds", "legendFormat": "{{instance}}" }
          ]
        },
        {
          "id": 4,
          "title": "7-day SLO (% successful probes)",
          "type": "table",
          "gridPos": { "h": 8, "w": 24, "x": 0, "y": 14 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "fieldConfig": {
            "defaults": {
              "unit": "percent",
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  { "value": 0, "color": "red" },
                  { "value": 99, "color": "yellow" },
                  { "value": 99.9, "color": "green" }
                ]
              }
            }
          },
          "targets": [
            {
              "expr": "avg_over_time(probe_success[7d]) * 100",
              "format": "table",
              "instant": true
            }
          ]
        },
        {
          "id": 5,
          "title": "Services DOWN right now",
          "type": "stat",
          "gridPos": { "h": 4, "w": 12, "x": 0, "y": 22 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "fieldConfig": {
            "defaults": {
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  { "value": 0, "color": "green" },
                  { "value": 1, "color": "red" }
                ]
              }
            }
          },
          "targets": [
            { "expr": "count(probe_success == 0) OR on() vector(0)", "legendFormat": "down" }
          ]
        },
        {
          "id": 6,
          "title": "Certs expiring in < 14 days",
          "type": "stat",
          "gridPos": { "h": 4, "w": 12, "x": 12, "y": 22 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "fieldConfig": {
            "defaults": {
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  { "value": 0, "color": "green" },
                  { "value": 1, "color": "red" }
                ]
              }
            }
          },
          "targets": [
            {
              "expr": "count((certmanager_certificate_expiration_timestamp_seconds - time()) / 86400 < 14) OR on() vector(0)",
              "legendFormat": "expiring"
            }
          ]
        },
        {
          "id": 7,
          "title": "Certificate expiry — days remaining",
          "type": "table",
          "gridPos": { "h": 8, "w": 24, "x": 0, "y": 26 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "fieldConfig": {
            "defaults": {
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  { "value": 0, "color": "red" },
                  { "value": 14, "color": "yellow" },
                  { "value": 30, "color": "green" }
                ]
              }
            }
          },
          "targets": [
            {
              "expr": "(certmanager_certificate_expiration_timestamp_seconds - time()) / 86400",
              "legendFormat": "{{name}}",
              "format": "table",
              "instant": true
            }
          ]
        }
      ]
    }
@@ -1,141 +0,0 @@
 # k8s/monitoring/dashboards/service-golden-signals.yaml
 # RED metrics (rate/errors/duration) for every service fronted by ingress-nginx.
 # Picked up automatically by Grafana's sidecar (grafana_dashboard=1 label) — see
 # sidecar.dashboards in k8s/logging/grafana-values.yaml.
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: service-golden-signals-dashboard
  namespace: logging
  labels:
    grafana_dashboard: "1"
 data:
  service-golden-signals.json: |
    {
      "title": "Latency & Golden Signals (Ingress RED)",
      "uid": "svc-golden-signals",
      "schemaVersion": 39,
      "timezone": "browser",
      "time": { "from": "now-6h", "to": "now" },
      "refresh": "30s",
      "templating": {
        "list": [
          {
            "name": "ingress",
            "type": "query",
            "datasource": { "type": "prometheus", "uid": "prometheus" },
            "query": "label_values(nginx_ingress_controller_requests, ingress)",
            "refresh": 2,
            "includeAll": false
          }
        ]
      },
      "panels": [
        {
          "id": 1,
          "title": "Request rate by status — $ingress",
          "type": "timeseries",
          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "targets": [
            {
              "expr": "sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\"}[5m])) by (status)",
              "legendFormat": "{{status}}"
            }
          ]
        },
        {
          "id": 2,
          "title": "Error rate % (4xx / 5xx) — $ingress",
          "type": "timeseries",
          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "fieldConfig": { "defaults": { "unit": "percent" } },
          "targets": [
            {
              "expr": "sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\", status=~\"5..\"}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\"}[5m])) * 100",
              "legendFormat": "5xx"
            },
            {
              "expr": "sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\", status=~\"4..\"}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\"}[5m])) * 100",
              "legendFormat": "4xx"
            }
          ]
        },
        {
          "id": 3,
          "title": "Latency p50 / p95 / p99 — $ingress",
          "type": "timeseries",
          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "fieldConfig": { "defaults": { "unit": "s" } },
          "targets": [
            {
              "expr": "histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"$ingress\"}[5m])) by (le))",
              "legendFormat": "p50"
            },
            {
              "expr": "histogram_quantile(0.95, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"$ingress\"}[5m])) by (le))",
              "legendFormat": "p95"
            },
            {
              "expr": "histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"$ingress\"}[5m])) by (le))",
              "legendFormat": "p99"
            }
          ]
        },
        {
          "id": 4,
          "title": "All services — traffic overview",
          "type": "table",
          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "targets": [
            {
              "expr": "topk(11, sum(rate(nginx_ingress_controller_requests[5m])) by (ingress))",
              "format": "table",
              "instant": true
            }
          ]
        },
        {
          "id": 5,
          "title": "Customer-facing failures (5xx count, window total)",
          "type": "stat",
          "gridPos": { "h": 5, "w": 12, "x": 0, "y": 16 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "fieldConfig": {
            "defaults": {
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  { "value": 0, "color": "green" },
                  { "value": 1, "color": "yellow" },
                  { "value": 50, "color": "red" }
                ]
              }
            }
          },
          "targets": [
            {
              "expr": "sum(increase(nginx_ingress_controller_requests{status=~\"5..\"}[$__range])) OR on() vector(0)",
              "legendFormat": "5xx total"
            }
          ]
        },
        {
          "id": 6,
          "title": "Top 5 error-contributing services",
          "type": "table",
          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "targets": [
            {
              "expr": "topk(5, sum(rate(nginx_ingress_controller_requests{status=~\"5..\"}[5m])) by (ingress))",
              "format": "table",
              "instant": true
            }
          ]
        }
      ]
    }
@@ -1,109 +0,0 @@
 # k8s/monitoring/dashboards/service-internals.yaml
 # Native per-service metrics — the "why" layer behind the ingress RED/uptime
 # dashboards (e.g. ingress shows MinIO is slow; this shows disk offline).
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: service-internals-dashboard
  namespace: logging
  labels:
    grafana_dashboard: "1"
 data:
  service-internals.json: |
    {
      "title": "Service Internals (MinIO / Forgejo / Argo CD / cert-manager / Vault / Longhorn)",
      "uid": "svc-internals",
      "schemaVersion": 39,
      "timezone": "browser",
      "time": { "from": "now-6h", "to": "now" },
      "refresh": "30s",
      "panels": [
        { "id": 1, "title": "MinIO — disk/node offline", "type": "timeseries",
          "gridPos": { "h": 6, "w": 12, "x": 0, "y": 0 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "targets": [
            { "expr": "minio_cluster_disk_offline_total", "legendFormat": "disks offline" },
            { "expr": "minio_cluster_nodes_offline_total", "legendFormat": "nodes offline" }
          ]
        },
        { "id": 2, "title": "MinIO — S3 request errors", "type": "timeseries",
          "gridPos": { "h": 6, "w": 12, "x": 12, "y": 0 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "targets": [
            { "expr": "sum(rate(minio_s3_requests_errors_total[5m])) by (api)", "legendFormat": "{{api}}" }
          ]
        },
        { "id": 3, "title": "MinIO — S3 TTFB latency", "type": "timeseries",
          "gridPos": { "h": 6, "w": 12, "x": 0, "y": 6 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "fieldConfig": { "defaults": { "unit": "s" } },
          "targets": [
            { "expr": "minio_s3_time_ttfb_seconds_distribution", "legendFormat": "{{api}}" }
          ]
        },
        { "id": 4, "title": "Forgejo — repos / orgs", "type": "stat",
          "gridPos": { "h": 6, "w": 12, "x": 12, "y": 6 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "targets": [
            { "expr": "gitea_repositories", "legendFormat": "repos" },
            { "expr": "gitea_organizations", "legendFormat": "orgs" }
          ]
        },
        { "id": 5, "title": "Forgejo — process health (CPU/mem)", "type": "timeseries",
          "gridPos": { "h": 6, "w": 12, "x": 0, "y": 12 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "targets": [
            { "expr": "rate(process_cpu_seconds_total{job=~\".*forgejo.*|.*gitea.*\"}[5m])", "legendFormat": "cpu" },
            { "expr": "process_resident_memory_bytes{job=~\".*forgejo.*|.*gitea.*\"}", "legendFormat": "mem" }
          ]
        },
        { "id": 6, "title": "Argo CD — app sync/health status", "type": "table",
          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "targets": [
            { "expr": "argocd_app_info", "format": "table", "instant": true }
          ]
        },
        { "id": 7, "title": "cert-manager — days to cert expiry", "type": "stat",
          "gridPos": { "h": 6, "w": 12, "x": 0, "y": 18 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "fieldConfig": {
            "defaults": {
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  { "value": 0, "color": "red" },
                  { "value": 14, "color": "yellow" },
                  { "value": 30, "color": "green" }
                ]
              }
            }
          },
          "targets": [
            { "expr": "(certmanager_certificate_expiration_timestamp_seconds - time()) / 86400", "legendFormat": "{{name}}" }
          ]
        },
        { "id": 8, "title": "Vault — sealed/unsealed", "type": "stat",
          "gridPos": { "h": 6, "w": 6, "x": 12, "y": 20 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "fieldConfig": {
            "defaults": {
              "mappings": [
                { "type": "value", "options": { "0": { "text": "SEALED", "color": "red" } } },
                { "type": "value", "options": { "1": { "text": "UNSEALED", "color": "green" } } }
              ]
            }
          },
          "targets": [
            { "expr": "vault_core_unsealed", "legendFormat": "vault" }
          ]
        },
        { "id": 9, "title": "Longhorn — volume robustness", "type": "table",
          "gridPos": { "h": 6, "w": 6, "x": 18, "y": 20 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "targets": [
            { "expr": "longhorn_volume_robustness", "format": "table", "instant": true }
          ]
        }
      ]
    }
@@ -1,12 +0,0 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: svc-argocd-dashboard
  namespace: logging
  labels:
    grafana_dashboard: "1"
  annotations:
    grafana_folder: "Argo CD"
 data:
  svc-argocd.json: |
    {"title":"Argo CD — Service Overview","uid":"svc-argocd","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"}}},"targets":[{"expr":"min(up{job=~\"argocd-.*\"})"}]},{"id":3,"title":"HTTP requests","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_http_request_total[5m])) by (status)","legendFormat":"{{status}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_http_request_total{status=~\"5..\"}[5m])) / sum(rate(argocd_http_request_total[5m])) * 100"}]},{"id":5,"title":"Request latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(argocd_http_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"argocd\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"argocd\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restarts","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"argocd\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Applications & Sync","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Applications","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"argocd_app_total"}]},{"id":22,"title":"Sync by status","type":"timeseries","gridPos":{"h":6,"w":9,"x":6,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_app_sync_total[5m])) by (sync_status)","legendFormat":"{{sync_status}}"}]},{"id":23,"title":"Degraded apps","type":"stat","gridPos":{"h":6,"w":6,"x":15,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"argocd_app_health_degraded_total"}]},{"id":24,"title":"Git sync ops","type":"timeseries","gridPos":{"h":6,"w":12,"x":0,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_git_sync_total[5m])) by (git_operation,git_status)","legendFormat":"{{git_operation}}/{{git_status}}"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"argocd\"}"}]}]}]}
@@ -1,143 +0,0 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: svc-authentik-dashboard
  namespace: logging
  labels:
    grafana_dashboard: "1"
  annotations:
    grafana_folder: "Authentik"
 data:
  svc-authentik.json: |
    {
      "title": "Authentik — Service Overview",
      "uid": "svc-authentik",
      "schemaVersion": 39,
      "timezone": "browser",
      "time": { "from": "now-6h", "to": "now" },
      "refresh": "30s",
      "panels": [
        {
          "id": 1, "title": "Row: Availability & Golden Signals", "type": "row",
          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
          "panels": [
            {
              "id": 2, "title": "Up", "type": "stat",
              "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
              "datasource": { "type": "prometheus", "uid": "prometheus" },
              "fieldConfig": {
                "defaults": {
                  "color": { "mode": "thresholds" },
                  "mappings": [
                    { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }
                  ],
                  "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" }, { "value": 1, "color": "green" } ] }
                }
              },
              "targets": [{ "expr": "min(up{job=\"authentik-server\"})" }]
            },
            {
              "id": 3, "title": "HTTP request rate by status", "type": "timeseries",
              "gridPos": { "h": 8, "w": 9, "x": 6, "y": 1 },
              "datasource": { "type": "prometheus", "uid": "prometheus" },
              "targets": [{ "expr": "sum(rate(authentik_flows_execution_stage_time_count[5m])) by (flow_slug)", "legendFormat": "{{flow_slug}}" }]
            },
            {
              "id": 4, "title": "Error rate % (5xx)", "type": "timeseries",
              "gridPos": { "h": 8, "w": 9, "x": 15, "y": 1 },
              "fieldConfig": { "defaults": { "unit": "percent" } },
              "datasource": { "type": "prometheus", "uid": "prometheus" },
              "targets": [{ "expr": "(1 - (authentik_flows_cached / authentik_flows_execution_stage_time_count)) * 100" }]
            },
            {
              "id": 5, "title": "Request duration p50/p95/p99", "type": "timeseries",
              "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
              "fieldConfig": { "defaults": { "unit": "s" } },
              "datasource": { "type": "prometheus", "uid": "prometheus" },
              "targets": [
                { "expr": "histogram_quantile(0.50, sum(rate(authentik_main_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p50" },
                { "expr": "histogram_quantile(0.95, sum(rate(authentik_main_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p95" },
                { "expr": "histogram_quantile(0.99, sum(rate(authentik_main_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p99" }
              ]
            }
          ]
        },
        {
          "id": 10, "title": "Row: Resource Usage", "type": "row",
          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 },
          "panels": [
            {
              "id": 11, "title": "CPU by pod", "type": "timeseries",
              "gridPos": { "h": 8, "w": 8, "x": 0, "y": 2 },
              "datasource": { "type": "prometheus", "uid": "prometheus" },
              "targets": [{ "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"iam\",pod=~\"authentik.*\"}[5m])) by (pod)", "legendFormat": "{{pod}}" }]
            },
            {
              "id": 12, "title": "Memory by pod", "type": "timeseries",
              "gridPos": { "h": 8, "w": 8, "x": 8, "y": 2 },
              "fieldConfig": { "defaults": { "unit": "bytes" } },
              "datasource": { "type": "prometheus", "uid": "prometheus" },
              "targets": [{ "expr": "sum(container_memory_working_set_bytes{namespace=\"iam\",pod=~\"authentik.*\"}) by (pod)", "legendFormat": "{{pod}}" }]
            },
            {
              "id": 13, "title": "Restart rate by pod", "type": "timeseries",
              "gridPos": { "h": 8, "w": 8, "x": 16, "y": 2 },
              "datasource": { "type": "prometheus", "uid": "prometheus" },
              "targets": [{ "expr": "sum(rate(kube_pod_container_status_restarts_total{namespace=\"iam\",pod=~\"authentik.*\"}[15m])) by (pod)", "legendFormat": "{{pod}}" }]
            }
          ]
        },
        {
          "id": 20, "title": "Row: Identity Provider (OIDC / OAuth2)", "type": "row",
          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 },
          "panels": [
            {
              "id": 21, "title": "Outpost connections", "type": "stat",
              "gridPos": { "h": 7, "w": 6, "x": 0, "y": 3 },
              "fieldConfig": { "defaults": { "unit": "short" } },
              "datasource": { "type": "prometheus", "uid": "prometheus" },
              "targets": [{ "expr": "authentik_outposts_connected" }]
            },
            {
              "id": 22, "title": "Flows cached", "type": "stat",
              "gridPos": { "h": 7, "w": 6, "x": 6, "y": 3 },
              "fieldConfig": { "defaults": { "unit": "short" } },
              "datasource": { "type": "prometheus", "uid": "prometheus" },
              "targets": [{ "expr": "authentik_flows_cached" }]
            },
            {
              "id": 23, "title": "Policies cached", "type": "stat",
              "gridPos": { "h": 7, "w": 6, "x": 12, "y": 3 },
              "fieldConfig": { "defaults": { "unit": "short" } },
              "datasource": { "type": "prometheus", "uid": "prometheus" },
              "targets": [{ "expr": "authentik_policies_cached" }]
            },
            {
              "id": 24, "title": "Queued tasks", "type": "stat",
              "gridPos": { "h": 7, "w": 6, "x": 18, "y": 3 },
              "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "unit": "short" } },
              "datasource": { "type": "prometheus", "uid": "prometheus" },
              "targets": [{ "expr": "authentik_tasks_queued" }]
            },
            {
              "id": 25, "title": "Admin workers", "type": "timeseries",
              "gridPos": { "h": 7, "w": 12, "x": 0, "y": 10 },
              "datasource": { "type": "prometheus", "uid": "prometheus" },
              "targets": [{ "expr": "authentik_admin_workers" }]
            }
          ]
        },
        {
          "id": 30, "title": "Row: Logs", "type": "row",
          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 },
          "panels": [
            {
              "id": 31, "title": "Recent logs", "type": "logs",
              "gridPos": { "h": 10, "w": 24, "x": 0, "y": 4 },
              "datasource": { "type": "loki", "uid": "loki" },
              "targets": [{ "expr": "{namespace=\"iam\"}" }]
            }
          ]
        }
      ]
    }
@@ -1,12 +0,0 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: svc-forgejo-dashboard
  namespace: logging
  labels:
    grafana_dashboard: "1"
  annotations:
    grafana_folder: "Forgejo"
 data:
  svc-forgejo.json: |
    {"title":"Forgejo — Service Overview","uid":"svc-forgejo","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"}}},"targets":[{"expr":"min(up{job=\"forgejo\"})"}]},{"id":3,"title":"HTTP requests by method","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_http_request_total[5m])) by (method)","legendFormat":"{{method}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_http_request_total{status=~\"5..\"}[5m])) / sum(rate(forgejo_http_request_total[5m])) * 100"}]},{"id":5,"title":"Request latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(forgejo_http_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"forgejo\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"forgejo\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restarts","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"forgejo\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Git Operations","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Repositories","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"forgejo_repositories_total"}]},{"id":22,"title":"Users","type":"stat","gridPos":{"h":6,"w":6,"x":6,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"forgejo_users_total"}]},{"id":23,"title":"Git ops rate","type":"timeseries","gridPos":{"h":6,"w":12,"x":12,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_git_operations_total[5m])) by (operation_type)","legendFormat":"{{operation_type}}"}]},{"id":24,"title":"Runner tasks","type":"timeseries","gridPos":{"h":6,"w":12,"x":0,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_runner_tasks_total[5m])) by (status)","legendFormat":"{{status}}"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"forgejo\"}"}]}]}]}
@@ -1,12 +0,0 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: svc-grafana-dashboard
  namespace: logging
  labels:
    grafana_dashboard: "1"
  annotations:
    grafana_folder: "Grafana"
 data:
  svc-grafana.json: |
    {"title":"Grafana — Service Overview","uid":"svc-grafana","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}]}},"targets":[{"expr":"min(up{job=\"grafana\"})"}]},{"id":3,"title":"HTTP requests","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(grafana_http_request_total[5m])) by (status)","legendFormat":"{{status}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(grafana_http_request_total{status=~\"5..\"}[5m])) / sum(rate(grafana_http_request_total[5m])) * 100"}]},{"id":5,"title":"Request latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(grafana_http_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"logging\",pod=~\"grafana.*\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"logging\",pod=~\"grafana.*\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restarts","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"logging\",pod=~\"grafana.*\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Dashboards & Users","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Total dashboards","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"grafana_dashboard_total"}]},{"id":22,"title":"Total users","type":"stat","gridPos":{"h":6,"w":6,"x":6,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"grafana_user_total"}]},{"id":23,"title":"Total alerts","type":"stat","gridPos":{"h":6,"w":6,"x":12,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"grafana_alerts_total"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"logging\",container=\"grafana\"}"}]}]}]}
@@ -1,143 +0,0 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: svc-minio-dashboard
  namespace: logging
  labels:
    grafana_dashboard: "1"
  annotations:
    grafana_folder: "MinIO"
 data:
  svc-minio.json: |
    {
      "title": "MinIO — Service Overview",
      "uid": "svc-minio",
      "schemaVersion": 39,
      "timezone": "browser",
      "time": { "from": "now-6h", "to": "now" },
      "refresh": "30s",
      "panels": [
        {
          "id": 1, "title": "Row: Availability & Golden Signals", "type": "row",
          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
          "panels": [
            {
              "id": 2, "title": "Up", "type": "stat",
              "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
              "datasource": { "type": "prometheus", "uid": "prometheus" },
              "fieldConfig": {
                "defaults": {
                  "color": { "mode": "thresholds" },
                  "mappings": [
                    { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }
                  ],
                  "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" }, { "value": 1, "color": "green" } ] }
                }
              },
              "targets": [{ "expr": "min(up{job=\"minio\"})" }]
            },
            {
              "id": 3, "title": "S3 request rate by method", "type": "timeseries",
              "gridPos": { "h": 8, "w": 9, "x": 6, "y": 1 },
              "datasource": { "type": "prometheus", "uid": "prometheus" },
              "targets": [{ "expr": "sum(rate(minio_s3_requests_total[5m])) by (method)", "legendFormat": "{{method}}" }]
            },
            {
              "id": 4, "title": "Error rate %", "type": "timeseries",
              "gridPos": { "h": 8, "w": 9, "x": 15, "y": 1 },
              "fieldConfig": { "defaults": { "unit": "percent" } },
              "datasource": { "type": "prometheus", "uid": "prometheus" },
              "targets": [{ "expr": "sum(rate(minio_s3_requests_total{error=\"true\"}[5m])) / sum(rate(minio_s3_requests_total[5m])) * 100" }]
            },
            {
              "id": 5, "title": "Request duration p50/p95/p99", "type": "timeseries",
              "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
              "fieldConfig": { "defaults": { "unit": "s" } },
              "datasource": { "type": "prometheus", "uid": "prometheus" },
              "targets": [
                { "expr": "histogram_quantile(0.50, sum(rate(minio_s3_requests_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p50" },
                { "expr": "histogram_quantile(0.95, sum(rate(minio_s3_requests_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p95" },
                { "expr": "histogram_quantile(0.99, sum(rate(minio_s3_requests_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p99" }
              ]
            }
          ]
        },
        {
          "id": 10, "title": "Row: Resource Usage", "type": "row",
          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 },
          "panels": [
            {
              "id": 11, "title": "CPU by pod", "type": "timeseries",
              "gridPos": { "h": 8, "w": 8, "x": 0, "y": 2 },
              "datasource": { "type": "prometheus", "uid": "prometheus" },
              "targets": [{ "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"storage\",pod=~\"minio.*\"}[5m])) by (pod)", "legendFormat": "{{pod}}" }]
            },
            {
              "id": 12, "title": "Memory by pod", "type": "timeseries",
              "gridPos": { "h": 8, "w": 8, "x": 8, "y": 2 },
              "fieldConfig": { "defaults": { "unit": "bytes" } },
              "datasource": { "type": "prometheus", "uid": "prometheus" },
              "targets": [{ "expr": "sum(container_memory_working_set_bytes{namespace=\"storage\",pod=~\"minio.*\"}) by (pod)", "legendFormat": "{{pod}}" }]
            },
            {
              "id": 13, "title": "Restart rate by pod", "type": "timeseries",
              "gridPos": { "h": 8, "w": 8, "x": 16, "y": 2 },
              "datasource": { "type": "prometheus", "uid": "prometheus" },
              "targets": [{ "expr": "sum(rate(kube_pod_container_status_restarts_total{namespace=\"storage\",pod=~\"minio.*\"}[15m])) by (pod)", "legendFormat": "{{pod}}" }]
            }
          ]
        },
        {
          "id": 20, "title": "Row: Storage & Replication", "type": "row",
          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 },
          "panels": [
            {
              "id": 21, "title": "Usable vs Raw capacity", "type": "timeseries",
              "gridPos": { "h": 8, "w": 12, "x": 0, "y": 3 },
              "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "lineWidth": 2 } } },
              "datasource": { "type": "prometheus", "uid": "prometheus" },
              "targets": [
                { "expr": "minio_cluster_capacity_usable_bytes", "legendFormat": "Usable" },
                { "expr": "minio_cluster_capacity_raw_total_bytes", "legendFormat": "Raw Total" }
              ]
            },
            {
              "id": 22, "title": "Drive health (online/offline)", "type": "timeseries",
              "gridPos": { "h": 8, "w": 12, "x": 12, "y": 3 },
              "fieldConfig": { "defaults": { "unit": "short" } },
              "datasource": { "type": "prometheus", "uid": "prometheus" },
              "targets": [
                { "expr": "minio_cluster_health_drives_online", "legendFormat": "Online" },
                { "expr": "minio_cluster_health_drives_offline", "legendFormat": "Offline" }
              ]
            },
            {
              "id": 23, "title": "Replication lag (bytes pending)", "type": "timeseries",
              "gridPos": { "h": 8, "w": 12, "x": 0, "y": 11 },
              "fieldConfig": { "defaults": { "unit": "bytes" } },
              "datasource": { "type": "prometheus", "uid": "prometheus" },
              "targets": [{ "expr": "minio_replication_metrics_replicating_byte_count", "legendFormat": "Pending replication" }]
            },
            {
              "id": 24, "title": "Replication failures (bytes)", "type": "timeseries",
              "gridPos": { "h": 8, "w": 12, "x": 12, "y": 11 },
              "fieldConfig": { "defaults": { "unit": "bytes" } },
              "datasource": { "type": "prometheus", "uid": "prometheus" },
              "targets": [{ "expr": "minio_replication_metrics_failed_byte_count", "legendFormat": "Failed replication" }]
            }
          ]
        },
        {
          "id": 30, "title": "Row: Logs", "type": "row",
          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 },
          "panels": [
            {
              "id": 31, "title": "Recent logs", "type": "logs",
              "gridPos": { "h": 10, "w": 24, "x": 0, "y": 4 },
              "datasource": { "type": "loki", "uid": "loki" },
              "targets": [{ "expr": "{namespace=\"storage\"}" }]
            }
          ]
        }
      ]
    }
@@ -1,12 +0,0 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: svc-vault-dashboard
  namespace: logging
  labels:
    grafana_dashboard: "1"
  annotations:
    grafana_folder: "Vault"
 data:
  svc-vault.json: |
    {"title":"Vault — Service Overview","uid":"svc-vault","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability & Golden Signals","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"red"},{"value":1,"color":"green"}]}}},"targets":[{"expr":"min(up{job=\"vault\"})"}]},{"id":3,"title":"Request rate by status","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(vault_core_handle_request_total[5m])) by (method)","legendFormat":"{{method}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(vault_core_handle_request_total{error=\"true\"}[5m])) / sum(rate(vault_core_handle_request_total[5m])) * 100"}]},{"id":5,"title":"Request duration p50/p95/p99","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.50, sum(rate(vault_core_handle_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95, sum(rate(vault_core_handle_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99, sum(rate(vault_core_handle_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p99"}]}]},{"id":10,"title":"Row: Resource Usage","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU by pod","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"iam\",pod=~\"vault.*\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory by pod","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"iam\",pod=~\"vault.*\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restart rate","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"iam\",pod=~\"vault.*\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Vault Seal State","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Sealed","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"UNSEALED","color":"green"},"1":{"text":"SEALED","color":"red"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]}}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_core_unsealed"}]},{"id":22,"title":"Active","type":"stat","gridPos":{"h":6,"w":6,"x":6,"y":3},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"INACTIVE","color":"red"},"1":{"text":"ACTIVE","color":"green"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"red"},{"value":1,"color":"green"}]}}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_core_active"}]},{"id":23,"title":"Replication (Primary)","type":"stat","gridPos":{"h":6,"w":6,"x":12,"y":3},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"SECONDARY","color":"orange"},"1":{"text":"PRIMARY","color":"green"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"orange"},{"value":1,"color":"green"}]}}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_core_replication_primary"}]},{"id":24,"title":"Active tokens","type":"stat","gridPos":{"h":6,"w":6,"x":18,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_token_total"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"iam\",container=\"vault\"}"}]}]}]}
@@ -12,20 +12,12 @@ resources:
  - alerts/svc-grafana-rules.yaml
  - alerts/svc-minio-rules.yaml
  - alerts/svc-vault-rules.yaml
  - alerts/cluster-alerts.yaml
  - alerts/api-gateway-alerts.yaml
  - servicemonitors/argocd.yaml
  - servicemonitors/authentik.yaml
  - servicemonitors/forgejo.yaml
  - servicemonitors/minio.yaml
-  - dashboards/control-plane-logs.yaml
+  - servicemonitors/ingress-nginx.yaml
-  - dashboards/hardware-overview.yaml
+  - dashboards/cluster-infrastructure.yaml
-  - dashboards/kube-controller-health.yaml
+  - dashboards/api-gateway.yaml
  - dashboards/llm-frontend.yaml
  - dashboards/service-availability.yaml
  - dashboards/service-golden-signals.yaml
  - dashboards/service-internals.yaml
  - dashboards/svc-argocd.yaml
  - dashboards/svc-authentik.yaml
  - dashboards/svc-forgejo.yaml
  - dashboards/svc-grafana.yaml
  - dashboards/svc-minio.yaml
  - dashboards/svc-vault.yaml
@@ -0,0 +1,18 @@
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
  name: ingress-nginx
  namespace: monitoring
  labels:
    release: prometheus
 spec:
  namespaceSelector:
    matchNames:
      - ingress-nginx
  selector:
    matchLabels:
      app.kubernetes.io/name: ingress-nginx
      app.kubernetes.io/component: controller
  endpoints:
    - port: metrics
      interval: 30s
@@ -0,0 +1,15 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
  name: homelab-admin-oidc
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
 subjects:
  # OIDC group for Authentik homelab-admins members
  # When rock logs in via OIDC, k8s sees:
  # - User: oidc:[email protected]
  # - Groups: oidc:homelab-admins (from Authentik group claim)
  - kind: Group
    name: oidc:homelab-admins
@@ -6,6 +6,7 @@ kind: Kustomization
 # already fixed once in k8s/infra/minio and k8s/infra/iam. Every resource
 # here sets its own explicit metadata.namespace.
 resources:
  - admin-oidc-binding.yaml
  - grafana-operator-role.yaml
  - minio-operator-role.yaml
  - forgejo-operator-role.yaml
@@ -37,8 +37,34 @@ import sys
 import urllib.error
 import urllib.request
-AUTHENTIK_URL = "http://authentik-server.iam.svc.cluster.local"
+AUTHENTIK_URL = "https://authentik.riotpiao.com"
-TOKEN = os.environ["AUTHENTIK_BOOTSTRAP_TOKEN"]
+
 # Try bootstrap token first, fallback to kubeconfig port-forward
 TOKEN = os.environ.get("AUTHENTIK_BOOTSTRAP_TOKEN")
 if not TOKEN:
    import subprocess
    # Use admin kubeconfig from Talos to port-forward and access Authentik
    # First, get bootstrap token from k8s secret via talosctl kubeconfig
    result = subprocess.run(
        ["kubectl", "--kubeconfig=/tmp/admin-kubeconfig.yaml", "-n", "authentik",
         "get", "secret", "authentik-bootstrap-token-secret", "-o", "jsonpath={.data.token}"],
        capture_output=True, text=True
    )
    if result.returncode == 0:
        TOKEN = result.stdout.strip()
        if TOKEN:
            # Decode if base64
            try:
                import base64
                decoded = base64.b64decode(TOKEN).decode()
                TOKEN = decoded
            except:
                pass  # Already decoded
    else:
        print("Error: AUTHENTIK_BOOTSTRAP_TOKEN not found")
        print(f"  Set env var: export AUTHENTIK_BOOTSTRAP_TOKEN=<token>")
        print(f"  Or use: kubectl -n authentik get secret ... ")
        sys.exit(1)
 def api(method, path, data=None):
@@ -185,6 +211,10 @@ GROUP_PERMISSIONS = {
    "llm-users": ["llm:inference"],
    "memory-users": ["memory:read"],
    "memory-writers": ["memory:read", "memory:write"],
    "s3-users": ["s3:read"],
    "s3-writers": ["s3:read", "s3:write"],
    "sqs-users": ["sqs:read"],
    "sqs-writers": ["sqs:read", "sqs:write"],
 }
 perms = set()
 for group in request.user.groups.all():
@@ -204,6 +234,38 @@ permissions_mapping = get_or_create(
 )
 PERMISSIONS_MAPPING_PK = permissions_mapping["pk"]
 # Roles claim for service accounts - for client_credentials flow, the auto-generated
 # user doesn't have roles, so we look up the actual service account by client_id.
 # Format: ["llm:inference", "memory:read", "memory:write"]
 _ROLES_EXPR = """
 from authentik.core.models import User
 # Try user's own roles first (password grant uses the actual user)
 roles = request.user.attributes.get("roles", [])
 # For client_credentials, user is auto-generated - look up by client_id
 if not roles and request.http_request:
    client_id = request.http_request.POST.get("client_id", "")
    if client_id:
        sa_user = User.objects.filter(username=client_id, path="service-accounts").first()
        if sa_user:
            roles = sa_user.attributes.get("roles", [])
 return {"roles": roles}
 """.strip()
 roles_mapping = get_or_create(
    "/api/v3/propertymappings/provider/scope/",
    "/api/v3/propertymappings/provider/scope/",
    "scope_name=roles",
    {
        "name": "homelab: roles claim",
        "scope_name": "roles",
        "expression": _ROLES_EXPR,
    },
    patch_existing={"expression": _ROLES_EXPR},
 )
 ROLES_MAPPING_PK = roles_mapping["pk"]
 # Immich reads a "immich_role" claim on every login (not just user-creation -
 # fixed upstream in immich-app/immich#29991) and syncs isAdmin from it, so
 # this is the actual mechanism that makes "rock" an Immich admin - not
@@ -680,6 +742,10 @@ for name, cfg in SERVICES.items():
 # Redirect URI matches kubelogin's (int128/kubelogin) documented default;
 # adjust here if a different kubectl OIDC plugin/port is actually used.
 print("Ensuring public OAuth2 client 'kubernetes' for kubectl OIDC login...")
 # Fetch explicit-consent flow for public authcode clients (kubelogin)
 status, res = api("GET", "/api/v3/flows/instances/?slug=default-provider-authorization-explicit-consent")
 k8s_authz_flow = res["results"][0]["pk"] if res["results"] else AUTHORIZATION_FLOW_PK
 k8s_provider = get_or_create(
    "/api/v3/providers/oauth2/", "/api/v3/providers/oauth2/",
    "name=kubernetes",
@@ -687,20 +753,20 @@ k8s_provider = get_or_create(
        "name": "kubernetes",
        "client_id": "kubernetes",
        "client_type": "public",
-        "authorization_flow": AUTHORIZATION_FLOW_PK,
+        "authorization_flow": k8s_authz_flow,
        "invalidation_flow": INVALIDATION_FLOW_PK,
        "signing_key": SIGNING_KEY_PK,
        "property_mappings": SCOPE_PKS,
        "sub_mode": "hashed_user_id",
        "include_claims_in_id_token": True,
-        "grant_types": ["authorization_code", "refresh_token"],
+        "grant_types": ["authorization_code", "refresh_token", "password"],
        "redirect_uris": [
            {"matching_mode": "strict", "url": "http://localhost:8000"},
        ],
    },
    patch_existing={
        "property_mappings": SCOPE_PKS,
-        "grant_types": ["authorization_code", "refresh_token"],
+        "grant_types": ["authorization_code", "refresh_token", "password"],
        "redirect_uris": [
            {"matching_mode": "strict", "url": "http://localhost:8000"},
        ],
@@ -720,14 +786,61 @@ app_pks_for_binding.append(("kubernetes", k8s_application["pk"]))
 print(f"  kubernetes: provider pk={k8s_provider['pk']} application pk={k8s_application['pk']}")
 # -----------------------------------------------------------------------------
-# Service accounts for programmatic API access (client_credentials flow).
+# Headless authentication flow for service accounts (password grant).
 # Default flow has MFA/interactive stages that break password grant.
 # This flow: identification -> password -> login (no MFA, no consent prompts).
 print("\n[SERVICE ACCOUNT FLOW] Ensuring headless authentication flow...")
 sa_flow = get_or_create(
    "/api/v3/flows/instances/", "/api/v3/flows/instances/",
    "slug=service-account-authentication",
    {
        "slug": "service-account-authentication",
        "name": "Service Account Authentication (Headless)",
        "title": "Service Account Login",
        "designation": "authentication",
        "policy_engine_mode": "any",
    },
 )
 SA_AUTHENTICATION_FLOW_PK = sa_flow["pk"]
 print(f"  flow pk={SA_AUTHENTICATION_FLOW_PK}")
 # Bind stages to the flow (identification -> password -> login)
 # Get default stages (reuse existing ones)
 status, ident_stages = api("GET", "/api/v3/stages/identification/")
 status, pass_stages = api("GET", "/api/v3/stages/password/")
 status, login_stages = api("GET", "/api/v3/stages/user_login/")
 if ident_stages.get("results") and pass_stages.get("results") and login_stages.get("results"):
    ident_pk = ident_stages["results"][0]["pk"]
    pass_pk = pass_stages["results"][0]["pk"]
    login_pk = login_stages["results"][0]["pk"]
    # Check if bindings already exist
    status, existing_bindings = api("GET", f"/api/v3/flows/bindings/?target={SA_AUTHENTICATION_FLOW_PK}")
    if not existing_bindings.get("results"):
        # Create bindings
        api("POST", "/api/v3/flows/bindings/", {"target": SA_AUTHENTICATION_FLOW_PK, "stage": ident_pk, "order": 10})
        api("POST", "/api/v3/flows/bindings/", {"target": SA_AUTHENTICATION_FLOW_PK, "stage": pass_pk, "order": 20})
        api("POST", "/api/v3/flows/bindings/", {"target": SA_AUTHENTICATION_FLOW_PK, "stage": login_pk, "order": 30})
        print("  bound stages: identification -> password -> login")
    else:
        print(f"  stages already bound ({len(existing_bindings['results'])} bindings)")
 else:
    print("  WARNING: Could not find default stages to bind")
 # -----------------------------------------------------------------------------
 # Service accounts for programmatic API access (password grant).
 # These are Authentik users with type=service_account, not OAuth applications.
-# They authenticate via client_credentials grant to get JWTs with claims.
+# They authenticate via password grant to get JWTs with user claims.
 print("\n[SERVICE ACCOUNTS] Creating service accounts for API access...")
 # Roles define what APIs a service can access - stored in user attributes,
 # output as "roles" claim in JWT. Gateway checks roles, not groups.
 # Format: "<api>:<action>" e.g. "llm:inference", "memory:write"
 SERVICE_ACCOUNTS = {
    "portfolio-agent": {
-        "groups": [],  # Permissions come from SA_CONFIGS in memory scope mapping
+        "roles": ["llm:inference", "memory:read", "s3:read", "sqs:read", "temporal:admin"],
        "attributes": {
            "memory_projects": ["homelab", "portfolio"],
            "memory_visibility": "public",
@@ -736,7 +849,7 @@ SERVICE_ACCOUNTS = {
        "secret_name": "portfolio-agent-oidc",
    },
    "memory-agent": {
-        "groups": [],  # Permissions come from SA_CONFIGS in memory scope mapping
+        "roles": ["llm:inference", "memory:read", "memory:write", "s3:read", "s3:write", "sqs:read", "sqs:write"],
        "attributes": {
            "memory_projects": ["*"],
            "memory_visibility": "private",
@@ -755,36 +868,40 @@ for sa_name, sa_cfg in SERVICE_ACCOUNTS.items():
    if status != 200:
        die(f"GET users for {sa_name} -> {status} {res}")
    # Merge roles into attributes
    sa_attributes = {**sa_cfg["attributes"], "roles": sa_cfg["roles"]}
    if res.get("results"):
        sa_user = res["results"][0]
-        # Update attributes and groups
+        # Update attributes (roles stored here, not in groups)
        group_pks = [service_admin_groups[g]["pk"] for g in sa_cfg["groups"] if g in service_admin_groups]
        status, sa_user = api("PATCH", f"/api/v3/core/users/{sa_user['pk']}/", {
-            "attributes": sa_cfg["attributes"],
+            "attributes": sa_attributes,
            "groups": group_pks,
            "is_active": True,
        })
        if status not in (200, 201):
            die(f"PATCH service account {sa_name} -> {status} {sa_user}")
-        print(f"  {sa_name}: updated existing service account")
+        print(f"  {sa_name}: updated (roles: {sa_cfg['roles']})")
    else:
        # Create new service account user
        group_pks = [service_admin_groups[g]["pk"] for g in sa_cfg["groups"] if g in service_admin_groups]
        status, sa_user = api("POST", "/api/v3/core/users/", {
            "username": sa_name,
            "name": f"Service Account: {sa_name}",
            "type": "service_account",
            "path": "service-accounts",
            "is_active": True,
-            "attributes": sa_cfg["attributes"],
+            "attributes": sa_attributes,
            "groups": group_pks,
        })
        if status not in (200, 201):
            die(f"POST service account {sa_name} -> {status} {sa_user}")
-        print(f"  {sa_name}: created new service account")
+        print(f"  {sa_name}: created (roles: {sa_cfg['roles']})")
-    # Create OAuth provider for this service account (client_credentials flow)
+    print(f"  {sa_name}: roles={sa_cfg['roles']}")
    # Create OAuth provider for this service account
    # Supports both password grant (user claims) and client_credentials (fallback)
    # Password grant requires authentication_flow and app_password token
    sa_client_secret = existing_secret or gen_secret(40)
    sa_grant_types = ["password", "client_credentials", "refresh_token"]
    sa_provider = get_or_create(
        "/api/v3/providers/oauth2/", "/api/v3/providers/oauth2/",
        f"name={sa_name}",
@@ -794,17 +911,19 @@ for sa_name, sa_cfg in SERVICE_ACCOUNTS.items():
            "client_secret": sa_client_secret,
            "client_type": "confidential",
            "authorization_flow": AUTHORIZATION_FLOW_PK,
            "authentication_flow": SA_AUTHENTICATION_FLOW_PK,  # Headless flow for password grant
            "invalidation_flow": INVALIDATION_FLOW_PK,
            "signing_key": SIGNING_KEY_PK,
-            "property_mappings": SCOPE_PKS + [MEMORY_MAPPING_PK],
+            "property_mappings": SCOPE_PKS + [ROLES_MAPPING_PK, MEMORY_MAPPING_PK],
            "sub_mode": "hashed_user_id",
            "include_claims_in_id_token": True,
-            "grant_types": ["client_credentials"],
+            "grant_types": sa_grant_types,
-            "redirect_uris": [],  # No redirects for client_credentials
+            "redirect_uris": [],
        },
        patch_existing={
-            "property_mappings": SCOPE_PKS + [MEMORY_MAPPING_PK],
+            "property_mappings": SCOPE_PKS + [ROLES_MAPPING_PK, MEMORY_MAPPING_PK],
-            "grant_types": ["client_credentials"],
+            "grant_types": sa_grant_types,
            "authentication_flow": SA_AUTHENTICATION_FLOW_PK,
        },
    )
@@ -819,17 +938,70 @@ for sa_name, sa_cfg in SERVICE_ACCOUNTS.items():
            "meta_launch_url": "",
        },
    )
-    app_pks_for_binding.append((sa_name, sa_application["pk"]))
+    
    # Service account apps have NO policy bindings - client_secret is the access control.
    # Clean up any existing bindings (from old provisioning logic).
    status, existing_bindings = api("GET", f"/api/v3/policies/bindings/?target={sa_application['pk']}")
    if status == 200:
        for binding in existing_bindings.get("results", []):
            api("DELETE", f"/api/v3/policies/bindings/{binding['pk']}/")
            print(f"  {sa_name}: removed stale binding {binding.get('group_obj', {}).get('name', binding['pk'])}")
    print(f"  {sa_name}: provider pk={sa_provider['pk']} application pk={sa_application['pk']}")
    # Create App Password token for password grant (optional, client_credentials also works)
    # Authentik's password grant checks against Token with intent=app_password, not user password
    # Note: API doesn't allow setting key at creation, must use set_key endpoint after
    existing_app_password = kubectl_get_secret_key(sa_cfg["secret_ns"], sa_cfg["secret_name"], "APP_PASSWORD")
    if not existing_app_password:
        token_identifier = f"{sa_name}-app-password"
        # Check if token exists
        status, existing_tokens = api("GET", f"/api/v3/core/tokens/?identifier={token_identifier}")
        if status == 200 and existing_tokens.get("results"):
            # Token exists, just set a new key
            pass
        else:
            # Create the token first
            status, token_resp = api("POST", "/api/v3/core/tokens/", {
                "identifier": token_identifier,
                "user": sa_user["pk"],
                "intent": "app_password",
                "expiring": False,
                "description": f"App password for {sa_name} OAuth password grant",
            })
            if status not in (200, 201):
                print(f"  WARNING: Failed to create app password token for {sa_name}: {token_resp}")
        # Set the key via set_key endpoint (works on existing or newly created token)
        app_password_key = gen_secret(40)
        status, _ = api("POST", f"/api/v3/core/tokens/{token_identifier}/set_key/", {
            "key": app_password_key,
        })
        if status not in (200, 204):
            print(f"  WARNING: Failed to set app password key for {sa_name}")
            app_password_key = None
        else:
            print(f"  {sa_name}: created/updated app password token")
    else:
        app_password_key = existing_app_password
        print(f"  {sa_name}: reusing existing app password")
    # Store credentials in k8s Secret
-    if not existing_secret:
+    # Supports both password grant (APP_PASSWORD) and client_credentials (CLIENT_SECRET)
-        kubectl_create_secret(sa_cfg["secret_ns"], sa_cfg["secret_name"], {
+    secret_data = {
        "CLIENT_ID": sa_name,
        "CLIENT_SECRET": sa_client_secret,
        "USERNAME": sa_name,
        "TOKEN_URL": "https://authentik.riotpiao.com/application/o/token/",
        "ISSUER": f"https://authentik.riotpiao.com/application/o/{sa_name}/",
-        })
+    }
-        print(f"  {sa_name}: generated credentials -> {sa_cfg['secret_ns']}/{sa_cfg['secret_name']}")
+    if app_password_key:
        secret_data["APP_PASSWORD"] = app_password_key
    if not existing_secret or (app_password_key and not existing_app_password):
        kubectl_create_secret(sa_cfg["secret_ns"], sa_cfg["secret_name"], secret_data)
        print(f"  {sa_name}: stored credentials -> {sa_cfg['secret_ns']}/{sa_cfg['secret_name']}")
    else:
        print(f"  {sa_name}: reusing existing credentials from {sa_cfg['secret_ns']}/{sa_cfg['secret_name']}")
@@ -35,6 +35,7 @@
    rewrite name longhorn.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    rewrite name paperless.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    rewrite name img.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    rewrite name api.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    rewrite name riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    kubernetes cluster.local in-addr.arpa ip6.arpa {