fix: use explicit-consent flow for kubernetes public OIDC client

- Kubernetes provider now uses default-provider-authorization-explicit-consent instead of implicit-consent flow - Better handling for public clients like kubelogin (authcode grant) - Prevents fallback to password grant prompts
fix: remove spec wrapper from ClusterRoleBinding
2026-09-05 15:09:56 -07:00 · 2026-09-05 15:04:00 -07:00 · 2026-09-05 14:53:59 -07:00 · 2026-09-05 14:48:21 -07:00 · 2026-09-05 14:42:17 -07:00 · 2026-09-05 14:33:27 -07:00
43 changed files with 1425 additions and 1496 deletions
@@ -0,0 +1,6 @@
+apiVersion: v2
+name: memory-queues
+description: Kafka queues (DLQ) for Poimen Memory service (Phase 6.6)
+type: application
+version: 0.1.0
+appVersion: "1.0"
@@ -0,0 +1,20 @@
+{{- range .Values.queues }}
+---
+apiVersion: kmsvc.io/v1alpha1
+kind: Queue
+metadata:
+  name: {{ .name }}
+  namespace: {{ $.Values.namespace }}
+  labels:
+    app: memory-service
+    queue: dlq
+spec:
+  name: {{ .name }}
+  description: {{ .description }}
+  partitions: {{ .partitions }}
+  replicationFactor: {{ .replicationFactor }}
+  config:
+    retention.ms: "{{ .config.retention.ms }}"
+    message.retention.seconds: "{{ .config.message.retention.seconds }}"
+    visibility.timeout.seconds: "{{ .config.visibility.timeout.seconds }}"
+{{- end }}
@@ -0,0 +1,25 @@
+# Poimen Memory Service Kafka Queues (kmsvc)
+# Phase 6.6: DLQ topics for webhook + metrics failures
+
+queues:
+  # DLQ for extraction, webhook, and agent failures
+  - name: poimen-memory-dlq
+    description: "DLQ for extraction, webhook, and agent failures"
+    partitions: 3
+    replicationFactor: 1
+    config:
+      retention.ms: "1209600000"  # 14 days
+      message.retention.seconds: "1209600"
+      visibility.timeout.seconds: "300"
+
+  # DLQ for metrics persistence failures  
+  - name: poimen-memory-metric-dlq
+    description: "DLQ for metrics persistence failures"
+    partitions: 3
+    replicationFactor: 1
+    config:
+      retention.ms: "1209600000"  # 14 days
+      message.retention.seconds: "1209600"
+      visibility.timeout.seconds: "300"
+
+namespace: sqs
@@ -0,0 +1,144 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: secretrotations.homelab.riotpiao.com
+spec:
+  group: homelab.riotpiao.com
+  names:
+    kind: SecretRotation
+    plural: secretrotations
+  scope: Namespaced
+  versions:
+  - name: v1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        properties:
+          metadata:
+            type: object
+          spec:
+            type: object
+            required:
+            - provider
+            - rotationInterval
+            properties:
+              # External system: authentik | forgejo | minio | vault
+              provider:
+                type: string
+                enum: [authentik, forgejo, minio, vault]
+              
+              # How often to rotate (hours)
+              rotationInterval:
+                type: integer
+                minimum: 24
+              
+              # Application ID in external system
+              appId:
+                type: string
+              
+              # k8s Secret to update (name, namespace, key)
+              secretRef:
+                type: object
+                required: [name, namespace]
+                properties:
+                  name:
+                    type: string
+                  namespace:
+                    type: string
+                  key:
+                    type: string
+                    description: "Secret key to update (e.g., MINIO_IDENTITY_OPENID_CLIENT_SECRET)"
+              
+              # Path to git file that holds the secret (for .enc.yaml files)
+              gitPath:
+                type: string
+                description: "Path in homelab repo to .enc.yaml file"
+              
+              # Ansible template values to substitute
+              templateValues:
+                type: object
+                additionalProperties:
+                  type: string
+          
+          status:
+            type: object
+            properties:
+              lastRotationTime:
+                type: string
+                format: date-time
+              nextRotationTime:
+                type: string
+                format: date-time
+              lastRotationStatus:
+                type: string
+                enum: [Success, Failed, Pending]
+              lastRotationError:
+                type: string
+              lastCommitHash:
+                type: string
+
+---
+# Example usage:
+apiVersion: homelab.riotpiao.com/v1
+kind: SecretRotation
+metadata:
+  name: minio-oidc
+  namespace: secret-rotation
+spec:
+  provider: authentik
+  rotationInterval: 2160  # 90 days in hours
+  appId: minio
+  secretRef:
+    name: minio-oidc
+    namespace: storage
+    key: MINIO_IDENTITY_OPENID_CLIENT_SECRET
+  gitPath: k8s/argocd/secrets/minio-oidc.enc.yaml
+
+---
+apiVersion: homelab.riotpiao.com/v1
+kind: SecretRotation
+metadata:
+  name: portfolio-agent-oidc
+  namespace: secret-rotation
+spec:
+  provider: authentik
+  rotationInterval: 2160
+  appId: portfolio-agent
+  secretRef:
+    name: portfolio-agent-oidc
+    namespace: portfolio
+    key: CLIENT_SECRET
+  gitPath: k8s/argocd/secrets/portfolio-agent-oidc.enc.yaml
+
+---
+apiVersion: homelab.riotpiao.com/v1
+kind: SecretRotation
+metadata:
+  name: forgejo-registry-token
+  namespace: secret-rotation
+spec:
+  provider: forgejo
+  rotationInterval: 2160
+  appId: rock/riotpiao.com
+  secretRef:
+    name: forgejo-registry-secret
+    namespace: kube-system
+    key: REGISTRY_TOKEN
+  gitPath: k8s/argocd/secrets/forgejo-registry-secret.enc.yaml
+
+---
+apiVersion: homelab.riotpiao.com/v1
+kind: SecretRotation
+metadata:
+  name: minio-root-credentials
+  namespace: secret-rotation
+spec:
+  provider: minio
+  rotationInterval: 4320  # 180 days in hours
+  appId: root
+  secretRef:
+    name: minio-creds
+    namespace: storage
+  gitPath: k8s/argocd/secrets/minio-secrets.enc.yaml
@@ -0,0 +1,92 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: secret-rotation-controller
+  namespace: secret-rotation
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: secret-rotation-controller
+  template:
+    metadata:
+      labels:
+        app: secret-rotation-controller
+    spec:
+      serviceAccountName: secret-rotation-controller
+      containers:
+      - name: controller
+        image: secret-rotation-controller:latest
+        imagePullPolicy: IfNotPresent
+        env:
+        # SOPS reads age key from this file
+        - name: SOPS_AGE_KEY_FILE
+          value: /etc/sops/age/private-key.txt
+        
+        # Vault auth (token in projected volume)
+        - name: VAULT_ADDR
+          value: http://vault.vault.svc.cluster.local:8200
+        - name: VAULT_TOKEN_FILE
+          value: /var/run/secrets/vault/token
+        
+        # Authentik
+        - name: AUTHENTIK_URL
+          value: http://authentik-server.iam.svc.cluster.local
+        - name: AUTHENTIK_BOOTSTRAP_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: authentik-bootstrap
+              key: token
+        
+        # Git
+        - name: GIT_REPO
+          value: https://forgejo.riotpiao.com/rock/homelab.git
+        - name: GIT_AUTHOR_EMAIL
+          value: [email protected]
+        - name: GIT_AUTHOR_NAME
+          value: Secret Rotation Controller
+        - name: FORGEJO_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: forgejo-registry-secret
+              key: REGISTRY_TOKEN
+        
+        volumeMounts:
+        # Age key from ExternalSecret (synced from Vault)
+        - name: age-key
+          mountPath: /etc/sops/age
+          readOnly: true
+        
+        # Vault auth token (projected)
+        - name: vault-token
+          mountPath: /var/run/secrets/vault
+          readOnly: true
+        
+        # Temp working dir
+        - name: tmp
+          mountPath: /tmp
+        
+        resources:
+          requests:
+            cpu: 100m
+            memory: 256Mi
+          limits:
+            cpu: 500m
+            memory: 512Mi
+      
+      volumes:
+      - name: age-key
+        secret:
+          secretName: sops-age-key
+          defaultMode: 0400
+      
+      - name: vault-token
+        projected:
+          sources:
+          - serviceAccountToken:
+              path: token
+              audience: vault
+              expirationSeconds: 3600
+      
+      - name: tmp
+        emptyDir: {}
@@ -0,0 +1,15 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: secret-rotation
+
+resources:
+  - rbac.yaml
+  - crd.yaml
+  - external-secret.yaml
+  - deployment.yaml
+
+commonLabels:
+  app.kubernetes.io/name: secret-rotation-controller
+  app.kubernetes.io/component: automation
+  managed-by: argocd
@@ -0,0 +1,53 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: secret-rotation-controller
+  namespace: secret-rotation
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: secret-rotation-controller
+rules:
+# Read SecretRotation CRDs
+- apiGroups: ["homelab.riotpiao.com"]
+  resources: ["secretrotations"]
+  verbs: ["get", "list", "watch"]
+
+# Update status
+- apiGroups: ["homelab.riotpiao.com"]
+  resources: ["secretrotations/status"]
+  verbs: ["get", "patch", "update"]
+
+# Read k8s secrets that will be rotated
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["get", "list"]
+
+# For recording events
+- apiGroups: [""]
+  resources: ["events"]
+  verbs: ["create", "patch"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: secret-rotation-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: secret-rotation-controller
+subjects:
+- kind: ServiceAccount
+  name: secret-rotation-controller
+  namespace: secret-rotation
+
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: secret-rotation
+  labels:
+    kubernetes.io/metadata.name: secret-rotation
@@ -0,0 +1,32 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: secret-rotation
+  namespace: argocd
+  labels:
+    app.kubernetes.io/name: secret-rotation
+spec:
+  project: homelab
+  
+  sources:
+  - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    path: k8s/apps/secret-rotation-controller
+    targetRevision: main
+  
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: secret-rotation
+  
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+    - CreateNamespace=true
+    - RespectIgnoreDifferences=true
+    retry:
+      limit: 5
+      backoff:
+        duration: 5s
+        factor: 2
+        maxDuration: 3m
@@ -0,0 +1,20 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: memory-queues
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "7"
+spec:
+  project: homelab
+  source:
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    targetRevision: main
+    path: k8s/apps/messaging/memory-queues
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: sqs
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
@@ -286,7 +286,8 @@ spec:
    path: k8s/infra/rbac
  destination:
    server: https://kubernetes.default.svc
-    namespace: default
+    # No namespace: cluster-scoped resources (ClusterRoleBinding, etc.)
+    # Namespace is set per-resource in kustomization
  syncPolicy:
    automated:
      prune: true
@@ -1,27 +1,35 @@
-# Poimen project collection — manages poimen-memory, poimen-workflows, and poiman
-# Each repo tracks its own main branch (no prod branch). Poiman is the primary
-# orchestrator with k8s/argocd/ containing the AppProject and deployment structure.
-#
-# CI: All three repos trigger on main branch pushes (no image builds yet).
-# Future: Add build workflows for poiman once container runtime needs are clear.
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
-  name: poimen-root
+  name: poimen
  namespace: argocd
-  labels:
-    app.kubernetes.io/name: poimen
-    app.kubernetes.io/component: orchestrator
  annotations:
    argocd.argoproj.io/sync-wave: "7"
+    # Image Updater: auto-update on new image push (SHA tag filter)
+    argocd-image-updater.argoproj.io/image-list: |
+      memory=forgejo.riotpiao.com/rock/poimen-memory
+      workflows=forgejo.riotpiao.com/rock/poimen-workflows
+      frontend=forgejo.riotpiao.com/rock/poimen-frontend
+    argocd-image-updater.argoproj.io/memory.update-strategy: newest-build
+    argocd-image-updater.argoproj.io/memory.allow-tags: regexp:^[0-9a-f]{7}$
+    argocd-image-updater.argoproj.io/workflows.update-strategy: newest-build
+    argocd-image-updater.argoproj.io/workflows.allow-tags: regexp:^[0-9a-f]{7}$
+    argocd-image-updater.argoproj.io/frontend.update-strategy: newest-build
+    argocd-image-updater.argoproj.io/frontend.allow-tags: regexp:^[0-9a-f]{7}$
+    argocd-image-updater.argoproj.io/write-back-method: git
+    argocd-image-updater.argoproj.io/git-branch: main
 spec:
  project: homelab
-  source:
-    repoURL: https://forgejo.riotpiao.com/rock/poimen.git
-    targetRevision: main
-    path: k8s/argocd
-    directory:
-      recurse: false
+  sources:
+    - repoURL: https://forgejo.riotpiao.com/rock/poimen-memory.git
+      targetRevision: main
+      path: k8s/argocd
+    - repoURL: https://forgejo.riotpiao.com/rock/poimen-workflows.git
+      targetRevision: main
+      path: k8s/argocd
+    - repoURL: https://forgejo.riotpiao.com/rock/poimen-frontend.git
+      targetRevision: main
+      path: k8s/argocd
  destination:
    server: https://kubernetes.default.svc
    namespace: poimen
@@ -20,10 +20,47 @@ config:
    insecure: true
    plaintext: true

+  # Git write-back configuration (for multi-source Applications)
+  git:
+    # Commit author for image updates
+    user:
+      name: "ArgoCD Image Updater"
+      email: "[email protected]"
+    # Use SSH keys from ArgoCD's known hosts + credentials
+    # Image Updater inherits ArgoCD's git credentials (mounted via ArgoCD secret)
+
+# Mount ArgoCD's git credentials for write-back
+extraVolumes:
+  - name: argocd-ssh-known-hosts-cm
+    configMap:
+      name: argocd-ssh-known-hosts-cm
+      defaultMode: 0644
+  - name: argocd-gpg-keys-cm
+    configMap:
+      name: argocd-gpg-keys-cm
+      optional: true
+      defaultMode: 0644
+  - name: argocd-gpg-pubring
+    configMap:
+      name: argocd-gpg-pubring-cm
+      optional: true
+      defaultMode: 0644
+
+extraVolumeMounts:
+  - name: argocd-ssh-known-hosts-cm
+    mountPath: /etc/ssh/ssh_known_hosts.d/argocd-ssh-known-hosts
+    subPath: ssh_known_hosts
+  - name: argocd-gpg-keys-cm
+    mountPath: /etc/gpg/source
+  - name: argocd-gpg-pubring
+    mountPath: /etc/gpg/pubring
+
 # Extra environment variables
 extraEnv:
  - name: ARGOCD_GRPC_WEB
    value: "true"
+  - name: GIT_SSH_KNOWN_HOSTS_CONFIG_MAP_ENABLED
+    value: "true"

 # Resources
 resources:
@@ -2,8 +2,8 @@
 # runner instance. Only runner.name and runner.labels differ -- everything
 # else (image, dind, persistence, tolerations, nodeSelector) is shared.
 #
-# node:22-bookworm ships Node natively, so unlike the golang/rust instances,
-# jobs on this runner need no "install node" step before actions/checkout.
+# node:22-bookworm ships Node natively. Docker client installed via workflow step if needed.
+# (homelab has no CI; custom runner images built manually if desired)
 runner:
  name: node-runner
  labels: "node:docker://node:22-bookworm"
@@ -61,6 +61,6 @@ gc:
  enabled: true
  schedule: "*/30 * * * *"     # every 30 minutes
  image: alpine/k8s:1.31.0
-  pruneAge: "72h"              # Docker artifacts unused longer than this get pruned
-  pruneAgeHours: 72             # Same as pruneAge but numeric for date arithmetic in shell
+  pruneAge: "30m"              # Docker artifacts unused longer than this get pruned
+  pruneAgeHours: 0.5            # Same as pruneAge but numeric for date arithmetic in shell
  actcacheMaxAgeDays: 1        # actcache files older than N days (aggressive for heavy Rust cargo builds)
@@ -1,252 +0,0 @@
-# Authentik OAuth provisioning — MANUAL operation, NOT auto-run.
-#
-# Security-sensitive IAM changes should be reviewed and run locally:
-#   export AUTHENTIK_BOOTSTRAP_TOKEN=$(kubectl -n iam get secret authentik-secrets \
-#     -o jsonpath='{.data.AUTHENTIK_BOOTSTRAP_TOKEN}' | base64 -d)
-#   sed 's|http://authentik-server.iam.svc.cluster.local|https://authentik.riotpiao.com|g' \
-#     k8s/infra/iam/scripts/authentik-provision.py | python3
-#
-# What it does (see scripts/authentik-provision.py docstring): creates scope
-# mappings (groups, permissions, memory, policy, immich_role), admin groups,
-# the "rock" admin user, OAuth2 providers + Applications, service accounts
-# (portfolio-agent, memory-agent), and binds groups to applications.
-#
-# This file provides the RBAC (ServiceAccount + RoleBindings) needed if you
-# ever want to run the Job in-cluster manually (kubectl create -f), but the
-# PostSync hook is deliberately removed — IAM is not GitOps-auto-deployed.
-#
-# RBAC: this Job only touches Secrets (get existing client secrets, create new
-# ones) across the namespaces those services live in.
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: authentik-provisioner
-  namespace: iam
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: authentik-provisioner
-rules:
-  - apiGroups: [""]
-    resources: ["secrets"]
-    verbs: ["get", "list", "create", "update", "patch"]
---
-# One RoleBinding per namespace the script touches (least-privilege: Secrets
-# only, and only in these 5 namespaces — not a cluster-wide ClusterRoleBinding).
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: iam
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: cicd
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: argocd
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: logging
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: storage
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: paperless
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: immich
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: llm-serving
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: portfolio
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: poimen
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: dashboard
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
---
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: authentik-provision
-  namespace: iam
-  # No ArgoCD hook - run manually when IAM changes are needed
-spec:
-  ttlSecondsAfterFinished: 600
-  backoffLimit: 3
-  template:
-    spec:
-      serviceAccountName: authentik-provisioner
-      restartPolicy: Never
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 1000
-        seccompProfile:
-          type: RuntimeDefault
-      containers:
-        - name: provision
-          image: python:3.12-alpine
-          securityContext:
-            allowPrivilegeEscalation: false
-            capabilities:
-              drop: ["ALL"]
-          env:
-            - name: AUTHENTIK_BOOTSTRAP_TOKEN
-              valueFrom:
-                secretKeyRef:
-                  name: authentik-secrets
-                  key: AUTHENTIK_BOOTSTRAP_TOKEN
-          volumeMounts:
-            - name: script
-              mountPath: /script
-          command:
-            - /bin/sh
-            - -c
-            - |
-              set -e
-              echo "waiting for authentik-server..."
-              until wget -q -O /dev/null http://authentik-server.iam.svc.cluster.local/-/health/ready/ 2>/dev/null; do
-                sleep 5
-              done
-              echo "installing kubectl (via python urllib - no apk/curl: this"
-              echo "container runs as non-root UID 1000 and can't write to"
-              echo "apk's directories or /usr/local/bin, both root-owned in"
-              echo "the python:3.12-alpine image; /tmp is world-writable)..."
-              python3 -c "
-              import urllib.request, os, stat
-              kver = urllib.request.urlopen('https://dl.k8s.io/release/stable.txt').read().decode().strip()
-              url = f'https://dl.k8s.io/release/{kver}/bin/linux/amd64/kubectl'
-              urllib.request.urlretrieve(url, '/tmp/kubectl')
-              st = os.stat('/tmp/kubectl')
-              os.chmod('/tmp/kubectl', st.st_mode | stat.S_IEXEC)
-              "
-              export PATH="/tmp:$PATH"
-              echo "running provisioning script..."
-              python3 /script/authentik-provision.py
-      volumes:
-        - name: script
-          configMap:
-            name: authentik-provision-script
@@ -1,35 +1,12 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-# NOTE: no top-level `namespace:` transformer here (removed) - it used to
-# force-rewrite metadata.namespace to "iam" on every resource in this
-# kustomization, which was harmless while every manifest here only ever
-# targeted the iam namespace itself. authentik-provision-job.yaml's
-# RoleBindings deliberately target cicd/argocd/logging/storage (least-
-# privilege access for the authentik-provisioner ServiceAccount to touch
-# Secrets in those namespaces) - the namespace transformer would have
-# silently rewritten all of them back to iam, breaking the RBAC. Every
-# manifest in this directory already sets its own explicit
-# metadata.namespace, so dropping the transformer changes nothing for the
-# existing resources/.
+
 resources:
-  - authentik-provision-job.yaml
  - rbac-dashboard-rolebinding.yaml

-# Provisioning/verification python lives in scripts/*.py (real files, linted +
-# diff-friendly) and is generated into ConfigMaps here rather than embedded in
-# the job YAML. disableNameSuffixHash keeps the names stable so the Jobs'
-# configMap volume refs and PostSync hook-delete semantics keep working; each
-# hook Job is recreated per sync so it always mounts the latest script.
-configMapGenerator:
-  - name: authentik-provision-script
-    namespace: iam
-    files:
-      - authentik-provision.py=scripts/authentik-provision.py
-
-generatorOptions:
-  disableNameSuffixHash: true
-# authentik-migrations-job.yaml removed — redundant + broken. The authentik
-# `server` entrypoint runs migrations itself; this standalone job lacked the
-# authentik-secrets envFrom (Secret key missing) and always failed.
-# SOPS secrets (*.enc.yaml) handled by ArgoCD SOPS plugin at sync time
-# authentik/vault deployed via ArgoCD Helm source
+# IAM provisioning is manual-only (security-sensitive).
+# Script: scripts/iam/authentik-provision.py
+# Run:
+#   export AUTHENTIK_BOOTSTRAP_TOKEN=$(kubectl -n iam get secret authentik-secrets \
+#     -o jsonpath='{.data.AUTHENTIK_BOOTSTRAP_TOKEN}' | base64 -d)
+#   python3 scripts/iam/authentik-provision.py
@@ -68,15 +68,14 @@ grafana.ini:
  # doesn't return localhost redirects in its token responses.
  #
  # role_attribute_path: JMESPath expression evaluated against the userinfo
-  # response. Members of the 'grafana-admins' Authentik group get Admin role;
-  # everyone else gets Viewer. The group name must match exactly what Authentik
-  # sends in the 'groups' claim.
+  # response. akadmin gets GrafanaAdmin (server admin, can impersonate);
+  # homelab-admins members get Admin (org admin); everyone else Viewer.
  auth.generic_oauth:
    enabled: true
    name: Authentik
    allow_sign_up: true
    client_id: grafana
-    scopes: openid email profile
+    scopes: openid email profile groups
    auth_url: https://authentik.riotpiao.com/application/o/authorize/
    token_url: https://authentik.riotpiao.com/application/o/token/
    api_url: https://authentik.riotpiao.com/application/o/userinfo/
@@ -87,7 +86,8 @@ grafana.ini:
    email_attribute_path: email
    login_attribute_path: preferred_username
    name_attribute_path: name
-    role_attribute_path: "contains(groups[*], 'grafana-admins') && 'Admin' || 'Viewer'"
+    role_attribute_path: "preferred_username == 'akadmin' && 'GrafanaAdmin' || contains(groups[*], 'homelab-admins') && 'Admin' || 'Viewer'"
+    allow_assign_grafana_admin: true
    use_pkce: false
    use_refresh_token: false
    skip_org_role_sync: false
@@ -96,8 +96,13 @@ spec:
      console: https://minio.riotpiao.com

  # ── OIDC via Authentik (server-side env, valid in v2 schema) ────────────────
+  # Use in-cluster URL for config fetch (pod→authentik); browser redirects use
+  # public URLs embedded in the OIDC metadata response (issuer stays public).
  env:
    - name: MINIO_IDENTITY_OPENID_CONFIG_URL
+      # Must use external URL — well-known response contains external issuer/jwks_uri.
+      # MinIO validates issuer in JWT matches well-known issuer. Internal URL = mismatch.
+      # Hairpins through ingress-nginx but stays in-cluster.
      value: "https://authentik.riotpiao.com/application/o/minio/.well-known/openid-configuration"
    - name: MINIO_IDENTITY_OPENID_CLIENT_ID
      value: "minio"
@@ -0,0 +1,160 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: api-gateway-alerts
+  namespace: monitoring
+  labels:
+    release: prometheus
+spec:
+  groups:
+    # ================================================================
+    # SLA Targets (based on canary traffic baselines):
+    #
+    # Availability:  99.9% (43.8 min downtime/month)
+    # LLM Chat:      p95 < 1s (qwen), p95 < 2s (reasoning), p95 < 5s (ornith)
+    # Embeddings:    p95 < 500ms
+    # Rerank:        p95 < 500ms
+    # Models list:   p95 < 300ms
+    # Error rate:    < 1% (5xx), < 5% (4xx excluding auth)
+    #
+    # Baselines from 200-request canary run:
+    #   qwen p99=609ms, reasoning p99=328ms, embeddings p99=287ms,
+    #   rerank p99=218ms, models p99=277ms
+    # SLA set at ~2x p99 for headroom.
+    # ================================================================
+
+    - name: api-gateway.availability
+      rules:
+        # Gateway pods not ready
+        - alert: APIGatewayDown
+          expr: sum(kube_pod_status_ready{namespace="api",condition="true"}) == 0
+          for: 1m
+          labels:
+            severity: critical
+          annotations:
+            summary: "API Gateway has zero ready pods"
+
+        # Gateway pod count below desired
+        - alert: APIGatewayDegraded
+          expr: |
+            sum(kube_pod_status_ready{namespace="api",condition="true"})
+            < kube_deployment_spec_replicas{namespace="api",deployment="api-gateway"}
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "API Gateway {{ $value }} ready pods below desired replica count"
+
+        # Blackbox probe down
+        - alert: APIGatewayProbeDown
+          expr: probe_success{instance=~".*api.riotpiao.com.*"} == 0
+          for: 2m
+          labels:
+            severity: critical
+          annotations:
+            summary: "API Gateway probe failed: {{ $labels.instance }}"
+
+        # LLM serving pods not ready
+        - alert: LLMServingDown
+          expr: sum(kube_pod_status_ready{namespace="llm-serving",condition="true"}) == 0
+          for: 2m
+          labels:
+            severity: critical
+          annotations:
+            summary: "All LLM serving pods down"
+
+        # Individual predictor down
+        - alert: LLMPredictorDown
+          expr: |
+            kube_deployment_status_replicas_ready{namespace="llm-serving"}
+            < kube_deployment_spec_replicas{namespace="llm-serving"}
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "{{ $labels.deployment }} has {{ $value }} ready (below desired)"
+
+    - name: api-gateway.latency
+      # SLA: latency thresholds at ~2x measured p99
+      rules:
+        # Ingress-level latency (all requests through nginx)
+        - alert: APIGatewayLatencyHigh
+          expr: |
+            histogram_quantile(0.95,
+              sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le)
+            ) > 2
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "API Gateway p95 latency {{ $value | printf \"%.1f\" }}s (SLA: <2s)"
+
+        # Extreme latency (p99 > 5s)
+        - alert: APIGatewayLatencyCritical
+          expr: |
+            histogram_quantile(0.99,
+              sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le)
+            ) > 5
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "API Gateway p99 latency {{ $value | printf \"%.1f\" }}s (SLA: <5s)"
+
+    - name: api-gateway.errors
+      rules:
+        # 5xx error rate > 1%
+        - alert: APIGateway5xxErrorRate
+          expr: |
+            sum(rate(nginx_ingress_controller_requests{ingress="api",status=~"5.."}[5m]))
+            / sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m]))
+            > 0.01
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "API Gateway 5xx rate {{ $value | humanizePercentage }} (SLA: <1%)"
+
+        # Total error rate > 10% (including 4xx)
+        - alert: APIGatewayHighErrorRate
+          expr: |
+            sum(rate(nginx_ingress_controller_requests{ingress="api",status=~"[45].."}[5m]))
+            / sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m]))
+            > 0.10
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "API Gateway total error rate {{ $value | humanizePercentage }} (SLA: <10%)"
+
+    - name: api-gateway.resources
+      rules:
+        # Gateway pod restart
+        - alert: APIGatewayRestarted
+          expr: increase(kube_pod_container_status_restarts_total{namespace="api"}[15m]) > 0
+          for: 0m
+          labels:
+            severity: warning
+          annotations:
+            summary: "API Gateway pod {{ $labels.pod }} restarted"
+
+        # LLM predictor restart
+        - alert: LLMPredictorRestarted
+          expr: increase(kube_pod_container_status_restarts_total{namespace="llm-serving"}[15m]) > 0
+          for: 0m
+          labels:
+            severity: warning
+          annotations:
+            summary: "LLM predictor {{ $labels.pod }} restarted"
+
+        # Gateway high memory (>80% of limit)
+        - alert: APIGatewayHighMemory
+          expr: |
+            sum(container_memory_working_set_bytes{namespace="api",container="gateway"}) by (pod)
+            / sum(kube_pod_container_resource_limits{namespace="api",container="gateway",resource="memory"}) by (pod)
+            > 0.8
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Gateway pod {{ $labels.pod }} memory at {{ $value | humanizePercentage }} of limit"
@@ -0,0 +1,177 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: cluster-alerts
+  namespace: monitoring
+  labels:
+    release: prometheus
+spec:
+  groups:
+    - name: cluster.availability
+      rules:
+        # Node down
+        - alert: NodeNotReady
+          expr: kube_node_status_condition{condition="Ready",status="true"} == 0
+          for: 2m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Node {{ $labels.node }} is NotReady"
+
+        # Pod stuck pending (scheduling failure)
+        - alert: PodStuckPending
+          expr: sum(kube_pod_status_phase{phase="Pending"}) > 0
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "{{ $value }} pod(s) stuck in Pending state for >10m"
+
+        # CrashLoopBackOff
+        - alert: PodCrashLooping
+          expr: sum(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}) by (namespace, pod) > 0
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "{{ $labels.namespace }}/{{ $labels.pod }} in CrashLoopBackOff"
+
+        # OOMKilled spike
+        - alert: OOMKilledSpike
+          expr: sum(increase(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[1h])) > 3
+          for: 0m
+          labels:
+            severity: warning
+          annotations:
+            summary: "{{ $value }} OOMKilled events in last hour"
+
+        # Deployment replicas unavailable
+        - alert: DeploymentReplicasUnavailable
+          expr: kube_deployment_status_replicas_unavailable > 0
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "{{ $labels.namespace }}/{{ $labels.deployment }} has {{ $value }} unavailable replicas"
+
+    - name: cluster.jobs
+      rules:
+        # Job failed
+        - alert: JobFailed
+          expr: kube_job_status_failed > 0
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Job {{ $labels.namespace }}/{{ $labels.job_name }} failed"
+
+        # Job stuck running >2h
+        - alert: JobStuckRunning
+          expr: |
+            kube_job_status_active == 1
+            and on(job_name,namespace)
+            (time() - kube_job_status_start_time) > 7200
+          for: 0m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Job {{ $labels.namespace }}/{{ $labels.job_name }} running >2h"
+
+        # CronJob missed schedule
+        - alert: CronJobMissedSchedule
+          expr: |
+            (time() - kube_cronjob_status_last_schedule_time) > 2 * (kube_cronjob_spec_next_schedule_time - kube_cronjob_status_last_schedule_time)
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} missed schedule"
+
+    - name: cluster.resources
+      rules:
+        # Node CPU >90% sustained
+        - alert: NodeHighCPU
+          expr: (1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m])) by (instance)) * 100 > 90
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Node {{ $labels.instance }} CPU at {{ $value | printf \"%.0f\" }}%"
+
+        # Node memory >90% sustained
+        - alert: NodeHighMemory
+          expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 90
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Node {{ $labels.instance }} memory at {{ $value | printf \"%.0f\" }}%"
+
+        # Node disk >85%
+        - alert: NodeDiskFull
+          expr: (1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 > 85
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Node {{ $labels.instance }} disk at {{ $value | printf \"%.0f\" }}%"
+
+        # Container restart storm (>5 restarts in 15m)
+        - alert: ContainerRestartStorm
+          expr: sum(increase(kube_pod_container_status_restarts_total[15m])) by (namespace, pod) > 5
+          for: 0m
+          labels:
+            severity: warning
+          annotations:
+            summary: "{{ $labels.namespace }}/{{ $labels.pod }} restarted {{ $value | printf \"%.0f\" }} times in 15m"
+
+    - name: cluster.storage
+      rules:
+        # Longhorn drive offline
+        - alert: LonghornDriveOffline
+          expr: longhorn_disk_health != 1
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Longhorn disk {{ $labels.node }} unhealthy"
+
+    - name: cluster.dns
+      rules:
+        # CoreDNS errors spike
+        - alert: CoreDNSErrorSpike
+          expr: sum(rate(coredns_dns_responses_total{rcode=~"SERVFAIL"}[5m])) > 0.5
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "CoreDNS SERVFAIL rate {{ $value | printf \"%.2f\" }}/s"
+
+    - name: cluster.probes
+      rules:
+        # Any blackbox probe down
+        - alert: ServiceProbeDown
+          expr: probe_success == 0
+          for: 3m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Probe failed: {{ $labels.instance }}"
+
+        # Probe latency >2s
+        - alert: ServiceProbeSlow
+          expr: probe_duration_seconds > 2
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Probe slow ({{ $value | printf \"%.1f\" }}s): {{ $labels.instance }}"
+
+        # Certificate expiry <14 days
+        - alert: CertificateExpiringSoon
+          expr: (certmanager_certificate_expiration_timestamp_seconds - time()) / 86400 < 14
+          for: 0m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Certificate {{ $labels.name }} expires in {{ $value | printf \"%.0f\" }} days"
@@ -61,6 +61,10 @@ serviceMonitor:
      url: https://argocd.riotpiao.com/healthz
    - name: longhorn
      url: https://longhorn.riotpiao.com/
+    - name: api-gateway
+      url: https://api.riotpiao.com/healthz
+    - name: api-gateway-models
+      url: https://api.riotpiao.com/v1/models

 prometheusRule:
  enabled: true
@@ -0,0 +1,36 @@
+apiVersion: v1
+data:
+  api-gateway.json: '{"title":"API Gateway","uid":"api-gateway","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","tags":["api","gateway","llm"],"panels":[{"id":1,"title":"Gateway
+    Health","type":"row","collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Gateway
+    Pods Ready","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"red"},{"value":3,"color":"green"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"api\",condition=\"true\"})"}]},{"id":3,"title":"Probe:
+    healthz","type":"stat","gridPos":{"h":4,"w":4,"x":4,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}]}},"targets":[{"expr":"probe_success{instance=~\".*api.riotpiao.com/healthz\"}"}]},{"id":4,"title":"Probe
+    Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"probe_duration_seconds{instance=~\".*api.riotpiao.com.*\"}","legendFormat":"{{instance}}"}]}]},{"id":10,"title":"Ingress
+    Traffic (nginx)","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"Request
+    Rate by Status","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(nginx_ingress_controller_requests{ingress=\"api\"}[5m]))
+    by (status)","legendFormat":"{{status}}"}]},{"id":12,"title":"Error Rate %","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percent"}},"targets":[{"expr":"sum(rate(nginx_ingress_controller_requests{ingress=\"api\",status=~\"5..\"}[5m]))
+    / sum(rate(nginx_ingress_controller_requests{ingress=\"api\"}[5m])) * 100","legendFormat":"5xx"},{"expr":"sum(rate(nginx_ingress_controller_requests{ingress=\"api\",status=~\"4..\"}[5m]))
+    / sum(rate(nginx_ingress_controller_requests{ingress=\"api\"}[5m])) * 100","legendFormat":"4xx"}]},{"id":13,"title":"Latency
+    p50/p95/p99","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.50,
+    sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"api\"}[5m]))
+    by (le))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"api\"}[5m]))
+    by (le))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"api\"}[5m]))
+    by (le))","legendFormat":"p99"}]}]},{"id":20,"title":"LLM Serving","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"LLM
+    Pods Ready","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"llm-serving\",condition=\"true\"})"}]},{"id":22,"title":"CPU
+    by Predictor","type":"timeseries","gridPos":{"h":8,"w":8,"x":4,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"llm-serving\"}[5m]))
+    by (pod)","legendFormat":"{{pod}}"}]},{"id":23,"title":"Memory by Predictor","type":"timeseries","gridPos":{"h":8,"w":8,"x":12,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"bytes"}},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"llm-serving\"})
+    by (pod)","legendFormat":"{{pod}}"}]},{"id":24,"title":"Predictor Restarts","type":"timeseries","gridPos":{"h":8,"w":4,"x":20,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"llm-serving\"}[15m]))
+    by (pod)","legendFormat":"{{pod}}"}]}]},{"id":30,"title":"Gateway Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"CPU
+    by Gateway Pod","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"api\"}[5m]))
+    by (pod)","legendFormat":"{{pod}}"}]},{"id":32,"title":"Memory by Gateway Pod","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"bytes"}},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"api\"})
+    by (pod)","legendFormat":"{{pod}}"}]},{"id":33,"title":"Gateway Restarts","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"api\"}[15m]))
+    by (pod)","legendFormat":"{{pod}}"}]}]},{"id":40,"title":"Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":4},"panels":[{"id":41,"title":"Gateway
+    Logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":5},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"api\",container=\"gateway\"}"}]},{"id":42,"title":"LLM
+    Serving Logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":15},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"llm-serving\"}"}]}]}]}'
+kind: ConfigMap
+metadata:
+  annotations:
+    grafana_folder: API
+  labels:
+    grafana_dashboard: '1'
+  name: api-gateway-dashboard
+  namespace: logging
@@ -0,0 +1,62 @@
+apiVersion: v1
+data:
+  cluster-infrastructure.json: '{"title":"Cluster Infrastructure","uid":"cluster-infra","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","tags":["infrastructure","k8s"],"panels":[{"id":1,"title":"Cluster
+    Health","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Nodes
+    Ready","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"red"},{"value":3,"color":"green"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"count(kube_node_status_condition{condition=\"Ready\",status=\"true\"}
+    == 1)"}]},{"id":3,"title":"Pods Pending","type":"stat","gridPos":{"h":4,"w":4,"x":4,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"sum(kube_pod_status_phase{phase=\"Pending\"})
+    OR on() vector(0)"}]},{"id":4,"title":"CrashLoopBackOff","type":"stat","gridPos":{"h":4,"w":4,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"sum(kube_pod_container_status_waiting_reason{reason=\"CrashLoopBackOff\"})
+    OR on() vector(0)"}]},{"id":5,"title":"OOMKilled (1h)","type":"stat","gridPos":{"h":4,"w":4,"x":12,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"sum(increase(kube_pod_container_status_last_terminated_reason{reason=\"OOMKilled\"}[1h]))
+    OR on() vector(0)"}]},{"id":6,"title":"Deploys Unavailable","type":"stat","gridPos":{"h":4,"w":4,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"count(kube_deployment_status_replicas_unavailable
+    > 0) OR on() vector(0)"}]},{"id":7,"title":"Services Down","type":"stat","gridPos":{"h":4,"w":4,"x":20,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"count(probe_success
+    == 0) OR on() vector(0)"}]}]},{"id":10,"title":"Jobs & CronJobs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"Failed
+    Jobs","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"count(kube_job_status_failed
+    > 0) OR on() vector(0)"}]},{"id":12,"title":"Failed Jobs Detail","type":"table","gridPos":{"h":8,"w":10,"x":4,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"kube_job_status_failed
+    > 0","format":"table","instant":true}]},{"id":13,"title":"Stuck Jobs (>1h)","type":"table","gridPos":{"h":8,"w":10,"x":14,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"kube_job_status_active
+    == 1 and on(job_name,namespace) (time() - kube_job_status_start_time) > 3600","format":"table","instant":true}]},{"id":14,"title":"CronJob
+    Last Success","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"dateTimeFromNow"}},"targets":[{"expr":"kube_cronjob_status_last_successful_time{namespace=~\"cicd|kube-system|paperless\"}","legendFormat":"{{namespace}}/{{cronjob}}"}]},{"id":15,"title":"Container
+    Restart Storm (top 10)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"topk(10,
+    sum(rate(kube_pod_container_status_restarts_total[15m])) by (namespace, pod))","legendFormat":"{{namespace}}/{{pod}}"}]}]},{"id":20,"title":"Node
+    Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"CPU
+    % by Node","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percent"}},"targets":[{"expr":"(1
+    - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) by (instance)) * 100","legendFormat":"{{instance}}"}]},{"id":22,"title":"Memory
+    % by Node","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percent"}},"targets":[{"expr":"(1
+    - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100","legendFormat":"{{instance}}"}]},{"id":23,"title":"Disk
+    % by Node","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percent"}},"targets":[{"expr":"(1
+    - node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"})
+    * 100","legendFormat":"{{instance}}"}]},{"id":24,"title":"Load Average","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"node_load1","legendFormat":"1m
+    {{instance}}"},{"expr":"node_load5","legendFormat":"5m {{instance}}"}]},{"id":25,"title":"Network
+    Errors & Drops","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(node_network_receive_errs_total[5m])","legendFormat":"rx-err
+    {{instance}}"},{"expr":"rate(node_network_transmit_errs_total[5m])","legendFormat":"tx-err
+    {{instance}}"},{"expr":"rate(node_network_receive_drop_total[5m])","legendFormat":"rx-drop
+    {{instance}}"}]}]},{"id":30,"title":"Control Plane","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"API
+    Server Up","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}]}},"targets":[{"expr":"min(up{job=\"apiserver\"})"}]},{"id":32,"title":"API
+    Server Request Rate","type":"timeseries","gridPos":{"h":8,"w":10,"x":4,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(apiserver_request_total[5m]))
+    by (verb, code)","legendFormat":"{{verb}} {{code}}"}]},{"id":33,"title":"API Server
+    Error Rate %","type":"timeseries","gridPos":{"h":8,"w":10,"x":14,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percent"}},"targets":[{"expr":"sum(rate(apiserver_request_total{code=~\"5..\"}[5m]))
+    / sum(rate(apiserver_request_total[5m])) * 100","legendFormat":"5xx %"}]},{"id":34,"title":"API
+    Server Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":12},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95,
+    sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
+    sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p99"}]},{"id":35,"title":"etcd
+    Request Duration","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":12},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.99,
+    sum(rate(etcd_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p99"}]}]},{"id":40,"title":"Storage","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":4},"panels":[{"id":41,"title":"Longhorn
+    Disk Capacity","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"bytes"}},"targets":[{"expr":"longhorn_disk_capacity_bytes","legendFormat":"capacity
+    {{node}}"},{"expr":"longhorn_disk_reservation_bytes","legendFormat":"reserved
+    {{node}}"}]},{"id":42,"title":"PVC Phase","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"kube_persistentvolumeclaim_status_phase","legendFormat":"{{namespace}}/{{persistentvolumeclaim}}
+    {{phase}}"}]}]},{"id":50,"title":"DNS & Networking","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":5},"panels":[{"id":51,"title":"CoreDNS
+    Cache Hit Rate","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"rate(coredns_cache_hits_total[5m])
+    / (rate(coredns_cache_hits_total[5m]) + rate(coredns_cache_misses_total[5m]))","legendFormat":"{{server}}"}]},{"id":52,"title":"CoreDNS
+    Errors","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(coredns_dns_responses_total{rcode=~\"SERVFAIL|NXDOMAIN\"}[5m]))
+    by (rcode)","legendFormat":"{{rcode}}"}]}]},{"id":60,"title":"Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":6},"panels":[{"id":61,"title":"Error
+    Rate by Namespace","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":7},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum
+    by (namespace) (count_over_time({namespace=~\"kube-system|cert-manager|ingress-nginx|longhorn-system\"}
+    |= \"error\" [5m]))","legendFormat":"{{namespace}}"}]},{"id":62,"title":"Control
+    Plane Logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":15},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"kube-system\"}"}]},{"id":63,"title":"Cluster
+    Addon Logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":25},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=~\"cert-manager|ingress-nginx|longhorn-system\"}"}]}]}]}'
+kind: ConfigMap
+metadata:
+  annotations:
+    grafana_folder: Infrastructure
+  labels:
+    grafana_dashboard: '1'
+  name: cluster-infrastructure-dashboard
+  namespace: logging
@@ -1,55 +0,0 @@
-# k8s/monitoring/dashboards/control-plane-logs.yaml
-# Surfaces controller/control-plane logs that are already in Loki today
-# (Promtail scrapes every namespace with no filter) — this dashboard is the
-# "make it visible" piece, not new log collection.
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: control-plane-logs-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-data:
-  control-plane-logs.json: |
-    {
-      "title": "Cluster Control Plane & Controllers (Logs)",
-      "uid": "control-plane-logs",
-      "schemaVersion": 39,
-      "timezone": "browser",
-      "time": { "from": "now-1h", "to": "now" },
-      "refresh": "30s",
-      "panels": [
-        {
-          "id": 1,
-          "title": "Error rate by namespace",
-          "type": "timeseries",
-          "gridPos": { "h": 6, "w": 24, "x": 0, "y": 0 },
-          "datasource": { "type": "loki", "uid": "loki" },
-          "targets": [
-            {
-              "expr": "sum by (namespace) (count_over_time({namespace=~\"kube-system|cert-manager|ingress-nginx|longhorn-system\"} |= \"error\" [5m]))"
-            }
-          ]
-        },
-        {
-          "id": 2,
-          "title": "Control plane (kube-apiserver, controller-manager, scheduler)",
-          "type": "logs",
-          "gridPos": { "h": 10, "w": 24, "x": 0, "y": 6 },
-          "datasource": { "type": "loki", "uid": "loki" },
-          "targets": [
-            { "expr": "{namespace=\"kube-system\"}" }
-          ]
-        },
-        {
-          "id": 3,
-          "title": "Cluster add-ons (cert-manager, ingress-nginx, longhorn)",
-          "type": "logs",
-          "gridPos": { "h": 10, "w": 24, "x": 0, "y": 16 },
-          "datasource": { "type": "loki", "uid": "loki" },
-          "targets": [
-            { "expr": "{namespace=~\"cert-manager|ingress-nginx|longhorn-system\"}" }
-          ]
-        }
-      ]
-    }
@@ -0,0 +1,280 @@
+#!/usr/bin/env python3
+"""Generate consolidated Grafana dashboards as k8s ConfigMap YAML files."""
+
+import json
+import os
+
+DASHBOARD_DIR = os.path.expanduser("~/workplace/homelab/k8s/infra/monitoring/dashboards")
+
+DS_PROM = {"type": "prometheus", "uid": "prometheus"}
+DS_LOKI = {"type": "loki", "uid": "loki"}
+
+
+def stat_panel(id, title, expr, x, y, w=4, h=4, unit="short", mappings=None, thresholds=None):
+    p = {
+        "id": id, "title": title, "type": "stat",
+        "gridPos": {"h": h, "w": w, "x": x, "y": y},
+        "datasource": DS_PROM,
+        "fieldConfig": {"defaults": {"unit": unit}},
+        "targets": [{"expr": expr}],
+    }
+    if mappings:
+        p["fieldConfig"]["defaults"]["mappings"] = mappings
+    if thresholds:
+        p["fieldConfig"]["defaults"]["thresholds"] = thresholds
+        p["fieldConfig"]["defaults"]["color"] = {"mode": "thresholds"}
+    return p
+
+
+def ts_panel(id, title, exprs, x, y, w=8, h=8, unit="short"):
+    targets = []
+    for e in exprs:
+        if isinstance(e, tuple):
+            targets.append({"expr": e[0], "legendFormat": e[1]})
+        else:
+            targets.append({"expr": e, "legendFormat": "{{pod}}"})
+    return {
+        "id": id, "title": title, "type": "timeseries",
+        "gridPos": {"h": h, "w": w, "x": x, "y": y},
+        "datasource": DS_PROM,
+        "fieldConfig": {"defaults": {"unit": unit}},
+        "targets": targets,
+    }
+
+
+def table_panel(id, title, expr, x, y, w=12, h=8):
+    return {
+        "id": id, "title": title, "type": "table",
+        "gridPos": {"h": h, "w": w, "x": x, "y": y},
+        "datasource": DS_PROM,
+        "targets": [{"expr": expr, "format": "table", "instant": True}],
+    }
+
+
+def log_panel(id, title, query, x, y, w=24, h=10):
+    return {
+        "id": id, "title": title, "type": "logs",
+        "gridPos": {"h": h, "w": w, "x": x, "y": y},
+        "datasource": DS_LOKI,
+        "targets": [{"expr": query}],
+    }
+
+
+def row(id, title, y, panels, collapsed=True):
+    return {
+        "id": id, "title": title, "type": "row",
+        "collapsed": collapsed, "gridPos": {"h": 1, "w": 24, "x": 0, "y": y},
+        "panels": panels,
+    }
+
+
+def write_dashboard(filename, dashboard, folder):
+    cm = {
+        "apiVersion": "v1",
+        "kind": "ConfigMap",
+        "metadata": {
+            "name": filename.replace(".yaml", "-dashboard"),
+            "namespace": "logging",
+            "labels": {"grafana_dashboard": "1"},
+            "annotations": {"grafana_folder": folder},
+        },
+        "data": {
+            filename.replace(".yaml", ".json"): json.dumps(dashboard, separators=(",", ":"))
+        },
+    }
+
+    import yaml
+    path = os.path.join(DASHBOARD_DIR, filename)
+    with open(path, "w") as f:
+        yaml.dump(cm, f, default_flow_style=False, allow_unicode=True)
+    print(f"  wrote {path}")
+
+
+# ============================================================================
+# Dashboard 1: Cluster Infrastructure
+# ============================================================================
+
+def build_cluster_infrastructure():
+    zero_thresholds = {"mode": "absolute", "steps": [
+        {"value": None, "color": "green"}, {"value": 1, "color": "red"}
+    ]}
+
+    panels = [
+        row(1, "Cluster Health", 0, [
+            stat_panel(2, "Nodes Ready", 'count(kube_node_status_condition{condition="Ready",status="true"} == 1)', 0, 1, thresholds={"mode":"absolute","steps":[{"value":None,"color":"red"},{"value":3,"color":"green"}]}),
+            stat_panel(3, "Pods Pending", 'sum(kube_pod_status_phase{phase="Pending"}) OR on() vector(0)', 4, 1, thresholds=zero_thresholds),
+            stat_panel(4, "CrashLoopBackOff", 'sum(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}) OR on() vector(0)', 8, 1, thresholds=zero_thresholds),
+            stat_panel(5, "OOMKilled (1h)", 'sum(increase(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[1h])) OR on() vector(0)', 12, 1, thresholds=zero_thresholds),
+            stat_panel(6, "Deploys Unavailable", 'count(kube_deployment_status_replicas_unavailable > 0) OR on() vector(0)', 16, 1, thresholds=zero_thresholds),
+            stat_panel(7, "Services Down", 'count(probe_success == 0) OR on() vector(0)', 20, 1, thresholds=zero_thresholds),
+        ]),
+        row(10, "Jobs & CronJobs", 1, [
+            stat_panel(11, "Failed Jobs", 'count(kube_job_status_failed > 0) OR on() vector(0)', 0, 2, thresholds=zero_thresholds),
+            table_panel(12, "Failed Jobs Detail", 'kube_job_status_failed > 0', 4, 2, w=10),
+            table_panel(13, "Stuck Jobs (>1h)", 'kube_job_status_active == 1 and on(job_name,namespace) (time() - kube_job_status_start_time) > 3600', 14, 2, w=10),
+            ts_panel(14, "CronJob Last Success", [
+                ('kube_cronjob_status_last_successful_time{namespace=~"cicd|kube-system|paperless"}', "{{namespace}}/{{cronjob}}")
+            ], 0, 10, w=12, unit="dateTimeFromNow"),
+            ts_panel(15, "Container Restart Storm (top 10)", [
+                ('topk(10, sum(rate(kube_pod_container_status_restarts_total[15m])) by (namespace, pod))', "{{namespace}}/{{pod}}")
+            ], 12, 10, w=12),
+        ]),
+        row(20, "Node Resources", 2, [
+            ts_panel(21, "CPU % by Node", [
+                ('(1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m])) by (instance)) * 100', "{{instance}}")
+            ], 0, 3, unit="percent"),
+            ts_panel(22, "Memory % by Node", [
+                ('(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100', "{{instance}}")
+            ], 8, 3, unit="percent"),
+            ts_panel(23, "Disk % by Node", [
+                ('(1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100', "{{instance}}")
+            ], 16, 3, unit="percent"),
+            ts_panel(24, "Load Average", [
+                ("node_load1", "1m {{instance}}"),
+                ("node_load5", "5m {{instance}}"),
+            ], 0, 11),
+            ts_panel(25, "Network Errors & Drops", [
+                ("rate(node_network_receive_errs_total[5m])", "rx-err {{instance}}"),
+                ("rate(node_network_transmit_errs_total[5m])", "tx-err {{instance}}"),
+                ("rate(node_network_receive_drop_total[5m])", "rx-drop {{instance}}"),
+            ], 8, 11),
+        ]),
+        row(30, "Control Plane", 3, [
+            stat_panel(31, "API Server Up", 'min(up{job="apiserver"})', 0, 4, mappings=[
+                {"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}
+            ]),
+            ts_panel(32, "API Server Request Rate", [
+                ('sum(rate(apiserver_request_total[5m])) by (verb, code)', "{{verb}} {{code}}")
+            ], 4, 4, w=10),
+            ts_panel(33, "API Server Error Rate %", [
+                ('sum(rate(apiserver_request_total{code=~"5.."}[5m])) / sum(rate(apiserver_request_total[5m])) * 100', "5xx %")
+            ], 14, 4, w=10, unit="percent"),
+            ts_panel(34, "API Server Latency", [
+                ('histogram_quantile(0.95, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))', "p95"),
+                ('histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))', "p99"),
+            ], 0, 12, unit="s"),
+            ts_panel(35, "etcd Request Duration", [
+                ('histogram_quantile(0.99, sum(rate(etcd_request_duration_seconds_bucket[5m])) by (le))', "p99"),
+            ], 8, 12, unit="s"),
+        ]),
+        row(40, "Storage", 4, [
+            ts_panel(41, "Longhorn Disk Capacity", [
+                ("longhorn_disk_capacity_bytes", "capacity {{node}}"),
+                ("longhorn_disk_reservation_bytes", "reserved {{node}}"),
+            ], 0, 5, unit="bytes"),
+            ts_panel(42, "PVC Phase", [
+                ('kube_persistentvolumeclaim_status_phase', "{{namespace}}/{{persistentvolumeclaim}} {{phase}}")
+            ], 8, 5),
+        ]),
+        row(50, "DNS & Networking", 5, [
+            ts_panel(51, "CoreDNS Cache Hit Rate", [
+                ('rate(coredns_cache_hits_total[5m]) / (rate(coredns_cache_hits_total[5m]) + rate(coredns_cache_misses_total[5m]))', "{{server}}")
+            ], 0, 6, unit="percentunit"),
+            ts_panel(52, "CoreDNS Errors", [
+                ('sum(rate(coredns_dns_responses_total{rcode=~"SERVFAIL|NXDOMAIN"}[5m])) by (rcode)', "{{rcode}}")
+            ], 8, 6),
+        ]),
+        row(60, "Logs", 6, [
+            ts_panel(61, "Error Rate by Namespace", [
+                ('sum by (namespace) (count_over_time({namespace=~"kube-system|cert-manager|ingress-nginx|longhorn-system"} |= "error" [5m]))', "{{namespace}}")
+            ], 0, 7),
+            log_panel(62, "Control Plane Logs", '{namespace="kube-system"}', 0, 15),
+            log_panel(63, "Cluster Addon Logs", '{namespace=~"cert-manager|ingress-nginx|longhorn-system"}', 0, 25),
+        ]),
+    ]
+
+    return {
+        "title": "Cluster Infrastructure",
+        "uid": "cluster-infra",
+        "schemaVersion": 39,
+        "timezone": "browser",
+        "time": {"from": "now-6h", "to": "now"},
+        "refresh": "30s",
+        "tags": ["infrastructure", "k8s"],
+        "panels": panels,
+    }
+
+
+# ============================================================================
+# Dashboard 3: API Gateway
+# ============================================================================
+
+def build_api_gateway():
+    panels = [
+        row(1, "Gateway Health", 0, [
+            stat_panel(2, "Gateway Pods Ready", 'sum(kube_pod_status_ready{namespace="api",condition="true"})', 0, 1, thresholds={"mode":"absolute","steps":[{"value":None,"color":"red"},{"value":3,"color":"green"}]}),
+            stat_panel(3, "Probe: healthz", 'probe_success{instance=~".*api.riotpiao.com/healthz"}', 4, 1, mappings=[
+                {"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}
+            ]),
+            ts_panel(4, "Probe Latency", [
+                ('probe_duration_seconds{instance=~".*api.riotpiao.com.*"}', "{{instance}}")
+            ], 8, 1, unit="s"),
+        ], collapsed=False),
+        row(10, "Ingress Traffic (nginx)", 1, [
+            ts_panel(11, "Request Rate by Status", [
+                ('sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m])) by (status)', "{{status}}")
+            ], 0, 2),
+            ts_panel(12, "Error Rate %", [
+                ('sum(rate(nginx_ingress_controller_requests{ingress="api",status=~"5.."}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m])) * 100', "5xx"),
+                ('sum(rate(nginx_ingress_controller_requests{ingress="api",status=~"4.."}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m])) * 100', "4xx"),
+            ], 8, 2, unit="percent"),
+            ts_panel(13, "Latency p50/p95/p99", [
+                ('histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le))', "p50"),
+                ('histogram_quantile(0.95, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le))', "p95"),
+                ('histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le))', "p99"),
+            ], 16, 2, unit="s"),
+        ]),
+        row(20, "LLM Serving", 2, [
+            stat_panel(21, "LLM Pods Ready", 'sum(kube_pod_status_ready{namespace="llm-serving",condition="true"})', 0, 3),
+            ts_panel(22, "CPU by Predictor", [
+                ('sum(rate(container_cpu_usage_seconds_total{namespace="llm-serving"}[5m])) by (pod)', "{{pod}}")
+            ], 4, 3),
+            ts_panel(23, "Memory by Predictor", [
+                ('sum(container_memory_working_set_bytes{namespace="llm-serving"}) by (pod)', "{{pod}}")
+            ], 12, 3, unit="bytes"),
+            ts_panel(24, "Predictor Restarts", [
+                ('sum(rate(kube_pod_container_status_restarts_total{namespace="llm-serving"}[15m])) by (pod)', "{{pod}}")
+            ], 20, 3, w=4),
+        ]),
+        row(30, "Gateway Resources", 3, [
+            ts_panel(31, "CPU by Gateway Pod", [
+                ('sum(rate(container_cpu_usage_seconds_total{namespace="api"}[5m])) by (pod)', "{{pod}}")
+            ], 0, 4),
+            ts_panel(32, "Memory by Gateway Pod", [
+                ('sum(container_memory_working_set_bytes{namespace="api"}) by (pod)', "{{pod}}")
+            ], 8, 4, unit="bytes"),
+            ts_panel(33, "Gateway Restarts", [
+                ('sum(rate(kube_pod_container_status_restarts_total{namespace="api"}[15m])) by (pod)', "{{pod}}")
+            ], 16, 4),
+        ]),
+        row(40, "Logs", 4, [
+            log_panel(41, "Gateway Logs", '{namespace="api",container="gateway"}', 0, 5),
+            log_panel(42, "LLM Serving Logs", '{namespace="llm-serving"}', 0, 15),
+        ]),
+    ]
+
+    return {
+        "title": "API Gateway",
+        "uid": "api-gateway",
+        "schemaVersion": 39,
+        "timezone": "browser",
+        "time": {"from": "now-6h", "to": "now"},
+        "refresh": "30s",
+        "tags": ["api", "gateway", "llm"],
+        "panels": panels,
+    }
+
+
+# ============================================================================
+# Generate
+# ============================================================================
+
+print("Generating dashboards...")
+
+# Dashboard 1
+write_dashboard("cluster-infrastructure.yaml", build_cluster_infrastructure(), "Infrastructure")
+
+# Dashboard 3
+write_dashboard("api-gateway.yaml", build_api_gateway(), "API")
+
+print("Done.")
@@ -1,121 +0,0 @@
-# k8s/monitoring/dashboards/hardware-overview.yaml
-# Trimmed operator at-a-glance view across all nodes — node-exporter already
-# powers the deep-dive "Node Exporter Full" (#1860, see grafana-values.yaml),
-# this is the quick health-check version, not a replacement for it.
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: hardware-overview-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-data:
-  hardware-overview.json: |
-    {
-      "title": "Hardware Statistics (Operator Overview)",
-      "uid": "hardware-overview",
-      "schemaVersion": 39,
-      "timezone": "browser",
-      "time": { "from": "now-6h", "to": "now" },
-      "refresh": "30s",
-      "panels": [
-        {
-          "id": 1,
-          "title": "Nodes up / down",
-          "type": "stat",
-          "gridPos": { "h": 5, "w": 24, "x": 0, "y": 0 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "mappings": [
-                { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" } } },
-                { "type": "value", "options": { "1": { "text": "UP", "color": "green" } } }
-              ]
-            }
-          },
-          "targets": [
-            { "expr": "up{job=~\".*node-exporter.*\"}", "legendFormat": "{{instance}}" }
-          ]
-        },
-        {
-          "id": 2,
-          "title": "CPU usage % by node",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 5 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } },
-          "targets": [
-            {
-              "expr": "(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) by (instance)) * 100",
-              "legendFormat": "{{instance}}"
-            }
-          ]
-        },
-        {
-          "id": 3,
-          "title": "Memory usage % by node",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 5 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } },
-          "targets": [
-            {
-              "expr": "(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100",
-              "legendFormat": "{{instance}}"
-            }
-          ]
-        },
-        {
-          "id": 4,
-          "title": "Root filesystem usage % by node",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } },
-          "targets": [
-            {
-              "expr": "(1 - node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"}) * 100",
-              "legendFormat": "{{instance}}"
-            }
-          ]
-        },
-        {
-          "id": 5,
-          "title": "Root filesystem space remaining",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 13 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "bytes" } },
-          "targets": [
-            {
-              "expr": "node_filesystem_avail_bytes{mountpoint=\"/\"}",
-              "legendFormat": "{{instance}}"
-            }
-          ]
-        },
-        {
-          "id": 6,
-          "title": "Network errors/drops by node",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 21 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            { "expr": "rate(node_network_receive_errs_total[5m])", "legendFormat": "{{instance}} rx errs" },
-            { "expr": "rate(node_network_transmit_errs_total[5m])", "legendFormat": "{{instance}} tx errs" },
-            { "expr": "rate(node_network_receive_drop_total[5m])", "legendFormat": "{{instance}} rx drops" },
-            { "expr": "rate(node_network_transmit_drop_total[5m])", "legendFormat": "{{instance}} tx drops" }
-          ]
-        },
-        {
-          "id": 7,
-          "title": "Load average (1m / 5m) by node",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 21 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            { "expr": "node_load1", "legendFormat": "{{instance}} load1" },
-            { "expr": "node_load5", "legendFormat": "{{instance}} load5" }
-          ]
-        }
-      ]
-    }
@@ -1,185 +0,0 @@
-# k8s/monitoring/dashboards/kube-controller-health.yaml
-# Talos binds controller-manager/scheduler/etcd to 127.0.0.1, so Prometheus
-# can't scrape them directly (see prometheus-values.yaml). kube-apiserver is
-# the one control-plane component that's still reachable (its ServiceMonitor
-# targets the in-cluster `kubernetes` service, not localhost) — paired with
-# kube-state-metrics signals as a proxy for controller/scheduler health.
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: kube-controller-health-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-data:
-  kube-controller-health.json: |
-    {
-      "title": "Kube-Controller Health",
-      "uid": "kube-controller-health",
-      "schemaVersion": 39,
-      "timezone": "browser",
-      "time": { "from": "now-6h", "to": "now" },
-      "refresh": "30s",
-      "panels": [
-        {
-          "id": 1,
-          "title": "API server — up",
-          "type": "stat",
-          "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "mappings": [
-                { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" } } },
-                { "type": "value", "options": { "1": { "text": "UP", "color": "green" } } }
-              ]
-            }
-          },
-          "targets": [
-            { "expr": "min(up{job=\"apiserver\"})", "legendFormat": "apiserver" }
-          ]
-        },
-        {
-          "id": 2,
-          "title": "API server — request rate by verb/code",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 18, "x": 6, "y": 0 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            {
-              "expr": "sum(rate(apiserver_request_total[5m])) by (verb, code)",
-              "legendFormat": "{{verb}} {{code}}"
-            }
-          ]
-        },
-        {
-          "id": 3,
-          "title": "API server — error rate % (5xx)",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "percent" } },
-          "targets": [
-            {
-              "expr": "sum(rate(apiserver_request_total{code=~\"5..\"}[5m])) / sum(rate(apiserver_request_total[5m])) * 100",
-              "legendFormat": "5xx %"
-            }
-          ]
-        },
-        {
-          "id": 4,
-          "title": "API server — latency p95 / p99",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "s" } },
-          "targets": [
-            {
-              "expr": "histogram_quantile(0.95, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))",
-              "legendFormat": "p95"
-            },
-            {
-              "expr": "histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))",
-              "legendFormat": "p99"
-            }
-          ]
-        },
-        {
-          "id": 5,
-          "title": "Pods stuck Pending",
-          "type": "stat",
-          "gridPos": { "h": 5, "w": 8, "x": 0, "y": 16 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  { "value": 0, "color": "green" },
-                  { "value": 1, "color": "yellow" },
-                  { "value": 5, "color": "red" }
-                ]
-              }
-            }
-          },
-          "targets": [
-            { "expr": "sum(kube_pod_status_phase{phase=\"Pending\"}) OR on() vector(0)", "legendFormat": "pending" }
-          ]
-        },
-        {
-          "id": 6,
-          "title": "CrashLoopBackOff containers",
-          "type": "stat",
-          "gridPos": { "h": 5, "w": 8, "x": 8, "y": 16 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  { "value": 0, "color": "green" },
-                  { "value": 1, "color": "red" }
-                ]
-              }
-            }
-          },
-          "targets": [
-            { "expr": "sum(kube_pod_container_status_waiting_reason{reason=\"CrashLoopBackOff\"}) OR on() vector(0)", "legendFormat": "crashlooping" }
-          ]
-        },
-        {
-          "id": 7,
-          "title": "Nodes NotReady",
-          "type": "stat",
-          "gridPos": { "h": 5, "w": 8, "x": 16, "y": 16 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  { "value": 0, "color": "green" },
-                  { "value": 1, "color": "red" }
-                ]
-              }
-            }
-          },
-          "targets": [
-            { "expr": "count(kube_node_status_condition{condition=\"Ready\", status=\"true\"} == 0) OR on() vector(0)", "legendFormat": "not ready" }
-          ]
-        },
-        {
-          "id": 8,
-          "title": "Failed Jobs",
-          "type": "table",
-          "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            { "expr": "kube_job_status_failed > 0", "format": "table", "instant": true }
-          ]
-        },
-        {
-          "id": 9,
-          "title": "Deployments with unavailable replicas",
-          "type": "table",
-          "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            { "expr": "kube_deployment_status_replicas_unavailable > 0", "format": "table", "instant": true }
-          ]
-        },
-        {
-          "id": 10,
-          "title": "Container restart rate by pod",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 24, "x": 0, "y": 28 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            {
-              "expr": "sum(rate(kube_pod_container_status_restarts_total[15m])) by (namespace, pod)",
-              "legendFormat": "{{namespace}}/{{pod}}"
-            }
-          ]
-        }
-      ]
-    }
@@ -1,28 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: llm-frontend-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-  annotations:
-    grafana_folder: "LLM"
-    # No request-level panels. The rate/error/latency/bandwidth row used to run
-    # on Kong's prometheus plugin; Kong was retired 2026-08-19 and the Go
-    # gateway that replaced it does not expose /metrics yet, so those panels
-    # were removed rather than left querying series that no longer exist.
-    # What is left is pod-level: readiness, CPU/memory, restarts, logs.
-    #
-    # Restoring request-level and per-model observability means wiring three
-    # sources, none of which are in place: gateway metrics (RED plus token
-    # counts and TTFT, which the gateway can measure because it sees the
-    # response stream), vLLM's own /metrics on reasoning-predictor (rich --
-    # vllm:time_to_first_token_seconds, vllm:inter_token_latency_seconds,
-    # vllm:e2e_request_latency_seconds, vllm:kv_cache_usage_perc), and TEI's
-    # /metrics on embeddings/reranker. Ollama exposes no Prometheus endpoint at
-    # all (verified: /metrics returns 404), so ornith can only ever be observed
-    # from the gateway side. No ServiceMonitor exists for the llm-serving
-    # namespace today, so none of the engine metrics are being scraped.
-data:
-  llm-frontend.json: |
-    {"title":"LLM Frontend","uid":"llm-frontend","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"llm-serving pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"llm-serving\",condition=\"true\"})"}]},{"id":3,"title":"agent-pod ready","type":"stat","gridPos":{"h":4,"w":8,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"agent-pod\",condition=\"true\"})"}]},{"id":4,"title":"api gateway pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"api\",condition=\"true\"})"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=~\"llm-serving|agent-pod|api\"}[5m])) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":12,"title":"Memory by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=~\"llm-serving|agent-pod|api\"}) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":13,"title":"GPU-node predictor restarts","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"llm-serving\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"llm-serving logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"llm-serving\"}"}]},{"id":22,"title":"agent-pod logs (pi runs)","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":14},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"agent-pod\"}"}]},{"id":23,"title":"api gateway logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":24},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"api\"}"}]}]}]}
@@ -1,172 +0,0 @@
-# k8s/monitoring/dashboards/service-availability.yaml
-# Active uptime/availability from blackbox-exporter probes — the signal that
-# covers low-traffic services (Vault, MinIO, Longhorn UI) where RED metrics
-# alone can't distinguish "idle" from "down".
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: service-availability-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-data:
-  service-availability.json: |
-    {
-      "title": "Service Availability & Certificate Expiration",
-      "uid": "svc-availability",
-      "schemaVersion": 39,
-      "timezone": "browser",
-      "time": { "from": "now-24h", "to": "now" },
-      "refresh": "30s",
-      "panels": [
-        {
-          "id": 1,
-          "title": "Up / Down — all probed services",
-          "type": "stat",
-          "gridPos": { "h": 6, "w": 24, "x": 0, "y": 0 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "mappings": [
-                { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" } } },
-                { "type": "value", "options": { "1": { "text": "UP", "color": "green" } } }
-              ],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  { "value": 0, "color": "red" },
-                  { "value": 1, "color": "green" }
-                ]
-              }
-            }
-          },
-          "targets": [
-            { "expr": "probe_success", "legendFormat": "{{instance}}" }
-          ]
-        },
-        {
-          "id": 2,
-          "title": "Uptime % trend",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } },
-          "targets": [
-            {
-              "expr": "avg_over_time(probe_success[$__rate_interval]) * 100",
-              "legendFormat": "{{instance}}"
-            }
-          ]
-        },
-        {
-          "id": 3,
-          "title": "Probe latency",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "s" } },
-          "targets": [
-            { "expr": "probe_duration_seconds", "legendFormat": "{{instance}}" }
-          ]
-        },
-        {
-          "id": 4,
-          "title": "7-day SLO (% successful probes)",
-          "type": "table",
-          "gridPos": { "h": 8, "w": 24, "x": 0, "y": 14 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "unit": "percent",
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  { "value": 0, "color": "red" },
-                  { "value": 99, "color": "yellow" },
-                  { "value": 99.9, "color": "green" }
-                ]
-              }
-            }
-          },
-          "targets": [
-            {
-              "expr": "avg_over_time(probe_success[7d]) * 100",
-              "format": "table",
-              "instant": true
-            }
-          ]
-        },
-        {
-          "id": 5,
-          "title": "Services DOWN right now",
-          "type": "stat",
-          "gridPos": { "h": 4, "w": 12, "x": 0, "y": 22 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  { "value": 0, "color": "green" },
-                  { "value": 1, "color": "red" }
-                ]
-              }
-            }
-          },
-          "targets": [
-            { "expr": "count(probe_success == 0) OR on() vector(0)", "legendFormat": "down" }
-          ]
-        },
-        {
-          "id": 6,
-          "title": "Certs expiring in < 14 days",
-          "type": "stat",
-          "gridPos": { "h": 4, "w": 12, "x": 12, "y": 22 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  { "value": 0, "color": "green" },
-                  { "value": 1, "color": "red" }
-                ]
-              }
-            }
-          },
-          "targets": [
-            {
-              "expr": "count((certmanager_certificate_expiration_timestamp_seconds - time()) / 86400 < 14) OR on() vector(0)",
-              "legendFormat": "expiring"
-            }
-          ]
-        },
-        {
-          "id": 7,
-          "title": "Certificate expiry — days remaining",
-          "type": "table",
-          "gridPos": { "h": 8, "w": 24, "x": 0, "y": 26 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  { "value": 0, "color": "red" },
-                  { "value": 14, "color": "yellow" },
-                  { "value": 30, "color": "green" }
-                ]
-              }
-            }
-          },
-          "targets": [
-            {
-              "expr": "(certmanager_certificate_expiration_timestamp_seconds - time()) / 86400",
-              "legendFormat": "{{name}}",
-              "format": "table",
-              "instant": true
-            }
-          ]
-        }
-      ]
-    }
@@ -1,141 +0,0 @@
-# k8s/monitoring/dashboards/service-golden-signals.yaml
-# RED metrics (rate/errors/duration) for every service fronted by ingress-nginx.
-# Picked up automatically by Grafana's sidecar (grafana_dashboard=1 label) — see
-# sidecar.dashboards in k8s/logging/grafana-values.yaml.
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: service-golden-signals-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-data:
-  service-golden-signals.json: |
-    {
-      "title": "Latency & Golden Signals (Ingress RED)",
-      "uid": "svc-golden-signals",
-      "schemaVersion": 39,
-      "timezone": "browser",
-      "time": { "from": "now-6h", "to": "now" },
-      "refresh": "30s",
-      "templating": {
-        "list": [
-          {
-            "name": "ingress",
-            "type": "query",
-            "datasource": { "type": "prometheus", "uid": "prometheus" },
-            "query": "label_values(nginx_ingress_controller_requests, ingress)",
-            "refresh": 2,
-            "includeAll": false
-          }
-        ]
-      },
-      "panels": [
-        {
-          "id": 1,
-          "title": "Request rate by status — $ingress",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            {
-              "expr": "sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\"}[5m])) by (status)",
-              "legendFormat": "{{status}}"
-            }
-          ]
-        },
-        {
-          "id": 2,
-          "title": "Error rate % (4xx / 5xx) — $ingress",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "percent" } },
-          "targets": [
-            {
-              "expr": "sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\", status=~\"5..\"}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\"}[5m])) * 100",
-              "legendFormat": "5xx"
-            },
-            {
-              "expr": "sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\", status=~\"4..\"}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\"}[5m])) * 100",
-              "legendFormat": "4xx"
-            }
-          ]
-        },
-        {
-          "id": 3,
-          "title": "Latency p50 / p95 / p99 — $ingress",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "s" } },
-          "targets": [
-            {
-              "expr": "histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"$ingress\"}[5m])) by (le))",
-              "legendFormat": "p50"
-            },
-            {
-              "expr": "histogram_quantile(0.95, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"$ingress\"}[5m])) by (le))",
-              "legendFormat": "p95"
-            },
-            {
-              "expr": "histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"$ingress\"}[5m])) by (le))",
-              "legendFormat": "p99"
-            }
-          ]
-        },
-        {
-          "id": 4,
-          "title": "All services — traffic overview",
-          "type": "table",
-          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            {
-              "expr": "topk(11, sum(rate(nginx_ingress_controller_requests[5m])) by (ingress))",
-              "format": "table",
-              "instant": true
-            }
-          ]
-        },
-        {
-          "id": 5,
-          "title": "Customer-facing failures (5xx count, window total)",
-          "type": "stat",
-          "gridPos": { "h": 5, "w": 12, "x": 0, "y": 16 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  { "value": 0, "color": "green" },
-                  { "value": 1, "color": "yellow" },
-                  { "value": 50, "color": "red" }
-                ]
-              }
-            }
-          },
-          "targets": [
-            {
-              "expr": "sum(increase(nginx_ingress_controller_requests{status=~\"5..\"}[$__range])) OR on() vector(0)",
-              "legendFormat": "5xx total"
-            }
-          ]
-        },
-        {
-          "id": 6,
-          "title": "Top 5 error-contributing services",
-          "type": "table",
-          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            {
-              "expr": "topk(5, sum(rate(nginx_ingress_controller_requests{status=~\"5..\"}[5m])) by (ingress))",
-              "format": "table",
-              "instant": true
-            }
-          ]
-        }
-      ]
-    }
@@ -1,109 +0,0 @@
-# k8s/monitoring/dashboards/service-internals.yaml
-# Native per-service metrics — the "why" layer behind the ingress RED/uptime
-# dashboards (e.g. ingress shows MinIO is slow; this shows disk offline).
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: service-internals-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-data:
-  service-internals.json: |
-    {
-      "title": "Service Internals (MinIO / Forgejo / Argo CD / cert-manager / Vault / Longhorn)",
-      "uid": "svc-internals",
-      "schemaVersion": 39,
-      "timezone": "browser",
-      "time": { "from": "now-6h", "to": "now" },
-      "refresh": "30s",
-      "panels": [
-        { "id": 1, "title": "MinIO — disk/node offline", "type": "timeseries",
-          "gridPos": { "h": 6, "w": 12, "x": 0, "y": 0 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            { "expr": "minio_cluster_disk_offline_total", "legendFormat": "disks offline" },
-            { "expr": "minio_cluster_nodes_offline_total", "legendFormat": "nodes offline" }
-          ]
-        },
-        { "id": 2, "title": "MinIO — S3 request errors", "type": "timeseries",
-          "gridPos": { "h": 6, "w": 12, "x": 12, "y": 0 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            { "expr": "sum(rate(minio_s3_requests_errors_total[5m])) by (api)", "legendFormat": "{{api}}" }
-          ]
-        },
-        { "id": 3, "title": "MinIO — S3 TTFB latency", "type": "timeseries",
-          "gridPos": { "h": 6, "w": 12, "x": 0, "y": 6 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "s" } },
-          "targets": [
-            { "expr": "minio_s3_time_ttfb_seconds_distribution", "legendFormat": "{{api}}" }
-          ]
-        },
-        { "id": 4, "title": "Forgejo — repos / orgs", "type": "stat",
-          "gridPos": { "h": 6, "w": 12, "x": 12, "y": 6 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            { "expr": "gitea_repositories", "legendFormat": "repos" },
-            { "expr": "gitea_organizations", "legendFormat": "orgs" }
-          ]
-        },
-        { "id": 5, "title": "Forgejo — process health (CPU/mem)", "type": "timeseries",
-          "gridPos": { "h": 6, "w": 12, "x": 0, "y": 12 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            { "expr": "rate(process_cpu_seconds_total{job=~\".*forgejo.*|.*gitea.*\"}[5m])", "legendFormat": "cpu" },
-            { "expr": "process_resident_memory_bytes{job=~\".*forgejo.*|.*gitea.*\"}", "legendFormat": "mem" }
-          ]
-        },
-        { "id": 6, "title": "Argo CD — app sync/health status", "type": "table",
-          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            { "expr": "argocd_app_info", "format": "table", "instant": true }
-          ]
-        },
-        { "id": 7, "title": "cert-manager — days to cert expiry", "type": "stat",
-          "gridPos": { "h": 6, "w": 12, "x": 0, "y": 18 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  { "value": 0, "color": "red" },
-                  { "value": 14, "color": "yellow" },
-                  { "value": 30, "color": "green" }
-                ]
-              }
-            }
-          },
-          "targets": [
-            { "expr": "(certmanager_certificate_expiration_timestamp_seconds - time()) / 86400", "legendFormat": "{{name}}" }
-          ]
-        },
-        { "id": 8, "title": "Vault — sealed/unsealed", "type": "stat",
-          "gridPos": { "h": 6, "w": 6, "x": 12, "y": 20 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "mappings": [
-                { "type": "value", "options": { "0": { "text": "SEALED", "color": "red" } } },
-                { "type": "value", "options": { "1": { "text": "UNSEALED", "color": "green" } } }
-              ]
-            }
-          },
-          "targets": [
-            { "expr": "vault_core_unsealed", "legendFormat": "vault" }
-          ]
-        },
-        { "id": 9, "title": "Longhorn — volume robustness", "type": "table",
-          "gridPos": { "h": 6, "w": 6, "x": 18, "y": 20 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            { "expr": "longhorn_volume_robustness", "format": "table", "instant": true }
-          ]
-        }
-      ]
-    }
@@ -1,12 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: svc-argocd-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-  annotations:
-    grafana_folder: "Argo CD"
-data:
-  svc-argocd.json: |
-    {"title":"Argo CD — Service Overview","uid":"svc-argocd","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"}}},"targets":[{"expr":"min(up{job=~\"argocd-.*\"})"}]},{"id":3,"title":"HTTP requests","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_http_request_total[5m])) by (status)","legendFormat":"{{status}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_http_request_total{status=~\"5..\"}[5m])) / sum(rate(argocd_http_request_total[5m])) * 100"}]},{"id":5,"title":"Request latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(argocd_http_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"argocd\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"argocd\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restarts","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"argocd\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Applications & Sync","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Applications","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"argocd_app_total"}]},{"id":22,"title":"Sync by status","type":"timeseries","gridPos":{"h":6,"w":9,"x":6,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_app_sync_total[5m])) by (sync_status)","legendFormat":"{{sync_status}}"}]},{"id":23,"title":"Degraded apps","type":"stat","gridPos":{"h":6,"w":6,"x":15,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"argocd_app_health_degraded_total"}]},{"id":24,"title":"Git sync ops","type":"timeseries","gridPos":{"h":6,"w":12,"x":0,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_git_sync_total[5m])) by (git_operation,git_status)","legendFormat":"{{git_operation}}/{{git_status}}"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"argocd\"}"}]}]}]}
@@ -1,143 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: svc-authentik-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-  annotations:
-    grafana_folder: "Authentik"
-data:
-  svc-authentik.json: |
-    {
-      "title": "Authentik — Service Overview",
-      "uid": "svc-authentik",
-      "schemaVersion": 39,
-      "timezone": "browser",
-      "time": { "from": "now-6h", "to": "now" },
-      "refresh": "30s",
-      "panels": [
-        {
-          "id": 1, "title": "Row: Availability & Golden Signals", "type": "row",
-          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
-          "panels": [
-            {
-              "id": 2, "title": "Up", "type": "stat",
-              "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "fieldConfig": {
-                "defaults": {
-                  "color": { "mode": "thresholds" },
-                  "mappings": [
-                    { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }
-                  ],
-                  "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" }, { "value": 1, "color": "green" } ] }
-                }
-              },
-              "targets": [{ "expr": "min(up{job=\"authentik-server\"})" }]
-            },
-            {
-              "id": 3, "title": "HTTP request rate by status", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 9, "x": 6, "y": 1 },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "sum(rate(authentik_flows_execution_stage_time_count[5m])) by (flow_slug)", "legendFormat": "{{flow_slug}}" }]
-            },
-            {
-              "id": 4, "title": "Error rate % (5xx)", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 9, "x": 15, "y": 1 },
-              "fieldConfig": { "defaults": { "unit": "percent" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "(1 - (authentik_flows_cached / authentik_flows_execution_stage_time_count)) * 100" }]
-            },
-            {
-              "id": 5, "title": "Request duration p50/p95/p99", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
-              "fieldConfig": { "defaults": { "unit": "s" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [
-                { "expr": "histogram_quantile(0.50, sum(rate(authentik_main_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p50" },
-                { "expr": "histogram_quantile(0.95, sum(rate(authentik_main_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p95" },
-                { "expr": "histogram_quantile(0.99, sum(rate(authentik_main_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p99" }
-              ]
-            }
-          ]
-        },
-        {
-          "id": 10, "title": "Row: Resource Usage", "type": "row",
-          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 },
-          "panels": [
-            {
-              "id": 11, "title": "CPU by pod", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 8, "x": 0, "y": 2 },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"iam\",pod=~\"authentik.*\"}[5m])) by (pod)", "legendFormat": "{{pod}}" }]
-            },
-            {
-              "id": 12, "title": "Memory by pod", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 8, "x": 8, "y": 2 },
-              "fieldConfig": { "defaults": { "unit": "bytes" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "sum(container_memory_working_set_bytes{namespace=\"iam\",pod=~\"authentik.*\"}) by (pod)", "legendFormat": "{{pod}}" }]
-            },
-            {
-              "id": 13, "title": "Restart rate by pod", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 8, "x": 16, "y": 2 },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "sum(rate(kube_pod_container_status_restarts_total{namespace=\"iam\",pod=~\"authentik.*\"}[15m])) by (pod)", "legendFormat": "{{pod}}" }]
-            }
-          ]
-        },
-        {
-          "id": 20, "title": "Row: Identity Provider (OIDC / OAuth2)", "type": "row",
-          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 },
-          "panels": [
-            {
-              "id": 21, "title": "Outpost connections", "type": "stat",
-              "gridPos": { "h": 7, "w": 6, "x": 0, "y": 3 },
-              "fieldConfig": { "defaults": { "unit": "short" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "authentik_outposts_connected" }]
-            },
-            {
-              "id": 22, "title": "Flows cached", "type": "stat",
-              "gridPos": { "h": 7, "w": 6, "x": 6, "y": 3 },
-              "fieldConfig": { "defaults": { "unit": "short" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "authentik_flows_cached" }]
-            },
-            {
-              "id": 23, "title": "Policies cached", "type": "stat",
-              "gridPos": { "h": 7, "w": 6, "x": 12, "y": 3 },
-              "fieldConfig": { "defaults": { "unit": "short" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "authentik_policies_cached" }]
-            },
-            {
-              "id": 24, "title": "Queued tasks", "type": "stat",
-              "gridPos": { "h": 7, "w": 6, "x": 18, "y": 3 },
-              "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "unit": "short" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "authentik_tasks_queued" }]
-            },
-            {
-              "id": 25, "title": "Admin workers", "type": "timeseries",
-              "gridPos": { "h": 7, "w": 12, "x": 0, "y": 10 },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "authentik_admin_workers" }]
-            }
-          ]
-        },
-        {
-          "id": 30, "title": "Row: Logs", "type": "row",
-          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 },
-          "panels": [
-            {
-              "id": 31, "title": "Recent logs", "type": "logs",
-              "gridPos": { "h": 10, "w": 24, "x": 0, "y": 4 },
-              "datasource": { "type": "loki", "uid": "loki" },
-              "targets": [{ "expr": "{namespace=\"iam\"}" }]
-            }
-          ]
-        }
-      ]
-    }
@@ -1,12 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: svc-forgejo-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-  annotations:
-    grafana_folder: "Forgejo"
-data:
-  svc-forgejo.json: |
-    {"title":"Forgejo — Service Overview","uid":"svc-forgejo","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"}}},"targets":[{"expr":"min(up{job=\"forgejo\"})"}]},{"id":3,"title":"HTTP requests by method","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_http_request_total[5m])) by (method)","legendFormat":"{{method}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_http_request_total{status=~\"5..\"}[5m])) / sum(rate(forgejo_http_request_total[5m])) * 100"}]},{"id":5,"title":"Request latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(forgejo_http_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"forgejo\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"forgejo\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restarts","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"forgejo\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Git Operations","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Repositories","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"forgejo_repositories_total"}]},{"id":22,"title":"Users","type":"stat","gridPos":{"h":6,"w":6,"x":6,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"forgejo_users_total"}]},{"id":23,"title":"Git ops rate","type":"timeseries","gridPos":{"h":6,"w":12,"x":12,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_git_operations_total[5m])) by (operation_type)","legendFormat":"{{operation_type}}"}]},{"id":24,"title":"Runner tasks","type":"timeseries","gridPos":{"h":6,"w":12,"x":0,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_runner_tasks_total[5m])) by (status)","legendFormat":"{{status}}"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"forgejo\"}"}]}]}]}
@@ -1,12 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: svc-grafana-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-  annotations:
-    grafana_folder: "Grafana"
-data:
-  svc-grafana.json: |
-    {"title":"Grafana — Service Overview","uid":"svc-grafana","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}]}},"targets":[{"expr":"min(up{job=\"grafana\"})"}]},{"id":3,"title":"HTTP requests","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(grafana_http_request_total[5m])) by (status)","legendFormat":"{{status}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(grafana_http_request_total{status=~\"5..\"}[5m])) / sum(rate(grafana_http_request_total[5m])) * 100"}]},{"id":5,"title":"Request latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(grafana_http_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"logging\",pod=~\"grafana.*\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"logging\",pod=~\"grafana.*\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restarts","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"logging\",pod=~\"grafana.*\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Dashboards & Users","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Total dashboards","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"grafana_dashboard_total"}]},{"id":22,"title":"Total users","type":"stat","gridPos":{"h":6,"w":6,"x":6,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"grafana_user_total"}]},{"id":23,"title":"Total alerts","type":"stat","gridPos":{"h":6,"w":6,"x":12,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"grafana_alerts_total"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"logging\",container=\"grafana\"}"}]}]}]}
@@ -1,143 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: svc-minio-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-  annotations:
-    grafana_folder: "MinIO"
-data:
-  svc-minio.json: |
-    {
-      "title": "MinIO — Service Overview",
-      "uid": "svc-minio",
-      "schemaVersion": 39,
-      "timezone": "browser",
-      "time": { "from": "now-6h", "to": "now" },
-      "refresh": "30s",
-      "panels": [
-        {
-          "id": 1, "title": "Row: Availability & Golden Signals", "type": "row",
-          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
-          "panels": [
-            {
-              "id": 2, "title": "Up", "type": "stat",
-              "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "fieldConfig": {
-                "defaults": {
-                  "color": { "mode": "thresholds" },
-                  "mappings": [
-                    { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }
-                  ],
-                  "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" }, { "value": 1, "color": "green" } ] }
-                }
-              },
-              "targets": [{ "expr": "min(up{job=\"minio\"})" }]
-            },
-            {
-              "id": 3, "title": "S3 request rate by method", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 9, "x": 6, "y": 1 },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "sum(rate(minio_s3_requests_total[5m])) by (method)", "legendFormat": "{{method}}" }]
-            },
-            {
-              "id": 4, "title": "Error rate %", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 9, "x": 15, "y": 1 },
-              "fieldConfig": { "defaults": { "unit": "percent" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "sum(rate(minio_s3_requests_total{error=\"true\"}[5m])) / sum(rate(minio_s3_requests_total[5m])) * 100" }]
-            },
-            {
-              "id": 5, "title": "Request duration p50/p95/p99", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
-              "fieldConfig": { "defaults": { "unit": "s" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [
-                { "expr": "histogram_quantile(0.50, sum(rate(minio_s3_requests_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p50" },
-                { "expr": "histogram_quantile(0.95, sum(rate(minio_s3_requests_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p95" },
-                { "expr": "histogram_quantile(0.99, sum(rate(minio_s3_requests_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p99" }
-              ]
-            }
-          ]
-        },
-        {
-          "id": 10, "title": "Row: Resource Usage", "type": "row",
-          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 },
-          "panels": [
-            {
-              "id": 11, "title": "CPU by pod", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 8, "x": 0, "y": 2 },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"storage\",pod=~\"minio.*\"}[5m])) by (pod)", "legendFormat": "{{pod}}" }]
-            },
-            {
-              "id": 12, "title": "Memory by pod", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 8, "x": 8, "y": 2 },
-              "fieldConfig": { "defaults": { "unit": "bytes" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "sum(container_memory_working_set_bytes{namespace=\"storage\",pod=~\"minio.*\"}) by (pod)", "legendFormat": "{{pod}}" }]
-            },
-            {
-              "id": 13, "title": "Restart rate by pod", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 8, "x": 16, "y": 2 },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "sum(rate(kube_pod_container_status_restarts_total{namespace=\"storage\",pod=~\"minio.*\"}[15m])) by (pod)", "legendFormat": "{{pod}}" }]
-            }
-          ]
-        },
-        {
-          "id": 20, "title": "Row: Storage & Replication", "type": "row",
-          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 },
-          "panels": [
-            {
-              "id": 21, "title": "Usable vs Raw capacity", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 12, "x": 0, "y": 3 },
-              "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "lineWidth": 2 } } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [
-                { "expr": "minio_cluster_capacity_usable_bytes", "legendFormat": "Usable" },
-                { "expr": "minio_cluster_capacity_raw_total_bytes", "legendFormat": "Raw Total" }
-              ]
-            },
-            {
-              "id": 22, "title": "Drive health (online/offline)", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 12, "x": 12, "y": 3 },
-              "fieldConfig": { "defaults": { "unit": "short" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [
-                { "expr": "minio_cluster_health_drives_online", "legendFormat": "Online" },
-                { "expr": "minio_cluster_health_drives_offline", "legendFormat": "Offline" }
-              ]
-            },
-            {
-              "id": 23, "title": "Replication lag (bytes pending)", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 12, "x": 0, "y": 11 },
-              "fieldConfig": { "defaults": { "unit": "bytes" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "minio_replication_metrics_replicating_byte_count", "legendFormat": "Pending replication" }]
-            },
-            {
-              "id": 24, "title": "Replication failures (bytes)", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 12, "x": 12, "y": 11 },
-              "fieldConfig": { "defaults": { "unit": "bytes" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "minio_replication_metrics_failed_byte_count", "legendFormat": "Failed replication" }]
-            }
-          ]
-        },
-        {
-          "id": 30, "title": "Row: Logs", "type": "row",
-          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 },
-          "panels": [
-            {
-              "id": 31, "title": "Recent logs", "type": "logs",
-              "gridPos": { "h": 10, "w": 24, "x": 0, "y": 4 },
-              "datasource": { "type": "loki", "uid": "loki" },
-              "targets": [{ "expr": "{namespace=\"storage\"}" }]
-            }
-          ]
-        }
-      ]
-    }
@@ -1,12 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: svc-vault-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-  annotations:
-    grafana_folder: "Vault"
-data:
-  svc-vault.json: |
-    {"title":"Vault — Service Overview","uid":"svc-vault","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability & Golden Signals","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"red"},{"value":1,"color":"green"}]}}},"targets":[{"expr":"min(up{job=\"vault\"})"}]},{"id":3,"title":"Request rate by status","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(vault_core_handle_request_total[5m])) by (method)","legendFormat":"{{method}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(vault_core_handle_request_total{error=\"true\"}[5m])) / sum(rate(vault_core_handle_request_total[5m])) * 100"}]},{"id":5,"title":"Request duration p50/p95/p99","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.50, sum(rate(vault_core_handle_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95, sum(rate(vault_core_handle_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99, sum(rate(vault_core_handle_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p99"}]}]},{"id":10,"title":"Row: Resource Usage","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU by pod","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"iam\",pod=~\"vault.*\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory by pod","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"iam\",pod=~\"vault.*\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restart rate","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"iam\",pod=~\"vault.*\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Vault Seal State","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Sealed","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"UNSEALED","color":"green"},"1":{"text":"SEALED","color":"red"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]}}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_core_unsealed"}]},{"id":22,"title":"Active","type":"stat","gridPos":{"h":6,"w":6,"x":6,"y":3},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"INACTIVE","color":"red"},"1":{"text":"ACTIVE","color":"green"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"red"},{"value":1,"color":"green"}]}}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_core_active"}]},{"id":23,"title":"Replication (Primary)","type":"stat","gridPos":{"h":6,"w":6,"x":12,"y":3},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"SECONDARY","color":"orange"},"1":{"text":"PRIMARY","color":"green"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"orange"},{"value":1,"color":"green"}]}}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_core_replication_primary"}]},{"id":24,"title":"Active tokens","type":"stat","gridPos":{"h":6,"w":6,"x":18,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_token_total"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"iam\",container=\"vault\"}"}]}]}]}
@@ -12,20 +12,12 @@ resources:
  - alerts/svc-grafana-rules.yaml
  - alerts/svc-minio-rules.yaml
  - alerts/svc-vault-rules.yaml
+  - alerts/cluster-alerts.yaml
+  - alerts/api-gateway-alerts.yaml
  - servicemonitors/argocd.yaml
  - servicemonitors/authentik.yaml
  - servicemonitors/forgejo.yaml
  - servicemonitors/minio.yaml
-  - dashboards/control-plane-logs.yaml
-  - dashboards/hardware-overview.yaml
-  - dashboards/kube-controller-health.yaml
-  - dashboards/llm-frontend.yaml
-  - dashboards/service-availability.yaml
-  - dashboards/service-golden-signals.yaml
-  - dashboards/service-internals.yaml
-  - dashboards/svc-argocd.yaml
-  - dashboards/svc-authentik.yaml
-  - dashboards/svc-forgejo.yaml
-  - dashboards/svc-grafana.yaml
-  - dashboards/svc-minio.yaml
-  - dashboards/svc-vault.yaml
+  - servicemonitors/ingress-nginx.yaml
+  - dashboards/cluster-infrastructure.yaml
+  - dashboards/api-gateway.yaml
@@ -0,0 +1,18 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: ingress-nginx
+  namespace: monitoring
+  labels:
+    release: prometheus
+spec:
+  namespaceSelector:
+    matchNames:
+      - ingress-nginx
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: ingress-nginx
+      app.kubernetes.io/component: controller
+  endpoints:
+    - port: metrics
+      interval: 30s
@@ -0,0 +1,15 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: homelab-admin-oidc
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+  # OIDC group for Authentik homelab-admins members
+  # When rock logs in via OIDC, k8s sees:
+  # - User: oidc:[email protected]
+  # - Groups: oidc:homelab-admins (from Authentik group claim)
+  - kind: Group
+    name: oidc:homelab-admins
@@ -6,6 +6,7 @@ kind: Kustomization
 # already fixed once in k8s/infra/minio and k8s/infra/iam. Every resource
 # here sets its own explicit metadata.namespace.
 resources:
+  - admin-oidc-binding.yaml
  - grafana-operator-role.yaml
  - minio-operator-role.yaml
  - forgejo-operator-role.yaml
@@ -37,7 +37,7 @@ import sys
 import urllib.error
 import urllib.request

-AUTHENTIK_URL = "http://authentik-server.iam.svc.cluster.local"
+AUTHENTIK_URL = "https://authentik.riotpiao.com"
 TOKEN = os.environ["AUTHENTIK_BOOTSTRAP_TOKEN"]


@@ -185,6 +185,10 @@ GROUP_PERMISSIONS = {
    "llm-users": ["llm:inference"],
    "memory-users": ["memory:read"],
    "memory-writers": ["memory:read", "memory:write"],
+    "s3-users": ["s3:read"],
+    "s3-writers": ["s3:read", "s3:write"],
+    "sqs-users": ["sqs:read"],
+    "sqs-writers": ["sqs:read", "sqs:write"],
 }
 perms = set()
 for group in request.user.groups.all():
@@ -204,6 +208,38 @@ permissions_mapping = get_or_create(
 )
 PERMISSIONS_MAPPING_PK = permissions_mapping["pk"]

+# Roles claim for service accounts - for client_credentials flow, the auto-generated
+# user doesn't have roles, so we look up the actual service account by client_id.
+# Format: ["llm:inference", "memory:read", "memory:write"]
+_ROLES_EXPR = """
+from authentik.core.models import User
+
+# Try user's own roles first (password grant uses the actual user)
+roles = request.user.attributes.get("roles", [])
+
+# For client_credentials, user is auto-generated - look up by client_id
+if not roles and request.http_request:
+    client_id = request.http_request.POST.get("client_id", "")
+    if client_id:
+        sa_user = User.objects.filter(username=client_id, path="service-accounts").first()
+        if sa_user:
+            roles = sa_user.attributes.get("roles", [])
+
+return {"roles": roles}
+""".strip()
+roles_mapping = get_or_create(
+    "/api/v3/propertymappings/provider/scope/",
+    "/api/v3/propertymappings/provider/scope/",
+    "scope_name=roles",
+    {
+        "name": "homelab: roles claim",
+        "scope_name": "roles",
+        "expression": _ROLES_EXPR,
+    },
+    patch_existing={"expression": _ROLES_EXPR},
+)
+ROLES_MAPPING_PK = roles_mapping["pk"]
+
 # Immich reads a "immich_role" claim on every login (not just user-creation -
 # fixed upstream in immich-app/immich#29991) and syncs isAdmin from it, so
 # this is the actual mechanism that makes "rock" an Immich admin - not
@@ -680,6 +716,10 @@ for name, cfg in SERVICES.items():
 # Redirect URI matches kubelogin's (int128/kubelogin) documented default;
 # adjust here if a different kubectl OIDC plugin/port is actually used.
 print("Ensuring public OAuth2 client 'kubernetes' for kubectl OIDC login...")
+# Fetch explicit-consent flow for public authcode clients (kubelogin)
+status, res = api("GET", "/api/v3/flows/instances/?slug=default-provider-authorization-explicit-consent")
+k8s_authz_flow = res["results"][0]["pk"] if res["results"] else AUTHORIZATION_FLOW_PK
+
 k8s_provider = get_or_create(
    "/api/v3/providers/oauth2/", "/api/v3/providers/oauth2/",
    "name=kubernetes",
@@ -687,7 +727,7 @@ k8s_provider = get_or_create(
        "name": "kubernetes",
        "client_id": "kubernetes",
        "client_type": "public",
-        "authorization_flow": AUTHORIZATION_FLOW_PK,
+        "authorization_flow": k8s_authz_flow,
        "invalidation_flow": INVALIDATION_FLOW_PK,
        "signing_key": SIGNING_KEY_PK,
        "property_mappings": SCOPE_PKS,
@@ -720,14 +760,61 @@ app_pks_for_binding.append(("kubernetes", k8s_application["pk"]))
 print(f"  kubernetes: provider pk={k8s_provider['pk']} application pk={k8s_application['pk']}")

 # -----------------------------------------------------------------------------
-# Service accounts for programmatic API access (client_credentials flow).
+# Headless authentication flow for service accounts (password grant).
+# Default flow has MFA/interactive stages that break password grant.
+# This flow: identification -> password -> login (no MFA, no consent prompts).
+print("\n[SERVICE ACCOUNT FLOW] Ensuring headless authentication flow...")
+
+sa_flow = get_or_create(
+    "/api/v3/flows/instances/", "/api/v3/flows/instances/",
+    "slug=service-account-authentication",
+    {
+        "slug": "service-account-authentication",
+        "name": "Service Account Authentication (Headless)",
+        "title": "Service Account Login",
+        "designation": "authentication",
+        "policy_engine_mode": "any",
+    },
+)
+SA_AUTHENTICATION_FLOW_PK = sa_flow["pk"]
+print(f"  flow pk={SA_AUTHENTICATION_FLOW_PK}")
+
+# Bind stages to the flow (identification -> password -> login)
+# Get default stages (reuse existing ones)
+status, ident_stages = api("GET", "/api/v3/stages/identification/")
+status, pass_stages = api("GET", "/api/v3/stages/password/")
+status, login_stages = api("GET", "/api/v3/stages/user_login/")
+
+if ident_stages.get("results") and pass_stages.get("results") and login_stages.get("results"):
+    ident_pk = ident_stages["results"][0]["pk"]
+    pass_pk = pass_stages["results"][0]["pk"]
+    login_pk = login_stages["results"][0]["pk"]
+    
+    # Check if bindings already exist
+    status, existing_bindings = api("GET", f"/api/v3/flows/bindings/?target={SA_AUTHENTICATION_FLOW_PK}")
+    if not existing_bindings.get("results"):
+        # Create bindings
+        api("POST", "/api/v3/flows/bindings/", {"target": SA_AUTHENTICATION_FLOW_PK, "stage": ident_pk, "order": 10})
+        api("POST", "/api/v3/flows/bindings/", {"target": SA_AUTHENTICATION_FLOW_PK, "stage": pass_pk, "order": 20})
+        api("POST", "/api/v3/flows/bindings/", {"target": SA_AUTHENTICATION_FLOW_PK, "stage": login_pk, "order": 30})
+        print("  bound stages: identification -> password -> login")
+    else:
+        print(f"  stages already bound ({len(existing_bindings['results'])} bindings)")
+else:
+    print("  WARNING: Could not find default stages to bind")
+
+# -----------------------------------------------------------------------------
+# Service accounts for programmatic API access (password grant).
 # These are Authentik users with type=service_account, not OAuth applications.
-# They authenticate via client_credentials grant to get JWTs with claims.
+# They authenticate via password grant to get JWTs with user claims.
 print("\n[SERVICE ACCOUNTS] Creating service accounts for API access...")

+# Roles define what APIs a service can access - stored in user attributes,
+# output as "roles" claim in JWT. Gateway checks roles, not groups.
+# Format: "<api>:<action>" e.g. "llm:inference", "memory:write"
 SERVICE_ACCOUNTS = {
    "portfolio-agent": {
-        "groups": [],  # Permissions come from SA_CONFIGS in memory scope mapping
+        "roles": ["llm:inference", "memory:read", "s3:read", "sqs:read", "temporal:admin"],
        "attributes": {
            "memory_projects": ["homelab", "portfolio"],
            "memory_visibility": "public",
@@ -736,7 +823,7 @@ SERVICE_ACCOUNTS = {
        "secret_name": "portfolio-agent-oidc",
    },
    "memory-agent": {
-        "groups": [],  # Permissions come from SA_CONFIGS in memory scope mapping
+        "roles": ["llm:inference", "memory:read", "memory:write", "s3:read", "s3:write", "sqs:read", "sqs:write"],
        "attributes": {
            "memory_projects": ["*"],
            "memory_visibility": "private",
@@ -755,36 +842,40 @@ for sa_name, sa_cfg in SERVICE_ACCOUNTS.items():
    if status != 200:
        die(f"GET users for {sa_name} -> {status} {res}")
    
+    # Merge roles into attributes
+    sa_attributes = {**sa_cfg["attributes"], "roles": sa_cfg["roles"]}
+    
    if res.get("results"):
        sa_user = res["results"][0]
-        # Update attributes and groups
-        group_pks = [service_admin_groups[g]["pk"] for g in sa_cfg["groups"] if g in service_admin_groups]
+        # Update attributes (roles stored here, not in groups)
        status, sa_user = api("PATCH", f"/api/v3/core/users/{sa_user['pk']}/", {
-            "attributes": sa_cfg["attributes"],
-            "groups": group_pks,
+            "attributes": sa_attributes,
            "is_active": True,
        })
        if status not in (200, 201):
            die(f"PATCH service account {sa_name} -> {status} {sa_user}")
-        print(f"  {sa_name}: updated existing service account")
+        print(f"  {sa_name}: updated (roles: {sa_cfg['roles']})")
    else:
        # Create new service account user
-        group_pks = [service_admin_groups[g]["pk"] for g in sa_cfg["groups"] if g in service_admin_groups]
        status, sa_user = api("POST", "/api/v3/core/users/", {
            "username": sa_name,
            "name": f"Service Account: {sa_name}",
            "type": "service_account",
            "path": "service-accounts",
            "is_active": True,
-            "attributes": sa_cfg["attributes"],
-            "groups": group_pks,
+            "attributes": sa_attributes,
        })
        if status not in (200, 201):
            die(f"POST service account {sa_name} -> {status} {sa_user}")
-        print(f"  {sa_name}: created new service account")
+        print(f"  {sa_name}: created (roles: {sa_cfg['roles']})")
    
-    # Create OAuth provider for this service account (client_credentials flow)
+    print(f"  {sa_name}: roles={sa_cfg['roles']}")
+    
+    # Create OAuth provider for this service account
+    # Supports both password grant (user claims) and client_credentials (fallback)
+    # Password grant requires authentication_flow and app_password token
    sa_client_secret = existing_secret or gen_secret(40)
+    sa_grant_types = ["password", "client_credentials", "refresh_token"]
    sa_provider = get_or_create(
        "/api/v3/providers/oauth2/", "/api/v3/providers/oauth2/",
        f"name={sa_name}",
@@ -794,17 +885,19 @@ for sa_name, sa_cfg in SERVICE_ACCOUNTS.items():
            "client_secret": sa_client_secret,
            "client_type": "confidential",
            "authorization_flow": AUTHORIZATION_FLOW_PK,
+            "authentication_flow": SA_AUTHENTICATION_FLOW_PK,  # Headless flow for password grant
            "invalidation_flow": INVALIDATION_FLOW_PK,
            "signing_key": SIGNING_KEY_PK,
-            "property_mappings": SCOPE_PKS + [MEMORY_MAPPING_PK],
+            "property_mappings": SCOPE_PKS + [ROLES_MAPPING_PK, MEMORY_MAPPING_PK],
            "sub_mode": "hashed_user_id",
            "include_claims_in_id_token": True,
-            "grant_types": ["client_credentials"],
-            "redirect_uris": [],  # No redirects for client_credentials
+            "grant_types": sa_grant_types,
+            "redirect_uris": [],
        },
        patch_existing={
-            "property_mappings": SCOPE_PKS + [MEMORY_MAPPING_PK],
-            "grant_types": ["client_credentials"],
+            "property_mappings": SCOPE_PKS + [ROLES_MAPPING_PK, MEMORY_MAPPING_PK],
+            "grant_types": sa_grant_types,
+            "authentication_flow": SA_AUTHENTICATION_FLOW_PK,
        },
    )
    
@@ -819,17 +912,70 @@ for sa_name, sa_cfg in SERVICE_ACCOUNTS.items():
            "meta_launch_url": "",
        },
    )
-    app_pks_for_binding.append((sa_name, sa_application["pk"]))
+    
+    # Service account apps have NO policy bindings - client_secret is the access control.
+    # Clean up any existing bindings (from old provisioning logic).
+    status, existing_bindings = api("GET", f"/api/v3/policies/bindings/?target={sa_application['pk']}")
+    if status == 200:
+        for binding in existing_bindings.get("results", []):
+            api("DELETE", f"/api/v3/policies/bindings/{binding['pk']}/")
+            print(f"  {sa_name}: removed stale binding {binding.get('group_obj', {}).get('name', binding['pk'])}")
+    
+    print(f"  {sa_name}: provider pk={sa_provider['pk']} application pk={sa_application['pk']}")
+    
+    # Create App Password token for password grant (optional, client_credentials also works)
+    # Authentik's password grant checks against Token with intent=app_password, not user password
+    # Note: API doesn't allow setting key at creation, must use set_key endpoint after
+    existing_app_password = kubectl_get_secret_key(sa_cfg["secret_ns"], sa_cfg["secret_name"], "APP_PASSWORD")
+    if not existing_app_password:
+        token_identifier = f"{sa_name}-app-password"
+        
+        # Check if token exists
+        status, existing_tokens = api("GET", f"/api/v3/core/tokens/?identifier={token_identifier}")
+        if status == 200 and existing_tokens.get("results"):
+            # Token exists, just set a new key
+            pass
+        else:
+            # Create the token first
+            status, token_resp = api("POST", "/api/v3/core/tokens/", {
+                "identifier": token_identifier,
+                "user": sa_user["pk"],
+                "intent": "app_password",
+                "expiring": False,
+                "description": f"App password for {sa_name} OAuth password grant",
+            })
+            if status not in (200, 201):
+                print(f"  WARNING: Failed to create app password token for {sa_name}: {token_resp}")
+        
+        # Set the key via set_key endpoint (works on existing or newly created token)
+        app_password_key = gen_secret(40)
+        status, _ = api("POST", f"/api/v3/core/tokens/{token_identifier}/set_key/", {
+            "key": app_password_key,
+        })
+        if status not in (200, 204):
+            print(f"  WARNING: Failed to set app password key for {sa_name}")
+            app_password_key = None
+        else:
+            print(f"  {sa_name}: created/updated app password token")
+    else:
+        app_password_key = existing_app_password
+        print(f"  {sa_name}: reusing existing app password")
    
    # Store credentials in k8s Secret
-    if not existing_secret:
-        kubectl_create_secret(sa_cfg["secret_ns"], sa_cfg["secret_name"], {
-            "CLIENT_ID": sa_name,
-            "CLIENT_SECRET": sa_client_secret,
-            "TOKEN_URL": "https://authentik.riotpiao.com/application/o/token/",
-            "ISSUER": f"https://authentik.riotpiao.com/application/o/{sa_name}/",
-        })
-        print(f"  {sa_name}: generated credentials -> {sa_cfg['secret_ns']}/{sa_cfg['secret_name']}")
+    # Supports both password grant (APP_PASSWORD) and client_credentials (CLIENT_SECRET)
+    secret_data = {
+        "CLIENT_ID": sa_name,
+        "CLIENT_SECRET": sa_client_secret,
+        "USERNAME": sa_name,
+        "TOKEN_URL": "https://authentik.riotpiao.com/application/o/token/",
+        "ISSUER": f"https://authentik.riotpiao.com/application/o/{sa_name}/",
+    }
+    if app_password_key:
+        secret_data["APP_PASSWORD"] = app_password_key
+    
+    if not existing_secret or (app_password_key and not existing_app_password):
+        kubectl_create_secret(sa_cfg["secret_ns"], sa_cfg["secret_name"], secret_data)
+        print(f"  {sa_name}: stored credentials -> {sa_cfg['secret_ns']}/{sa_cfg['secret_name']}")
    else:
        print(f"  {sa_name}: reusing existing credentials from {sa_cfg['secret_ns']}/{sa_cfg['secret_name']}")
Author	SHA1	Message	Date
rock	ef2228fdfb	fix: use explicit-consent flow for kubernetes public OIDC client - Kubernetes provider now uses default-provider-authorization-explicit-consent instead of implicit-consent flow - Better handling for public clients like kubelogin (authcode grant) - Prevents fallback to password grant prompts	2026-09-05 15:09:56 -07:00
rock	7ac37c7ab5	fix: remove spec wrapper from ClusterRoleBinding - ClusterRoleBinding doesn't use spec: wrapper (unlike Deployment/StatefulSet) - roleRef and subjects go at top level with metadata - Fixes: 'strict decoding error: unknown field "spec"'	2026-09-05 15:04:00 -07:00
rock	f740076694	fix: remove namespace from rbac Application destination - RBAC kustomization contains cluster-scoped (ClusterRoleBinding) and namespace-scoped (Role/RoleBinding) resources - Each resource has explicit metadata.namespace, so Application shouldn't force a default namespace - Fixes: ClusterRoleBinding gets namespace=default, causing sync failure with 'unsupported role reference kind: ""'	2026-09-05 14:53:59 -07:00
rock	c183978d6d	fix: remove backslash line continuations from YAML multiline string - YAML block scalars (\|) don't use backslash continuation - Just indent lines properly, block scalar handles them automatically - Fixes ArgoCD ComparisonError on poimen app	2026-09-05 14:48:21 -07:00
rock	19cf9277d9	fix: add admin-oidc-binding to kustomization resources - admin-oidc-binding.yaml wasn't listed in resources - Now kustomize will include it when building manifests - ArgoCD can sync the OIDC group binding	2026-09-05 14:42:17 -07:00
rock	74d587ca1b	chore: revert node-runner to stock image, remove Dockerfile - Reverted to node:22-bookworm (no custom image) - Removed Dockerfile.node (no CI to build it) - Docker install step in riotpiao workflow is already the workaround	2026-09-05 14:33:27 -07:00
rock	21156ccc1f	chore: remove homelab CI workflow - Removed .gitea/workflows/build-runner-node.yml - Homelab is GitOps only, not a buildable artifact - Runner images managed via direct Dockerfile edits + manual pushes	2026-09-05 14:33:13 -07:00
rock	6ec61d1c2c	feat: add OIDC group binding for cluster-admin access - Binds oidc:homelab-admins group to cluster-admin ClusterRole - Allows OIDC users (via Authentik) to have admin access - Groups claim from Authentik with oidc: prefix per kube-apiserver config - Enables kubectl access via 'kubectl login' + kubelogin	2026-09-05 14:30:41 -07:00
rock	acc11d5e87	feat: custom forgejo-runner-node image with docker.io pre-installed Build forgejo-runner-node image / build (push) Failing after 30s Details - Dockerfile.node extends node:22-bookworm + docker.io - CI workflow builds and pushes to forgejo registry on changes - values-node.yaml references custom image instead of stock node - Removes need to install docker in every workflow using node runner	2026-09-05 14:20:42 -07:00
rock	ba4261fb39	feat: Image Updater git write-back for multi-source poimen Application - write-back-method: git (commits image updates back to repos) - git-branch: main - Mounts ArgoCD SSH credentials for git pushes - Image Updater commits new SHAs → repos → ArgoCD syncs	2026-09-05 13:56:40 -07:00
rock	30ab2de36e	feat: multi-source poimen Application (memory, workflows, frontend) - Single Application syncs 3 independent repos - All deploy to poimen namespace - Image Updater tracks all 3 services (7-char SHA tags) - Auto-sync: prune + selfHeal enabled	2026-09-05 13:55:29 -07:00
rock	2954a9a0a4	fix: remove poimen-root Application (external repo dependency) - Removed dependency on external poimen.git repo - Poimen manifests should be managed locally or via separate workflow - Simplifies homelab GitOps to only manage homelab-owned services	2026-09-05 13:52:53 -07:00
rock	dc027cecb6	Revert "feat: enable Image Updater for poimen services" This reverts commit `8a7ee29e93`.	2026-09-05 13:52:50 -07:00
rock	8a7ee29e93	feat: enable Image Updater for poimen services - Track poimen-memory, poimen-workflows, poimen-frontend images - Auto-update on new 7-char SHA tags from Forgejo - Filter: regexp:^[0-9a-f]{7}$ (commit SHA) - write-back-method: argocd (updates Application)	2026-09-05 13:51:29 -07:00
rock	f5e100ee32	feat: add temporal:admin role to portfolio-agent - portfolio-agent can now call Temporal API in addition to LLM, memory, S3, SQS	2026-09-05 06:02:08 -07:00
rock	6743f7c25f	Phase 6.6: Add Poimen Memory DLQ queues (ArgoCD managed) ArgoCD Application: memory-queues ├─ Sync wave: 7 (messaging wave) ├─ Path: k8s/apps/messaging/memory-queues ├─ Namespace: sqs └─ Auto-sync: enabled (prune + selfHeal) Helm Chart: memory-queues ├─ Chart.yaml: v0.1.0 ├─ values.yaml: Queue config └─ templates/queues.yaml: Queue CRD resources Queues Created: 1. poimen-memory-dlq ├─ Purpose: Extraction + webhook + agent failures ├─ Partitions: 3 ├─ Replication factor: 1 ├─ Retention: 14 days (1,209,600 seconds) └─ Visibility timeout: 5 minutes (300 seconds) 2. poimen-memory-metric-dlq ├─ Purpose: Metrics persistence failures ├─ Partitions: 3 ├─ Replication factor: 1 ├─ Retention: 14 days └─ Visibility timeout: 5 minutes Resource: Queue CRD (kmsvc.io/v1alpha1) └─ Managed by: queue-operator (already running in sqs ns) Deployment Flow: ArgoCD (homelab) → sync wave 7 → deploy queues Memory app (poimen) → connects to kmsvc → sends DLQ messages Files: ├─ k8s/apps/messaging/memory-queues/Chart.yaml (new) ├─ k8s/apps/messaging/memory-queues/values.yaml (new) ├─ k8s/apps/messaging/memory-queues/templates/queues.yaml (new) └─ k8s/argocd/apps/50-memory-queues.yaml (new)	2026-09-05 01:10:51 -07:00
rock	266f0637a4	Revert "feat: add memory service queues (processing, indexing, dlq)" This reverts commit `af6fc84a18`.	2026-09-05 01:09:30 -07:00
rock	af6fc84a18	feat: add memory service queues (processing, indexing, dlq) - processing-queue: high-throughput, auto-scaling (1-4 shards) - indexing-queue: FIFO with deduplication (1-2 shards) - memory-dlq: dead letter queue for redelivery failures - ArgoCD Application (wave 7) to auto-sync queue lifecycle	2026-09-05 01:05:01 -07:00
rock	ed644b2c83	feat: add S3 and SQS permissions to service accounts and capability groups - New capability groups: s3-users, s3-writers, sqs-users, sqs-writers - portfolio-agent: add s3:read, sqs:read - memory-agent: add s3:read, s3:write, sqs:read, sqs:write - Enables portfolio and memory services to access MinIO S3 and message queues via JWT	2026-09-05 00:14:43 -07:00
rock	6db4d7dcb0	fix(grafana): give homelab-admins Admin org role, akadmin GrafanaAdmin GrafanaAdmin is server admin only — no org membership, so users couldn't see dashboards. Now: - akadmin: GrafanaAdmin (server admin, can impersonate) - homelab-admins: Admin (org admin, dashboard access) - others: Viewer	2026-09-04 23:41:22 -07:00
rock	ed794befdb	fix: grant GrafanaAdmin (server admin) to homelab-admins for Administration menu	2026-09-04 23:18:19 -07:00
rock	adb5c3597c	fix: add groups scope to grafana OIDC so role mapping works	2026-09-04 23:14:11 -07:00
rock	60bdd16a66	feat: add nginx-ingress ServiceMonitor for gateway traffic metrics	2026-09-04 23:07:44 -07:00
rock	823d5c6a3f	fix: map grafana admin role from homelab-admins group (grafana-admins deleted)	2026-09-04 23:04:01 -07:00
rock	176ec44b42	refactor: consolidate 13 dashboards into 2 (cluster-infrastructure + api-gateway)	2026-09-04 22:58:53 -07:00
rock	09fac8ada6	feat: add cluster and api-gateway alert rules with SLA targets	2026-09-04 22:37:48 -07:00
rock	75bb105e52	feat: add cluster-infrastructure and api-gateway grafana dashboards	2026-09-04 22:31:51 -07:00
rock	eafcb2397e	feat: add api-gateway blackbox probes for healthz and /v1/models	2026-09-04 22:13:00 -07:00
rock	539ef848d0	fix: use external Authentik URL for MinIO OIDC config discovery	2026-09-04 21:25:30 -07:00
rock	449c2a9109	gitops: add secret-rotation controller ArgoCD Application - Syncs k8s/apps/secret-rotation-controller/ kustomization - Auto-prune and self-heal enabled - Creates secret-rotation namespace - ArgoCD will deploy CRD, RBAC, ExternalSecret, Deployment	2026-09-04 13:51:26 -07:00
rock	667bca0f44	feat: automated secret rotation controller - ExternalSecret syncs age key from Vault to pod - CRD defines rotation schedule for each secret - Controller watches CRD, rotates on schedule: * Call provider API (Authentik/Forgejo/MinIO) for new secret * Update k8s Secret * Update .enc.yaml via sops (uses age key from Vault) * Git commit and push - Vault is source of truth for age key (never on disk) - Examples: minio-oidc (90d), portfolio-agent (90d), forgejo-token (90d), minio-root (180d)	2026-09-03 23:37:24 -07:00
rock	f9654986ad	fix(minio): use in-cluster URL for OIDC config fetch MinIO pod was getting 503 from public URL at startup. Use in-cluster authentik-server.iam.svc for metadata fetch; browser redirects still use public URLs from OIDC metadata response.	2026-09-03 23:15:19 -07:00
rock	8f7004c946	iam: switch service accounts to roles-based auth - Roles stored in user attributes, not groups - Property mapping looks up roles by client_id for client_credentials - Service account apps have no policy bindings (client_secret = access control) - Cleanup stale bindings on re-provision - JWT claims: azp (service identity) + roles (capabilities)	2026-09-03 19:23:06 -07:00
rock	f1e5fe58f4	iam: move provisioning script to scripts/iam, remove k8s job - Move authentik-provision.py to scripts/iam/ (manual-only) - Remove job/RBAC resources (not needed for local runs) - Use public URL directly (no sed substitution needed) - Add app password support via set_key endpoint - Support both password grant and client_credentials	2026-09-03 19:03:58 -07:00