fix(coredns): declare the poimen rewrite in the source of truth

poimen.riotpiao.com resolved only because someone edited the live coredns
ConfigMap by hand. Neither terraform/files/coredns/Corefile nor the Talos
machine config carried the line, so the next Talos reconcile would have
deleted it and broken in-cluster resolution of a heavily used service.

Live and source now agree on both hostnames the retirement touched.

k8s/apps/cloudflared/configmap.yaml

@@ -45,8 +45,6 @@ data:
         service: https://igotify.riotpiao.com
       - hostname: poimen.riotpiao.com
         service: https://poimen.riotpiao.com
-      - hostname: comfyui.riotpiao.com
-        service: https://comfyui.riotpiao.com
       - hostname: forgejo.riotpiao.com
         service: https://forgejo.riotpiao.com
       # ZITADEL. Without this rule the connector answers 404 for the hostname

k8s/apps/comfyui/deployment.yaml

@@ -1,93 +0,0 @@
-# ComfyUI — GPU-accelerated image generation on worker-1.
-# Uses 1x V100 32GB (sm70). Freed by scaling ornith 2→1.
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: comfyui
-  namespace: comfyui
-  labels:
-    app: comfyui
-spec:
-  # Retired on purpose. This Deployment held the 4th V100 on worker-1 while
-  # serving nothing: its oauth2-proxy roll-out has been stuck for 42h and
-  # failing 11621 times, because the Secret comfyui/comfyui-oauth2-proxy carries
-  # only 1 key while the pod requests client-id, client-secret and
-  # cookie-secret. Every request was therefore refused, so the card sat at 0%
-  # utilisation and blocked a GPU that llm-serving can use.
-  #
-  # Scaling to 0 is the whole change. The Deployment, the Service, the Ingress
-  # and the 50Gi comfyui-models PVC all stay, so nothing is lost and no data is
-  # touched. To bring it back: set replicas to 1 here, then fix the Secret to
-  # carry all three keys, otherwise the same stuck roll-out returns.
-  replicas: 0
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app: comfyui
-  template:
-    metadata:
-      labels:
-        app: comfyui
-    spec:
-      nodeSelector:
-        kubernetes.io/hostname: worker-1
-      runtimeClassName: nvidia
-      # k8s Service named 'comfyui' injects COMFYUI_PORT=tcp://... into pod env,
-      # which clobbers ai-dock's own COMFYUI_PORT variable (expects a port number).
-      # Disable service link injection to avoid the collision.
-      enableServiceLinks: false
-      containers:
-        - name: comfyui
-          image: ghcr.io/ai-dock/comfyui:v2-cuda-12.1.1-base-22.04
-          ports:
-            - containerPort: 8188
-              protocol: TCP
-          env:
-            - name: NVIDIA_VISIBLE_DEVICES
-              value: "all"
-          resources:
-            requests:
-              cpu: "4"
-              memory: 8Gi
-              nvidia.com/gpu: "1"
-            limits:
-              cpu: "8"
-              memory: 16Gi
-              nvidia.com/gpu: "1"
-          volumeMounts:
-            - mountPath: /workspace/ComfyUI/models
-              name: models
-            - mountPath: /workspace/ComfyUI/output
-              name: output
-          readinessProbe:
-            httpGet:
-              path: /
-              port: 8188
-            periodSeconds: 10
-            initialDelaySeconds: 30
-          startupProbe:
-            httpGet:
-              path: /
-              port: 8188
-            failureThreshold: 120
-            periodSeconds: 10
-      volumes:
-        - name: models
-          persistentVolumeClaim:
-            claimName: comfyui-models
-        - name: output
-          emptyDir: {}
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: comfyui-models
-  namespace: comfyui
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: longhorn
-  resources:
-    requests:
-      storage: 50Gi

k8s/apps/comfyui/ingress.yaml

@@ -1,38 +0,0 @@
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: comfyui
-  namespace: comfyui
-  annotations:
-    # No cert-manager annotation and no tls: block. This host is covered by
-    # the *.riotpiao.com wildcard, which nginx serves as its
-    # default-ssl-certificate (ingress-nginx/riotpiao-com-tls).
-    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
-    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
-    nginx.ingress.kubernetes.io/proxy-body-size: "0"
-    # WebSocket support for ComfyUI's live preview
-    nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
-    nginx.ingress.kubernetes.io/upstream-hash-by: "$remote_addr"
-    # Single sign-on. ComfyUI has no auth of its own, so every request is handed to
-    # oauth2-proxy for an allow or deny decision.
-    nginx.ingress.kubernetes.io/auth-url: "http://comfyui-oauth2-proxy.comfyui.svc.cluster.local:4180/oauth2/auth"
-    nginx.ingress.kubernetes.io/auth-signin: "https://comfyui.riotpiao.com/oauth2/start?rd=$scheme://$host$request_uri"
-    nginx.ingress.kubernetes.io/auth-response-headers: "x-auth-request-user,x-auth-request-email"
-    # No configuration-snippet. The controller rejects that annotation as risky
-    # ("annotation group ConfigurationSnippet contains risky annotation"), and a
-    # rejected annotation drops the WHOLE ingress from nginx.conf: the host then
-    # answers 404 with no entry in the access log. ingress-nginx sets the Upgrade
-    # and Connection headers itself for a websocket upgrade.
-spec:
-  ingressClassName: nginx
-  rules:
-    - host: comfyui.riotpiao.com
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: comfyui
-                port:
-                  number: 80

k8s/apps/comfyui/kustomization.yaml

@@ -1,8 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-  - deployment.yaml
-  - service.yaml
-  - oauth2-proxy.yaml
-  - ingress.yaml

k8s/apps/comfyui/oauth2-proxy.yaml

@@ -1,105 +0,0 @@
-# ComfyUI has no authentication of its own: comfyui.riotpiao.com is open to anyone
-# who can resolve the host. It also has no OIDC support, so ZITADEL cannot be a
-# login method inside it. oauth2-proxy is the front door instead: nginx asks it for
-# a decision on every request and it answers from a ZITADEL session.
-#
-# No API client calls this host today, so the whole host is gated. If a client ever
-# needs direct access, add a skip rule for the exact paths it uses rather than
-# opening the whole host.
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: comfyui-oauth2-proxy
-  namespace: comfyui
-  labels:
-    app: comfyui-oauth2-proxy
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: comfyui-oauth2-proxy
-  template:
-    metadata:
-      labels:
-        app: comfyui-oauth2-proxy
-    spec:
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 65534
-        seccompProfile: {type: RuntimeDefault}
-      containers:
-        - name: oauth2-proxy
-          image: quay.io/oauth2-proxy/oauth2-proxy:v7.7.1
-          args:
-            - --provider=oidc
-            - --oidc-issuer-url=https://zitadel.riotpiao.com
-            - --email-domain=*
-            - --upstream=static://202
-            - --reverse-proxy=true
-            - --set-xauthrequest=true
-            - --cookie-secure=true
-            - --cookie-expire=12h
-            - --cookie-refresh=1h
-            - --skip-provider-button=true
-            - --redirect-url=https://comfyui.riotpiao.com/oauth2/callback
-            - --http-address=0.0.0.0:4180
-          env:
-            - name: OAUTH2_PROXY_CLIENT_ID
-              valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: client-id}}
-            - name: OAUTH2_PROXY_CLIENT_SECRET
-              valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: client-secret}}
-            - name: OAUTH2_PROXY_COOKIE_SECRET
-              valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: cookie-secret}}
-          ports:
-            - {name: http, containerPort: 4180}
-          readinessProbe:
-            httpGet: {path: /ping, port: http}
-            initialDelaySeconds: 5
-          livenessProbe:
-            httpGet: {path: /ping, port: http}
-            initialDelaySeconds: 15
-          securityContext:
-            allowPrivilegeEscalation: false
-            capabilities: {drop: ["ALL"]}
-          resources:
-            requests: {cpu: 10m, memory: 32Mi}
-            limits: {cpu: 200m, memory: 128Mi}
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: comfyui-oauth2-proxy
-  namespace: comfyui
-  labels:
-    app: comfyui-oauth2-proxy
-spec:
-  type: ClusterIP
-  selector:
-    app: comfyui-oauth2-proxy
-  ports:
-    - {name: http, port: 4180, targetPort: http}
----
-# The login path must NOT be gated: a request to /oauth2/start that itself demands a
-# login redirects to /oauth2/start forever. ingress-nginx applies the annotations of
-# an ingress to the locations it defines, so a separate object without auth-url keeps
-# this path open. The earlier failure of this shape was not the split — it was the
-# configuration-snippet on the app ingress, which made the controller drop the whole
-# object and left only this one alive.
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: comfyui-oauth2
-  namespace: comfyui
-spec:
-  ingressClassName: nginx
-  rules:
-    - host: comfyui.riotpiao.com
-      http:
-        paths:
-          - path: /oauth2
-            pathType: Prefix
-            backend:
-              service:
-                name: comfyui-oauth2-proxy
-                port:
-                  number: 4180

k8s/apps/comfyui/service.yaml

@@ -1,14 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: comfyui
-  namespace: comfyui
-  labels:
-    app: comfyui
-spec:
-  selector:
-    app: comfyui
-  ports:
-    - port: 80
-      targetPort: 8188
-      protocol: TCP

k8s/apps/key-rotation-controller/README.md

@@ -40,7 +40,7 @@ target is one root-of-trust file per thing that cannot rotate itself:
 | controller bootstrap key | chicken and egg: it creates every other key |
 | etcd defrag credentials | needed by a job that runs outside the controller |
 
-Everything else — Grafana, Argo CD, Forgejo, MinIO, Paperless, Immich, ComfyUI,
+Everything else — Grafana, Argo CD, Forgejo, MinIO, Paperless, Immich,
 api-gw, Gotify — is a KeyRotation object, and its secret leaves git.
 
 ## Status

k8s/apps/key-rotation-controller/rotations.yaml

@@ -64,32 +64,6 @@ spec:
 ---
 apiVersion: iam.homelab.riotpiao.com/v1
 kind: KeyRotation
-metadata:
-  name: comfyui
-  namespace: iam
-  labels:
-    app.kubernetes.io/part-of: key-rotation
-spec:
-  provider: zitadel-oidc-client
-  strategy: atomic
-  intervalHours: 2160
-  suspend: true
-  target:
-    zitadelProjectID: "396757796898471937"
-    zitadelAppID: "397015743994003854"  # app resource id, not the client id 397015744010781070
-    zitadelClientID: "397015744010781070"
-    secretName: comfyui-oauth2-proxy
-    secretNamespace: comfyui
-    secretKeys:
-      secret: client-secret
-    reload: restart
-    consumers:
-      - {kind: Deployment, name: comfyui-oauth2-proxy, namespace: comfyui}
-  verify:
-    introspect: true
----
-apiVersion: iam.homelab.riotpiao.com/v1
-kind: KeyRotation
 metadata:
   name: minio
   namespace: iam
@@ -107,7 +81,11 @@ spec:
     secretName: minio-oidc
     secretNamespace: storage
     secretKeys:
-      secret: client-secret
+      # The key name here must match the consumer's secretKeyRef, not a house style.
+      # minio-tenant.yaml reads MINIO_IDENTITY_OPENID_CLIENT_SECRET. While this said
+      # `client-secret`, the tenant failed with CreateContainerConfigError on every
+      # restart and the rotation wrote a key nobody read.
+      secret: MINIO_IDENTITY_OPENID_CLIENT_SECRET
     reload: hot
     consumers:
       - {kind: StatefulSet, name: minio-cluster-az-a, namespace: storage}

k8s/apps/llm-serving/qwen-cpu.yaml

@@ -1,5 +1,6 @@
 # qwen2.5:3b-instruct on CPU (talos-cp-2, 144GB RAM, 24 cores).
-# Moved off GPU to free a V100 for ComfyUI. Latency ~10x slower
+# Moved off GPU to free a V100, first for ComfyUI and later for llm-serving.
+# Latency ~10x slower
 # than GPU but sufficient for lightweight tasks (summarization,
 # classification, quick answers).
 apiVersion: apps/v1

k8s/argocd/apps/58-comfyui.yaml

@@ -1,32 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: comfyui
-  namespace: argocd
-  labels:
-    app.kubernetes.io/name: comfyui
-    app.kubernetes.io/component: image-generation
-  annotations:
-    argocd.argoproj.io/sync-wave: "8"
-spec:
-  project: homelab
-  revisionHistoryLimit: 3
-  source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
-    targetRevision: main
-    path: k8s/apps/comfyui
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: comfyui
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
-    syncOptions:
-      - CreateNamespace=true
-    retry:
-      limit: 5
-      backoff:
-        duration: 5s
-        factor: 2
-        maxDuration: 3m

k8s/bootstrap/phase3-forgejo/forgejo-values.yaml

@@ -97,10 +97,25 @@ gitea:
       # USER and PASSWD injected via env vars below (GITEA__MAILER__USER, GITEA__MAILER__PASSWD)
 
 # Persistence (shared storage for repos)
+#
+# 60Gi, not the original 20Gi. The container registry stores its blobs here,
+# and a single candle-vllm sm_70 image is 14GB uncompressed. At 20Gi the volume
+# filled, and every blob upload failed with:
+#   Package registry API internal error: 500 write /data/tmp/package-upload/...:
+#   no space left on device
+# The push client surfaces that as a bare `received unexpected HTTP status: 500`
+# after retrying every layer, which looks like a network fault and is not one.
+#
+# Note for whoever expands this again: Longhorn's admission webhook checks the
+# PHYSICAL free space of the disk holding each replica, not the thin volume
+# size. A request for 100Gi was rejected while the disk had 23.5Gi free:
+#   disk ... does not have sufficient physical space for expansion:
+#   physical free space would drop below minimal: left=14596177920 < minimal=49931236556
+# 60Gi fit. Raise it in steps and watch the webhook message.
 persistence:
   enabled: true
   storageClass: longhorn
-  size: 20Gi
+  size: 60Gi
   accessModes:
     - ReadWriteOnce
 
@@ -146,19 +161,19 @@ ingress:
 # The database was never the bottleneck. A real action_runner UPDATE measured
 # 2.133ms with zero lock waits, so this is CPU starvation in Gitea itself.
 #
-# requests.cpu rises to 750m because a request below real usage earns the pod
-# little scheduling weight during contention. limits.cpu rises to 3000m
-# because diff computation is bursty and needs room to run without throttling.
+# requests.cpu stays 200m and limits.cpu stays 1000m. The 3000m limit was a
+# temporary measure while the Actions stall was diagnosed; the real fix landed
+# elsewhere, so the limit is back to the original.
 #
-# Caution: this pod runs on talos-cp-3, a control-plane node. 3000m is 25% of
-# that node's 11950m allocatable CPU. If Gitea sustains near this limit, move
-# the workload off the control plane rather than raising the limit again.
+# Caution: this pod runs on talos-cp-3, a control-plane node. If Gitea
+# throttles again at 1000m, move the workload off the control plane rather
+# than raising the limit.
 resources:
   requests:
-    cpu: 750m
+    cpu: 200m
     memory: 768Mi
   limits:
-    cpu: 3000m
+    cpu: 1000m
     memory: 2Gi
 
 # Tolerations for control-plane

k8s/infra/longhorn/disk-scan-job.yaml

@@ -0,0 +1,90 @@
+# One-shot destructive health scan of a candidate Longhorn disk.
+#
+# WHY THIS IS A MANUAL JOB AND NOT AN ARGOCD HOOK
+#   It writes over the entire raw device. An ArgoCD PostSync hook would run it
+#   on every sync and destroy whatever the volume holds. Apply it by hand,
+#   once, on a disk that has no partitions and no mounts.
+#
+# WHY NOT TALOS
+#   Talos declares disks by mounting them (machine.disks in the machine
+#   config). It has no primitive for "run this command on the host", so a scan
+#   cannot live in the Talos config. This Job is the IaC-correct home.
+#
+# ORDER OF OPERATIONS
+#   1. Scan the raw device with this Job        <- the disk must be UNMOUNTED
+#   2. If it passes, add the disk to terraform.tfvars.local `longhorn_disks`
+#      with device /dev/disk/by-id/<wwid> and run `make apply-cp`
+#   3. Register it with Longhorn via the pattern in longhorn-add-disks-job.yaml
+#
+# What it does
+#   fio writes a CRC32C-verified pattern across the whole device, then reads
+#   every block back and compares. Any unreadable or mismatched sector fails
+#   the Job. This is a full write and read pass, not a sample.
+#
+# Expected duration
+#   2 TB on a 5400 rpm 2.5" drive at roughly 90 MB/s is about 6 h to write and
+#   6 h to verify. Budget 12 hours. Check progress with:
+#     kubectl -n llm-io logs job/disk-scan-cp1-sdb -f
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: disk-scan-cp1-sdb
+  namespace: llm-io
+  labels:
+    app: disk-scan
+spec:
+  backoffLimit: 0
+  # Do not let a stuck scan retry silently.
+  template:
+    metadata:
+      labels:
+        app: disk-scan
+    spec:
+      restartPolicy: Never
+      nodeSelector:
+        kubernetes.io/hostname: talos-cp-1
+      containers:
+        - name: scan
+          image: ubuntu:22.04
+          securityContext:
+            privileged: true
+          command:
+            - /bin/bash
+            - -c
+            - |
+              set -euo pipefail
+              apt-get update -qq
+              apt-get install -y -qq smartmontools fio
+
+              echo "=== device identity ==="
+              smartctl -i /dev/sdb | tail -12
+
+              echo "=== SMART before ==="
+              smartctl -H /dev/sdb | tail -2
+
+              echo "=== full write + verify (destructive) ==="
+              fio --name=devverify \
+                  --filename=/dev/sdb \
+                  --rw=write \
+                  --bs=1M \
+                  --iodepth=4 \
+                  --ioengine=libaio \
+                  --direct=1 \
+                  --verify=crc32c \
+                  --do_verify=1 \
+                  --verify_fatal=1 \
+                  --eta=always \
+                  --output-format=normal
+
+              echo "=== SMART after ==="
+              smartctl -H /dev/sdb | tail -2
+              smartctl -A /dev/sdb | awk '/Reallocated_Sector|Current_Pending|Offline_Uncorrect|UDMA_CRC/{print "  "$0}'
+
+              echo "=== SCAN PASSED ==="
+          volumeMounts:
+            - name: dev
+              mountPath: /dev
+      volumes:
+        - name: dev
+          hostPath:
+            path: /dev

scripts/iam/rotate-oidc.py

@@ -32,7 +32,6 @@ ISSUER = "https://zitadel.riotpiao.com"
 APPS = [
     ("grafana",   "grafana",   "grafana-oidc",         "logging",   "GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET", "deploy", "grafana"),
     ("argocd",    "argocd",    "oidc-secret",           "argocd",    "client-secret",                       "deploy", "argocd-server"),
-    ("comfyui",   "comfyui",   "comfyui-oauth2-proxy", "comfyui",   "client-secret",                       "deploy", "comfyui-oauth2-proxy"),
     ("minio",     "minio",     "minio-oidc",           "storage",   "MINIO_IDENTITY_OPENID_CLIENT_SECRET",  "sts",    "minio-cluster-az-a"),
     ("paperless", "paperless", "paperless-oidc",       "paperless", "client-secret",                       "deploy", "paperless"),
     ("immich",    "immich",    "immich-oidc",          "immich",    "client-secret",                       "deploy", "immich-server"),

scripts/iam/zitadel-provision.py

@@ -128,11 +128,6 @@ OIDC_APPS = {
         "https://forgejo.riotpiao.com/user/oauth2/zitadel/callback",
         "http://localhost:3000/user/oauth2/authentik/callback",
         "https://forgejo.riotpiao.com/user/oauth2/authentik/callback"]},
-    # ComfyUI has no native OIDC. oauth2-proxy is its front door and this app is
-    # the proxy's client, so the callback path belongs to the proxy.
-    "comfyui": {**_WEB, "redirect_uris": [
-        "http://localhost:4180/oauth2/callback",
-        "https://comfyui.riotpiao.com/oauth2/callback"]},
     "grafana": {**_WEB, "redirect_uris": [
         "http://localhost:3000/login/generic_oauth",
         "https://grafana.riotpiao.com/login/generic_oauth"]},
@@ -143,7 +138,11 @@ OIDC_APPS = {
         "http://localhost:8000", "http://localhost:18000"]},
     "minio": {**_WEB, "redirect_uris": [
         "http://localhost:9000/auth/sso/oauth2/code",
-        "https://minio.riotpiao.com/auth/sso/oauth2/code"]},
+        "https://minio.riotpiao.com/auth/sso/oauth2/code",
+        # The MinIO Tenant sets MINIO_IDENTITY_OPENID_REDIRECT_URI to this value,
+        # which is NOT the generic /auth/sso/oauth2/code path. Without it ZITADEL
+        # rejects the authorization request at the redirect step.
+        "https://minio.riotpiao.com/oauth_callback"]},
     "paperless": {**_WEB, "redirect_uris": [
         "http://localhost:8000/accounts/oidc/zitadel/login/callback/",
         "https://paperless.riotpiao.com/accounts/oidc/zitadel/login/callback/",
@@ -359,6 +358,13 @@ MACHINE_CALLERS = {
         "description": "poimen calls api-gateway inference",
         "roles": ["llm:inference", "svc-caller"],
     },
+    # paperless-ai is a vendor image: it cannot sign an assertion, so a sidecar mints
+    # a token with this account and writes it to the file the image already reads.
+    "paperless-llm": {
+        "name": "Paperless AI LLM caller",
+        "description": "paperless-ai calls api-gateway inference through the token vendor",
+        "roles": ["llm:inference", "svc-caller"],
+    },
 }
 
 

terraform/files/coredns/Corefile

@@ -36,8 +36,10 @@
     rewrite name paperless.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name img.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name api.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
-    rewrite name comfy.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
-    rewrite name comfyui.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    # poimen ran only in the live ConfigMap, never in this file or the Talos
+    # machine config. The next Talos reconcile would have deleted it and broken
+    # in-cluster resolution. Declared here so the source of truth carries it.
+    rewrite name poimen.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name gotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name pi.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name igotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local