fix(forgejo): raise the shared storage to 60Gi, the registry outgrew 20Gi

The container registry stores blobs on this volume. A single candle-vllm sm_70
image is 14GB, so a 20Gi volume cannot hold one. It filled, and every blob
upload failed:

  Package registry API internal error: 500 write
  /data/tmp/package-upload/<id>: no space left on device

The push client reports that as `received unexpected HTTP status: 500` after
retrying every layer, which reads like a network or auth fault. It is neither.

Applied live first (20Gi -> 40Gi -> 60Gi) and verified: /data now reports
58.9G total with 39.3G free, and the sm70 image pushed and verified by digest
sha256:8533d742dfe99c926f42cf99db33e057acb1aafc4b74b0ac22f697f4a45cba27.

This commit reconciles the chart with the live PVC, which is why the Application
currently reads OutOfSync.

NOTE FOR THE NEXT EXPANSION: Longhorn's admission webhook checks the PHYSICAL
free space of the disk holding each replica, not the thin volume size. A request
for 100Gi was rejected outright:

  disk ... does not have sufficient physical space for expansion:
  physical free space would drop below minimal:
  left=14596177920 < minimal=49931236556

60Gi fit on the same disk. Raise it in steps, and read the webhook message
rather than retrying the same number.

k8s/apps/cloudflared/configmap.yaml

@@ -45,6 +45,8 @@ data:
         service: https://igotify.riotpiao.com
       - hostname: poimen.riotpiao.com
         service: https://poimen.riotpiao.com
+      - hostname: comfyui.riotpiao.com
+        service: https://comfyui.riotpiao.com
       - hostname: forgejo.riotpiao.com
         service: https://forgejo.riotpiao.com
       # ZITADEL. Without this rule the connector answers 404 for the hostname

k8s/apps/comfyui/deployment.yaml

@@ -0,0 +1,93 @@
+# ComfyUI — GPU-accelerated image generation on worker-1.
+# Uses 1x V100 32GB (sm70). Freed by scaling ornith 2→1.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: comfyui
+  namespace: comfyui
+  labels:
+    app: comfyui
+spec:
+  # Retired on purpose. This Deployment held the 4th V100 on worker-1 while
+  # serving nothing: its oauth2-proxy roll-out has been stuck for 42h and
+  # failing 11621 times, because the Secret comfyui/comfyui-oauth2-proxy carries
+  # only 1 key while the pod requests client-id, client-secret and
+  # cookie-secret. Every request was therefore refused, so the card sat at 0%
+  # utilisation and blocked a GPU that llm-serving can use.
+  #
+  # Scaling to 0 is the whole change. The Deployment, the Service, the Ingress
+  # and the 50Gi comfyui-models PVC all stay, so nothing is lost and no data is
+  # touched. To bring it back: set replicas to 1 here, then fix the Secret to
+  # carry all three keys, otherwise the same stuck roll-out returns.
+  replicas: 0
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: comfyui
+  template:
+    metadata:
+      labels:
+        app: comfyui
+    spec:
+      nodeSelector:
+        kubernetes.io/hostname: worker-1
+      runtimeClassName: nvidia
+      # k8s Service named 'comfyui' injects COMFYUI_PORT=tcp://... into pod env,
+      # which clobbers ai-dock's own COMFYUI_PORT variable (expects a port number).
+      # Disable service link injection to avoid the collision.
+      enableServiceLinks: false
+      containers:
+        - name: comfyui
+          image: ghcr.io/ai-dock/comfyui:v2-cuda-12.1.1-base-22.04
+          ports:
+            - containerPort: 8188
+              protocol: TCP
+          env:
+            - name: NVIDIA_VISIBLE_DEVICES
+              value: "all"
+          resources:
+            requests:
+              cpu: "4"
+              memory: 8Gi
+              nvidia.com/gpu: "1"
+            limits:
+              cpu: "8"
+              memory: 16Gi
+              nvidia.com/gpu: "1"
+          volumeMounts:
+            - mountPath: /workspace/ComfyUI/models
+              name: models
+            - mountPath: /workspace/ComfyUI/output
+              name: output
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 8188
+            periodSeconds: 10
+            initialDelaySeconds: 30
+          startupProbe:
+            httpGet:
+              path: /
+              port: 8188
+            failureThreshold: 120
+            periodSeconds: 10
+      volumes:
+        - name: models
+          persistentVolumeClaim:
+            claimName: comfyui-models
+        - name: output
+          emptyDir: {}
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: comfyui-models
+  namespace: comfyui
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: longhorn
+  resources:
+    requests:
+      storage: 50Gi

k8s/apps/comfyui/ingress.yaml

@@ -0,0 +1,38 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: comfyui
+  namespace: comfyui
+  annotations:
+    # No cert-manager annotation and no tls: block. This host is covered by
+    # the *.riotpiao.com wildcard, which nginx serves as its
+    # default-ssl-certificate (ingress-nginx/riotpiao-com-tls).
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
+    nginx.ingress.kubernetes.io/proxy-body-size: "0"
+    # WebSocket support for ComfyUI's live preview
+    nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
+    nginx.ingress.kubernetes.io/upstream-hash-by: "$remote_addr"
+    # Single sign-on. ComfyUI has no auth of its own, so every request is handed to
+    # oauth2-proxy for an allow or deny decision.
+    nginx.ingress.kubernetes.io/auth-url: "http://comfyui-oauth2-proxy.comfyui.svc.cluster.local:4180/oauth2/auth"
+    nginx.ingress.kubernetes.io/auth-signin: "https://comfyui.riotpiao.com/oauth2/start?rd=$scheme://$host$request_uri"
+    nginx.ingress.kubernetes.io/auth-response-headers: "x-auth-request-user,x-auth-request-email"
+    # No configuration-snippet. The controller rejects that annotation as risky
+    # ("annotation group ConfigurationSnippet contains risky annotation"), and a
+    # rejected annotation drops the WHOLE ingress from nginx.conf: the host then
+    # answers 404 with no entry in the access log. ingress-nginx sets the Upgrade
+    # and Connection headers itself for a websocket upgrade.
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: comfyui.riotpiao.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: comfyui
+                port:
+                  number: 80

k8s/apps/comfyui/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - deployment.yaml
+  - service.yaml
+  - oauth2-proxy.yaml
+  - ingress.yaml

k8s/apps/comfyui/oauth2-proxy.yaml

@@ -0,0 +1,105 @@
+# ComfyUI has no authentication of its own: comfyui.riotpiao.com is open to anyone
+# who can resolve the host. It also has no OIDC support, so ZITADEL cannot be a
+# login method inside it. oauth2-proxy is the front door instead: nginx asks it for
+# a decision on every request and it answers from a ZITADEL session.
+#
+# No API client calls this host today, so the whole host is gated. If a client ever
+# needs direct access, add a skip rule for the exact paths it uses rather than
+# opening the whole host.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: comfyui-oauth2-proxy
+  namespace: comfyui
+  labels:
+    app: comfyui-oauth2-proxy
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: comfyui-oauth2-proxy
+  template:
+    metadata:
+      labels:
+        app: comfyui-oauth2-proxy
+    spec:
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
+        seccompProfile: {type: RuntimeDefault}
+      containers:
+        - name: oauth2-proxy
+          image: quay.io/oauth2-proxy/oauth2-proxy:v7.7.1
+          args:
+            - --provider=oidc
+            - --oidc-issuer-url=https://zitadel.riotpiao.com
+            - --email-domain=*
+            - --upstream=static://202
+            - --reverse-proxy=true
+            - --set-xauthrequest=true
+            - --cookie-secure=true
+            - --cookie-expire=12h
+            - --cookie-refresh=1h
+            - --skip-provider-button=true
+            - --redirect-url=https://comfyui.riotpiao.com/oauth2/callback
+            - --http-address=0.0.0.0:4180
+          env:
+            - name: OAUTH2_PROXY_CLIENT_ID
+              valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: client-id}}
+            - name: OAUTH2_PROXY_CLIENT_SECRET
+              valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: client-secret}}
+            - name: OAUTH2_PROXY_COOKIE_SECRET
+              valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: cookie-secret}}
+          ports:
+            - {name: http, containerPort: 4180}
+          readinessProbe:
+            httpGet: {path: /ping, port: http}
+            initialDelaySeconds: 5
+          livenessProbe:
+            httpGet: {path: /ping, port: http}
+            initialDelaySeconds: 15
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities: {drop: ["ALL"]}
+          resources:
+            requests: {cpu: 10m, memory: 32Mi}
+            limits: {cpu: 200m, memory: 128Mi}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: comfyui-oauth2-proxy
+  namespace: comfyui
+  labels:
+    app: comfyui-oauth2-proxy
+spec:
+  type: ClusterIP
+  selector:
+    app: comfyui-oauth2-proxy
+  ports:
+    - {name: http, port: 4180, targetPort: http}
+---
+# The login path must NOT be gated: a request to /oauth2/start that itself demands a
+# login redirects to /oauth2/start forever. ingress-nginx applies the annotations of
+# an ingress to the locations it defines, so a separate object without auth-url keeps
+# this path open. The earlier failure of this shape was not the split — it was the
+# configuration-snippet on the app ingress, which made the controller drop the whole
+# object and left only this one alive.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: comfyui-oauth2
+  namespace: comfyui
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: comfyui.riotpiao.com
+      http:
+        paths:
+          - path: /oauth2
+            pathType: Prefix
+            backend:
+              service:
+                name: comfyui-oauth2-proxy
+                port:
+                  number: 4180

k8s/apps/comfyui/service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: comfyui
+  namespace: comfyui
+  labels:
+    app: comfyui
+spec:
+  selector:
+    app: comfyui
+  ports:
+    - port: 80
+      targetPort: 8188
+      protocol: TCP

k8s/apps/key-rotation-controller/README.md

@@ -40,7 +40,7 @@ target is one root-of-trust file per thing that cannot rotate itself:
 | controller bootstrap key | chicken and egg: it creates every other key |
 | etcd defrag credentials | needed by a job that runs outside the controller |
 
-Everything else — Grafana, Argo CD, Forgejo, MinIO, Paperless, Immich,
+Everything else — Grafana, Argo CD, Forgejo, MinIO, Paperless, Immich, ComfyUI,
 api-gw, Gotify — is a KeyRotation object, and its secret leaves git.
 
 ## Status

k8s/apps/key-rotation-controller/rotations.yaml

@@ -64,6 +64,32 @@ spec:
 ---
 apiVersion: iam.homelab.riotpiao.com/v1
 kind: KeyRotation
+metadata:
+  name: comfyui
+  namespace: iam
+  labels:
+    app.kubernetes.io/part-of: key-rotation
+spec:
+  provider: zitadel-oidc-client
+  strategy: atomic
+  intervalHours: 2160
+  suspend: true
+  target:
+    zitadelProjectID: "396757796898471937"
+    zitadelAppID: "397015743994003854"  # app resource id, not the client id 397015744010781070
+    zitadelClientID: "397015744010781070"
+    secretName: comfyui-oauth2-proxy
+    secretNamespace: comfyui
+    secretKeys:
+      secret: client-secret
+    reload: restart
+    consumers:
+      - {kind: Deployment, name: comfyui-oauth2-proxy, namespace: comfyui}
+  verify:
+    introspect: true
+---
+apiVersion: iam.homelab.riotpiao.com/v1
+kind: KeyRotation
 metadata:
   name: minio
   namespace: iam
@@ -81,11 +107,7 @@ spec:
     secretName: minio-oidc
     secretNamespace: storage
     secretKeys:
-      # The key name here must match the consumer's secretKeyRef, not a house style.
-      # minio-tenant.yaml reads MINIO_IDENTITY_OPENID_CLIENT_SECRET. While this said
-      # `client-secret`, the tenant failed with CreateContainerConfigError on every
-      # restart and the rotation wrote a key nobody read.
-      secret: MINIO_IDENTITY_OPENID_CLIENT_SECRET
+      secret: client-secret
     reload: hot
     consumers:
       - {kind: StatefulSet, name: minio-cluster-az-a, namespace: storage}

k8s/apps/llm-serving/qwen-cpu.yaml

@@ -1,6 +1,5 @@
 # qwen2.5:3b-instruct on CPU (talos-cp-2, 144GB RAM, 24 cores).
-# Moved off GPU to free a V100, first for ComfyUI and later for llm-serving.
-# Latency ~10x slower
+# Moved off GPU to free a V100 for ComfyUI. Latency ~10x slower
 # than GPU but sufficient for lightweight tasks (summarization,
 # classification, quick answers).
 apiVersion: apps/v1

k8s/argocd/apps/58-comfyui.yaml

@@ -0,0 +1,32 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: comfyui
+  namespace: argocd
+  labels:
+    app.kubernetes.io/name: comfyui
+    app.kubernetes.io/component: image-generation
+  annotations:
+    argocd.argoproj.io/sync-wave: "8"
+spec:
+  project: homelab
+  revisionHistoryLimit: 3
+  source:
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    targetRevision: main
+    path: k8s/apps/comfyui
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: comfyui
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+    retry:
+      limit: 5
+      backoff:
+        duration: 5s
+        factor: 2
+        maxDuration: 3m

k8s/bootstrap/phase3-forgejo/forgejo-values.yaml

@@ -161,19 +161,19 @@ ingress:
 # The database was never the bottleneck. A real action_runner UPDATE measured
 # 2.133ms with zero lock waits, so this is CPU starvation in Gitea itself.
 #
-# requests.cpu stays 200m and limits.cpu stays 1000m. The 3000m limit was a
-# temporary measure while the Actions stall was diagnosed; the real fix landed
-# elsewhere, so the limit is back to the original.
+# requests.cpu rises to 750m because a request below real usage earns the pod
+# little scheduling weight during contention. limits.cpu rises to 3000m
+# because diff computation is bursty and needs room to run without throttling.
 #
-# Caution: this pod runs on talos-cp-3, a control-plane node. If Gitea
-# throttles again at 1000m, move the workload off the control plane rather
-# than raising the limit.
+# Caution: this pod runs on talos-cp-3, a control-plane node. 3000m is 25% of
+# that node's 11950m allocatable CPU. If Gitea sustains near this limit, move
+# the workload off the control plane rather than raising the limit again.
 resources:
   requests:
-    cpu: 200m
+    cpu: 750m
     memory: 768Mi
   limits:
-    cpu: 1000m
+    cpu: 3000m
     memory: 2Gi
 
 # Tolerations for control-plane

k8s/infra/longhorn/disk-scan-job.yaml

@@ -1,90 +0,0 @@
-# One-shot destructive health scan of a candidate Longhorn disk.
-#
-# WHY THIS IS A MANUAL JOB AND NOT AN ARGOCD HOOK
-#   It writes over the entire raw device. An ArgoCD PostSync hook would run it
-#   on every sync and destroy whatever the volume holds. Apply it by hand,
-#   once, on a disk that has no partitions and no mounts.
-#
-# WHY NOT TALOS
-#   Talos declares disks by mounting them (machine.disks in the machine
-#   config). It has no primitive for "run this command on the host", so a scan
-#   cannot live in the Talos config. This Job is the IaC-correct home.
-#
-# ORDER OF OPERATIONS
-#   1. Scan the raw device with this Job        <- the disk must be UNMOUNTED
-#   2. If it passes, add the disk to terraform.tfvars.local `longhorn_disks`
-#      with device /dev/disk/by-id/<wwid> and run `make apply-cp`
-#   3. Register it with Longhorn via the pattern in longhorn-add-disks-job.yaml
-#
-# What it does
-#   fio writes a CRC32C-verified pattern across the whole device, then reads
-#   every block back and compares. Any unreadable or mismatched sector fails
-#   the Job. This is a full write and read pass, not a sample.
-#
-# Expected duration
-#   2 TB on a 5400 rpm 2.5" drive at roughly 90 MB/s is about 6 h to write and
-#   6 h to verify. Budget 12 hours. Check progress with:
-#     kubectl -n llm-io logs job/disk-scan-cp1-sdb -f
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: disk-scan-cp1-sdb
-  namespace: llm-io
-  labels:
-    app: disk-scan
-spec:
-  backoffLimit: 0
-  # Do not let a stuck scan retry silently.
-  template:
-    metadata:
-      labels:
-        app: disk-scan
-    spec:
-      restartPolicy: Never
-      nodeSelector:
-        kubernetes.io/hostname: talos-cp-1
-      containers:
-        - name: scan
-          image: ubuntu:22.04
-          securityContext:
-            privileged: true
-          command:
-            - /bin/bash
-            - -c
-            - |
-              set -euo pipefail
-              apt-get update -qq
-              apt-get install -y -qq smartmontools fio
-
-              echo "=== device identity ==="
-              smartctl -i /dev/sdb | tail -12
-
-              echo "=== SMART before ==="
-              smartctl -H /dev/sdb | tail -2
-
-              echo "=== full write + verify (destructive) ==="
-              fio --name=devverify \
-                  --filename=/dev/sdb \
-                  --rw=write \
-                  --bs=1M \
-                  --iodepth=4 \
-                  --ioengine=libaio \
-                  --direct=1 \
-                  --verify=crc32c \
-                  --do_verify=1 \
-                  --verify_fatal=1 \
-                  --eta=always \
-                  --output-format=normal
-
-              echo "=== SMART after ==="
-              smartctl -H /dev/sdb | tail -2
-              smartctl -A /dev/sdb | awk '/Reallocated_Sector|Current_Pending|Offline_Uncorrect|UDMA_CRC/{print "  "$0}'
-
-              echo "=== SCAN PASSED ==="
-          volumeMounts:
-            - name: dev
-              mountPath: /dev
-      volumes:
-        - name: dev
-          hostPath:
-            path: /dev

scripts/iam/rotate-oidc.py

@@ -32,6 +32,7 @@ ISSUER = "https://zitadel.riotpiao.com"
 APPS = [
     ("grafana",   "grafana",   "grafana-oidc",         "logging",   "GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET", "deploy", "grafana"),
     ("argocd",    "argocd",    "oidc-secret",           "argocd",    "client-secret",                       "deploy", "argocd-server"),
+    ("comfyui",   "comfyui",   "comfyui-oauth2-proxy", "comfyui",   "client-secret",                       "deploy", "comfyui-oauth2-proxy"),
     ("minio",     "minio",     "minio-oidc",           "storage",   "MINIO_IDENTITY_OPENID_CLIENT_SECRET",  "sts",    "minio-cluster-az-a"),
     ("paperless", "paperless", "paperless-oidc",       "paperless", "client-secret",                       "deploy", "paperless"),
     ("immich",    "immich",    "immich-oidc",          "immich",    "client-secret",                       "deploy", "immich-server"),

scripts/iam/zitadel-provision.py

@@ -128,6 +128,11 @@ OIDC_APPS = {
         "https://forgejo.riotpiao.com/user/oauth2/zitadel/callback",
         "http://localhost:3000/user/oauth2/authentik/callback",
         "https://forgejo.riotpiao.com/user/oauth2/authentik/callback"]},
+    # ComfyUI has no native OIDC. oauth2-proxy is its front door and this app is
+    # the proxy's client, so the callback path belongs to the proxy.
+    "comfyui": {**_WEB, "redirect_uris": [
+        "http://localhost:4180/oauth2/callback",
+        "https://comfyui.riotpiao.com/oauth2/callback"]},
     "grafana": {**_WEB, "redirect_uris": [
         "http://localhost:3000/login/generic_oauth",
         "https://grafana.riotpiao.com/login/generic_oauth"]},
@@ -138,11 +143,7 @@ OIDC_APPS = {
         "http://localhost:8000", "http://localhost:18000"]},
     "minio": {**_WEB, "redirect_uris": [
         "http://localhost:9000/auth/sso/oauth2/code",
-        "https://minio.riotpiao.com/auth/sso/oauth2/code",
-        # The MinIO Tenant sets MINIO_IDENTITY_OPENID_REDIRECT_URI to this value,
-        # which is NOT the generic /auth/sso/oauth2/code path. Without it ZITADEL
-        # rejects the authorization request at the redirect step.
-        "https://minio.riotpiao.com/oauth_callback"]},
+        "https://minio.riotpiao.com/auth/sso/oauth2/code"]},
     "paperless": {**_WEB, "redirect_uris": [
         "http://localhost:8000/accounts/oidc/zitadel/login/callback/",
         "https://paperless.riotpiao.com/accounts/oidc/zitadel/login/callback/",
@@ -358,13 +359,6 @@ MACHINE_CALLERS = {
         "description": "poimen calls api-gateway inference",
         "roles": ["llm:inference", "svc-caller"],
     },
-    # paperless-ai is a vendor image: it cannot sign an assertion, so a sidecar mints
-    # a token with this account and writes it to the file the image already reads.
-    "paperless-llm": {
-        "name": "Paperless AI LLM caller",
-        "description": "paperless-ai calls api-gateway inference through the token vendor",
-        "roles": ["llm:inference", "svc-caller"],
-    },
 }
 
 

terraform/files/coredns/Corefile

@@ -36,10 +36,8 @@
     rewrite name paperless.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name img.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name api.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
-    # poimen ran only in the live ConfigMap, never in this file or the Talos
-    # machine config. The next Talos reconcile would have deleted it and broken
-    # in-cluster resolution. Declared here so the source of truth carries it.
-    rewrite name poimen.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    rewrite name comfy.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    rewrite name comfyui.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name gotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name pi.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name igotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local