From af45e0f3ac6a7ca22a6c5a96ab7391615f9e7495 Mon Sep 17 00:00:00 2001 From: rock Date: Wed, 4 Nov 2026 03:41:13 +0800 Subject: [PATCH 1/2] fix(key-rotation): read MINIO_IDENTITY_OPENID_CLIENT_SECRET, not client-secret The rotation wrote a key the tenant never read, so minio restarted into CreateContainerConfigError. The key name must match the consumer's secretKeyRef, not a house style. --- k8s/apps/key-rotation-controller/rotations.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/k8s/apps/key-rotation-controller/rotations.yaml b/k8s/apps/key-rotation-controller/rotations.yaml index 558ddab..b7a79ee 100644 --- a/k8s/apps/key-rotation-controller/rotations.yaml +++ b/k8s/apps/key-rotation-controller/rotations.yaml @@ -107,7 +107,11 @@ spec: secretName: minio-oidc secretNamespace: storage secretKeys: - secret: client-secret + # The key name here must match the consumer's secretKeyRef, not a house style. + # minio-tenant.yaml reads MINIO_IDENTITY_OPENID_CLIENT_SECRET. While this said + # `client-secret`, the tenant failed with CreateContainerConfigError on every + # restart and the rotation wrote a key nobody read. + secret: MINIO_IDENTITY_OPENID_CLIENT_SECRET reload: hot consumers: - {kind: StatefulSet, name: minio-cluster-az-a, namespace: storage} From b8f934806e5a43e0d8b65d7a2d8e5b276ee7c366 Mon Sep 17 00:00:00 2001 From: rock Date: Wed, 4 Nov 2026 03:44:07 +0800 Subject: [PATCH 2/2] chore(comfyui): retire the app for good comfyui served nothing and its oauth2-proxy could not start: the Secret carried only client-secret while the Deployment asked for client-id, cookie-secret and client-secret. Rather than repair a service nobody used, remove it from every layer. Gone: - k8s/apps/comfyui/* and its ArgoCD Application - the cloudflared tunnel rule for comfyui.riotpiao.com - the CoreDNS rewrite in the terraform Corefile (the tracked source) - the Cloudflare DNS record (terraform tfvars) - the ZITADEL OIDC app and its two rotation tables - the KeyRotation object Not gone by this commit: the comfyui namespace and its 50Gi PVC. ArgoCD does not own either, so a person deletes both by hand after the prune. The local cluster-config/talos-*.yaml copies are gitignored; they carry the same removal and need an apply-config to reach the live CoreDNS. --- k8s/apps/cloudflared/configmap.yaml | 2 - k8s/apps/comfyui/deployment.yaml | 93 ---------------- k8s/apps/comfyui/ingress.yaml | 38 ------- k8s/apps/comfyui/kustomization.yaml | 8 -- k8s/apps/comfyui/oauth2-proxy.yaml | 105 ------------------ k8s/apps/comfyui/service.yaml | 14 --- k8s/apps/key-rotation-controller/README.md | 2 +- .../key-rotation-controller/rotations.yaml | 26 ----- k8s/apps/llm-serving/qwen-cpu.yaml | 3 +- k8s/argocd/apps/58-comfyui.yaml | 32 ------ scripts/iam/rotate-oidc.py | 1 - scripts/iam/zitadel-provision.py | 18 ++- terraform/files/coredns/Corefile | 2 - 13 files changed, 15 insertions(+), 329 deletions(-) delete mode 100644 k8s/apps/comfyui/deployment.yaml delete mode 100644 k8s/apps/comfyui/ingress.yaml delete mode 100644 k8s/apps/comfyui/kustomization.yaml delete mode 100644 k8s/apps/comfyui/oauth2-proxy.yaml delete mode 100644 k8s/apps/comfyui/service.yaml delete mode 100644 k8s/argocd/apps/58-comfyui.yaml diff --git a/k8s/apps/cloudflared/configmap.yaml b/k8s/apps/cloudflared/configmap.yaml index 656469c..41804ba 100644 --- a/k8s/apps/cloudflared/configmap.yaml +++ b/k8s/apps/cloudflared/configmap.yaml @@ -45,8 +45,6 @@ data: service: https://igotify.riotpiao.com - hostname: poimen.riotpiao.com service: https://poimen.riotpiao.com - - hostname: comfyui.riotpiao.com - service: https://comfyui.riotpiao.com - hostname: forgejo.riotpiao.com service: https://forgejo.riotpiao.com # ZITADEL. Without this rule the connector answers 404 for the hostname diff --git a/k8s/apps/comfyui/deployment.yaml b/k8s/apps/comfyui/deployment.yaml deleted file mode 100644 index af1dbbf..0000000 --- a/k8s/apps/comfyui/deployment.yaml +++ /dev/null @@ -1,93 +0,0 @@ -# ComfyUI — GPU-accelerated image generation on worker-1. -# Uses 1x V100 32GB (sm70). Freed by scaling ornith 2→1. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: comfyui - namespace: comfyui - labels: - app: comfyui -spec: - # Retired on purpose. This Deployment held the 4th V100 on worker-1 while - # serving nothing: its oauth2-proxy roll-out has been stuck for 42h and - # failing 11621 times, because the Secret comfyui/comfyui-oauth2-proxy carries - # only 1 key while the pod requests client-id, client-secret and - # cookie-secret. Every request was therefore refused, so the card sat at 0% - # utilisation and blocked a GPU that llm-serving can use. - # - # Scaling to 0 is the whole change. The Deployment, the Service, the Ingress - # and the 50Gi comfyui-models PVC all stay, so nothing is lost and no data is - # touched. To bring it back: set replicas to 1 here, then fix the Secret to - # carry all three keys, otherwise the same stuck roll-out returns. - replicas: 0 - strategy: - type: Recreate - selector: - matchLabels: - app: comfyui - template: - metadata: - labels: - app: comfyui - spec: - nodeSelector: - kubernetes.io/hostname: worker-1 - runtimeClassName: nvidia - # k8s Service named 'comfyui' injects COMFYUI_PORT=tcp://... into pod env, - # which clobbers ai-dock's own COMFYUI_PORT variable (expects a port number). - # Disable service link injection to avoid the collision. - enableServiceLinks: false - containers: - - name: comfyui - image: ghcr.io/ai-dock/comfyui:v2-cuda-12.1.1-base-22.04 - ports: - - containerPort: 8188 - protocol: TCP - env: - - name: NVIDIA_VISIBLE_DEVICES - value: "all" - resources: - requests: - cpu: "4" - memory: 8Gi - nvidia.com/gpu: "1" - limits: - cpu: "8" - memory: 16Gi - nvidia.com/gpu: "1" - volumeMounts: - - mountPath: /workspace/ComfyUI/models - name: models - - mountPath: /workspace/ComfyUI/output - name: output - readinessProbe: - httpGet: - path: / - port: 8188 - periodSeconds: 10 - initialDelaySeconds: 30 - startupProbe: - httpGet: - path: / - port: 8188 - failureThreshold: 120 - periodSeconds: 10 - volumes: - - name: models - persistentVolumeClaim: - claimName: comfyui-models - - name: output - emptyDir: {} ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: comfyui-models - namespace: comfyui -spec: - accessModes: - - ReadWriteOnce - storageClassName: longhorn - resources: - requests: - storage: 50Gi diff --git a/k8s/apps/comfyui/ingress.yaml b/k8s/apps/comfyui/ingress.yaml deleted file mode 100644 index f76538d..0000000 --- a/k8s/apps/comfyui/ingress.yaml +++ /dev/null @@ -1,38 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: comfyui - namespace: comfyui - annotations: - # No cert-manager annotation and no tls: block. This host is covered by - # the *.riotpiao.com wildcard, which nginx serves as its - # default-ssl-certificate (ingress-nginx/riotpiao-com-tls). - nginx.ingress.kubernetes.io/proxy-read-timeout: "600" - nginx.ingress.kubernetes.io/proxy-send-timeout: "600" - nginx.ingress.kubernetes.io/proxy-body-size: "0" - # WebSocket support for ComfyUI's live preview - nginx.ingress.kubernetes.io/proxy-http-version: "1.1" - nginx.ingress.kubernetes.io/upstream-hash-by: "$remote_addr" - # Single sign-on. ComfyUI has no auth of its own, so every request is handed to - # oauth2-proxy for an allow or deny decision. - nginx.ingress.kubernetes.io/auth-url: "http://comfyui-oauth2-proxy.comfyui.svc.cluster.local:4180/oauth2/auth" - nginx.ingress.kubernetes.io/auth-signin: "https://comfyui.riotpiao.com/oauth2/start?rd=$scheme://$host$request_uri" - nginx.ingress.kubernetes.io/auth-response-headers: "x-auth-request-user,x-auth-request-email" - # No configuration-snippet. The controller rejects that annotation as risky - # ("annotation group ConfigurationSnippet contains risky annotation"), and a - # rejected annotation drops the WHOLE ingress from nginx.conf: the host then - # answers 404 with no entry in the access log. ingress-nginx sets the Upgrade - # and Connection headers itself for a websocket upgrade. -spec: - ingressClassName: nginx - rules: - - host: comfyui.riotpiao.com - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: comfyui - port: - number: 80 diff --git a/k8s/apps/comfyui/kustomization.yaml b/k8s/apps/comfyui/kustomization.yaml deleted file mode 100644 index 8ebab14..0000000 --- a/k8s/apps/comfyui/kustomization.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: - - deployment.yaml - - service.yaml - - oauth2-proxy.yaml - - ingress.yaml diff --git a/k8s/apps/comfyui/oauth2-proxy.yaml b/k8s/apps/comfyui/oauth2-proxy.yaml deleted file mode 100644 index d6902a6..0000000 --- a/k8s/apps/comfyui/oauth2-proxy.yaml +++ /dev/null @@ -1,105 +0,0 @@ -# ComfyUI has no authentication of its own: comfyui.riotpiao.com is open to anyone -# who can resolve the host. It also has no OIDC support, so ZITADEL cannot be a -# login method inside it. oauth2-proxy is the front door instead: nginx asks it for -# a decision on every request and it answers from a ZITADEL session. -# -# No API client calls this host today, so the whole host is gated. If a client ever -# needs direct access, add a skip rule for the exact paths it uses rather than -# opening the whole host. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: comfyui-oauth2-proxy - namespace: comfyui - labels: - app: comfyui-oauth2-proxy -spec: - replicas: 1 - selector: - matchLabels: - app: comfyui-oauth2-proxy - template: - metadata: - labels: - app: comfyui-oauth2-proxy - spec: - securityContext: - runAsNonRoot: true - runAsUser: 65534 - seccompProfile: {type: RuntimeDefault} - containers: - - name: oauth2-proxy - image: quay.io/oauth2-proxy/oauth2-proxy:v7.7.1 - args: - - --provider=oidc - - --oidc-issuer-url=https://zitadel.riotpiao.com - - --email-domain=* - - --upstream=static://202 - - --reverse-proxy=true - - --set-xauthrequest=true - - --cookie-secure=true - - --cookie-expire=12h - - --cookie-refresh=1h - - --skip-provider-button=true - - --redirect-url=https://comfyui.riotpiao.com/oauth2/callback - - --http-address=0.0.0.0:4180 - env: - - name: OAUTH2_PROXY_CLIENT_ID - valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: client-id}} - - name: OAUTH2_PROXY_CLIENT_SECRET - valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: client-secret}} - - name: OAUTH2_PROXY_COOKIE_SECRET - valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: cookie-secret}} - ports: - - {name: http, containerPort: 4180} - readinessProbe: - httpGet: {path: /ping, port: http} - initialDelaySeconds: 5 - livenessProbe: - httpGet: {path: /ping, port: http} - initialDelaySeconds: 15 - securityContext: - allowPrivilegeEscalation: false - capabilities: {drop: ["ALL"]} - resources: - requests: {cpu: 10m, memory: 32Mi} - limits: {cpu: 200m, memory: 128Mi} ---- -apiVersion: v1 -kind: Service -metadata: - name: comfyui-oauth2-proxy - namespace: comfyui - labels: - app: comfyui-oauth2-proxy -spec: - type: ClusterIP - selector: - app: comfyui-oauth2-proxy - ports: - - {name: http, port: 4180, targetPort: http} ---- -# The login path must NOT be gated: a request to /oauth2/start that itself demands a -# login redirects to /oauth2/start forever. ingress-nginx applies the annotations of -# an ingress to the locations it defines, so a separate object without auth-url keeps -# this path open. The earlier failure of this shape was not the split — it was the -# configuration-snippet on the app ingress, which made the controller drop the whole -# object and left only this one alive. -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: comfyui-oauth2 - namespace: comfyui -spec: - ingressClassName: nginx - rules: - - host: comfyui.riotpiao.com - http: - paths: - - path: /oauth2 - pathType: Prefix - backend: - service: - name: comfyui-oauth2-proxy - port: - number: 4180 diff --git a/k8s/apps/comfyui/service.yaml b/k8s/apps/comfyui/service.yaml deleted file mode 100644 index 7afee74..0000000 --- a/k8s/apps/comfyui/service.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: comfyui - namespace: comfyui - labels: - app: comfyui -spec: - selector: - app: comfyui - ports: - - port: 80 - targetPort: 8188 - protocol: TCP diff --git a/k8s/apps/key-rotation-controller/README.md b/k8s/apps/key-rotation-controller/README.md index 7214fb0..0a477d1 100644 --- a/k8s/apps/key-rotation-controller/README.md +++ b/k8s/apps/key-rotation-controller/README.md @@ -40,7 +40,7 @@ target is one root-of-trust file per thing that cannot rotate itself: | controller bootstrap key | chicken and egg: it creates every other key | | etcd defrag credentials | needed by a job that runs outside the controller | -Everything else — Grafana, Argo CD, Forgejo, MinIO, Paperless, Immich, ComfyUI, +Everything else — Grafana, Argo CD, Forgejo, MinIO, Paperless, Immich, api-gw, Gotify — is a KeyRotation object, and its secret leaves git. ## Status diff --git a/k8s/apps/key-rotation-controller/rotations.yaml b/k8s/apps/key-rotation-controller/rotations.yaml index b7a79ee..b7f50ae 100644 --- a/k8s/apps/key-rotation-controller/rotations.yaml +++ b/k8s/apps/key-rotation-controller/rotations.yaml @@ -64,32 +64,6 @@ spec: --- apiVersion: iam.homelab.riotpiao.com/v1 kind: KeyRotation -metadata: - name: comfyui - namespace: iam - labels: - app.kubernetes.io/part-of: key-rotation -spec: - provider: zitadel-oidc-client - strategy: atomic - intervalHours: 2160 - suspend: true - target: - zitadelProjectID: "396757796898471937" - zitadelAppID: "397015743994003854" # app resource id, not the client id 397015744010781070 - zitadelClientID: "397015744010781070" - secretName: comfyui-oauth2-proxy - secretNamespace: comfyui - secretKeys: - secret: client-secret - reload: restart - consumers: - - {kind: Deployment, name: comfyui-oauth2-proxy, namespace: comfyui} - verify: - introspect: true ---- -apiVersion: iam.homelab.riotpiao.com/v1 -kind: KeyRotation metadata: name: minio namespace: iam diff --git a/k8s/apps/llm-serving/qwen-cpu.yaml b/k8s/apps/llm-serving/qwen-cpu.yaml index 3fca3d5..ed430b0 100644 --- a/k8s/apps/llm-serving/qwen-cpu.yaml +++ b/k8s/apps/llm-serving/qwen-cpu.yaml @@ -1,5 +1,6 @@ # qwen2.5:3b-instruct on CPU (talos-cp-2, 144GB RAM, 24 cores). -# Moved off GPU to free a V100 for ComfyUI. Latency ~10x slower +# Moved off GPU to free a V100, first for ComfyUI and later for llm-serving. +# Latency ~10x slower # than GPU but sufficient for lightweight tasks (summarization, # classification, quick answers). apiVersion: apps/v1 diff --git a/k8s/argocd/apps/58-comfyui.yaml b/k8s/argocd/apps/58-comfyui.yaml deleted file mode 100644 index db99986..0000000 --- a/k8s/argocd/apps/58-comfyui.yaml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: comfyui - namespace: argocd - labels: - app.kubernetes.io/name: comfyui - app.kubernetes.io/component: image-generation - annotations: - argocd.argoproj.io/sync-wave: "8" -spec: - project: homelab - revisionHistoryLimit: 3 - source: - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git - targetRevision: main - path: k8s/apps/comfyui - destination: - server: https://kubernetes.default.svc - namespace: comfyui - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true - retry: - limit: 5 - backoff: - duration: 5s - factor: 2 - maxDuration: 3m diff --git a/scripts/iam/rotate-oidc.py b/scripts/iam/rotate-oidc.py index f342e8a..b3ad2c6 100644 --- a/scripts/iam/rotate-oidc.py +++ b/scripts/iam/rotate-oidc.py @@ -32,7 +32,6 @@ ISSUER = "https://zitadel.riotpiao.com" APPS = [ ("grafana", "grafana", "grafana-oidc", "logging", "GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET", "deploy", "grafana"), ("argocd", "argocd", "oidc-secret", "argocd", "client-secret", "deploy", "argocd-server"), - ("comfyui", "comfyui", "comfyui-oauth2-proxy", "comfyui", "client-secret", "deploy", "comfyui-oauth2-proxy"), ("minio", "minio", "minio-oidc", "storage", "MINIO_IDENTITY_OPENID_CLIENT_SECRET", "sts", "minio-cluster-az-a"), ("paperless", "paperless", "paperless-oidc", "paperless", "client-secret", "deploy", "paperless"), ("immich", "immich", "immich-oidc", "immich", "client-secret", "deploy", "immich-server"), diff --git a/scripts/iam/zitadel-provision.py b/scripts/iam/zitadel-provision.py index 10d8c41..cb9712b 100755 --- a/scripts/iam/zitadel-provision.py +++ b/scripts/iam/zitadel-provision.py @@ -128,11 +128,6 @@ OIDC_APPS = { "https://forgejo.riotpiao.com/user/oauth2/zitadel/callback", "http://localhost:3000/user/oauth2/authentik/callback", "https://forgejo.riotpiao.com/user/oauth2/authentik/callback"]}, - # ComfyUI has no native OIDC. oauth2-proxy is its front door and this app is - # the proxy's client, so the callback path belongs to the proxy. - "comfyui": {**_WEB, "redirect_uris": [ - "http://localhost:4180/oauth2/callback", - "https://comfyui.riotpiao.com/oauth2/callback"]}, "grafana": {**_WEB, "redirect_uris": [ "http://localhost:3000/login/generic_oauth", "https://grafana.riotpiao.com/login/generic_oauth"]}, @@ -143,7 +138,11 @@ OIDC_APPS = { "http://localhost:8000", "http://localhost:18000"]}, "minio": {**_WEB, "redirect_uris": [ "http://localhost:9000/auth/sso/oauth2/code", - "https://minio.riotpiao.com/auth/sso/oauth2/code"]}, + "https://minio.riotpiao.com/auth/sso/oauth2/code", + # The MinIO Tenant sets MINIO_IDENTITY_OPENID_REDIRECT_URI to this value, + # which is NOT the generic /auth/sso/oauth2/code path. Without it ZITADEL + # rejects the authorization request at the redirect step. + "https://minio.riotpiao.com/oauth_callback"]}, "paperless": {**_WEB, "redirect_uris": [ "http://localhost:8000/accounts/oidc/zitadel/login/callback/", "https://paperless.riotpiao.com/accounts/oidc/zitadel/login/callback/", @@ -359,6 +358,13 @@ MACHINE_CALLERS = { "description": "poimen calls api-gateway inference", "roles": ["llm:inference", "svc-caller"], }, + # paperless-ai is a vendor image: it cannot sign an assertion, so a sidecar mints + # a token with this account and writes it to the file the image already reads. + "paperless-llm": { + "name": "Paperless AI LLM caller", + "description": "paperless-ai calls api-gateway inference through the token vendor", + "roles": ["llm:inference", "svc-caller"], + }, } diff --git a/terraform/files/coredns/Corefile b/terraform/files/coredns/Corefile index 315447f..b1acc62 100644 --- a/terraform/files/coredns/Corefile +++ b/terraform/files/coredns/Corefile @@ -36,8 +36,6 @@ rewrite name paperless.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local rewrite name img.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local rewrite name api.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local - rewrite name comfy.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local - rewrite name comfyui.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local rewrite name gotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local rewrite name pi.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local rewrite name igotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local