riotpiao-poimen/homelab
2
0
Fork 0
Code Issues 24 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Merge pull request 'chore(comfyui): retire the app for good' (#97) from chore/retire-comfyui-fully into main

Browse Source
This commit was merged in pull request #97.
This commit is contained in:
rock
2026-11-03 19:44:18 +00:00
parent 814f96040f b8f934806e
commit b45ae269fd
13 changed files with 20 additions and 330 deletions
Download Patch File Download Diff File Expand all files Collapse all files
k8s/apps/cloudflared/configmap.yaml
-2
View File
@@ -45,8 +45,6 @@ data:
service: https://igotify.riotpiao.com
- hostname: poimen.riotpiao.com
service: https://poimen.riotpiao.com
- hostname: comfyui.riotpiao.com
service: https://comfyui.riotpiao.com
- hostname: forgejo.riotpiao.com
service: https://forgejo.riotpiao.com
# ZITADEL. Without this rule the connector answers 404 for the hostname
k8s/apps/comfyui/deployment.yaml
-93
View File
@@ -1,93 +0,0 @@
# ComfyUI — GPU-accelerated image generation on worker-1.
# Uses 1x V100 32GB (sm70). Freed by scaling ornith 2→1.
apiVersion: apps/v1
kind: Deployment
metadata:
name: comfyui
namespace: comfyui
labels:
app: comfyui
spec:
# Retired on purpose. This Deployment held the 4th V100 on worker-1 while
# serving nothing: its oauth2-proxy roll-out has been stuck for 42h and
# failing 11621 times, because the Secret comfyui/comfyui-oauth2-proxy carries
# only 1 key while the pod requests client-id, client-secret and
# cookie-secret. Every request was therefore refused, so the card sat at 0%
# utilisation and blocked a GPU that llm-serving can use.
#
# Scaling to 0 is the whole change. The Deployment, the Service, the Ingress
# and the 50Gi comfyui-models PVC all stay, so nothing is lost and no data is
# touched. To bring it back: set replicas to 1 here, then fix the Secret to
# carry all three keys, otherwise the same stuck roll-out returns.
replicas: 0
strategy:
type: Recreate
selector:
matchLabels:
app: comfyui
template:
metadata:
labels:
app: comfyui
spec:
nodeSelector:
kubernetes.io/hostname: worker-1
runtimeClassName: nvidia
# k8s Service named 'comfyui' injects COMFYUI_PORT=tcp://... into pod env,
# which clobbers ai-dock's own COMFYUI_PORT variable (expects a port number).
# Disable service link injection to avoid the collision.
enableServiceLinks: false
containers:
- name: comfyui
image: ghcr.io/ai-dock/comfyui:v2-cuda-12.1.1-base-22.04
ports:
- containerPort: 8188
protocol: TCP
env:
- name: NVIDIA_VISIBLE_DEVICES
value: "all"
resources:
requests:
cpu: "4"
memory: 8Gi
nvidia.com/gpu: "1"
limits:
cpu: "8"
memory: 16Gi
nvidia.com/gpu: "1"
volumeMounts:
- mountPath: /workspace/ComfyUI/models
name: models
- mountPath: /workspace/ComfyUI/output
name: output
readinessProbe:
httpGet:
path: /
port: 8188
periodSeconds: 10
initialDelaySeconds: 30
startupProbe:
httpGet:
path: /
port: 8188
failureThreshold: 120
periodSeconds: 10
volumes:
- name: models
persistentVolumeClaim:
claimName: comfyui-models
- name: output
emptyDir: {}
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: comfyui-models
namespace: comfyui
spec:
accessModes:
- ReadWriteOnce
storageClassName: longhorn
resources:
requests:
storage: 50Gi
k8s/apps/comfyui/ingress.yaml
-38
View File
@@ -1,38 +0,0 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: comfyui
namespace: comfyui
annotations:
# No cert-manager annotation and no tls: block. This host is covered by
# the *.riotpiao.com wildcard, which nginx serves as its
# default-ssl-certificate (ingress-nginx/riotpiao-com-tls).
nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
nginx.ingress.kubernetes.io/proxy-body-size: "0"
# WebSocket support for ComfyUI's live preview
nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
nginx.ingress.kubernetes.io/upstream-hash-by: "$remote_addr"
# Single sign-on. ComfyUI has no auth of its own, so every request is handed to
# oauth2-proxy for an allow or deny decision.
nginx.ingress.kubernetes.io/auth-url: "http://comfyui-oauth2-proxy.comfyui.svc.cluster.local:4180/oauth2/auth"
nginx.ingress.kubernetes.io/auth-signin: "https://comfyui.riotpiao.com/oauth2/start?rd=$scheme://$host$request_uri"
nginx.ingress.kubernetes.io/auth-response-headers: "x-auth-request-user,x-auth-request-email"
# No configuration-snippet. The controller rejects that annotation as risky
# ("annotation group ConfigurationSnippet contains risky annotation"), and a
# rejected annotation drops the WHOLE ingress from nginx.conf: the host then
# answers 404 with no entry in the access log. ingress-nginx sets the Upgrade
# and Connection headers itself for a websocket upgrade.
spec:
ingressClassName: nginx
rules:
- host: comfyui.riotpiao.com
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: comfyui
port:
number: 80
k8s/apps/comfyui/kustomization.yaml
-8
View File
@@ -1,8 +0,0 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- deployment.yaml
- service.yaml
- oauth2-proxy.yaml
- ingress.yaml
k8s/apps/comfyui/oauth2-proxy.yaml
-105
View File
@@ -1,105 +0,0 @@
# ComfyUI has no authentication of its own: comfyui.riotpiao.com is open to anyone
# who can resolve the host. It also has no OIDC support, so ZITADEL cannot be a
# login method inside it. oauth2-proxy is the front door instead: nginx asks it for
# a decision on every request and it answers from a ZITADEL session.
#
# No API client calls this host today, so the whole host is gated. If a client ever
# needs direct access, add a skip rule for the exact paths it uses rather than
# opening the whole host.
apiVersion: apps/v1
kind: Deployment
metadata:
name: comfyui-oauth2-proxy
namespace: comfyui
labels:
app: comfyui-oauth2-proxy
spec:
replicas: 1
selector:
matchLabels:
app: comfyui-oauth2-proxy
template:
metadata:
labels:
app: comfyui-oauth2-proxy
spec:
securityContext:
runAsNonRoot: true
runAsUser: 65534
seccompProfile: {type: RuntimeDefault}
containers:
- name: oauth2-proxy
image: quay.io/oauth2-proxy/oauth2-proxy:v7.7.1
args:
- --provider=oidc
- --oidc-issuer-url=https://zitadel.riotpiao.com
- --email-domain=*
- --upstream=static://202
- --reverse-proxy=true
- --set-xauthrequest=true
- --cookie-secure=true
- --cookie-expire=12h
- --cookie-refresh=1h
- --skip-provider-button=true
- --redirect-url=https://comfyui.riotpiao.com/oauth2/callback
- --http-address=0.0.0.0:4180
env:
- name: OAUTH2_PROXY_CLIENT_ID
valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: client-id}}
- name: OAUTH2_PROXY_CLIENT_SECRET
valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: client-secret}}
- name: OAUTH2_PROXY_COOKIE_SECRET
valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: cookie-secret}}
ports:
- {name: http, containerPort: 4180}
readinessProbe:
httpGet: {path: /ping, port: http}
initialDelaySeconds: 5
livenessProbe:
httpGet: {path: /ping, port: http}
initialDelaySeconds: 15
securityContext:
allowPrivilegeEscalation: false
capabilities: {drop: ["ALL"]}
resources:
requests: {cpu: 10m, memory: 32Mi}
limits: {cpu: 200m, memory: 128Mi}
---
apiVersion: v1
kind: Service
metadata:
name: comfyui-oauth2-proxy
namespace: comfyui
labels:
app: comfyui-oauth2-proxy
spec:
type: ClusterIP
selector:
app: comfyui-oauth2-proxy
ports:
- {name: http, port: 4180, targetPort: http}
---
# The login path must NOT be gated: a request to /oauth2/start that itself demands a
# login redirects to /oauth2/start forever. ingress-nginx applies the annotations of
# an ingress to the locations it defines, so a separate object without auth-url keeps
# this path open. The earlier failure of this shape was not the split — it was the
# configuration-snippet on the app ingress, which made the controller drop the whole
# object and left only this one alive.
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: comfyui-oauth2
namespace: comfyui
spec:
ingressClassName: nginx
rules:
- host: comfyui.riotpiao.com
http:
paths:
- path: /oauth2
pathType: Prefix
backend:
service:
name: comfyui-oauth2-proxy
port:
number: 4180
k8s/apps/comfyui/service.yaml
-14
View File
@@ -1,14 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: comfyui
namespace: comfyui
labels:
app: comfyui
spec:
selector:
app: comfyui
ports:
- port: 80
targetPort: 8188
protocol: TCP
k8s/apps/key-rotation-controller/README.md
+1 -1
View File
@@ -40,7 +40,7 @@ target is one root-of-trust file per thing that cannot rotate itself:
| controller bootstrap key | chicken and egg: it creates every other key |
| etcd defrag credentials | needed by a job that runs outside the controller |
Everything else — Grafana, Argo CD, Forgejo, MinIO, Paperless, Immich, ComfyUI,
Everything else — Grafana, Argo CD, Forgejo, MinIO, Paperless, Immich,
api-gw, Gotify — is a KeyRotation object, and its secret leaves git.
## Status
k8s/apps/key-rotation-controller/rotations.yaml
+5 -27
View File
@@ -64,32 +64,6 @@ spec:
---
apiVersion: iam.homelab.riotpiao.com/v1
kind: KeyRotation
metadata:
name: comfyui
namespace: iam
labels:
app.kubernetes.io/part-of: key-rotation
spec:
provider: zitadel-oidc-client
strategy: atomic
intervalHours: 2160
suspend: true
target:
zitadelProjectID: "396757796898471937"
zitadelAppID: "397015743994003854" # app resource id, not the client id 397015744010781070
zitadelClientID: "397015744010781070"
secretName: comfyui-oauth2-proxy
secretNamespace: comfyui
secretKeys:
secret: client-secret
reload: restart
consumers:
- {kind: Deployment, name: comfyui-oauth2-proxy, namespace: comfyui}
verify:
introspect: true
---
apiVersion: iam.homelab.riotpiao.com/v1
kind: KeyRotation
metadata:
name: minio
namespace: iam
@@ -107,7 +81,11 @@ spec:
secretName: minio-oidc
secretNamespace: storage
secretKeys:
secret: client-secret
# The key name here must match the consumer's secretKeyRef, not a house style.
# minio-tenant.yaml reads MINIO_IDENTITY_OPENID_CLIENT_SECRET. While this said
# `client-secret`, the tenant failed with CreateContainerConfigError on every
# restart and the rotation wrote a key nobody read.
secret: MINIO_IDENTITY_OPENID_CLIENT_SECRET
reload: hot
consumers:
- {kind: StatefulSet, name: minio-cluster-az-a, namespace: storage}
k8s/apps/llm-serving/qwen-cpu.yaml
+2 -1
View File
@@ -1,5 +1,6 @@
# qwen2.5:3b-instruct on CPU (talos-cp-2, 144GB RAM, 24 cores).
# Moved off GPU to free a V100 for ComfyUI. Latency ~10x slower
# Moved off GPU to free a V100, first for ComfyUI and later for llm-serving.
# Latency ~10x slower
# than GPU but sufficient for lightweight tasks (summarization,
# classification, quick answers).
apiVersion: apps/v1
k8s/argocd/apps/58-comfyui.yaml
-32
View File
@@ -1,32 +0,0 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: comfyui
namespace: argocd
labels:
app.kubernetes.io/name: comfyui
app.kubernetes.io/component: image-generation
annotations:
argocd.argoproj.io/sync-wave: "8"
spec:
project: homelab
revisionHistoryLimit: 3
source:
repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
targetRevision: main
path: k8s/apps/comfyui
destination:
server: https://kubernetes.default.svc
namespace: comfyui
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true
retry:
limit: 5
backoff:
duration: 5s
factor: 2
maxDuration: 3m
scripts/iam/rotate-oidc.py
-1
View File
@@ -32,7 +32,6 @@ ISSUER = "https://zitadel.riotpiao.com"
APPS = [
("grafana", "grafana", "grafana-oidc", "logging", "GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET", "deploy", "grafana"),
("argocd", "argocd", "oidc-secret", "argocd", "client-secret", "deploy", "argocd-server"),
("comfyui", "comfyui", "comfyui-oauth2-proxy", "comfyui", "client-secret", "deploy", "comfyui-oauth2-proxy"),
("minio", "minio", "minio-oidc", "storage", "MINIO_IDENTITY_OPENID_CLIENT_SECRET", "sts", "minio-cluster-az-a"),
("paperless", "paperless", "paperless-oidc", "paperless", "client-secret", "deploy", "paperless"),
("immich", "immich", "immich-oidc", "immich", "client-secret", "deploy", "immich-server"),
scripts/iam/zitadel-provision.py
+12 -6
View File
@@ -128,11 +128,6 @@ OIDC_APPS = {
"https://forgejo.riotpiao.com/user/oauth2/zitadel/callback",
"http://localhost:3000/user/oauth2/authentik/callback",
"https://forgejo.riotpiao.com/user/oauth2/authentik/callback"]},
# ComfyUI has no native OIDC. oauth2-proxy is its front door and this app is
# the proxy's client, so the callback path belongs to the proxy.
"comfyui": {**_WEB, "redirect_uris": [
"http://localhost:4180/oauth2/callback",
"https://comfyui.riotpiao.com/oauth2/callback"]},
"grafana": {**_WEB, "redirect_uris": [
"http://localhost:3000/login/generic_oauth",
"https://grafana.riotpiao.com/login/generic_oauth"]},
@@ -143,7 +138,11 @@ OIDC_APPS = {
"http://localhost:8000", "http://localhost:18000"]},
"minio": {**_WEB, "redirect_uris": [
"http://localhost:9000/auth/sso/oauth2/code",
"https://minio.riotpiao.com/auth/sso/oauth2/code"]},
"https://minio.riotpiao.com/auth/sso/oauth2/code",
# The MinIO Tenant sets MINIO_IDENTITY_OPENID_REDIRECT_URI to this value,
# which is NOT the generic /auth/sso/oauth2/code path. Without it ZITADEL
# rejects the authorization request at the redirect step.
"https://minio.riotpiao.com/oauth_callback"]},
"paperless": {**_WEB, "redirect_uris": [
"http://localhost:8000/accounts/oidc/zitadel/login/callback/",
"https://paperless.riotpiao.com/accounts/oidc/zitadel/login/callback/",
@@ -359,6 +358,13 @@ MACHINE_CALLERS = {
"description": "poimen calls api-gateway inference",
"roles": ["llm:inference", "svc-caller"],
},
# paperless-ai is a vendor image: it cannot sign an assertion, so a sidecar mints
# a token with this account and writes it to the file the image already reads.
"paperless-llm": {
"name": "Paperless AI LLM caller",
"description": "paperless-ai calls api-gateway inference through the token vendor",
"roles": ["llm:inference", "svc-caller"],
},
}
terraform/files/coredns/Corefile
-2
View File
@@ -36,8 +36,6 @@
rewrite name paperless.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
rewrite name img.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
rewrite name api.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
rewrite name comfy.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
rewrite name comfyui.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
rewrite name gotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
rewrite name pi.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
rewrite name igotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local