k8s(llm-serving): stop managing the coder InferenceService

coder.yaml requests 2 V100s and never reaches Ready, so it permanently
occupies half the GPU pool. candle-vllm serves the coder model on the
other two cards; the unmanaged predictor starves clef-serve and blocks
its own rollout on every sync.

ArgoCD has prune:false, so this removes management only. The live object
is deleted manually after the sync, otherwise selfHeal recreates it.

k8s/apps/cloudflared/configmap.yaml

@@ -45,8 +45,6 @@ data:
         service: https://igotify.riotpiao.com
       - hostname: poimen.riotpiao.com
         service: https://poimen.riotpiao.com
-      - hostname: comfyui.riotpiao.com
-        service: https://comfyui.riotpiao.com
       - hostname: forgejo.riotpiao.com
         service: https://forgejo.riotpiao.com
       # ZITADEL. Without this rule the connector answers 404 for the hostname

k8s/apps/comfyui/deployment.yaml

@@ -1,93 +0,0 @@
-# ComfyUI — GPU-accelerated image generation on worker-1.
-# Uses 1x V100 32GB (sm70). Freed by scaling ornith 2→1.
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: comfyui
-  namespace: comfyui
-  labels:
-    app: comfyui
-spec:
-  # Retired on purpose. This Deployment held the 4th V100 on worker-1 while
-  # serving nothing: its oauth2-proxy roll-out has been stuck for 42h and
-  # failing 11621 times, because the Secret comfyui/comfyui-oauth2-proxy carries
-  # only 1 key while the pod requests client-id, client-secret and
-  # cookie-secret. Every request was therefore refused, so the card sat at 0%
-  # utilisation and blocked a GPU that llm-serving can use.
-  #
-  # Scaling to 0 is the whole change. The Deployment, the Service, the Ingress
-  # and the 50Gi comfyui-models PVC all stay, so nothing is lost and no data is
-  # touched. To bring it back: set replicas to 1 here, then fix the Secret to
-  # carry all three keys, otherwise the same stuck roll-out returns.
-  replicas: 0
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app: comfyui
-  template:
-    metadata:
-      labels:
-        app: comfyui
-    spec:
-      nodeSelector:
-        kubernetes.io/hostname: worker-1
-      runtimeClassName: nvidia
-      # k8s Service named 'comfyui' injects COMFYUI_PORT=tcp://... into pod env,
-      # which clobbers ai-dock's own COMFYUI_PORT variable (expects a port number).
-      # Disable service link injection to avoid the collision.
-      enableServiceLinks: false
-      containers:
-        - name: comfyui
-          image: ghcr.io/ai-dock/comfyui:v2-cuda-12.1.1-base-22.04
-          ports:
-            - containerPort: 8188
-              protocol: TCP
-          env:
-            - name: NVIDIA_VISIBLE_DEVICES
-              value: "all"
-          resources:
-            requests:
-              cpu: "4"
-              memory: 8Gi
-              nvidia.com/gpu: "1"
-            limits:
-              cpu: "8"
-              memory: 16Gi
-              nvidia.com/gpu: "1"
-          volumeMounts:
-            - mountPath: /workspace/ComfyUI/models
-              name: models
-            - mountPath: /workspace/ComfyUI/output
-              name: output
-          readinessProbe:
-            httpGet:
-              path: /
-              port: 8188
-            periodSeconds: 10
-            initialDelaySeconds: 30
-          startupProbe:
-            httpGet:
-              path: /
-              port: 8188
-            failureThreshold: 120
-            periodSeconds: 10
-      volumes:
-        - name: models
-          persistentVolumeClaim:
-            claimName: comfyui-models
-        - name: output
-          emptyDir: {}
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: comfyui-models
-  namespace: comfyui
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: longhorn
-  resources:
-    requests:
-      storage: 50Gi

k8s/apps/comfyui/ingress.yaml

@@ -1,38 +0,0 @@
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: comfyui
-  namespace: comfyui
-  annotations:
-    # No cert-manager annotation and no tls: block. This host is covered by
-    # the *.riotpiao.com wildcard, which nginx serves as its
-    # default-ssl-certificate (ingress-nginx/riotpiao-com-tls).
-    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
-    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
-    nginx.ingress.kubernetes.io/proxy-body-size: "0"
-    # WebSocket support for ComfyUI's live preview
-    nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
-    nginx.ingress.kubernetes.io/upstream-hash-by: "$remote_addr"
-    # Single sign-on. ComfyUI has no auth of its own, so every request is handed to
-    # oauth2-proxy for an allow or deny decision.
-    nginx.ingress.kubernetes.io/auth-url: "http://comfyui-oauth2-proxy.comfyui.svc.cluster.local:4180/oauth2/auth"
-    nginx.ingress.kubernetes.io/auth-signin: "https://comfyui.riotpiao.com/oauth2/start?rd=$scheme://$host$request_uri"
-    nginx.ingress.kubernetes.io/auth-response-headers: "x-auth-request-user,x-auth-request-email"
-    # No configuration-snippet. The controller rejects that annotation as risky
-    # ("annotation group ConfigurationSnippet contains risky annotation"), and a
-    # rejected annotation drops the WHOLE ingress from nginx.conf: the host then
-    # answers 404 with no entry in the access log. ingress-nginx sets the Upgrade
-    # and Connection headers itself for a websocket upgrade.
-spec:
-  ingressClassName: nginx
-  rules:
-    - host: comfyui.riotpiao.com
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: comfyui
-                port:
-                  number: 80

k8s/apps/comfyui/kustomization.yaml

@@ -1,8 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-  - deployment.yaml
-  - service.yaml
-  - oauth2-proxy.yaml
-  - ingress.yaml

k8s/apps/comfyui/oauth2-proxy.yaml

@@ -1,105 +0,0 @@
-# ComfyUI has no authentication of its own: comfyui.riotpiao.com is open to anyone
-# who can resolve the host. It also has no OIDC support, so ZITADEL cannot be a
-# login method inside it. oauth2-proxy is the front door instead: nginx asks it for
-# a decision on every request and it answers from a ZITADEL session.
-#
-# No API client calls this host today, so the whole host is gated. If a client ever
-# needs direct access, add a skip rule for the exact paths it uses rather than
-# opening the whole host.
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: comfyui-oauth2-proxy
-  namespace: comfyui
-  labels:
-    app: comfyui-oauth2-proxy
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: comfyui-oauth2-proxy
-  template:
-    metadata:
-      labels:
-        app: comfyui-oauth2-proxy
-    spec:
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 65534
-        seccompProfile: {type: RuntimeDefault}
-      containers:
-        - name: oauth2-proxy
-          image: quay.io/oauth2-proxy/oauth2-proxy:v7.7.1
-          args:
-            - --provider=oidc
-            - --oidc-issuer-url=https://zitadel.riotpiao.com
-            - --email-domain=*
-            - --upstream=static://202
-            - --reverse-proxy=true
-            - --set-xauthrequest=true
-            - --cookie-secure=true
-            - --cookie-expire=12h
-            - --cookie-refresh=1h
-            - --skip-provider-button=true
-            - --redirect-url=https://comfyui.riotpiao.com/oauth2/callback
-            - --http-address=0.0.0.0:4180
-          env:
-            - name: OAUTH2_PROXY_CLIENT_ID
-              valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: client-id}}
-            - name: OAUTH2_PROXY_CLIENT_SECRET
-              valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: client-secret}}
-            - name: OAUTH2_PROXY_COOKIE_SECRET
-              valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: cookie-secret}}
-          ports:
-            - {name: http, containerPort: 4180}
-          readinessProbe:
-            httpGet: {path: /ping, port: http}
-            initialDelaySeconds: 5
-          livenessProbe:
-            httpGet: {path: /ping, port: http}
-            initialDelaySeconds: 15
-          securityContext:
-            allowPrivilegeEscalation: false
-            capabilities: {drop: ["ALL"]}
-          resources:
-            requests: {cpu: 10m, memory: 32Mi}
-            limits: {cpu: 200m, memory: 128Mi}
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: comfyui-oauth2-proxy
-  namespace: comfyui
-  labels:
-    app: comfyui-oauth2-proxy
-spec:
-  type: ClusterIP
-  selector:
-    app: comfyui-oauth2-proxy
-  ports:
-    - {name: http, port: 4180, targetPort: http}
----
-# The login path must NOT be gated: a request to /oauth2/start that itself demands a
-# login redirects to /oauth2/start forever. ingress-nginx applies the annotations of
-# an ingress to the locations it defines, so a separate object without auth-url keeps
-# this path open. The earlier failure of this shape was not the split — it was the
-# configuration-snippet on the app ingress, which made the controller drop the whole
-# object and left only this one alive.
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: comfyui-oauth2
-  namespace: comfyui
-spec:
-  ingressClassName: nginx
-  rules:
-    - host: comfyui.riotpiao.com
-      http:
-        paths:
-          - path: /oauth2
-            pathType: Prefix
-            backend:
-              service:
-                name: comfyui-oauth2-proxy
-                port:
-                  number: 4180

k8s/apps/comfyui/service.yaml

@@ -1,14 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: comfyui
-  namespace: comfyui
-  labels:
-    app: comfyui
-spec:
-  selector:
-    app: comfyui
-  ports:
-    - port: 80
-      targetPort: 8188
-      protocol: TCP

k8s/apps/gotify/db.yaml

@@ -8,7 +8,11 @@ metadata:
   annotations:
     argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
 spec:
-  instances: 3
+  # 2, not 3. The file's own header always said 2, and the third instance never
+  # existed as a running pod: its volume only ever held a replica that could not
+  # start, because talos-cp-2 disk2 has no free space. A third replica on a disk
+  # that cannot host it is not redundancy, it is a permanent degraded volume.
+  instances: 2
   imageName: ghcr.io/cloudnative-pg/postgresql:16.2
   bootstrap:
     initdb:

k8s/apps/key-rotation-controller/README.md

@@ -40,7 +40,7 @@ target is one root-of-trust file per thing that cannot rotate itself:
 | controller bootstrap key | chicken and egg: it creates every other key |
 | etcd defrag credentials | needed by a job that runs outside the controller |
 
-Everything else — Grafana, Argo CD, Forgejo, MinIO, Paperless, Immich, ComfyUI,
+Everything else — Grafana, Argo CD, Forgejo, MinIO, Paperless, Immich,
 api-gw, Gotify — is a KeyRotation object, and its secret leaves git.
 
 ## Status

k8s/apps/key-rotation-controller/rotations.yaml

@@ -64,32 +64,6 @@ spec:
 ---
 apiVersion: iam.homelab.riotpiao.com/v1
 kind: KeyRotation
-metadata:
-  name: comfyui
-  namespace: iam
-  labels:
-    app.kubernetes.io/part-of: key-rotation
-spec:
-  provider: zitadel-oidc-client
-  strategy: atomic
-  intervalHours: 2160
-  suspend: true
-  target:
-    zitadelProjectID: "396757796898471937"
-    zitadelAppID: "397015743994003854"  # app resource id, not the client id 397015744010781070
-    zitadelClientID: "397015744010781070"
-    secretName: comfyui-oauth2-proxy
-    secretNamespace: comfyui
-    secretKeys:
-      secret: client-secret
-    reload: restart
-    consumers:
-      - {kind: Deployment, name: comfyui-oauth2-proxy, namespace: comfyui}
-  verify:
-    introspect: true
----
-apiVersion: iam.homelab.riotpiao.com/v1
-kind: KeyRotation
 metadata:
   name: minio
   namespace: iam
@@ -107,7 +81,11 @@ spec:
     secretName: minio-oidc
     secretNamespace: storage
     secretKeys:
-      secret: client-secret
+      # The key name here must match the consumer's secretKeyRef, not a house style.
+      # minio-tenant.yaml reads MINIO_IDENTITY_OPENID_CLIENT_SECRET. While this said
+      # `client-secret`, the tenant failed with CreateContainerConfigError on every
+      # restart and the rotation wrote a key nobody read.
+      secret: MINIO_IDENTITY_OPENID_CLIENT_SECRET
     reload: hot
     consumers:
       - {kind: StatefulSet, name: minio-cluster-az-a, namespace: storage}

k8s/apps/llm-serving/clef-serve.yaml

@@ -33,7 +33,19 @@ metadata:
     app: clef-serve
     app.kubernetes.io/part-of: llm-serving
 spec:
-  replicas: 1
+  # Parked at 0 to free the only V100 that was not serving anything, so candle-vllm can
+  # run its Phase 0 on one card. All four cards were allocated (coder 2, reasoning 1,
+  # this 1) and a new GPU workload had nowhere to land.
+  #
+  # This is the only GPU consumer in the namespace that is not vLLM, which is why it is
+  # the first card released: the standing instruction is to keep the vLLM pods healthy
+  # until candle-vllm answers, and this pod is not one of them.
+  #
+  # It was serving nothing. The last 30 log lines at park time were 30/30 kubelet
+  # /health probes and no inference request since start (2026-10-30T06:23:40Z). Nothing
+  # in git selects this service. Restore it once the rollout leaves a card free:
+  # 30B-A3B takes 1, Coder-Next TP=2 takes 2, so card 4 is this one.
+  replicas: 0
   # Recreate, not RollingUpdate. Every GPU card is allocated, so a rolling update leaves
   # the new pod Pending forever waiting for a card the old pod still holds.
   strategy:

k8s/apps/llm-serving/coder.yaml

@@ -0,0 +1,134 @@
+apiVersion: serving.kserve.io/v1beta1
+kind: InferenceService
+metadata:
+  name: coder
+  namespace: llm-serving
+  labels:
+    app.kubernetes.io/name: llm-coder
+    app.kubernetes.io/part-of: llm-serving
+  annotations:
+    serving.kserve.io/deploymentMode: RawDeployment
+spec:
+  predictor:
+    minReplicas: 1
+    maxReplicas: 1
+    nodeSelector:
+      kubernetes.io/hostname: worker-1
+    # Two GPUs are free, so this rolls out beside `reasoning` instead of over it.
+    # Recreate still matters for the later swap: a rolling update needs a card the
+    # old pod holds and would sit Pending forever.
+    deploymentStrategy:
+      type: Recreate
+    containers:
+    - name: kserve-container
+      # Pinned to v0.11.0 on purpose. This is the last image measured to work on this
+      # node: torch 2.8.0+cu128, arch list sm_70..sm_120, driver 580.159.04.
+      # v0.15.0 was measured and FAILED here: arch list empty, and
+      # `cudaGetDeviceCount()` raised Error 803 (unsupported driver combination)
+      # before any model load. Do not raise this tag without re-running that probe.
+      image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
+      args:
+      # Qwen3-Coder-30B-A3B-Instruct: 30.5B total, 3.3B active. Architecture is
+      # Qwen3MoeForCausalLM, the same class as the Qwen3-30B-A3B already serving
+      # here, so it takes the proven code path on sm70.
+      #
+      # Qwen ships this model only as BF16 (61.1 GB) and FP8 (no FP8 on Volta).
+      # 4-bit GPTQ is the only form that fits one 32 GB card. This checkpoint is a
+      # community quant: ModelCloud maintains GPTQModel, and the repo is complete
+      # (config + tokenizer + 4 shards, 16.9 GB). Verified by loading, not by trust.
+      #
+      # Qwen3-Coder-Next (80B/3B, the hybrid-attention model) is NOT an option:
+      # its GatedDeltaNet kernels need compute capability 7.5+ and the `fla` module,
+      # and sm70 has neither. Measured, not assumed.
+      - --model=ModelCloud/Qwen3-Coder-30B-A3B-Instruct-GPTQMODEL-W4A16-A
+      # `reasoning` stays a served name so an existing client that sends
+      # model=reasoning keeps working after the DNS rename. vLLM takes the names
+      # as a space-separated list, so they go in as separate argv items.
+      - --served-model-name
+      - coder
+      - reasoning
+      - --quantization=gptq
+      - --dtype=float16
+      - --kv-cache-dtype=auto
+      # Native window is 262144 and rope_scaling is null, so 256K needs no YaRN.
+      # One card cannot hold it: measured KV pool on a single V100 was 132,672
+      # tokens. TP=2 halves the weights per card (7.8 GiB each), which frees room
+      # for the larger pool. Measured after boot: see the KV line in the log.
+      - --max-model-len=262144
+      - --tensor-parallel-size=2
+      - --pipeline-parallel-size=1
+      - --gpu-memory-utilization=0.90
+      # A single 262,144-token request consumes the whole pool, so keep few slots.
+      - --max-num-seqs=4
+      - --max-num-batched-tokens=8192
+      - --long-prefill-token-threshold=2048
+      - --enable-chunked-prefill
+      - --enable-prefix-caching
+      # Measured: with the default (custom all-reduce ON) the engine finished CUDA graph
+      # capture in 6s at 17:01:56 and then went silent for 14+ minutes, emitting only
+      # "No available shared memory broadcast block found in 60 seconds". The front-end
+      # never received the ready message. vLLM's custom all-reduce registers its own
+      # CUDA graph addresses and deadlocks on this Volta/PCIe pair. NCCL is the fallback
+      # and is correct here, because TP=2 across two PCIe V100s gains nothing from the
+      # custom kernel anyway.
+      - --disable-custom-all-reduce
+      - --reasoning-parser=qwen3
+      - --enable-auto-tool-choice
+      - --tool-call-parser=hermes
+      - --host=0.0.0.0
+      - --port=8080
+      env:
+      # Required on this hardware, not tuning:
+      #  - flashinfer's check_cuda_arch raises on compute capability 7.0,
+      #    so the sampler must stay off
+      #  - FlashInfer and FlashAttention-2 need sm75+, so Triton serves attention
+      - name: VLLM_USE_FLASHINFER_SAMPLER
+        value: "0"
+      - name: VLLM_ATTENTION_BACKEND
+        value: TRITON_ATTN
+      - name: HF_HOME
+        value: /mnt/models
+      ports:
+      - containerPort: 8080
+        protocol: TCP
+      readinessProbe:
+        httpGet:
+          path: /health
+          port: 8080
+        periodSeconds: 10
+      startupProbe:
+        httpGet:
+          path: /health
+          port: 8080
+        # TP=2 at 256K needs ~10 min before the API binds: 22s weights, 80s torch.compile,
+        # 85s KV profiling, then graph capture. 120 x 15s = 30 min of headroom.
+        failureThreshold: 120
+        periodSeconds: 15
+      resources:
+        # worker-1 exposes only 63,495 MiB of allocatable memory, so a 24Gi request
+        # never schedules. Two vLLM workers keep the weights on the GPU and need
+        # little host RAM, so the request stays small and the limit allows burst.
+        requests:
+          cpu: "8"
+          memory: 12Gi
+          nvidia.com/gpu: "2"
+        limits:
+          cpu: "16"
+          memory: 40Gi
+          nvidia.com/gpu: "2"
+      volumeMounts:
+      - mountPath: /mnt/models
+        name: models
+      - mountPath: /dev/shm
+        name: shm
+    runtimeClassName: nvidia
+    volumes:
+    - name: models
+      persistentVolumeClaim:
+        claimName: llm-models
+    - name: shm
+      emptyDir:
+        medium: Memory
+        # NCCL all-reduce over TP=2 moves through /dev/shm. The 64Mi default is
+        # too small and shows up as an init hang or a bus error.
+        sizeLimit: 8Gi

k8s/apps/llm-serving/kustomization.yaml

@@ -14,7 +14,9 @@ resources:
   # v0.5.22+, published only on CUDA 13, and CUDA 13 dropped Volta kernels. The model
   # instead runs the implementation it ships (joint_schema_model.py) in fp16 on one V100.
   - clef-serve.yaml
-  - reasoning.yaml
+  # Stable client-facing name for the chat model. poimen-memory points at this Service,
+  # so the vLLM -> candle-vllm handoff is a selector edit here and no client change.
+  - reasoning-long-service.yaml
   # ornith is retired: its InferenceService still reports Ready=True while running zero
   # replicas, because desired equals available at 0. Its manifest was never committed,
   # so it is left unmanaged and is not listed here. Delete the CR when convenient.

k8s/apps/llm-serving/networkpolicy.yaml

@@ -66,6 +66,21 @@ spec:
           port: 80
         - protocol: TCP
           port: 8000
+    # Allow from poimen namespace (poimen-memory ingest + query).
+    # The client-side egress rule is the other half and must stay in step:
+    # NetworkPolicy poimen/poimen-memory-llm-egress permits port 8080 to pods
+    # labelled part-of=llm-serving, which is the same selector used here. This
+    # policy had no matching ingress rule at all, so the packet was allowed out
+    # and dropped here. The client then saw a bare timeout and the predictor log
+    # stayed empty, which is why this failure looked like a hung inference engine
+    # for 34 days.
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: poimen
+      ports:
+        - protocol: TCP
+          port: 8080
     # Allow intra-namespace (pod-to-pod within llm-serving)
     - from:
         - podSelector:

k8s/apps/llm-serving/qwen-cpu.yaml

@@ -1,5 +1,6 @@
 # qwen2.5:3b-instruct on CPU (talos-cp-2, 144GB RAM, 24 cores).
-# Moved off GPU to free a V100 for ComfyUI. Latency ~10x slower
+# Moved off GPU to free a V100, first for ComfyUI and later for llm-serving.
+# Latency ~10x slower
 # than GPU but sufficient for lightweight tasks (summarization,
 # classification, quick answers).
 apiVersion: apps/v1

k8s/apps/llm-serving/reasoning-long-service.yaml

@@ -0,0 +1,43 @@
+# Stable client-facing name for the chat model, deliberately decoupled from the
+# process that currently serves it.
+#
+# poimen-memory addresses this Service, not a pod and not an InferenceService. That
+# means the vLLM -> candle-vllm handoff changes only this file's selector. The client
+# config never changes again, so the ingest cutover has no window in which the host
+# name is wrong.
+#
+# Two ports on purpose:
+#   - 80    matches every other predictor Service in this namespace, and matches
+#           poimen-memory's LLM_CHAT_BASE, which carries no port number.
+#   - 8080  is the container port. It is also the only port llm-serving-ingress and
+#           poimen-memory-llm-egress permit, and the port the handoff readiness check
+#           uses. Publishing it directly avoids a port translation that is easy to get
+#           wrong in a curl.
+#
+# The selector names the vLLM reasoning pod today. To hand off, replace the three
+# selector labels with candle-vllm's pod labels and nothing else. Do not delete and
+# recreate this Service: a delete drops the ClusterIP, and a client that cached the
+# name during the gap will fail until it re-resolves.
+apiVersion: v1
+kind: Service
+metadata:
+  name: reasoning-long-predictor
+  namespace: llm-serving
+  labels:
+    app.kubernetes.io/part-of: llm-serving
+    app: reasoning-long-predictor
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: llm-reasoning
+    component: predictor
+    serving.kserve.io/inferenceservice: reasoning
+  ports:
+    - name: http
+      protocol: TCP
+      port: 80
+      targetPort: 8080
+    - name: http-8080
+      protocol: TCP
+      port: 8080
+      targetPort: 8080

k8s/apps/llm-serving/reasoning.yaml

@@ -30,7 +30,28 @@ spec:
       # Measured on this cluster, one V100-32GB, 64k context: 15.55 GiB of weights,
       # 137k tokens of KV cache, 2.09x concurrency at 65536 tokens per request.
       - --model=Qwen/Qwen3-30B-A3B-GPTQ-Int4
-      - --served-model-name=reasoning
+      # Two served names, one pod, on purpose. `reasoning-long` is the name the goal map
+      # uses for this model, and the name qwen-finetune wiring targets. It is NOT a
+      # separate InferenceService: this model already occupies the only free card, and a
+      # second pod on the same weights cannot schedule (4 of 4 GPUs allocated: coder 2,
+      # this 1, clef-serve 1). Serving both names from one process costs nothing and
+      # keeps the two callers working without a second weights copy.
+      #
+      # `reasoning` is deliberately still served here. poimen-memory pins
+      # LLM_CHAT_BASE=reasoning-predictor, so dropping the name here breaks ingest
+      # immediately. coder-predictor also serves `reasoning`; that ambiguity is known
+      # and documented in the PR. Resolve it by moving the client host, not by
+      # removing the name here.
+      #
+      # The two names are separate list entries, NOT one comma-joined value. This
+      # image's parser returns ['reasoning,reasoning-long'] for the comma form, i.e. a
+      # single model whose name contains a comma, and api_server.py then iterates
+      # args.served_model_name to build the model list. Verified against vLLM 0.11.0:
+      #   FlexibleArgumentParser().parse_args([... "--served-model-name", "a", "b"])
+      #     -> ['a', 'b']
+      - --served-model-name
+      - reasoning
+      - reasoning-long
       - --quantization=gptq
       - --dtype=float16
       - --kv-cache-dtype=auto

k8s/apps/poimen-frontend/api-ingress.yaml

@@ -0,0 +1,38 @@
+---
+# Public entry point for the poimen-memory HTTP API.
+#
+# The API was reachable only inside the cluster (ClusterIP :8080), so a local pi
+# session had no way to use it. This Ingress exposes the same handler set the
+# in-cluster clients use: /mcp, /memory/*, /graphql, /health, /ready.
+#
+# Cloudflare terminates TLS in front of the nginx ingress, which is why this rule
+# carries no `tls:` section — the same shape as ingress.yaml next to it.
+#
+# Auth: poimen-memory validates ZITADEL JWTs (MEM_AUTH_MODE=jwt). A caller
+# without a valid bearer token gets 401, so exposing the route does not expose
+# the data.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: poimen-api
+  namespace: poimen
+  annotations:
+    # MCP calls and ingest can be slow; the provider may stream. Match the
+    # frontend ingress so a long tool call is not cut at 60s.
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-buffering: "off"
+    nginx.ingress.kubernetes.io/proxy-body-size: "50m"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: api-poimen.riotpiao.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: poimen-memory
+                port:
+                  number: 8080

k8s/apps/poimen-frontend/kustomization.yaml

@@ -6,3 +6,5 @@ kind: Kustomization
 # riotpiao-poimen/poimen-frontend.git k8s/ — each repo owns its own k8s.
 resources:
   - ingress.yaml
+  # The memory API. Without it a local pi session has no route to :8080.
+  - api-ingress.yaml

k8s/argocd/apps/00-substrate.yaml

@@ -159,54 +159,3 @@ spec:
     automated:
       prune: true
       selfHeal: true
----
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: kyverno
-  namespace: argocd
-  annotations:
-    argocd.argoproj.io/sync-wave: "0"
-spec:
-  project: homelab
-  source:
-    repoURL: https://kyverno.github.io/kyverno/
-    chart: kyverno
-    targetRevision: "1.14.0"
-    helm:
-      valueFiles:
-        - $values/k8s/bootstrap/kyverno/kyverno-values.yaml
-  sources:
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
-      targetRevision: main
-      ref: values
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: kyverno
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
-    syncOptions:
-      - CreateNamespace=true
----
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: kyverno-policies
-  namespace: argocd
-  annotations:
-    argocd.argoproj.io/sync-wave: "0"
-spec:
-  project: homelab
-  source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
-    targetRevision: main
-    path: k8s/bootstrap/kyverno
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: kyverno
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true

k8s/argocd/apps/58-comfyui.yaml

@@ -1,32 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: comfyui
-  namespace: argocd
-  labels:
-    app.kubernetes.io/name: comfyui
-    app.kubernetes.io/component: image-generation
-  annotations:
-    argocd.argoproj.io/sync-wave: "8"
-spec:
-  project: homelab
-  revisionHistoryLimit: 3
-  source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
-    targetRevision: main
-    path: k8s/apps/comfyui
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: comfyui
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
-    syncOptions:
-      - CreateNamespace=true
-    retry:
-      limit: 5
-      backoff:
-        duration: 5s
-        factor: 2
-        maxDuration: 3m

k8s/bootstrap/kyverno/kustomization.yaml

@@ -1,6 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-namespace: kyverno
-
-resources:
-  - policies.yaml

k8s/bootstrap/kyverno/kyverno-values.yaml

@@ -1,42 +0,0 @@
-# Kyverno: Policy engine for Kubernetes image scanning, Pod security, and admission control
-# Scan all images, enforce baseline Pod Security Standard, prevent privilege escalation
-
-replicaCount: 1
-
-image:
-  registry: ghcr.io
-  repository: kyverno/kyverno
-  tag: "v1.14.0"
-
-config:
-  # Webhook timeout for policy evaluation. Increase if scanning takes longer.
-  webhookTimeoutSeconds: 30
-  # Failure policy: fail-open (audit/log) vs fail-closed (reject on error)
-  failurePolicy: fail
-  # Resource limits for webhook
-  webhookAnnotations:
-    rules: "allow"
-
-# Pod security via Kyverno instead of Pod Security Policies (deprecated)
-# Enforces baseline restrictions cluster-wide, with exceptions for privileged namespaces
-podSecurityContext:
-  runAsNonRoot: true
-  runAsUser: 1000
-
-rbac:
-  create: true
-
-resources:
-  requests:
-    memory: "256Mi"
-    cpu: "100m"
-  limits:
-    memory: "512Mi"
-    cpu: "500m"
-
-# Webhook configuration
-webhook:
-  timeoutSeconds: 30
-  # Failure policy: "Fail" (reject on error) or "Ignore" (audit-only)
-  # Set to "Ignore" for initial testing, then change to "Fail"
-  failurePolicy: ignore

k8s/bootstrap/kyverno/policies.yaml

@@ -1,204 +0,0 @@
-# Kyverno ClusterPolicies: Image scanning, Pod security, and admission control
----
-# Policy 1: Require non-root containers
-apiVersion: kyverno.io/v1
-kind: ClusterPolicy
-metadata:
-  name: require-non-root
-  namespace: kyverno
-spec:
-  validationFailureAction: audit  # audit first, then change to enforce
-  rules:
-    - name: check-runAsNonRoot
-      match:
-        any:
-          - resources:
-              kinds:
-                - Pod
-              selector:
-                matchLabels:
-                  pod-security.kubernetes.io/enforce: "!privileged"
-      validate:
-        message: "Container must not run as root"
-        pattern:
-          spec:
-            containers:
-              - securityContext:
-                  runAsNonRoot: true
----
-# Policy 2: Drop all Linux capabilities, add only required ones
-apiVersion: kyverno.io/v1
-kind: ClusterPolicy
-metadata:
-  name: require-dropped-caps
-  namespace: kyverno
-spec:
-  validationFailureAction: audit
-  rules:
-    - name: drop-all-capabilities
-      match:
-        any:
-          - resources:
-              kinds:
-                - Pod
-              selector:
-                matchLabels:
-                  pod-security.kubernetes.io/enforce: "!privileged"
-      validate:
-        message: "All Linux capabilities must be dropped"
-        pattern:
-          spec:
-            containers:
-              - securityContext:
-                  capabilities:
-                    drop:
-                      - ALL
----
-# Policy 3: Require image tags (no 'latest')
-apiVersion: kyverno.io/v1
-kind: ClusterPolicy
-metadata:
-  name: disallow-latest-tag
-  namespace: kyverno
-spec:
-  validationFailureAction: audit  # Change to enforce after testing
-  rules:
-    - name: disallow-latest
-      match:
-        any:
-          - resources:
-              kinds:
-                - Pod
-                - Deployment
-                - StatefulSet
-                - DaemonSet
-                - Job
-      validate:
-        message: "Image tag 'latest' is not allowed. Use explicit version tags."
-        pattern:
-          spec:
-            =(template):
-              spec:
-                containers:
-                  - image: "!*:latest"
-                =(initContainers):
-                  - image: "!*:latest"
----
-# Policy 4: Restrict images to trusted registries
-apiVersion: kyverno.io/v1
-kind: ClusterPolicy
-metadata:
-  name: restrict-registries
-  namespace: kyverno
-spec:
-  validationFailureAction: audit
-  rules:
-    - name: trusted-registries
-      match:
-        any:
-          - resources:
-              kinds:
-                - Pod
-                - Deployment
-                - StatefulSet
-                - DaemonSet
-                - Job
-              selector:
-                matchLabels:
-                  pod-security.kubernetes.io/enforce: "!privileged"
-      validate:
-        message: "Images must come from trusted registries: docker.io, ghcr.io, quay.io, k8s.gcr.io, registry.k8s.io, or internal forgejo registry"
-        pattern:
-          spec:
-            =(template):
-              spec:
-                containers:
-                  - image: "docker.io/* | ghcr.io/* | quay.io/* | k8s.gcr.io/* | registry.k8s.io/* | forgejo.riotpiao.com/* | *"
----
-# Policy 5: Require read-only root filesystem (audit only, exceptions for apps that need writes)
-apiVersion: kyverno.io/v1
-kind: ClusterPolicy
-metadata:
-  name: require-readonly-filesystem
-  namespace: kyverno
-spec:
-  validationFailureAction: audit
-  rules:
-    - name: check-readOnlyRootFilesystem
-      match:
-        any:
-          - resources:
-              kinds:
-                - Pod
-              selector:
-                matchLabels:
-                  pod-security.kubernetes.io/enforce: "!privileged"
-      validate:
-        message: "Root filesystem should be read-only for defense-in-depth"
-        pattern:
-          spec:
-            containers:
-              - securityContext:
-                  readOnlyRootFilesystem: true
----
-# Policy 6: Require resource requests and limits (prevent resource starvation)
-apiVersion: kyverno.io/v1
-kind: ClusterPolicy
-metadata:
-  name: require-resource-limits
-  namespace: kyverno
-spec:
-  validationFailureAction: audit
-  rules:
-    - name: check-resources
-      match:
-        any:
-          - resources:
-              kinds:
-                - Pod
-                - Deployment
-                - StatefulSet
-                - DaemonSet
-              excludeResources:
-                namespaceSelector:
-                  matchLabels:
-                    kubernetes.io/metadata.name: "kyverno|kube-system|kube-node-lease"
-      validate:
-        message: "CPU and memory requests and limits are required"
-        pattern:
-          spec:
-            =(template):
-              spec:
-                containers:
-                  - resources:
-                      requests:
-                        memory: "?*"
-                        cpu: "?*"
-                      limits:
-                        memory: "?*"
-                        cpu: "?*"
----
-# Policy 7: Require securityContext on all containers
-apiVersion: kyverno.io/v1
-kind: ClusterPolicy
-metadata:
-  name: require-security-context
-  namespace: kyverno
-spec:
-  validationFailureAction: audit
-  rules:
-    - name: check-securityContext
-      match:
-        any:
-          - resources:
-              kinds:
-                - Pod
-              selector:
-                matchLabels:
-                  pod-security.kubernetes.io/enforce: "!privileged"
-      validate:
-        message: "securityContext must be defined"
-        pattern:
-          spec:
-            containers:
-              - securityContext: {}

k8s/bootstrap/phase3-forgejo/forgejo-values.yaml

@@ -97,10 +97,25 @@ gitea:
       # USER and PASSWD injected via env vars below (GITEA__MAILER__USER, GITEA__MAILER__PASSWD)
 
 # Persistence (shared storage for repos)
+#
+# 60Gi, not the original 20Gi. The container registry stores its blobs here,
+# and a single candle-vllm sm_70 image is 14GB uncompressed. At 20Gi the volume
+# filled, and every blob upload failed with:
+#   Package registry API internal error: 500 write /data/tmp/package-upload/...:
+#   no space left on device
+# The push client surfaces that as a bare `received unexpected HTTP status: 500`
+# after retrying every layer, which looks like a network fault and is not one.
+#
+# Note for whoever expands this again: Longhorn's admission webhook checks the
+# PHYSICAL free space of the disk holding each replica, not the thin volume
+# size. A request for 100Gi was rejected while the disk had 23.5Gi free:
+#   disk ... does not have sufficient physical space for expansion:
+#   physical free space would drop below minimal: left=14596177920 < minimal=49931236556
+# 60Gi fit. Raise it in steps and watch the webhook message.
 persistence:
   enabled: true
   storageClass: longhorn
-  size: 20Gi
+  size: 60Gi
   accessModes:
     - ReadWriteOnce
 
@@ -146,19 +161,19 @@ ingress:
 # The database was never the bottleneck. A real action_runner UPDATE measured
 # 2.133ms with zero lock waits, so this is CPU starvation in Gitea itself.
 #
-# requests.cpu rises to 750m because a request below real usage earns the pod
-# little scheduling weight during contention. limits.cpu rises to 3000m
-# because diff computation is bursty and needs room to run without throttling.
+# requests.cpu stays 200m and limits.cpu stays 1000m. The 3000m limit was a
+# temporary measure while the Actions stall was diagnosed; the real fix landed
+# elsewhere, so the limit is back to the original.
 #
-# Caution: this pod runs on talos-cp-3, a control-plane node. 3000m is 25% of
-# that node's 11950m allocatable CPU. If Gitea sustains near this limit, move
-# the workload off the control plane rather than raising the limit again.
+# Caution: this pod runs on talos-cp-3, a control-plane node. If Gitea
+# throttles again at 1000m, move the workload off the control plane rather
+# than raising the limit.
 resources:
   requests:
-    cpu: 750m
+    cpu: 200m
     memory: 768Mi
   limits:
-    cpu: 3000m
+    cpu: 1000m
     memory: 2Gi
 
 # Tolerations for control-plane

k8s/infra/longhorn/disk-scan-job.yaml

@@ -0,0 +1,90 @@
+# One-shot destructive health scan of a candidate Longhorn disk.
+#
+# WHY THIS IS A MANUAL JOB AND NOT AN ARGOCD HOOK
+#   It writes over the entire raw device. An ArgoCD PostSync hook would run it
+#   on every sync and destroy whatever the volume holds. Apply it by hand,
+#   once, on a disk that has no partitions and no mounts.
+#
+# WHY NOT TALOS
+#   Talos declares disks by mounting them (machine.disks in the machine
+#   config). It has no primitive for "run this command on the host", so a scan
+#   cannot live in the Talos config. This Job is the IaC-correct home.
+#
+# ORDER OF OPERATIONS
+#   1. Scan the raw device with this Job        <- the disk must be UNMOUNTED
+#   2. If it passes, add the disk to terraform.tfvars.local `longhorn_disks`
+#      with device /dev/disk/by-id/<wwid> and run `make apply-cp`
+#   3. Register it with Longhorn via the pattern in longhorn-add-disks-job.yaml
+#
+# What it does
+#   fio writes a CRC32C-verified pattern across the whole device, then reads
+#   every block back and compares. Any unreadable or mismatched sector fails
+#   the Job. This is a full write and read pass, not a sample.
+#
+# Expected duration
+#   2 TB on a 5400 rpm 2.5" drive at roughly 90 MB/s is about 6 h to write and
+#   6 h to verify. Budget 12 hours. Check progress with:
+#     kubectl -n llm-io logs job/disk-scan-cp1-sdb -f
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: disk-scan-cp1-sdb
+  namespace: llm-io
+  labels:
+    app: disk-scan
+spec:
+  backoffLimit: 0
+  # Do not let a stuck scan retry silently.
+  template:
+    metadata:
+      labels:
+        app: disk-scan
+    spec:
+      restartPolicy: Never
+      nodeSelector:
+        kubernetes.io/hostname: talos-cp-1
+      containers:
+        - name: scan
+          image: ubuntu:22.04
+          securityContext:
+            privileged: true
+          command:
+            - /bin/bash
+            - -c
+            - |
+              set -euo pipefail
+              apt-get update -qq
+              apt-get install -y -qq smartmontools fio
+
+              echo "=== device identity ==="
+              smartctl -i /dev/sdb | tail -12
+
+              echo "=== SMART before ==="
+              smartctl -H /dev/sdb | tail -2
+
+              echo "=== full write + verify (destructive) ==="
+              fio --name=devverify \
+                  --filename=/dev/sdb \
+                  --rw=write \
+                  --bs=1M \
+                  --iodepth=4 \
+                  --ioengine=libaio \
+                  --direct=1 \
+                  --verify=crc32c \
+                  --do_verify=1 \
+                  --verify_fatal=1 \
+                  --eta=always \
+                  --output-format=normal
+
+              echo "=== SMART after ==="
+              smartctl -H /dev/sdb | tail -2
+              smartctl -A /dev/sdb | awk '/Reallocated_Sector|Current_Pending|Offline_Uncorrect|UDMA_CRC/{print "  "$0}'
+
+              echo "=== SCAN PASSED ==="
+          volumeMounts:
+            - name: dev
+              mountPath: /dev
+      volumes:
+        - name: dev
+          hostPath:
+            path: /dev

k8s/infra/poimen/graphql-netpol.yaml

@@ -0,0 +1,28 @@
+---
+# Keep the GraphQL server reachable only from the one service that proxies it.
+#
+# poimen-graphql is already off the internet: its Service is ClusterIP on 8090 and
+# no Ingress names it. What it lacked is in-cluster isolation — a ClusterIP answers
+# every pod in the cluster.
+#
+# The only caller is poimen-memory, whose /graphql route forwards the request body
+# to MEM_GQL_PORT (8090). No other workload references this service.
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: poimen-graphql-ingress
+  namespace: poimen
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: poimen-graphql
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: poimen-memory
+      ports:
+        - protocol: TCP
+          port: 8090

k8s/infra/poimen/kustomization.yaml

@@ -6,6 +6,10 @@ kind: Kustomization
 # deployments (wave 0) are applied.
 resources:
   - memory-db.yaml
+  # ClusterIP alone is not isolation: every pod in the cluster can reach it.
+  - memory-db-netpol.yaml
+  # Only poimen-memory's /graphql proxy calls 8090.
+  - graphql-netpol.yaml
   - poimen-sandbox-namespace.yaml
   - poimen-sandbox-quota.yaml
   - pi-session-netpol.yaml

k8s/infra/poimen/memory-db-netpol.yaml

@@ -0,0 +1,42 @@
+---
+# Keep the memory database reachable only from inside its own namespace.
+#
+# What this does NOT change: the database was already unreachable from the
+# internet. `memory-db-rw`, `memory-db-ro` and `memory-db-r` are ClusterIP
+# services, no Ingress backs them, and no public DNS name resolves to them.
+#
+# What it fixes: a ClusterIP service is reachable by EVERY pod in the cluster,
+# because Kubernetes has no implicit isolation. Cilium enforces NetworkPolicy, so
+# stating the allowed sources is what turns "not on the internet" into "not
+# reachable from another tenant's namespace".
+#
+# Two sources, both required:
+#  - the poimen namespace, which holds every client: poimen-memory, poimen-graphql,
+#    poimen-worker, poimen-workflows and the migration Job.
+#  - cnpg-system, where the CloudNativePG operator lives. It connects to the
+#    instance for switchover and failover, so a narrower rule breaks recovery.
+#
+# No other port is opened. The operator's metrics endpoint stays closed because no
+# PodMonitor for this cluster exists, so nothing scrapes it.
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: memory-db-ingress
+  namespace: poimen
+spec:
+  podSelector:
+    matchLabels:
+      cnpg.io/cluster: memory-db
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: poimen
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: cnpg-system
+      ports:
+        - protocol: TCP
+          port: 5432

scripts/iam/rotate-oidc.py

@@ -32,7 +32,6 @@ ISSUER = "https://zitadel.riotpiao.com"
 APPS = [
     ("grafana",   "grafana",   "grafana-oidc",         "logging",   "GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET", "deploy", "grafana"),
     ("argocd",    "argocd",    "oidc-secret",           "argocd",    "client-secret",                       "deploy", "argocd-server"),
-    ("comfyui",   "comfyui",   "comfyui-oauth2-proxy", "comfyui",   "client-secret",                       "deploy", "comfyui-oauth2-proxy"),
     ("minio",     "minio",     "minio-oidc",           "storage",   "MINIO_IDENTITY_OPENID_CLIENT_SECRET",  "sts",    "minio-cluster-az-a"),
     ("paperless", "paperless", "paperless-oidc",       "paperless", "client-secret",                       "deploy", "paperless"),
     ("immich",    "immich",    "immich-oidc",          "immich",    "client-secret",                       "deploy", "immich-server"),

scripts/iam/zitadel-provision.py

@@ -128,11 +128,6 @@ OIDC_APPS = {
         "https://forgejo.riotpiao.com/user/oauth2/zitadel/callback",
         "http://localhost:3000/user/oauth2/authentik/callback",
         "https://forgejo.riotpiao.com/user/oauth2/authentik/callback"]},
-    # ComfyUI has no native OIDC. oauth2-proxy is its front door and this app is
-    # the proxy's client, so the callback path belongs to the proxy.
-    "comfyui": {**_WEB, "redirect_uris": [
-        "http://localhost:4180/oauth2/callback",
-        "https://comfyui.riotpiao.com/oauth2/callback"]},
     "grafana": {**_WEB, "redirect_uris": [
         "http://localhost:3000/login/generic_oauth",
         "https://grafana.riotpiao.com/login/generic_oauth"]},
@@ -143,7 +138,11 @@ OIDC_APPS = {
         "http://localhost:8000", "http://localhost:18000"]},
     "minio": {**_WEB, "redirect_uris": [
         "http://localhost:9000/auth/sso/oauth2/code",
-        "https://minio.riotpiao.com/auth/sso/oauth2/code"]},
+        "https://minio.riotpiao.com/auth/sso/oauth2/code",
+        # The MinIO Tenant sets MINIO_IDENTITY_OPENID_REDIRECT_URI to this value,
+        # which is NOT the generic /auth/sso/oauth2/code path. Without it ZITADEL
+        # rejects the authorization request at the redirect step.
+        "https://minio.riotpiao.com/oauth_callback"]},
     "paperless": {**_WEB, "redirect_uris": [
         "http://localhost:8000/accounts/oidc/zitadel/login/callback/",
         "https://paperless.riotpiao.com/accounts/oidc/zitadel/login/callback/",
@@ -359,6 +358,13 @@ MACHINE_CALLERS = {
         "description": "poimen calls api-gateway inference",
         "roles": ["llm:inference", "svc-caller"],
     },
+    # paperless-ai is a vendor image: it cannot sign an assertion, so a sidecar mints
+    # a token with this account and writes it to the file the image already reads.
+    "paperless-llm": {
+        "name": "Paperless AI LLM caller",
+        "description": "paperless-ai calls api-gateway inference through the token vendor",
+        "roles": ["llm:inference", "svc-caller"],
+    },
 }
 
 

terraform/files/coredns/Corefile

@@ -36,8 +36,10 @@
     rewrite name paperless.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name img.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name api.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
-    rewrite name comfy.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
-    rewrite name comfyui.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    # poimen ran only in the live ConfigMap, never in this file or the Talos
+    # machine config. The next Talos reconcile would have deleted it and broken
+    # in-cluster resolution. Declared here so the source of truth carries it.
+    rewrite name poimen.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name gotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name pi.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name igotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local