fix(forgejo): raise the shared storage to 60Gi, the registry outgrew 20Gi

The container registry stores blobs on this volume. A single candle-vllm sm_70 image is 14GB, so a 20Gi volume cannot hold one. It filled, and every blob upload failed: Package registry API internal error: 500 write /data/tmp/package-upload/<id>: no space left on device The push client reports that as `received unexpected HTTP status: 500` after retrying every layer, which reads like a network or auth fault. It is neither. Applied live first (20Gi -> 40Gi -> 60Gi) and verified: /data now reports 58.9G total with 39.3G free, and the sm70 image pushed and verified by digest sha256:8533d742dfe99c926f42cf99db33e057acb1aafc4b74b0ac22f697f4a45cba27. This commit reconciles the chart with the live PVC, which is why the Application currently reads OutOfSync. NOTE FOR THE NEXT EXPANSION: Longhorn's admission webhook checks the PHYSICAL free space of the disk holding each replica, not the thin volume size. A request for 100Gi was rejected outright: disk ... does not have sufficient physical space for expansion: physical free space would drop below minimal: left=14596177920 < minimal=49931236556 60Gi fit on the same disk. Raise it in steps, and read the webhook message rather than retrying the same number.
2026-11-01 13:54:41 -07:00
31 changed files with 645 additions and 471 deletions
@@ -45,6 +45,8 @@ data:
        service: https://igotify.riotpiao.com
      - hostname: poimen.riotpiao.com
        service: https://poimen.riotpiao.com
      - hostname: comfyui.riotpiao.com
        service: https://comfyui.riotpiao.com
      - hostname: forgejo.riotpiao.com
        service: https://forgejo.riotpiao.com
      # ZITADEL. Without this rule the connector answers 404 for the hostname
@@ -0,0 +1,93 @@
 # ComfyUI — GPU-accelerated image generation on worker-1.
 # Uses 1x V100 32GB (sm70). Freed by scaling ornith 2→1.
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: comfyui
  namespace: comfyui
  labels:
    app: comfyui
 spec:
  # Retired on purpose. This Deployment held the 4th V100 on worker-1 while
  # serving nothing: its oauth2-proxy roll-out has been stuck for 42h and
  # failing 11621 times, because the Secret comfyui/comfyui-oauth2-proxy carries
  # only 1 key while the pod requests client-id, client-secret and
  # cookie-secret. Every request was therefore refused, so the card sat at 0%
  # utilisation and blocked a GPU that llm-serving can use.
  #
  # Scaling to 0 is the whole change. The Deployment, the Service, the Ingress
  # and the 50Gi comfyui-models PVC all stay, so nothing is lost and no data is
  # touched. To bring it back: set replicas to 1 here, then fix the Secret to
  # carry all three keys, otherwise the same stuck roll-out returns.
  replicas: 0
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app: comfyui
  template:
    metadata:
      labels:
        app: comfyui
    spec:
      nodeSelector:
        kubernetes.io/hostname: worker-1
      runtimeClassName: nvidia
      # k8s Service named 'comfyui' injects COMFYUI_PORT=tcp://... into pod env,
      # which clobbers ai-dock's own COMFYUI_PORT variable (expects a port number).
      # Disable service link injection to avoid the collision.
      enableServiceLinks: false
      containers:
        - name: comfyui
          image: ghcr.io/ai-dock/comfyui:v2-cuda-12.1.1-base-22.04
          ports:
            - containerPort: 8188
              protocol: TCP
          env:
            - name: NVIDIA_VISIBLE_DEVICES
              value: "all"
          resources:
            requests:
              cpu: "4"
              memory: 8Gi
              nvidia.com/gpu: "1"
            limits:
              cpu: "8"
              memory: 16Gi
              nvidia.com/gpu: "1"
          volumeMounts:
            - mountPath: /workspace/ComfyUI/models
              name: models
            - mountPath: /workspace/ComfyUI/output
              name: output
          readinessProbe:
            httpGet:
              path: /
              port: 8188
            periodSeconds: 10
            initialDelaySeconds: 30
          startupProbe:
            httpGet:
              path: /
              port: 8188
            failureThreshold: 120
            periodSeconds: 10
      volumes:
        - name: models
          persistentVolumeClaim:
            claimName: comfyui-models
        - name: output
          emptyDir: {}
 ---
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
  name: comfyui-models
  namespace: comfyui
 spec:
  accessModes:
    - ReadWriteOnce
  storageClassName: longhorn
  resources:
    requests:
      storage: 50Gi
@@ -0,0 +1,38 @@
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
  name: comfyui
  namespace: comfyui
  annotations:
    # No cert-manager annotation and no tls: block. This host is covered by
    # the *.riotpiao.com wildcard, which nginx serves as its
    # default-ssl-certificate (ingress-nginx/riotpiao-com-tls).
    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
    nginx.ingress.kubernetes.io/proxy-body-size: "0"
    # WebSocket support for ComfyUI's live preview
    nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
    nginx.ingress.kubernetes.io/upstream-hash-by: "$remote_addr"
    # Single sign-on. ComfyUI has no auth of its own, so every request is handed to
    # oauth2-proxy for an allow or deny decision.
    nginx.ingress.kubernetes.io/auth-url: "http://comfyui-oauth2-proxy.comfyui.svc.cluster.local:4180/oauth2/auth"
    nginx.ingress.kubernetes.io/auth-signin: "https://comfyui.riotpiao.com/oauth2/start?rd=$scheme://$host$request_uri"
    nginx.ingress.kubernetes.io/auth-response-headers: "x-auth-request-user,x-auth-request-email"
    # No configuration-snippet. The controller rejects that annotation as risky
    # ("annotation group ConfigurationSnippet contains risky annotation"), and a
    # rejected annotation drops the WHOLE ingress from nginx.conf: the host then
    # answers 404 with no entry in the access log. ingress-nginx sets the Upgrade
    # and Connection headers itself for a websocket upgrade.
 spec:
  ingressClassName: nginx
  rules:
    - host: comfyui.riotpiao.com
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: comfyui
                port:
                  number: 80
@@ -0,0 +1,8 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
  - deployment.yaml
  - service.yaml
  - oauth2-proxy.yaml
  - ingress.yaml
@@ -0,0 +1,105 @@
 # ComfyUI has no authentication of its own: comfyui.riotpiao.com is open to anyone
 # who can resolve the host. It also has no OIDC support, so ZITADEL cannot be a
 # login method inside it. oauth2-proxy is the front door instead: nginx asks it for
 # a decision on every request and it answers from a ZITADEL session.
 #
 # No API client calls this host today, so the whole host is gated. If a client ever
 # needs direct access, add a skip rule for the exact paths it uses rather than
 # opening the whole host.
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: comfyui-oauth2-proxy
  namespace: comfyui
  labels:
    app: comfyui-oauth2-proxy
 spec:
  replicas: 1
  selector:
    matchLabels:
      app: comfyui-oauth2-proxy
  template:
    metadata:
      labels:
        app: comfyui-oauth2-proxy
    spec:
      securityContext:
        runAsNonRoot: true
        runAsUser: 65534
        seccompProfile: {type: RuntimeDefault}
      containers:
        - name: oauth2-proxy
          image: quay.io/oauth2-proxy/oauth2-proxy:v7.7.1
          args:
            - --provider=oidc
            - --oidc-issuer-url=https://zitadel.riotpiao.com
            - --email-domain=*
            - --upstream=static://202
            - --reverse-proxy=true
            - --set-xauthrequest=true
            - --cookie-secure=true
            - --cookie-expire=12h
            - --cookie-refresh=1h
            - --skip-provider-button=true
            - --redirect-url=https://comfyui.riotpiao.com/oauth2/callback
            - --http-address=0.0.0.0:4180
          env:
            - name: OAUTH2_PROXY_CLIENT_ID
              valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: client-id}}
            - name: OAUTH2_PROXY_CLIENT_SECRET
              valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: client-secret}}
            - name: OAUTH2_PROXY_COOKIE_SECRET
              valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: cookie-secret}}
          ports:
            - {name: http, containerPort: 4180}
          readinessProbe:
            httpGet: {path: /ping, port: http}
            initialDelaySeconds: 5
          livenessProbe:
            httpGet: {path: /ping, port: http}
            initialDelaySeconds: 15
          securityContext:
            allowPrivilegeEscalation: false
            capabilities: {drop: ["ALL"]}
          resources:
            requests: {cpu: 10m, memory: 32Mi}
            limits: {cpu: 200m, memory: 128Mi}
 ---
 apiVersion: v1
 kind: Service
 metadata:
  name: comfyui-oauth2-proxy
  namespace: comfyui
  labels:
    app: comfyui-oauth2-proxy
 spec:
  type: ClusterIP
  selector:
    app: comfyui-oauth2-proxy
  ports:
    - {name: http, port: 4180, targetPort: http}
 ---
 # The login path must NOT be gated: a request to /oauth2/start that itself demands a
 # login redirects to /oauth2/start forever. ingress-nginx applies the annotations of
 # an ingress to the locations it defines, so a separate object without auth-url keeps
 # this path open. The earlier failure of this shape was not the split — it was the
 # configuration-snippet on the app ingress, which made the controller drop the whole
 # object and left only this one alive.
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
  name: comfyui-oauth2
  namespace: comfyui
 spec:
  ingressClassName: nginx
  rules:
    - host: comfyui.riotpiao.com
      http:
        paths:
          - path: /oauth2
            pathType: Prefix
            backend:
              service:
                name: comfyui-oauth2-proxy
                port:
                  number: 4180
@@ -0,0 +1,14 @@
 apiVersion: v1
 kind: Service
 metadata:
  name: comfyui
  namespace: comfyui
  labels:
    app: comfyui
 spec:
  selector:
    app: comfyui
  ports:
    - port: 80
      targetPort: 8188
      protocol: TCP
@@ -8,11 +8,7 @@ metadata:
  annotations:
    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
 spec:
-  # 2, not 3. The file's own header always said 2, and the third instance never
+  instances: 3
  # existed as a running pod: its volume only ever held a replica that could not
  # start, because talos-cp-2 disk2 has no free space. A third replica on a disk
  # that cannot host it is not redundancy, it is a permanent degraded volume.
  instances: 2
  imageName: ghcr.io/cloudnative-pg/postgresql:16.2
  bootstrap:
    initdb:
@@ -40,7 +40,7 @@ target is one root-of-trust file per thing that cannot rotate itself:
 | controller bootstrap key | chicken and egg: it creates every other key |
 | etcd defrag credentials | needed by a job that runs outside the controller |
-Everything else — Grafana, Argo CD, Forgejo, MinIO, Paperless, Immich,
+Everything else — Grafana, Argo CD, Forgejo, MinIO, Paperless, Immich, ComfyUI,
 api-gw, Gotify — is a KeyRotation object, and its secret leaves git.
 ## Status
@@ -64,6 +64,32 @@ spec:
 ---
 apiVersion: iam.homelab.riotpiao.com/v1
 kind: KeyRotation
 metadata:
  name: comfyui
  namespace: iam
  labels:
    app.kubernetes.io/part-of: key-rotation
 spec:
  provider: zitadel-oidc-client
  strategy: atomic
  intervalHours: 2160
  suspend: true
  target:
    zitadelProjectID: "396757796898471937"
    zitadelAppID: "397015743994003854"  # app resource id, not the client id 397015744010781070
    zitadelClientID: "397015744010781070"
    secretName: comfyui-oauth2-proxy
    secretNamespace: comfyui
    secretKeys:
      secret: client-secret
    reload: restart
    consumers:
      - {kind: Deployment, name: comfyui-oauth2-proxy, namespace: comfyui}
  verify:
    introspect: true
 ---
 apiVersion: iam.homelab.riotpiao.com/v1
 kind: KeyRotation
 metadata:
  name: minio
  namespace: iam
@@ -81,11 +107,7 @@ spec:
    secretName: minio-oidc
    secretNamespace: storage
    secretKeys:
-      # The key name here must match the consumer's secretKeyRef, not a house style.
+      secret: client-secret
      # minio-tenant.yaml reads MINIO_IDENTITY_OPENID_CLIENT_SECRET. While this said
      # `client-secret`, the tenant failed with CreateContainerConfigError on every
      # restart and the rotation wrote a key nobody read.
      secret: MINIO_IDENTITY_OPENID_CLIENT_SECRET
    reload: hot
    consumers:
      - {kind: StatefulSet, name: minio-cluster-az-a, namespace: storage}
@@ -33,19 +33,7 @@ metadata:
    app: clef-serve
    app.kubernetes.io/part-of: llm-serving
 spec:
-  # Parked at 0 to free the only V100 that was not serving anything, so candle-vllm can
+  replicas: 1
  # run its Phase 0 on one card. All four cards were allocated (coder 2, reasoning 1,
  # this 1) and a new GPU workload had nowhere to land.
  #
  # This is the only GPU consumer in the namespace that is not vLLM, which is why it is
  # the first card released: the standing instruction is to keep the vLLM pods healthy
  # until candle-vllm answers, and this pod is not one of them.
  #
  # It was serving nothing. The last 30 log lines at park time were 30/30 kubelet
  # /health probes and no inference request since start (2026-10-30T06:23:40Z). Nothing
  # in git selects this service. Restore it once the rollout leaves a card free:
  # 30B-A3B takes 1, Coder-Next TP=2 takes 2, so card 4 is this one.
  replicas: 0
  # Recreate, not RollingUpdate. Every GPU card is allocated, so a rolling update leaves
  # the new pod Pending forever waiting for a card the old pod still holds.
  strategy:
@@ -1,134 +0,0 @@
 apiVersion: serving.kserve.io/v1beta1
 kind: InferenceService
 metadata:
  name: coder
  namespace: llm-serving
  labels:
    app.kubernetes.io/name: llm-coder
    app.kubernetes.io/part-of: llm-serving
  annotations:
    serving.kserve.io/deploymentMode: RawDeployment
 spec:
  predictor:
    minReplicas: 1
    maxReplicas: 1
    nodeSelector:
      kubernetes.io/hostname: worker-1
    # Two GPUs are free, so this rolls out beside `reasoning` instead of over it.
    # Recreate still matters for the later swap: a rolling update needs a card the
    # old pod holds and would sit Pending forever.
    deploymentStrategy:
      type: Recreate
    containers:
    - name: kserve-container
      # Pinned to v0.11.0 on purpose. This is the last image measured to work on this
      # node: torch 2.8.0+cu128, arch list sm_70..sm_120, driver 580.159.04.
      # v0.15.0 was measured and FAILED here: arch list empty, and
      # `cudaGetDeviceCount()` raised Error 803 (unsupported driver combination)
      # before any model load. Do not raise this tag without re-running that probe.
      image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
      args:
      # Qwen3-Coder-30B-A3B-Instruct: 30.5B total, 3.3B active. Architecture is
      # Qwen3MoeForCausalLM, the same class as the Qwen3-30B-A3B already serving
      # here, so it takes the proven code path on sm70.
      #
      # Qwen ships this model only as BF16 (61.1 GB) and FP8 (no FP8 on Volta).
      # 4-bit GPTQ is the only form that fits one 32 GB card. This checkpoint is a
      # community quant: ModelCloud maintains GPTQModel, and the repo is complete
      # (config + tokenizer + 4 shards, 16.9 GB). Verified by loading, not by trust.
      #
      # Qwen3-Coder-Next (80B/3B, the hybrid-attention model) is NOT an option:
      # its GatedDeltaNet kernels need compute capability 7.5+ and the `fla` module,
      # and sm70 has neither. Measured, not assumed.
      - --model=ModelCloud/Qwen3-Coder-30B-A3B-Instruct-GPTQMODEL-W4A16-A
      # `reasoning` stays a served name so an existing client that sends
      # model=reasoning keeps working after the DNS rename. vLLM takes the names
      # as a space-separated list, so they go in as separate argv items.
      - --served-model-name
      - coder
      - reasoning
      - --quantization=gptq
      - --dtype=float16
      - --kv-cache-dtype=auto
      # Native window is 262144 and rope_scaling is null, so 256K needs no YaRN.
      # One card cannot hold it: measured KV pool on a single V100 was 132,672
      # tokens. TP=2 halves the weights per card (7.8 GiB each), which frees room
      # for the larger pool. Measured after boot: see the KV line in the log.
      - --max-model-len=262144
      - --tensor-parallel-size=2
      - --pipeline-parallel-size=1
      - --gpu-memory-utilization=0.90
      # A single 262,144-token request consumes the whole pool, so keep few slots.
      - --max-num-seqs=4
      - --max-num-batched-tokens=8192
      - --long-prefill-token-threshold=2048
      - --enable-chunked-prefill
      - --enable-prefix-caching
      # Measured: with the default (custom all-reduce ON) the engine finished CUDA graph
      # capture in 6s at 17:01:56 and then went silent for 14+ minutes, emitting only
      # "No available shared memory broadcast block found in 60 seconds". The front-end
      # never received the ready message. vLLM's custom all-reduce registers its own
      # CUDA graph addresses and deadlocks on this Volta/PCIe pair. NCCL is the fallback
      # and is correct here, because TP=2 across two PCIe V100s gains nothing from the
      # custom kernel anyway.
      - --disable-custom-all-reduce
      - --reasoning-parser=qwen3
      - --enable-auto-tool-choice
      - --tool-call-parser=hermes
      - --host=0.0.0.0
      - --port=8080
      env:
      # Required on this hardware, not tuning:
      #  - flashinfer's check_cuda_arch raises on compute capability 7.0,
      #    so the sampler must stay off
      #  - FlashInfer and FlashAttention-2 need sm75+, so Triton serves attention
      - name: VLLM_USE_FLASHINFER_SAMPLER
        value: "0"
      - name: VLLM_ATTENTION_BACKEND
        value: TRITON_ATTN
      - name: HF_HOME
        value: /mnt/models
      ports:
      - containerPort: 8080
        protocol: TCP
      readinessProbe:
        httpGet:
          path: /health
          port: 8080
        periodSeconds: 10
      startupProbe:
        httpGet:
          path: /health
          port: 8080
        # TP=2 at 256K needs ~10 min before the API binds: 22s weights, 80s torch.compile,
        # 85s KV profiling, then graph capture. 120 x 15s = 30 min of headroom.
        failureThreshold: 120
        periodSeconds: 15
      resources:
        # worker-1 exposes only 63,495 MiB of allocatable memory, so a 24Gi request
        # never schedules. Two vLLM workers keep the weights on the GPU and need
        # little host RAM, so the request stays small and the limit allows burst.
        requests:
          cpu: "8"
          memory: 12Gi
          nvidia.com/gpu: "2"
        limits:
          cpu: "16"
          memory: 40Gi
          nvidia.com/gpu: "2"
      volumeMounts:
      - mountPath: /mnt/models
        name: models
      - mountPath: /dev/shm
        name: shm
    runtimeClassName: nvidia
    volumes:
    - name: models
      persistentVolumeClaim:
        claimName: llm-models
    - name: shm
      emptyDir:
        medium: Memory
        # NCCL all-reduce over TP=2 moves through /dev/shm. The 64Mi default is
        # too small and shows up as an init hang or a bus error.
        sizeLimit: 8Gi
@@ -14,9 +14,7 @@ resources:
  # v0.5.22+, published only on CUDA 13, and CUDA 13 dropped Volta kernels. The model
  # instead runs the implementation it ships (joint_schema_model.py) in fp16 on one V100.
  - clef-serve.yaml
-  # Stable client-facing name for the chat model. poimen-memory points at this Service,
+  - reasoning.yaml
  # so the vLLM -> candle-vllm handoff is a selector edit here and no client change.
  - reasoning-long-service.yaml
  # ornith is retired: its InferenceService still reports Ready=True while running zero
  # replicas, because desired equals available at 0. Its manifest was never committed,
  # so it is left unmanaged and is not listed here. Delete the CR when convenient.
@@ -66,21 +66,6 @@ spec:
          port: 80
        - protocol: TCP
          port: 8000
    # Allow from poimen namespace (poimen-memory ingest + query).
    # The client-side egress rule is the other half and must stay in step:
    # NetworkPolicy poimen/poimen-memory-llm-egress permits port 8080 to pods
    # labelled part-of=llm-serving, which is the same selector used here. This
    # policy had no matching ingress rule at all, so the packet was allowed out
    # and dropped here. The client then saw a bare timeout and the predictor log
    # stayed empty, which is why this failure looked like a hung inference engine
    # for 34 days.
    - from:
        - namespaceSelector:
            matchLabels:
              kubernetes.io/metadata.name: poimen
      ports:
        - protocol: TCP
          port: 8080
    # Allow intra-namespace (pod-to-pod within llm-serving)
    - from:
        - podSelector:
@@ -1,6 +1,5 @@
 # qwen2.5:3b-instruct on CPU (talos-cp-2, 144GB RAM, 24 cores).
-# Moved off GPU to free a V100, first for ComfyUI and later for llm-serving.
+# Moved off GPU to free a V100 for ComfyUI. Latency ~10x slower
 # Latency ~10x slower
 # than GPU but sufficient for lightweight tasks (summarization,
 # classification, quick answers).
 apiVersion: apps/v1
@@ -1,43 +0,0 @@
 # Stable client-facing name for the chat model, deliberately decoupled from the
 # process that currently serves it.
 #
 # poimen-memory addresses this Service, not a pod and not an InferenceService. That
 # means the vLLM -> candle-vllm handoff changes only this file's selector. The client
 # config never changes again, so the ingest cutover has no window in which the host
 # name is wrong.
 #
 # Two ports on purpose:
 #   - 80    matches every other predictor Service in this namespace, and matches
 #           poimen-memory's LLM_CHAT_BASE, which carries no port number.
 #   - 8080  is the container port. It is also the only port llm-serving-ingress and
 #           poimen-memory-llm-egress permit, and the port the handoff readiness check
 #           uses. Publishing it directly avoids a port translation that is easy to get
 #           wrong in a curl.
 #
 # The selector names the vLLM reasoning pod today. To hand off, replace the three
 # selector labels with candle-vllm's pod labels and nothing else. Do not delete and
 # recreate this Service: a delete drops the ClusterIP, and a client that cached the
 # name during the gap will fail until it re-resolves.
 apiVersion: v1
 kind: Service
 metadata:
  name: reasoning-long-predictor
  namespace: llm-serving
  labels:
    app.kubernetes.io/part-of: llm-serving
    app: reasoning-long-predictor
 spec:
  type: ClusterIP
  selector:
    app.kubernetes.io/name: llm-reasoning
    component: predictor
    serving.kserve.io/inferenceservice: reasoning
  ports:
    - name: http
      protocol: TCP
      port: 80
      targetPort: 8080
    - name: http-8080
      protocol: TCP
      port: 8080
      targetPort: 8080
@@ -30,28 +30,7 @@ spec:
      # Measured on this cluster, one V100-32GB, 64k context: 15.55 GiB of weights,
      # 137k tokens of KV cache, 2.09x concurrency at 65536 tokens per request.
      - --model=Qwen/Qwen3-30B-A3B-GPTQ-Int4
-      # Two served names, one pod, on purpose. `reasoning-long` is the name the goal map
+      - --served-model-name=reasoning
      # uses for this model, and the name qwen-finetune wiring targets. It is NOT a
      # separate InferenceService: this model already occupies the only free card, and a
      # second pod on the same weights cannot schedule (4 of 4 GPUs allocated: coder 2,
      # this 1, clef-serve 1). Serving both names from one process costs nothing and
      # keeps the two callers working without a second weights copy.
      #
      # `reasoning` is deliberately still served here. poimen-memory pins
      # LLM_CHAT_BASE=reasoning-predictor, so dropping the name here breaks ingest
      # immediately. coder-predictor also serves `reasoning`; that ambiguity is known
      # and documented in the PR. Resolve it by moving the client host, not by
      # removing the name here.
      #
      # The two names are separate list entries, NOT one comma-joined value. This
      # image's parser returns ['reasoning,reasoning-long'] for the comma form, i.e. a
      # single model whose name contains a comma, and api_server.py then iterates
      # args.served_model_name to build the model list. Verified against vLLM 0.11.0:
      #   FlexibleArgumentParser().parse_args([... "--served-model-name", "a", "b"])
      #     -> ['a', 'b']
      - --served-model-name
      - reasoning
      - reasoning-long
      - --quantization=gptq
      - --dtype=float16
      - --kv-cache-dtype=auto
@@ -1,38 +0,0 @@
 ---
 # Public entry point for the poimen-memory HTTP API.
 #
 # The API was reachable only inside the cluster (ClusterIP :8080), so a local pi
 # session had no way to use it. This Ingress exposes the same handler set the
 # in-cluster clients use: /mcp, /memory/*, /graphql, /health, /ready.
 #
 # Cloudflare terminates TLS in front of the nginx ingress, which is why this rule
 # carries no `tls:` section — the same shape as ingress.yaml next to it.
 #
 # Auth: poimen-memory validates ZITADEL JWTs (MEM_AUTH_MODE=jwt). A caller
 # without a valid bearer token gets 401, so exposing the route does not expose
 # the data.
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
  name: poimen-api
  namespace: poimen
  annotations:
    # MCP calls and ingest can be slow; the provider may stream. Match the
    # frontend ingress so a long tool call is not cut at 60s.
    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
    nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
    nginx.ingress.kubernetes.io/proxy-buffering: "off"
    nginx.ingress.kubernetes.io/proxy-body-size: "50m"
 spec:
  ingressClassName: nginx
  rules:
    - host: api-poimen.riotpiao.com
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: poimen-memory
                port:
                  number: 8080
@@ -6,5 +6,3 @@ kind: Kustomization
 # riotpiao-poimen/poimen-frontend.git k8s/ — each repo owns its own k8s.
 resources:
  - ingress.yaml
  # The memory API. Without it a local pi session has no route to :8080.
  - api-ingress.yaml
@@ -159,3 +159,54 @@ spec:
    automated:
      prune: true
      selfHeal: true
 ---
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: kyverno
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "0"
 spec:
  project: homelab
  source:
    repoURL: https://kyverno.github.io/kyverno/
    chart: kyverno
    targetRevision: "1.14.0"
    helm:
      valueFiles:
        - $values/k8s/bootstrap/kyverno/kyverno-values.yaml
  sources:
    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
      targetRevision: main
      ref: values
  destination:
    server: https://kubernetes.default.svc
    namespace: kyverno
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - CreateNamespace=true
 ---
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: kyverno-policies
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "0"
 spec:
  project: homelab
  source:
    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
    targetRevision: main
    path: k8s/bootstrap/kyverno
  destination:
    server: https://kubernetes.default.svc
    namespace: kyverno
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
@@ -0,0 +1,32 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: comfyui
  namespace: argocd
  labels:
    app.kubernetes.io/name: comfyui
    app.kubernetes.io/component: image-generation
  annotations:
    argocd.argoproj.io/sync-wave: "8"
 spec:
  project: homelab
  revisionHistoryLimit: 3
  source:
    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
    targetRevision: main
    path: k8s/apps/comfyui
  destination:
    server: https://kubernetes.default.svc
    namespace: comfyui
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - CreateNamespace=true
    retry:
      limit: 5
      backoff:
        duration: 5s
        factor: 2
        maxDuration: 3m
@@ -0,0 +1,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: kyverno
 resources:
  - policies.yaml
@@ -0,0 +1,42 @@
 # Kyverno: Policy engine for Kubernetes image scanning, Pod security, and admission control
 # Scan all images, enforce baseline Pod Security Standard, prevent privilege escalation
 replicaCount: 1
 image:
  registry: ghcr.io
  repository: kyverno/kyverno
  tag: "v1.14.0"
 config:
  # Webhook timeout for policy evaluation. Increase if scanning takes longer.
  webhookTimeoutSeconds: 30
  # Failure policy: fail-open (audit/log) vs fail-closed (reject on error)
  failurePolicy: fail
  # Resource limits for webhook
  webhookAnnotations:
    rules: "allow"
 # Pod security via Kyverno instead of Pod Security Policies (deprecated)
 # Enforces baseline restrictions cluster-wide, with exceptions for privileged namespaces
 podSecurityContext:
  runAsNonRoot: true
  runAsUser: 1000
 rbac:
  create: true
 resources:
  requests:
    memory: "256Mi"
    cpu: "100m"
  limits:
    memory: "512Mi"
    cpu: "500m"
 # Webhook configuration
 webhook:
  timeoutSeconds: 30
  # Failure policy: "Fail" (reject on error) or "Ignore" (audit-only)
  # Set to "Ignore" for initial testing, then change to "Fail"
  failurePolicy: ignore
@@ -0,0 +1,204 @@
 # Kyverno ClusterPolicies: Image scanning, Pod security, and admission control
 ---
 # Policy 1: Require non-root containers
 apiVersion: kyverno.io/v1
 kind: ClusterPolicy
 metadata:
  name: require-non-root
  namespace: kyverno
 spec:
  validationFailureAction: audit  # audit first, then change to enforce
  rules:
    - name: check-runAsNonRoot
      match:
        any:
          - resources:
              kinds:
                - Pod
              selector:
                matchLabels:
                  pod-security.kubernetes.io/enforce: "!privileged"
      validate:
        message: "Container must not run as root"
        pattern:
          spec:
            containers:
              - securityContext:
                  runAsNonRoot: true
 ---
 # Policy 2: Drop all Linux capabilities, add only required ones
 apiVersion: kyverno.io/v1
 kind: ClusterPolicy
 metadata:
  name: require-dropped-caps
  namespace: kyverno
 spec:
  validationFailureAction: audit
  rules:
    - name: drop-all-capabilities
      match:
        any:
          - resources:
              kinds:
                - Pod
              selector:
                matchLabels:
                  pod-security.kubernetes.io/enforce: "!privileged"
      validate:
        message: "All Linux capabilities must be dropped"
        pattern:
          spec:
            containers:
              - securityContext:
                  capabilities:
                    drop:
                      - ALL
 ---
 # Policy 3: Require image tags (no 'latest')
 apiVersion: kyverno.io/v1
 kind: ClusterPolicy
 metadata:
  name: disallow-latest-tag
  namespace: kyverno
 spec:
  validationFailureAction: audit  # Change to enforce after testing
  rules:
    - name: disallow-latest
      match:
        any:
          - resources:
              kinds:
                - Pod
                - Deployment
                - StatefulSet
                - DaemonSet
                - Job
      validate:
        message: "Image tag 'latest' is not allowed. Use explicit version tags."
        pattern:
          spec:
            =(template):
              spec:
                containers:
                  - image: "!*:latest"
                =(initContainers):
                  - image: "!*:latest"
 ---
 # Policy 4: Restrict images to trusted registries
 apiVersion: kyverno.io/v1
 kind: ClusterPolicy
 metadata:
  name: restrict-registries
  namespace: kyverno
 spec:
  validationFailureAction: audit
  rules:
    - name: trusted-registries
      match:
        any:
          - resources:
              kinds:
                - Pod
                - Deployment
                - StatefulSet
                - DaemonSet
                - Job
              selector:
                matchLabels:
                  pod-security.kubernetes.io/enforce: "!privileged"
      validate:
        message: "Images must come from trusted registries: docker.io, ghcr.io, quay.io, k8s.gcr.io, registry.k8s.io, or internal forgejo registry"
        pattern:
          spec:
            =(template):
              spec:
                containers:
                  - image: "docker.io/* | ghcr.io/* | quay.io/* | k8s.gcr.io/* | registry.k8s.io/* | forgejo.riotpiao.com/* | *"
 ---
 # Policy 5: Require read-only root filesystem (audit only, exceptions for apps that need writes)
 apiVersion: kyverno.io/v1
 kind: ClusterPolicy
 metadata:
  name: require-readonly-filesystem
  namespace: kyverno
 spec:
  validationFailureAction: audit
  rules:
    - name: check-readOnlyRootFilesystem
      match:
        any:
          - resources:
              kinds:
                - Pod
              selector:
                matchLabels:
                  pod-security.kubernetes.io/enforce: "!privileged"
      validate:
        message: "Root filesystem should be read-only for defense-in-depth"
        pattern:
          spec:
            containers:
              - securityContext:
                  readOnlyRootFilesystem: true
 ---
 # Policy 6: Require resource requests and limits (prevent resource starvation)
 apiVersion: kyverno.io/v1
 kind: ClusterPolicy
 metadata:
  name: require-resource-limits
  namespace: kyverno
 spec:
  validationFailureAction: audit
  rules:
    - name: check-resources
      match:
        any:
          - resources:
              kinds:
                - Pod
                - Deployment
                - StatefulSet
                - DaemonSet
              excludeResources:
                namespaceSelector:
                  matchLabels:
                    kubernetes.io/metadata.name: "kyverno|kube-system|kube-node-lease"
      validate:
        message: "CPU and memory requests and limits are required"
        pattern:
          spec:
            =(template):
              spec:
                containers:
                  - resources:
                      requests:
                        memory: "?*"
                        cpu: "?*"
                      limits:
                        memory: "?*"
                        cpu: "?*"
 ---
 # Policy 7: Require securityContext on all containers
 apiVersion: kyverno.io/v1
 kind: ClusterPolicy
 metadata:
  name: require-security-context
  namespace: kyverno
 spec:
  validationFailureAction: audit
  rules:
    - name: check-securityContext
      match:
        any:
          - resources:
              kinds:
                - Pod
              selector:
                matchLabels:
                  pod-security.kubernetes.io/enforce: "!privileged"
      validate:
        message: "securityContext must be defined"
        pattern:
          spec:
            containers:
              - securityContext: {}
@@ -161,19 +161,19 @@ ingress:
 # The database was never the bottleneck. A real action_runner UPDATE measured
 # 2.133ms with zero lock waits, so this is CPU starvation in Gitea itself.
 #
-# requests.cpu stays 200m and limits.cpu stays 1000m. The 3000m limit was a
+# requests.cpu rises to 750m because a request below real usage earns the pod
-# temporary measure while the Actions stall was diagnosed; the real fix landed
+# little scheduling weight during contention. limits.cpu rises to 3000m
-# elsewhere, so the limit is back to the original.
+# because diff computation is bursty and needs room to run without throttling.
 #
-# Caution: this pod runs on talos-cp-3, a control-plane node. If Gitea
+# Caution: this pod runs on talos-cp-3, a control-plane node. 3000m is 25% of
-# throttles again at 1000m, move the workload off the control plane rather
+# that node's 11950m allocatable CPU. If Gitea sustains near this limit, move
-# than raising the limit.
+# the workload off the control plane rather than raising the limit again.
 resources:
  requests:
-    cpu: 200m
+    cpu: 750m
    memory: 768Mi
  limits:
-    cpu: 1000m
+    cpu: 3000m
    memory: 2Gi
 # Tolerations for control-plane
@@ -1,90 +0,0 @@
 # One-shot destructive health scan of a candidate Longhorn disk.
 #
 # WHY THIS IS A MANUAL JOB AND NOT AN ARGOCD HOOK
 #   It writes over the entire raw device. An ArgoCD PostSync hook would run it
 #   on every sync and destroy whatever the volume holds. Apply it by hand,
 #   once, on a disk that has no partitions and no mounts.
 #
 # WHY NOT TALOS
 #   Talos declares disks by mounting them (machine.disks in the machine
 #   config). It has no primitive for "run this command on the host", so a scan
 #   cannot live in the Talos config. This Job is the IaC-correct home.
 #
 # ORDER OF OPERATIONS
 #   1. Scan the raw device with this Job        <- the disk must be UNMOUNTED
 #   2. If it passes, add the disk to terraform.tfvars.local `longhorn_disks`
 #      with device /dev/disk/by-id/<wwid> and run `make apply-cp`
 #   3. Register it with Longhorn via the pattern in longhorn-add-disks-job.yaml
 #
 # What it does
 #   fio writes a CRC32C-verified pattern across the whole device, then reads
 #   every block back and compares. Any unreadable or mismatched sector fails
 #   the Job. This is a full write and read pass, not a sample.
 #
 # Expected duration
 #   2 TB on a 5400 rpm 2.5" drive at roughly 90 MB/s is about 6 h to write and
 #   6 h to verify. Budget 12 hours. Check progress with:
 #     kubectl -n llm-io logs job/disk-scan-cp1-sdb -f
 apiVersion: batch/v1
 kind: Job
 metadata:
  name: disk-scan-cp1-sdb
  namespace: llm-io
  labels:
    app: disk-scan
 spec:
  backoffLimit: 0
  # Do not let a stuck scan retry silently.
  template:
    metadata:
      labels:
        app: disk-scan
    spec:
      restartPolicy: Never
      nodeSelector:
        kubernetes.io/hostname: talos-cp-1
      containers:
        - name: scan
          image: ubuntu:22.04
          securityContext:
            privileged: true
          command:
            - /bin/bash
            - -c
            - |
              set -euo pipefail
              apt-get update -qq
              apt-get install -y -qq smartmontools fio
              echo "=== device identity ==="
              smartctl -i /dev/sdb | tail -12
              echo "=== SMART before ==="
              smartctl -H /dev/sdb | tail -2
              echo "=== full write + verify (destructive) ==="
              fio --name=devverify \
                  --filename=/dev/sdb \
                  --rw=write \
                  --bs=1M \
                  --iodepth=4 \
                  --ioengine=libaio \
                  --direct=1 \
                  --verify=crc32c \
                  --do_verify=1 \
                  --verify_fatal=1 \
                  --eta=always \
                  --output-format=normal
              echo "=== SMART after ==="
              smartctl -H /dev/sdb | tail -2
              smartctl -A /dev/sdb | awk '/Reallocated_Sector|Current_Pending|Offline_Uncorrect|UDMA_CRC/{print "  "$0}'
              echo "=== SCAN PASSED ==="
          volumeMounts:
            - name: dev
              mountPath: /dev
      volumes:
        - name: dev
          hostPath:
            path: /dev
@@ -1,28 +0,0 @@
 ---
 # Keep the GraphQL server reachable only from the one service that proxies it.
 #
 # poimen-graphql is already off the internet: its Service is ClusterIP on 8090 and
 # no Ingress names it. What it lacked is in-cluster isolation — a ClusterIP answers
 # every pod in the cluster.
 #
 # The only caller is poimen-memory, whose /graphql route forwards the request body
 # to MEM_GQL_PORT (8090). No other workload references this service.
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy
 metadata:
  name: poimen-graphql-ingress
  namespace: poimen
 spec:
  podSelector:
    matchLabels:
      app.kubernetes.io/name: poimen-graphql
  policyTypes:
    - Ingress
  ingress:
    - from:
        - podSelector:
            matchLabels:
              app.kubernetes.io/name: poimen-memory
      ports:
        - protocol: TCP
          port: 8090
@@ -6,10 +6,6 @@ kind: Kustomization
 # deployments (wave 0) are applied.
 resources:
  - memory-db.yaml
  # ClusterIP alone is not isolation: every pod in the cluster can reach it.
  - memory-db-netpol.yaml
  # Only poimen-memory's /graphql proxy calls 8090.
  - graphql-netpol.yaml
  - poimen-sandbox-namespace.yaml
  - poimen-sandbox-quota.yaml
  - pi-session-netpol.yaml
@@ -1,42 +0,0 @@
 ---
 # Keep the memory database reachable only from inside its own namespace.
 #
 # What this does NOT change: the database was already unreachable from the
 # internet. `memory-db-rw`, `memory-db-ro` and `memory-db-r` are ClusterIP
 # services, no Ingress backs them, and no public DNS name resolves to them.
 #
 # What it fixes: a ClusterIP service is reachable by EVERY pod in the cluster,
 # because Kubernetes has no implicit isolation. Cilium enforces NetworkPolicy, so
 # stating the allowed sources is what turns "not on the internet" into "not
 # reachable from another tenant's namespace".
 #
 # Two sources, both required:
 #  - the poimen namespace, which holds every client: poimen-memory, poimen-graphql,
 #    poimen-worker, poimen-workflows and the migration Job.
 #  - cnpg-system, where the CloudNativePG operator lives. It connects to the
 #    instance for switchover and failover, so a narrower rule breaks recovery.
 #
 # No other port is opened. The operator's metrics endpoint stays closed because no
 # PodMonitor for this cluster exists, so nothing scrapes it.
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy
 metadata:
  name: memory-db-ingress
  namespace: poimen
 spec:
  podSelector:
    matchLabels:
      cnpg.io/cluster: memory-db
  policyTypes:
    - Ingress
  ingress:
    - from:
        - namespaceSelector:
            matchLabels:
              kubernetes.io/metadata.name: poimen
        - namespaceSelector:
            matchLabels:
              kubernetes.io/metadata.name: cnpg-system
      ports:
        - protocol: TCP
          port: 5432
@@ -32,6 +32,7 @@ ISSUER = "https://zitadel.riotpiao.com"
 APPS = [
    ("grafana",   "grafana",   "grafana-oidc",         "logging",   "GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET", "deploy", "grafana"),
    ("argocd",    "argocd",    "oidc-secret",           "argocd",    "client-secret",                       "deploy", "argocd-server"),
    ("comfyui",   "comfyui",   "comfyui-oauth2-proxy", "comfyui",   "client-secret",                       "deploy", "comfyui-oauth2-proxy"),
    ("minio",     "minio",     "minio-oidc",           "storage",   "MINIO_IDENTITY_OPENID_CLIENT_SECRET",  "sts",    "minio-cluster-az-a"),
    ("paperless", "paperless", "paperless-oidc",       "paperless", "client-secret",                       "deploy", "paperless"),
    ("immich",    "immich",    "immich-oidc",          "immich",    "client-secret",                       "deploy", "immich-server"),
@@ -128,6 +128,11 @@ OIDC_APPS = {
        "https://forgejo.riotpiao.com/user/oauth2/zitadel/callback",
        "http://localhost:3000/user/oauth2/authentik/callback",
        "https://forgejo.riotpiao.com/user/oauth2/authentik/callback"]},
    # ComfyUI has no native OIDC. oauth2-proxy is its front door and this app is
    # the proxy's client, so the callback path belongs to the proxy.
    "comfyui": {**_WEB, "redirect_uris": [
        "http://localhost:4180/oauth2/callback",
        "https://comfyui.riotpiao.com/oauth2/callback"]},
    "grafana": {**_WEB, "redirect_uris": [
        "http://localhost:3000/login/generic_oauth",
        "https://grafana.riotpiao.com/login/generic_oauth"]},
@@ -138,11 +143,7 @@ OIDC_APPS = {
        "http://localhost:8000", "http://localhost:18000"]},
    "minio": {**_WEB, "redirect_uris": [
        "http://localhost:9000/auth/sso/oauth2/code",
-        "https://minio.riotpiao.com/auth/sso/oauth2/code",
+        "https://minio.riotpiao.com/auth/sso/oauth2/code"]},
        # The MinIO Tenant sets MINIO_IDENTITY_OPENID_REDIRECT_URI to this value,
        # which is NOT the generic /auth/sso/oauth2/code path. Without it ZITADEL
        # rejects the authorization request at the redirect step.
        "https://minio.riotpiao.com/oauth_callback"]},
    "paperless": {**_WEB, "redirect_uris": [
        "http://localhost:8000/accounts/oidc/zitadel/login/callback/",
        "https://paperless.riotpiao.com/accounts/oidc/zitadel/login/callback/",
@@ -358,13 +359,6 @@ MACHINE_CALLERS = {
        "description": "poimen calls api-gateway inference",
        "roles": ["llm:inference", "svc-caller"],
    },
    # paperless-ai is a vendor image: it cannot sign an assertion, so a sidecar mints
    # a token with this account and writes it to the file the image already reads.
    "paperless-llm": {
        "name": "Paperless AI LLM caller",
        "description": "paperless-ai calls api-gateway inference through the token vendor",
        "roles": ["llm:inference", "svc-caller"],
    },
 }
@@ -36,10 +36,8 @@
    rewrite name paperless.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    rewrite name img.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    rewrite name api.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
-    # poimen ran only in the live ConfigMap, never in this file or the Talos
+    rewrite name comfy.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
-    # machine config. The next Talos reconcile would have deleted it and broken
+    rewrite name comfyui.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    # in-cluster resolution. Declared here so the source of truth carries it.
    rewrite name poimen.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    rewrite name gotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    rewrite name pi.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    rewrite name igotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local