chore(kyverno): drop the two Application manifests from 00-substrate

PR #100 removed k8s/bootstrap/kyverno but not this file: the git add for it
failed and the error was redirected away, so the commit landed with only the
deletions. main therefore still declared both Applications and ArgoCD kept
them alive as Unknown / OutOfSync-Missing.

Removes the kyverno and kyverno-policies Application documents. The bootstrap
directory is already gone from main.

k8s/apps/cloudflared/configmap.yaml

@@ -45,8 +45,6 @@ data:
         service: https://igotify.riotpiao.com
       - hostname: poimen.riotpiao.com
         service: https://poimen.riotpiao.com
-      - hostname: comfyui.riotpiao.com
-        service: https://comfyui.riotpiao.com
       - hostname: forgejo.riotpiao.com
         service: https://forgejo.riotpiao.com
       # ZITADEL. Without this rule the connector answers 404 for the hostname

k8s/apps/comfyui/deployment.yaml

@@ -1,93 +0,0 @@
-# ComfyUI — GPU-accelerated image generation on worker-1.
-# Uses 1x V100 32GB (sm70). Freed by scaling ornith 2→1.
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: comfyui
-  namespace: comfyui
-  labels:
-    app: comfyui
-spec:
-  # Retired on purpose. This Deployment held the 4th V100 on worker-1 while
-  # serving nothing: its oauth2-proxy roll-out has been stuck for 42h and
-  # failing 11621 times, because the Secret comfyui/comfyui-oauth2-proxy carries
-  # only 1 key while the pod requests client-id, client-secret and
-  # cookie-secret. Every request was therefore refused, so the card sat at 0%
-  # utilisation and blocked a GPU that llm-serving can use.
-  #
-  # Scaling to 0 is the whole change. The Deployment, the Service, the Ingress
-  # and the 50Gi comfyui-models PVC all stay, so nothing is lost and no data is
-  # touched. To bring it back: set replicas to 1 here, then fix the Secret to
-  # carry all three keys, otherwise the same stuck roll-out returns.
-  replicas: 0
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app: comfyui
-  template:
-    metadata:
-      labels:
-        app: comfyui
-    spec:
-      nodeSelector:
-        kubernetes.io/hostname: worker-1
-      runtimeClassName: nvidia
-      # k8s Service named 'comfyui' injects COMFYUI_PORT=tcp://... into pod env,
-      # which clobbers ai-dock's own COMFYUI_PORT variable (expects a port number).
-      # Disable service link injection to avoid the collision.
-      enableServiceLinks: false
-      containers:
-        - name: comfyui
-          image: ghcr.io/ai-dock/comfyui:v2-cuda-12.1.1-base-22.04
-          ports:
-            - containerPort: 8188
-              protocol: TCP
-          env:
-            - name: NVIDIA_VISIBLE_DEVICES
-              value: "all"
-          resources:
-            requests:
-              cpu: "4"
-              memory: 8Gi
-              nvidia.com/gpu: "1"
-            limits:
-              cpu: "8"
-              memory: 16Gi
-              nvidia.com/gpu: "1"
-          volumeMounts:
-            - mountPath: /workspace/ComfyUI/models
-              name: models
-            - mountPath: /workspace/ComfyUI/output
-              name: output
-          readinessProbe:
-            httpGet:
-              path: /
-              port: 8188
-            periodSeconds: 10
-            initialDelaySeconds: 30
-          startupProbe:
-            httpGet:
-              path: /
-              port: 8188
-            failureThreshold: 120
-            periodSeconds: 10
-      volumes:
-        - name: models
-          persistentVolumeClaim:
-            claimName: comfyui-models
-        - name: output
-          emptyDir: {}
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: comfyui-models
-  namespace: comfyui
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: longhorn
-  resources:
-    requests:
-      storage: 50Gi

k8s/apps/comfyui/ingress.yaml

@@ -1,38 +0,0 @@
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: comfyui
-  namespace: comfyui
-  annotations:
-    # No cert-manager annotation and no tls: block. This host is covered by
-    # the *.riotpiao.com wildcard, which nginx serves as its
-    # default-ssl-certificate (ingress-nginx/riotpiao-com-tls).
-    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
-    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
-    nginx.ingress.kubernetes.io/proxy-body-size: "0"
-    # WebSocket support for ComfyUI's live preview
-    nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
-    nginx.ingress.kubernetes.io/upstream-hash-by: "$remote_addr"
-    # Single sign-on. ComfyUI has no auth of its own, so every request is handed to
-    # oauth2-proxy for an allow or deny decision.
-    nginx.ingress.kubernetes.io/auth-url: "http://comfyui-oauth2-proxy.comfyui.svc.cluster.local:4180/oauth2/auth"
-    nginx.ingress.kubernetes.io/auth-signin: "https://comfyui.riotpiao.com/oauth2/start?rd=$scheme://$host$request_uri"
-    nginx.ingress.kubernetes.io/auth-response-headers: "x-auth-request-user,x-auth-request-email"
-    # No configuration-snippet. The controller rejects that annotation as risky
-    # ("annotation group ConfigurationSnippet contains risky annotation"), and a
-    # rejected annotation drops the WHOLE ingress from nginx.conf: the host then
-    # answers 404 with no entry in the access log. ingress-nginx sets the Upgrade
-    # and Connection headers itself for a websocket upgrade.
-spec:
-  ingressClassName: nginx
-  rules:
-    - host: comfyui.riotpiao.com
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: comfyui
-                port:
-                  number: 80

k8s/apps/comfyui/kustomization.yaml

@@ -1,8 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-  - deployment.yaml
-  - service.yaml
-  - oauth2-proxy.yaml
-  - ingress.yaml

k8s/apps/comfyui/oauth2-proxy.yaml

@@ -1,105 +0,0 @@
-# ComfyUI has no authentication of its own: comfyui.riotpiao.com is open to anyone
-# who can resolve the host. It also has no OIDC support, so ZITADEL cannot be a
-# login method inside it. oauth2-proxy is the front door instead: nginx asks it for
-# a decision on every request and it answers from a ZITADEL session.
-#
-# No API client calls this host today, so the whole host is gated. If a client ever
-# needs direct access, add a skip rule for the exact paths it uses rather than
-# opening the whole host.
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: comfyui-oauth2-proxy
-  namespace: comfyui
-  labels:
-    app: comfyui-oauth2-proxy
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: comfyui-oauth2-proxy
-  template:
-    metadata:
-      labels:
-        app: comfyui-oauth2-proxy
-    spec:
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 65534
-        seccompProfile: {type: RuntimeDefault}
-      containers:
-        - name: oauth2-proxy
-          image: quay.io/oauth2-proxy/oauth2-proxy:v7.7.1
-          args:
-            - --provider=oidc
-            - --oidc-issuer-url=https://zitadel.riotpiao.com
-            - --email-domain=*
-            - --upstream=static://202
-            - --reverse-proxy=true
-            - --set-xauthrequest=true
-            - --cookie-secure=true
-            - --cookie-expire=12h
-            - --cookie-refresh=1h
-            - --skip-provider-button=true
-            - --redirect-url=https://comfyui.riotpiao.com/oauth2/callback
-            - --http-address=0.0.0.0:4180
-          env:
-            - name: OAUTH2_PROXY_CLIENT_ID
-              valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: client-id}}
-            - name: OAUTH2_PROXY_CLIENT_SECRET
-              valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: client-secret}}
-            - name: OAUTH2_PROXY_COOKIE_SECRET
-              valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: cookie-secret}}
-          ports:
-            - {name: http, containerPort: 4180}
-          readinessProbe:
-            httpGet: {path: /ping, port: http}
-            initialDelaySeconds: 5
-          livenessProbe:
-            httpGet: {path: /ping, port: http}
-            initialDelaySeconds: 15
-          securityContext:
-            allowPrivilegeEscalation: false
-            capabilities: {drop: ["ALL"]}
-          resources:
-            requests: {cpu: 10m, memory: 32Mi}
-            limits: {cpu: 200m, memory: 128Mi}
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: comfyui-oauth2-proxy
-  namespace: comfyui
-  labels:
-    app: comfyui-oauth2-proxy
-spec:
-  type: ClusterIP
-  selector:
-    app: comfyui-oauth2-proxy
-  ports:
-    - {name: http, port: 4180, targetPort: http}
----
-# The login path must NOT be gated: a request to /oauth2/start that itself demands a
-# login redirects to /oauth2/start forever. ingress-nginx applies the annotations of
-# an ingress to the locations it defines, so a separate object without auth-url keeps
-# this path open. The earlier failure of this shape was not the split — it was the
-# configuration-snippet on the app ingress, which made the controller drop the whole
-# object and left only this one alive.
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: comfyui-oauth2
-  namespace: comfyui
-spec:
-  ingressClassName: nginx
-  rules:
-    - host: comfyui.riotpiao.com
-      http:
-        paths:
-          - path: /oauth2
-            pathType: Prefix
-            backend:
-              service:
-                name: comfyui-oauth2-proxy
-                port:
-                  number: 4180

k8s/apps/comfyui/service.yaml

@@ -1,14 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: comfyui
-  namespace: comfyui
-  labels:
-    app: comfyui
-spec:
-  selector:
-    app: comfyui
-  ports:
-    - port: 80
-      targetPort: 8188
-      protocol: TCP

k8s/apps/gotify/db.yaml

@@ -8,7 +8,11 @@ metadata:
   annotations:
     argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
 spec:
-  instances: 3
+  # 2, not 3. The file's own header always said 2, and the third instance never
+  # existed as a running pod: its volume only ever held a replica that could not
+  # start, because talos-cp-2 disk2 has no free space. A third replica on a disk
+  # that cannot host it is not redundancy, it is a permanent degraded volume.
+  instances: 2
   imageName: ghcr.io/cloudnative-pg/postgresql:16.2
   bootstrap:
     initdb:

k8s/apps/key-rotation-controller/README.md

@@ -40,7 +40,7 @@ target is one root-of-trust file per thing that cannot rotate itself:
 | controller bootstrap key | chicken and egg: it creates every other key |
 | etcd defrag credentials | needed by a job that runs outside the controller |
 
-Everything else — Grafana, Argo CD, Forgejo, MinIO, Paperless, Immich, ComfyUI,
+Everything else — Grafana, Argo CD, Forgejo, MinIO, Paperless, Immich,
 api-gw, Gotify — is a KeyRotation object, and its secret leaves git.
 
 ## Status

k8s/apps/key-rotation-controller/rotations.yaml

@@ -64,32 +64,6 @@ spec:
 ---
 apiVersion: iam.homelab.riotpiao.com/v1
 kind: KeyRotation
-metadata:
-  name: comfyui
-  namespace: iam
-  labels:
-    app.kubernetes.io/part-of: key-rotation
-spec:
-  provider: zitadel-oidc-client
-  strategy: atomic
-  intervalHours: 2160
-  suspend: true
-  target:
-    zitadelProjectID: "396757796898471937"
-    zitadelAppID: "397015743994003854"  # app resource id, not the client id 397015744010781070
-    zitadelClientID: "397015744010781070"
-    secretName: comfyui-oauth2-proxy
-    secretNamespace: comfyui
-    secretKeys:
-      secret: client-secret
-    reload: restart
-    consumers:
-      - {kind: Deployment, name: comfyui-oauth2-proxy, namespace: comfyui}
-  verify:
-    introspect: true
----
-apiVersion: iam.homelab.riotpiao.com/v1
-kind: KeyRotation
 metadata:
   name: minio
   namespace: iam
@@ -107,7 +81,11 @@ spec:
     secretName: minio-oidc
     secretNamespace: storage
     secretKeys:
-      secret: client-secret
+      # The key name here must match the consumer's secretKeyRef, not a house style.
+      # minio-tenant.yaml reads MINIO_IDENTITY_OPENID_CLIENT_SECRET. While this said
+      # `client-secret`, the tenant failed with CreateContainerConfigError on every
+      # restart and the rotation wrote a key nobody read.
+      secret: MINIO_IDENTITY_OPENID_CLIENT_SECRET
     reload: hot
     consumers:
       - {kind: StatefulSet, name: minio-cluster-az-a, namespace: storage}

k8s/apps/llm-serving/qwen-cpu.yaml

@@ -1,5 +1,6 @@
 # qwen2.5:3b-instruct on CPU (talos-cp-2, 144GB RAM, 24 cores).
-# Moved off GPU to free a V100 for ComfyUI. Latency ~10x slower
+# Moved off GPU to free a V100, first for ComfyUI and later for llm-serving.
+# Latency ~10x slower
 # than GPU but sufficient for lightweight tasks (summarization,
 # classification, quick answers).
 apiVersion: apps/v1

k8s/apps/poimen-frontend/api-ingress.yaml

@@ -0,0 +1,38 @@
+---
+# Public entry point for the poimen-memory HTTP API.
+#
+# The API was reachable only inside the cluster (ClusterIP :8080), so a local pi
+# session had no way to use it. This Ingress exposes the same handler set the
+# in-cluster clients use: /mcp, /memory/*, /graphql, /health, /ready.
+#
+# Cloudflare terminates TLS in front of the nginx ingress, which is why this rule
+# carries no `tls:` section — the same shape as ingress.yaml next to it.
+#
+# Auth: poimen-memory validates ZITADEL JWTs (MEM_AUTH_MODE=jwt). A caller
+# without a valid bearer token gets 401, so exposing the route does not expose
+# the data.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: poimen-api
+  namespace: poimen
+  annotations:
+    # MCP calls and ingest can be slow; the provider may stream. Match the
+    # frontend ingress so a long tool call is not cut at 60s.
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-buffering: "off"
+    nginx.ingress.kubernetes.io/proxy-body-size: "50m"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: api-poimen.riotpiao.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: poimen-memory
+                port:
+                  number: 8080

k8s/apps/poimen-frontend/kustomization.yaml

@@ -6,3 +6,5 @@ kind: Kustomization
 # riotpiao-poimen/poimen-frontend.git k8s/ — each repo owns its own k8s.
 resources:
   - ingress.yaml
+  # The memory API. Without it a local pi session has no route to :8080.
+  - api-ingress.yaml

k8s/argocd/apps/00-substrate.yaml

@@ -159,54 +159,3 @@ spec:
     automated:
       prune: true
       selfHeal: true
----
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: kyverno
-  namespace: argocd
-  annotations:
-    argocd.argoproj.io/sync-wave: "0"
-spec:
-  project: homelab
-  source:
-    repoURL: https://kyverno.github.io/kyverno/
-    chart: kyverno
-    targetRevision: "1.14.0"
-    helm:
-      valueFiles:
-        - $values/k8s/bootstrap/kyverno/kyverno-values.yaml
-  sources:
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
-      targetRevision: main
-      ref: values
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: kyverno
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
-    syncOptions:
-      - CreateNamespace=true
----
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: kyverno-policies
-  namespace: argocd
-  annotations:
-    argocd.argoproj.io/sync-wave: "0"
-spec:
-  project: homelab
-  source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
-    targetRevision: main
-    path: k8s/bootstrap/kyverno
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: kyverno
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true

k8s/argocd/apps/58-comfyui.yaml

@@ -1,32 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: comfyui
-  namespace: argocd
-  labels:
-    app.kubernetes.io/name: comfyui
-    app.kubernetes.io/component: image-generation
-  annotations:
-    argocd.argoproj.io/sync-wave: "8"
-spec:
-  project: homelab
-  revisionHistoryLimit: 3
-  source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
-    targetRevision: main
-    path: k8s/apps/comfyui
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: comfyui
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
-    syncOptions:
-      - CreateNamespace=true
-    retry:
-      limit: 5
-      backoff:
-        duration: 5s
-        factor: 2
-        maxDuration: 3m

k8s/bootstrap/kyverno/kustomization.yaml

@@ -1,6 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-namespace: kyverno
-
-resources:
-  - policies.yaml

k8s/bootstrap/kyverno/kyverno-values.yaml

@@ -1,42 +0,0 @@
-# Kyverno: Policy engine for Kubernetes image scanning, Pod security, and admission control
-# Scan all images, enforce baseline Pod Security Standard, prevent privilege escalation
-
-replicaCount: 1
-
-image:
-  registry: ghcr.io
-  repository: kyverno/kyverno
-  tag: "v1.14.0"
-
-config:
-  # Webhook timeout for policy evaluation. Increase if scanning takes longer.
-  webhookTimeoutSeconds: 30
-  # Failure policy: fail-open (audit/log) vs fail-closed (reject on error)
-  failurePolicy: fail
-  # Resource limits for webhook
-  webhookAnnotations:
-    rules: "allow"
-
-# Pod security via Kyverno instead of Pod Security Policies (deprecated)
-# Enforces baseline restrictions cluster-wide, with exceptions for privileged namespaces
-podSecurityContext:
-  runAsNonRoot: true
-  runAsUser: 1000
-
-rbac:
-  create: true
-
-resources:
-  requests:
-    memory: "256Mi"
-    cpu: "100m"
-  limits:
-    memory: "512Mi"
-    cpu: "500m"
-
-# Webhook configuration
-webhook:
-  timeoutSeconds: 30
-  # Failure policy: "Fail" (reject on error) or "Ignore" (audit-only)
-  # Set to "Ignore" for initial testing, then change to "Fail"
-  failurePolicy: ignore

k8s/bootstrap/kyverno/policies.yaml

@@ -1,204 +0,0 @@
-# Kyverno ClusterPolicies: Image scanning, Pod security, and admission control
----
-# Policy 1: Require non-root containers
-apiVersion: kyverno.io/v1
-kind: ClusterPolicy
-metadata:
-  name: require-non-root
-  namespace: kyverno
-spec:
-  validationFailureAction: audit  # audit first, then change to enforce
-  rules:
-    - name: check-runAsNonRoot
-      match:
-        any:
-          - resources:
-              kinds:
-                - Pod
-              selector:
-                matchLabels:
-                  pod-security.kubernetes.io/enforce: "!privileged"
-      validate:
-        message: "Container must not run as root"
-        pattern:
-          spec:
-            containers:
-              - securityContext:
-                  runAsNonRoot: true
----
-# Policy 2: Drop all Linux capabilities, add only required ones
-apiVersion: kyverno.io/v1
-kind: ClusterPolicy
-metadata:
-  name: require-dropped-caps
-  namespace: kyverno
-spec:
-  validationFailureAction: audit
-  rules:
-    - name: drop-all-capabilities
-      match:
-        any:
-          - resources:
-              kinds:
-                - Pod
-              selector:
-                matchLabels:
-                  pod-security.kubernetes.io/enforce: "!privileged"
-      validate:
-        message: "All Linux capabilities must be dropped"
-        pattern:
-          spec:
-            containers:
-              - securityContext:
-                  capabilities:
-                    drop:
-                      - ALL
----
-# Policy 3: Require image tags (no 'latest')
-apiVersion: kyverno.io/v1
-kind: ClusterPolicy
-metadata:
-  name: disallow-latest-tag
-  namespace: kyverno
-spec:
-  validationFailureAction: audit  # Change to enforce after testing
-  rules:
-    - name: disallow-latest
-      match:
-        any:
-          - resources:
-              kinds:
-                - Pod
-                - Deployment
-                - StatefulSet
-                - DaemonSet
-                - Job
-      validate:
-        message: "Image tag 'latest' is not allowed. Use explicit version tags."
-        pattern:
-          spec:
-            =(template):
-              spec:
-                containers:
-                  - image: "!*:latest"
-                =(initContainers):
-                  - image: "!*:latest"
----
-# Policy 4: Restrict images to trusted registries
-apiVersion: kyverno.io/v1
-kind: ClusterPolicy
-metadata:
-  name: restrict-registries
-  namespace: kyverno
-spec:
-  validationFailureAction: audit
-  rules:
-    - name: trusted-registries
-      match:
-        any:
-          - resources:
-              kinds:
-                - Pod
-                - Deployment
-                - StatefulSet
-                - DaemonSet
-                - Job
-              selector:
-                matchLabels:
-                  pod-security.kubernetes.io/enforce: "!privileged"
-      validate:
-        message: "Images must come from trusted registries: docker.io, ghcr.io, quay.io, k8s.gcr.io, registry.k8s.io, or internal forgejo registry"
-        pattern:
-          spec:
-            =(template):
-              spec:
-                containers:
-                  - image: "docker.io/* | ghcr.io/* | quay.io/* | k8s.gcr.io/* | registry.k8s.io/* | forgejo.riotpiao.com/* | *"
----
-# Policy 5: Require read-only root filesystem (audit only, exceptions for apps that need writes)
-apiVersion: kyverno.io/v1
-kind: ClusterPolicy
-metadata:
-  name: require-readonly-filesystem
-  namespace: kyverno
-spec:
-  validationFailureAction: audit
-  rules:
-    - name: check-readOnlyRootFilesystem
-      match:
-        any:
-          - resources:
-              kinds:
-                - Pod
-              selector:
-                matchLabels:
-                  pod-security.kubernetes.io/enforce: "!privileged"
-      validate:
-        message: "Root filesystem should be read-only for defense-in-depth"
-        pattern:
-          spec:
-            containers:
-              - securityContext:
-                  readOnlyRootFilesystem: true
----
-# Policy 6: Require resource requests and limits (prevent resource starvation)
-apiVersion: kyverno.io/v1
-kind: ClusterPolicy
-metadata:
-  name: require-resource-limits
-  namespace: kyverno
-spec:
-  validationFailureAction: audit
-  rules:
-    - name: check-resources
-      match:
-        any:
-          - resources:
-              kinds:
-                - Pod
-                - Deployment
-                - StatefulSet
-                - DaemonSet
-              excludeResources:
-                namespaceSelector:
-                  matchLabels:
-                    kubernetes.io/metadata.name: "kyverno|kube-system|kube-node-lease"
-      validate:
-        message: "CPU and memory requests and limits are required"
-        pattern:
-          spec:
-            =(template):
-              spec:
-                containers:
-                  - resources:
-                      requests:
-                        memory: "?*"
-                        cpu: "?*"
-                      limits:
-                        memory: "?*"
-                        cpu: "?*"
----
-# Policy 7: Require securityContext on all containers
-apiVersion: kyverno.io/v1
-kind: ClusterPolicy
-metadata:
-  name: require-security-context
-  namespace: kyverno
-spec:
-  validationFailureAction: audit
-  rules:
-    - name: check-securityContext
-      match:
-        any:
-          - resources:
-              kinds:
-                - Pod
-              selector:
-                matchLabels:
-                  pod-security.kubernetes.io/enforce: "!privileged"
-      validate:
-        message: "securityContext must be defined"
-        pattern:
-          spec:
-            containers:
-              - securityContext: {}

k8s/bootstrap/phase3-forgejo/forgejo-values.yaml

@@ -97,10 +97,25 @@ gitea:
       # USER and PASSWD injected via env vars below (GITEA__MAILER__USER, GITEA__MAILER__PASSWD)
 
 # Persistence (shared storage for repos)
+#
+# 60Gi, not the original 20Gi. The container registry stores its blobs here,
+# and a single candle-vllm sm_70 image is 14GB uncompressed. At 20Gi the volume
+# filled, and every blob upload failed with:
+#   Package registry API internal error: 500 write /data/tmp/package-upload/...:
+#   no space left on device
+# The push client surfaces that as a bare `received unexpected HTTP status: 500`
+# after retrying every layer, which looks like a network fault and is not one.
+#
+# Note for whoever expands this again: Longhorn's admission webhook checks the
+# PHYSICAL free space of the disk holding each replica, not the thin volume
+# size. A request for 100Gi was rejected while the disk had 23.5Gi free:
+#   disk ... does not have sufficient physical space for expansion:
+#   physical free space would drop below minimal: left=14596177920 < minimal=49931236556
+# 60Gi fit. Raise it in steps and watch the webhook message.
 persistence:
   enabled: true
   storageClass: longhorn
-  size: 20Gi
+  size: 60Gi
   accessModes:
     - ReadWriteOnce
 
@@ -134,10 +149,29 @@ ingress:
   # wildcard. The wildcard covers this host.
 
 # Resources
+#
+# The 1000m ceiling throttled Gitea badly. Measured on a running pod:
+#   nr_throttled   189528 of 1169014 CFS periods  (16.2%)
+#   throttled_usec 4424898602 us                   (~74 minutes lost)
+#   actual usage   951m against the 1000m limit
+# The visible symptom was slow Actions traffic: FetchTask took 3469.9ms and
+# repository compare endpoints took 3091-3811ms, so runners timed out and
+# retried, which added more load to an already saturated process.
+#
+# The database was never the bottleneck. A real action_runner UPDATE measured
+# 2.133ms with zero lock waits, so this is CPU starvation in Gitea itself.
+#
+# requests.cpu stays 200m and limits.cpu stays 1000m. The 3000m limit was a
+# temporary measure while the Actions stall was diagnosed; the real fix landed
+# elsewhere, so the limit is back to the original.
+#
+# Caution: this pod runs on talos-cp-3, a control-plane node. If Gitea
+# throttles again at 1000m, move the workload off the control plane rather
+# than raising the limit.
 resources:
   requests:
     cpu: 200m
-    memory: 512Mi
+    memory: 768Mi
   limits:
     cpu: 1000m
     memory: 2Gi

k8s/infra/longhorn/disk-scan-job.yaml

@@ -0,0 +1,90 @@
+# One-shot destructive health scan of a candidate Longhorn disk.
+#
+# WHY THIS IS A MANUAL JOB AND NOT AN ARGOCD HOOK
+#   It writes over the entire raw device. An ArgoCD PostSync hook would run it
+#   on every sync and destroy whatever the volume holds. Apply it by hand,
+#   once, on a disk that has no partitions and no mounts.
+#
+# WHY NOT TALOS
+#   Talos declares disks by mounting them (machine.disks in the machine
+#   config). It has no primitive for "run this command on the host", so a scan
+#   cannot live in the Talos config. This Job is the IaC-correct home.
+#
+# ORDER OF OPERATIONS
+#   1. Scan the raw device with this Job        <- the disk must be UNMOUNTED
+#   2. If it passes, add the disk to terraform.tfvars.local `longhorn_disks`
+#      with device /dev/disk/by-id/<wwid> and run `make apply-cp`
+#   3. Register it with Longhorn via the pattern in longhorn-add-disks-job.yaml
+#
+# What it does
+#   fio writes a CRC32C-verified pattern across the whole device, then reads
+#   every block back and compares. Any unreadable or mismatched sector fails
+#   the Job. This is a full write and read pass, not a sample.
+#
+# Expected duration
+#   2 TB on a 5400 rpm 2.5" drive at roughly 90 MB/s is about 6 h to write and
+#   6 h to verify. Budget 12 hours. Check progress with:
+#     kubectl -n llm-io logs job/disk-scan-cp1-sdb -f
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: disk-scan-cp1-sdb
+  namespace: llm-io
+  labels:
+    app: disk-scan
+spec:
+  backoffLimit: 0
+  # Do not let a stuck scan retry silently.
+  template:
+    metadata:
+      labels:
+        app: disk-scan
+    spec:
+      restartPolicy: Never
+      nodeSelector:
+        kubernetes.io/hostname: talos-cp-1
+      containers:
+        - name: scan
+          image: ubuntu:22.04
+          securityContext:
+            privileged: true
+          command:
+            - /bin/bash
+            - -c
+            - |
+              set -euo pipefail
+              apt-get update -qq
+              apt-get install -y -qq smartmontools fio
+
+              echo "=== device identity ==="
+              smartctl -i /dev/sdb | tail -12
+
+              echo "=== SMART before ==="
+              smartctl -H /dev/sdb | tail -2
+
+              echo "=== full write + verify (destructive) ==="
+              fio --name=devverify \
+                  --filename=/dev/sdb \
+                  --rw=write \
+                  --bs=1M \
+                  --iodepth=4 \
+                  --ioengine=libaio \
+                  --direct=1 \
+                  --verify=crc32c \
+                  --do_verify=1 \
+                  --verify_fatal=1 \
+                  --eta=always \
+                  --output-format=normal
+
+              echo "=== SMART after ==="
+              smartctl -H /dev/sdb | tail -2
+              smartctl -A /dev/sdb | awk '/Reallocated_Sector|Current_Pending|Offline_Uncorrect|UDMA_CRC/{print "  "$0}'
+
+              echo "=== SCAN PASSED ==="
+          volumeMounts:
+            - name: dev
+              mountPath: /dev
+      volumes:
+        - name: dev
+          hostPath:
+            path: /dev

scripts/iam/rotate-oidc.py

@@ -32,7 +32,6 @@ ISSUER = "https://zitadel.riotpiao.com"
 APPS = [
     ("grafana",   "grafana",   "grafana-oidc",         "logging",   "GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET", "deploy", "grafana"),
     ("argocd",    "argocd",    "oidc-secret",           "argocd",    "client-secret",                       "deploy", "argocd-server"),
-    ("comfyui",   "comfyui",   "comfyui-oauth2-proxy", "comfyui",   "client-secret",                       "deploy", "comfyui-oauth2-proxy"),
     ("minio",     "minio",     "minio-oidc",           "storage",   "MINIO_IDENTITY_OPENID_CLIENT_SECRET",  "sts",    "minio-cluster-az-a"),
     ("paperless", "paperless", "paperless-oidc",       "paperless", "client-secret",                       "deploy", "paperless"),
     ("immich",    "immich",    "immich-oidc",          "immich",    "client-secret",                       "deploy", "immich-server"),

scripts/iam/zitadel-provision.py

@@ -128,11 +128,6 @@ OIDC_APPS = {
         "https://forgejo.riotpiao.com/user/oauth2/zitadel/callback",
         "http://localhost:3000/user/oauth2/authentik/callback",
         "https://forgejo.riotpiao.com/user/oauth2/authentik/callback"]},
-    # ComfyUI has no native OIDC. oauth2-proxy is its front door and this app is
-    # the proxy's client, so the callback path belongs to the proxy.
-    "comfyui": {**_WEB, "redirect_uris": [
-        "http://localhost:4180/oauth2/callback",
-        "https://comfyui.riotpiao.com/oauth2/callback"]},
     "grafana": {**_WEB, "redirect_uris": [
         "http://localhost:3000/login/generic_oauth",
         "https://grafana.riotpiao.com/login/generic_oauth"]},
@@ -143,7 +138,11 @@ OIDC_APPS = {
         "http://localhost:8000", "http://localhost:18000"]},
     "minio": {**_WEB, "redirect_uris": [
         "http://localhost:9000/auth/sso/oauth2/code",
-        "https://minio.riotpiao.com/auth/sso/oauth2/code"]},
+        "https://minio.riotpiao.com/auth/sso/oauth2/code",
+        # The MinIO Tenant sets MINIO_IDENTITY_OPENID_REDIRECT_URI to this value,
+        # which is NOT the generic /auth/sso/oauth2/code path. Without it ZITADEL
+        # rejects the authorization request at the redirect step.
+        "https://minio.riotpiao.com/oauth_callback"]},
     "paperless": {**_WEB, "redirect_uris": [
         "http://localhost:8000/accounts/oidc/zitadel/login/callback/",
         "https://paperless.riotpiao.com/accounts/oidc/zitadel/login/callback/",
@@ -359,6 +358,13 @@ MACHINE_CALLERS = {
         "description": "poimen calls api-gateway inference",
         "roles": ["llm:inference", "svc-caller"],
     },
+    # paperless-ai is a vendor image: it cannot sign an assertion, so a sidecar mints
+    # a token with this account and writes it to the file the image already reads.
+    "paperless-llm": {
+        "name": "Paperless AI LLM caller",
+        "description": "paperless-ai calls api-gateway inference through the token vendor",
+        "roles": ["llm:inference", "svc-caller"],
+    },
 }
 
 

terraform/files/coredns/Corefile

@@ -36,8 +36,10 @@
     rewrite name paperless.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name img.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name api.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
-    rewrite name comfy.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
-    rewrite name comfyui.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    # poimen ran only in the live ConfigMap, never in this file or the Talos
+    # machine config. The next Talos reconcile would have deleted it and broken
+    # in-cluster resolution. Declared here so the source of truth carries it.
+    rewrite name poimen.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name gotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name pi.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name igotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local