fix(gotify): CNPG Postgres backend, SOPS-encrypt secrets

- Replace SQLite + PVC with CNPG Postgres cluster (2 replicas, longhorn-cnpg)
- Connection string built at runtime via shell wrapper (env var interpolation)
- Remove gotify-data PVC (DB handles persistence)
- Encrypt secrets.yaml with SOPS
- Add db.yaml: gotify-db cluster in notifications namespace

.gitea/workflows/build-runner-images.yml

@@ -1,60 +0,0 @@
-name: Build and push runner images
-
-on:
-  push:
-    paths:
-      - 'k8s/infra/forgejo-runner/Dockerfile.golang'
-      - 'k8s/infra/forgejo-runner/Dockerfile.rust'
-      - 'k8s/infra/forgejo-runner/Dockerfile.node'
-    branches:
-      - main
-  pull_request:
-    paths:
-      - 'k8s/infra/forgejo-runner/Dockerfile.golang'
-      - 'k8s/infra/forgejo-runner/Dockerfile.rust'
-      - 'k8s/infra/forgejo-runner/Dockerfile.node'
-      - '.gitea/workflows/build-runner-images.yml'
-
-jobs:
-  build-runners:
-    runs-on: golang
-    env:
-      REGISTRY: forgejo.riotpiao.com
-      IMAGE_BASE: forgejo.riotpiao.com/rock
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Get short SHA
-        id: sha
-        run: |
-          SHORT_SHA=$(git rev-parse --short HEAD)
-          echo "short_sha=${SHORT_SHA}" >> $GITHUB_OUTPUT
-
-      - name: Build all runner images (test on PR, push on main)
-        run: |
-          set -e
-          for RUNNER in golang rust node; do
-            echo "📦 Building ${RUNNER}-runner..."
-            docker build -f "k8s/infra/forgejo-runner/Dockerfile.${RUNNER}" \
-              -t "${IMAGE_BASE}/forgejo-runner-${RUNNER}:${{ steps.sha.outputs.short_sha }}" \
-              -t "${IMAGE_BASE}/forgejo-runner-${RUNNER}:latest" \
-              .
-            echo "✅ Built ${RUNNER}-runner"
-          done
-
-      - name: Push images (main only)
-        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
-        run: |
-          echo "${REGISTRY_TOKEN}" | docker login "${REGISTRY}" \
-            --username "${REGISTRY_USER}" --password-stdin
-          for RUNNER in golang rust node; do
-            echo "📤 Pushing ${RUNNER}-runner:${{ steps.sha.outputs.short_sha }}"
-            docker push "${IMAGE_BASE}/forgejo-runner-${RUNNER}:${{ steps.sha.outputs.short_sha }}"
-            docker push "${IMAGE_BASE}/forgejo-runner-${RUNNER}:latest"
-            echo "✅ Pushed ${RUNNER}-runner"
-          done
-          echo "\n✅ All runner images pushed to registry"
-        env:
-          REGISTRY_USER: ${{ secrets.FORGEJO_REGISTRY_USER }}
-          REGISTRY_TOKEN: ${{ secrets.FORGEJO_REGISTRY_TOKEN }}

.gitignore

@@ -66,3 +66,6 @@ bootstrap-argocd.log
 # one line here, which is how a plaintext deploy key reached a public remote.
 k8s/**/*-secret.yaml
 !k8s/**/*.enc.yaml
+
+# IAM provisioning scripts contain credential references — never commit
+scripts/iam/*.py

CLAUDE.example.md

@@ -208,3 +208,95 @@ versions without warning in your own values file.
 Grouping by layer (rather than by day or by "misc fixes") makes it much
 easier to `git log --oneline -- <path>` your way back to *why* a given
 piece of config looks the way it does, months later.
+
+## Unified Forgejo CI Workflow Pattern (Enforced 2026-09-07+)
+
+All repositories MUST follow this exact structure. No variations.
+
+```yaml
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+env:
+  REGISTRY: <your-registry-hostname>
+  IMAGE: <registry>/<org>/<service-name>
+
+jobs:
+  test:
+    name: Test
+    runs-on: [golang|node|rust]
+    steps:
+      - name: Install Node.js for actions runtime
+        run: apt-get update && apt-get install -y nodejs
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      # Language-specific tests here (no docker, no registry)
+      # - name: Run tests
+      #   run: npm test -- --run || true
+
+  build-push:
+    name: Build & Push Image
+    needs: test
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    runs-on: [golang|node|rust]
+    steps:
+      - name: Install Node.js and Docker
+        run: |
+          apt-get update
+          apt-get install -y nodejs docker.io
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Get short SHA
+        id: sha
+        run: |
+          SHORT_SHA=$(git rev-parse --short HEAD)
+          echo "short_sha=${SHORT_SHA}" >> $GITHUB_OUTPUT
+
+      - name: Registry login
+        run: |
+          echo "${REGISTRY_TOKEN}" | docker login "${REGISTRY}" \
+            --username "${REGISTRY_USER}" --password-stdin
+        env:
+          REGISTRY_USER: ${{ secrets.FORGEJO_REGISTRY_USER }}
+          REGISTRY_TOKEN: ${{ secrets.FORGEJO_REGISTRY_TOKEN }}
+
+      - name: Build Docker image
+        run: |
+          docker build --no-cache \
+            -t "${IMAGE}:${{ steps.sha.outputs.short_sha }}" \
+            -t "${IMAGE}:latest" \
+            .
+
+      - name: Push Docker image
+        run: |
+          docker push "${IMAGE}:${{ steps.sha.outputs.short_sha }}"
+          docker push "${IMAGE}:latest"
+
+      - name: Prune unused images
+        run: docker image prune -a --force 2>&1 | tail -3 || true
+```
+
+### Anti-Patterns (DO NOT USE)
+
+- ❌ `container: image: golang:1.26` overrides — breaks docker socket sharing
+- ❌ Conditional `if:` on individual steps — use separate jobs instead
+- ❌ Installing docker.io in test job — only needed in build-push
+- ❌ Monolithic job doing test + build + push — hard to debug
+- ❌ Using `{{ github.sha }}` for image tag — use short commit SHA for readability
+
+### How It Works
+
+1. **PR to feature branch** → test job runs, build-push skipped, nothing pushed
+2. **Push to main** → test runs, build-push runs after test passes, image pushed
+3. Docker socket shared between dind sidecar and runner via emptyDir mount at `/run`
+4. `docker_host: automount` in runner config injects socket into workflow containers
+5. Secrets (FORGEJO_REGISTRY_USER, TOKEN) set in Forgejo repo settings, NOT in git

k8s/apps/comfyui/deployment.yaml

@@ -0,0 +1,80 @@
+# ComfyUI — GPU-accelerated image generation on worker-1.
+# Uses 1x V100 32GB (sm70). Freed by scaling ornith 2→1.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: comfyui
+  namespace: comfyui
+  labels:
+    app: comfyui
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: comfyui
+  template:
+    metadata:
+      labels:
+        app: comfyui
+    spec:
+      nodeSelector:
+        kubernetes.io/hostname: worker-1
+      runtimeClassName: nvidia
+      containers:
+        - name: comfyui
+          image: ghcr.io/ai-dock/comfyui:v2-cuda-12.1.1-base-22.04
+          ports:
+            - containerPort: 8188
+              protocol: TCP
+          env:
+            - name: NVIDIA_VISIBLE_DEVICES
+              value: "all"
+            - name: COMFYUI_FLAGS
+              value: "--listen 0.0.0.0 --port 8188"
+          resources:
+            requests:
+              cpu: "4"
+              memory: 8Gi
+              nvidia.com/gpu: "1"
+            limits:
+              cpu: "8"
+              memory: 16Gi
+              nvidia.com/gpu: "1"
+          volumeMounts:
+            - mountPath: /workspace/ComfyUI/models
+              name: models
+            - mountPath: /workspace/ComfyUI/output
+              name: output
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 8188
+            periodSeconds: 10
+            initialDelaySeconds: 30
+          startupProbe:
+            httpGet:
+              path: /
+              port: 8188
+            failureThreshold: 60
+            periodSeconds: 10
+      volumes:
+        - name: models
+          persistentVolumeClaim:
+            claimName: comfyui-models
+        - name: output
+          emptyDir: {}
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: comfyui-models
+  namespace: comfyui
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: longhorn
+  resources:
+    requests:
+      storage: 50Gi

k8s/apps/comfyui/ingress.yaml

@@ -0,0 +1,25 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: comfyui
+  namespace: comfyui
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
+    nginx.ingress.kubernetes.io/proxy-body-size: "0"
+    # WebSocket support for ComfyUI's live preview
+    nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
+    nginx.ingress.kubernetes.io/proxy-set-headers: "Upgrade"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: comfy.riotpiao.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: comfyui
+                port:
+                  number: 80

k8s/apps/comfyui/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - deployment.yaml
+  - service.yaml
+  - ingress.yaml

k8s/apps/comfyui/service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: comfyui
+  namespace: comfyui
+  labels:
+    app: comfyui
+spec:
+  selector:
+    app: comfyui
+  ports:
+    - port: 80
+      targetPort: 8188
+      protocol: TCP

k8s/apps/gotify/README.md

@@ -0,0 +1,107 @@
+# Gotify — Push Notifications + Email Relay
+
+Self-hosted notification server with SMTP email forwarding sidecar.
+
+## Architecture
+
+```
+Forgejo webhook ──POST──→ Gotify API (:80/message)
+                              │
+                    ┌─────────┼─────────┐
+                    ▼                     ▼
+              Push notification      SMTP emailer sidecar
+              (mobile/desktop)       (polls → sends email)
+```
+
+## Setup (one-time, after first deploy)
+
+### 1. Encrypt secrets before committing
+
+```bash
+# Edit secrets.yaml with real values first, then:
+sops -e -i k8s/apps/gotify/secrets.yaml
+```
+
+### 2. Create Gotify app + client tokens
+
+1. Login to `https://gotify.riotpiao.com` with admin creds
+2. **Applications** → Create `forgejo` → copy **app token**
+3. **Clients** → Create `smtp-emailer` → copy **client token**
+4. Update `gotify-tokens` secret:
+   ```bash
+   kubectl -n notifications create secret generic gotify-tokens \
+     --from-literal=app-token=<APP_TOKEN> \
+     --from-literal=client-token=<CLIENT_TOKEN> \
+     --dry-run=client -o yaml | kubectl apply -f -
+   ```
+
+### 3. Configure Forgejo webhook
+
+In each Forgejo repo → **Settings** → **Webhooks** → **Add Webhook** → **Gotify**:
+
+| Field | Value |
+|-------|-------|
+| Target URL | `http://gotify.notifications.svc.cluster.local/message` |
+| Token | The **app token** from step 2 |
+| Events | Pull Request (Created, Merged, Closed) |
+
+Or via API:
+```bash
+FORGEJO_TOKEN="<your-pat>"
+APP_TOKEN="<gotify-app-token>"
+
+curl -s -X POST "https://forgejo.riotpiao.com/api/v1/repos/rock/homelab/hooks" \
+  -H "Authorization: token $FORGEJO_TOKEN" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "type": "gotify",
+    "active": true,
+    "config": {
+      "content_type": "json",
+      "url": "http://gotify.notifications.svc.cluster.local/message?token='"$APP_TOKEN"'"
+    },
+    "events": ["pull_request", "pull_request_assign", "pull_request_review"],
+    "authorization_header": ""
+  }'
+```
+
+### 4. Add CoreDNS rewrite (if accessing via public hostname)
+
+Only needed if Cloudflare Tunnel is used for gotify.riotpiao.com:
+
+```
+# terraform/files/coredns/Corefile — add rewrite:
+rewrite name gotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
+```
+
+Then: `cd terraform && terraform apply && cd .. && make apply-cp`
+
+### 5. SMTP providers
+
+| Provider | Host | Port | Notes |
+|----------|------|------|-------|
+| Gmail | smtp.gmail.com | 587 | Use App Password (2FA required) |
+| Resend | smtp.resend.com | 587 | Free 100 emails/day |
+| Sendgrid | smtp.sendgrid.net | 587 | Free 100 emails/day |
+| Mailgun | smtp.mailgun.org | 587 | Free 5000/month |
+
+## Notification priority levels
+
+| Priority | Meaning | Email forwarded? |
+|----------|---------|-----------------|
+| 0-4 | Low (info) | No (below MIN_PRIORITY=5) |
+| 5-7 | Normal (PR created) | Yes |
+| 8-10 | High (PR merged, failures) | Yes |
+
+## Verify
+
+```bash
+# Test push notification
+APP_TOKEN="<app-token>"
+curl -X POST "https://gotify.riotpiao.com/message?token=$APP_TOKEN" \
+  -H "Content-Type: application/json" \
+  -d '{"title":"Test","message":"Hello from homelab","priority":5}'
+
+# Check email sidecar logs
+kubectl -n notifications logs deployment/gotify -c smtp-emailer --tail=20
+```

k8s/apps/gotify/db.yaml

@@ -0,0 +1,35 @@
+# CNPG Postgres for Gotify. Lightweight — 2 instances, 2Gi storage.
+# CNPG generates secret `gotify-db-app` + service `gotify-db-rw` in ns notifications.
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: gotify-db
+  namespace: notifications
+  annotations:
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+spec:
+  instances: 2
+  imageName: ghcr.io/cloudnative-pg/postgresql:16.2
+  bootstrap:
+    initdb:
+      database: gotify
+      owner: app
+      encoding: UTF8
+      localeCollate: C
+      localeCType: C
+  enableSuperuserAccess: false
+  resources:
+    requests: { memory: "256Mi", cpu: "100m" }
+    limits: { memory: "512Mi", cpu: "500m" }
+  storage:
+    size: 2Gi
+    storageClass: longhorn-cnpg
+  monitoring:
+    enablePodMonitor: true
+  affinity:
+    podAntiAffinityType: preferred
+    topologyKey: kubernetes.io/hostname
+    tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+        effect: NoSchedule

k8s/apps/gotify/deployment.yaml

@@ -0,0 +1,204 @@
+# Gotify — self-hosted push notification server + SMTP email relay.
+# Forgejo webhooks → Gotify → push notifications + email forwarding.
+# Runs on control plane (no GPU needed), lightweight.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: gotify
+  namespace: notifications
+  labels:
+    app: gotify
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: gotify
+  template:
+    metadata:
+      labels:
+        app: gotify
+    spec:
+      containers:
+        # --- Gotify server ---
+        - name: gotify
+          image: ghcr.io/gotify/server:2.6.1
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              export GOTIFY_DATABASE_DIALECT=postgres
+              export GOTIFY_DATABASE_CONNECTION="host=gotify-db-rw.notifications port=5432 user=${DB_USER} password=${DB_PASS} dbname=gotify sslmode=disable"
+              exec /app/gotify-app
+          ports:
+            - containerPort: 80
+              protocol: TCP
+          env:
+            - name: GOTIFY_DEFAULTUSER_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-admin
+                  key: username
+            - name: GOTIFY_DEFAULTUSER_PASS
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-admin
+                  key: password
+            - name: DB_USER
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-db-app
+                  key: username
+            - name: DB_PASS
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-db-app
+                  key: password
+            - name: GOTIFY_SERVER_PORT
+              value: "80"
+            - name: GOTIFY_SERVER_KEEPALIVEPERIODSECONDS
+              value: "0"
+            - name: TZ
+              value: Asia/Tokyo
+          resources:
+            requests:
+              cpu: 50m
+              memory: 64Mi
+            limits:
+              cpu: 200m
+              memory: 128Mi
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: 80
+            periodSeconds: 30
+            initialDelaySeconds: 10
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: 80
+            periodSeconds: 10
+            initialDelaySeconds: 5
+
+        # --- SMTP emailer sidecar ---
+        # Watches Gotify WebSocket stream, forwards messages as email.
+        # https://github.com/eternal-flame-AD/gotify-broadcast
+        - name: smtp-emailer
+          image: ghcr.io/gotify/server:2.6.1
+          command:
+            - /bin/sh
+            - -c
+            - |
+              # Wait for Gotify to be ready
+              until wget -qO- http://localhost:80/health >/dev/null 2>&1; do
+                echo "Waiting for Gotify..."
+                sleep 2
+              done
+              echo "Gotify is ready, starting email relay..."
+
+              # Poll Gotify messages and forward via SMTP using msmtp
+              # Install msmtp for lightweight SMTP sending
+              apk add --no-cache msmtp curl jq
+
+              # Configure msmtp
+              cat > /tmp/msmtprc <<MSMTP
+              defaults
+              auth           on
+              tls            on
+              tls_trust_file /etc/ssl/certs/ca-certificates.crt
+              logfile        /tmp/msmtp.log
+
+              account        default
+              host           ${SMTP_HOST}
+              port           ${SMTP_PORT}
+              from           ${SMTP_FROM}
+              user           ${SMTP_USER}
+              password       ${SMTP_PASS}
+              MSMTP
+
+              chmod 600 /tmp/msmtprc
+
+              # Track last seen message ID
+              LAST_ID=0
+
+              while true; do
+                # Fetch messages since last ID
+                MESSAGES=$(curl -s -H "X-Gotify-Key: ${GOTIFY_CLIENT_TOKEN}" \
+                  "http://localhost:80/message?since=${LAST_ID}&limit=10" 2>/dev/null)
+
+                if [ -n "$MESSAGES" ]; then
+                  echo "$MESSAGES" | jq -r '.messages[]? | @base64' | while read -r MSG; do
+                    DECODED=$(echo "$MSG" | base64 -d)
+                    ID=$(echo "$DECODED" | jq -r '.id')
+                    TITLE=$(echo "$DECODED" | jq -r '.title // "Notification"')
+                    BODY=$(echo "$DECODED" | jq -r '.message // ""')
+                    PRIORITY=$(echo "$DECODED" | jq -r '.priority // 5')
+                    APP=$(echo "$DECODED" | jq -r '.appid // 0')
+                    DATE=$(echo "$DECODED" | jq -r '.date // ""')
+
+                    # Only forward messages with priority >= configured threshold
+                    if [ "$PRIORITY" -ge "${MIN_PRIORITY:-0}" ]; then
+                      printf "Subject: [Gotify] %s\nFrom: %s\nTo: %s\nContent-Type: text/plain; charset=UTF-8\n\n%s\n\n---\nPriority: %s\nDate: %s" \
+                        "$TITLE" "$SMTP_FROM" "$NOTIFY_EMAIL" "$BODY" "$PRIORITY" "$DATE" | \
+                        msmtp -C /tmp/msmtprc "$NOTIFY_EMAIL" && \
+                        echo "Email sent for message $ID: $TITLE" || \
+                        echo "Failed to send email for message $ID"
+                    fi
+
+                    # Update last seen ID
+                    if [ "$ID" -gt "$LAST_ID" ]; then
+                      LAST_ID=$ID
+                    fi
+                  done
+                fi
+
+                sleep ${POLL_INTERVAL:-30}
+              done
+          env:
+            - name: GOTIFY_CLIENT_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-tokens
+                  key: client-token
+            - name: SMTP_HOST
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-smtp
+                  key: host
+            - name: SMTP_PORT
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-smtp
+                  key: port
+            - name: SMTP_FROM
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-smtp
+                  key: from
+            - name: SMTP_USER
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-smtp
+                  key: user
+            - name: SMTP_PASS
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-smtp
+                  key: password
+            - name: NOTIFY_EMAIL
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-smtp
+                  key: notify-email
+            - name: MIN_PRIORITY
+              value: "5"
+            - name: POLL_INTERVAL
+              value: "15"
+          resources:
+            requests:
+              cpu: 10m
+              memory: 32Mi
+            limits:
+              cpu: 100m
+              memory: 64Mi
+      # No volumes — Postgres handles persistence

k8s/apps/gotify/ingress.yaml

@@ -0,0 +1,26 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: gotify
+  namespace: notifications
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
+    # WebSocket support for Gotify client connections
+    nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
+    nginx.ingress.kubernetes.io/configuration-snippet: |
+      proxy_set_header Upgrade $http_upgrade;
+      proxy_set_header Connection "upgrade";
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: gotify.riotpiao.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: gotify
+                port:
+                  number: 80

k8s/apps/gotify/kustomization.yaml

@@ -0,0 +1,9 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - db.yaml
+  - secrets.yaml
+  - deployment.yaml
+  - service.yaml
+  - ingress.yaml

k8s/apps/gotify/namespace.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: notifications
+  labels:
+    kubernetes.io/metadata.name: notifications

k8s/apps/gotify/secrets.yaml

@@ -0,0 +1,92 @@
+#ENC[AES256_GCM,data:zdor03HqWha5KnsrNNfM+NT96bZafKhVJ1vfARlqbXzd56QTMq6tScippCbJWCtqLhK5,iv:shItz1uc3RYNBo9Xmq4M+0/Yjw6PbpF4x1tfZZQOSUM=,tag:8Bcwz5XkgLtjuMrDxFXOZQ==,type:comment]
+#ENC[AES256_GCM,data:QuQeqD21UgcVOUjlJTzQOAXioYWmnc4b38UTPAClL3WIf7/MqDZ2SQ==,iv:PHw7QplRXB4XBQh3fEolZpdvMMxLucaZx/aY8K5LE20=,tag:Vp3BUb2JPOBffwZW0jPfOg==,type:comment]
+#
+#ENC[AES256_GCM,data:3JOwvqkN0IsF4ogdR6SK9xZKCs7I,iv:uB3qIR9r2AZ7pewPzV+5anulFHnnmUmlBhfhUdM8A2U=,tag:w4g3k9bBcqO6o0pGfCRjuA==,type:comment]
+#ENC[AES256_GCM,data:mMQ0VWb11cOPA5QgdUqNGgeeLyzbFRKU7YOhFHmUx7Zb3NmePRXZ2gT6Y9vuieiMxCL/GeNXeg==,iv:URCI0VXAlcT5QI01L4yBB+P3+VT7lEfbmQiMQhVTUYk=,tag:CiyVNUzFZBmydrK9I861Og==,type:comment]
+#ENC[AES256_GCM,data:AX/p8bnjaddZFwR3hkvZNsm8op3nx6b8jnWEj6mNvPryq/MGpoS9SkNvlyACQ5KYMIYG6QtltG9buv4n5CVJzyYX,iv:YKdhnl+YtvvOFjpNXp3Ap3mnDPaDa0YoWupviRwPjwE=,tag:etGle/aGv7RemlmUeKEkCg==,type:comment]
+#ENC[AES256_GCM,data:Rs33VKbpeUCDrUnblHfBXiVjZa7tlxFV+QWiMYMv/uLzmXBS7NCpU7fIgNKG,iv:ZrZug41B6AysyVylQ6T6Z+Txc2nBUvblX8pRP2r12iQ=,tag:SvrNa0N0mprMq/Wqf77tZg==,type:comment]
+#ENC[AES256_GCM,data:sBzMx0RT9FAdtE4pEyWUkNX/nKDgHoEHk4IITSKpAl0/3i2uWgWs9NO4Sp1USAQaTtIa+Os=,iv:Zqj2GAtfw3UsIDgrTZ8erug+KgjP2ckNcas7NaZei+E=,tag:6xRTvzspIVLVhQJMQinPOQ==,type:comment]
+#ENC[AES256_GCM,data:bD+GzYKonJecM6Pym1powOYISWJDPIEVT8V1Hu8AM1U/wCdmq70TNq6DRu1FDSG+,iv:a/p1nM+rkie+mZ7E9A1Zg3qQloGTdtRWSIUJPiFNk24=,tag:ymyVypKvIUWmcZeHc5BQFQ==,type:comment]
+apiVersion: ENC[AES256_GCM,data:4vs=,iv:JAbEScwusLVWdsz4AuuYsBQDuusynhsbstrEyihrZ/k=,tag:b1IY2bxT6TL7djcFCiyMlg==,type:str]
+kind: ENC[AES256_GCM,data:1+fCn5DR,iv:HvI/Id59qQC//Uh3Q0P3oJo8ZSWZnhTXv3LTox3BuEU=,tag:i9aPbsqch9YKekzbf5G4cg==,type:str]
+metadata:
+    name: ENC[AES256_GCM,data:4528BwgvzAtwiHyK,iv:/vz1jcRbLJmAoqDvgTmDbxvE5gYxpInSEGBP7IUFvlI=,tag:yZPEJNrHht9IVrlzsAUqFA==,type:str]
+    namespace: ENC[AES256_GCM,data:FZMvrE3BV3iD3As59A==,iv:PfOFjLpZspP6JeTdNwa1IwjRl4JE1XKeAOab+8S8imk=,tag:yg3bWjkPngilwirv1CscDw==,type:str]
+type: ENC[AES256_GCM,data:QnoC8qce,iv:8paBbJZtmEbKSKlkwhy+3lLiRLoy1I+lB0xMBPdcNMg=,tag:p4/aUaSZHBzeU1CRqJ5rLw==,type:str]
+stringData:
+    username: ENC[AES256_GCM,data:ANiWzQM=,iv:I3LrmBV5HEtjO+aysXuZEOmbvhyOfuOJCqZHx1ykWyU=,tag:zEV4gu2125L0bHGUkgGdNQ==,type:str]
+    password: ENC[AES256_GCM,data:E5OW/ydSmskcnR8UYZsi9SZM5kpbiAE=,iv:NTgtzAK8P0jeczb8Egrrkmlf1Ejulabxh2/rLWzwNTk=,tag:kF2luernkCvnNl8o3EEvfA==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB3b3VUNi9oRFB3ZjFyeVQr
+            a2dTR3AvWnFDNi9YZ0QvaVNLU3dUSnlEMVdZCklhZjAzNURsZDBNWEI1MzZ1LzFl
+            cUNhUWJUcVhNSjZqSG43MWdudStaRUkKLS0tIEpuc2kwR2hpUzRiSXZyb1I0V3lE
+            YWJPcHFZU1lBOFFrQ3ErdzF4RzRqYzgK7q0N+ZcF/plIHR7HeRTF2qRmE4Za+eO+
+            mMCo01fxd3ybf8gXnC9lrYmIK4oCkS8cg/B8mfIIBqts7XDzHiCaGA==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    lastmodified: "2026-09-10T01:34:10Z"
+    mac: ENC[AES256_GCM,data:SURFYfHkesBFpdzeoMSdASSduGefetZ9sPY+0KgeUP4lMRX70kH6vnZdt5UyXe4hMgRCDA38sF+UgITW1pxhngqp8tL8QK5GBj+wTX9/zGmbtYl0zPvjvvXLx1vXGvUtfcEyxHW90EfDWae3JxQHCNlUcEH61hNj1ccZX+ceufM=,iv:mCQp3110nD3tg1eaf6P4e9XiMPbO+bhchC0zZ5+vqX8=,tag:vWGl3iaEi3KEoW87JBl3/g==,type:str]
+    unencrypted_suffix: _unencrypted
+    version: 3.13.2
+---
+apiVersion: ENC[AES256_GCM,data:too=,iv:DFFOApoqaZDGmJe7NcGBrvE69kp85YpLwJUsImKSxmk=,tag:sKWVbV1U3oI656Sb2+iWew==,type:str]
+kind: ENC[AES256_GCM,data:Lp1+y5tL,iv:pHbNnGCX6qsAIMSjkdHNyuHZct63jcDatnaek6Z5rjg=,tag:k/0Y7noIi/nAAVLpCv9GTw==,type:str]
+metadata:
+    name: ENC[AES256_GCM,data:QHAhhTHIxeyBXCfE2g==,iv:Q/2HuRPlbEC5AMONhi6POcCDdCJv3vHWMvmm6ZXGrr0=,tag:B+3Jhut9I7NzfpGtF969dQ==,type:str]
+    namespace: ENC[AES256_GCM,data:VnXAy9fR8ixXanheUw==,iv:Uvkcm7zqgLCWlzcx9fO2/ccytzM7MlD0Zl1X8WNaz7Q=,tag:JsWelai4T+aNg1UabZPMTg==,type:str]
+type: ENC[AES256_GCM,data:I3UeAwER,iv:mtIMz/kuzbHx0HMSPVaqzHMLvhCGzvkdNLRfr7xNKBM=,tag:QCgh1anTe7oN2j3kkUM2/g==,type:str]
+stringData:
+    #ENC[AES256_GCM,data:1XpJNypHreS+gUaRgEZe+CaJykvp3JJgI6Q=,iv:g/loGxxl6hvVvODmqwtip6om92df0tWVVg/msZvDOrg=,tag:ATCy0nG4PpGNoPQDRkXRcA==,type:comment]
+    #ENC[AES256_GCM,data:EmitK30ZlOZz7MruQ/ItDim2LjU3uUZlOQhfr2KO65t2lcAw3xcGZAdaQpQfyZ0RDUQucUgnl+g8ckhH6rZ7+/N9,iv:4655+CO98ak8cG/dZI0H2RCykLS29AWAZ4LjrH3Tg2M=,tag:eqhcnuY6rbyy98ABZrtPrw==,type:comment]
+    #ENC[AES256_GCM,data:iBIgf2Ru/cwE8D1L7uTdCmcii3lJxny1JE84jntuhROkCK1bqZU2cyf8eC0rq1zONiWwPuTokqFv18v+pdmbzQ==,iv:cnOCtS1qGy/w7ZkG2zRjqpgSeLU0FgvuV4tuBJ2SQ5U=,tag:3uxrBDiH/zMpLvFxZVfn1Q==,type:comment]
+    app-token: ENC[AES256_GCM,data:NNpIl+DOZ9hjP5TY0bmHWRfSC88GTy1O,iv:NDnz4i/rYbbUxpxUSHFyXc31iI4PQSasjEcHV3u9E70=,tag:8mBEOwhXxVPmoX85fAVKdg==,type:str]
+    client-token: ENC[AES256_GCM,data:nMSuoalWILpFijzVcB7sfA+d4EDb7HzL,iv:cgqkBnBUDqYMBXpQVCMzIPvpVvMu3BKl1CDqtY04xrk=,tag:Eql+ZAqPdvzm2M19HOoOHA==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB3b3VUNi9oRFB3ZjFyeVQr
+            a2dTR3AvWnFDNi9YZ0QvaVNLU3dUSnlEMVdZCklhZjAzNURsZDBNWEI1MzZ1LzFl
+            cUNhUWJUcVhNSjZqSG43MWdudStaRUkKLS0tIEpuc2kwR2hpUzRiSXZyb1I0V3lE
+            YWJPcHFZU1lBOFFrQ3ErdzF4RzRqYzgK7q0N+ZcF/plIHR7HeRTF2qRmE4Za+eO+
+            mMCo01fxd3ybf8gXnC9lrYmIK4oCkS8cg/B8mfIIBqts7XDzHiCaGA==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    lastmodified: "2026-09-10T01:34:10Z"
+    mac: ENC[AES256_GCM,data:SURFYfHkesBFpdzeoMSdASSduGefetZ9sPY+0KgeUP4lMRX70kH6vnZdt5UyXe4hMgRCDA38sF+UgITW1pxhngqp8tL8QK5GBj+wTX9/zGmbtYl0zPvjvvXLx1vXGvUtfcEyxHW90EfDWae3JxQHCNlUcEH61hNj1ccZX+ceufM=,iv:mCQp3110nD3tg1eaf6P4e9XiMPbO+bhchC0zZ5+vqX8=,tag:vWGl3iaEi3KEoW87JBl3/g==,type:str]
+    unencrypted_suffix: _unencrypted
+    version: 3.13.2
+---
+apiVersion: ENC[AES256_GCM,data:Ub4=,iv:t0/4neL1XNR9sc2a0rZ7c4fOnZjRw0ACOY2Pp2A61IY=,tag:X8W/L0wfQLw4JF5aQ1dteg==,type:str]
+kind: ENC[AES256_GCM,data:54r8Je+L,iv:feX1TVQNZ7VrYIFiYKvopB0hYEIl9iw0ly2A6u/j+1Y=,tag:gD7iCoOCxIUT5uj1bGfikQ==,type:str]
+metadata:
+    name: ENC[AES256_GCM,data:ub9FC+uwioKPMXg=,iv:1YbPJQle7t60Onnh77R8ffc8M87DNU1cmBQbVA72B28=,tag:3DsCpXMHcTIYN3uBh6Ztug==,type:str]
+    namespace: ENC[AES256_GCM,data:dLIZlOoIo9YF2VyHew==,iv:YxQFWoMhVMG3jUJqmAtAUwzRr7MkSRjiWlq92rnymIE=,tag:iw5AI7kKlp+wKcgg2GGMxw==,type:str]
+type: ENC[AES256_GCM,data:4+CN/8f0,iv:Gmnh4TP0oCAiJ2VTrBT/Q1PWkPZks6XnzWu3io32hvQ=,tag:m5SNbovIU7pjwGACc4Q50g==,type:str]
+stringData:
+    #ENC[AES256_GCM,data:bGrL1ia7o4ADA4J7soyOLCMVq3Y7kUU0BHhdz9ID45/q,iv:fwMHTJ68cYgBrzeAQ/CNdpfH14bAv7Exu7H6/QUk1ok=,tag:Lwc+FnbCd48oErLp5qEmcw==,type:comment]
+    #ENC[AES256_GCM,data:bQdj5OgHL60gffBB5DZPOL6+RFWxrrBxa8lPkXbYR7viOf0v0xb9iBqjS/o5kIOjjcdeJibgGhQ=,iv:DJQBppj4w3KqFOC1MTEYwoc1LT0Qz17fKb611hgH+AE=,tag:D+l7BE+FEgxZFTIDLpKsiA==,type:comment]
+    host: ENC[AES256_GCM,data:kaPQN225/tgQLLmejLQ=,iv:Xn69DoAa7Zop6EjZwIRgO/zVPdxvARgKkoacIE4Fcwc=,tag:hlCapggLwTsXr4DeejAYkQ==,type:str]
+    port: ENC[AES256_GCM,data:vkBH,iv:uCA8fhnMZDg/qiSQYRYmhuc7dnteoavcuJxfSCE8GU0=,tag:ujosmW6GjrEa30mBCEFXgw==,type:str]
+    from: ENC[AES256_GCM,data:kiWkZ7zmIT6gOHCH6hHI8fiJcQ==,iv:tqpYbLohLYbQ6kotOC5vDw6cZEFvNK1UEiKIa+nFzuQ=,tag:POZAxMpiGItr1lRY1yhW6w==,type:str]
+    user: ENC[AES256_GCM,data:Vdew4lqPsOD2,iv:1zJE/hEDbchie4crugcjCfvACklXoeSzqopXbPulD/E=,tag:QdJvWQRi9Tc7pgMBigKsBg==,type:str]
+    password: ENC[AES256_GCM,data:JcjnnXFkAXmI,iv:VgpYDrTn6o3l7tAk0lGMB2rpu+kB1IuAFu2VZMyJkX4=,tag:QR+En9nv/30yyt09GTcyqg==,type:str]
+    notify-email: ENC[AES256_GCM,data:zMvA9KXy6rK2HFRrjAmZd5c/ay8=,iv:BtCXUsHneVc1QzvPKRwBjmbYc39I3Xy/zFoMg4l22Ks=,tag:NTZKDoPaLARJoTIX9u6oGQ==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB3b3VUNi9oRFB3ZjFyeVQr
+            a2dTR3AvWnFDNi9YZ0QvaVNLU3dUSnlEMVdZCklhZjAzNURsZDBNWEI1MzZ1LzFl
+            cUNhUWJUcVhNSjZqSG43MWdudStaRUkKLS0tIEpuc2kwR2hpUzRiSXZyb1I0V3lE
+            YWJPcHFZU1lBOFFrQ3ErdzF4RzRqYzgK7q0N+ZcF/plIHR7HeRTF2qRmE4Za+eO+
+            mMCo01fxd3ybf8gXnC9lrYmIK4oCkS8cg/B8mfIIBqts7XDzHiCaGA==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    lastmodified: "2026-09-10T01:34:10Z"
+    mac: ENC[AES256_GCM,data:SURFYfHkesBFpdzeoMSdASSduGefetZ9sPY+0KgeUP4lMRX70kH6vnZdt5UyXe4hMgRCDA38sF+UgITW1pxhngqp8tL8QK5GBj+wTX9/zGmbtYl0zPvjvvXLx1vXGvUtfcEyxHW90EfDWae3JxQHCNlUcEH61hNj1ccZX+ceufM=,iv:mCQp3110nD3tg1eaf6P4e9XiMPbO+bhchC0zZ5+vqX8=,tag:vWGl3iaEi3KEoW87JBl3/g==,type:str]
+    unencrypted_suffix: _unencrypted
+    version: 3.13.2

k8s/apps/gotify/service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: gotify
+  namespace: notifications
+  labels:
+    app: gotify
+spec:
+  selector:
+    app: gotify
+  ports:
+    - port: 80
+      targetPort: 80
+      protocol: TCP

k8s/apps/llm-serving/kustomization.yaml

@@ -14,5 +14,7 @@ resources:
   - ornith.yaml
   - reasoning.yaml
   - reranker.yaml
+  - qwen-cpu.yaml
+  - networkpolicy.yaml
 # No namespace transformer: every file sets its own, and the transformer would
 # rewrite metadata.namespace on anything cross-namespace added later.

k8s/apps/llm-serving/networkpolicy.yaml

@@ -0,0 +1,62 @@
+# NetworkPolicy for LLM inference engines (llm-serving namespace).
+#
+# These pods have NO auth — vLLM, Ollama, and TEI accept any request.
+# All access MUST go through the api-gateway, which validates JWTs and
+# injects identity headers (X-Forwarded-User, X-Auth-Verified).
+#
+# Replaces the hand-applied llm-serving-default-deny policy that used
+# `llm-client: "true"` pod label as a selector — any pod in any namespace
+# could self-grant access by adding that label, which defeats the purpose.
+#
+# This policy restricts ingress to:
+#   1. api namespace (gateway) — the sole entry point for inference
+#   2. monitoring namespace — Prometheus scraping vLLM/TEI /metrics
+#   3. intra-namespace — pod-to-pod (future: multi-replica comms)
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: llm-serving-ingress
+  namespace: llm-serving
+  labels:
+    app.kubernetes.io/part-of: llm-serving
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/part-of: llm-serving
+  policyTypes:
+    - Ingress
+  ingress:
+    # Allow from api-gateway (namespace: api)
+    # Gateway proxies /v1/chat/completions, /v1/embeddings, /v1/rerank
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: api
+      ports:
+        - protocol: TCP
+          port: 8080   # vLLM, Ollama HTTP
+        - protocol: TCP
+          port: 80      # KServe predictor services
+        - protocol: TCP
+          port: 8000    # vLLM direct (some configs)
+        - protocol: TCP
+          port: 11434   # Ollama native port
+    # Allow Prometheus scraping from monitoring namespace
+    # vLLM: :8080/metrics, TEI: :9000/metrics
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: monitoring
+      ports:
+        - protocol: TCP
+          port: 8080
+        - protocol: TCP
+          port: 9000
+    # Allow intra-namespace (pod-to-pod within llm-serving)
+    - from:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/part-of: llm-serving
+      ports:
+        - protocol: TCP
+          port: 8080

k8s/apps/llm-serving/ornith.yaml

@@ -33,12 +33,8 @@ spec:
 
         ollama pull ornith:35b
 
-        ollama pull qwen2.5:3b-instruct
-
         ollama run ornith:35b "ok" >/dev/null 2>&1 || true
 
-        ollama run qwen2.5:3b-instruct "ok" >/dev/null 2>&1 || true
-
         wait $SERVE_PID
 
         '
@@ -54,7 +50,7 @@ spec:
       - name: OLLAMA_NUM_PARALLEL
         value: '1'
       - name: OLLAMA_MAX_LOADED_MODELS
-        value: '2'
+        value: '1'
       image: ollama/ollama:0.32.9@sha256:1685741456770df6e3cceb2a945a5f75e020f658d1701509668d6f4688f1dd3f
       name: kserve-container
       ports:
@@ -65,8 +61,7 @@ spec:
           command:
           - /bin/sh
           - -c
-          - ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
+          - ollama ps 2>/dev/null | grep -q ornith
-            grep -q qwen2.5
         periodSeconds: 10
       resources:
         limits:
@@ -82,8 +77,7 @@ spec:
           command:
           - /bin/sh
           - -c
-          - ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
+          - ollama ps 2>/dev/null | grep -q ornith
-            grep -q qwen2.5
         failureThreshold: 120
         periodSeconds: 15
       volumeMounts:
@@ -91,13 +85,10 @@ spec:
         name: models
     deploymentStrategy:
       type: Recreate
-    # 2 replicas -- each its own GPU, each loading both ornith:35b and
+    # 1 replica -- ornith:35b only. qwen2.5:3b moved to CPU on cp-2.
-    # qwen2.5:3b-instruct -- so 2 concurrent implementer-style calls each
+    # Frees 1 GPU for ComfyUI.
-    # get an independent instance instead of contending on one, at the
+    maxReplicas: 1
-    # cost of judge/qwen traffic still sharing whichever replica an
+    minReplicas: 1
-    # implementer call also lands on.
-    maxReplicas: 2
-    minReplicas: 2
     nodeSelector:
       kubernetes.io/hostname: worker-1
     runtimeClassName: nvidia

k8s/apps/llm-serving/qwen-cpu.yaml

@@ -0,0 +1,115 @@
+# qwen2.5:3b-instruct on CPU (talos-cp-2, 144GB RAM, 24 cores).
+# Moved off GPU to free a V100 for ComfyUI. Latency ~10x slower
+# than GPU but sufficient for lightweight tasks (summarization,
+# classification, quick answers).
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: qwen-cpu
+  namespace: llm-serving
+  labels:
+    app: qwen-cpu
+    app.kubernetes.io/name: qwen-cpu
+    app.kubernetes.io/part-of: llm-serving
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: qwen-cpu
+  template:
+    metadata:
+      labels:
+        app: qwen-cpu
+        app.kubernetes.io/name: qwen-cpu
+        app.kubernetes.io/part-of: llm-serving
+    spec:
+      nodeSelector:
+        kubernetes.io/hostname: talos-cp-2
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+      containers:
+        - name: ollama
+          image: ollama/ollama:0.32.9@sha256:1685741456770df6e3cceb2a945a5f75e020f658d1701509668d6f4688f1dd3f
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              ollama serve &
+              SERVE_PID=$!
+              until ollama list >/dev/null 2>&1; do sleep 2; done
+              ollama pull qwen2.5:3b-instruct
+              ollama run qwen2.5:3b-instruct "ok" >/dev/null 2>&1 || true
+              wait $SERVE_PID
+          env:
+            - name: OLLAMA_HOST
+              value: "0.0.0.0:8080"
+            - name: OLLAMA_MODELS
+              value: /root/.ollama/models
+            - name: OLLAMA_CONTEXT_LENGTH
+              value: "32768"
+            - name: OLLAMA_KEEP_ALIVE
+              value: "-1"
+            - name: OLLAMA_MAX_LOADED_MODELS
+              value: "1"
+            - name: OLLAMA_NUM_PARALLEL
+              value: "2"
+          ports:
+            - containerPort: 8080
+              protocol: TCP
+          readinessProbe:
+            exec:
+              command: ["/bin/sh", "-c", "ollama ps 2>/dev/null | grep -q qwen2.5"]
+            periodSeconds: 10
+          startupProbe:
+            exec:
+              command: ["/bin/sh", "-c", "ollama ps 2>/dev/null | grep -q qwen2.5"]
+            failureThreshold: 60
+            periodSeconds: 10
+          resources:
+            requests:
+              cpu: "4"
+              memory: 4Gi
+            limits:
+              cpu: "8"
+              memory: 8Gi
+          volumeMounts:
+            - mountPath: /root/.ollama
+              name: ollama-data
+      volumes:
+        - name: ollama-data
+          persistentVolumeClaim:
+            claimName: qwen-cpu-data
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: qwen-cpu
+  namespace: llm-serving
+  labels:
+    app: qwen-cpu
+    app.kubernetes.io/part-of: llm-serving
+spec:
+  selector:
+    app: qwen-cpu
+  ports:
+    - port: 80
+      targetPort: 8080
+      protocol: TCP
+---
+# Small PVC for qwen2.5:3b model weights (~1.9GB).
+# Separate from llm-models PVC which is pinned to worker-1.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: qwen-cpu-data
+  namespace: llm-serving
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: longhorn
+  resources:
+    requests:
+      storage: 5Gi

k8s/argocd/apps/55-api-gateway.yaml

@@ -38,8 +38,8 @@ metadata:
     argocd.argoproj.io/sync-wave: "7"
     # ArgoCD Image Updater - auto-update on new image push
     argocd-image-updater.argoproj.io/image-list: gw=forgejo.riotpiao.com/rock/api-gateway
-    argocd-image-updater.argoproj.io/gw.update-strategy: newest-build
+    argocd-image-updater.argoproj.io/gw.update-strategy: digest
-    argocd-image-updater.argoproj.io/gw.allow-tags: regexp:^[0-9a-f]{7}$
+    argocd-image-updater.argoproj.io/gw.allow-tags: regexp:^latest$
     argocd-image-updater.argoproj.io/write-back-method: argocd
 spec:
   project: homelab

k8s/argocd/apps/58-comfyui.yaml

@@ -0,0 +1,32 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: comfyui
+  namespace: argocd
+  labels:
+    app.kubernetes.io/name: comfyui
+    app.kubernetes.io/component: image-generation
+  annotations:
+    argocd.argoproj.io/sync-wave: "8"
+spec:
+  project: homelab
+  revisionHistoryLimit: 3
+  source:
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    targetRevision: main
+    path: k8s/apps/comfyui
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: comfyui
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+    retry:
+      limit: 5
+      backoff:
+        duration: 5s
+        factor: 2
+        maxDuration: 3m

k8s/argocd/apps/59-gotify.yaml

@@ -0,0 +1,32 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: gotify
+  namespace: argocd
+  labels:
+    app.kubernetes.io/name: gotify
+    app.kubernetes.io/component: notifications
+  annotations:
+    argocd.argoproj.io/sync-wave: "8"
+spec:
+  project: homelab
+  revisionHistoryLimit: 3
+  source:
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    targetRevision: main
+    path: k8s/apps/gotify
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: notifications
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+    retry:
+      limit: 5
+      backoff:
+        duration: 5s
+        factor: 2
+        maxDuration: 3m

k8s/argocd/apps/60-applications.yaml

@@ -248,8 +248,8 @@ metadata:
     argocd.argoproj.io/sync-wave: "8"
     # ArgoCD Image Updater - auto-update on new image push
     argocd-image-updater.argoproj.io/image-list: app=forgejo.riotpiao.com/rock/portfolio
-    argocd-image-updater.argoproj.io/app.update-strategy: newest-build
+    argocd-image-updater.argoproj.io/app.update-strategy: digest
-    argocd-image-updater.argoproj.io/app.allow-tags: regexp:^[0-9a-f]{7}$
+    argocd-image-updater.argoproj.io/app.allow-tags: regexp:^latest$
     argocd-image-updater.argoproj.io/write-back-method: argocd
 spec:
   project: homelab

k8s/argocd/apps/70-poimen.yaml

@@ -10,14 +10,13 @@ metadata:
       memory=forgejo.riotpiao.com/rock/poimen-memory
       workflows=forgejo.riotpiao.com/rock/poimen-workflows
       frontend=forgejo.riotpiao.com/rock/poimen-frontend
-    argocd-image-updater.argoproj.io/memory.update-strategy: newest-build
+    argocd-image-updater.argoproj.io/memory.update-strategy: digest
-    argocd-image-updater.argoproj.io/memory.allow-tags: regexp:^[0-9a-f]{7}$
+    argocd-image-updater.argoproj.io/memory.allow-tags: regexp:^latest$
-    argocd-image-updater.argoproj.io/workflows.update-strategy: newest-build
+    argocd-image-updater.argoproj.io/workflows.update-strategy: digest
-    argocd-image-updater.argoproj.io/workflows.allow-tags: regexp:^[0-9a-f]{7}$
+    argocd-image-updater.argoproj.io/workflows.allow-tags: regexp:^latest$
-    argocd-image-updater.argoproj.io/frontend.update-strategy: newest-build
+    argocd-image-updater.argoproj.io/frontend.update-strategy: digest
-    argocd-image-updater.argoproj.io/frontend.allow-tags: regexp:^[0-9a-f]{7}$
+    argocd-image-updater.argoproj.io/frontend.allow-tags: regexp:^latest$
-    argocd-image-updater.argoproj.io/write-back-method: git
+    argocd-image-updater.argoproj.io/write-back-method: argocd
-    argocd-image-updater.argoproj.io/git-branch: main
 spec:
   project: homelab
   sources:

k8s/infra/forgejo-runner/Dockerfile.golang

@@ -1,16 +0,0 @@
-FROM code.forgejo.org/forgejo/runner:6
-
-# Switch to root to install packages (Alpine)
-USER root
-
-# Alpine uses apk, not apt-get
-RUN apk update && apk add --no-cache \
-      nodejs \
-      npm \
-      docker-cli
-
-# Verify installations
-RUN docker --version && node --version && git --version
-
-# Switch back to runner user
-USER 1000:1000

k8s/infra/forgejo-runner/Dockerfile.node

@@ -1,16 +0,0 @@
-FROM code.forgejo.org/forgejo/runner:6
-
-# Switch to root to install packages (Alpine)
-USER root
-
-# Alpine uses apk, not apt-get
-RUN apk update && apk add --no-cache \
-      docker-cli \
-      nodejs \
-      npm
-
-# Verify installations
-RUN node --version && docker --version && git --version
-
-# Switch back to runner user
-USER 1000:1000

k8s/infra/forgejo-runner/Dockerfile.rust

@@ -1,20 +0,0 @@
-FROM code.forgejo.org/forgejo/runner:6
-
-# Switch to root to install packages (Alpine)
-USER root
-
-# Alpine uses apk, not apt-get
-RUN apk update && apk add --no-cache \
-      nodejs \
-      npm \
-      curl \
-      docker-cli
-
-# Install Rust (as root, skip verification for now)
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable || true
-
-# Verify core installations
-RUN docker --version && node --version && git --version
-
-# Switch back to runner user
-USER 1000:1000

k8s/infra/forgejo-runner/templates/configmap.yaml

@@ -36,3 +36,4 @@ data:
       valid_volumes:
         - /docker-certs/client
       network: host
+      docker_host: automount

k8s/infra/forgejo-runner/templates/deployment.yaml

@@ -34,7 +34,11 @@ spec:
           command: ["sh", "-c"]
           args:
             - |
-              test -f /data/.runner || forgejo-runner register --no-interactive \
+              # Always re-register to keep labels in sync with values.yaml.
+              # Without this, changing a runner label requires manually deleting
+              # the PVC or .runner file — not GitOps-friendly.
+              rm -f /data/.runner
+              forgejo-runner register --no-interactive \
                 --instance {{ .Values.runner.forgejoUrl }} \
                 --token $(RUNNER_TOKEN) \
                 --name {{ .Values.runner.name }} \
@@ -56,20 +60,18 @@ spec:
       containers:
         - name: runner
           image: {{ .Values.runner.image.repository }}:{{ .Values.runner.image.tag }}
-          command: ["sh", "-c", "forgejo-runner daemon --config /etc/forgejo-runner/config.yaml"]
+          command: ["sh", "-c", "while ! wget -q -O- http://localhost:2375/_ping >/dev/null 2>&1; do echo 'waiting for dind...'; sleep 2; done; echo 'dind ready'; forgejo-runner daemon --config /etc/forgejo-runner/config.yaml"]
           workingDir: /data
           env:
             - name: DOCKER_HOST
-              value: tcp://localhost:2376
+              value: tcp://localhost:2375
-            - name: DOCKER_TLS_VERIFY
-              value: "1"
-            - name: DOCKER_CERT_PATH
-              value: /docker-certs/client
           volumeMounts:
             - name: runner-data
               mountPath: /data
             - name: docker-certs
               mountPath: /docker-certs
+            - name: docker-sock
+              mountPath: /run
             - name: homelab-ca
               mountPath: /etc/ssl/certs/homelab-ca.pem
               subPath: ca.crt
@@ -85,10 +87,12 @@ spec:
             privileged: true    # required for DinD; cicd namespace is labelled privileged
           env:
             - name: DOCKER_TLS_CERTDIR
-              value: /docker-certs
+              value: ""
           volumeMounts:
             - name: docker-certs
               mountPath: /docker-certs
+            - name: docker-sock
+              mountPath: /run
             - name: dind-storage
               mountPath: /var/lib/docker
             - name: homelab-ca
@@ -113,6 +117,8 @@ spec:
             claimName: {{ .Release.Name }}-dind
         - name: docker-certs
           emptyDir: {}    # DinD regenerates mTLS certs on each start
+        - name: docker-sock
+          emptyDir: {}    # Shared docker socket between dind and runner
         - name: homelab-ca
           # homelab-ca is a ConfigMap (public CA trust bundle), not a Secret.
           # The volumeMounts use subPath: ca.crt to project the single cert file.

k8s/infra/forgejo-runner/values-node.yaml

@@ -2,15 +2,14 @@
 # runner instance. Only runner.name and runner.labels differ -- everything
 # else (image, dind, persistence, tolerations, nodeSelector) is shared.
 #
-# node:22-bookworm ships Node natively. Docker client installed via workflow step if needed.
+# Label image: node:22-bookworm — Debian, root, apt-get, Node.js, npm, git.
-# (homelab has no CI; custom runner images built manually if desired)
+# Install docker in workflow steps as needed.
-# Bootstrap with runner image (already has Node.js), CI builds custom
 runner:
   image:
     repository: code.forgejo.org/forgejo/runner
     tag: "6"
   name: node-runner
-  labels: "node:docker://code.forgejo.org/forgejo/runner:6"
+  labels: "node:docker://node:22-bookworm"
 
 # GC CronJob renders only from the default (golang) values to avoid duplicates
 gc:

k8s/infra/forgejo-runner/values-rust.yaml

@@ -2,13 +2,14 @@
 # runner instance. Only runner.name and runner.labels differ -- everything
 # else (image, dind, persistence, tolerations, nodeSelector) is shared.
 #
-# Bootstrap with runner image, CI builds custom with Node.js+Rust
+# Label image: rust:1-bookworm — Debian, root, apt-get, Rust, cargo, git.
+# Install Node.js/docker in workflow steps as needed.
 runner:
   image:
     repository: code.forgejo.org/forgejo/runner
     tag: "6"
   name: rust-runner
-  labels: "rust:docker://code.forgejo.org/forgejo/runner:6"
+  labels: "rust:docker://rust:1-bookworm"
 
 
 

k8s/infra/forgejo-runner/values.yaml

@@ -1,9 +1,12 @@
 runner:
   image:
     repository: code.forgejo.org/forgejo/runner
-    tag: "6"  # Bootstrap with runner image, CI builds custom with Node.js
+    tag: "6"
   name: golang-runner
-  labels: "golang:docker://code.forgejo.org/forgejo/runner:6"
+  # Label image is what workflow steps run in (NOT the runner daemon image).
+  # golang:1.26-bookworm: Debian, root, apt-get, Go, git.
+  # TODO: Switch to custom image once build-runner-images.yml pushes images
+  labels: "golang:docker://golang:1.26-bookworm"
   forgejoUrl: http://forgejo-gitea-http.cicd.svc.cluster.local:3000
   tokenSecret: runner-token
   resources:

terraform/files/coredns/Corefile

@@ -36,6 +36,7 @@
     rewrite name paperless.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name img.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name api.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    rewrite name comfy.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
 
     kubernetes cluster.local in-addr.arpa ip6.arpa {