fix(ingress): remove stale ingress-nginx-controller-alias Service — its selfHeal kept clobbering the helm LoadBalancer Service (same name, dead ingress-nginx-bootstrap selector, 0 endpoints), unannouncing LB IP .160 and taking down all ingress

- namespace: PodSecurity privileged, needed for /dev/kvm + privileged QEMU
- storageclass: 1 replica, strict-local, WaitForFirstConsumer
- deployment: nodeSelector workload=imessage + matching NoSchedule toleration,
  Recreate strategy (two QEMU procs on one qcow2 corrupts it), no readiness
  probe (guest install is interactive and takes many minutes)
- services: ClusterIP only; VNC is an unauthenticated console, reach it with
  port-forward, never an Ingress
- networkpolicy: default-deny, opt-in via sms-client=true on port 1234

k8s/apps/api/ingress.yaml

@@ -0,0 +1,43 @@
+# Edge route for the API gateway.
+#
+# Lives here rather than in the central k8s/bootstrap/ingress/ingress.yaml
+# because that Application syncs in wave 1, before namespace `api` exists.
+#
+# nginx terminates TLS with the wildcard *.riotpiao.com cert (served as its
+# default-ssl-certificate, so no per-rule `tls:` block is needed) and forwards
+# plain HTTP to kong-proxy. Kong then does the real routing, from Ingresses
+# carrying `ingressClassName: kong`.
+#
+# Catch-all `/` on purpose: everything under this host belongs to Kong. Listing
+# per-API paths here would duplicate Kong's routing table inside nginx, and the
+# two copies would drift.
+#
+# In-cluster callers should prefer http://kong-proxy.api.svc.cluster.local
+# directly. Resolving api.riotpiao.com sends them out to nginx and back in,
+# which is a pointless hairpin unless they need TLS or the public hostname.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: api
+  namespace: api
+  annotations:
+    # An API gateway carries streaming responses (SSE, gRPC-web, LLM token
+    # streams). nginx's 60s default read timeout and its response buffering
+    # would truncate or stall those.
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-buffering: "off"
+    nginx.ingress.kubernetes.io/proxy-body-size: "0"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: api.riotpiao.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: kong-proxy
+                port:
+                  number: 80

k8s/apps/api/kong-values.yaml

@@ -0,0 +1,97 @@
+# Kong Gateway — cluster-internal API gateway (namespace `api`).
+#
+# Chart: kong/kong 3.4.1 (appVersion 3.9). Only overrides are listed; every key
+# here was checked against `helm show values kong/kong --version 3.4.1`, because
+# Helm silently ignores unknown keys — a typo is a no-op, not an error.
+#
+# ── Topology ────────────────────────────────────────────────────────────────
+#   external:  client -> nginx (TLS, wildcard *.riotpiao.com) -> kong-proxy:80
+#   internal:  pod    -> kong-proxy.api.svc.cluster.local:80
+#
+# nginx stays the single edge and the only LoadBalancer (192.168.1.160). Kong is
+# the policy/routing layer behind it, so it needs no LB IP and no TLS of its own
+# — hence ClusterIP and proxy.tls disabled. Giving Kong its own IP from
+# homelab-pool would mean duplicating cert-manager wiring and diverging from the
+# CoreDNS convention that sends every *.riotpiao.com host to nginx.
+#
+# ── Routing model ───────────────────────────────────────────────────────────
+# Consumers publish an Ingress with `ingressClassName: kong`; the controller
+# turns it into a Kong route. `nginx` remains the default IngressClass, so this
+# is strictly opt-in and no existing Ingress changes behaviour.
+
+# Without this the release name is prefixed onto everything (`kong-kong-proxy`).
+# Pinning it keeps the Service name stable and independent of the release name,
+# which matters because the nginx Ingress in k8s/bootstrap/ingress/ingress.yaml
+# references it by name.
+fullnameOverride: kong
+
+# Two replicas so a node drain or rollout doesn't take the gateway down. Kong is
+# stateless in DB-less mode, so replicas are pure redundancy.
+replicaCount: 2
+
+env:
+  # DB-less. Config comes from Kubernetes objects via the ingress controller, so
+  # git stays the source of truth. A Postgres-backed Kong would put live routing
+  # config in a database mutated through the Admin API — state outside git, plus
+  # migration Jobs on every upgrade.
+  database: "off"
+
+ingressController:
+  enabled: true
+  ingressClass: kong
+  # The chart's ingress-class template is gated on
+  # `.Capabilities.APIVersions.Has "networking.k8s.io/v1/IngressClass"`, so a
+  # bare `helm template` renders nothing. ArgoCD passes --api-versions from the
+  # live cluster, so it does render there — verify `kubectl get ingressclass
+  # kong` after the first sync rather than assuming it.
+  createIngressClass: true
+  # Deliberately empty: setting is-default-class here would hijack every Ingress
+  # in the cluster that omits ingressClassName. nginx keeps that role.
+  ingressClassAnnotations: {}
+
+proxy:
+  enabled: true
+  # Chart default is LoadBalancer, which would claim an IP from homelab-pool.
+  type: ClusterIP
+  http:
+    enabled: true
+    servicePort: 80
+    containerPort: 8000
+  # nginx already terminated TLS; a second handshake to the same cluster buys
+  # nothing and would need Kong to hold its own certificate.
+  tls:
+    enabled: false
+
+# No Service for the Admin API. The controller reaches it over localhost inside
+# the pod, so exposing it would only create an unauthenticated write path to the
+# gateway's entire configuration.
+admin:
+  enabled: false
+
+# Kong Manager UI — chart default is `enabled: true` with type NodePort, which
+# would open a port on every node. Not wanted.
+manager:
+  enabled: false
+
+resources:
+  requests:
+    cpu: 200m
+    memory: 256Mi
+  limits:
+    cpu: "2"
+    memory: 1Gi
+
+podDisruptionBudget:
+  enabled: true
+  minAvailable: 1
+
+# Spread the two replicas across nodes; `ScheduleAnyway` so a single-node
+# situation degrades to co-location instead of leaving a pod Pending.
+topologySpreadConstraints:
+  - maxSkew: 1
+    topologyKey: kubernetes.io/hostname
+    whenUnsatisfiable: ScheduleAnyway
+    labelSelector:
+      matchLabels:
+        app.kubernetes.io/name: kong
+        app.kubernetes.io/instance: kong

k8s/apps/api/kustomization.yaml

@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+# Explicit allowlist so kong-values.yaml in this directory is NOT treated as a
+# manifest — it is Helm input consumed by the chart source of the `kong`
+# Application, not a Kubernetes object. Anything new added here must be listed
+# or it is silently dropped with no error and no drift shown.
+resources:
+  - ingress.yaml
+# No top-level `namespace:` transformer on purpose: ingress.yaml sets its own
+# namespace, and the transformer rewrites metadata.namespace on every resource
+# it builds, which is a trap for anything cross-namespace added later.

k8s/apps/gpu-runtimeclass/kustomization.yaml

@@ -1,6 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-# No namespace: RuntimeClass is cluster-scoped.
-resources:
-  - namespace.yaml
-  - runtimeclass.yaml

k8s/apps/gpu-runtimeclass/namespace.yaml

@@ -1,23 +0,0 @@
-# Namespace for GPU node-level plumbing (device plugin, and later DCGM).
-#
-# PodSecurity must be `privileged` here. The cluster default from the Talos
-# controlplane config is `enforce: baseline` with exemptions only for
-# kube-system, and a device plugin cannot satisfy baseline: it has to mount the
-# kubelet device-plugin socket and the CDI/driver directories as hostPath
-# volumes, which baseline forbids outright:
-#
-#   Error creating: pods "nvidia-device-plugin-xxxxx" is forbidden:
-#   violates PodSecurity "baseline:latest": hostPath volumes
-#   (volumes "kubelet-device-plugins-dir", "mps-root", "mps-shm", "cdi-root")
-#
-# This is inherent to how device plugins work, not a workaround. Scope is
-# limited to this namespace; the engine namespace (llm-serving) stays on the
-# cluster default.
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: gpu-system
-  labels:
-    pod-security.kubernetes.io/enforce: privileged
-    pod-security.kubernetes.io/audit: privileged
-    pod-security.kubernetes.io/warn: privileged

k8s/apps/gpu-runtimeclass/runtimeclass.yaml

@@ -1,18 +0,0 @@
-# Cluster-scoped prerequisite for every GPU workload on worker-1.
-#
-# The Talos nvidia-container-toolkit extension already registers the containerd
-# runtime handler (/etc/cri/conf.d/10-nvidia-container-runtime.part ->
-# plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia), but the
-# Kubernetes RuntimeClass object is separate and is NOT created by the
-# extension. Without it every pod carrying runtimeClassName: nvidia is rejected
-# at admission with:
-#   pods "..." is forbidden: pod rejected: RuntimeClass "nvidia" not found
-#
-# Deliberately NOT setting nvidia as containerd's default_runtime_name (the
-# 20-customization.part patch in the Talos guide): that would route every pod on
-# the node through the NVIDIA runtime. Opting in per-pod is narrower.
-apiVersion: node.k8s.io/v1
-kind: RuntimeClass
-metadata:
-  name: nvidia
-handler: nvidia

k8s/apps/homarr/homarr-values.yaml

@@ -30,10 +30,16 @@ replicaCount: 1
 env:
   AUTH_PROVIDERS: "oidc,credentials"
   AUTH_OIDC_ISSUER: "https://authentik.riotpiao.com/application/o/homarr/"
+  # AUTH_OIDC_URI (authorize endpoint) is REQUIRED in addition to ISSUER — homarr
+  # hides the "Sign in with Authentik" button entirely when it's absent (per the
+  # authentik Homarr integration + homarr SSO docs). This was the missing var.
+  AUTH_OIDC_URI: "https://authentik.riotpiao.com/application/o/authorize/"
   AUTH_OIDC_CLIENT_NAME: "Authentik"
   AUTH_OIDC_GROUPS_ATTRIBUTE: "groups"
   AUTH_OIDC_SCOPE_OVERWRITE: "openid email profile groups"
   AUTH_OIDC_AUTO_LOGIN: "false"
+  # Link the OIDC identity to an existing homarr account with the same email.
+  OAUTH_ALLOW_DANGEROUS_EMAIL_ACCOUNT_LINKING: "true"
   BASE_URL: "https://homarr.riotpiao.com"
   NEXTAUTH_URL: "https://homarr.riotpiao.com"
 

k8s/apps/llm-serving/inferenceservice-embeddings.yaml

@@ -1,74 +0,0 @@
-# Embeddings — Nomic Embed Text v2 (MoE), on CPU via HuggingFace TEI.
-#
-# CPU, not GPU, deliberately. All 4 V100s are claimed by the generation models,
-# and the device plugin hands out WHOLE GPUs — a 5th GPU-requesting pod is
-# unschedulable no matter how much VRAM is free. Sharing would need global
-# time-slicing, which on a single node cannot be scoped to one card and would let
-# the scheduler co-locate two ~20GB models and OOM both.
-#
-# worker-1 has 96 cores with ~250m requested, and this is a 475M-param encoder
-# (305M active). Retrieval runs once per agent-loop iteration, not per token, so
-# CPU latency here is immaterial. This is also what Plan 1 originally specified.
-#
-# TEI (not vLLM) because it is purpose-built for encoders and explicitly lists
-# nomic-embed-text-v2-moe as supported.
-#
-# NOTE: Nomic v2 requires task prefixes on the CLIENT side —
-#   documents: "search_document: <text>"
-#   queries:   "search_query: <text>"
-# Embedding without the prefix silently degrades retrieval quality.
-apiVersion: serving.kserve.io/v1beta1
-kind: InferenceService
-metadata:
-  name: embeddings
-  labels:
-    app.kubernetes.io/name: llm-embeddings
-    app.kubernetes.io/part-of: llm-serving
-spec:
-  predictor:
-    minReplicas: 1
-    maxReplicas: 1
-    # Pinned to worker-1 only so it can share the RWO models PVC with the GPU
-    # pods (RWO = single node, any number of pods on it).
-    nodeSelector:
-      kubernetes.io/hostname: worker-1
-    containers:
-      - name: kserve-container
-        image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.8.2@sha256:4d632b76bd14cb57044a1ffb0ad48ab0ba4939e705a9a615ccc740658575c26e
-        args:
-          - --model-id=nomic-ai/nomic-embed-text-v2-moe
-          - --port=8080
-          - --hostname=0.0.0.0
-          # Truncate rather than 413 on over-long input.
-          - --auto-truncate
-        env:
-          - name: HUGGINGFACE_HUB_CACHE
-            value: /mnt/models
-        ports:
-          - containerPort: 8080
-            protocol: TCP
-        resources:
-          requests:
-            cpu: "8"
-            memory: 4Gi
-          limits:
-            cpu: "16"
-            memory: 8Gi
-        volumeMounts:
-          - name: models
-            mountPath: /mnt/models
-        startupProbe:
-          httpGet:
-            path: /health
-            port: 8080
-          periodSeconds: 10
-          failureThreshold: 60
-        readinessProbe:
-          httpGet:
-            path: /health
-            port: 8080
-          periodSeconds: 10
-    volumes:
-      - name: models
-        persistentVolumeClaim:
-          claimName: llm-models

k8s/apps/llm-serving/inferenceservice-ornith.yaml

@@ -1,107 +0,0 @@
-# Action engine — Ornith-1.0-35B on Ollama.
-#
-# Why not vLLM like the other two: Ornith is
-# Qwen3_5MoeForConditionalGeneration (Qwen3.5 MoE, 256 experts / 8 active,
-# hybrid attention — 30 linear_attention + 10 full_attention layers). vLLM's
-# Qwen3.5 support landed 2026-07-29, AFTER vLLM dropped Volta (sm_70) at
-# v0.11.1. No vLLM build has both, so Ornith cannot run on vLLM on a V100.
-#
-# Ollama ships `ornith:35b` in its library and runs a llama-server runner
-# underneath, which keeps Volta support. q4 is ~21GB — fits one 32GB V100 with
-# room for KV.
-#
-# OLLAMA_KEEP_ALIVE=-1 is load-bearing: the harness calls this every loop
-# iteration, and Ollama's default is to evict an idle model after 5m, which
-# would add a ~21GB reload to a random future request.
-apiVersion: serving.kserve.io/v1beta1
-kind: InferenceService
-metadata:
-  name: ornith
-  labels:
-    app.kubernetes.io/name: llm-ornith
-    app.kubernetes.io/part-of: llm-serving
-spec:
-  predictor:
-    minReplicas: 1
-    # Recreate, not the default RollingUpdate: GPUs are allocated exactly 4/4,
-    # so a surge pod has no card to claim and sits Pending while the old pod is
-    # never torn down — a deadlock. Recreate tears down first, accepting a brief
-    # gap during updates.
-    deploymentStrategy:
-      type: Recreate
-    maxReplicas: 1
-    nodeSelector:
-      kubernetes.io/hostname: worker-1
-    runtimeClassName: nvidia
-    containers:
-      - name: kserve-container
-        image: ollama/ollama:0.32.9@sha256:1685741456770df6e3cceb2a945a5f75e020f658d1701509668d6f4688f1dd3f
-        # `ollama serve` does not pull models, and `ollama pull` needs a running
-        # server — so background the server, wait for it, pull, then hand the
-        # foreground back to serve.
-        command:
-          - /bin/sh
-          - -c
-          - |
-            set -e
-            ollama serve &
-            SERVE_PID=$!
-            until ollama list >/dev/null 2>&1; do sleep 2; done
-            ollama pull ornith:35b
-            ollama pull qwen2.5:3b-instruct
-            wait $SERVE_PID
-        env:
-          # Match the port the other two engines use.
-          - name: OLLAMA_HOST
-            value: "0.0.0.0:8080"
-          - name: OLLAMA_MODELS
-            value: /mnt/models/ollama
-          # Ollama defaults to a 4096 context, far too small for an agentic
-          # coding model. Ornith's hybrid attention means only 10 of its 40
-          # layers hold a conventional KV cache, so 32K is affordable in the
-          # ~11GiB left after its 21GB of weights.
-          - name: OLLAMA_CONTEXT_LENGTH
-            value: "32768"
-          # Never evict — this model is on the harness's hot path.
-          - name: OLLAMA_KEEP_ALIVE
-            value: "-1"
-          # Serial agent loop; no benefit from parallel slots.
-          - name: OLLAMA_NUM_PARALLEL
-            value: "1"
-          # 2, so ornith and the small utility model stay co-resident on GPU2
-          # instead of evicting one another on every alternating request.
-          - name: OLLAMA_MAX_LOADED_MODELS
-            value: "2"
-        ports:
-          - containerPort: 8080
-            protocol: TCP
-        resources:
-          requests:
-            cpu: "8"
-            memory: 8Gi
-            nvidia.com/gpu: "1"
-          limits:
-            cpu: "16"
-            memory: 16Gi
-            nvidia.com/gpu: "1"
-        volumeMounts:
-          - name: models
-            mountPath: /mnt/models
-        # Probes must confirm the MODEL is present, not just that the server
-        # answers. Ollama's `GET /` returns 200 ("Ollama is running") the moment
-        # `ollama serve` binds — which is before the ~21GB pull finishes. An
-        # httpGet probe would therefore mark this pod Ready with no model
-        # loaded, and KServe would route traffic to it.
-        startupProbe:
-          exec:
-            command: ["/bin/sh", "-c", "ollama list 2>/dev/null | grep -q ornith && ollama list 2>/dev/null | grep -q qwen2.5"]
-          periodSeconds: 15
-          failureThreshold: 120
-        readinessProbe:
-          exec:
-            command: ["/bin/sh", "-c", "ollama list 2>/dev/null | grep -q ornith && ollama list 2>/dev/null | grep -q qwen2.5"]
-          periodSeconds: 10
-    volumes:
-      - name: models
-        persistentVolumeClaim:
-          claimName: llm-models

k8s/apps/llm-serving/inferenceservice-reasoning.yaml

@@ -1,133 +0,0 @@
-# Reasoning engine — DeepSeek-R1-Distill-Qwen-32B, GPTQ INT4, vLLM.
-#
-# TP=1 with 2 data-parallel replicas (GPU0 + GPU1) rather than one TP=2 engine:
-# worker-1 has NO NVLink, so tensor-parallel's per-token all-reduce would cross
-# PCIe on every decode step. Two independent replicas need zero inter-GPU
-# communication and KServe load-balances them behind one Service.
-#
-# vLLM is pinned to v0.11.0 — the LAST release that compiles sm_70 (Volta)
-# kernels. v0.11.1 dropped 7.0 from CUDA_SUPPORTED_ARCHS. Do not bump this
-# without re-checking CMakeLists.txt, or every pod dies with "no kernel image".
-apiVersion: serving.kserve.io/v1beta1
-kind: InferenceService
-metadata:
-  name: reasoning
-  labels:
-    app.kubernetes.io/name: llm-reasoning
-    app.kubernetes.io/part-of: llm-serving
-spec:
-  predictor:
-    minReplicas: 2
-    # Recreate, not the default RollingUpdate: GPUs are allocated exactly 4/4,
-    # so a surge pod has no card to claim and sits Pending while the old pod is
-    # never torn down — a deadlock. Recreate tears down first, accepting a brief
-    # gap during updates.
-    deploymentStrategy:
-      type: Recreate
-    maxReplicas: 2
-    nodeSelector:
-      kubernetes.io/hostname: worker-1
-    runtimeClassName: nvidia
-    containers:
-      - name: kserve-container
-        image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
-        args:
-          # GPTQ, NOT AWQ. vLLM hard-refuses AWQ below compute capability 75:
-          #   "The quantization method awq is not supported for the current GPU.
-          #    Minimum capability: 75. Current capability: 70."
-          # V100 is sm_70. GPTQ's min capability is 60, so it runs. (gptq_marlin
-          # needs 80 and fp8 needs 80 — both also out.) Same 19.3GB footprint.
-          # desc_act=False in this build: no activation reordering, faster.
-          - --model=unsloth/DeepSeek-R1-Distill-Qwen-32B-bnb-4bit
-          - --served-model-name=reasoning
-          # bitsandbytes nf4. GPTQ passed vLLM's min_capability=60 check but was
-          # numerically WRONG on sm_70 (garbage logits) — proven by the fp16
-          # control run producing correct text with the identical backend. bnb
-          # declares min_capability=70, but treat that as unverified until the
-          # output itself is checked.
-          # NOTE the repo sets bnb_4bit_compute_dtype=bfloat16, which Volta does
-          # not have; --dtype=float16 must override it.
-          - --quantization=bitsandbytes
-          # Volta has no bf16 — must be explicit, the repo's weights are bf16.
-          - --dtype=float16
-          # No FP8 KV on Volta; stays fp16.
-          - --kv-cache-dtype=auto
-          - --tensor-parallel-size=1
-          - --max-model-len=16384
-          # VRAM budget on a 32GiB V100: 0.92 => ~29.4GiB, minus ~18GiB of GPTQ
-          # weights leaves ~11GiB for KV + activations. One full 32K sequence
-          # costs 32768 x 256KB = 8GiB of KV, so 8 concurrent full-length
-          # sequences is not physically possible here — 4 is honest, and a
-          # serial single-user harness never needs more.
-          - --gpu-memory-utilization=0.90
-          - --max-num-seqs=4
-          # Smooths Volta's slow prefill (no FlashAttention2 on sm_70).
-          - --enable-chunked-prefill
-          - --enable-prefix-caching
-          # Splits <think>…</think> into its own channel.
-          - --reasoning-parser=deepseek_r1
-          - --host=0.0.0.0
-          - --port=8080
-        env:
-          # FlashAttention2 requires sm_80; Volta must fall back to xformers.
-          # flashinfer's check_cuda_arch() has an upstream bug that crashes on
-          # ANY sm_7x GPU: `elif major == 7 and minor.isdigit()` calls .isdigit()
-          # on an int, so instead of reporting "unsupported" it raises
-          #   AttributeError: 'int' object has no attribute 'isdigit'
-          # and engine init dies. Default is None (auto-detect), which walks
-          # straight into that path. 0 disables the flashinfer sampler outright.
-          # Only affects the generate runner — the verifier (pooling) never hits
-          # the sampler, which is why it started fine and this did not.
-          - name: VLLM_USE_FLASHINFER_SAMPLER
-            value: "0"
-          # TRITON_ATTN, not XFORMERS. On sm_70 every xformers kernel is
-          # rejected for V1's paged-attention bias type:
-          #   fa2F / triton_splitKF -> require sm_80
-          #   cutlassF              -> supports sm_70 but not
-          #                            PagedBlockDiagonalCausalWithOffsetPaddedKeysMask
-          # -> NotImplementedError kills EngineCore on the FIRST request, which
-          # takes the whole pod down (vLLM treats engine death as fatal).
-          # V0, whose hand-written paged kernels did support sm_70, was REMOVED
-          # in v0.11.0, so VLLM_USE_V1=0 has nothing to fall back to.
-          # Triton JIT-compiles for the local arch, so it is the last option.
-          - name: VLLM_ATTENTION_BACKEND
-            value: TRITON_ATTN
-          - name: HF_HOME
-            value: /mnt/models
-        ports:
-          - containerPort: 8080
-            protocol: TCP
-        resources:
-          requests:
-            cpu: "8"
-            memory: 8Gi
-            nvidia.com/gpu: "1"
-          limits:
-            cpu: "16"
-            memory: 16Gi
-            nvidia.com/gpu: "1"
-        volumeMounts:
-          - name: models
-            mountPath: /mnt/models
-          - name: shm
-            mountPath: /dev/shm
-        startupProbe:
-          httpGet:
-            path: /health
-            port: 8080
-          # Cold start pulls ~18Gi of weights over Longhorn, then loads to VRAM.
-          periodSeconds: 15
-          failureThreshold: 80
-        readinessProbe:
-          httpGet:
-            path: /health
-            port: 8080
-          periodSeconds: 10
-    volumes:
-      - name: models
-        persistentVolumeClaim:
-          claimName: llm-models
-      - name: shm
-        emptyDir:
-          medium: Memory
-          sizeLimit: 2Gi

k8s/apps/llm-serving/inferenceservice-reranker.yaml

@@ -1,71 +0,0 @@
-# Reranker — BAAI/bge-reranker-base, on CPU via HuggingFace TEI.
-#
-# Second stage of retrieval: the embedding model fetches a coarse top-k by
-# vector similarity, this cross-encoder re-scores those candidates against the
-# query directly. That is what fixes the "semantic dilution" problem in Plan 1 —
-# a single embedding vector cannot represent a large chunk faithfully, so
-# ranking by cosine alone surfaces near-misses.
-#
-# CPU for the same reason as the embedding service: all 4 GPUs are claimed and
-# the device plugin allocates whole cards. A 568M cross-encoder scoring ~20-50
-# candidates per query is well within CPU budget.
-#
-# Arch is XLMRobertaForSequenceClassification, which TEI serves as /rerank.
-# 278M params — smaller than v2-m3 (568M) and English/Chinese rather than
-# multilingual, which suits code+docs retrieval and is faster on CPU.
-apiVersion: serving.kserve.io/v1beta1
-kind: InferenceService
-metadata:
-  name: reranker
-  labels:
-    app.kubernetes.io/name: llm-reranker
-    app.kubernetes.io/part-of: llm-serving
-spec:
-  predictor:
-    minReplicas: 1
-    maxReplicas: 1
-    # Same worker-1 pin as the embedding service, to share the RWO models PVC.
-    nodeSelector:
-      kubernetes.io/hostname: worker-1
-    containers:
-      - name: kserve-container
-        image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.8.2@sha256:4d632b76bd14cb57044a1ffb0ad48ab0ba4939e705a9a615ccc740658575c26e
-        args:
-          # bge-reranker-base, NOT v2-m3. TEI's CPU image starts the ONNX
-          # Runtime backend and v2-m3 ships no ONNX files, so it dies with
-          # "Model ONNX files not found in the repository". This build does.
-          - --model-id=BAAI/bge-reranker-base
-          - --port=8080
-          - --hostname=0.0.0.0
-          - --auto-truncate
-        env:
-          - name: HUGGINGFACE_HUB_CACHE
-            value: /mnt/models
-        ports:
-          - containerPort: 8080
-            protocol: TCP
-        resources:
-          requests:
-            cpu: "8"
-            memory: 4Gi
-          limits:
-            cpu: "16"
-            memory: 8Gi
-        volumeMounts:
-          - name: models
-            mountPath: /mnt/models
-        startupProbe:
-          httpGet:
-            path: /health
-            port: 8080
-          periodSeconds: 10
-          failureThreshold: 60
-        readinessProbe:
-          httpGet:
-            path: /health
-            port: 8080
-          periodSeconds: 10
-    volumes:
-      - name: models
-        persistentVolumeClaim:
-          claimName: llm-models

k8s/apps/llm-serving/inferenceservice-verifier.yaml

@@ -1,101 +0,0 @@
-# Verifier — Qwen2.5-Math-PRM-7B, step-level process reward model, vLLM.
-#
-# Model choice was constrained by vLLM v0.11.0's registry: its arch
-# (Qwen2ForProcessRewardModel) is natively registered, whereas the smaller
-# community PRMs are Qwen2ForTokenClassification / Qwen2ForPrmModel, neither of
-# which v0.11.0 can load (ForTokenClassification is absent from
-# _SUFFIX_TO_DEFAULTS, so it won't even auto-convert).
-#
-# --runner pooling, NOT --task reward: --task is [DEPRECATED] in v0.11.0.
-# Scoring goes to /pooling, not /v1/completions — this is a reward model, it
-# returns scores, not tokens.
-#
-# Gets a whole dedicated GPU despite only needing ~15Gi: it sits on the
-# harness's critical path (every reasoning->action->verify iteration waits on
-# it), so isolation from the generation engines' decode loops matters more than
-# the idle VRAM.
-apiVersion: serving.kserve.io/v1beta1
-kind: InferenceService
-metadata:
-  name: verifier
-  labels:
-    app.kubernetes.io/name: llm-verifier
-    app.kubernetes.io/part-of: llm-serving
-spec:
-  predictor:
-    minReplicas: 1
-    # Recreate, not the default RollingUpdate: GPUs are allocated exactly 4/4,
-    # so a surge pod has no card to claim and sits Pending while the old pod is
-    # never torn down — a deadlock. Recreate tears down first, accepting a brief
-    # gap during updates.
-    deploymentStrategy:
-      type: Recreate
-    maxReplicas: 1
-    nodeSelector:
-      kubernetes.io/hostname: worker-1
-    runtimeClassName: nvidia
-    containers:
-      - name: kserve-container
-        image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
-        args:
-          - --model=Qwen/Qwen2.5-Math-PRM-7B
-          - --served-model-name=verifier
-          # Pooling runner => reward scoring. Weights are bf16; Volta needs fp16.
-          - --runner=pooling
-          - --dtype=float16
-          - --tensor-parallel-size=1
-          - --max-model-len=4096
-          - --max-num-seqs=8
-          - --host=0.0.0.0
-          - --port=8080
-        env:
-          # flashinfer's check_cuda_arch() has an upstream bug that crashes on
-          # ANY sm_7x GPU: `elif major == 7 and minor.isdigit()` calls .isdigit()
-          # on an int, so instead of reporting "unsupported" it raises
-          #   AttributeError: 'int' object has no attribute 'isdigit'
-          # and engine init dies. Default is None (auto-detect), which walks
-          # straight into that path. 0 disables the flashinfer sampler outright.
-          # Only affects the generate runner — the verifier (pooling) never hits
-          # the sampler, which is why it started fine and this did not.
-          - name: VLLM_USE_FLASHINFER_SAMPLER
-            value: "0"
-          - name: VLLM_ATTENTION_BACKEND
-            value: XFORMERS
-          - name: HF_HOME
-            value: /mnt/models
-        ports:
-          - containerPort: 8080
-            protocol: TCP
-        resources:
-          requests:
-            cpu: "4"
-            memory: 8Gi
-            nvidia.com/gpu: "1"
-          limits:
-            cpu: "16"
-            memory: 16Gi
-            nvidia.com/gpu: "1"
-        volumeMounts:
-          - name: models
-            mountPath: /mnt/models
-          - name: shm
-            mountPath: /dev/shm
-        startupProbe:
-          httpGet:
-            path: /health
-            port: 8080
-          periodSeconds: 15
-          failureThreshold: 60
-        readinessProbe:
-          httpGet:
-            path: /health
-            port: 8080
-          periodSeconds: 10
-    volumes:
-      - name: models
-        persistentVolumeClaim:
-          claimName: llm-models
-      - name: shm
-        emptyDir:
-          medium: Memory
-          sizeLimit: 1Gi

k8s/apps/llm-serving/kustomization.yaml

@@ -1,13 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-namespace: llm-serving
-resources:
-  - namespace.yaml
-  - storageclass.yaml
-  - pvc-models.yaml
-  - inferenceservice-reasoning.yaml
-  - inferenceservice-ornith.yaml
-  - inferenceservice-verifier.yaml
-  - inferenceservice-embeddings.yaml
-  - inferenceservice-reranker.yaml
-  - networkpolicy.yaml

k8s/apps/llm-serving/namespace.yaml

@@ -1,4 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: llm-serving

k8s/apps/llm-serving/networkpolicy.yaml

@@ -1,43 +0,0 @@
-# Default-deny ingress for the serving pods.
-#
-# This is a real compensating control, not hygiene: vLLM is pinned to v0.11.0
-# (forced — last release with Volta kernels), which sits below the patch line on
-# several advisories that will never be backported to that branch, incl.
-# CVE-2026-54234 (remote DoS) and GHSA-7m6h-x95x-82q5 (cross-user data leak).
-# Those are all remote/unauthenticated attack surface, so keeping the engines
-# reachable only from opted-in in-cluster clients is what keeps exposure low.
-#
-# Consumers opt in with label `llm-client: "true"`. Do NOT expose these via
-# Ingress.
-apiVersion: networking.k8s.io/v1
-kind: NetworkPolicy
-metadata:
-  name: llm-serving-default-deny
-spec:
-  podSelector:
-    matchLabels:
-      app.kubernetes.io/part-of: llm-serving
-  policyTypes:
-    - Ingress
-  ingress:
-    - from:
-        # Any pod, any namespace, that explicitly opts in as an LLM client.
-        - namespaceSelector: {}
-          podSelector:
-            matchLabels:
-              llm-client: "true"
-        # Sibling engines (harness may chain calls between them).
-        - podSelector:
-            matchLabels:
-              app.kubernetes.io/part-of: llm-serving
-      ports:
-        - protocol: TCP
-          port: 8080
-    - from:
-        # Prometheus scraping /metrics.
-        - namespaceSelector:
-            matchLabels:
-              kubernetes.io/metadata.name: monitoring
-      ports:
-        - protocol: TCP
-          port: 8080

k8s/apps/llm-serving/pvc-models.yaml

@@ -1,32 +0,0 @@
-# Shared HuggingFace cache for all three engines.
-#
-# ReadWriteOnce is correct here despite six pods mounting it: RWO means "one
-# NODE", and every pod in this app is pinned to worker-1 via nodeSelector, so
-# they share the volume legally. If a pod is ever allowed onto another node,
-# this must become RWX first.
-#
-# StorageClass is longhorn-llm-local (1 replica, strict-local, disk tag `llm`)
-# — NOT the default 3-replica class, which could not place this volume at all:
-# every control-plane disk was already at its over-provisioning ceiling.
-#
-# Sizing (measured, not estimated):
-#   reasoning GPTQ INT4                19.3 GB
-#   ornith:35b q4 (ollama)             21.0 GB
-#   verifier Qwen2.5-Math-PRM-7B fp16  15.3 GB
-#   nomic-embed-text-v2-moe (CPU)       1.9 GB
-#   bge-reranker-base (CPU)             1.1 GB
-#   ------------------------------------------
-#   total                            ~58.6 GB   (+ HF temp during pulls)
-# The two reasoning replicas share ONE on-disk copy; they differ only in which
-# GPU they load it onto.
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: llm-models
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: longhorn-llm-local
-  resources:
-    requests:
-      storage: 120Gi

k8s/apps/llm-serving/storageclass.yaml

@@ -1,38 +0,0 @@
-# Dedicated StorageClass for model weights on worker-1's local NVMe.
-#
-# Why not the default `longhorn` class (3 replicas, network-attached):
-#
-# 1. numberOfReplicas: 1 — model weights are re-downloadable from HuggingFace.
-#    Replicating them 3x buys nothing; losing a replica costs a re-pull, not
-#    data. The repo's "never delete a PVC without replicas/backups" rule exists
-#    for irreplaceable data, which this is not.
-#
-# 2. dataLocality: strict-local — keeps the single replica on the SAME node as
-#    the pod. All engines are pinned to worker-1, so weights are read from its
-#    local 751GB NVMe instead of over the network from a control-plane node.
-#    Removes ~60GB of network reads on every cold start.
-#
-# 3. diskSelector: llm — restricts this class to disks tagged `llm`, i.e. only
-#    worker-1's disk. Equally important, worker-1's disk carries that tag so
-#    UNTAGGED volumes (any ordinary cluster PVC) will not land on it. Before
-#    tagging, worker-1 had been silently hosting a replica of cicd/runner-dind,
-#    consuming GPU-node storage for general cluster workloads.
-#
-# The default 3-replica class also physically could not place this volume: all
-# three control-plane disks were already at their over-provisioning ceiling
-# (storage-over-provisioning-percentage=100, 30% reserved), so a 120Gi x3
-# request failed with ReplicaSchedulingFailure on every node.
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  name: longhorn-llm-local
-provisioner: driver.longhorn.io
-allowVolumeExpansion: true
-reclaimPolicy: Delete
-volumeBindingMode: Immediate
-parameters:
-  numberOfReplicas: "1"
-  dataLocality: "strict-local"
-  diskSelector: "llm"
-  staleReplicaTimeout: "30"
-  fsType: "ext4"

k8s/apps/sms/deployment-macos.yaml

@@ -0,0 +1,115 @@
+# macOS VM (Docker-OSX) hosting the BlueBubbles server.
+#
+# ── Why a VM and not a container ────────────────────────────────────────────
+# Containers share the host kernel. macOS binaries are Mach-O and need XNU plus
+# Cocoa/IOKit, which a Linux kernel cannot provide, so no macOS container exists
+# or can exist. Docker-OSX is QEMU running a macOS guest, packaged in a
+# container — a VM in a box, not a macOS container.
+#
+# ── Why this works on worker-2 ──────────────────────────────────────────────
+# Verified on the existing hardware: amd64, `vmx` (Intel VT-x) present, and
+# /dev/kvm exists on Talos nodes (KVM is compiled into Talos' kernel, not a
+# module). Bare metal, so no nested virtualisation needed.
+#
+# ── Read this before relying on it ──────────────────────────────────────────
+# 1. Setup is INTERACTIVE. First boot runs the macOS installer: connect over
+#    VNC (:5999), erase the disk in Disk Utility, install, create a user, sign
+#    into iMessage, THEN install BlueBubbles inside the guest. This manifest
+#    only provides the machine; it does not provision macOS.
+# 2. iMessage activation on non-Apple hardware is a coin flip. BlueBubbles'
+#    own guidance: "test sending an iMessage to yourself. If it does not
+#    succeed, it's likely best to restart from the beginning."
+# 3. Apple's macOS licence permits virtualisation only on Apple hardware. This
+#    is a Hackintosh. Use a throwaway Apple ID, not a primary one.
+# 4. BlueBubbles labels this path "not for beginners", "no guarantees or
+#    warranty".
+#
+# Private API (reactions, typing indicators, edit/unsend) needs SIP disabled
+# inside the guest and is NOT required for plain send/receive. Skip it.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: macos-bluebubbles
+  labels:
+    app.kubernetes.io/name: macos-bluebubbles
+    app.kubernetes.io/part-of: sms
+spec:
+  replicas: 1
+  # Recreate: the qcow2 disk is RWO and a second pod must never attach it
+  # concurrently — two QEMU processes on one image corrupts it.
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: macos-bluebubbles
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: macos-bluebubbles
+        app.kubernetes.io/part-of: sms
+    spec:
+      # Dedicated node. The taint keeps everything else off worker-2; this
+      # toleration is what lets the VM on. Both halves are required.
+      nodeSelector:
+        workload: imessage
+      tolerations:
+        - key: workload
+          operator: Equal
+          value: imessage
+          effect: NoSchedule
+      containers:
+        - name: macos
+          image: sickcodes/docker-osx:latest@sha256:3a3c82c79bc4e73531f819ccdfa4053b3084efd7c1f645678dbf8b4b3a24369c
+          # QEMU needs /dev/kvm; Talos enforces `baseline` cluster-wide, so this
+          # only schedules because the sms namespace is labelled privileged.
+          securityContext:
+            privileged: true
+          env:
+            # Generates a unique serial / board-serial / UUID / MAC and persists
+            # them to bootdisk.qcow2. This synthetic identity is what iMessage
+            # activates against — it must stay stable across restarts, which is
+            # why the PVC matters.
+            - name: GENERATE_UNIQUE
+              value: "true"
+            # Identity is only plausible if it matches a real product line.
+            - name: DEVICE_MODEL
+              value: "iMacPro1,1"
+            - name: RAM
+              value: "12"
+            - name: CORES
+              value: "6"
+            - name: EXTRA
+              # Expose the BlueBubbles server port from the guest to the pod.
+              # Guest :1234 (BlueBubbles default) -> pod :1234.
+              value: "-device virtio-net-pci,netdev=net0 -netdev user,id=net0,hostfwd=tcp::1234-:1234"
+          ports:
+            - name: vnc
+              containerPort: 5999
+              protocol: TCP
+            - name: bluebubbles
+              containerPort: 1234
+              protocol: TCP
+          resources:
+            requests:
+              cpu: "6"
+              memory: 14Gi
+            limits:
+              cpu: "12"
+              memory: 20Gi
+          volumeMounts:
+            - name: macos-disk
+              mountPath: /home/arch/OSX-KVM/disk
+            - name: kvm
+              mountPath: /dev/kvm
+          # No readiness probe on purpose. The guest takes many minutes to boot,
+          # and until macOS + BlueBubbles are installed BY HAND there is nothing
+          # listening on 1234. A probe here would crash-loop the pod through the
+          # entire interactive install.
+      volumes:
+        - name: macos-disk
+          persistentVolumeClaim:
+            claimName: macos-disk
+        - name: kvm
+          hostPath:
+            path: /dev/kvm
+            type: CharDevice

k8s/apps/sms/kustomization.yaml

@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: sms
+resources:
+  - namespace.yaml
+  - storageclass.yaml
+  - pvc-macos.yaml
+  - deployment-macos.yaml
+  - service.yaml
+  - networkpolicy.yaml

k8s/apps/sms/namespace.yaml

@@ -0,0 +1,19 @@
+# iMessage delivery for the cluster.
+#
+# BlueBubbles' server is a macOS Electron app paired with an Objective-C helper
+# that hooks Messages.app private APIs — it cannot be containerised on Linux,
+# because containers share the host kernel and macOS needs XNU + Cocoa. The only
+# way to run it on Talos is a full macOS VM under QEMU/KVM (Docker-OSX), which
+# needs a privileged pod with /dev/kvm.
+#
+# Hence privileged PodSecurity: the cluster default from the Talos controlplane
+# is `enforce: baseline`, which forbids privileged containers and host devices.
+# Scope is limited to this namespace.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: sms
+  labels:
+    pod-security.kubernetes.io/enforce: privileged
+    pod-security.kubernetes.io/audit: privileged
+    pod-security.kubernetes.io/warn: privileged

k8s/apps/sms/networkpolicy.yaml

@@ -0,0 +1,22 @@
+# Default-deny. This namespace runs a privileged QEMU VM signed into an Apple
+# ID and exposes an unauthenticated VNC console; nothing should reach it except
+# opted-in clients.
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: sms-default-deny
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/part-of: sms
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector: {}
+          podSelector:
+            matchLabels:
+              sms-client: "true"
+      ports:
+        - protocol: TCP
+          port: 1234

k8s/apps/sms/pvc-macos.yaml

@@ -0,0 +1,25 @@
+# Persistent macOS disk image + generated hardware identity (bootdisk.qcow2).
+#
+# This volume is NOT disposable: it holds the VM's serial number, board serial,
+# UUID and MAC, which together form the identity iMessage was activated against.
+# Losing it means re-running activation, which is the least reliable step of the
+# whole setup.
+#
+# Docker-OSX documents 128GB minimum for the guest image; 200Gi leaves room for
+# the installer, the base system, and qcow2 growth.
+#
+# ⚠️ Single replica (see storageclass.yaml — capacity and IO both rule out 3).
+# Losing worker-2's disk therefore means losing the activated identity and
+# redoing iMessage activation. Once the guest is installed and activated, take
+# a Longhorn snapshot/backup of this volume; that is the only redundancy here.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: macos-disk
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: longhorn-imessage-local
+  resources:
+    requests:
+      storage: 200Gi

k8s/apps/sms/service.yaml

@@ -0,0 +1,31 @@
+# VNC is how you drive the interactive macOS install. Deliberately ClusterIP —
+# it is an unauthenticated console onto a machine holding a live Apple ID
+# session. Reach it with `kubectl port-forward`, never an Ingress.
+apiVersion: v1
+kind: Service
+metadata:
+  name: macos-vnc
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: macos-bluebubbles
+  ports:
+    - name: vnc
+      port: 5999
+      targetPort: vnc
+---
+# The BlueBubbles REST API, once installed inside the guest. This is the stable
+# name cluster services use, so callers never depend on the pod IP or on whether
+# the backend is this VM or a real Mac mini later.
+apiVersion: v1
+kind: Service
+metadata:
+  name: bluebubbles
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: macos-bluebubbles
+  ports:
+    - name: http
+      port: 1234
+      targetPort: bluebubbles

k8s/apps/sms/storageclass.yaml

@@ -0,0 +1,32 @@
+# Dedicated StorageClass for the macOS VM disk.
+#
+# The default `longhorn` class does not work here, for two independent reasons:
+#
+# 1. Replica count. Default is 3, and Longhorn schedules against
+#    storageMaximum - storageScheduled with over-provisioning at 100%. Free
+#    space is cp-1 146Gi / cp-2 8Gi / cp-3 146Gi / worker-1 292Gi, so a 200Gi
+#    volume has only one node that can hold even a single replica — a 3-replica
+#    volume fails outright with ReplicaSchedulingFailure.
+# 2. Binding mode. `Immediate` provisions the volume the moment the PVC is
+#    created, before any pod is scheduled. Combined with strict-local that
+#    pins the data to an arbitrary node, not the one the VM runs on.
+#
+# So: one replica, kept local to the VM, bound only once the pod has a node.
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: longhorn-imessage-local
+provisioner: driver.longhorn.io
+allowVolumeExpansion: true
+reclaimPolicy: Delete
+# The pod is pinned to worker-2 by nodeSelector; wait for it to be scheduled so
+# the replica is placed on that node and not somewhere else.
+volumeBindingMode: WaitForFirstConsumer
+parameters:
+  # A qcow2 backing a live VM is latency-sensitive and rewritten constantly.
+  # Serving it over the network from another node's disk would be the single
+  # worst thing for guest responsiveness, so force it local.
+  numberOfReplicas: "1"
+  dataLocality: "strict-local"
+  staleReplicaTimeout: "30"
+  fsType: "ext4"

k8s/argocd/apps/00-ingress-nginx.yaml

@@ -1,42 +0,0 @@
-# Wave 0 — Nginx Ingress Controller
-# Foundational infrastructure required for all ingress resources and ArgoCD UI access.
-# Must be wave 0 to ensure LoadBalancer IP is available before other apps deploy.
----
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: ingress-nginx
-  namespace: argocd
-  annotations:
-    argocd.argoproj.io/sync-wave: "0"
-  finalizers:
-    - resources-finalizer.argocd.argoproj.io
-spec:
-  project: homelab
-  revisionHistoryLimit: 3
-  sources:
-    - repoURL: https://kubernetes.github.io/ingress-nginx
-      chart: ingress-nginx
-      targetRevision: "4.15.1"
-      helm:
-        valueFiles:
-          - $values/k8s/bootstrap/ingress/nginx-values.yaml
-    - repoURL: [email protected]:Riotpiaole/riotpiao.homelab.com.git
-      targetRevision: main
-      ref: values
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: ingress-nginx
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
-    syncOptions:
-      - CreateNamespace=true
-      - ServerSideApply=true
-    retry:
-      limit: 3
-      backoff:
-        duration: 10s
-        factor: 2
-        maxDuration: 3m

k8s/argocd/apps/05-networking.yaml

@@ -1,26 +1,8 @@
-# Wave 0 — networking policies layered on the Cilium CNI + CoreDNS that the
+# Wave 0 — networking substrate is Talos-owned (terraform inlineManifests), not
-# cluster bootstrap already installed (substrate). These are raw manifests only.
+# ArgoCD:
-apiVersion: argoproj.io/v1alpha1
+#   - CoreDNS Corefile + hostname rewrites  -> terraform/files/coredns/Corefile
-kind: Application
+#   - Cilium LB-IPAM pool + L2 announcement -> terraform/files/cilium/*.yaml
-metadata:
+# Both were previously ArgoCD apps here whose empty `resources: []`
-  name: cilium-policy
+# kustomizations never actually applied them (live objects came from manual
-  namespace: argocd
+# kubectl). Managing them from ArgoCD too would let two reconcilers fight. This
-  annotations:
+# file intentionally defines no Applications now.
-    argocd.argoproj.io/sync-wave: "0"
-spec:
-  project: homelab
-  source:
-    repoURL: [email protected]:Riotpiaole/riotpiao.homelab.com.git
-    targetRevision: main
-    path: k8s/bootstrap/cilium
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: kube-system
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
-# coredns-config Application removed: CoreDNS (incl. homelab hostname rewrites)
-# is owned by Talos via an inlineManifest (terraform/files/coredns/Corefile).
-# Managing the coredns ConfigMap from ArgoCD too would let the two reconcilers
-# fight and revert the rewrites.

k8s/argocd/apps/55-api-gateway.yaml

@@ -0,0 +1,58 @@
+# Wave 7 — Kong, the cluster's internal API gateway (namespace `api`).
+#
+# Sits between nginx and the backend services: nginx owns the edge and TLS,
+# Kong owns routing policy, auth and rate limiting. Wave 7 puts it after the
+# data/messaging tiers it fronts and before the wave-8 applications that
+# publish routes into it.
+#
+# DB-less: routing config comes from Kubernetes objects (Ingress with
+# `ingressClassName: kong`, plus KongPlugin/KongConsumer CRDs), so git remains
+# the source of truth and there are no migration Jobs on upgrade.
+#
+# CRDs ship in the chart's crds/ directory; ArgoCD applies those by default
+# (helm.skipCrds is left false).
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: kong
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "7"
+spec:
+  project: homelab
+  revisionHistoryLimit: 3
+  sources:
+    - repoURL: https://charts.konghq.com
+      chart: kong
+      targetRevision: "3.4.1"
+      helm:
+        valueFiles:
+          - $values/k8s/apps/api/kong-values.yaml
+    - repoURL: [email protected]:Riotpiaole/riotpiao.homelab.com.git
+      targetRevision: main
+      ref: values
+    # The nginx Ingress for api.riotpiao.com. Kept in this Application rather
+    # than the central k8s/bootstrap/ingress/ingress.yaml because that one syncs
+    # in wave 1, before namespace `api` exists.
+    - repoURL: [email protected]:Riotpiaole/riotpiao.homelab.com.git
+      targetRevision: main
+      path: k8s/apps/api
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: api
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      # The chart's CRDs exceed the annotation size limit that client-side
+      # apply relies on; server-side apply avoids the
+      # "metadata.annotations: Too long" failure CRDs commonly hit.
+      - ServerSideApply=true
+    retry:
+      limit: 3
+      backoff:
+        duration: 10s
+        factor: 2
+        maxDuration: 3m

k8s/argocd/apps/60-applications.yaml

@@ -84,6 +84,37 @@ spec:
     syncOptions:
       - CreateNamespace=true
 ---
+# iMessage/SMS delivery. Raw manifests: a privileged macOS VM (Docker-OSX)
+# running the BlueBubbles server, plus its dedicated local StorageClass.
+#
+# Pinned to worker-2 via nodeSelector `workload: imessage` + a matching
+# toleration for that node's taint. Until worker-2 is provisioned this app
+# syncs everything except the pod, which stays Pending — that is expected.
+#
+# No CreateNamespace: namespace.yaml carries `pod-security: privileged`, which
+# the VM needs (/dev/kvm, privileged), and an ArgoCD-created namespace would
+# not have those labels.
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: sms
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "8"
+spec:
+  project: homelab
+  source:
+    repoURL: [email protected]:Riotpiaole/riotpiao.homelab.com.git
+    targetRevision: main
+    path: k8s/apps/sms
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: sms
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+---
 # Consolidated: homarr + homarr-patches → homarr
 # Helm chart + values + PostSync hook patch (fix-probes-job.yaml)
 apiVersion: argoproj.io/v1alpha1

k8s/argocd/apps/70-gpu-serving.yaml

@@ -1,156 +0,0 @@
-# Wave 9-11 — GPU serving stack on worker-1 (4x Tesla V100 32GB).
-#
-# Ordering matters: device plugin must expose nvidia.com/gpu and the KServe CRDs
-# must exist before any InferenceService is applied, hence three waves.
-#
-# NOTE the chart versions below are v-PREFIXED (v0.15.2, not 0.15.2) — that is
-# how the KServe OCI tags are published; the unprefixed form 404s.
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: gpu-runtimeclass
-  namespace: argocd
-  annotations:
-    # Wave 8: must precede the device plugin, whose DaemonSet sets
-    # runtimeClassName: nvidia and is rejected at admission if the
-    # RuntimeClass does not exist yet.
-    argocd.argoproj.io/sync-wave: "8"
-spec:
-  project: homelab
-  source:
-    repoURL: [email protected]:Riotpiaole/riotpiao.homelab.com.git
-    targetRevision: main
-    path: k8s/apps/gpu-runtimeclass
-  destination:
-    server: https://kubernetes.default.svc
-    # Cluster-scoped resource; namespace is only the app's default context.
-    namespace: gpu-system
-  syncPolicy:
-    syncOptions:
-      - CreateNamespace=true
----
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: nvidia-device-plugin
-  namespace: argocd
-  annotations:
-    argocd.argoproj.io/sync-wave: "9"
-spec:
-  project: homelab
-  source:
-    repoURL: https://nvidia.github.io/k8s-device-plugin
-    chart: nvidia-device-plugin
-    targetRevision: "0.19.3"
-    helm:
-      values: |
-        # Driver + container toolkit are supplied by Talos system extensions
-        # baked into the installer image (nonfree-kmod-nvidia-lts /
-        # nvidia-container-toolkit-lts). This chart ONLY advertises the GPUs to
-        # the kubelet — it does not and must not install drivers.
-        runtimeClassName: nvidia
-        nodeSelector:
-          nvidia.com/gpu: "true"
-        # Drop the chart's default nodeAffinity. It requires one of three
-        # Node-Feature-Discovery labels (feature.node.kubernetes.io/pci-10de.present,
-        # .../cpu-model.vendor_id=NVIDIA, or nvidia.com/gpu.present). NFD is not
-        # installed and Talos sets nvidia.com/gpu (no ".present" suffix), so the
-        # affinity matches zero nodes and the DaemonSet silently reports
-        # desiredNumberScheduled=0 with no events. nodeSelector is the constraint.
-        affinity: null
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: gpu-system
-  syncPolicy:
-    # Manual sync for first bring-up: watch device-plugin -> KServe -> models
-    # come up in order, and avoid auto-deploying while worker-1 is cordoned.
-    # Switch to `automated: {prune: true, selfHeal: true}` once proven.
-    syncOptions:
-      - CreateNamespace=true
----
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: kserve-crd
-  namespace: argocd
-  annotations:
-    argocd.argoproj.io/sync-wave: "9"
-spec:
-  project: homelab
-  source:
-    repoURL: oci://ghcr.io/kserve/charts
-    chart: kserve-crd
-    targetRevision: v0.15.2
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: kserve
-  syncPolicy:
-    # Manual sync for first bring-up: watch device-plugin -> KServe -> models
-    # come up in order, and avoid auto-deploying while worker-1 is cordoned.
-    # Switch to `automated: {prune: true, selfHeal: true}` once proven.
-    syncOptions:
-      - CreateNamespace=true
-      # InferenceService CRD exceeds the annotation size limit for
-      # client-side apply.
-      - ServerSideApply=true
----
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: kserve
-  namespace: argocd
-  annotations:
-    argocd.argoproj.io/sync-wave: "10"
-spec:
-  project: homelab
-  source:
-    repoURL: oci://ghcr.io/kserve/charts
-    chart: kserve
-    targetRevision: v0.15.2
-    helm:
-      values: |
-        kserve:
-          controller:
-            # RawDeployment => plain Deployments/Services, no Knative, no Istio.
-            # v0.18 renames this mode "Standard"; do not bump without checking.
-            deploymentMode: RawDeployment
-            gateway:
-              ingressGateway:
-                # Route through the existing ingress-nginx, not Gateway API.
-                # NOTE the nesting: it is gateway.ingressGateway.enableGatewayApi,
-                # not gateway.enableGatewayApi — Helm silently ignores the wrong
-                # key rather than erroring.
-                enableGatewayApi: false
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: kserve
-  syncPolicy:
-    # Manual sync for first bring-up: watch device-plugin -> KServe -> models
-    # come up in order, and avoid auto-deploying while worker-1 is cordoned.
-    # Switch to `automated: {prune: true, selfHeal: true}` once proven.
-    syncOptions:
-      - CreateNamespace=true
-      - ServerSideApply=true
----
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: llm-serving
-  namespace: argocd
-  annotations:
-    argocd.argoproj.io/sync-wave: "11"
-spec:
-  project: homelab
-  source:
-    repoURL: [email protected]:Riotpiaole/riotpiao.homelab.com.git
-    targetRevision: main
-    path: k8s/apps/llm-serving
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: llm-serving
-  syncPolicy:
-    # Manual sync for first bring-up: watch device-plugin -> KServe -> models
-    # come up in order, and avoid auto-deploying while worker-1 is cordoned.
-    # Switch to `automated: {prune: true, selfHeal: true}` once proven.
-    syncOptions:
-      - CreateNamespace=true

k8s/bootstrap/cilium/kustomization.yaml

@@ -1,5 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-namespace: kube-system
-resources: []
-# Cilium deployed via Helm chart

k8s/bootstrap/cilium/l2-announcement-policy.yaml

@@ -1,39 +0,0 @@
-# k8s/cilium/l2-announcement-policy.yaml
-# CiliumL2AnnouncementPolicy — without this, LB-IPAM (lb-ipam-pool.yaml)
-# assigns IPs to LoadBalancer Services but nothing ARPs for them on the LAN,
-# so they're unreachable from outside the cluster even though `kubectl get
-# svc` shows a real EXTERNAL-IP. Confirmed both forgejo's .165 and
-# shadowsocks' .166 were 100% packet loss / incomplete ARP before this.
-#
-# loadBalancerIPs: true makes Cilium announce every Service's LB-IPAM IP via
-# ARP from whichever node currently holds the lease for it (one node per IP,
-# decided by leaderElection — not all nodes simultaneously, which would
-# otherwise cause ARP flapping/duplicate-IP confusion on the LAN).
-#
-# externalIPs/loadBalancerIPs split exists because Cilium also supports
-# announcing Service externalIPs (a different field, unused in this repo);
-# we only need loadBalancerIPs since every exposed Service here is type
-# LoadBalancer via lb-ipam-pool.yaml.
-#
-# requires kube-proxy replacement (already the case — see
-# k8s/talos-iam or helmfile.yaml.gotmpl kubeProxyReplacement=true) and a
-# Cilium build with L2 announcements enabled (default since v1.14).
-#
-# Apply once after cluster bootstrap, alongside lb-ipam-pool.yaml:
-#   kubectl apply -f k8s/cilium/l2-announcement-policy.yaml
-#
-# Verify:
-#   kubectl get ciliuml2announcementpolicy
-#   ping 192.168.1.165 && ping 192.168.1.166   # both should now respond
-#   arp -a | grep 192.168.1.16                 # should resolve to a real MAC
-apiVersion: cilium.io/v2alpha1
-kind: CiliumL2AnnouncementPolicy
-metadata:
-  name: homelab-l2-announce
-spec:
-  loadBalancerIPs: true
-  interfaces:
-    - eno1
-  # No nodeSelector restriction — all 3 nodes already run workloads
-  # (allowSchedulingOnControlPlanes: true in controlplane.yaml), and with
-  # 3 zone-labeled nodes, redundancy for per-IP leader election is maintained.

k8s/bootstrap/cilium/lb-ipam-pool.yaml

@@ -1,36 +0,0 @@
-# k8s/cilium/lb-ipam-pool.yaml
-# CiliumLoadBalancerIPPool — tells Cilium LB-IPAM which IPs it can assign
-# to LoadBalancer services in this cluster.
-#
-# CIDR 192.168.1.160/28 covers .160–.175 on the LAN:
-#   .160  talos-cp-1       (node — not assignable to services)
-#   .161  reserved
-#   .162  talos-worker-1   (node — not assignable to services)
-#   .163–.175  free for LoadBalancer services
-#
-# Current service IP assignments (via io.cilium/lb-ipam-ips annotation):
-#   192.168.1.165  forgejo-gitea-http (cicd)
-#   192.168.1.165  forgejo-gitea-ssh  (cicd)  — same IP, different ports
-#   192.168.1.166  shadowsocks        (vpn)
-#
-# Apply once after cluster bootstrap:
-#   kubectl apply -f k8s/cilium/lb-ipam-pool.yaml
-#
-# Verify assignment:
-#   kubectl get svc -n cicd forgejo-gitea-http forgejo-gitea-ssh
-#   # EXTERNAL-IP should change from <pending> to 192.168.1.165
-
-apiVersion: "cilium.io/v2alpha1"
-kind: CiliumLoadBalancerIPPool
-metadata:
-  name: homelab-pool
-spec:
-  blocks:
-    - cidr: "192.168.1.160/28"
-    # DO NOT add any 10.6.0.0/24 block here. That is the WireGuard subnet
-    # (10.6.0.1 = talos-cp-1 tunnel IP, 10.6.0.2 = DNS — see
-    # cluster-config/controlplane.yaml). A 10.6.0.x block let Cilium LB-IPAM
-    # auto-assign the CP's own tunnel IP to a Service, which broke the
-    # WireGuard tunnel and locked out the default kubectl context. It also
-    # can't work over WireGuard anyway — L2 announcements only ARP on eno1
-    # (the LAN interface), not wg0. Keep this pool LAN-only.

k8s/bootstrap/ingress/ingress-nginx-controller-alias.yaml

@@ -1,22 +0,0 @@
-# Service alias for CoreDNS compatibility
-# CoreDNS rewrites *.riotpiao.com → ingress-nginx-controller.ingress-nginx.svc
-# But bootstrap deployed as ingress-nginx-bootstrap-controller
-# This alias makes both names work
-apiVersion: v1
-kind: Service
-metadata:
-  name: ingress-nginx-controller
-  namespace: ingress-nginx
-spec:
-  type: ClusterIP
-  selector:
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/instance: ingress-nginx-bootstrap
-    app.kubernetes.io/component: controller
-  ports:
-  - name: http
-    port: 80
-    targetPort: http
-  - name: https
-    port: 443
-    targetPort: https

k8s/bootstrap/ingress/ingress.yaml

@@ -300,6 +300,10 @@ spec:
                 port:
                   number: 8080
 ---
+# NOTE: api.riotpiao.com (Kong) is deliberately NOT here. Its namespace `api` is
+# created in wave 7, and this Application syncs in wave 1 — an Ingress into a
+# namespace that doesn't exist yet would fail and mark this whole app
+# SyncFailed. It lives in k8s/apps/api/ingress.yaml, synced with Kong itself.
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:

k8s/bootstrap/ingress/kustomization.yaml

@@ -2,6 +2,12 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 # No top-level namespace - resources declare their own namespaces
 resources:
-  - ingress-nginx-controller-alias.yaml  # Service alias for CoreDNS compatibility
+  # ingress-nginx-controller-alias.yaml REMOVED — it was a ClusterIP Service named
+  # ingress-nginx-controller with a stale selector (instance: ingress-nginx-bootstrap,
+  # a release that no longer exists). ingress-config's selfHeal kept re-applying it
+  # over the helm release's real LoadBalancer Service of the same name, reverting it
+  # to a ClusterIP with zero endpoints -> LB IP .160 unannounced -> cluster-wide
+  # outage. CoreDNS rewrites *.riotpiao.com to ingress-nginx-controller.ingress-nginx
+  # .svc, which is the helm Service directly — no alias needed.
   - riotpiao-com-cert.yaml                # Certificate for *.riotpiao.com (ingress-nginx namespace)
   - ingress.yaml                           # Ingress rules for all services (multiple namespaces)

k8s/bootstrap/phase3-forgejo/forgejo-values.yaml

@@ -14,6 +14,18 @@ valkey-cluster:
 redis:
   enabled: false
 
+# External SSH access for git over the LAN. The chart's ssh Service becomes a
+# LoadBalancer with a stable IP from the Cilium homelab-pool (192.168.1.160/28,
+# L2-announced) so `git clone ssh://[email protected]:2222/...` works from the
+# LAN. gitea's sshd listens on 2222 in-pod; port 2222 is exposed directly to
+# avoid needing privileged :22.
+service:
+  ssh:
+    type: LoadBalancer
+    port: 2222
+    annotations:
+      lbipam.cilium.io/ips: "192.168.1.161"
+
 gitea:
   admin:
     existingSecret: forgejo-admin
@@ -22,8 +34,10 @@ gitea:
     server:
       DOMAIN: forgejo.riotpiao.com
       ROOT_URL: https://forgejo.riotpiao.com
-      SSH_DOMAIN: forgejo.riotpiao.com
+      # SSH clone URLs advertise git.riotpiao.com:2222 (the LoadBalancer above).
-      SSH_PORT: 22
+      SSH_DOMAIN: git.riotpiao.com
+      SSH_PORT: 2222
+      SSH_LISTEN_PORT: 2222
       
     database:
       DB_TYPE: postgres

k8s/infra/monitoring/alerts/svc-forgejo-rules.yaml

@@ -2,7 +2,7 @@ apiVersion: monitoring.coreos.com/v1
 kind: PrometheusRule
 metadata:
   name: forgejo-rules
-  namespace: forgejo
+  namespace: cicd
 spec:
   groups:
     - name: forgejo.rules

k8s/infra/monitoring/prometheus-values.yaml

@@ -20,8 +20,36 @@ grafana:
   enabled: false
 
 # ── Alertmanager ──────────────────────────────────────────────────────────────
+# Enabled with a default (null) receiver — every firing PrometheusRule lands in
+# the Alertmanager UI and Grafana's Alerting view; no external Slack/email/
+# PagerDuty notifier is wired yet (add a receiver + route later). Storage pinned
+# to az-a (sole Longhorn node) like Prometheus so the RWO PVC can attach.
 alertmanager:
-  enabled: false
+  enabled: true
+  alertmanagerSpec:
+    nodeSelector:
+      topology.kubernetes.io/zone: az-a
+    tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+        effect: NoSchedule
+    storage:
+      volumeClaimTemplate:
+        spec:
+          storageClassName: longhorn
+          accessModes: ["ReadWriteOnce"]
+          resources:
+            requests:
+              storage: 2Gi
+  config:
+    route:
+      group_by: ["alertname", "namespace"]
+      group_wait: 30s
+      group_interval: 5m
+      repeat_interval: 4h
+      receiver: "null"
+    receivers:
+      - name: "null"
 
 # ── Prometheus ────────────────────────────────────────────────────────────────
 prometheus:

scripts/deploy-gpu-serving.sh

@@ -1,381 +0,0 @@
-#!/usr/bin/env bash
-#
-# Deploy the GPU serving stack on worker-1:
-#   NVIDIA device plugin -> KServe (RawDeployment) -> 3 engines
-#     GPU0,1  vLLM v0.11.0  deepseek-r1-distill-qwen-32b-awq  (2 replicas)
-#     GPU2    Ollama        ornith:35b
-#     GPU3    vLLM v0.11.0  Qwen2.5-Math-PRM-7B  (reward model)
-#
-# Two modes:
-#   argocd (default) -- syncs the ArgoCD Applications from
-#                       k8s/argocd/apps/70-gpu-serving.yaml. Git is the source
-#                       of truth; this only sequences the syncs.
-#   manual  (--manual) -- bootstraps directly with helm + kubectl, for first
-#                       bring-up before the Applications are committed. Applies
-#                       the SAME manifests from k8s/apps/llm-serving/, so ArgoCD
-#                       adopts them cleanly later (matches the repo's existing
-#                       k8s/bootstrap/phaseN-* pattern).
-#
-# Usage:
-#   ./scripts/deploy-gpu-serving.sh --manual              # full manual bootstrap
-#   ./scripts/deploy-gpu-serving.sh --manual gpu-plugin   # one stage
-#   ./scripts/deploy-gpu-serving.sh --manual --dry-run    # show, don't run
-#   ./scripts/deploy-gpu-serving.sh                       # via ArgoCD
-#
-set -euo pipefail
-
-NODE=worker-1
-NS=llm-serving
-KSERVE_NS=kserve
-GPU_NS=gpu-system
-KSERVE_VER=v0.15.2       # v-PREFIXED; the unprefixed tag 404s
-NVDP_VER=0.19.3
-REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
-MODE=argocd
-DRY=false
-
-log()  { printf '\n\033[1m==> %s\033[0m\n' "$*"; }
-info() { printf '    %s\n' "$*"; }
-die()  { printf '\033[31mERROR: %s\033[0m\n' "$*" >&2; exit 1; }
-run()  { if $DRY; then info "[dry-run] $*"; else "$@"; fi; }
-need() { command -v "$1" >/dev/null 2>&1 || die "missing required tool: $1"; }
-
-sync_app() {
-  local app=$1
-  log "argocd sync: $app"
-  run argocd app sync "$app" --timeout 600
-  run argocd app wait "$app" --health --timeout 600
-}
-
-# ---------------------------------------------------------------- stages
-
-preflight() {
-  log "preflight"
-  need kubectl
-  [[ $MODE == argocd ]] && need argocd
-  [[ $MODE == manual ]] && need helm
-
-  kubectl get node "$NODE" >/dev/null 2>&1 || die "node $NODE not found"
-  local ready
-  ready=$(kubectl get node "$NODE" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}')
-  [[ $ready == True ]] || die "node $NODE is not Ready"
-  info "node $NODE: Ready"
-
-  # Driver ships in the Talos installer image as a system extension, not from
-  # the cluster. Missing label => node was built from the wrong schematic and
-  # no amount of device-plugin will help.
-  kubectl get node "$NODE" -o jsonpath='{.metadata.labels}' | grep -q 'nonfree-kmod-nvidia' \
-    || die "NVIDIA driver extension label absent — worker-1 not on the GPU schematic"
-  info "NVIDIA driver extension: present"
-
-  kubectl get node "$NODE" -o jsonpath='{.metadata.labels.nvidia\.com/gpu}' 2>/dev/null | grep -q true \
-    || die "node label nvidia.com/gpu=true missing — device plugin nodeSelector will not match"
-  info "node label nvidia.com/gpu=true: present"
-
-  [[ $(kubectl get node "$NODE" -o jsonpath='{.spec.unschedulable}') == true ]] \
-    && info "node is cordoned (expected; uncordoned before engines deploy)"
-  info "mode: $MODE"
-}
-
-runtimeclass() {
-  # Must exist before ANY pod with runtimeClassName: nvidia is admitted —
-  # including the device plugin itself. The Talos extension registers the
-  # containerd handler but not this object.
-  if [[ $MODE == manual ]]; then
-    log "apply RuntimeClass nvidia"
-    run kubectl apply -k "$REPO_ROOT/k8s/apps/gpu-runtimeclass"
-  else
-    sync_app gpu-runtimeclass
-  fi
-  $DRY || kubectl get runtimeclass nvidia >/dev/null 2>&1 \
-    && info "RuntimeClass nvidia: present"
-}
-
-gpu_plugin() {
-  runtimeclass
-  if [[ $MODE == manual ]]; then
-    log "helm install: nvidia-device-plugin $NVDP_VER"
-    # Driver + container toolkit come from Talos system extensions. This chart
-    # only advertises the GPUs to the kubelet — it must not install drivers.
-    run helm upgrade --install nvidia-device-plugin nvidia-device-plugin \
-      --repo https://nvidia.github.io/k8s-device-plugin \
-      --version "$NVDP_VER" \
-      -n "$GPU_NS" --create-namespace \
-      --set runtimeClassName=nvidia \
-      `# --set-string, NOT --set: nodeSelector is map[string]string in the` \
-      `# PodSpec schema, and plain --set coerces "true" to a YAML boolean,` \
-      `# which the API server rejects as a type violation.` \
-      --set-string nodeSelector."nvidia\.com/gpu"=true \
-      `# Drop the chart's default nodeAffinity. It requires one of three` \
-      `# Node-Feature-Discovery labels (feature.node.kubernetes.io/pci-10de.present,` \
-      `# .../cpu-model.vendor_id=NVIDIA, or nvidia.com/gpu.present). NFD is not` \
-      `# installed here and Talos sets nvidia.com/gpu (no ".present" suffix), so` \
-      `# the affinity matches zero nodes and the DaemonSet silently reports` \
-      `# desiredNumberScheduled=0. The nodeSelector above is our constraint.` \
-      `# Deliberately NO --wait: it blocks on DaemonSet readiness, and any` \
-      `# admission failure then times helm out and wedges the release in` \
-      `# pending-upgrade, blocking all later upgrades. The allocatable-GPU` \
-      `# poll below is the real readiness signal.` \
-      --set affinity=null
-  else
-    sync_app nvidia-device-plugin
-  fi
-
-  log "waiting for nvidia.com/gpu to register on $NODE"
-  $DRY && { info "[dry-run] would verify allocatable == 4"; return 0; }
-  local n=""
-  for _ in $(seq 1 60); do
-    n=$(kubectl get node "$NODE" -o jsonpath='{.status.allocatable.nvidia\.com/gpu}' 2>/dev/null || true)
-    [[ -n $n && $n != 0 ]] && break
-    sleep 5
-  done
-  [[ $n == 4 ]] || die "expected 4 allocatable GPUs, got '${n:-none}'"
-  info "allocatable nvidia.com/gpu: $n"
-}
-
-kserve() {
-  if [[ $MODE == manual ]]; then
-    log "helm install: kserve-crd $KSERVE_VER"
-    # No --wait on helm (a timeout leaves the release wedged in pending-upgrade,
-    # blocking every later upgrade). kubectl wait below is the readiness signal
-    # and does not touch helm release state.
-    run helm upgrade --install kserve-crd "oci://ghcr.io/kserve/charts/kserve-crd" \
-      --version "$KSERVE_VER" -n "$KSERVE_NS" --create-namespace
-
-    run kubectl wait --for=condition=established --timeout=120s \
-      crd/inferenceservices.serving.kserve.io
-
-    log "helm install: kserve $KSERVE_VER (RawDeployment)"
-    # NOTE the nesting on enableGatewayApi: gateway.ingressGateway.enableGatewayApi.
-    # Helm silently ignores a wrong key rather than erroring.
-    run helm upgrade --install kserve "oci://ghcr.io/kserve/charts/kserve" \
-      --version "$KSERVE_VER" -n "$KSERVE_NS" \
-      --set kserve.controller.deploymentMode=RawDeployment \
-      --set kserve.controller.gateway.ingressGateway.enableGatewayApi=false
-    # Controller readiness via kubectl, not helm --wait, for the same reason.
-    run kubectl rollout status deploy -n "$KSERVE_NS" --timeout=600s
-  else
-    sync_app kserve-crd
-    run kubectl wait --for=condition=established --timeout=120s \
-      crd/inferenceservices.serving.kserve.io
-    sync_app kserve
-  fi
-  info "KServe ready (RawDeployment mode)"
-}
-
-uncordon() {
-  log "uncordon $NODE"
-  # Must precede the engines. The device-plugin DaemonSet tolerates the
-  # unschedulable taint automatically; the engine Deployments do not and would
-  # sit Pending forever.
-  run kubectl uncordon "$NODE"
-}
-
-engines() {
-  if [[ $MODE == manual ]]; then
-    log "kubectl apply -k k8s/apps/llm-serving"
-    run kubectl apply -k "$REPO_ROOT/k8s/apps/llm-serving"
-  else
-    sync_app llm-serving
-  fi
-
-  log "waiting for InferenceServices (first start pulls ~60GB of weights)"
-  run kubectl wait --for=condition=Ready --timeout=2400s \
-    inferenceservice --all -n "$NS" \
-    || info "not all Ready yet — check: kubectl get pods -n $NS"
-}
-
-smoke() {
-  log "smoke test"
-  $DRY && { info "[dry-run] would curl the three engines"; return 0; }
-
-  # Engines are ClusterIP behind a default-deny NetworkPolicy, so probe from
-  # inside the cluster with the llm-client label that the policy allows.
-  #
-  # KServe names the Service "<isvc>-predictor" (constants.PredictorServiceName)
-  # on port 80 -> containerPort 8080 — NOT "<isvc>" on 8080.
-  probe() {
-    local isvc=$1 path=$2 body=$3
-    printf '    %-24s ' "$isvc"
-    if kubectl run "smoke-$RANDOM" -n "$NS" --rm -i --restart=Never -q \
-         --image=curlimages/curl:8.11.1 \
-         --labels="llm-client=true" \
-         --command -- curl -sf -m 300 -X POST \
-         "http://${isvc}-predictor.${NS}.svc.cluster.local/${path}" \
-         -H 'Content-Type: application/json' -d "$body" >/dev/null 2>&1; then
-      printf '\033[32mOK\033[0m\n'
-    else
-      printf '\033[31mFAIL\033[0m\n'
-    fi
-  }
-
-  # Thinking models (reasoning, ornith) spend their first tokens inside a
-  # <think> block, so a small max_tokens returns EMPTY content and looks like a
-  # failure. Give them room.
-  probe reasoning v1/completions \
-    '{"model":"reasoning","prompt":"2+2=","max_tokens":200}'
-  probe ornith    v1/chat/completions \
-    '{"model":"ornith:35b","messages":[{"role":"user","content":"hi"}],"max_tokens":400}'
-  # Same Ollama endpoint, second co-resident model on GPU2.
-  probe ornith    v1/chat/completions \
-    '{"model":"qwen2.5:3b-instruct","messages":[{"role":"user","content":"hi"}],"max_tokens":64}'
-  # Reward model: scores via /pooling with an `input` field, NOT /v1/completions.
-  probe verifier  pooling \
-    '{"model":"verifier","input":"Step 1: 2+2=4."}'
-  # TEI encoders (CPU): /embed and /rerank, not an OpenAI-shaped API.
-  # Nomic v2 needs the search_document:/search_query: prefix from the client.
-  probe embeddings embed \
-    '{"inputs":"search_query: hello"}'
-  probe reranker   rerank \
-    '{"query":"how to sort a list","texts":["use sorted()","unrelated text"]}'
-}
-
-status() {
-  log "status"
-  kubectl get node "$NODE" -o wide
-  echo
-  kubectl get inferenceservice -n "$NS" 2>/dev/null || info "no InferenceServices yet"
-  echo
-  kubectl get pods -n "$NS" -o wide 2>/dev/null || true
-  echo
-  info "GPU allocatable: $(kubectl get node "$NODE" -o jsonpath='{.status.allocatable.nvidia\.com/gpu}' 2>/dev/null || echo none)"
-}
-
-# Every check that this bring-up has actually needed, in one command, so
-# troubleshooting never requires ad-hoc kubectl archaeology again. Read-only.
-doctor() {
-  log "doctor — full diagnostic"
-  set +e   # advisory only: never abort on an absent resource
-  local fail=0
-  chk() { # chk <label> <expected> <actual>
-    if [[ "$2" == "$3" ]]; then printf '    \033[32m✓\033[0m %-38s %s\n' "$1" "$3"
-    else printf '    \033[31m✗\033[0m %-38s got=%s want=%s\n' "$1" "${3:-<empty>}" "$2"; fail=$((fail+1)); fi
-  }
-
-  echo "  [node]"
-  chk "Ready" True "$(kubectl get node "$NODE" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null)"
-  local cordon; cordon=$(kubectl get node "$NODE" -o jsonpath='{.spec.unschedulable}' 2>/dev/null || true)
-  printf '    %-40s %s\n' "cordoned" "${cordon:-false} (must be false before engines)"
-  chk "driver extension label" present \
-    "$(kubectl get node "$NODE" -o jsonpath='{.metadata.labels}' 2>/dev/null | grep -q nonfree-kmod-nvidia && echo present)"
-  chk "label nvidia.com/gpu" true "$(kubectl get node "$NODE" -o jsonpath='{.metadata.labels.nvidia\.com/gpu}' 2>/dev/null)"
-  chk "allocatable nvidia.com/gpu" 4 "$(kubectl get node "$NODE" -o jsonpath='{.status.allocatable.nvidia\.com/gpu}' 2>/dev/null)"
-
-  echo "  [containerd / runtime]"
-  # The Talos extension registers the handler; the RuntimeClass object is ours.
-  chk "RuntimeClass nvidia" nvidia "$(kubectl get runtimeclass nvidia -o jsonpath='{.handler}' 2>/dev/null)"
-
-  echo "  [device plugin]"
-  local st; st=$(helm list -n "$GPU_NS" -a -o json 2>/dev/null \
-    | python3 -c 'import json,sys;print(next((r["status"] for r in json.load(sys.stdin) if r["name"]=="nvidia-device-plugin"),""))' 2>/dev/null || true)
-  chk "helm release status" deployed "$st"
-  [[ $st == pending-* || $st == failed ]] && \
-    info "  -> stuck release: run '$0 unstick' (helm refuses upgrades in this state)"
-  chk "DaemonSet desired" 1 "$(kubectl get ds -n "$GPU_NS" nvidia-device-plugin -o jsonpath='{.status.desiredNumberScheduled}' 2>/dev/null)"
-  chk "DaemonSet ready"   1 "$(kubectl get ds -n "$GPU_NS" nvidia-device-plugin -o jsonpath='{.status.numberReady}' 2>/dev/null)"
-  local aff; aff=$(kubectl get ds -n "$GPU_NS" nvidia-device-plugin -o jsonpath='{.spec.template.spec.affinity}' 2>/dev/null || true)
-  [[ -z $aff ]] && printf '    \033[32m✓\033[0m %-38s removed\n' "chart nodeAffinity (needs NFD)" \
-                || { printf '    \033[31m✗\033[0m %-38s present — requires NFD labels, will match 0 nodes\n' "chart nodeAffinity"; fail=$((fail+1)); }
-  local ev; ev=$(kubectl get events -n "$GPU_NS" --field-selector reason=FailedCreate \
-    -o jsonpath='{.items[-1:].message}' 2>/dev/null || true)
-  [[ -n $ev ]] && info "  last FailedCreate: ${ev:0:110}"
-
-  echo "  [kserve]"
-  chk "InferenceService CRD" true \
-    "$(kubectl get crd inferenceservices.serving.kserve.io >/dev/null 2>&1 && echo true)"
-  local dm; dm=$(kubectl get cm inferenceservice-config -n "$KSERVE_NS" -o jsonpath='{.data.deploy}' 2>/dev/null \
-    | grep -o '"defaultDeploymentMode": *"[^"]*"' | sed 's/.*"\([^"]*\)"$/\1/' || true)
-  chk "deploymentMode" RawDeployment "$dm"
-
-  echo "  [engines]"
-  if kubectl get ns "$NS" >/dev/null 2>&1; then
-    kubectl get inferenceservice -n "$NS" --no-headers 2>/dev/null | awk '{printf "    %-24s ready=%s\n",$1,$3}'
-    kubectl get pods -n "$NS" --no-headers 2>/dev/null \
-      | awk '{printf "    %-40s %s %s\n",$1,$3,$5}'
-  else
-    info "namespace $NS absent (engines not deployed yet)"
-  fi
-
-  echo
-  [[ $fail -eq 0 ]] && info "all checks passed" || info "$fail check(s) failed"
-  set -e
-  return 0
-}
-
-# Clear a helm release wedged in pending-* / failed. Helm blocks every
-# subsequent upgrade with "another operation is in progress" until this is done.
-unstick() {
-  log "unstick helm release"
-  local st; st=$(helm list -n "$GPU_NS" -a -o json 2>/dev/null \
-    | python3 -c 'import json,sys;print(next((r["status"] for r in json.load(sys.stdin) if r["name"]=="nvidia-device-plugin"),""))' 2>/dev/null)
-  info "current status: ${st:-<no release>}"
-  case $st in
-    deployed) info "nothing to do" ;;
-    "")       info "no release; nothing to do" ;;
-    *)        # Roll back to the last deployed revision; if none, uninstall.
-              if helm history nvidia-device-plugin -n "$GPU_NS" 2>/dev/null | grep -q deployed; then
-                run helm rollback nvidia-device-plugin -n "$GPU_NS"
-              else
-                run helm uninstall nvidia-device-plugin -n "$GPU_NS"
-              fi ;;
-  esac
-  # The DaemonSet controller backs off after repeated FailedCreate and will not
-  # retry for many minutes even after the underlying cause is fixed. Force it.
-  if kubectl get ds -n "$GPU_NS" nvidia-device-plugin >/dev/null 2>&1; then
-    log "nudging DaemonSet (clears admission backoff)"
-    run kubectl rollout restart ds/nvidia-device-plugin -n "$GPU_NS"
-  fi
-}
-
-# Remove everything this script creates, so a retry starts from a clean slate.
-# Does NOT touch the node's Talos config or the models PVC by default.
-teardown() {
-  log "teardown"
-  info "this removes: llm-serving ns, kserve, device plugin, RuntimeClass"
-  if ! $DRY; then
-    printf '    type "yes" to proceed: '; local a; read -r a
-    [[ $a == yes ]] || die "aborted"
-  fi
-  run kubectl delete -k "$REPO_ROOT/k8s/apps/llm-serving" --ignore-not-found
-  run helm uninstall kserve -n "$KSERVE_NS" --ignore-not-found 2>/dev/null || true
-  run helm uninstall kserve-crd -n "$KSERVE_NS" --ignore-not-found 2>/dev/null || true
-  run helm uninstall nvidia-device-plugin -n "$GPU_NS" --ignore-not-found 2>/dev/null || true
-  run kubectl delete -k "$REPO_ROOT/k8s/apps/gpu-runtimeclass" --ignore-not-found
-  info "PVC llm-models in ns $NS was NOT deleted (holds ~56Gi of downloaded weights)"
-  info "delete explicitly if you want a cold re-download:"
-  info "  kubectl delete pvc llm-models -n $NS"
-}
-
-all() { preflight; gpu_plugin; kserve; uncordon; engines; smoke; status; }
-
-# ---------------------------------------------------------------- main
-
-STAGE=all
-for a in "$@"; do
-  case $a in
-    --manual)  MODE=manual ;;
-    --argocd)  MODE=argocd ;;
-    --dry-run) DRY=true ;;
-    preflight|runtimeclass|gpu-plugin|kserve|uncordon|engines|smoke|status|doctor|unstick|teardown|all) STAGE=$a ;;
-    -h|--help) sed -n '2,28p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
-    *) die "unknown argument: $a" ;;
-  esac
-done
-
-$DRY && info "DRY RUN — no changes will be made"
-case $STAGE in
-  preflight)  preflight ;;
-  runtimeclass) runtimeclass ;;
-  gpu-plugin) preflight; gpu_plugin ;;
-  kserve)     preflight; kserve ;;
-  uncordon)   uncordon ;;
-  engines)    preflight; engines ;;
-  smoke)      smoke ;;
-  status)     status ;;
-  doctor)     doctor ;;
-  unstick)    unstick ;;
-  teardown)   teardown ;;
-  all)        all ;;
-esac
-
-log "done"

terraform/files/cilium/l2-announcement.yaml

@@ -0,0 +1,13 @@
+# CiliumL2AnnouncementPolicy — ARPs each LoadBalancer IP (from lb-ippool) on the
+# LAN so the EXTERNAL-IP is actually reachable. Without it, LB-IPAM assigns IPs
+# but nothing answers ARP (100% packet loss / incomplete ARP). One node per IP
+# holds the lease (leaderElection) to avoid ARP flapping. Requires kube-proxy
+# replacement (enabled) and Cilium L2 announcements (default since v1.14).
+apiVersion: cilium.io/v2alpha1
+kind: CiliumL2AnnouncementPolicy
+metadata:
+  name: homelab-l2-announce
+spec:
+  loadBalancerIPs: true
+  interfaces:
+    - eno1

terraform/files/cilium/lb-ippool.yaml

@@ -0,0 +1,15 @@
+# CiliumLoadBalancerIPPool — the IPs Cilium LB-IPAM may assign to LoadBalancer
+# Services. CIDR 192.168.1.160/28 covers .160–.175 on the LAN.
+#   .160  ingress-nginx (LoadBalancer)
+#   .161  forgejo-ssh   (LoadBalancer)
+#   .162–.175  free
+# Do NOT add a 10.6.0.0/24 block — that is the WireGuard subnet; letting LB-IPAM
+# hand out a CP tunnel IP breaks the tunnel, and L2 ARP only works on the LAN
+# interface (eno1) anyway. Keep this pool LAN-only.
+apiVersion: cilium.io/v2alpha1
+kind: CiliumLoadBalancerIPPool
+metadata:
+  name: homelab-pool
+spec:
+  blocks:
+    - cidr: "192.168.1.160/28"

terraform/files/coredns/Corefile

@@ -33,6 +33,11 @@
     rewrite name homarr.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name portainer.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name longhorn.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    # Kong API gateway. Points at nginx, not kong-proxy, for the same reason as
+    # the rest: a direct rewrite would skip TLS termination. Pods that don't
+    # need TLS should call kong-proxy.api.svc.cluster.local instead of using
+    # this name at all.
+    rewrite name api.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
 
     kubernetes cluster.local in-addr.arpa ip6.arpa {
         pods insecure

terraform/main.tf

@@ -42,6 +42,8 @@ resource "local_file" "controlplane_configs" {
     zone                   = each.value.zone
     allow_scheduling       = each.value.allow_scheduling
     coredns_corefile       = file("${path.module}/files/coredns/Corefile")
+    cilium_lb_ippool       = file("${path.module}/files/cilium/lb-ippool.yaml")
+    cilium_l2_announcement = file("${path.module}/files/cilium/l2-announcement.yaml")
 
     # Cloudflare Tunnel cert SANs (talos :50000 and kube-apiserver :6443)
     cloudflare_talos_sans     = each.value.cloudflare_talos_sans
@@ -101,6 +103,8 @@ resource "local_file" "worker_configs" {
     forgejo_hostname    = var.forgejo_hostname
     zone                = each.value.zone
     gpu_count           = each.value.gpu_count
+    node_labels         = each.value.node_labels
+    node_taints         = each.value.node_taints
     extra_disks         = each.value.extra_disks
     swap_size           = each.value.swap_size
     ephemeral_max_size  = each.value.ephemeral_max_size

terraform/templates/controlplane.tftpl

@@ -164,6 +164,18 @@ cluster:
         kind: Namespace
         metadata:
           name: kube-system
+    # Cilium LoadBalancer IPAM pool + L2 announcement policy. Substrate networking
+    # (owned here alongside the Cilium install), single source of truth in
+    # terraform/files/cilium/*.yaml. Provides LAN LoadBalancer IPs for ingress-nginx
+    # (.160) and forgejo-ssh (.161). Was previously an ArgoCD app whose empty
+    # kustomization never actually applied it (the live pool came from manual
+    # kubectl); moved here so LB-IPAM exists before any LoadBalancer Service syncs.
+    - name: cilium-lb-ippool
+      contents: |
+${indent(8, cilium_lb_ippool)}
+    - name: cilium-l2-announcement
+      contents: |
+${indent(8, cilium_l2_announcement)}
     # CoreDNS Corefile with homelab hostname rewrites (single source of truth in
     # terraform/files/coredns/Corefile). In-cluster pods resolve *.riotpiao.com to
     # the nginx ingress controller so OIDC auto-discovery against

terraform/templates/worker.tftpl

@@ -71,11 +71,20 @@ machine:
   nodeLabels:
     topology.kubernetes.io/region: homelab
     topology.kubernetes.io/zone: ${zone}
-    node-role.kubernetes.io/gpu-node: ""
 %{ if gpu_count > 0 ~}
+    node-role.kubernetes.io/gpu-node: ""
     nvidia.com/gpu: "true"
     gpu-count: "${gpu_count}"
 %{ endif ~}
+%{ for k, v in node_labels ~}
+    ${k}: "${v}"
+%{ endfor ~}
+%{ if length(node_taints) > 0 ~}
+  nodeTaints:
+%{ for t in node_taints ~}
+    ${t.key}: "${t.value}:${t.effect}"
+%{ endfor ~}
+%{ endif ~}
 
 cluster:
   id: ${cluster_id}

terraform/variables.tf

@@ -131,6 +131,15 @@ variable "worker_configs" {
     network_interface = optional(string, "eno1")
     zone              = string
     gpu_count         = optional(number, 0)
+    # Extra node labels beyond the topology/GPU defaults.
+    node_labels = optional(map(string), {})
+    # Taints make a node dedicated: only pods carrying a matching toleration
+    # schedule there. effect is NoSchedule | PreferNoSchedule | NoExecute.
+    node_taints = optional(list(object({
+      key    = string
+      value  = string
+      effect = string
+    })), [])
     factory_image      = optional(string)
     swap_size          = optional(string, "")
     ephemeral_max_size = optional(string, "700GiB")