Compare commits

..
15 changed files with 30 additions and 490 deletions
-118
View File
@@ -1,118 +0,0 @@
apiVersion: serving.kserve.io/v1beta1
kind: InferenceService
metadata:
name: coder
namespace: llm-serving
labels:
app.kubernetes.io/name: llm-coder
app.kubernetes.io/part-of: llm-serving
annotations:
serving.kserve.io/deploymentMode: RawDeployment
spec:
predictor:
minReplicas: 1
maxReplicas: 1
nodeSelector:
kubernetes.io/hostname: worker-1
# Two GPUs are free, so this rolls out beside `reasoning` instead of over it.
# Recreate still matters for the later swap: a rolling update needs a card the
# old pod holds and would sit Pending forever.
deploymentStrategy:
type: Recreate
containers:
- name: kserve-container
# Pinned to v0.11.0 on purpose. This is the last image measured to work on this
# node: torch 2.8.0+cu128, arch list sm_70..sm_120, driver 580.159.04.
# v0.15.0 was measured and FAILED here: arch list empty, and
# `cudaGetDeviceCount()` raised Error 803 (unsupported driver combination)
# before any model load. Do not raise this tag without re-running that probe.
image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
args:
# Qwen3-Coder-30B-A3B-Instruct: 30.5B total, 3.3B active. Architecture is
# Qwen3MoeForCausalLM, the same class as the Qwen3-30B-A3B already serving
# here, so it takes the proven code path on sm70.
#
# Qwen ships this model only as BF16 (61.1 GB) and FP8 (no FP8 on Volta).
# 4-bit GPTQ is the only form that fits one 32 GB card. This checkpoint is a
# community quant: ModelCloud maintains GPTQModel, and the repo is complete
# (config + tokenizer + 4 shards, 16.9 GB). Verified by loading, not by trust.
#
# Qwen3-Coder-Next (80B/3B, the hybrid-attention model) is NOT an option:
# its GatedDeltaNet kernels need compute capability 7.5+ and the `fla` module,
# and sm70 has neither. Measured, not assumed.
- --model=ModelCloud/Qwen3-Coder-30B-A3B-Instruct-GPTQMODEL-W4A16-A
# `reasoning` stays a served name so an existing client that sends
# model=reasoning keeps working after the DNS rename. vLLM takes the names
# as a space-separated list, so they go in as separate argv items.
- --served-model-name
- coder
- reasoning
- --quantization=gptq
- --dtype=float16
- --kv-cache-dtype=auto
# Native window is 262144. 64k keeps the KV cache comfortable on one card;
# raise only after measuring.
- --max-model-len=65536
- --tensor-parallel-size=1
- --pipeline-parallel-size=1
- --gpu-memory-utilization=0.90
- --max-num-seqs=16
- --max-num-batched-tokens=8192
- --long-prefill-token-threshold=2048
- --enable-chunked-prefill
- --enable-prefix-caching
- --reasoning-parser=qwen3
- --enable-auto-tool-choice
- --tool-call-parser=hermes
- --host=0.0.0.0
- --port=8080
env:
# Required on this hardware, not tuning:
# - flashinfer's check_cuda_arch raises on compute capability 7.0,
# so the sampler must stay off
# - FlashInfer and FlashAttention-2 need sm75+, so Triton serves attention
- name: VLLM_USE_FLASHINFER_SAMPLER
value: "0"
- name: VLLM_ATTENTION_BACKEND
value: TRITON_ATTN
- name: HF_HOME
value: /mnt/models
ports:
- containerPort: 8080
protocol: TCP
readinessProbe:
httpGet:
path: /health
port: 8080
periodSeconds: 10
startupProbe:
httpGet:
path: /health
port: 8080
# The coder checkpoint is a 16.9 GB cold download on first start, then
# ~20 s of weight load and ~80 s of torch.compile. 80 x 15 s covers it.
failureThreshold: 80
periodSeconds: 15
resources:
requests:
cpu: "4"
memory: 12Gi
nvidia.com/gpu: "1"
limits:
cpu: "8"
memory: 36Gi
nvidia.com/gpu: "1"
volumeMounts:
- mountPath: /mnt/models
name: models
- mountPath: /dev/shm
name: shm
runtimeClassName: nvidia
volumes:
- name: models
persistentVolumeClaim:
claimName: llm-models
- name: shm
emptyDir:
medium: Memory
sizeLimit: 2Gi
-4
View File
@@ -10,12 +10,8 @@ kind: Kustomization
# way after any edit here: a GPU predictor restart is a weights reload measured
# in tens of seconds, not a rolling update.
resources:
- coder.yaml
- embeddings.yaml
- ornith.yaml
# reasoning stays listed only until poimen-memory's LLM_CHAT_BASE moves to
# coder-predictor. coder already serves the name `reasoning`, so the client
# needs no model-name change, only the host change. Drop this line after that.
- reasoning.yaml
- reranker.yaml
- qwen-cpu.yaml
-13
View File
@@ -66,19 +66,6 @@ spec:
port: 80
- protocol: TCP
port: 8000
# Allow from poimen namespace (poimen-memory ingest + query).
# The client-side egress rule already exists and is not the gap:
# NetworkPolicy poimen/poimen-memory-llm-egress permits port 8080 to pods
# labelled llm-reasoning, llm-embeddings and llm-reranker. This policy had no
# matching ingress rule, so the packet was allowed out and dropped here.
# The client then sees a bare timeout and the predictor log stays empty.
- from:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: poimen
ports:
- protocol: TCP
port: 8080
# Allow intra-namespace (pod-to-pod within llm-serving)
- from:
- podSelector:
+1 -1
View File
@@ -48,7 +48,7 @@ spec:
mountPath: /backup
containers:
- name: mc-mirror
image: docker.io/bitnamilegacy/minio-client:2025.7.21-debian-12-r3@sha256:73bd39f7899a0cef12b8dd5df13aa93a3ed1aaa44236542442e9ac76819ac158
image: quay.io/minio/mc:latest
env:
- name: ACCESS_KEY
valueFrom:
-20
View File
@@ -1,20 +0,0 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: oidc-bootstrap
namespace: argocd
annotations:
argocd.argoproj.io/sync-wave: "5"
spec:
project: homelab
source:
repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
targetRevision: main
path: k8s/infra/oidc-bootstrap
destination:
server: https://kubernetes.default.svc
namespace: kube-system
syncPolicy:
automated:
prune: true
selfHeal: true
@@ -1,24 +0,0 @@
apiVersion: v1
kind: Secret
type: Opaque
metadata:
name: pi-code-agent-api-keys
namespace: poimen-sandbox
stringData:
ANTHROPIC_API_KEY: ENC[AES256_GCM,data:TBcno/dzj3qykHjeX96UKZekuA/WMQT2E7a1ziPyYo5J6lWLPydgli84KeJZhH62/8CnnFtFOhVIe4FWCGPGjvqKH6M8WbiQHaP9y7FN0qQUpFHKNVAQ2R+s0wInhw0eUmANvd7bRx8aIZ2X,iv:oex19yVtWUevoJ0A3mkMLCfWu85o1gPSNp43x1iV0LE=,tag:BW3zQnwvagPceeNird9CZg==,type:str]
DEEPSEEK_API_KEY: ENC[AES256_GCM,data:vq/5gOsMVjlVMd0oXiBtHFHFEr+IF2YR2iP2OBFWx3ZFNoQ=,iv:QhLU/BRIJb/7QKDCRyw+HWPphZifVEhIOLywYDgOb20=,tag:BZdtXw+WsuK+P4E0caTa0Q==,type:str]
sops:
age:
- enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqNm1LNE5nWktnd3c3STg3
UlVWbWJxWElxUHMwa0JVMk81Z0c4d0JaQm1vCnpLQWJWV3hxOU00NzJIaG51Mmlr
ZWpNTFJWcmdtV0w4YndQeHpvSUx2MTAKLS0tIDVnNkNuVWhPUFZ5V2hFOEt1eitG
blVqRy8yNnYzSU1FNzZGNkxrRElvWVEKL3J77INje75+CvKXaJHC3CvONgHv7d0a
a10MITvfreFr6M4AN3Z/mno49SDMp15Q4oQRagHHxSTS1K54mWHo8g==
-----END AGE ENCRYPTED FILE-----
recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
encrypted_regex: ^(data|stringData)$
lastmodified: "2026-10-06T01:19:49Z"
mac: ENC[AES256_GCM,data:GpYl8IRefms36GvO3RW41c0US7SOgoOhd4rumxsCdEf5ME9kJ0eOxPx8klMgnlYrkBnwHmad8jj7C+HcA5Eim7mdUaYOZZWPFohvC6N4GLHX85aH78P09CelqG5tdm/Gm+1QfxYu0vANYGCAVjLK7YbBbZjbZmbvVfQvAbs3mOY=,iv:y3lglGu8FhTHQpO1poSIKgGYJFu5J9yjIaOjulda6fY=,tag:5Sgh4TbtNFMw7xOJjYRn0g==,type:str]
version: 3.13.2
+26 -27
View File
@@ -7,33 +7,32 @@ metadata:
exec:
path: ksops
files:
- agent-pod-models.enc.yaml
- agent-pod-ssh-key.enc.yaml
- authentik-secrets.enc.yaml
- cloudflare-secrets.enc.yaml
- forgejo-registry-pat.enc.yaml
- forgejo-registry-pat-api.enc.yaml
- forgejo-registry-pull.enc.yaml
- forgejo-runner-token.enc.yaml
- forgejo-secrets.enc.yaml
- grafana-oidc-secrets.enc.yaml
- grafana-secrets.enc.yaml
- homarr-auth-oidc.enc.yaml
- homarr-db-encryption.enc.yaml
- homarr-secrets.enc.yaml
- homelab-ca-secrets.enc.yaml
- loki-secrets.enc.yaml
- minio-secrets.enc.yaml
- paperless-secrets.enc.yaml
- vault-secrets.enc.yaml
- vault-unseal-keys.enc.yaml
- portfolio-secrets.enc.yaml
- gotify-admin-secrets.enc.yaml
- gotify-tokens-secrets.enc.yaml
- gotify-smtp-secrets.enc.yaml
- forgejo-smtp-secrets.enc.yaml
- paperless-ai-secrets.enc.yaml
- agent-pod-models.enc.yaml
- agent-pod-ssh-key.enc.yaml
- authentik-secrets.enc.yaml
- cloudflare-secrets.enc.yaml
- forgejo-registry-pat.enc.yaml
- forgejo-registry-pat-api.enc.yaml
- forgejo-registry-pull.enc.yaml
- forgejo-runner-token.enc.yaml
- forgejo-secrets.enc.yaml
- grafana-oidc-secrets.enc.yaml
- grafana-secrets.enc.yaml
- homarr-auth-oidc.enc.yaml
- homarr-db-encryption.enc.yaml
- homarr-secrets.enc.yaml
- homelab-ca-secrets.enc.yaml
- loki-secrets.enc.yaml
- minio-secrets.enc.yaml
- paperless-secrets.enc.yaml
- vault-secrets.enc.yaml
- vault-unseal-keys.enc.yaml
- portfolio-secrets.enc.yaml
- gotify-admin-secrets.enc.yaml
- gotify-tokens-secrets.enc.yaml
- gotify-smtp-secrets.enc.yaml
- forgejo-smtp-secrets.enc.yaml
- paperless-ai-secrets.enc.yaml
- memory-agent-oidc.enc.yaml
- oauth2-credentials.enc.yaml
- forgejo-registry-token.enc.yaml
- pi-sandbox-secrets.enc.yaml
@@ -52,14 +52,14 @@ spec:
restartPolicy: Never
initContainers:
- name: kubectl-copy
image: bitnami/kubectl:latest@sha256:f7f9e4f64d9e114650c115a4ac6fd383394b3d494061a8f17b1a0f6c8d55bc25
image: bitnami/kubectl:latest
command: ["sh", "-c", "cp $(which kubectl) /shared/kubectl"]
volumeMounts:
- name: shared
mountPath: /shared
containers:
- name: provision
image: docker.io/bitnamilegacy/minio-client:2025.7.21-debian-12-r3@sha256:73bd39f7899a0cef12b8dd5df13aa93a3ed1aaa44236542442e9ac76819ac158
image: quay.io/minio/mc:latest
volumeMounts:
- name: shared
mountPath: /shared
+1 -4
View File
@@ -16,10 +16,7 @@ spec:
# backend could not resolve minio-cluster-hl. Official images now need registry auth
# (quay.io/minio/minio returns 401), so this pins a public mirror of the same upstream
# release; swap it for the official image once MinIO registry credentials exist.
# The digest freezes those bytes, so a retag or a deletion on the mirror cannot break
# the pull again. The same digest already sits in the containerd cache on talos-cp-1,
# so the first start needs no registry access.
image: docker.io/coollabsio/minio:RELEASE.2025-10-15T17-29-55Z@sha256:69b55a1c1c5dc285ce04db96689f5b2102317fc77a50680a1874ca6efd1c87f9
image: coollabsio/minio:RELEASE.2025-10-15T17-29-55Z
# Disable operator auto-TLS: MinIO serves plain HTTP internally on 9000.
# External TLS is terminated at nginx ingress (wildcard riotpiao-com-tls cert).
@@ -27,24 +27,6 @@ spec:
annotations:
summary: "{{ $value }} pod(s) stuck in Pending state for >10m"
# ImagePullBackOff on a vendor-deleted image produced no alert for 13 days
- alert: PodImagePullFailure
expr: sum(kube_pod_container_status_waiting_reason{reason=~"ImagePullBackOff|ErrImagePull"}) by (namespace, pod) > 0
for: 10m
labels:
severity: critical
annotations:
summary: "{{ $labels.namespace }}/{{ $labels.pod }} cannot pull its image"
# Pod not ready
- alert: PodNotReady
expr: sum(kube_pod_status_ready{condition="false"}) by (namespace, pod) > 0
for: 15m
labels:
severity: warning
annotations:
summary: "{{ $labels.namespace }}/{{ $labels.pod }} is not ready"
# CrashLoopBackOff
- alert: PodCrashLooping
expr: sum(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}) by (namespace, pod) > 0
-59
View File
@@ -1,59 +0,0 @@
# One-shot bootstrap: restart the control-plane apiservers ONCE so they pick up the
# OIDC authenticator (they last started before the Authentik provider existed, so OIDC
# tokens were never accepted -> every login got 401). Delete this whole directory after.
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: oidc-bootstrap
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: oidc-bootstrap
namespace: kube-system
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list", "delete"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: oidc-bootstrap
namespace: kube-system
subjects:
- kind: ServiceAccount
name: oidc-bootstrap
namespace: kube-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: oidc-bootstrap
---
apiVersion: batch/v1
kind: Job
metadata:
name: oidc-bootstrap-restart-apiserver-v2
namespace: kube-system
spec:
backoffLimit: 4
ttlSecondsAfterFinished: 7200
template:
spec:
serviceAccountName: oidc-bootstrap
restartPolicy: Never
containers:
- name: restart
image: docker.io/curlimages/curl:8.22.0
command: ["/bin/sh", "-c"]
args:
- |
set -u
TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
CA=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
for pod in kube-apiserver-talos-cp-1 kube-apiserver-talos-cp-2 kube-apiserver-talos-cp-3; do
curl -sS --max-time 15 --cacert "$CA" -H "Authorization: Bearer $TOKEN" -X DELETE "https://kubernetes.default.svc/api/v1/namespaces/kube-system/pods/$pod?gracePeriodSeconds=0" -o /dev/null -w "$pod -> HTTP %{http_code}\n" || echo "$pod -> request failed"
done
echo "deletes requested; kubelet recreates each static pod in ~20-40s"
@@ -1,141 +0,0 @@
# One-shot: restart the control-plane API servers so their OIDC authenticator initialises.
#
# Two earlier mechanisms failed and are documented so nobody repeats them:
# 1. deleting the mirror pod does NOT restart a static pod (deletes returned 200, no restart);
# 2. matching /proc/*/cmdline for 'kube-apiserver' also matched this script's own cmdline,
# so every pod SIGTERMed itself (exit 143) before reaching the real process.
# This version matches /proc/<pid>/comm (the process name) and skips \$\$, so self-kill is
# impossible. hostPID + privileged are permitted in the 'cicd' namespace. Delays stagger the
# three control planes by 2 minutes each, so only one API server is down at a time.
---
apiVersion: batch/v1
kind: Job
metadata:
name: oidc-restart-apiserver-cp1-v2
namespace: cicd
spec:
backoffLimit: 0
ttlSecondsAfterFinished: 3600
template:
spec:
hostPID: true
restartPolicy: Never
nodeSelector:
kubernetes.io/hostname: talos-cp-1
tolerations:
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule
containers:
- name: kill
image: docker.io/library/busybox:1.36
securityContext:
privileged: true
command: ["/bin/sh", "-c"]
args:
- |
echo "sleeping 10s before restarting the API server on talos-cp-1"
sleep 10
self=$$
killed=0
for d in /proc/[0-9]*; do
pid=${d#/proc/}
[ "$pid" = "$self" ] && continue
comm=$(cat "$d/comm" 2>/dev/null)
[ "$comm" = "kube-apiserver" ] || continue
echo "SIGTERM -> pid $pid"
kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
done
echo "signalled $killed API server process(es); kubelet recreates the static pod"
---
apiVersion: batch/v1
kind: Job
metadata:
name: oidc-restart-apiserver-cp2-v2
namespace: cicd
spec:
backoffLimit: 0
ttlSecondsAfterFinished: 3600
template:
spec:
hostPID: true
restartPolicy: Never
nodeSelector:
kubernetes.io/hostname: talos-cp-2
tolerations:
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule
containers:
- name: kill
image: docker.io/library/busybox:1.36
securityContext:
privileged: true
command: ["/bin/sh", "-c"]
args:
- |
echo "sleeping 120s before restarting the API server on talos-cp-2"
sleep 120
self=$$
killed=0
for d in /proc/[0-9]*; do
pid=${d#/proc/}
[ "$pid" = "$self" ] && continue
comm=$(cat "$d/comm" 2>/dev/null)
[ "$comm" = "kube-apiserver" ] || continue
echo "SIGTERM -> pid $pid"
kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
done
echo "signalled $killed API server process(es); kubelet recreates the static pod"
---
apiVersion: batch/v1
kind: Job
metadata:
name: oidc-restart-apiserver-cp3-v2
namespace: cicd
spec:
backoffLimit: 0
ttlSecondsAfterFinished: 3600
template:
spec:
hostPID: true
restartPolicy: Never
nodeSelector:
kubernetes.io/hostname: talos-cp-3
tolerations:
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule
containers:
- name: kill
image: docker.io/library/busybox:1.36
securityContext:
privileged: true
command: ["/bin/sh", "-c"]
args:
- |
echo "sleeping 240s before restarting the API server on talos-cp-3"
sleep 240
self=$$
killed=0
for d in /proc/[0-9]*; do
pid=${d#/proc/}
[ "$pid" = "$self" ] && continue
comm=$(cat "$d/comm" 2>/dev/null)
[ "$comm" = "kube-apiserver" ] || continue
echo "SIGTERM -> pid $pid"
kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
done
echo "signalled $killed API server process(es); kubelet recreates the static pod"
-1
View File
@@ -9,4 +9,3 @@ resources:
- poimen-sandbox-namespace.yaml
- poimen-sandbox-quota.yaml
- pi-session-netpol.yaml
- pi-agent-workspace-pv.yaml
@@ -1,57 +0,0 @@
apiVersion: v1
kind: PersistentVolume
metadata:
finalizers:
- kubernetes.io/pv-protection
labels:
app: pi-coding-agent-sandbox
part-of: poimen-toolkit
name: pi-agent-workspace-pv
annotations:
argocd.argoproj.io/sync-wave: '-1'
spec:
accessModes:
- ReadWriteOnce
capacity:
storage: 10Gi
claimRef:
apiVersion: v1
kind: PersistentVolumeClaim
name: pi-agent-workspace
namespace: poimen-sandbox
hostPath:
path: /var/lib/pi-agent-sandbox
type: DirectoryOrCreate
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
values:
- talos-cp-2
persistentVolumeReclaimPolicy: Retain
volumeMode: Filesystem
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
annotations:
pv.kubernetes.io/bind-completed: 'yes'
argocd.argoproj.io/sync-wave: '-1'
finalizers:
- kubernetes.io/pvc-protection
labels:
app: pi-coding-agent-sandbox
part-of: poimen-toolkit
name: pi-agent-workspace
namespace: poimen-sandbox
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10Gi
storageClassName: ''
volumeMode: Filesystem
volumeName: pi-agent-workspace-pv
-1
View File
@@ -39,7 +39,6 @@
rewrite name comfy.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
rewrite name comfyui.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
rewrite name gotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
rewrite name pi.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
rewrite name igotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
rewrite name riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local