Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 rock
|
1714b99b71 |
@@ -45,10 +45,33 @@ data:
|
||||
service: https://igotify.riotpiao.com
|
||||
- hostname: poimen.riotpiao.com
|
||||
service: https://poimen.riotpiao.com
|
||||
- hostname: comfyui.riotpiao.com
|
||||
service: https://comfyui.riotpiao.com
|
||||
- hostname: forgejo.riotpiao.com
|
||||
service: https://forgejo.riotpiao.com
|
||||
# ZITADEL. Without this rule the connector answers 404 for the hostname
|
||||
# even though the DNS CNAME points at the tunnel.
|
||||
- hostname: zitadel.riotpiao.com
|
||||
service: https://zitadel.riotpiao.com
|
||||
# KUBERNETES API, for kubectl from outside the LAN.
|
||||
#
|
||||
# Origin is the in-cluster Service, not the node IP: that survives a
|
||||
# control-plane address change and lets the connectors reach whichever
|
||||
# apiserver is local-healthy.
|
||||
#
|
||||
# noTLSVerify is required. The apiserver serves a certificate for the
|
||||
# cluster CA plus its node/LB addresses, so its SANs cannot match this
|
||||
# public hostname. Cloudflare terminates the public TLS with the
|
||||
# *.riotpiao.com edge certificate, and the connector hop is already
|
||||
# inside the cluster, so skipping verification on that short hop is the
|
||||
# least-bad option. Do not widen this to other rules.
|
||||
#
|
||||
# SECURITY: this exposes the API to the internet. Authentication is still
|
||||
# ZITADEL OIDC plus client certificates, but put a Cloudflare Access
|
||||
# application in front of this hostname as well -- see the note in the
|
||||
# PR. Without Access, every scanner on the internet reaches the apiserver.
|
||||
- hostname: k8s.riotpiao.com
|
||||
service: https://kubernetes.default.svc.cluster.local:443
|
||||
originRequest:
|
||||
noTLSVerify: true
|
||||
- service: http_status:404
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
# ComfyUI — GPU-accelerated image generation on worker-1.
|
||||
# Uses 1x V100 32GB (sm70). Freed by scaling ornith 2→1.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: comfyui
|
||||
namespace: comfyui
|
||||
labels:
|
||||
app: comfyui
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: comfyui
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: comfyui
|
||||
spec:
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: worker-1
|
||||
runtimeClassName: nvidia
|
||||
# k8s Service named 'comfyui' injects COMFYUI_PORT=tcp://... into pod env,
|
||||
# which clobbers ai-dock's own COMFYUI_PORT variable (expects a port number).
|
||||
# Disable service link injection to avoid the collision.
|
||||
enableServiceLinks: false
|
||||
containers:
|
||||
- name: comfyui
|
||||
image: ghcr.io/ai-dock/comfyui:v2-cuda-12.1.1-base-22.04
|
||||
ports:
|
||||
- containerPort: 8188
|
||||
protocol: TCP
|
||||
env:
|
||||
- name: NVIDIA_VISIBLE_DEVICES
|
||||
value: "all"
|
||||
resources:
|
||||
requests:
|
||||
cpu: "4"
|
||||
memory: 8Gi
|
||||
nvidia.com/gpu: "1"
|
||||
limits:
|
||||
cpu: "8"
|
||||
memory: 16Gi
|
||||
nvidia.com/gpu: "1"
|
||||
volumeMounts:
|
||||
- mountPath: /workspace/ComfyUI/models
|
||||
name: models
|
||||
- mountPath: /workspace/ComfyUI/output
|
||||
name: output
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: 8188
|
||||
periodSeconds: 10
|
||||
initialDelaySeconds: 30
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: 8188
|
||||
failureThreshold: 120
|
||||
periodSeconds: 10
|
||||
volumes:
|
||||
- name: models
|
||||
persistentVolumeClaim:
|
||||
claimName: comfyui-models
|
||||
- name: output
|
||||
emptyDir: {}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: comfyui-models
|
||||
namespace: comfyui
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: longhorn
|
||||
resources:
|
||||
requests:
|
||||
storage: 50Gi
|
||||
@@ -0,0 +1,38 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: comfyui
|
||||
namespace: comfyui
|
||||
annotations:
|
||||
# No cert-manager annotation and no tls: block. This host is covered by
|
||||
# the *.riotpiao.com wildcard, which nginx serves as its
|
||||
# default-ssl-certificate (ingress-nginx/riotpiao-com-tls).
|
||||
nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
|
||||
nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
|
||||
nginx.ingress.kubernetes.io/proxy-body-size: "0"
|
||||
# WebSocket support for ComfyUI's live preview
|
||||
nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
|
||||
nginx.ingress.kubernetes.io/upstream-hash-by: "$remote_addr"
|
||||
# Single sign-on. ComfyUI has no auth of its own, so every request is handed to
|
||||
# oauth2-proxy for an allow or deny decision.
|
||||
nginx.ingress.kubernetes.io/auth-url: "http://comfyui-oauth2-proxy.comfyui.svc.cluster.local:4180/oauth2/auth"
|
||||
nginx.ingress.kubernetes.io/auth-signin: "https://comfyui.riotpiao.com/oauth2/start?rd=$scheme://$host$request_uri"
|
||||
nginx.ingress.kubernetes.io/auth-response-headers: "x-auth-request-user,x-auth-request-email"
|
||||
# No configuration-snippet. The controller rejects that annotation as risky
|
||||
# ("annotation group ConfigurationSnippet contains risky annotation"), and a
|
||||
# rejected annotation drops the WHOLE ingress from nginx.conf: the host then
|
||||
# answers 404 with no entry in the access log. ingress-nginx sets the Upgrade
|
||||
# and Connection headers itself for a websocket upgrade.
|
||||
spec:
|
||||
ingressClassName: nginx
|
||||
rules:
|
||||
- host: comfyui.riotpiao.com
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: comfyui
|
||||
port:
|
||||
number: 80
|
||||
@@ -0,0 +1,8 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
resources:
|
||||
- deployment.yaml
|
||||
- service.yaml
|
||||
- oauth2-proxy.yaml
|
||||
- ingress.yaml
|
||||
@@ -0,0 +1,105 @@
|
||||
# ComfyUI has no authentication of its own: comfyui.riotpiao.com is open to anyone
|
||||
# who can resolve the host. It also has no OIDC support, so ZITADEL cannot be a
|
||||
# login method inside it. oauth2-proxy is the front door instead: nginx asks it for
|
||||
# a decision on every request and it answers from a ZITADEL session.
|
||||
#
|
||||
# No API client calls this host today, so the whole host is gated. If a client ever
|
||||
# needs direct access, add a skip rule for the exact paths it uses rather than
|
||||
# opening the whole host.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: comfyui-oauth2-proxy
|
||||
namespace: comfyui
|
||||
labels:
|
||||
app: comfyui-oauth2-proxy
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: comfyui-oauth2-proxy
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: comfyui-oauth2-proxy
|
||||
spec:
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65534
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
containers:
|
||||
- name: oauth2-proxy
|
||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.7.1
|
||||
args:
|
||||
- --provider=oidc
|
||||
- --oidc-issuer-url=https://zitadel.riotpiao.com
|
||||
- --email-domain=*
|
||||
- --upstream=static://202
|
||||
- --reverse-proxy=true
|
||||
- --set-xauthrequest=true
|
||||
- --cookie-secure=true
|
||||
- --cookie-expire=12h
|
||||
- --cookie-refresh=1h
|
||||
- --skip-provider-button=true
|
||||
- --redirect-url=https://comfyui.riotpiao.com/oauth2/callback
|
||||
- --http-address=0.0.0.0:4180
|
||||
env:
|
||||
- name: OAUTH2_PROXY_CLIENT_ID
|
||||
valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: client-id}}
|
||||
- name: OAUTH2_PROXY_CLIENT_SECRET
|
||||
valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: client-secret}}
|
||||
- name: OAUTH2_PROXY_COOKIE_SECRET
|
||||
valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: cookie-secret}}
|
||||
ports:
|
||||
- {name: http, containerPort: 4180}
|
||||
readinessProbe:
|
||||
httpGet: {path: /ping, port: http}
|
||||
initialDelaySeconds: 5
|
||||
livenessProbe:
|
||||
httpGet: {path: /ping, port: http}
|
||||
initialDelaySeconds: 15
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities: {drop: ["ALL"]}
|
||||
resources:
|
||||
requests: {cpu: 10m, memory: 32Mi}
|
||||
limits: {cpu: 200m, memory: 128Mi}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: comfyui-oauth2-proxy
|
||||
namespace: comfyui
|
||||
labels:
|
||||
app: comfyui-oauth2-proxy
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: comfyui-oauth2-proxy
|
||||
ports:
|
||||
- {name: http, port: 4180, targetPort: http}
|
||||
---
|
||||
# The login path must NOT be gated: a request to /oauth2/start that itself demands a
|
||||
# login redirects to /oauth2/start forever. ingress-nginx applies the annotations of
|
||||
# an ingress to the locations it defines, so a separate object without auth-url keeps
|
||||
# this path open. The earlier failure of this shape was not the split — it was the
|
||||
# configuration-snippet on the app ingress, which made the controller drop the whole
|
||||
# object and left only this one alive.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: comfyui-oauth2
|
||||
namespace: comfyui
|
||||
spec:
|
||||
ingressClassName: nginx
|
||||
rules:
|
||||
- host: comfyui.riotpiao.com
|
||||
http:
|
||||
paths:
|
||||
- path: /oauth2
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: comfyui-oauth2-proxy
|
||||
port:
|
||||
number: 4180
|
||||
@@ -0,0 +1,14 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: comfyui
|
||||
namespace: comfyui
|
||||
labels:
|
||||
app: comfyui
|
||||
spec:
|
||||
selector:
|
||||
app: comfyui
|
||||
ports:
|
||||
- port: 80
|
||||
targetPort: 8188
|
||||
protocol: TCP
|
||||
@@ -8,11 +8,7 @@ metadata:
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
|
||||
spec:
|
||||
# 2, not 3. The file's own header always said 2, and the third instance never
|
||||
# existed as a running pod: its volume only ever held a replica that could not
|
||||
# start, because talos-cp-2 disk2 has no free space. A third replica on a disk
|
||||
# that cannot host it is not redundancy, it is a permanent degraded volume.
|
||||
instances: 2
|
||||
instances: 3
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16.2
|
||||
bootstrap:
|
||||
initdb:
|
||||
|
||||
@@ -40,7 +40,7 @@ target is one root-of-trust file per thing that cannot rotate itself:
|
||||
| controller bootstrap key | chicken and egg: it creates every other key |
|
||||
| etcd defrag credentials | needed by a job that runs outside the controller |
|
||||
|
||||
Everything else — Grafana, Argo CD, Forgejo, MinIO, Paperless, Immich,
|
||||
Everything else — Grafana, Argo CD, Forgejo, MinIO, Paperless, Immich, ComfyUI,
|
||||
api-gw, Gotify — is a KeyRotation object, and its secret leaves git.
|
||||
|
||||
## Status
|
||||
|
||||
@@ -64,6 +64,32 @@ spec:
|
||||
---
|
||||
apiVersion: iam.homelab.riotpiao.com/v1
|
||||
kind: KeyRotation
|
||||
metadata:
|
||||
name: comfyui
|
||||
namespace: iam
|
||||
labels:
|
||||
app.kubernetes.io/part-of: key-rotation
|
||||
spec:
|
||||
provider: zitadel-oidc-client
|
||||
strategy: atomic
|
||||
intervalHours: 2160
|
||||
suspend: true
|
||||
target:
|
||||
zitadelProjectID: "396757796898471937"
|
||||
zitadelAppID: "397015743994003854" # app resource id, not the client id 397015744010781070
|
||||
zitadelClientID: "397015744010781070"
|
||||
secretName: comfyui-oauth2-proxy
|
||||
secretNamespace: comfyui
|
||||
secretKeys:
|
||||
secret: client-secret
|
||||
reload: restart
|
||||
consumers:
|
||||
- {kind: Deployment, name: comfyui-oauth2-proxy, namespace: comfyui}
|
||||
verify:
|
||||
introspect: true
|
||||
---
|
||||
apiVersion: iam.homelab.riotpiao.com/v1
|
||||
kind: KeyRotation
|
||||
metadata:
|
||||
name: minio
|
||||
namespace: iam
|
||||
@@ -81,11 +107,7 @@ spec:
|
||||
secretName: minio-oidc
|
||||
secretNamespace: storage
|
||||
secretKeys:
|
||||
# The key name here must match the consumer's secretKeyRef, not a house style.
|
||||
# minio-tenant.yaml reads MINIO_IDENTITY_OPENID_CLIENT_SECRET. While this said
|
||||
# `client-secret`, the tenant failed with CreateContainerConfigError on every
|
||||
# restart and the rotation wrote a key nobody read.
|
||||
secret: MINIO_IDENTITY_OPENID_CLIENT_SECRET
|
||||
secret: client-secret
|
||||
reload: hot
|
||||
consumers:
|
||||
- {kind: StatefulSet, name: minio-cluster-az-a, namespace: storage}
|
||||
|
||||
@@ -33,19 +33,7 @@ metadata:
|
||||
app: clef-serve
|
||||
app.kubernetes.io/part-of: llm-serving
|
||||
spec:
|
||||
# Parked at 0 to free the only V100 that was not serving anything, so candle-vllm can
|
||||
# run its Phase 0 on one card. All four cards were allocated (coder 2, reasoning 1,
|
||||
# this 1) and a new GPU workload had nowhere to land.
|
||||
#
|
||||
# This is the only GPU consumer in the namespace that is not vLLM, which is why it is
|
||||
# the first card released: the standing instruction is to keep the vLLM pods healthy
|
||||
# until candle-vllm answers, and this pod is not one of them.
|
||||
#
|
||||
# It was serving nothing. The last 30 log lines at park time were 30/30 kubelet
|
||||
# /health probes and no inference request since start (2026-10-30T06:23:40Z). Nothing
|
||||
# in git selects this service. Restore it once the rollout leaves a card free:
|
||||
# 30B-A3B takes 1, Coder-Next TP=2 takes 2, so card 4 is this one.
|
||||
replicas: 0
|
||||
replicas: 1
|
||||
# Recreate, not RollingUpdate. Every GPU card is allocated, so a rolling update leaves
|
||||
# the new pod Pending forever waiting for a card the old pod still holds.
|
||||
strategy:
|
||||
|
||||
@@ -1,134 +0,0 @@
|
||||
apiVersion: serving.kserve.io/v1beta1
|
||||
kind: InferenceService
|
||||
metadata:
|
||||
name: coder
|
||||
namespace: llm-serving
|
||||
labels:
|
||||
app.kubernetes.io/name: llm-coder
|
||||
app.kubernetes.io/part-of: llm-serving
|
||||
annotations:
|
||||
serving.kserve.io/deploymentMode: RawDeployment
|
||||
spec:
|
||||
predictor:
|
||||
minReplicas: 1
|
||||
maxReplicas: 1
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: worker-1
|
||||
# Two GPUs are free, so this rolls out beside `reasoning` instead of over it.
|
||||
# Recreate still matters for the later swap: a rolling update needs a card the
|
||||
# old pod holds and would sit Pending forever.
|
||||
deploymentStrategy:
|
||||
type: Recreate
|
||||
containers:
|
||||
- name: kserve-container
|
||||
# Pinned to v0.11.0 on purpose. This is the last image measured to work on this
|
||||
# node: torch 2.8.0+cu128, arch list sm_70..sm_120, driver 580.159.04.
|
||||
# v0.15.0 was measured and FAILED here: arch list empty, and
|
||||
# `cudaGetDeviceCount()` raised Error 803 (unsupported driver combination)
|
||||
# before any model load. Do not raise this tag without re-running that probe.
|
||||
image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
|
||||
args:
|
||||
# Qwen3-Coder-30B-A3B-Instruct: 30.5B total, 3.3B active. Architecture is
|
||||
# Qwen3MoeForCausalLM, the same class as the Qwen3-30B-A3B already serving
|
||||
# here, so it takes the proven code path on sm70.
|
||||
#
|
||||
# Qwen ships this model only as BF16 (61.1 GB) and FP8 (no FP8 on Volta).
|
||||
# 4-bit GPTQ is the only form that fits one 32 GB card. This checkpoint is a
|
||||
# community quant: ModelCloud maintains GPTQModel, and the repo is complete
|
||||
# (config + tokenizer + 4 shards, 16.9 GB). Verified by loading, not by trust.
|
||||
#
|
||||
# Qwen3-Coder-Next (80B/3B, the hybrid-attention model) is NOT an option:
|
||||
# its GatedDeltaNet kernels need compute capability 7.5+ and the `fla` module,
|
||||
# and sm70 has neither. Measured, not assumed.
|
||||
- --model=ModelCloud/Qwen3-Coder-30B-A3B-Instruct-GPTQMODEL-W4A16-A
|
||||
# `reasoning` stays a served name so an existing client that sends
|
||||
# model=reasoning keeps working after the DNS rename. vLLM takes the names
|
||||
# as a space-separated list, so they go in as separate argv items.
|
||||
- --served-model-name
|
||||
- coder
|
||||
- reasoning
|
||||
- --quantization=gptq
|
||||
- --dtype=float16
|
||||
- --kv-cache-dtype=auto
|
||||
# Native window is 262144 and rope_scaling is null, so 256K needs no YaRN.
|
||||
# One card cannot hold it: measured KV pool on a single V100 was 132,672
|
||||
# tokens. TP=2 halves the weights per card (7.8 GiB each), which frees room
|
||||
# for the larger pool. Measured after boot: see the KV line in the log.
|
||||
- --max-model-len=262144
|
||||
- --tensor-parallel-size=2
|
||||
- --pipeline-parallel-size=1
|
||||
- --gpu-memory-utilization=0.90
|
||||
# A single 262,144-token request consumes the whole pool, so keep few slots.
|
||||
- --max-num-seqs=4
|
||||
- --max-num-batched-tokens=8192
|
||||
- --long-prefill-token-threshold=2048
|
||||
- --enable-chunked-prefill
|
||||
- --enable-prefix-caching
|
||||
# Measured: with the default (custom all-reduce ON) the engine finished CUDA graph
|
||||
# capture in 6s at 17:01:56 and then went silent for 14+ minutes, emitting only
|
||||
# "No available shared memory broadcast block found in 60 seconds". The front-end
|
||||
# never received the ready message. vLLM's custom all-reduce registers its own
|
||||
# CUDA graph addresses and deadlocks on this Volta/PCIe pair. NCCL is the fallback
|
||||
# and is correct here, because TP=2 across two PCIe V100s gains nothing from the
|
||||
# custom kernel anyway.
|
||||
- --disable-custom-all-reduce
|
||||
- --reasoning-parser=qwen3
|
||||
- --enable-auto-tool-choice
|
||||
- --tool-call-parser=hermes
|
||||
- --host=0.0.0.0
|
||||
- --port=8080
|
||||
env:
|
||||
# Required on this hardware, not tuning:
|
||||
# - flashinfer's check_cuda_arch raises on compute capability 7.0,
|
||||
# so the sampler must stay off
|
||||
# - FlashInfer and FlashAttention-2 need sm75+, so Triton serves attention
|
||||
- name: VLLM_USE_FLASHINFER_SAMPLER
|
||||
value: "0"
|
||||
- name: VLLM_ATTENTION_BACKEND
|
||||
value: TRITON_ATTN
|
||||
- name: HF_HOME
|
||||
value: /mnt/models
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
protocol: TCP
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
periodSeconds: 10
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
# TP=2 at 256K needs ~10 min before the API binds: 22s weights, 80s torch.compile,
|
||||
# 85s KV profiling, then graph capture. 120 x 15s = 30 min of headroom.
|
||||
failureThreshold: 120
|
||||
periodSeconds: 15
|
||||
resources:
|
||||
# worker-1 exposes only 63,495 MiB of allocatable memory, so a 24Gi request
|
||||
# never schedules. Two vLLM workers keep the weights on the GPU and need
|
||||
# little host RAM, so the request stays small and the limit allows burst.
|
||||
requests:
|
||||
cpu: "8"
|
||||
memory: 12Gi
|
||||
nvidia.com/gpu: "2"
|
||||
limits:
|
||||
cpu: "16"
|
||||
memory: 40Gi
|
||||
nvidia.com/gpu: "2"
|
||||
volumeMounts:
|
||||
- mountPath: /mnt/models
|
||||
name: models
|
||||
- mountPath: /dev/shm
|
||||
name: shm
|
||||
runtimeClassName: nvidia
|
||||
volumes:
|
||||
- name: models
|
||||
persistentVolumeClaim:
|
||||
claimName: llm-models
|
||||
- name: shm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
# NCCL all-reduce over TP=2 moves through /dev/shm. The 64Mi default is
|
||||
# too small and shows up as an init hang or a bus error.
|
||||
sizeLimit: 8Gi
|
||||
@@ -14,9 +14,7 @@ resources:
|
||||
# v0.5.22+, published only on CUDA 13, and CUDA 13 dropped Volta kernels. The model
|
||||
# instead runs the implementation it ships (joint_schema_model.py) in fp16 on one V100.
|
||||
- clef-serve.yaml
|
||||
# Stable client-facing name for the chat model. poimen-memory points at this Service,
|
||||
# so the vLLM -> candle-vllm handoff is a selector edit here and no client change.
|
||||
- reasoning-long-service.yaml
|
||||
- reasoning.yaml
|
||||
# ornith is retired: its InferenceService still reports Ready=True while running zero
|
||||
# replicas, because desired equals available at 0. Its manifest was never committed,
|
||||
# so it is left unmanaged and is not listed here. Delete the CR when convenient.
|
||||
|
||||
@@ -66,21 +66,6 @@ spec:
|
||||
port: 80
|
||||
- protocol: TCP
|
||||
port: 8000
|
||||
# Allow from poimen namespace (poimen-memory ingest + query).
|
||||
# The client-side egress rule is the other half and must stay in step:
|
||||
# NetworkPolicy poimen/poimen-memory-llm-egress permits port 8080 to pods
|
||||
# labelled part-of=llm-serving, which is the same selector used here. This
|
||||
# policy had no matching ingress rule at all, so the packet was allowed out
|
||||
# and dropped here. The client then saw a bare timeout and the predictor log
|
||||
# stayed empty, which is why this failure looked like a hung inference engine
|
||||
# for 34 days.
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: poimen
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 8080
|
||||
# Allow intra-namespace (pod-to-pod within llm-serving)
|
||||
- from:
|
||||
- podSelector:
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
# qwen2.5:3b-instruct on CPU (talos-cp-2, 144GB RAM, 24 cores).
|
||||
# Moved off GPU to free a V100, first for ComfyUI and later for llm-serving.
|
||||
# Latency ~10x slower
|
||||
# Moved off GPU to free a V100 for ComfyUI. Latency ~10x slower
|
||||
# than GPU but sufficient for lightweight tasks (summarization,
|
||||
# classification, quick answers).
|
||||
apiVersion: apps/v1
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
# Stable client-facing name for the chat model, deliberately decoupled from the
|
||||
# process that currently serves it.
|
||||
#
|
||||
# poimen-memory addresses this Service, not a pod and not an InferenceService. That
|
||||
# means the vLLM -> candle-vllm handoff changes only this file's selector. The client
|
||||
# config never changes again, so the ingest cutover has no window in which the host
|
||||
# name is wrong.
|
||||
#
|
||||
# Two ports on purpose:
|
||||
# - 80 matches every other predictor Service in this namespace, and matches
|
||||
# poimen-memory's LLM_CHAT_BASE, which carries no port number.
|
||||
# - 8080 is the container port. It is also the only port llm-serving-ingress and
|
||||
# poimen-memory-llm-egress permit, and the port the handoff readiness check
|
||||
# uses. Publishing it directly avoids a port translation that is easy to get
|
||||
# wrong in a curl.
|
||||
#
|
||||
# The selector names the vLLM reasoning pod today. To hand off, replace the three
|
||||
# selector labels with candle-vllm's pod labels and nothing else. Do not delete and
|
||||
# recreate this Service: a delete drops the ClusterIP, and a client that cached the
|
||||
# name during the gap will fail until it re-resolves.
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: reasoning-long-predictor
|
||||
namespace: llm-serving
|
||||
labels:
|
||||
app.kubernetes.io/part-of: llm-serving
|
||||
app: reasoning-long-predictor
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app.kubernetes.io/name: llm-reasoning
|
||||
component: predictor
|
||||
serving.kserve.io/inferenceservice: reasoning
|
||||
ports:
|
||||
- name: http
|
||||
protocol: TCP
|
||||
port: 80
|
||||
targetPort: 8080
|
||||
- name: http-8080
|
||||
protocol: TCP
|
||||
port: 8080
|
||||
targetPort: 8080
|
||||
@@ -30,28 +30,7 @@ spec:
|
||||
# Measured on this cluster, one V100-32GB, 64k context: 15.55 GiB of weights,
|
||||
# 137k tokens of KV cache, 2.09x concurrency at 65536 tokens per request.
|
||||
- --model=Qwen/Qwen3-30B-A3B-GPTQ-Int4
|
||||
# Two served names, one pod, on purpose. `reasoning-long` is the name the goal map
|
||||
# uses for this model, and the name qwen-finetune wiring targets. It is NOT a
|
||||
# separate InferenceService: this model already occupies the only free card, and a
|
||||
# second pod on the same weights cannot schedule (4 of 4 GPUs allocated: coder 2,
|
||||
# this 1, clef-serve 1). Serving both names from one process costs nothing and
|
||||
# keeps the two callers working without a second weights copy.
|
||||
#
|
||||
# `reasoning` is deliberately still served here. poimen-memory pins
|
||||
# LLM_CHAT_BASE=reasoning-predictor, so dropping the name here breaks ingest
|
||||
# immediately. coder-predictor also serves `reasoning`; that ambiguity is known
|
||||
# and documented in the PR. Resolve it by moving the client host, not by
|
||||
# removing the name here.
|
||||
#
|
||||
# The two names are separate list entries, NOT one comma-joined value. This
|
||||
# image's parser returns ['reasoning,reasoning-long'] for the comma form, i.e. a
|
||||
# single model whose name contains a comma, and api_server.py then iterates
|
||||
# args.served_model_name to build the model list. Verified against vLLM 0.11.0:
|
||||
# FlexibleArgumentParser().parse_args([... "--served-model-name", "a", "b"])
|
||||
# -> ['a', 'b']
|
||||
- --served-model-name
|
||||
- reasoning
|
||||
- reasoning-long
|
||||
- --served-model-name=reasoning
|
||||
- --quantization=gptq
|
||||
- --dtype=float16
|
||||
- --kv-cache-dtype=auto
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
---
|
||||
# Public entry point for the poimen-memory HTTP API.
|
||||
#
|
||||
# The API was reachable only inside the cluster (ClusterIP :8080), so a local pi
|
||||
# session had no way to use it. This Ingress exposes the same handler set the
|
||||
# in-cluster clients use: /mcp, /memory/*, /graphql, /health, /ready.
|
||||
#
|
||||
# Cloudflare terminates TLS in front of the nginx ingress, which is why this rule
|
||||
# carries no `tls:` section — the same shape as ingress.yaml next to it.
|
||||
#
|
||||
# Auth: poimen-memory validates ZITADEL JWTs (MEM_AUTH_MODE=jwt). A caller
|
||||
# without a valid bearer token gets 401, so exposing the route does not expose
|
||||
# the data.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: poimen-api
|
||||
namespace: poimen
|
||||
annotations:
|
||||
# MCP calls and ingest can be slow; the provider may stream. Match the
|
||||
# frontend ingress so a long tool call is not cut at 60s.
|
||||
nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
|
||||
nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
|
||||
nginx.ingress.kubernetes.io/proxy-buffering: "off"
|
||||
nginx.ingress.kubernetes.io/proxy-body-size: "50m"
|
||||
spec:
|
||||
ingressClassName: nginx
|
||||
rules:
|
||||
- host: api-poimen.riotpiao.com
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: poimen-memory
|
||||
port:
|
||||
number: 8080
|
||||
@@ -6,5 +6,3 @@ kind: Kustomization
|
||||
# riotpiao-poimen/poimen-frontend.git k8s/ — each repo owns its own k8s.
|
||||
resources:
|
||||
- ingress.yaml
|
||||
# The memory API. Without it a local pi session has no route to :8080.
|
||||
- api-ingress.yaml
|
||||
|
||||
@@ -159,3 +159,54 @@ spec:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
---
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: kyverno
|
||||
namespace: argocd
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "0"
|
||||
spec:
|
||||
project: homelab
|
||||
source:
|
||||
repoURL: https://kyverno.github.io/kyverno/
|
||||
chart: kyverno
|
||||
targetRevision: "1.14.0"
|
||||
helm:
|
||||
valueFiles:
|
||||
- $values/k8s/bootstrap/kyverno/kyverno-values.yaml
|
||||
sources:
|
||||
- repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
|
||||
targetRevision: main
|
||||
ref: values
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: kyverno
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
---
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: kyverno-policies
|
||||
namespace: argocd
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "0"
|
||||
spec:
|
||||
project: homelab
|
||||
source:
|
||||
repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
|
||||
targetRevision: main
|
||||
path: k8s/bootstrap/kyverno
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: kyverno
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: comfyui
|
||||
namespace: argocd
|
||||
labels:
|
||||
app.kubernetes.io/name: comfyui
|
||||
app.kubernetes.io/component: image-generation
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "8"
|
||||
spec:
|
||||
project: homelab
|
||||
revisionHistoryLimit: 3
|
||||
source:
|
||||
repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
|
||||
targetRevision: main
|
||||
path: k8s/apps/comfyui
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: comfyui
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
retry:
|
||||
limit: 5
|
||||
backoff:
|
||||
duration: 5s
|
||||
factor: 2
|
||||
maxDuration: 3m
|
||||
@@ -0,0 +1,6 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: kyverno
|
||||
|
||||
resources:
|
||||
- policies.yaml
|
||||
@@ -0,0 +1,42 @@
|
||||
# Kyverno: Policy engine for Kubernetes image scanning, Pod security, and admission control
|
||||
# Scan all images, enforce baseline Pod Security Standard, prevent privilege escalation
|
||||
|
||||
replicaCount: 1
|
||||
|
||||
image:
|
||||
registry: ghcr.io
|
||||
repository: kyverno/kyverno
|
||||
tag: "v1.14.0"
|
||||
|
||||
config:
|
||||
# Webhook timeout for policy evaluation. Increase if scanning takes longer.
|
||||
webhookTimeoutSeconds: 30
|
||||
# Failure policy: fail-open (audit/log) vs fail-closed (reject on error)
|
||||
failurePolicy: fail
|
||||
# Resource limits for webhook
|
||||
webhookAnnotations:
|
||||
rules: "allow"
|
||||
|
||||
# Pod security via Kyverno instead of Pod Security Policies (deprecated)
|
||||
# Enforces baseline restrictions cluster-wide, with exceptions for privileged namespaces
|
||||
podSecurityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
|
||||
rbac:
|
||||
create: true
|
||||
|
||||
resources:
|
||||
requests:
|
||||
memory: "256Mi"
|
||||
cpu: "100m"
|
||||
limits:
|
||||
memory: "512Mi"
|
||||
cpu: "500m"
|
||||
|
||||
# Webhook configuration
|
||||
webhook:
|
||||
timeoutSeconds: 30
|
||||
# Failure policy: "Fail" (reject on error) or "Ignore" (audit-only)
|
||||
# Set to "Ignore" for initial testing, then change to "Fail"
|
||||
failurePolicy: ignore
|
||||
@@ -0,0 +1,204 @@
|
||||
# Kyverno ClusterPolicies: Image scanning, Pod security, and admission control
|
||||
---
|
||||
# Policy 1: Require non-root containers
|
||||
apiVersion: kyverno.io/v1
|
||||
kind: ClusterPolicy
|
||||
metadata:
|
||||
name: require-non-root
|
||||
namespace: kyverno
|
||||
spec:
|
||||
validationFailureAction: audit # audit first, then change to enforce
|
||||
rules:
|
||||
- name: check-runAsNonRoot
|
||||
match:
|
||||
any:
|
||||
- resources:
|
||||
kinds:
|
||||
- Pod
|
||||
selector:
|
||||
matchLabels:
|
||||
pod-security.kubernetes.io/enforce: "!privileged"
|
||||
validate:
|
||||
message: "Container must not run as root"
|
||||
pattern:
|
||||
spec:
|
||||
containers:
|
||||
- securityContext:
|
||||
runAsNonRoot: true
|
||||
---
|
||||
# Policy 2: Drop all Linux capabilities, add only required ones
|
||||
apiVersion: kyverno.io/v1
|
||||
kind: ClusterPolicy
|
||||
metadata:
|
||||
name: require-dropped-caps
|
||||
namespace: kyverno
|
||||
spec:
|
||||
validationFailureAction: audit
|
||||
rules:
|
||||
- name: drop-all-capabilities
|
||||
match:
|
||||
any:
|
||||
- resources:
|
||||
kinds:
|
||||
- Pod
|
||||
selector:
|
||||
matchLabels:
|
||||
pod-security.kubernetes.io/enforce: "!privileged"
|
||||
validate:
|
||||
message: "All Linux capabilities must be dropped"
|
||||
pattern:
|
||||
spec:
|
||||
containers:
|
||||
- securityContext:
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
---
|
||||
# Policy 3: Require image tags (no 'latest')
|
||||
apiVersion: kyverno.io/v1
|
||||
kind: ClusterPolicy
|
||||
metadata:
|
||||
name: disallow-latest-tag
|
||||
namespace: kyverno
|
||||
spec:
|
||||
validationFailureAction: audit # Change to enforce after testing
|
||||
rules:
|
||||
- name: disallow-latest
|
||||
match:
|
||||
any:
|
||||
- resources:
|
||||
kinds:
|
||||
- Pod
|
||||
- Deployment
|
||||
- StatefulSet
|
||||
- DaemonSet
|
||||
- Job
|
||||
validate:
|
||||
message: "Image tag 'latest' is not allowed. Use explicit version tags."
|
||||
pattern:
|
||||
spec:
|
||||
=(template):
|
||||
spec:
|
||||
containers:
|
||||
- image: "!*:latest"
|
||||
=(initContainers):
|
||||
- image: "!*:latest"
|
||||
---
|
||||
# Policy 4: Restrict images to trusted registries
|
||||
apiVersion: kyverno.io/v1
|
||||
kind: ClusterPolicy
|
||||
metadata:
|
||||
name: restrict-registries
|
||||
namespace: kyverno
|
||||
spec:
|
||||
validationFailureAction: audit
|
||||
rules:
|
||||
- name: trusted-registries
|
||||
match:
|
||||
any:
|
||||
- resources:
|
||||
kinds:
|
||||
- Pod
|
||||
- Deployment
|
||||
- StatefulSet
|
||||
- DaemonSet
|
||||
- Job
|
||||
selector:
|
||||
matchLabels:
|
||||
pod-security.kubernetes.io/enforce: "!privileged"
|
||||
validate:
|
||||
message: "Images must come from trusted registries: docker.io, ghcr.io, quay.io, k8s.gcr.io, registry.k8s.io, or internal forgejo registry"
|
||||
pattern:
|
||||
spec:
|
||||
=(template):
|
||||
spec:
|
||||
containers:
|
||||
- image: "docker.io/* | ghcr.io/* | quay.io/* | k8s.gcr.io/* | registry.k8s.io/* | forgejo.riotpiao.com/* | *"
|
||||
---
|
||||
# Policy 5: Require read-only root filesystem (audit only, exceptions for apps that need writes)
|
||||
apiVersion: kyverno.io/v1
|
||||
kind: ClusterPolicy
|
||||
metadata:
|
||||
name: require-readonly-filesystem
|
||||
namespace: kyverno
|
||||
spec:
|
||||
validationFailureAction: audit
|
||||
rules:
|
||||
- name: check-readOnlyRootFilesystem
|
||||
match:
|
||||
any:
|
||||
- resources:
|
||||
kinds:
|
||||
- Pod
|
||||
selector:
|
||||
matchLabels:
|
||||
pod-security.kubernetes.io/enforce: "!privileged"
|
||||
validate:
|
||||
message: "Root filesystem should be read-only for defense-in-depth"
|
||||
pattern:
|
||||
spec:
|
||||
containers:
|
||||
- securityContext:
|
||||
readOnlyRootFilesystem: true
|
||||
---
|
||||
# Policy 6: Require resource requests and limits (prevent resource starvation)
|
||||
apiVersion: kyverno.io/v1
|
||||
kind: ClusterPolicy
|
||||
metadata:
|
||||
name: require-resource-limits
|
||||
namespace: kyverno
|
||||
spec:
|
||||
validationFailureAction: audit
|
||||
rules:
|
||||
- name: check-resources
|
||||
match:
|
||||
any:
|
||||
- resources:
|
||||
kinds:
|
||||
- Pod
|
||||
- Deployment
|
||||
- StatefulSet
|
||||
- DaemonSet
|
||||
excludeResources:
|
||||
namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: "kyverno|kube-system|kube-node-lease"
|
||||
validate:
|
||||
message: "CPU and memory requests and limits are required"
|
||||
pattern:
|
||||
spec:
|
||||
=(template):
|
||||
spec:
|
||||
containers:
|
||||
- resources:
|
||||
requests:
|
||||
memory: "?*"
|
||||
cpu: "?*"
|
||||
limits:
|
||||
memory: "?*"
|
||||
cpu: "?*"
|
||||
---
|
||||
# Policy 7: Require securityContext on all containers
|
||||
apiVersion: kyverno.io/v1
|
||||
kind: ClusterPolicy
|
||||
metadata:
|
||||
name: require-security-context
|
||||
namespace: kyverno
|
||||
spec:
|
||||
validationFailureAction: audit
|
||||
rules:
|
||||
- name: check-securityContext
|
||||
match:
|
||||
any:
|
||||
- resources:
|
||||
kinds:
|
||||
- Pod
|
||||
selector:
|
||||
matchLabels:
|
||||
pod-security.kubernetes.io/enforce: "!privileged"
|
||||
validate:
|
||||
message: "securityContext must be defined"
|
||||
pattern:
|
||||
spec:
|
||||
containers:
|
||||
- securityContext: {}
|
||||
@@ -97,25 +97,10 @@ gitea:
|
||||
# USER and PASSWD injected via env vars below (GITEA__MAILER__USER, GITEA__MAILER__PASSWD)
|
||||
|
||||
# Persistence (shared storage for repos)
|
||||
#
|
||||
# 60Gi, not the original 20Gi. The container registry stores its blobs here,
|
||||
# and a single candle-vllm sm_70 image is 14GB uncompressed. At 20Gi the volume
|
||||
# filled, and every blob upload failed with:
|
||||
# Package registry API internal error: 500 write /data/tmp/package-upload/...:
|
||||
# no space left on device
|
||||
# The push client surfaces that as a bare `received unexpected HTTP status: 500`
|
||||
# after retrying every layer, which looks like a network fault and is not one.
|
||||
#
|
||||
# Note for whoever expands this again: Longhorn's admission webhook checks the
|
||||
# PHYSICAL free space of the disk holding each replica, not the thin volume
|
||||
# size. A request for 100Gi was rejected while the disk had 23.5Gi free:
|
||||
# disk ... does not have sufficient physical space for expansion:
|
||||
# physical free space would drop below minimal: left=14596177920 < minimal=49931236556
|
||||
# 60Gi fit. Raise it in steps and watch the webhook message.
|
||||
persistence:
|
||||
enabled: true
|
||||
storageClass: longhorn
|
||||
size: 60Gi
|
||||
size: 20Gi
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
|
||||
@@ -149,29 +134,10 @@ ingress:
|
||||
# wildcard. The wildcard covers this host.
|
||||
|
||||
# Resources
|
||||
#
|
||||
# The 1000m ceiling throttled Gitea badly. Measured on a running pod:
|
||||
# nr_throttled 189528 of 1169014 CFS periods (16.2%)
|
||||
# throttled_usec 4424898602 us (~74 minutes lost)
|
||||
# actual usage 951m against the 1000m limit
|
||||
# The visible symptom was slow Actions traffic: FetchTask took 3469.9ms and
|
||||
# repository compare endpoints took 3091-3811ms, so runners timed out and
|
||||
# retried, which added more load to an already saturated process.
|
||||
#
|
||||
# The database was never the bottleneck. A real action_runner UPDATE measured
|
||||
# 2.133ms with zero lock waits, so this is CPU starvation in Gitea itself.
|
||||
#
|
||||
# requests.cpu stays 200m and limits.cpu stays 1000m. The 3000m limit was a
|
||||
# temporary measure while the Actions stall was diagnosed; the real fix landed
|
||||
# elsewhere, so the limit is back to the original.
|
||||
#
|
||||
# Caution: this pod runs on talos-cp-3, a control-plane node. If Gitea
|
||||
# throttles again at 1000m, move the workload off the control plane rather
|
||||
# than raising the limit.
|
||||
resources:
|
||||
requests:
|
||||
cpu: 200m
|
||||
memory: 768Mi
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: 1000m
|
||||
memory: 2Gi
|
||||
|
||||
@@ -1,90 +0,0 @@
|
||||
# One-shot destructive health scan of a candidate Longhorn disk.
|
||||
#
|
||||
# WHY THIS IS A MANUAL JOB AND NOT AN ARGOCD HOOK
|
||||
# It writes over the entire raw device. An ArgoCD PostSync hook would run it
|
||||
# on every sync and destroy whatever the volume holds. Apply it by hand,
|
||||
# once, on a disk that has no partitions and no mounts.
|
||||
#
|
||||
# WHY NOT TALOS
|
||||
# Talos declares disks by mounting them (machine.disks in the machine
|
||||
# config). It has no primitive for "run this command on the host", so a scan
|
||||
# cannot live in the Talos config. This Job is the IaC-correct home.
|
||||
#
|
||||
# ORDER OF OPERATIONS
|
||||
# 1. Scan the raw device with this Job <- the disk must be UNMOUNTED
|
||||
# 2. If it passes, add the disk to terraform.tfvars.local `longhorn_disks`
|
||||
# with device /dev/disk/by-id/<wwid> and run `make apply-cp`
|
||||
# 3. Register it with Longhorn via the pattern in longhorn-add-disks-job.yaml
|
||||
#
|
||||
# What it does
|
||||
# fio writes a CRC32C-verified pattern across the whole device, then reads
|
||||
# every block back and compares. Any unreadable or mismatched sector fails
|
||||
# the Job. This is a full write and read pass, not a sample.
|
||||
#
|
||||
# Expected duration
|
||||
# 2 TB on a 5400 rpm 2.5" drive at roughly 90 MB/s is about 6 h to write and
|
||||
# 6 h to verify. Budget 12 hours. Check progress with:
|
||||
# kubectl -n llm-io logs job/disk-scan-cp1-sdb -f
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: disk-scan-cp1-sdb
|
||||
namespace: llm-io
|
||||
labels:
|
||||
app: disk-scan
|
||||
spec:
|
||||
backoffLimit: 0
|
||||
# Do not let a stuck scan retry silently.
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: disk-scan
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: talos-cp-1
|
||||
containers:
|
||||
- name: scan
|
||||
image: ubuntu:22.04
|
||||
securityContext:
|
||||
privileged: true
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
set -euo pipefail
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq smartmontools fio
|
||||
|
||||
echo "=== device identity ==="
|
||||
smartctl -i /dev/sdb | tail -12
|
||||
|
||||
echo "=== SMART before ==="
|
||||
smartctl -H /dev/sdb | tail -2
|
||||
|
||||
echo "=== full write + verify (destructive) ==="
|
||||
fio --name=devverify \
|
||||
--filename=/dev/sdb \
|
||||
--rw=write \
|
||||
--bs=1M \
|
||||
--iodepth=4 \
|
||||
--ioengine=libaio \
|
||||
--direct=1 \
|
||||
--verify=crc32c \
|
||||
--do_verify=1 \
|
||||
--verify_fatal=1 \
|
||||
--eta=always \
|
||||
--output-format=normal
|
||||
|
||||
echo "=== SMART after ==="
|
||||
smartctl -H /dev/sdb | tail -2
|
||||
smartctl -A /dev/sdb | awk '/Reallocated_Sector|Current_Pending|Offline_Uncorrect|UDMA_CRC/{print " "$0}'
|
||||
|
||||
echo "=== SCAN PASSED ==="
|
||||
volumeMounts:
|
||||
- name: dev
|
||||
mountPath: /dev
|
||||
volumes:
|
||||
- name: dev
|
||||
hostPath:
|
||||
path: /dev
|
||||
@@ -1,28 +0,0 @@
|
||||
---
|
||||
# Keep the GraphQL server reachable only from the one service that proxies it.
|
||||
#
|
||||
# poimen-graphql is already off the internet: its Service is ClusterIP on 8090 and
|
||||
# no Ingress names it. What it lacked is in-cluster isolation — a ClusterIP answers
|
||||
# every pod in the cluster.
|
||||
#
|
||||
# The only caller is poimen-memory, whose /graphql route forwards the request body
|
||||
# to MEM_GQL_PORT (8090). No other workload references this service.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: poimen-graphql-ingress
|
||||
namespace: poimen
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: poimen-graphql
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: poimen-memory
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 8090
|
||||
@@ -6,10 +6,6 @@ kind: Kustomization
|
||||
# deployments (wave 0) are applied.
|
||||
resources:
|
||||
- memory-db.yaml
|
||||
# ClusterIP alone is not isolation: every pod in the cluster can reach it.
|
||||
- memory-db-netpol.yaml
|
||||
# Only poimen-memory's /graphql proxy calls 8090.
|
||||
- graphql-netpol.yaml
|
||||
- poimen-sandbox-namespace.yaml
|
||||
- poimen-sandbox-quota.yaml
|
||||
- pi-session-netpol.yaml
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
---
|
||||
# Keep the memory database reachable only from inside its own namespace.
|
||||
#
|
||||
# What this does NOT change: the database was already unreachable from the
|
||||
# internet. `memory-db-rw`, `memory-db-ro` and `memory-db-r` are ClusterIP
|
||||
# services, no Ingress backs them, and no public DNS name resolves to them.
|
||||
#
|
||||
# What it fixes: a ClusterIP service is reachable by EVERY pod in the cluster,
|
||||
# because Kubernetes has no implicit isolation. Cilium enforces NetworkPolicy, so
|
||||
# stating the allowed sources is what turns "not on the internet" into "not
|
||||
# reachable from another tenant's namespace".
|
||||
#
|
||||
# Two sources, both required:
|
||||
# - the poimen namespace, which holds every client: poimen-memory, poimen-graphql,
|
||||
# poimen-worker, poimen-workflows and the migration Job.
|
||||
# - cnpg-system, where the CloudNativePG operator lives. It connects to the
|
||||
# instance for switchover and failover, so a narrower rule breaks recovery.
|
||||
#
|
||||
# No other port is opened. The operator's metrics endpoint stays closed because no
|
||||
# PodMonitor for this cluster exists, so nothing scrapes it.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: memory-db-ingress
|
||||
namespace: poimen
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
cnpg.io/cluster: memory-db
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: poimen
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: cnpg-system
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 5432
|
||||
@@ -32,6 +32,7 @@ ISSUER = "https://zitadel.riotpiao.com"
|
||||
APPS = [
|
||||
("grafana", "grafana", "grafana-oidc", "logging", "GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET", "deploy", "grafana"),
|
||||
("argocd", "argocd", "oidc-secret", "argocd", "client-secret", "deploy", "argocd-server"),
|
||||
("comfyui", "comfyui", "comfyui-oauth2-proxy", "comfyui", "client-secret", "deploy", "comfyui-oauth2-proxy"),
|
||||
("minio", "minio", "minio-oidc", "storage", "MINIO_IDENTITY_OPENID_CLIENT_SECRET", "sts", "minio-cluster-az-a"),
|
||||
("paperless", "paperless", "paperless-oidc", "paperless", "client-secret", "deploy", "paperless"),
|
||||
("immich", "immich", "immich-oidc", "immich", "client-secret", "deploy", "immich-server"),
|
||||
|
||||
@@ -128,6 +128,11 @@ OIDC_APPS = {
|
||||
"https://forgejo.riotpiao.com/user/oauth2/zitadel/callback",
|
||||
"http://localhost:3000/user/oauth2/authentik/callback",
|
||||
"https://forgejo.riotpiao.com/user/oauth2/authentik/callback"]},
|
||||
# ComfyUI has no native OIDC. oauth2-proxy is its front door and this app is
|
||||
# the proxy's client, so the callback path belongs to the proxy.
|
||||
"comfyui": {**_WEB, "redirect_uris": [
|
||||
"http://localhost:4180/oauth2/callback",
|
||||
"https://comfyui.riotpiao.com/oauth2/callback"]},
|
||||
"grafana": {**_WEB, "redirect_uris": [
|
||||
"http://localhost:3000/login/generic_oauth",
|
||||
"https://grafana.riotpiao.com/login/generic_oauth"]},
|
||||
@@ -138,11 +143,7 @@ OIDC_APPS = {
|
||||
"http://localhost:8000", "http://localhost:18000"]},
|
||||
"minio": {**_WEB, "redirect_uris": [
|
||||
"http://localhost:9000/auth/sso/oauth2/code",
|
||||
"https://minio.riotpiao.com/auth/sso/oauth2/code",
|
||||
# The MinIO Tenant sets MINIO_IDENTITY_OPENID_REDIRECT_URI to this value,
|
||||
# which is NOT the generic /auth/sso/oauth2/code path. Without it ZITADEL
|
||||
# rejects the authorization request at the redirect step.
|
||||
"https://minio.riotpiao.com/oauth_callback"]},
|
||||
"https://minio.riotpiao.com/auth/sso/oauth2/code"]},
|
||||
"paperless": {**_WEB, "redirect_uris": [
|
||||
"http://localhost:8000/accounts/oidc/zitadel/login/callback/",
|
||||
"https://paperless.riotpiao.com/accounts/oidc/zitadel/login/callback/",
|
||||
@@ -358,13 +359,6 @@ MACHINE_CALLERS = {
|
||||
"description": "poimen calls api-gateway inference",
|
||||
"roles": ["llm:inference", "svc-caller"],
|
||||
},
|
||||
# paperless-ai is a vendor image: it cannot sign an assertion, so a sidecar mints
|
||||
# a token with this account and writes it to the file the image already reads.
|
||||
"paperless-llm": {
|
||||
"name": "Paperless AI LLM caller",
|
||||
"description": "paperless-ai calls api-gateway inference through the token vendor",
|
||||
"roles": ["llm:inference", "svc-caller"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -36,10 +36,8 @@
|
||||
rewrite name paperless.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
|
||||
rewrite name img.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
|
||||
rewrite name api.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
|
||||
# poimen ran only in the live ConfigMap, never in this file or the Talos
|
||||
# machine config. The next Talos reconcile would have deleted it and broken
|
||||
# in-cluster resolution. Declared here so the source of truth carries it.
|
||||
rewrite name poimen.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
|
||||
rewrite name comfy.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
|
||||
rewrite name comfyui.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
|
||||
rewrite name gotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
|
||||
rewrite name pi.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
|
||||
rewrite name igotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
|
||||
|
||||
@@ -39,9 +39,6 @@ resource "local_file" "controlplane_configs" {
|
||||
dns_servers = var.cluster_config.dns_servers
|
||||
forgejo_registry_ip = var.forgejo_registry_ip
|
||||
forgejo_hostname = var.forgejo_hostname
|
||||
oidc_issuer_hostname = var.oidc_issuer_hostname
|
||||
oidc_issuer_ip = var.oidc_issuer_ip
|
||||
oidc_client_id = var.oidc_client_id
|
||||
zone = each.value.zone
|
||||
allow_scheduling = each.value.allow_scheduling
|
||||
coredns_corefile = file("${path.module}/files/coredns/Corefile")
|
||||
|
||||
@@ -33,14 +33,6 @@ machine:
|
||||
- ip: ${forgejo_registry_ip}
|
||||
aliases:
|
||||
- ${forgejo_hostname}
|
||||
# kube-apiserver's static pod runs on the host network, so it resolves the
|
||||
# OIDC issuer through THIS node's DNS, not cluster CoreDNS. Pin the issuer
|
||||
# to the ingress LoadBalancer so discovery stays on the LAN and presents
|
||||
# the publicly-trusted Let's Encrypt wildcard certificate. Without this
|
||||
# entry the lookup leaves via public DNS and Cloudflare.
|
||||
- ip: ${oidc_issuer_ip}
|
||||
aliases:
|
||||
- ${oidc_issuer_hostname}
|
||||
kubelet:
|
||||
image: ${kubelet_image}
|
||||
defaultRuntimeSeccompProfileEnabled: true
|
||||
@@ -128,31 +120,10 @@ cluster:
|
||||
- ${san}
|
||||
%{ endfor ~}
|
||||
image: ${kube_apiserver_img}
|
||||
# OIDC authentication against ZITADEL. Additive: client-certificate auth
|
||||
# (this session's own admin@homelab-cluster kubeconfig, system:masters)
|
||||
# keeps working unchanged, so a bad value here degrades OIDC logins rather
|
||||
# than locking anyone out.
|
||||
#
|
||||
# The issuer has NO trailing slash and NO /application/o/<app>/ suffix.
|
||||
# That suffix belonged to the Authentik scheme; the apiserver compares `iss`
|
||||
# in the id_token by exact string equality, so the stale value made every
|
||||
# OIDC login fail once Authentik stopped running.
|
||||
#
|
||||
# Groups: ZITADEL asserts roles as a nested urn claim that the apiserver
|
||||
# cannot read. The `groupsClaim` action in ZITADEL mirrors the bare role
|
||||
# names into a flat `groups` claim, which is what oidc-groups-claim consumes.
|
||||
# Verified end to end: a PKCE id_token for this client carries
|
||||
# groups=[homelab-admins, ...], email and preferred_username.
|
||||
#
|
||||
# Prefixes keep OIDC identities from colliding with built-in system:* users
|
||||
# and groups. k8s RBAC bindings therefore target `oidc:homelab-admins`.
|
||||
extraArgs:
|
||||
oidc-issuer-url: https://${oidc_issuer_hostname}
|
||||
oidc-client-id: "${oidc_client_id}"
|
||||
oidc-username-claim: email
|
||||
oidc-groups-claim: groups
|
||||
oidc-username-prefix: "oidc:"
|
||||
oidc-groups-prefix: "oidc:"
|
||||
# No OIDC block. The cluster authenticates with client certificates. An
|
||||
# earlier Authentik-based kube-apiserver OIDC config was removed from the
|
||||
# live nodes, and leaving it here would re-add an issuer that no longer
|
||||
# resolves on any future machine-config render.
|
||||
admissionControl:
|
||||
- name: PodSecurity
|
||||
configuration:
|
||||
|
||||
@@ -198,42 +198,3 @@ variable "forgejo_hostname" {
|
||||
default = "forgejo.riotpiao.com"
|
||||
description = "Forgejo external hostname"
|
||||
}
|
||||
|
||||
# ── kube-apiserver OIDC (ZITADEL) ────────────────────────────────────────────
|
||||
# Authenticates `kubectl` against ZITADEL instead of client certificates only.
|
||||
# Client-certificate auth keeps working in parallel, so a wrong value here
|
||||
# degrades OIDC logins without locking anyone out.
|
||||
#
|
||||
# The issuer is ZITADEL's external host with NO trailing slash and NO
|
||||
# /application/o/<app>/ suffix — a leftover of the Authentik scheme. The
|
||||
# apiserver validates `iss` in the id_token for exact string equality.
|
||||
variable "oidc_issuer_hostname" {
|
||||
type = string
|
||||
default = "zitadel.riotpiao.com"
|
||||
description = "Hostname of the OIDC issuer (ZITADEL). Becomes https://<host> as the issuer."
|
||||
}
|
||||
|
||||
variable "oidc_issuer_ip" {
|
||||
type = string
|
||||
default = "192.168.1.160"
|
||||
description = <<-EOT
|
||||
Address the nodes resolve the OIDC issuer to, via extraHostEntries.
|
||||
|
||||
kube-apiserver's static pod runs on the host network, so it resolves the
|
||||
issuer through the node's own DNS, not cluster CoreDNS. Pointing it at the
|
||||
ingress-nginx LoadBalancer keeps OIDC discovery on the LAN and serves the
|
||||
publicly-trusted Let's Encrypt *.riotpiao.com certificate. Without this the
|
||||
lookup leaves via public DNS and Cloudflare, and discovery either hairpins
|
||||
or fails.
|
||||
EOT
|
||||
}
|
||||
|
||||
variable "oidc_client_id" {
|
||||
type = string
|
||||
default = "396757803089330177"
|
||||
description = <<-EOT
|
||||
ZITADEL client id of the `kubernetes` OIDC application (PKCE, no secret).
|
||||
Must be a STRING: the id is an 18-digit number and an unquoted YAML scalar
|
||||
would be parsed as a float and rejected.
|
||||
EOT
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user