Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 rock
|
747a6fb9cb |
@@ -45,6 +45,8 @@ data:
|
||||
service: https://igotify.riotpiao.com
|
||||
- hostname: poimen.riotpiao.com
|
||||
service: https://poimen.riotpiao.com
|
||||
- hostname: comfyui.riotpiao.com
|
||||
service: https://comfyui.riotpiao.com
|
||||
- hostname: forgejo.riotpiao.com
|
||||
service: https://forgejo.riotpiao.com
|
||||
# ZITADEL. Without this rule the connector answers 404 for the hostname
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
# ComfyUI — GPU-accelerated image generation on worker-1.
|
||||
# Uses 1x V100 32GB (sm70). Freed by scaling ornith 2→1.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: comfyui
|
||||
namespace: comfyui
|
||||
labels:
|
||||
app: comfyui
|
||||
spec:
|
||||
# Retired on purpose. This Deployment held the 4th V100 on worker-1 while
|
||||
# serving nothing: its oauth2-proxy roll-out has been stuck for 42h and
|
||||
# failing 11621 times, because the Secret comfyui/comfyui-oauth2-proxy carries
|
||||
# only 1 key while the pod requests client-id, client-secret and
|
||||
# cookie-secret. Every request was therefore refused, so the card sat at 0%
|
||||
# utilisation and blocked a GPU that llm-serving can use.
|
||||
#
|
||||
# Scaling to 0 is the whole change. The Deployment, the Service, the Ingress
|
||||
# and the 50Gi comfyui-models PVC all stay, so nothing is lost and no data is
|
||||
# touched. To bring it back: set replicas to 1 here, then fix the Secret to
|
||||
# carry all three keys, otherwise the same stuck roll-out returns.
|
||||
replicas: 0
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: comfyui
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: comfyui
|
||||
spec:
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: worker-1
|
||||
runtimeClassName: nvidia
|
||||
# k8s Service named 'comfyui' injects COMFYUI_PORT=tcp://... into pod env,
|
||||
# which clobbers ai-dock's own COMFYUI_PORT variable (expects a port number).
|
||||
# Disable service link injection to avoid the collision.
|
||||
enableServiceLinks: false
|
||||
containers:
|
||||
- name: comfyui
|
||||
image: ghcr.io/ai-dock/comfyui:v2-cuda-12.1.1-base-22.04
|
||||
ports:
|
||||
- containerPort: 8188
|
||||
protocol: TCP
|
||||
env:
|
||||
- name: NVIDIA_VISIBLE_DEVICES
|
||||
value: "all"
|
||||
resources:
|
||||
requests:
|
||||
cpu: "4"
|
||||
memory: 8Gi
|
||||
nvidia.com/gpu: "1"
|
||||
limits:
|
||||
cpu: "8"
|
||||
memory: 16Gi
|
||||
nvidia.com/gpu: "1"
|
||||
volumeMounts:
|
||||
- mountPath: /workspace/ComfyUI/models
|
||||
name: models
|
||||
- mountPath: /workspace/ComfyUI/output
|
||||
name: output
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: 8188
|
||||
periodSeconds: 10
|
||||
initialDelaySeconds: 30
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: 8188
|
||||
failureThreshold: 120
|
||||
periodSeconds: 10
|
||||
volumes:
|
||||
- name: models
|
||||
persistentVolumeClaim:
|
||||
claimName: comfyui-models
|
||||
- name: output
|
||||
emptyDir: {}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: comfyui-models
|
||||
namespace: comfyui
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: longhorn
|
||||
resources:
|
||||
requests:
|
||||
storage: 50Gi
|
||||
@@ -0,0 +1,38 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: comfyui
|
||||
namespace: comfyui
|
||||
annotations:
|
||||
# No cert-manager annotation and no tls: block. This host is covered by
|
||||
# the *.riotpiao.com wildcard, which nginx serves as its
|
||||
# default-ssl-certificate (ingress-nginx/riotpiao-com-tls).
|
||||
nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
|
||||
nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
|
||||
nginx.ingress.kubernetes.io/proxy-body-size: "0"
|
||||
# WebSocket support for ComfyUI's live preview
|
||||
nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
|
||||
nginx.ingress.kubernetes.io/upstream-hash-by: "$remote_addr"
|
||||
# Single sign-on. ComfyUI has no auth of its own, so every request is handed to
|
||||
# oauth2-proxy for an allow or deny decision.
|
||||
nginx.ingress.kubernetes.io/auth-url: "http://comfyui-oauth2-proxy.comfyui.svc.cluster.local:4180/oauth2/auth"
|
||||
nginx.ingress.kubernetes.io/auth-signin: "https://comfyui.riotpiao.com/oauth2/start?rd=$scheme://$host$request_uri"
|
||||
nginx.ingress.kubernetes.io/auth-response-headers: "x-auth-request-user,x-auth-request-email"
|
||||
# No configuration-snippet. The controller rejects that annotation as risky
|
||||
# ("annotation group ConfigurationSnippet contains risky annotation"), and a
|
||||
# rejected annotation drops the WHOLE ingress from nginx.conf: the host then
|
||||
# answers 404 with no entry in the access log. ingress-nginx sets the Upgrade
|
||||
# and Connection headers itself for a websocket upgrade.
|
||||
spec:
|
||||
ingressClassName: nginx
|
||||
rules:
|
||||
- host: comfyui.riotpiao.com
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: comfyui
|
||||
port:
|
||||
number: 80
|
||||
@@ -0,0 +1,8 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
resources:
|
||||
- deployment.yaml
|
||||
- service.yaml
|
||||
- oauth2-proxy.yaml
|
||||
- ingress.yaml
|
||||
@@ -0,0 +1,105 @@
|
||||
# ComfyUI has no authentication of its own: comfyui.riotpiao.com is open to anyone
|
||||
# who can resolve the host. It also has no OIDC support, so ZITADEL cannot be a
|
||||
# login method inside it. oauth2-proxy is the front door instead: nginx asks it for
|
||||
# a decision on every request and it answers from a ZITADEL session.
|
||||
#
|
||||
# No API client calls this host today, so the whole host is gated. If a client ever
|
||||
# needs direct access, add a skip rule for the exact paths it uses rather than
|
||||
# opening the whole host.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: comfyui-oauth2-proxy
|
||||
namespace: comfyui
|
||||
labels:
|
||||
app: comfyui-oauth2-proxy
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: comfyui-oauth2-proxy
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: comfyui-oauth2-proxy
|
||||
spec:
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65534
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
containers:
|
||||
- name: oauth2-proxy
|
||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.7.1
|
||||
args:
|
||||
- --provider=oidc
|
||||
- --oidc-issuer-url=https://zitadel.riotpiao.com
|
||||
- --email-domain=*
|
||||
- --upstream=static://202
|
||||
- --reverse-proxy=true
|
||||
- --set-xauthrequest=true
|
||||
- --cookie-secure=true
|
||||
- --cookie-expire=12h
|
||||
- --cookie-refresh=1h
|
||||
- --skip-provider-button=true
|
||||
- --redirect-url=https://comfyui.riotpiao.com/oauth2/callback
|
||||
- --http-address=0.0.0.0:4180
|
||||
env:
|
||||
- name: OAUTH2_PROXY_CLIENT_ID
|
||||
valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: client-id}}
|
||||
- name: OAUTH2_PROXY_CLIENT_SECRET
|
||||
valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: client-secret}}
|
||||
- name: OAUTH2_PROXY_COOKIE_SECRET
|
||||
valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: cookie-secret}}
|
||||
ports:
|
||||
- {name: http, containerPort: 4180}
|
||||
readinessProbe:
|
||||
httpGet: {path: /ping, port: http}
|
||||
initialDelaySeconds: 5
|
||||
livenessProbe:
|
||||
httpGet: {path: /ping, port: http}
|
||||
initialDelaySeconds: 15
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities: {drop: ["ALL"]}
|
||||
resources:
|
||||
requests: {cpu: 10m, memory: 32Mi}
|
||||
limits: {cpu: 200m, memory: 128Mi}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: comfyui-oauth2-proxy
|
||||
namespace: comfyui
|
||||
labels:
|
||||
app: comfyui-oauth2-proxy
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: comfyui-oauth2-proxy
|
||||
ports:
|
||||
- {name: http, port: 4180, targetPort: http}
|
||||
---
|
||||
# The login path must NOT be gated: a request to /oauth2/start that itself demands a
|
||||
# login redirects to /oauth2/start forever. ingress-nginx applies the annotations of
|
||||
# an ingress to the locations it defines, so a separate object without auth-url keeps
|
||||
# this path open. The earlier failure of this shape was not the split — it was the
|
||||
# configuration-snippet on the app ingress, which made the controller drop the whole
|
||||
# object and left only this one alive.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: comfyui-oauth2
|
||||
namespace: comfyui
|
||||
spec:
|
||||
ingressClassName: nginx
|
||||
rules:
|
||||
- host: comfyui.riotpiao.com
|
||||
http:
|
||||
paths:
|
||||
- path: /oauth2
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: comfyui-oauth2-proxy
|
||||
port:
|
||||
number: 4180
|
||||
@@ -0,0 +1,14 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: comfyui
|
||||
namespace: comfyui
|
||||
labels:
|
||||
app: comfyui
|
||||
spec:
|
||||
selector:
|
||||
app: comfyui
|
||||
ports:
|
||||
- port: 80
|
||||
targetPort: 8188
|
||||
protocol: TCP
|
||||
@@ -8,11 +8,7 @@ metadata:
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
|
||||
spec:
|
||||
# 2, not 3. The file's own header always said 2, and the third instance never
|
||||
# existed as a running pod: its volume only ever held a replica that could not
|
||||
# start, because talos-cp-2 disk2 has no free space. A third replica on a disk
|
||||
# that cannot host it is not redundancy, it is a permanent degraded volume.
|
||||
instances: 2
|
||||
instances: 3
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16.2
|
||||
bootstrap:
|
||||
initdb:
|
||||
|
||||
@@ -40,7 +40,7 @@ target is one root-of-trust file per thing that cannot rotate itself:
|
||||
| controller bootstrap key | chicken and egg: it creates every other key |
|
||||
| etcd defrag credentials | needed by a job that runs outside the controller |
|
||||
|
||||
Everything else — Grafana, Argo CD, Forgejo, MinIO, Paperless, Immich,
|
||||
Everything else — Grafana, Argo CD, Forgejo, MinIO, Paperless, Immich, ComfyUI,
|
||||
api-gw, Gotify — is a KeyRotation object, and its secret leaves git.
|
||||
|
||||
## Status
|
||||
|
||||
@@ -64,6 +64,32 @@ spec:
|
||||
---
|
||||
apiVersion: iam.homelab.riotpiao.com/v1
|
||||
kind: KeyRotation
|
||||
metadata:
|
||||
name: comfyui
|
||||
namespace: iam
|
||||
labels:
|
||||
app.kubernetes.io/part-of: key-rotation
|
||||
spec:
|
||||
provider: zitadel-oidc-client
|
||||
strategy: atomic
|
||||
intervalHours: 2160
|
||||
suspend: true
|
||||
target:
|
||||
zitadelProjectID: "396757796898471937"
|
||||
zitadelAppID: "397015743994003854" # app resource id, not the client id 397015744010781070
|
||||
zitadelClientID: "397015744010781070"
|
||||
secretName: comfyui-oauth2-proxy
|
||||
secretNamespace: comfyui
|
||||
secretKeys:
|
||||
secret: client-secret
|
||||
reload: restart
|
||||
consumers:
|
||||
- {kind: Deployment, name: comfyui-oauth2-proxy, namespace: comfyui}
|
||||
verify:
|
||||
introspect: true
|
||||
---
|
||||
apiVersion: iam.homelab.riotpiao.com/v1
|
||||
kind: KeyRotation
|
||||
metadata:
|
||||
name: minio
|
||||
namespace: iam
|
||||
@@ -81,11 +107,7 @@ spec:
|
||||
secretName: minio-oidc
|
||||
secretNamespace: storage
|
||||
secretKeys:
|
||||
# The key name here must match the consumer's secretKeyRef, not a house style.
|
||||
# minio-tenant.yaml reads MINIO_IDENTITY_OPENID_CLIENT_SECRET. While this said
|
||||
# `client-secret`, the tenant failed with CreateContainerConfigError on every
|
||||
# restart and the rotation wrote a key nobody read.
|
||||
secret: MINIO_IDENTITY_OPENID_CLIENT_SECRET
|
||||
secret: client-secret
|
||||
reload: hot
|
||||
consumers:
|
||||
- {kind: StatefulSet, name: minio-cluster-az-a, namespace: storage}
|
||||
|
||||
@@ -33,19 +33,7 @@ metadata:
|
||||
app: clef-serve
|
||||
app.kubernetes.io/part-of: llm-serving
|
||||
spec:
|
||||
# Parked at 0 to free the only V100 that was not serving anything, so candle-vllm can
|
||||
# run its Phase 0 on one card. All four cards were allocated (coder 2, reasoning 1,
|
||||
# this 1) and a new GPU workload had nowhere to land.
|
||||
#
|
||||
# This is the only GPU consumer in the namespace that is not vLLM, which is why it is
|
||||
# the first card released: the standing instruction is to keep the vLLM pods healthy
|
||||
# until candle-vllm answers, and this pod is not one of them.
|
||||
#
|
||||
# It was serving nothing. The last 30 log lines at park time were 30/30 kubelet
|
||||
# /health probes and no inference request since start (2026-10-30T06:23:40Z). Nothing
|
||||
# in git selects this service. Restore it once the rollout leaves a card free:
|
||||
# 30B-A3B takes 1, Coder-Next TP=2 takes 2, so card 4 is this one.
|
||||
replicas: 0
|
||||
replicas: 1
|
||||
# Recreate, not RollingUpdate. Every GPU card is allocated, so a rolling update leaves
|
||||
# the new pod Pending forever waiting for a card the old pod still holds.
|
||||
strategy:
|
||||
|
||||
@@ -1,134 +0,0 @@
|
||||
apiVersion: serving.kserve.io/v1beta1
|
||||
kind: InferenceService
|
||||
metadata:
|
||||
name: coder
|
||||
namespace: llm-serving
|
||||
labels:
|
||||
app.kubernetes.io/name: llm-coder
|
||||
app.kubernetes.io/part-of: llm-serving
|
||||
annotations:
|
||||
serving.kserve.io/deploymentMode: RawDeployment
|
||||
spec:
|
||||
predictor:
|
||||
minReplicas: 1
|
||||
maxReplicas: 1
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: worker-1
|
||||
# Two GPUs are free, so this rolls out beside `reasoning` instead of over it.
|
||||
# Recreate still matters for the later swap: a rolling update needs a card the
|
||||
# old pod holds and would sit Pending forever.
|
||||
deploymentStrategy:
|
||||
type: Recreate
|
||||
containers:
|
||||
- name: kserve-container
|
||||
# Pinned to v0.11.0 on purpose. This is the last image measured to work on this
|
||||
# node: torch 2.8.0+cu128, arch list sm_70..sm_120, driver 580.159.04.
|
||||
# v0.15.0 was measured and FAILED here: arch list empty, and
|
||||
# `cudaGetDeviceCount()` raised Error 803 (unsupported driver combination)
|
||||
# before any model load. Do not raise this tag without re-running that probe.
|
||||
image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
|
||||
args:
|
||||
# Qwen3-Coder-30B-A3B-Instruct: 30.5B total, 3.3B active. Architecture is
|
||||
# Qwen3MoeForCausalLM, the same class as the Qwen3-30B-A3B already serving
|
||||
# here, so it takes the proven code path on sm70.
|
||||
#
|
||||
# Qwen ships this model only as BF16 (61.1 GB) and FP8 (no FP8 on Volta).
|
||||
# 4-bit GPTQ is the only form that fits one 32 GB card. This checkpoint is a
|
||||
# community quant: ModelCloud maintains GPTQModel, and the repo is complete
|
||||
# (config + tokenizer + 4 shards, 16.9 GB). Verified by loading, not by trust.
|
||||
#
|
||||
# Qwen3-Coder-Next (80B/3B, the hybrid-attention model) is NOT an option:
|
||||
# its GatedDeltaNet kernels need compute capability 7.5+ and the `fla` module,
|
||||
# and sm70 has neither. Measured, not assumed.
|
||||
- --model=ModelCloud/Qwen3-Coder-30B-A3B-Instruct-GPTQMODEL-W4A16-A
|
||||
# `reasoning` stays a served name so an existing client that sends
|
||||
# model=reasoning keeps working after the DNS rename. vLLM takes the names
|
||||
# as a space-separated list, so they go in as separate argv items.
|
||||
- --served-model-name
|
||||
- coder
|
||||
- reasoning
|
||||
- --quantization=gptq
|
||||
- --dtype=float16
|
||||
- --kv-cache-dtype=auto
|
||||
# Native window is 262144 and rope_scaling is null, so 256K needs no YaRN.
|
||||
# One card cannot hold it: measured KV pool on a single V100 was 132,672
|
||||
# tokens. TP=2 halves the weights per card (7.8 GiB each), which frees room
|
||||
# for the larger pool. Measured after boot: see the KV line in the log.
|
||||
- --max-model-len=262144
|
||||
- --tensor-parallel-size=2
|
||||
- --pipeline-parallel-size=1
|
||||
- --gpu-memory-utilization=0.90
|
||||
# A single 262,144-token request consumes the whole pool, so keep few slots.
|
||||
- --max-num-seqs=4
|
||||
- --max-num-batched-tokens=8192
|
||||
- --long-prefill-token-threshold=2048
|
||||
- --enable-chunked-prefill
|
||||
- --enable-prefix-caching
|
||||
# Measured: with the default (custom all-reduce ON) the engine finished CUDA graph
|
||||
# capture in 6s at 17:01:56 and then went silent for 14+ minutes, emitting only
|
||||
# "No available shared memory broadcast block found in 60 seconds". The front-end
|
||||
# never received the ready message. vLLM's custom all-reduce registers its own
|
||||
# CUDA graph addresses and deadlocks on this Volta/PCIe pair. NCCL is the fallback
|
||||
# and is correct here, because TP=2 across two PCIe V100s gains nothing from the
|
||||
# custom kernel anyway.
|
||||
- --disable-custom-all-reduce
|
||||
- --reasoning-parser=qwen3
|
||||
- --enable-auto-tool-choice
|
||||
- --tool-call-parser=hermes
|
||||
- --host=0.0.0.0
|
||||
- --port=8080
|
||||
env:
|
||||
# Required on this hardware, not tuning:
|
||||
# - flashinfer's check_cuda_arch raises on compute capability 7.0,
|
||||
# so the sampler must stay off
|
||||
# - FlashInfer and FlashAttention-2 need sm75+, so Triton serves attention
|
||||
- name: VLLM_USE_FLASHINFER_SAMPLER
|
||||
value: "0"
|
||||
- name: VLLM_ATTENTION_BACKEND
|
||||
value: TRITON_ATTN
|
||||
- name: HF_HOME
|
||||
value: /mnt/models
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
protocol: TCP
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
periodSeconds: 10
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
# TP=2 at 256K needs ~10 min before the API binds: 22s weights, 80s torch.compile,
|
||||
# 85s KV profiling, then graph capture. 120 x 15s = 30 min of headroom.
|
||||
failureThreshold: 120
|
||||
periodSeconds: 15
|
||||
resources:
|
||||
# worker-1 exposes only 63,495 MiB of allocatable memory, so a 24Gi request
|
||||
# never schedules. Two vLLM workers keep the weights on the GPU and need
|
||||
# little host RAM, so the request stays small and the limit allows burst.
|
||||
requests:
|
||||
cpu: "8"
|
||||
memory: 12Gi
|
||||
nvidia.com/gpu: "2"
|
||||
limits:
|
||||
cpu: "16"
|
||||
memory: 40Gi
|
||||
nvidia.com/gpu: "2"
|
||||
volumeMounts:
|
||||
- mountPath: /mnt/models
|
||||
name: models
|
||||
- mountPath: /dev/shm
|
||||
name: shm
|
||||
runtimeClassName: nvidia
|
||||
volumes:
|
||||
- name: models
|
||||
persistentVolumeClaim:
|
||||
claimName: llm-models
|
||||
- name: shm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
# NCCL all-reduce over TP=2 moves through /dev/shm. The 64Mi default is
|
||||
# too small and shows up as an init hang or a bus error.
|
||||
sizeLimit: 8Gi
|
||||
@@ -14,14 +14,7 @@ resources:
|
||||
# v0.5.22+, published only on CUDA 13, and CUDA 13 dropped Volta kernels. The model
|
||||
# instead runs the implementation it ships (joint_schema_model.py) in fp16 on one V100.
|
||||
- clef-serve.yaml
|
||||
# coder = Qwen3-Coder-30B-A3B-Instruct, 4-bit GPTQ, TP=2 at 262,144 tokens. It serves
|
||||
# both `coder` and `reasoning`, so poimen-memory needs a host change only, never a
|
||||
# model-name change. reasoning.yaml below can be dropped once that host change lands.
|
||||
- coder.yaml
|
||||
- reasoning.yaml
|
||||
# Stable client-facing name for the chat model. poimen-memory points at this Service,
|
||||
# so the vLLM -> candle-vllm handoff is a selector edit here and no client change.
|
||||
- reasoning-long-service.yaml
|
||||
# ornith is retired: its InferenceService still reports Ready=True while running zero
|
||||
# replicas, because desired equals available at 0. Its manifest was never committed,
|
||||
# so it is left unmanaged and is not listed here. Delete the CR when convenient.
|
||||
|
||||
@@ -66,21 +66,6 @@ spec:
|
||||
port: 80
|
||||
- protocol: TCP
|
||||
port: 8000
|
||||
# Allow from poimen namespace (poimen-memory ingest + query).
|
||||
# The client-side egress rule is the other half and must stay in step:
|
||||
# NetworkPolicy poimen/poimen-memory-llm-egress permits port 8080 to pods
|
||||
# labelled part-of=llm-serving, which is the same selector used here. This
|
||||
# policy had no matching ingress rule at all, so the packet was allowed out
|
||||
# and dropped here. The client then saw a bare timeout and the predictor log
|
||||
# stayed empty, which is why this failure looked like a hung inference engine
|
||||
# for 34 days.
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: poimen
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 8080
|
||||
# Allow intra-namespace (pod-to-pod within llm-serving)
|
||||
- from:
|
||||
- podSelector:
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
# qwen2.5:3b-instruct on CPU (talos-cp-2, 144GB RAM, 24 cores).
|
||||
# Moved off GPU to free a V100, first for ComfyUI and later for llm-serving.
|
||||
# Latency ~10x slower
|
||||
# Moved off GPU to free a V100 for ComfyUI. Latency ~10x slower
|
||||
# than GPU but sufficient for lightweight tasks (summarization,
|
||||
# classification, quick answers).
|
||||
apiVersion: apps/v1
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
# Stable client-facing name for the chat model, deliberately decoupled from the
|
||||
# process that currently serves it.
|
||||
#
|
||||
# poimen-memory addresses this Service, not a pod and not an InferenceService. That
|
||||
# means the vLLM -> candle-vllm handoff changes only this file's selector. The client
|
||||
# config never changes again, so the ingest cutover has no window in which the host
|
||||
# name is wrong.
|
||||
#
|
||||
# Two ports on purpose:
|
||||
# - 80 matches every other predictor Service in this namespace, and matches
|
||||
# poimen-memory's LLM_CHAT_BASE, which carries no port number.
|
||||
# - 8080 is the container port. It is also the only port llm-serving-ingress and
|
||||
# poimen-memory-llm-egress permit, and the port the handoff readiness check
|
||||
# uses. Publishing it directly avoids a port translation that is easy to get
|
||||
# wrong in a curl.
|
||||
#
|
||||
# The selector names the vLLM reasoning pod today. To hand off, replace the three
|
||||
# selector labels with candle-vllm's pod labels and nothing else. Do not delete and
|
||||
# recreate this Service: a delete drops the ClusterIP, and a client that cached the
|
||||
# name during the gap will fail until it re-resolves.
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: reasoning-long-predictor
|
||||
namespace: llm-serving
|
||||
labels:
|
||||
app.kubernetes.io/part-of: llm-serving
|
||||
app: reasoning-long-predictor
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app.kubernetes.io/name: llm-reasoning
|
||||
component: predictor
|
||||
serving.kserve.io/inferenceservice: reasoning
|
||||
ports:
|
||||
- name: http
|
||||
protocol: TCP
|
||||
port: 80
|
||||
targetPort: 8080
|
||||
- name: http-8080
|
||||
protocol: TCP
|
||||
port: 8080
|
||||
targetPort: 8080
|
||||
@@ -30,28 +30,7 @@ spec:
|
||||
# Measured on this cluster, one V100-32GB, 64k context: 15.55 GiB of weights,
|
||||
# 137k tokens of KV cache, 2.09x concurrency at 65536 tokens per request.
|
||||
- --model=Qwen/Qwen3-30B-A3B-GPTQ-Int4
|
||||
# Two served names, one pod, on purpose. `reasoning-long` is the name the goal map
|
||||
# uses for this model, and the name qwen-finetune wiring targets. It is NOT a
|
||||
# separate InferenceService: this model already occupies the only free card, and a
|
||||
# second pod on the same weights cannot schedule (4 of 4 GPUs allocated: coder 2,
|
||||
# this 1, clef-serve 1). Serving both names from one process costs nothing and
|
||||
# keeps the two callers working without a second weights copy.
|
||||
#
|
||||
# `reasoning` is deliberately still served here. poimen-memory pins
|
||||
# LLM_CHAT_BASE=reasoning-predictor, so dropping the name here breaks ingest
|
||||
# immediately. coder-predictor also serves `reasoning`; that ambiguity is known
|
||||
# and documented in the PR. Resolve it by moving the client host, not by
|
||||
# removing the name here.
|
||||
#
|
||||
# The two names are separate list entries, NOT one comma-joined value. This
|
||||
# image's parser returns ['reasoning,reasoning-long'] for the comma form, i.e. a
|
||||
# single model whose name contains a comma, and api_server.py then iterates
|
||||
# args.served_model_name to build the model list. Verified against vLLM 0.11.0:
|
||||
# FlexibleArgumentParser().parse_args([... "--served-model-name", "a", "b"])
|
||||
# -> ['a', 'b']
|
||||
- --served-model-name
|
||||
- reasoning
|
||||
- reasoning-long
|
||||
- --served-model-name=reasoning
|
||||
- --quantization=gptq
|
||||
- --dtype=float16
|
||||
- --kv-cache-dtype=auto
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
---
|
||||
# Public entry point for the poimen-memory HTTP API.
|
||||
#
|
||||
# The API was reachable only inside the cluster (ClusterIP :8080), so a local pi
|
||||
# session had no way to use it. This Ingress exposes the same handler set the
|
||||
# in-cluster clients use: /mcp, /memory/*, /graphql, /health, /ready.
|
||||
#
|
||||
# Cloudflare terminates TLS in front of the nginx ingress, which is why this rule
|
||||
# carries no `tls:` section — the same shape as ingress.yaml next to it.
|
||||
#
|
||||
# Auth: poimen-memory validates ZITADEL JWTs (MEM_AUTH_MODE=jwt). A caller
|
||||
# without a valid bearer token gets 401, so exposing the route does not expose
|
||||
# the data.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: poimen-api
|
||||
namespace: poimen
|
||||
annotations:
|
||||
# MCP calls and ingest can be slow; the provider may stream. Match the
|
||||
# frontend ingress so a long tool call is not cut at 60s.
|
||||
nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
|
||||
nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
|
||||
nginx.ingress.kubernetes.io/proxy-buffering: "off"
|
||||
nginx.ingress.kubernetes.io/proxy-body-size: "50m"
|
||||
spec:
|
||||
ingressClassName: nginx
|
||||
rules:
|
||||
- host: api-poimen.riotpiao.com
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: poimen-memory
|
||||
port:
|
||||
number: 8080
|
||||
@@ -6,5 +6,3 @@ kind: Kustomization
|
||||
# riotpiao-poimen/poimen-frontend.git k8s/ — each repo owns its own k8s.
|
||||
resources:
|
||||
- ingress.yaml
|
||||
# The memory API. Without it a local pi session has no route to :8080.
|
||||
- api-ingress.yaml
|
||||
|
||||
@@ -159,3 +159,54 @@ spec:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
---
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: kyverno
|
||||
namespace: argocd
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "0"
|
||||
spec:
|
||||
project: homelab
|
||||
source:
|
||||
repoURL: https://kyverno.github.io/kyverno/
|
||||
chart: kyverno
|
||||
targetRevision: "1.14.0"
|
||||
helm:
|
||||
valueFiles:
|
||||
- $values/k8s/bootstrap/kyverno/kyverno-values.yaml
|
||||
sources:
|
||||
- repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
|
||||
targetRevision: main
|
||||
ref: values
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: kyverno
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
---
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: kyverno-policies
|
||||
namespace: argocd
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "0"
|
||||
spec:
|
||||
project: homelab
|
||||
source:
|
||||
repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
|
||||
targetRevision: main
|
||||
path: k8s/bootstrap/kyverno
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: kyverno
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: comfyui
|
||||
namespace: argocd
|
||||
labels:
|
||||
app.kubernetes.io/name: comfyui
|
||||
app.kubernetes.io/component: image-generation
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "8"
|
||||
spec:
|
||||
project: homelab
|
||||
revisionHistoryLimit: 3
|
||||
source:
|
||||
repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
|
||||
targetRevision: main
|
||||
path: k8s/apps/comfyui
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: comfyui
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
retry:
|
||||
limit: 5
|
||||
backoff:
|
||||
duration: 5s
|
||||
factor: 2
|
||||
maxDuration: 3m
|
||||
@@ -0,0 +1,6 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: kyverno
|
||||
|
||||
resources:
|
||||
- policies.yaml
|
||||
@@ -0,0 +1,42 @@
|
||||
# Kyverno: Policy engine for Kubernetes image scanning, Pod security, and admission control
|
||||
# Scan all images, enforce baseline Pod Security Standard, prevent privilege escalation
|
||||
|
||||
replicaCount: 1
|
||||
|
||||
image:
|
||||
registry: ghcr.io
|
||||
repository: kyverno/kyverno
|
||||
tag: "v1.14.0"
|
||||
|
||||
config:
|
||||
# Webhook timeout for policy evaluation. Increase if scanning takes longer.
|
||||
webhookTimeoutSeconds: 30
|
||||
# Failure policy: fail-open (audit/log) vs fail-closed (reject on error)
|
||||
failurePolicy: fail
|
||||
# Resource limits for webhook
|
||||
webhookAnnotations:
|
||||
rules: "allow"
|
||||
|
||||
# Pod security via Kyverno instead of Pod Security Policies (deprecated)
|
||||
# Enforces baseline restrictions cluster-wide, with exceptions for privileged namespaces
|
||||
podSecurityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
|
||||
rbac:
|
||||
create: true
|
||||
|
||||
resources:
|
||||
requests:
|
||||
memory: "256Mi"
|
||||
cpu: "100m"
|
||||
limits:
|
||||
memory: "512Mi"
|
||||
cpu: "500m"
|
||||
|
||||
# Webhook configuration
|
||||
webhook:
|
||||
timeoutSeconds: 30
|
||||
# Failure policy: "Fail" (reject on error) or "Ignore" (audit-only)
|
||||
# Set to "Ignore" for initial testing, then change to "Fail"
|
||||
failurePolicy: ignore
|
||||
@@ -0,0 +1,204 @@
|
||||
# Kyverno ClusterPolicies: Image scanning, Pod security, and admission control
|
||||
---
|
||||
# Policy 1: Require non-root containers
|
||||
apiVersion: kyverno.io/v1
|
||||
kind: ClusterPolicy
|
||||
metadata:
|
||||
name: require-non-root
|
||||
namespace: kyverno
|
||||
spec:
|
||||
validationFailureAction: audit # audit first, then change to enforce
|
||||
rules:
|
||||
- name: check-runAsNonRoot
|
||||
match:
|
||||
any:
|
||||
- resources:
|
||||
kinds:
|
||||
- Pod
|
||||
selector:
|
||||
matchLabels:
|
||||
pod-security.kubernetes.io/enforce: "!privileged"
|
||||
validate:
|
||||
message: "Container must not run as root"
|
||||
pattern:
|
||||
spec:
|
||||
containers:
|
||||
- securityContext:
|
||||
runAsNonRoot: true
|
||||
---
|
||||
# Policy 2: Drop all Linux capabilities, add only required ones
|
||||
apiVersion: kyverno.io/v1
|
||||
kind: ClusterPolicy
|
||||
metadata:
|
||||
name: require-dropped-caps
|
||||
namespace: kyverno
|
||||
spec:
|
||||
validationFailureAction: audit
|
||||
rules:
|
||||
- name: drop-all-capabilities
|
||||
match:
|
||||
any:
|
||||
- resources:
|
||||
kinds:
|
||||
- Pod
|
||||
selector:
|
||||
matchLabels:
|
||||
pod-security.kubernetes.io/enforce: "!privileged"
|
||||
validate:
|
||||
message: "All Linux capabilities must be dropped"
|
||||
pattern:
|
||||
spec:
|
||||
containers:
|
||||
- securityContext:
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
---
|
||||
# Policy 3: Require image tags (no 'latest')
|
||||
apiVersion: kyverno.io/v1
|
||||
kind: ClusterPolicy
|
||||
metadata:
|
||||
name: disallow-latest-tag
|
||||
namespace: kyverno
|
||||
spec:
|
||||
validationFailureAction: audit # Change to enforce after testing
|
||||
rules:
|
||||
- name: disallow-latest
|
||||
match:
|
||||
any:
|
||||
- resources:
|
||||
kinds:
|
||||
- Pod
|
||||
- Deployment
|
||||
- StatefulSet
|
||||
- DaemonSet
|
||||
- Job
|
||||
validate:
|
||||
message: "Image tag 'latest' is not allowed. Use explicit version tags."
|
||||
pattern:
|
||||
spec:
|
||||
=(template):
|
||||
spec:
|
||||
containers:
|
||||
- image: "!*:latest"
|
||||
=(initContainers):
|
||||
- image: "!*:latest"
|
||||
---
|
||||
# Policy 4: Restrict images to trusted registries
|
||||
apiVersion: kyverno.io/v1
|
||||
kind: ClusterPolicy
|
||||
metadata:
|
||||
name: restrict-registries
|
||||
namespace: kyverno
|
||||
spec:
|
||||
validationFailureAction: audit
|
||||
rules:
|
||||
- name: trusted-registries
|
||||
match:
|
||||
any:
|
||||
- resources:
|
||||
kinds:
|
||||
- Pod
|
||||
- Deployment
|
||||
- StatefulSet
|
||||
- DaemonSet
|
||||
- Job
|
||||
selector:
|
||||
matchLabels:
|
||||
pod-security.kubernetes.io/enforce: "!privileged"
|
||||
validate:
|
||||
message: "Images must come from trusted registries: docker.io, ghcr.io, quay.io, k8s.gcr.io, registry.k8s.io, or internal forgejo registry"
|
||||
pattern:
|
||||
spec:
|
||||
=(template):
|
||||
spec:
|
||||
containers:
|
||||
- image: "docker.io/* | ghcr.io/* | quay.io/* | k8s.gcr.io/* | registry.k8s.io/* | forgejo.riotpiao.com/* | *"
|
||||
---
|
||||
# Policy 5: Require read-only root filesystem (audit only, exceptions for apps that need writes)
|
||||
apiVersion: kyverno.io/v1
|
||||
kind: ClusterPolicy
|
||||
metadata:
|
||||
name: require-readonly-filesystem
|
||||
namespace: kyverno
|
||||
spec:
|
||||
validationFailureAction: audit
|
||||
rules:
|
||||
- name: check-readOnlyRootFilesystem
|
||||
match:
|
||||
any:
|
||||
- resources:
|
||||
kinds:
|
||||
- Pod
|
||||
selector:
|
||||
matchLabels:
|
||||
pod-security.kubernetes.io/enforce: "!privileged"
|
||||
validate:
|
||||
message: "Root filesystem should be read-only for defense-in-depth"
|
||||
pattern:
|
||||
spec:
|
||||
containers:
|
||||
- securityContext:
|
||||
readOnlyRootFilesystem: true
|
||||
---
|
||||
# Policy 6: Require resource requests and limits (prevent resource starvation)
|
||||
apiVersion: kyverno.io/v1
|
||||
kind: ClusterPolicy
|
||||
metadata:
|
||||
name: require-resource-limits
|
||||
namespace: kyverno
|
||||
spec:
|
||||
validationFailureAction: audit
|
||||
rules:
|
||||
- name: check-resources
|
||||
match:
|
||||
any:
|
||||
- resources:
|
||||
kinds:
|
||||
- Pod
|
||||
- Deployment
|
||||
- StatefulSet
|
||||
- DaemonSet
|
||||
excludeResources:
|
||||
namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: "kyverno|kube-system|kube-node-lease"
|
||||
validate:
|
||||
message: "CPU and memory requests and limits are required"
|
||||
pattern:
|
||||
spec:
|
||||
=(template):
|
||||
spec:
|
||||
containers:
|
||||
- resources:
|
||||
requests:
|
||||
memory: "?*"
|
||||
cpu: "?*"
|
||||
limits:
|
||||
memory: "?*"
|
||||
cpu: "?*"
|
||||
---
|
||||
# Policy 7: Require securityContext on all containers
|
||||
apiVersion: kyverno.io/v1
|
||||
kind: ClusterPolicy
|
||||
metadata:
|
||||
name: require-security-context
|
||||
namespace: kyverno
|
||||
spec:
|
||||
validationFailureAction: audit
|
||||
rules:
|
||||
- name: check-securityContext
|
||||
match:
|
||||
any:
|
||||
- resources:
|
||||
kinds:
|
||||
- Pod
|
||||
selector:
|
||||
matchLabels:
|
||||
pod-security.kubernetes.io/enforce: "!privileged"
|
||||
validate:
|
||||
message: "securityContext must be defined"
|
||||
pattern:
|
||||
spec:
|
||||
containers:
|
||||
- securityContext: {}
|
||||
@@ -161,19 +161,19 @@ ingress:
|
||||
# The database was never the bottleneck. A real action_runner UPDATE measured
|
||||
# 2.133ms with zero lock waits, so this is CPU starvation in Gitea itself.
|
||||
#
|
||||
# requests.cpu stays 200m and limits.cpu stays 1000m. The 3000m limit was a
|
||||
# temporary measure while the Actions stall was diagnosed; the real fix landed
|
||||
# elsewhere, so the limit is back to the original.
|
||||
# requests.cpu rises to 750m because a request below real usage earns the pod
|
||||
# little scheduling weight during contention. limits.cpu rises to 3000m
|
||||
# because diff computation is bursty and needs room to run without throttling.
|
||||
#
|
||||
# Caution: this pod runs on talos-cp-3, a control-plane node. If Gitea
|
||||
# throttles again at 1000m, move the workload off the control plane rather
|
||||
# than raising the limit.
|
||||
# Caution: this pod runs on talos-cp-3, a control-plane node. 3000m is 25% of
|
||||
# that node's 11950m allocatable CPU. If Gitea sustains near this limit, move
|
||||
# the workload off the control plane rather than raising the limit again.
|
||||
resources:
|
||||
requests:
|
||||
cpu: 200m
|
||||
cpu: 750m
|
||||
memory: 768Mi
|
||||
limits:
|
||||
cpu: 1000m
|
||||
cpu: 3000m
|
||||
memory: 2Gi
|
||||
|
||||
# Tolerations for control-plane
|
||||
|
||||
@@ -1,90 +0,0 @@
|
||||
# One-shot destructive health scan of a candidate Longhorn disk.
|
||||
#
|
||||
# WHY THIS IS A MANUAL JOB AND NOT AN ARGOCD HOOK
|
||||
# It writes over the entire raw device. An ArgoCD PostSync hook would run it
|
||||
# on every sync and destroy whatever the volume holds. Apply it by hand,
|
||||
# once, on a disk that has no partitions and no mounts.
|
||||
#
|
||||
# WHY NOT TALOS
|
||||
# Talos declares disks by mounting them (machine.disks in the machine
|
||||
# config). It has no primitive for "run this command on the host", so a scan
|
||||
# cannot live in the Talos config. This Job is the IaC-correct home.
|
||||
#
|
||||
# ORDER OF OPERATIONS
|
||||
# 1. Scan the raw device with this Job <- the disk must be UNMOUNTED
|
||||
# 2. If it passes, add the disk to terraform.tfvars.local `longhorn_disks`
|
||||
# with device /dev/disk/by-id/<wwid> and run `make apply-cp`
|
||||
# 3. Register it with Longhorn via the pattern in longhorn-add-disks-job.yaml
|
||||
#
|
||||
# What it does
|
||||
# fio writes a CRC32C-verified pattern across the whole device, then reads
|
||||
# every block back and compares. Any unreadable or mismatched sector fails
|
||||
# the Job. This is a full write and read pass, not a sample.
|
||||
#
|
||||
# Expected duration
|
||||
# 2 TB on a 5400 rpm 2.5" drive at roughly 90 MB/s is about 6 h to write and
|
||||
# 6 h to verify. Budget 12 hours. Check progress with:
|
||||
# kubectl -n llm-io logs job/disk-scan-cp1-sdb -f
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: disk-scan-cp1-sdb
|
||||
namespace: llm-io
|
||||
labels:
|
||||
app: disk-scan
|
||||
spec:
|
||||
backoffLimit: 0
|
||||
# Do not let a stuck scan retry silently.
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: disk-scan
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: talos-cp-1
|
||||
containers:
|
||||
- name: scan
|
||||
image: ubuntu:22.04
|
||||
securityContext:
|
||||
privileged: true
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
set -euo pipefail
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq smartmontools fio
|
||||
|
||||
echo "=== device identity ==="
|
||||
smartctl -i /dev/sdb | tail -12
|
||||
|
||||
echo "=== SMART before ==="
|
||||
smartctl -H /dev/sdb | tail -2
|
||||
|
||||
echo "=== full write + verify (destructive) ==="
|
||||
fio --name=devverify \
|
||||
--filename=/dev/sdb \
|
||||
--rw=write \
|
||||
--bs=1M \
|
||||
--iodepth=4 \
|
||||
--ioengine=libaio \
|
||||
--direct=1 \
|
||||
--verify=crc32c \
|
||||
--do_verify=1 \
|
||||
--verify_fatal=1 \
|
||||
--eta=always \
|
||||
--output-format=normal
|
||||
|
||||
echo "=== SMART after ==="
|
||||
smartctl -H /dev/sdb | tail -2
|
||||
smartctl -A /dev/sdb | awk '/Reallocated_Sector|Current_Pending|Offline_Uncorrect|UDMA_CRC/{print " "$0}'
|
||||
|
||||
echo "=== SCAN PASSED ==="
|
||||
volumeMounts:
|
||||
- name: dev
|
||||
mountPath: /dev
|
||||
volumes:
|
||||
- name: dev
|
||||
hostPath:
|
||||
path: /dev
|
||||
@@ -1,28 +0,0 @@
|
||||
---
|
||||
# Keep the GraphQL server reachable only from the one service that proxies it.
|
||||
#
|
||||
# poimen-graphql is already off the internet: its Service is ClusterIP on 8090 and
|
||||
# no Ingress names it. What it lacked is in-cluster isolation — a ClusterIP answers
|
||||
# every pod in the cluster.
|
||||
#
|
||||
# The only caller is poimen-memory, whose /graphql route forwards the request body
|
||||
# to MEM_GQL_PORT (8090). No other workload references this service.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: poimen-graphql-ingress
|
||||
namespace: poimen
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: poimen-graphql
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: poimen-memory
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 8090
|
||||
@@ -6,10 +6,6 @@ kind: Kustomization
|
||||
# deployments (wave 0) are applied.
|
||||
resources:
|
||||
- memory-db.yaml
|
||||
# ClusterIP alone is not isolation: every pod in the cluster can reach it.
|
||||
- memory-db-netpol.yaml
|
||||
# Only poimen-memory's /graphql proxy calls 8090.
|
||||
- graphql-netpol.yaml
|
||||
- poimen-sandbox-namespace.yaml
|
||||
- poimen-sandbox-quota.yaml
|
||||
- pi-session-netpol.yaml
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
---
|
||||
# Keep the memory database reachable only from inside its own namespace.
|
||||
#
|
||||
# What this does NOT change: the database was already unreachable from the
|
||||
# internet. `memory-db-rw`, `memory-db-ro` and `memory-db-r` are ClusterIP
|
||||
# services, no Ingress backs them, and no public DNS name resolves to them.
|
||||
#
|
||||
# What it fixes: a ClusterIP service is reachable by EVERY pod in the cluster,
|
||||
# because Kubernetes has no implicit isolation. Cilium enforces NetworkPolicy, so
|
||||
# stating the allowed sources is what turns "not on the internet" into "not
|
||||
# reachable from another tenant's namespace".
|
||||
#
|
||||
# Two sources, both required:
|
||||
# - the poimen namespace, which holds every client: poimen-memory, poimen-graphql,
|
||||
# poimen-worker, poimen-workflows and the migration Job.
|
||||
# - cnpg-system, where the CloudNativePG operator lives. It connects to the
|
||||
# instance for switchover and failover, so a narrower rule breaks recovery.
|
||||
#
|
||||
# No other port is opened. The operator's metrics endpoint stays closed because no
|
||||
# PodMonitor for this cluster exists, so nothing scrapes it.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: memory-db-ingress
|
||||
namespace: poimen
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
cnpg.io/cluster: memory-db
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: poimen
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: cnpg-system
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 5432
|
||||
@@ -32,6 +32,7 @@ ISSUER = "https://zitadel.riotpiao.com"
|
||||
APPS = [
|
||||
("grafana", "grafana", "grafana-oidc", "logging", "GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET", "deploy", "grafana"),
|
||||
("argocd", "argocd", "oidc-secret", "argocd", "client-secret", "deploy", "argocd-server"),
|
||||
("comfyui", "comfyui", "comfyui-oauth2-proxy", "comfyui", "client-secret", "deploy", "comfyui-oauth2-proxy"),
|
||||
("minio", "minio", "minio-oidc", "storage", "MINIO_IDENTITY_OPENID_CLIENT_SECRET", "sts", "minio-cluster-az-a"),
|
||||
("paperless", "paperless", "paperless-oidc", "paperless", "client-secret", "deploy", "paperless"),
|
||||
("immich", "immich", "immich-oidc", "immich", "client-secret", "deploy", "immich-server"),
|
||||
|
||||
@@ -128,6 +128,11 @@ OIDC_APPS = {
|
||||
"https://forgejo.riotpiao.com/user/oauth2/zitadel/callback",
|
||||
"http://localhost:3000/user/oauth2/authentik/callback",
|
||||
"https://forgejo.riotpiao.com/user/oauth2/authentik/callback"]},
|
||||
# ComfyUI has no native OIDC. oauth2-proxy is its front door and this app is
|
||||
# the proxy's client, so the callback path belongs to the proxy.
|
||||
"comfyui": {**_WEB, "redirect_uris": [
|
||||
"http://localhost:4180/oauth2/callback",
|
||||
"https://comfyui.riotpiao.com/oauth2/callback"]},
|
||||
"grafana": {**_WEB, "redirect_uris": [
|
||||
"http://localhost:3000/login/generic_oauth",
|
||||
"https://grafana.riotpiao.com/login/generic_oauth"]},
|
||||
@@ -138,11 +143,7 @@ OIDC_APPS = {
|
||||
"http://localhost:8000", "http://localhost:18000"]},
|
||||
"minio": {**_WEB, "redirect_uris": [
|
||||
"http://localhost:9000/auth/sso/oauth2/code",
|
||||
"https://minio.riotpiao.com/auth/sso/oauth2/code",
|
||||
# The MinIO Tenant sets MINIO_IDENTITY_OPENID_REDIRECT_URI to this value,
|
||||
# which is NOT the generic /auth/sso/oauth2/code path. Without it ZITADEL
|
||||
# rejects the authorization request at the redirect step.
|
||||
"https://minio.riotpiao.com/oauth_callback"]},
|
||||
"https://minio.riotpiao.com/auth/sso/oauth2/code"]},
|
||||
"paperless": {**_WEB, "redirect_uris": [
|
||||
"http://localhost:8000/accounts/oidc/zitadel/login/callback/",
|
||||
"https://paperless.riotpiao.com/accounts/oidc/zitadel/login/callback/",
|
||||
@@ -358,13 +359,6 @@ MACHINE_CALLERS = {
|
||||
"description": "poimen calls api-gateway inference",
|
||||
"roles": ["llm:inference", "svc-caller"],
|
||||
},
|
||||
# paperless-ai is a vendor image: it cannot sign an assertion, so a sidecar mints
|
||||
# a token with this account and writes it to the file the image already reads.
|
||||
"paperless-llm": {
|
||||
"name": "Paperless AI LLM caller",
|
||||
"description": "paperless-ai calls api-gateway inference through the token vendor",
|
||||
"roles": ["llm:inference", "svc-caller"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -36,10 +36,8 @@
|
||||
rewrite name paperless.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
|
||||
rewrite name img.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
|
||||
rewrite name api.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
|
||||
# poimen ran only in the live ConfigMap, never in this file or the Talos
|
||||
# machine config. The next Talos reconcile would have deleted it and broken
|
||||
# in-cluster resolution. Declared here so the source of truth carries it.
|
||||
rewrite name poimen.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
|
||||
rewrite name comfy.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
|
||||
rewrite name comfyui.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
|
||||
rewrite name gotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
|
||||
rewrite name pi.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
|
||||
rewrite name igotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
|
||||
|
||||
Reference in New Issue
Block a user