Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 rock
|
747a6fb9cb |
@@ -45,6 +45,8 @@ data:
|
||||
service: https://igotify.riotpiao.com
|
||||
- hostname: poimen.riotpiao.com
|
||||
service: https://poimen.riotpiao.com
|
||||
- hostname: comfyui.riotpiao.com
|
||||
service: https://comfyui.riotpiao.com
|
||||
- hostname: forgejo.riotpiao.com
|
||||
service: https://forgejo.riotpiao.com
|
||||
# ZITADEL. Without this rule the connector answers 404 for the hostname
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
# ComfyUI — GPU-accelerated image generation on worker-1.
|
||||
# Uses 1x V100 32GB (sm70). Freed by scaling ornith 2→1.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: comfyui
|
||||
namespace: comfyui
|
||||
labels:
|
||||
app: comfyui
|
||||
spec:
|
||||
# Retired on purpose. This Deployment held the 4th V100 on worker-1 while
|
||||
# serving nothing: its oauth2-proxy roll-out has been stuck for 42h and
|
||||
# failing 11621 times, because the Secret comfyui/comfyui-oauth2-proxy carries
|
||||
# only 1 key while the pod requests client-id, client-secret and
|
||||
# cookie-secret. Every request was therefore refused, so the card sat at 0%
|
||||
# utilisation and blocked a GPU that llm-serving can use.
|
||||
#
|
||||
# Scaling to 0 is the whole change. The Deployment, the Service, the Ingress
|
||||
# and the 50Gi comfyui-models PVC all stay, so nothing is lost and no data is
|
||||
# touched. To bring it back: set replicas to 1 here, then fix the Secret to
|
||||
# carry all three keys, otherwise the same stuck roll-out returns.
|
||||
replicas: 0
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: comfyui
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: comfyui
|
||||
spec:
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: worker-1
|
||||
runtimeClassName: nvidia
|
||||
# k8s Service named 'comfyui' injects COMFYUI_PORT=tcp://... into pod env,
|
||||
# which clobbers ai-dock's own COMFYUI_PORT variable (expects a port number).
|
||||
# Disable service link injection to avoid the collision.
|
||||
enableServiceLinks: false
|
||||
containers:
|
||||
- name: comfyui
|
||||
image: ghcr.io/ai-dock/comfyui:v2-cuda-12.1.1-base-22.04
|
||||
ports:
|
||||
- containerPort: 8188
|
||||
protocol: TCP
|
||||
env:
|
||||
- name: NVIDIA_VISIBLE_DEVICES
|
||||
value: "all"
|
||||
resources:
|
||||
requests:
|
||||
cpu: "4"
|
||||
memory: 8Gi
|
||||
nvidia.com/gpu: "1"
|
||||
limits:
|
||||
cpu: "8"
|
||||
memory: 16Gi
|
||||
nvidia.com/gpu: "1"
|
||||
volumeMounts:
|
||||
- mountPath: /workspace/ComfyUI/models
|
||||
name: models
|
||||
- mountPath: /workspace/ComfyUI/output
|
||||
name: output
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: 8188
|
||||
periodSeconds: 10
|
||||
initialDelaySeconds: 30
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: 8188
|
||||
failureThreshold: 120
|
||||
periodSeconds: 10
|
||||
volumes:
|
||||
- name: models
|
||||
persistentVolumeClaim:
|
||||
claimName: comfyui-models
|
||||
- name: output
|
||||
emptyDir: {}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: comfyui-models
|
||||
namespace: comfyui
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: longhorn
|
||||
resources:
|
||||
requests:
|
||||
storage: 50Gi
|
||||
@@ -0,0 +1,38 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: comfyui
|
||||
namespace: comfyui
|
||||
annotations:
|
||||
# No cert-manager annotation and no tls: block. This host is covered by
|
||||
# the *.riotpiao.com wildcard, which nginx serves as its
|
||||
# default-ssl-certificate (ingress-nginx/riotpiao-com-tls).
|
||||
nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
|
||||
nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
|
||||
nginx.ingress.kubernetes.io/proxy-body-size: "0"
|
||||
# WebSocket support for ComfyUI's live preview
|
||||
nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
|
||||
nginx.ingress.kubernetes.io/upstream-hash-by: "$remote_addr"
|
||||
# Single sign-on. ComfyUI has no auth of its own, so every request is handed to
|
||||
# oauth2-proxy for an allow or deny decision.
|
||||
nginx.ingress.kubernetes.io/auth-url: "http://comfyui-oauth2-proxy.comfyui.svc.cluster.local:4180/oauth2/auth"
|
||||
nginx.ingress.kubernetes.io/auth-signin: "https://comfyui.riotpiao.com/oauth2/start?rd=$scheme://$host$request_uri"
|
||||
nginx.ingress.kubernetes.io/auth-response-headers: "x-auth-request-user,x-auth-request-email"
|
||||
# No configuration-snippet. The controller rejects that annotation as risky
|
||||
# ("annotation group ConfigurationSnippet contains risky annotation"), and a
|
||||
# rejected annotation drops the WHOLE ingress from nginx.conf: the host then
|
||||
# answers 404 with no entry in the access log. ingress-nginx sets the Upgrade
|
||||
# and Connection headers itself for a websocket upgrade.
|
||||
spec:
|
||||
ingressClassName: nginx
|
||||
rules:
|
||||
- host: comfyui.riotpiao.com
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: comfyui
|
||||
port:
|
||||
number: 80
|
||||
@@ -0,0 +1,8 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
resources:
|
||||
- deployment.yaml
|
||||
- service.yaml
|
||||
- oauth2-proxy.yaml
|
||||
- ingress.yaml
|
||||
@@ -0,0 +1,105 @@
|
||||
# ComfyUI has no authentication of its own: comfyui.riotpiao.com is open to anyone
|
||||
# who can resolve the host. It also has no OIDC support, so ZITADEL cannot be a
|
||||
# login method inside it. oauth2-proxy is the front door instead: nginx asks it for
|
||||
# a decision on every request and it answers from a ZITADEL session.
|
||||
#
|
||||
# No API client calls this host today, so the whole host is gated. If a client ever
|
||||
# needs direct access, add a skip rule for the exact paths it uses rather than
|
||||
# opening the whole host.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: comfyui-oauth2-proxy
|
||||
namespace: comfyui
|
||||
labels:
|
||||
app: comfyui-oauth2-proxy
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: comfyui-oauth2-proxy
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: comfyui-oauth2-proxy
|
||||
spec:
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65534
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
containers:
|
||||
- name: oauth2-proxy
|
||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.7.1
|
||||
args:
|
||||
- --provider=oidc
|
||||
- --oidc-issuer-url=https://zitadel.riotpiao.com
|
||||
- --email-domain=*
|
||||
- --upstream=static://202
|
||||
- --reverse-proxy=true
|
||||
- --set-xauthrequest=true
|
||||
- --cookie-secure=true
|
||||
- --cookie-expire=12h
|
||||
- --cookie-refresh=1h
|
||||
- --skip-provider-button=true
|
||||
- --redirect-url=https://comfyui.riotpiao.com/oauth2/callback
|
||||
- --http-address=0.0.0.0:4180
|
||||
env:
|
||||
- name: OAUTH2_PROXY_CLIENT_ID
|
||||
valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: client-id}}
|
||||
- name: OAUTH2_PROXY_CLIENT_SECRET
|
||||
valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: client-secret}}
|
||||
- name: OAUTH2_PROXY_COOKIE_SECRET
|
||||
valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: cookie-secret}}
|
||||
ports:
|
||||
- {name: http, containerPort: 4180}
|
||||
readinessProbe:
|
||||
httpGet: {path: /ping, port: http}
|
||||
initialDelaySeconds: 5
|
||||
livenessProbe:
|
||||
httpGet: {path: /ping, port: http}
|
||||
initialDelaySeconds: 15
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities: {drop: ["ALL"]}
|
||||
resources:
|
||||
requests: {cpu: 10m, memory: 32Mi}
|
||||
limits: {cpu: 200m, memory: 128Mi}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: comfyui-oauth2-proxy
|
||||
namespace: comfyui
|
||||
labels:
|
||||
app: comfyui-oauth2-proxy
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: comfyui-oauth2-proxy
|
||||
ports:
|
||||
- {name: http, port: 4180, targetPort: http}
|
||||
---
|
||||
# The login path must NOT be gated: a request to /oauth2/start that itself demands a
|
||||
# login redirects to /oauth2/start forever. ingress-nginx applies the annotations of
|
||||
# an ingress to the locations it defines, so a separate object without auth-url keeps
|
||||
# this path open. The earlier failure of this shape was not the split — it was the
|
||||
# configuration-snippet on the app ingress, which made the controller drop the whole
|
||||
# object and left only this one alive.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: comfyui-oauth2
|
||||
namespace: comfyui
|
||||
spec:
|
||||
ingressClassName: nginx
|
||||
rules:
|
||||
- host: comfyui.riotpiao.com
|
||||
http:
|
||||
paths:
|
||||
- path: /oauth2
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: comfyui-oauth2-proxy
|
||||
port:
|
||||
number: 4180
|
||||
@@ -0,0 +1,14 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: comfyui
|
||||
namespace: comfyui
|
||||
labels:
|
||||
app: comfyui
|
||||
spec:
|
||||
selector:
|
||||
app: comfyui
|
||||
ports:
|
||||
- port: 80
|
||||
targetPort: 8188
|
||||
protocol: TCP
|
||||
@@ -8,11 +8,7 @@ metadata:
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
|
||||
spec:
|
||||
# 2, not 3. The file's own header always said 2, and the third instance never
|
||||
# existed as a running pod: its volume only ever held a replica that could not
|
||||
# start, because talos-cp-2 disk2 has no free space. A third replica on a disk
|
||||
# that cannot host it is not redundancy, it is a permanent degraded volume.
|
||||
instances: 2
|
||||
instances: 3
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16.2
|
||||
bootstrap:
|
||||
initdb:
|
||||
|
||||
@@ -40,7 +40,7 @@ target is one root-of-trust file per thing that cannot rotate itself:
|
||||
| controller bootstrap key | chicken and egg: it creates every other key |
|
||||
| etcd defrag credentials | needed by a job that runs outside the controller |
|
||||
|
||||
Everything else — Grafana, Argo CD, Forgejo, MinIO, Paperless, Immich,
|
||||
Everything else — Grafana, Argo CD, Forgejo, MinIO, Paperless, Immich, ComfyUI,
|
||||
api-gw, Gotify — is a KeyRotation object, and its secret leaves git.
|
||||
|
||||
## Status
|
||||
|
||||
@@ -64,6 +64,32 @@ spec:
|
||||
---
|
||||
apiVersion: iam.homelab.riotpiao.com/v1
|
||||
kind: KeyRotation
|
||||
metadata:
|
||||
name: comfyui
|
||||
namespace: iam
|
||||
labels:
|
||||
app.kubernetes.io/part-of: key-rotation
|
||||
spec:
|
||||
provider: zitadel-oidc-client
|
||||
strategy: atomic
|
||||
intervalHours: 2160
|
||||
suspend: true
|
||||
target:
|
||||
zitadelProjectID: "396757796898471937"
|
||||
zitadelAppID: "397015743994003854" # app resource id, not the client id 397015744010781070
|
||||
zitadelClientID: "397015744010781070"
|
||||
secretName: comfyui-oauth2-proxy
|
||||
secretNamespace: comfyui
|
||||
secretKeys:
|
||||
secret: client-secret
|
||||
reload: restart
|
||||
consumers:
|
||||
- {kind: Deployment, name: comfyui-oauth2-proxy, namespace: comfyui}
|
||||
verify:
|
||||
introspect: true
|
||||
---
|
||||
apiVersion: iam.homelab.riotpiao.com/v1
|
||||
kind: KeyRotation
|
||||
metadata:
|
||||
name: minio
|
||||
namespace: iam
|
||||
@@ -81,11 +107,7 @@ spec:
|
||||
secretName: minio-oidc
|
||||
secretNamespace: storage
|
||||
secretKeys:
|
||||
# The key name here must match the consumer's secretKeyRef, not a house style.
|
||||
# minio-tenant.yaml reads MINIO_IDENTITY_OPENID_CLIENT_SECRET. While this said
|
||||
# `client-secret`, the tenant failed with CreateContainerConfigError on every
|
||||
# restart and the rotation wrote a key nobody read.
|
||||
secret: MINIO_IDENTITY_OPENID_CLIENT_SECRET
|
||||
secret: client-secret
|
||||
reload: hot
|
||||
consumers:
|
||||
- {kind: StatefulSet, name: minio-cluster-az-a, namespace: storage}
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
# qwen2.5:3b-instruct on CPU (talos-cp-2, 144GB RAM, 24 cores).
|
||||
# Moved off GPU to free a V100, first for ComfyUI and later for llm-serving.
|
||||
# Latency ~10x slower
|
||||
# Moved off GPU to free a V100 for ComfyUI. Latency ~10x slower
|
||||
# than GPU but sufficient for lightweight tasks (summarization,
|
||||
# classification, quick answers).
|
||||
apiVersion: apps/v1
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
---
|
||||
# Public entry point for the poimen-memory HTTP API.
|
||||
#
|
||||
# The API was reachable only inside the cluster (ClusterIP :8080), so a local pi
|
||||
# session had no way to use it. This Ingress exposes the same handler set the
|
||||
# in-cluster clients use: /mcp, /memory/*, /graphql, /health, /ready.
|
||||
#
|
||||
# Cloudflare terminates TLS in front of the nginx ingress, which is why this rule
|
||||
# carries no `tls:` section — the same shape as ingress.yaml next to it.
|
||||
#
|
||||
# Auth: poimen-memory validates ZITADEL JWTs (MEM_AUTH_MODE=jwt). A caller
|
||||
# without a valid bearer token gets 401, so exposing the route does not expose
|
||||
# the data.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: poimen-api
|
||||
namespace: poimen
|
||||
annotations:
|
||||
# MCP calls and ingest can be slow; the provider may stream. Match the
|
||||
# frontend ingress so a long tool call is not cut at 60s.
|
||||
nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
|
||||
nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
|
||||
nginx.ingress.kubernetes.io/proxy-buffering: "off"
|
||||
nginx.ingress.kubernetes.io/proxy-body-size: "50m"
|
||||
spec:
|
||||
ingressClassName: nginx
|
||||
rules:
|
||||
- host: api-poimen.riotpiao.com
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: poimen-memory
|
||||
port:
|
||||
number: 8080
|
||||
@@ -6,5 +6,3 @@ kind: Kustomization
|
||||
# riotpiao-poimen/poimen-frontend.git k8s/ — each repo owns its own k8s.
|
||||
resources:
|
||||
- ingress.yaml
|
||||
# The memory API. Without it a local pi session has no route to :8080.
|
||||
- api-ingress.yaml
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: comfyui
|
||||
namespace: argocd
|
||||
labels:
|
||||
app.kubernetes.io/name: comfyui
|
||||
app.kubernetes.io/component: image-generation
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "8"
|
||||
spec:
|
||||
project: homelab
|
||||
revisionHistoryLimit: 3
|
||||
source:
|
||||
repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
|
||||
targetRevision: main
|
||||
path: k8s/apps/comfyui
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: comfyui
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
retry:
|
||||
limit: 5
|
||||
backoff:
|
||||
duration: 5s
|
||||
factor: 2
|
||||
maxDuration: 3m
|
||||
@@ -0,0 +1,6 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: kyverno
|
||||
|
||||
resources:
|
||||
- policies.yaml
|
||||
@@ -0,0 +1,42 @@
|
||||
# Kyverno: Policy engine for Kubernetes image scanning, Pod security, and admission control
|
||||
# Scan all images, enforce baseline Pod Security Standard, prevent privilege escalation
|
||||
|
||||
replicaCount: 1
|
||||
|
||||
image:
|
||||
registry: ghcr.io
|
||||
repository: kyverno/kyverno
|
||||
tag: "v1.14.0"
|
||||
|
||||
config:
|
||||
# Webhook timeout for policy evaluation. Increase if scanning takes longer.
|
||||
webhookTimeoutSeconds: 30
|
||||
# Failure policy: fail-open (audit/log) vs fail-closed (reject on error)
|
||||
failurePolicy: fail
|
||||
# Resource limits for webhook
|
||||
webhookAnnotations:
|
||||
rules: "allow"
|
||||
|
||||
# Pod security via Kyverno instead of Pod Security Policies (deprecated)
|
||||
# Enforces baseline restrictions cluster-wide, with exceptions for privileged namespaces
|
||||
podSecurityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
|
||||
rbac:
|
||||
create: true
|
||||
|
||||
resources:
|
||||
requests:
|
||||
memory: "256Mi"
|
||||
cpu: "100m"
|
||||
limits:
|
||||
memory: "512Mi"
|
||||
cpu: "500m"
|
||||
|
||||
# Webhook configuration
|
||||
webhook:
|
||||
timeoutSeconds: 30
|
||||
# Failure policy: "Fail" (reject on error) or "Ignore" (audit-only)
|
||||
# Set to "Ignore" for initial testing, then change to "Fail"
|
||||
failurePolicy: ignore
|
||||
@@ -0,0 +1,204 @@
|
||||
# Kyverno ClusterPolicies: Image scanning, Pod security, and admission control
|
||||
---
|
||||
# Policy 1: Require non-root containers
|
||||
apiVersion: kyverno.io/v1
|
||||
kind: ClusterPolicy
|
||||
metadata:
|
||||
name: require-non-root
|
||||
namespace: kyverno
|
||||
spec:
|
||||
validationFailureAction: audit # audit first, then change to enforce
|
||||
rules:
|
||||
- name: check-runAsNonRoot
|
||||
match:
|
||||
any:
|
||||
- resources:
|
||||
kinds:
|
||||
- Pod
|
||||
selector:
|
||||
matchLabels:
|
||||
pod-security.kubernetes.io/enforce: "!privileged"
|
||||
validate:
|
||||
message: "Container must not run as root"
|
||||
pattern:
|
||||
spec:
|
||||
containers:
|
||||
- securityContext:
|
||||
runAsNonRoot: true
|
||||
---
|
||||
# Policy 2: Drop all Linux capabilities, add only required ones
|
||||
apiVersion: kyverno.io/v1
|
||||
kind: ClusterPolicy
|
||||
metadata:
|
||||
name: require-dropped-caps
|
||||
namespace: kyverno
|
||||
spec:
|
||||
validationFailureAction: audit
|
||||
rules:
|
||||
- name: drop-all-capabilities
|
||||
match:
|
||||
any:
|
||||
- resources:
|
||||
kinds:
|
||||
- Pod
|
||||
selector:
|
||||
matchLabels:
|
||||
pod-security.kubernetes.io/enforce: "!privileged"
|
||||
validate:
|
||||
message: "All Linux capabilities must be dropped"
|
||||
pattern:
|
||||
spec:
|
||||
containers:
|
||||
- securityContext:
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
---
|
||||
# Policy 3: Require image tags (no 'latest')
|
||||
apiVersion: kyverno.io/v1
|
||||
kind: ClusterPolicy
|
||||
metadata:
|
||||
name: disallow-latest-tag
|
||||
namespace: kyverno
|
||||
spec:
|
||||
validationFailureAction: audit # Change to enforce after testing
|
||||
rules:
|
||||
- name: disallow-latest
|
||||
match:
|
||||
any:
|
||||
- resources:
|
||||
kinds:
|
||||
- Pod
|
||||
- Deployment
|
||||
- StatefulSet
|
||||
- DaemonSet
|
||||
- Job
|
||||
validate:
|
||||
message: "Image tag 'latest' is not allowed. Use explicit version tags."
|
||||
pattern:
|
||||
spec:
|
||||
=(template):
|
||||
spec:
|
||||
containers:
|
||||
- image: "!*:latest"
|
||||
=(initContainers):
|
||||
- image: "!*:latest"
|
||||
---
|
||||
# Policy 4: Restrict images to trusted registries
|
||||
apiVersion: kyverno.io/v1
|
||||
kind: ClusterPolicy
|
||||
metadata:
|
||||
name: restrict-registries
|
||||
namespace: kyverno
|
||||
spec:
|
||||
validationFailureAction: audit
|
||||
rules:
|
||||
- name: trusted-registries
|
||||
match:
|
||||
any:
|
||||
- resources:
|
||||
kinds:
|
||||
- Pod
|
||||
- Deployment
|
||||
- StatefulSet
|
||||
- DaemonSet
|
||||
- Job
|
||||
selector:
|
||||
matchLabels:
|
||||
pod-security.kubernetes.io/enforce: "!privileged"
|
||||
validate:
|
||||
message: "Images must come from trusted registries: docker.io, ghcr.io, quay.io, k8s.gcr.io, registry.k8s.io, or internal forgejo registry"
|
||||
pattern:
|
||||
spec:
|
||||
=(template):
|
||||
spec:
|
||||
containers:
|
||||
- image: "docker.io/* | ghcr.io/* | quay.io/* | k8s.gcr.io/* | registry.k8s.io/* | forgejo.riotpiao.com/* | *"
|
||||
---
|
||||
# Policy 5: Require read-only root filesystem (audit only, exceptions for apps that need writes)
|
||||
apiVersion: kyverno.io/v1
|
||||
kind: ClusterPolicy
|
||||
metadata:
|
||||
name: require-readonly-filesystem
|
||||
namespace: kyverno
|
||||
spec:
|
||||
validationFailureAction: audit
|
||||
rules:
|
||||
- name: check-readOnlyRootFilesystem
|
||||
match:
|
||||
any:
|
||||
- resources:
|
||||
kinds:
|
||||
- Pod
|
||||
selector:
|
||||
matchLabels:
|
||||
pod-security.kubernetes.io/enforce: "!privileged"
|
||||
validate:
|
||||
message: "Root filesystem should be read-only for defense-in-depth"
|
||||
pattern:
|
||||
spec:
|
||||
containers:
|
||||
- securityContext:
|
||||
readOnlyRootFilesystem: true
|
||||
---
|
||||
# Policy 6: Require resource requests and limits (prevent resource starvation)
|
||||
apiVersion: kyverno.io/v1
|
||||
kind: ClusterPolicy
|
||||
metadata:
|
||||
name: require-resource-limits
|
||||
namespace: kyverno
|
||||
spec:
|
||||
validationFailureAction: audit
|
||||
rules:
|
||||
- name: check-resources
|
||||
match:
|
||||
any:
|
||||
- resources:
|
||||
kinds:
|
||||
- Pod
|
||||
- Deployment
|
||||
- StatefulSet
|
||||
- DaemonSet
|
||||
excludeResources:
|
||||
namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: "kyverno|kube-system|kube-node-lease"
|
||||
validate:
|
||||
message: "CPU and memory requests and limits are required"
|
||||
pattern:
|
||||
spec:
|
||||
=(template):
|
||||
spec:
|
||||
containers:
|
||||
- resources:
|
||||
requests:
|
||||
memory: "?*"
|
||||
cpu: "?*"
|
||||
limits:
|
||||
memory: "?*"
|
||||
cpu: "?*"
|
||||
---
|
||||
# Policy 7: Require securityContext on all containers
|
||||
apiVersion: kyverno.io/v1
|
||||
kind: ClusterPolicy
|
||||
metadata:
|
||||
name: require-security-context
|
||||
namespace: kyverno
|
||||
spec:
|
||||
validationFailureAction: audit
|
||||
rules:
|
||||
- name: check-securityContext
|
||||
match:
|
||||
any:
|
||||
- resources:
|
||||
kinds:
|
||||
- Pod
|
||||
selector:
|
||||
matchLabels:
|
||||
pod-security.kubernetes.io/enforce: "!privileged"
|
||||
validate:
|
||||
message: "securityContext must be defined"
|
||||
pattern:
|
||||
spec:
|
||||
containers:
|
||||
- securityContext: {}
|
||||
@@ -161,19 +161,19 @@ ingress:
|
||||
# The database was never the bottleneck. A real action_runner UPDATE measured
|
||||
# 2.133ms with zero lock waits, so this is CPU starvation in Gitea itself.
|
||||
#
|
||||
# requests.cpu stays 200m and limits.cpu stays 1000m. The 3000m limit was a
|
||||
# temporary measure while the Actions stall was diagnosed; the real fix landed
|
||||
# elsewhere, so the limit is back to the original.
|
||||
# requests.cpu rises to 750m because a request below real usage earns the pod
|
||||
# little scheduling weight during contention. limits.cpu rises to 3000m
|
||||
# because diff computation is bursty and needs room to run without throttling.
|
||||
#
|
||||
# Caution: this pod runs on talos-cp-3, a control-plane node. If Gitea
|
||||
# throttles again at 1000m, move the workload off the control plane rather
|
||||
# than raising the limit.
|
||||
# Caution: this pod runs on talos-cp-3, a control-plane node. 3000m is 25% of
|
||||
# that node's 11950m allocatable CPU. If Gitea sustains near this limit, move
|
||||
# the workload off the control plane rather than raising the limit again.
|
||||
resources:
|
||||
requests:
|
||||
cpu: 200m
|
||||
cpu: 750m
|
||||
memory: 768Mi
|
||||
limits:
|
||||
cpu: 1000m
|
||||
cpu: 3000m
|
||||
memory: 2Gi
|
||||
|
||||
# Tolerations for control-plane
|
||||
|
||||
@@ -1,90 +0,0 @@
|
||||
# One-shot destructive health scan of a candidate Longhorn disk.
|
||||
#
|
||||
# WHY THIS IS A MANUAL JOB AND NOT AN ARGOCD HOOK
|
||||
# It writes over the entire raw device. An ArgoCD PostSync hook would run it
|
||||
# on every sync and destroy whatever the volume holds. Apply it by hand,
|
||||
# once, on a disk that has no partitions and no mounts.
|
||||
#
|
||||
# WHY NOT TALOS
|
||||
# Talos declares disks by mounting them (machine.disks in the machine
|
||||
# config). It has no primitive for "run this command on the host", so a scan
|
||||
# cannot live in the Talos config. This Job is the IaC-correct home.
|
||||
#
|
||||
# ORDER OF OPERATIONS
|
||||
# 1. Scan the raw device with this Job <- the disk must be UNMOUNTED
|
||||
# 2. If it passes, add the disk to terraform.tfvars.local `longhorn_disks`
|
||||
# with device /dev/disk/by-id/<wwid> and run `make apply-cp`
|
||||
# 3. Register it with Longhorn via the pattern in longhorn-add-disks-job.yaml
|
||||
#
|
||||
# What it does
|
||||
# fio writes a CRC32C-verified pattern across the whole device, then reads
|
||||
# every block back and compares. Any unreadable or mismatched sector fails
|
||||
# the Job. This is a full write and read pass, not a sample.
|
||||
#
|
||||
# Expected duration
|
||||
# 2 TB on a 5400 rpm 2.5" drive at roughly 90 MB/s is about 6 h to write and
|
||||
# 6 h to verify. Budget 12 hours. Check progress with:
|
||||
# kubectl -n llm-io logs job/disk-scan-cp1-sdb -f
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: disk-scan-cp1-sdb
|
||||
namespace: llm-io
|
||||
labels:
|
||||
app: disk-scan
|
||||
spec:
|
||||
backoffLimit: 0
|
||||
# Do not let a stuck scan retry silently.
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: disk-scan
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: talos-cp-1
|
||||
containers:
|
||||
- name: scan
|
||||
image: ubuntu:22.04
|
||||
securityContext:
|
||||
privileged: true
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
set -euo pipefail
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq smartmontools fio
|
||||
|
||||
echo "=== device identity ==="
|
||||
smartctl -i /dev/sdb | tail -12
|
||||
|
||||
echo "=== SMART before ==="
|
||||
smartctl -H /dev/sdb | tail -2
|
||||
|
||||
echo "=== full write + verify (destructive) ==="
|
||||
fio --name=devverify \
|
||||
--filename=/dev/sdb \
|
||||
--rw=write \
|
||||
--bs=1M \
|
||||
--iodepth=4 \
|
||||
--ioengine=libaio \
|
||||
--direct=1 \
|
||||
--verify=crc32c \
|
||||
--do_verify=1 \
|
||||
--verify_fatal=1 \
|
||||
--eta=always \
|
||||
--output-format=normal
|
||||
|
||||
echo "=== SMART after ==="
|
||||
smartctl -H /dev/sdb | tail -2
|
||||
smartctl -A /dev/sdb | awk '/Reallocated_Sector|Current_Pending|Offline_Uncorrect|UDMA_CRC/{print " "$0}'
|
||||
|
||||
echo "=== SCAN PASSED ==="
|
||||
volumeMounts:
|
||||
- name: dev
|
||||
mountPath: /dev
|
||||
volumes:
|
||||
- name: dev
|
||||
hostPath:
|
||||
path: /dev
|
||||
@@ -32,6 +32,7 @@ ISSUER = "https://zitadel.riotpiao.com"
|
||||
APPS = [
|
||||
("grafana", "grafana", "grafana-oidc", "logging", "GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET", "deploy", "grafana"),
|
||||
("argocd", "argocd", "oidc-secret", "argocd", "client-secret", "deploy", "argocd-server"),
|
||||
("comfyui", "comfyui", "comfyui-oauth2-proxy", "comfyui", "client-secret", "deploy", "comfyui-oauth2-proxy"),
|
||||
("minio", "minio", "minio-oidc", "storage", "MINIO_IDENTITY_OPENID_CLIENT_SECRET", "sts", "minio-cluster-az-a"),
|
||||
("paperless", "paperless", "paperless-oidc", "paperless", "client-secret", "deploy", "paperless"),
|
||||
("immich", "immich", "immich-oidc", "immich", "client-secret", "deploy", "immich-server"),
|
||||
|
||||
@@ -128,6 +128,11 @@ OIDC_APPS = {
|
||||
"https://forgejo.riotpiao.com/user/oauth2/zitadel/callback",
|
||||
"http://localhost:3000/user/oauth2/authentik/callback",
|
||||
"https://forgejo.riotpiao.com/user/oauth2/authentik/callback"]},
|
||||
# ComfyUI has no native OIDC. oauth2-proxy is its front door and this app is
|
||||
# the proxy's client, so the callback path belongs to the proxy.
|
||||
"comfyui": {**_WEB, "redirect_uris": [
|
||||
"http://localhost:4180/oauth2/callback",
|
||||
"https://comfyui.riotpiao.com/oauth2/callback"]},
|
||||
"grafana": {**_WEB, "redirect_uris": [
|
||||
"http://localhost:3000/login/generic_oauth",
|
||||
"https://grafana.riotpiao.com/login/generic_oauth"]},
|
||||
@@ -138,11 +143,7 @@ OIDC_APPS = {
|
||||
"http://localhost:8000", "http://localhost:18000"]},
|
||||
"minio": {**_WEB, "redirect_uris": [
|
||||
"http://localhost:9000/auth/sso/oauth2/code",
|
||||
"https://minio.riotpiao.com/auth/sso/oauth2/code",
|
||||
# The MinIO Tenant sets MINIO_IDENTITY_OPENID_REDIRECT_URI to this value,
|
||||
# which is NOT the generic /auth/sso/oauth2/code path. Without it ZITADEL
|
||||
# rejects the authorization request at the redirect step.
|
||||
"https://minio.riotpiao.com/oauth_callback"]},
|
||||
"https://minio.riotpiao.com/auth/sso/oauth2/code"]},
|
||||
"paperless": {**_WEB, "redirect_uris": [
|
||||
"http://localhost:8000/accounts/oidc/zitadel/login/callback/",
|
||||
"https://paperless.riotpiao.com/accounts/oidc/zitadel/login/callback/",
|
||||
@@ -358,13 +359,6 @@ MACHINE_CALLERS = {
|
||||
"description": "poimen calls api-gateway inference",
|
||||
"roles": ["llm:inference", "svc-caller"],
|
||||
},
|
||||
# paperless-ai is a vendor image: it cannot sign an assertion, so a sidecar mints
|
||||
# a token with this account and writes it to the file the image already reads.
|
||||
"paperless-llm": {
|
||||
"name": "Paperless AI LLM caller",
|
||||
"description": "paperless-ai calls api-gateway inference through the token vendor",
|
||||
"roles": ["llm:inference", "svc-caller"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -36,10 +36,8 @@
|
||||
rewrite name paperless.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
|
||||
rewrite name img.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
|
||||
rewrite name api.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
|
||||
# poimen ran only in the live ConfigMap, never in this file or the Talos
|
||||
# machine config. The next Talos reconcile would have deleted it and broken
|
||||
# in-cluster resolution. Declared here so the source of truth carries it.
|
||||
rewrite name poimen.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
|
||||
rewrite name comfy.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
|
||||
rewrite name comfyui.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
|
||||
rewrite name gotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
|
||||
rewrite name pi.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
|
||||
rewrite name igotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
|
||||
|
||||
Reference in New Issue
Block a user