refactor(k8s): consolidate to infra/+apps/ single-source tree, dedicated per-app CNPG (authentik-db/temporal-db), wire monitoring-config, forgejo→cicd ns, drop orphan/stale (data-schemas, ollama, story-crater, sqs/argocd, key-rotation)
This commit is contained in:
Story Crater Bot
@@ -1,6 +0,0 @@
|
||||
apiVersion: v2
|
||||
name: claude-terminal
|
||||
description: Persistent Claude CLI terminal running in tmux with web access via gotty
|
||||
type: application
|
||||
version: 1.0.0
|
||||
appVersion: "1.0"
|
||||
@@ -1,31 +0,0 @@
|
||||
FROM --platform=linux/amd64 ubuntu:24.04
|
||||
|
||||
RUN apt-get update && apt-get install -y \
|
||||
tmux \
|
||||
curl \
|
||||
git \
|
||||
build-essential \
|
||||
nodejs \
|
||||
npm \
|
||||
bash \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install gotty (web terminal access)
|
||||
RUN curl -sL https://github.com/sorenisanerd/gotty/releases/download/v1.5.0/gotty_linux_amd64.tar.gz | \
|
||||
tar xz -C /usr/local/bin && chmod +x /usr/local/bin/gotty
|
||||
|
||||
# Install Claude CLI
|
||||
RUN npm install -g claude-code-cli 2>&1 || echo "Note: Claude CLI will be available after NPM package is published"
|
||||
|
||||
WORKDIR /root
|
||||
|
||||
# Create persistent storage dir
|
||||
RUN mkdir -p /root/.claude /root/.config /root/.cache
|
||||
|
||||
# Entrypoint: start tmux session and gotty
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
RUN chmod +x /entrypoint.sh
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
@@ -1,49 +0,0 @@
|
||||
# Claude Terminal — Persistent Remote Dev Environment
|
||||
|
||||
Runs Claude CLI in a persistent tmux session with web-based terminal access via gotty.
|
||||
|
||||
## Building the Image
|
||||
|
||||
Build for `linux/amd64`:
|
||||
|
||||
```bash
|
||||
cd homelab
|
||||
docker buildx build --platform linux/amd64 \
|
||||
-t forgejo.riotpiao.com/rock/claude-terminal:latest \
|
||||
-f k8s/dev-tools/Dockerfile \
|
||||
k8s/dev-tools
|
||||
|
||||
# Log in to Forgejo registry
|
||||
docker login forgejo.riotpiao.com \
|
||||
--username ci-bot \
|
||||
--password "$(talos get cluster/iam/agents/ci-bot --key token)"
|
||||
|
||||
# Push
|
||||
docker push forgejo.riotpiao.com/rock/claude-terminal:latest
|
||||
```
|
||||
|
||||
Or use the provided build script:
|
||||
|
||||
```bash
|
||||
./k8s/dev-tools/build.sh
|
||||
```
|
||||
|
||||
## Deployment
|
||||
|
||||
Update `values.yaml` if needed, then deploy via helmfile:
|
||||
|
||||
```bash
|
||||
helmfile apply -l name=claude-terminal
|
||||
```
|
||||
|
||||
Access the terminal at: **https://claude.riotpiao.com**
|
||||
|
||||
## Persistent Storage
|
||||
|
||||
- All Claude configuration stored in `/root/.claude` (persistent PVC, 10Gi Longhorn)
|
||||
- Survives pod restarts and node reboots
|
||||
- Accessible immediately after reconnecting
|
||||
|
||||
## SSH Access (Optional)
|
||||
|
||||
To add SSH access, extend the Dockerfile to include openssh-server and mount the PVC as home directory.
|
||||
@@ -1,24 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
REGISTRY="forgejo.riotpiao.com"
|
||||
IMAGE_NAME="rock/claude-terminal"
|
||||
TAG="latest"
|
||||
FULL_IMAGE="${REGISTRY}/${IMAGE_NAME}:${TAG}"
|
||||
|
||||
echo "🔨 Building Claude Terminal image for linux/amd64..."
|
||||
docker buildx build --platform linux/amd64 \
|
||||
-t "${FULL_IMAGE}" \
|
||||
-f Dockerfile \
|
||||
. || { echo "❌ Build failed"; exit 1; }
|
||||
|
||||
echo "🔓 Logging in to Forgejo registry..."
|
||||
REGISTRY_TOKEN=$(talos get cluster/iam/agents/ci-bot --key token)
|
||||
echo "${REGISTRY_TOKEN}" | docker login "${REGISTRY}" \
|
||||
--username ci-bot \
|
||||
--password-stdin || { echo "❌ Login failed"; exit 1; }
|
||||
|
||||
echo "📤 Pushing image to registry..."
|
||||
docker push "${FULL_IMAGE}" || { echo "❌ Push failed"; exit 1; }
|
||||
|
||||
echo "✅ Successfully pushed ${FULL_IMAGE}"
|
||||
@@ -1,13 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Start tmux server in background
|
||||
tmux new-session -d -s claude -c /root "bash"
|
||||
|
||||
# Give tmux a moment to stabilize
|
||||
sleep 1
|
||||
|
||||
# Start gotty serving the tmux session
|
||||
# -w: allow write (make terminal interactive)
|
||||
# -p 8080: listen on port 8080
|
||||
exec gotty -p 8080 -w tmux attach-session -t claude
|
||||
@@ -1,5 +0,0 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: dev-tools
|
||||
resources: []
|
||||
# Helm chart deployed via ArgoCD Helm source
|
||||
@@ -1,49 +0,0 @@
|
||||
{{/*
|
||||
Expand the name of the chart.
|
||||
*/}}
|
||||
{{- define "claude-terminal.name" -}}
|
||||
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Create a default fully qualified app name.
|
||||
*/}}
|
||||
{{- define "claude-terminal.fullname" -}}
|
||||
{{- if .Values.fullnameOverride }}
|
||||
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
|
||||
{{- else }}
|
||||
{{- $name := default .Chart.Name .Values.nameOverride }}
|
||||
{{- if contains $name .Release.Name }}
|
||||
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
|
||||
{{- else }}
|
||||
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Create chart name and version as used by the chart label.
|
||||
*/}}
|
||||
{{- define "claude-terminal.chart" -}}
|
||||
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Common labels
|
||||
*/}}
|
||||
{{- define "claude-terminal.labels" -}}
|
||||
helm.sh/chart: {{ include "claude-terminal.chart" . }}
|
||||
{{ include "claude-terminal.selectorLabels" . }}
|
||||
{{- if .Chart.AppVersion }}
|
||||
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
|
||||
{{- end }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Selector labels
|
||||
*/}}
|
||||
{{- define "claude-terminal.selectorLabels" -}}
|
||||
app.kubernetes.io/name: {{ include "claude-terminal.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
{{- end }}
|
||||
@@ -1,57 +0,0 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "claude-terminal.fullname" . }}
|
||||
labels:
|
||||
{{- include "claude-terminal.labels" . | nindent 4 }}
|
||||
spec:
|
||||
replicas: {{ .Values.replicaCount }}
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "claude-terminal.selectorLabels" . | nindent 6 }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "claude-terminal.selectorLabels" . | nindent 8 }}
|
||||
spec:
|
||||
containers:
|
||||
- name: claude-terminal
|
||||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
protocol: TCP
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: http
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: http
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
resources:
|
||||
{{- toYaml .Values.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
- name: claude-storage
|
||||
mountPath: {{ .Values.persistence.mountPath }}
|
||||
volumes:
|
||||
- name: claude-storage
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ include "claude-terminal.fullname" . }}-pvc
|
||||
{{- with .Values.nodeSelector }}
|
||||
nodeSelector:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.affinity }}
|
||||
affinity:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.tolerations }}
|
||||
tolerations:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
@@ -1,41 +0,0 @@
|
||||
{{- if .Values.ingress.enabled }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: {{ include "claude-terminal.fullname" . }}
|
||||
labels:
|
||||
{{- include "claude-terminal.labels" . | nindent 4 }}
|
||||
{{- with .Values.ingress.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if .Values.ingress.className }}
|
||||
ingressClassName: {{ .Values.ingress.className }}
|
||||
{{- end }}
|
||||
{{- if .Values.ingress.tls }}
|
||||
tls:
|
||||
{{- range .Values.ingress.tls }}
|
||||
- hosts:
|
||||
{{- range .hosts }}
|
||||
- {{ . | quote }}
|
||||
{{- end }}
|
||||
secretName: {{ .secretName }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
rules:
|
||||
{{- range .Values.ingress.hosts }}
|
||||
- host: {{ .host | quote }}
|
||||
http:
|
||||
paths:
|
||||
{{- range .paths }}
|
||||
- path: {{ .path }}
|
||||
pathType: {{ .pathType }}
|
||||
backend:
|
||||
service:
|
||||
name: {{ include "claude-terminal.fullname" $ }}
|
||||
port:
|
||||
number: {{ $.Values.service.port }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -1,15 +0,0 @@
|
||||
{{- if .Values.persistence.enabled }}
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: {{ include "claude-terminal.fullname" . }}-pvc
|
||||
labels:
|
||||
{{- include "claude-terminal.labels" . | nindent 4 }}
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: {{ .Values.persistence.storageClass }}
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.persistence.size }}
|
||||
{{- end }}
|
||||
@@ -1,15 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "claude-terminal.fullname" . }}
|
||||
labels:
|
||||
{{- include "claude-terminal.labels" . | nindent 4 }}
|
||||
spec:
|
||||
type: {{ .Values.service.type }}
|
||||
ports:
|
||||
- port: {{ .Values.service.port }}
|
||||
targetPort: http
|
||||
protocol: TCP
|
||||
name: http
|
||||
selector:
|
||||
{{- include "claude-terminal.selectorLabels" . | nindent 4 }}
|
||||
@@ -1,45 +0,0 @@
|
||||
replicaCount: 1
|
||||
|
||||
image:
|
||||
repository: localhost:5000/claude-terminal
|
||||
pullPolicy: IfNotPresent
|
||||
tag: latest
|
||||
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 8080
|
||||
|
||||
ingress:
|
||||
enabled: true
|
||||
className: nginx
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: homelab-ca
|
||||
hosts:
|
||||
- host: claude.riotpiao.com
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
tls:
|
||||
- secretName: claude-terminal-tls
|
||||
hosts:
|
||||
- claude.riotpiao.com
|
||||
|
||||
persistence:
|
||||
enabled: true
|
||||
storageClass: longhorn
|
||||
size: 10Gi
|
||||
mountPath: /root/.claude
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
|
||||
nodeSelector: {}
|
||||
|
||||
tolerations: []
|
||||
|
||||
affinity: {}
|
||||
@@ -1,5 +0,0 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: forge
|
||||
resources:
|
||||
- pki/
|
||||
@@ -1,5 +0,0 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
resources: []
|
||||
# PKI configuration, not K8s manifests
|
||||
@@ -1,95 +0,0 @@
|
||||
# k8s/forge/runner-gc-cronjob.yaml
|
||||
# Garbage-collects the forgejo-runner's DinD layer cache (runner-dind PVC,
|
||||
# 30Gi). Every CI build/pull only adds images and build-cache layers — there
|
||||
# is no automatic pruning, so without this the PVC fills up and breaks builds.
|
||||
#
|
||||
# Runs `docker image prune` / `docker builder prune` inside the live dind
|
||||
# container via `kubectl exec`, rather than a sidecar in the runner pod itself,
|
||||
# so it can run on its own schedule independent of runner restarts.
|
||||
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: runner-gc
|
||||
namespace: cicd
|
||||
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: runner-gc
|
||||
namespace: cicd
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["pods"]
|
||||
verbs: ["get", "list"]
|
||||
- apiGroups: [""]
|
||||
resources: ["pods/exec"]
|
||||
verbs: ["create"]
|
||||
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: runner-gc
|
||||
namespace: cicd
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: runner-gc
|
||||
namespace: cicd
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: runner-gc
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: forgejo-runner-image-gc
|
||||
namespace: cicd
|
||||
spec:
|
||||
schedule: "0 3 * * *" # daily 03:00
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 1
|
||||
activeDeadlineSeconds: 600
|
||||
template:
|
||||
spec:
|
||||
serviceAccountName: runner-gc
|
||||
restartPolicy: Never
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
containers:
|
||||
- name: gc
|
||||
image: alpine/k8s:1.31.0
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -e
|
||||
POD=$(kubectl -n cicd get pod -l app=forgejo-runner -o jsonpath='{.items[0].metadata.name}')
|
||||
if [ -z "$POD" ]; then
|
||||
echo "no forgejo-runner pod found, skipping"
|
||||
exit 0
|
||||
fi
|
||||
echo "before:"
|
||||
kubectl -n cicd exec "$POD" -c dind -- df -h /var/lib/docker
|
||||
echo "pruning images unused for >72h on $POD"
|
||||
kubectl -n cicd exec "$POD" -c dind -- docker image prune -af --filter "until=72h"
|
||||
echo "pruning build cache unused for >72h on $POD"
|
||||
kubectl -n cicd exec "$POD" -c dind -- docker builder prune -af --filter "until=72h"
|
||||
echo "after:"
|
||||
kubectl -n cicd exec "$POD" -c dind -- df -h /var/lib/docker
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
cpu: 250m
|
||||
memory: 128Mi
|
||||
@@ -1,438 +0,0 @@
|
||||
# k8s/forge/runner.yaml
|
||||
# Forgejo Actions runner with Docker-in-Docker (DinD) sidecar.
|
||||
# Phase 3.2 of talos_version_control.html.
|
||||
#
|
||||
# Pod layout (two containers, one pod):
|
||||
# register initContainer — registers with Forgejo once; skips if .runner exists
|
||||
# runner main container — long-running daemon that polls Forgejo for jobs
|
||||
# dind sidecar — Docker daemon the runner talks to via mTLS on tcp://localhost:2376
|
||||
#
|
||||
# Prerequisites (Phase 3.1):
|
||||
# TOKEN=$(kubectl -n cicd exec deploy/forgejo-gitea -- \
|
||||
# gitea actions generate-runner-token 2>/dev/null | tr -d '\r\n')
|
||||
# kubectl -n cicd create secret generic runner-token --from-literal=token="$TOKEN"
|
||||
# # CA must come from cert-manager's homelab-ca-secret (the org-wide CA that
|
||||
# # signs the live ingress cert), NOT k8s/forge/pki/ca.crt — that file is a
|
||||
# # stale CA from before the "unified certificate" migration.
|
||||
# kubectl get secret homelab-ca-secret -n cert-manager -o jsonpath='{.data.tls\.crt}' \
|
||||
# | base64 -d | kubectl -n cicd create secret generic homelab-ca --from-file=ca.crt=/dev/stdin
|
||||
#
|
||||
# CA trust for ephemeral job containers (Phase 3.3):
|
||||
# The homelab-ca secret above only reaches the runner/register/dind containers.
|
||||
# It does NOT reach the per-job containers DinD spawns (e.g. golangci-lint,
|
||||
# node:22-bookworm) — those are fresh sibling containers with their own trust
|
||||
# store. Without this, any git/curl/apk call to forgejo.riotpiao.com
|
||||
# from inside a job fails with "SSL certificate problem: unable to get local
|
||||
# issuer certificate". Build a merged bundle (public roots + homelab CA) and
|
||||
# bind-mount it over /etc/ssl/certs/ca-certificates.crt in every job
|
||||
# container via forgejo-runner's config.yaml container.options.
|
||||
#
|
||||
# IMPORTANT: source the CA from the live cluster secret, NOT from
|
||||
# k8s/forge/pki/ca.crt — that repo file is a stale CA from before the
|
||||
# "unified certificate" migration (different key, fails verification
|
||||
# against the cert actually served by forgejo.riotpiao.com). The
|
||||
# org-wide CA that signs the live ingress cert lives in
|
||||
# cert-manager/homelab-ca-secret, and cicd/homelab-ca above is already
|
||||
# synced from it.
|
||||
# docker run --rm docker:27-dind cat /etc/ssl/certs/ca-certificates.crt > /tmp/ca-bundle.crt
|
||||
# kubectl -n cicd get secret homelab-ca -o jsonpath='{.data.ca\.crt}' | base64 -d >> /tmp/ca-bundle.crt
|
||||
# kubectl -n cicd create secret generic ca-bundle --from-file=ca-certificates.crt=/tmp/ca-bundle.crt
|
||||
# Re-run this whenever the homelab CA rotates (see talos-forge-trust.yaml).
|
||||
#
|
||||
# Apply:
|
||||
# kubectl apply -f k8s/forge/runner.yaml
|
||||
# kubectl -n cicd rollout status deploy/forgejo-runner
|
||||
# kubectl -n cicd logs deploy/forgejo-runner -c runner -f
|
||||
# # expect: "runner: daemon started" / "connected to Forgejo"
|
||||
|
||||
# ── PVCs ──────────────────────────────────────────────────────────────────────
|
||||
# runner-reg — persists the .runner registration file so the runner doesn't
|
||||
# re-register on every pod restart (token is one-use-per-registration)
|
||||
# runner-dind — persists the Docker layer cache across pod restarts; keeps
|
||||
# rebuilds fast — images don't need to be re-pulled every time
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: runner-reg
|
||||
namespace: cicd
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
storageClassName: longhorn
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Gi
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: runner-dind
|
||||
namespace: cicd
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
storageClassName: longhorn
|
||||
resources:
|
||||
requests:
|
||||
storage: 30Gi
|
||||
|
||||
---
|
||||
# ── DinD TLS certs, issued by the homelab's unified CA ───────────────────────
|
||||
# DinD's own entrypoint (dockerd-entrypoint.sh) self-generates a throwaway CA
|
||||
# + server/client cert pair on every container start if none is supplied. Its
|
||||
# server cert's SAN list only ever covers "docker", the pod hostname, and
|
||||
# "localhost" - so anything reaching it via a stable Service DNS name (added
|
||||
# below for story-crater-backend's release.yaml to build/push images) fails
|
||||
# TLS hostname verification, even though the handshake itself succeeds.
|
||||
#
|
||||
# Fix: supply our own server+client cert pair, both issued by the same
|
||||
# ClusterIssuer (homelab-ca) that already signs the live ingress cert, so
|
||||
# they share one trust root. dockerd-entrypoint.sh skips its own generation
|
||||
# step entirely once it finds $DOCKER_TLS_CERTDIR/server/{ca,cert,key}.pem
|
||||
# already present and no CA private key alongside them (confirmed by reading
|
||||
# the script directly: `kubectl exec -n cicd <pod> -c dind -- cat
|
||||
# /usr/local/bin/dockerd-entrypoint.sh`) - exactly the "bring your own CA"
|
||||
# path it's designed for.
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: forgejo-runner-dind-server
|
||||
namespace: cicd
|
||||
spec:
|
||||
secretName: forgejo-runner-dind-server-tls
|
||||
issuerRef:
|
||||
name: homelab-ca
|
||||
kind: ClusterIssuer
|
||||
commonName: docker:dind server
|
||||
dnsNames:
|
||||
- forgejo-runner-dind.cicd.svc.cluster.local
|
||||
- forgejo-runner-dind.cicd.svc
|
||||
- forgejo-runner-dind
|
||||
- docker
|
||||
- localhost
|
||||
usages:
|
||||
- server auth
|
||||
- digital signature
|
||||
- key encipherment
|
||||
|
||||
---
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: forgejo-runner-dind-client
|
||||
namespace: cicd
|
||||
spec:
|
||||
secretName: forgejo-runner-dind-client-tls
|
||||
issuerRef:
|
||||
name: homelab-ca
|
||||
kind: ClusterIssuer
|
||||
commonName: docker:dind client
|
||||
usages:
|
||||
- client auth
|
||||
- digital signature
|
||||
- key encipherment
|
||||
|
||||
---
|
||||
# Stable address for the dind sidecar's docker API (2376, mTLS) - lets
|
||||
# CI workflows (e.g. story-crater-backend's release.yaml) build/push images
|
||||
# by reaching this runner's own already-working outer dind directly, instead
|
||||
# of the per-job `services:` sidecar pattern (confirmed broken: act-runner
|
||||
# never registers a DNS alias for service containers - job container's
|
||||
# /etc/hosts has no entry for it, `docker info` fails with a DNS lookup
|
||||
# error, not a TLS/connection error).
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: forgejo-runner-dind
|
||||
namespace: cicd
|
||||
spec:
|
||||
selector:
|
||||
app: forgejo-runner
|
||||
ports:
|
||||
- port: 2376
|
||||
targetPort: 2376
|
||||
|
||||
---
|
||||
# ── Runner config ─────────────────────────────────────────────────────────────
|
||||
# container.options is appended to every `docker run` DinD issues for a job
|
||||
# container, so this is what actually gets the merged CA bundle (ca-bundle
|
||||
# secret, see header comment) trusted inside golangci-lint, node, etc. - and
|
||||
# now also what propagates the homelab-CA-signed client cert (above) into
|
||||
# job containers that need to talk back to dind themselves (e.g. building
|
||||
# and pushing images).
|
||||
# Source paths resolve against the dind container's filesystem (it's the
|
||||
# daemon creating these containers), so both are mounted into dind below.
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: forgejo-runner-config
|
||||
namespace: cicd
|
||||
data:
|
||||
config.yaml: |
|
||||
container:
|
||||
options: -v /etc/forgejo-ca/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt:ro -v /docker-certs/client:/docker-certs/client:ro
|
||||
# Without this, sanitizeConfig() in forgejo/act silently drops any bind
|
||||
# mount whose source isn't allowlisted here — including ones injected
|
||||
# via container.options above, not just workflow-declared volumes.
|
||||
valid_volumes:
|
||||
- /etc/forgejo-ca/ca-certificates.crt
|
||||
- /docker-certs/client
|
||||
|
||||
---
|
||||
# ── Deployment ────────────────────────────────────────────────────────────────
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: forgejo-runner
|
||||
namespace: cicd
|
||||
spec:
|
||||
replicas: 1
|
||||
# RWO PVCs mean only one pod can mount them at a time.
|
||||
# Recreate ensures the old pod fully terminates before the new one starts.
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: forgejo-runner
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: forgejo-runner
|
||||
spec:
|
||||
# runner/register containers run as uid 1000 (image default); fsGroup
|
||||
# makes kubelet chown+chmod the Longhorn PVC's group to 1000 with
|
||||
# write access, otherwise writes to /data (.runner config) fail with
|
||||
# "permission denied" since the volume is root:root 755 by default.
|
||||
securityContext:
|
||||
fsGroup: 1000
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
|
||||
initContainers:
|
||||
# Registers the runner with Forgejo exactly once.
|
||||
# test -f /data/.runner makes it idempotent — re-registration would
|
||||
# consume the one-time token and break the runner.
|
||||
- name: register
|
||||
image: code.forgejo.org/forgejo/runner:6
|
||||
command: ["sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
test -f /data/.runner && echo "already registered, skipping" && exit 0
|
||||
forgejo-runner register --no-interactive \
|
||||
--instance https://forgejo.riotpiao.com \
|
||||
--token "$RUNNER_TOKEN" \
|
||||
--name talos-runner \
|
||||
--labels "docker:docker://node:22-bookworm"
|
||||
env:
|
||||
- name: RUNNER_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: runner-token
|
||||
key: token
|
||||
volumeMounts:
|
||||
- name: runner-data
|
||||
mountPath: /data
|
||||
# CA cert so the register call can verify Forgejo's TLS cert
|
||||
- name: homelab-ca
|
||||
mountPath: /etc/ssl/certs/homelab-ca.pem
|
||||
subPath: ca.crt
|
||||
workingDir: /data
|
||||
|
||||
containers:
|
||||
# ── Runner daemon ────────────────────────────────────────────────────
|
||||
# Polls Forgejo for pending jobs and executes them inside DinD.
|
||||
# The `until docker info` loop waits for the DinD sidecar to finish
|
||||
# its TLS setup before starting the daemon — without this the runner
|
||||
# starts before Docker is ready and immediately errors out.
|
||||
- name: runner
|
||||
image: code.forgejo.org/forgejo/runner:6
|
||||
command: ["sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
until nc -z localhost 2376 >/dev/null 2>&1; do
|
||||
echo "waiting for docker daemon..."; sleep 2
|
||||
done
|
||||
forgejo-runner daemon --config /data/config.yaml
|
||||
workingDir: /data
|
||||
env:
|
||||
# Connect to the DinD sidecar via mTLS on localhost
|
||||
- name: DOCKER_HOST
|
||||
value: tcp://localhost:2376
|
||||
- name: DOCKER_TLS_VERIFY
|
||||
value: "1"
|
||||
- name: DOCKER_CERT_PATH
|
||||
value: /docker-certs/client
|
||||
volumeMounts:
|
||||
- name: runner-data
|
||||
mountPath: /data
|
||||
- name: docker-certs
|
||||
mountPath: /docker-certs
|
||||
- name: homelab-ca
|
||||
mountPath: /etc/ssl/certs/homelab-ca.pem
|
||||
subPath: ca.crt
|
||||
# forgejo-runner's container.options, read from this file, is what
|
||||
# propagates the CA bundle into per-job containers (see ca-bundle
|
||||
# secret + dind mount below)
|
||||
- name: runner-config
|
||||
mountPath: /data/config.yaml
|
||||
subPath: config.yaml
|
||||
# Homelab-CA-signed client cert (overlays whatever's in the
|
||||
# docker-certs emptyDir at this subpath) - matches the server
|
||||
# cert dind now presents, see Certificates above.
|
||||
- name: dind-client-tls
|
||||
mountPath: /docker-certs/client
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 4Gi
|
||||
|
||||
# ── DinD sidecar ──────────────────────────────────────────────────────
|
||||
# Full Docker daemon running inside the pod.
|
||||
# privileged: true is required for DinD — the cicd namespace is labelled
|
||||
# pod-security.kubernetes.io/enforce=privileged to allow this.
|
||||
# DOCKER_TLS_CERTDIR causes DinD to generate mTLS certs in /docker-certs
|
||||
# on startup; the runner reads the client certs from /docker-certs/client.
|
||||
# runner-dind PVC mounts /var/lib/docker so the layer cache persists
|
||||
# across pod restarts.
|
||||
- name: dind
|
||||
image: docker:27-dind
|
||||
securityContext:
|
||||
privileged: true
|
||||
env:
|
||||
- name: DOCKER_TLS_CERTDIR
|
||||
value: /docker-certs
|
||||
volumeMounts:
|
||||
- name: docker-certs
|
||||
mountPath: /docker-certs
|
||||
- name: dind-storage
|
||||
mountPath: /var/lib/docker
|
||||
# Trust the homelab CA so DinD can pull from Forgejo's OCI registry
|
||||
- name: homelab-ca
|
||||
mountPath: /etc/ssl/certs/homelab-ca.pem
|
||||
subPath: ca.crt
|
||||
# Merged CA bundle (public roots + homelab CA), bind-mounted from
|
||||
# here into every job container by container.options above —
|
||||
# this path is resolved against dind's filesystem since dind is
|
||||
# the daemon actually creating those containers.
|
||||
- name: ca-bundle
|
||||
mountPath: /etc/forgejo-ca/ca-certificates.crt
|
||||
subPath: ca-certificates.crt
|
||||
# Homelab-CA-signed server+client certs (see Certificates above),
|
||||
# overlaying the matching subpaths of the docker-certs emptyDir.
|
||||
# dockerd-entrypoint.sh detects these and skips its own
|
||||
# self-signed generation entirely (no CA private key is supplied
|
||||
# alongside them, so it can't regenerate even if it wanted to).
|
||||
- name: dind-server-tls
|
||||
mountPath: /docker-certs/server
|
||||
- name: dind-client-tls
|
||||
mountPath: /docker-certs/client
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 4Gi
|
||||
|
||||
volumes:
|
||||
- name: runner-data
|
||||
persistentVolumeClaim:
|
||||
claimName: runner-reg
|
||||
- name: dind-storage
|
||||
persistentVolumeClaim:
|
||||
claimName: runner-dind
|
||||
# emptyDir parent mount for /docker-certs - the server/ and client/
|
||||
# subpaths are now overlaid by the homelab-CA-signed dind-server-tls/
|
||||
# dind-client-tls secret mounts below (dockerd-entrypoint.sh no
|
||||
# longer self-generates once it finds those present). This emptyDir
|
||||
# just needs to exist as the parent directory; nothing writes
|
||||
# directly to it anymore.
|
||||
- name: docker-certs
|
||||
emptyDir: {}
|
||||
- name: homelab-ca
|
||||
secret:
|
||||
secretName: homelab-ca
|
||||
- name: ca-bundle
|
||||
secret:
|
||||
secretName: ca-bundle
|
||||
- name: runner-config
|
||||
configMap:
|
||||
name: forgejo-runner-config
|
||||
# cert-manager issues these as tls.crt/tls.key/ca.crt - remapped to
|
||||
# the ca.pem/cert.pem/key.pem filenames dockerd-entrypoint.sh expects
|
||||
# under $DOCKER_TLS_CERTDIR/{server,client}/.
|
||||
- name: dind-server-tls
|
||||
secret:
|
||||
secretName: forgejo-runner-dind-server-tls
|
||||
items:
|
||||
- key: ca.crt
|
||||
path: ca.pem
|
||||
- key: tls.crt
|
||||
path: cert.pem
|
||||
- key: tls.key
|
||||
path: key.pem
|
||||
- name: dind-client-tls
|
||||
secret:
|
||||
secretName: forgejo-runner-dind-client-tls
|
||||
items:
|
||||
- key: ca.crt
|
||||
path: ca.pem
|
||||
- key: tls.crt
|
||||
path: cert.pem
|
||||
- key: tls.key
|
||||
path: key.pem
|
||||
|
||||
---
|
||||
# ── NetworkPolicy ─────────────────────────────────────────────────────────────
|
||||
# Restrict runner egress: it may only reach Forgejo (cicd ns), CoreDNS, and
|
||||
# the public internet for action dependencies and base images.
|
||||
# LAN (192.168.1.0/24) and the pod network (10.244.0.0/16) are blocked to
|
||||
# prevent a compromised CI job from pivoting into the cluster or LAN.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: runner-egress
|
||||
namespace: cicd
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: forgejo-runner
|
||||
policyTypes: [Egress]
|
||||
egress:
|
||||
# Forgejo (same namespace — git push, OCI registry push/pull)
|
||||
- to:
|
||||
- podSelector: {}
|
||||
# ingress-nginx (the runner talks to Forgejo via its public hostname,
|
||||
# https://forgejo.riotpiao.com, which resolves to the ingress
|
||||
# controller's ClusterIP — a different namespace on the pod network)
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: ingress-nginx
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 443
|
||||
- protocol: TCP
|
||||
port: 80
|
||||
# CoreDNS (DNS resolution for action deps and Forgejo hostname)
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 53
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
# Public internet for action dependencies and base images
|
||||
# LAN and pod network are explicitly excluded
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 0.0.0.0/0
|
||||
except:
|
||||
- 192.168.1.0/24
|
||||
- 10.244.0.0/16
|
||||
@@ -1,19 +0,0 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: applications
|
||||
|
||||
# Layer 5: Applications — user services, workloads
|
||||
# Dependencies: all previous layers (bootstrap, platform, security)
|
||||
# Order: Applied fifth
|
||||
|
||||
resources:
|
||||
- temporal/
|
||||
- sqs/
|
||||
- llm/
|
||||
- portainer/
|
||||
- forge/
|
||||
- shadowsocks/
|
||||
- dev-tools/
|
||||
- duckdns/
|
||||
- cloudflared/
|
||||
@@ -1,299 +0,0 @@
|
||||
# Ollama LLM Inference Service
|
||||
|
||||
CPU-only LLM inference server on talos-cp-1. Single model hot-loaded (DeepSeek-R1:70b), 42GB, 70Gi memory limit.
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Access via port-forward
|
||||
```bash
|
||||
kubectl -n llm port-forward svc/ollama 11434:11434
|
||||
curl http://localhost:11434/api/tags
|
||||
```
|
||||
|
||||
### Debug pod (in-cluster)
|
||||
```bash
|
||||
kubectl run debug --rm -it -n llm --image=curlimages/curl \
|
||||
--labels="app.kubernetes.io/role=llm-debug" \
|
||||
--serviceaccount=llm-worker -- sh
|
||||
|
||||
# Inside pod
|
||||
TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
|
||||
curl -H "Authorization: Bearer $TOKEN" \
|
||||
http://ollama.llm.svc.cluster.local:11434/api/tags
|
||||
```
|
||||
|
||||
## Architecture
|
||||
|
||||
| Component | Value |
|
||||
|-----------|-------|
|
||||
| Service | ClusterIP `ollama.llm.svc.cluster.local:11434` |
|
||||
| Namespace | `llm` |
|
||||
| Node | talos-cp-1 (pinned via nodeAffinity) |
|
||||
| Memory request | 50Gi |
|
||||
| Memory limit | 70Gi |
|
||||
| Storage | 115Gi PVC (Longhorn) |
|
||||
| Model | `deepseek-r1:70b` (~42GB) |
|
||||
| Max loaded | 1 model |
|
||||
| Parallelism | 1 request at a time |
|
||||
|
||||
## API Endpoints
|
||||
|
||||
### List models
|
||||
```bash
|
||||
curl http://ollama.llm.svc.cluster.local:11434/api/tags
|
||||
```
|
||||
|
||||
Response:
|
||||
```json
|
||||
{
|
||||
"models": [
|
||||
{"name": "deepseek-r1:70b", "size": 42000000000, ...}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Generate (non-streaming)
|
||||
```bash
|
||||
curl -X POST http://ollama.llm.svc.cluster.local:11434/api/generate \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "deepseek-r1:70b",
|
||||
"prompt": "Why is the sky blue?",
|
||||
"stream": false
|
||||
}'
|
||||
```
|
||||
|
||||
### Pull model
|
||||
```bash
|
||||
curl -X POST http://ollama.llm.svc.cluster.local:11434/api/pull \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"name": "deepseek-r1:70b", "stream": false}'
|
||||
```
|
||||
|
||||
## Operations
|
||||
|
||||
### Check pod status
|
||||
```bash
|
||||
kubectl -n llm get pod -l app.kubernetes.io/name=ollama
|
||||
kubectl -n llm describe pod -l app.kubernetes.io/name=ollama
|
||||
```
|
||||
|
||||
### View logs
|
||||
```bash
|
||||
kubectl -n llm logs deployment/ollama -f
|
||||
```
|
||||
|
||||
### Monitor download progress (bootstrap)
|
||||
```bash
|
||||
kubectl -n llm logs -f job/bootstrap-models -c model-download
|
||||
```
|
||||
|
||||
### Restart deployment
|
||||
```bash
|
||||
kubectl -n llm rollout restart deployment/ollama
|
||||
```
|
||||
|
||||
## Storage
|
||||
|
||||
- **PVC:** `ollama-models-cache`, 115Gi, Longhorn StorageClass
|
||||
- **Mount:** `/root/.ollama/models` (Ollama model cache)
|
||||
- **Lifecycle:** RWO (Read-Write-Once), tied to talos-cp-1
|
||||
|
||||
### Resize PVC
|
||||
⚠️ PVCs can only expand, not shrink. Edit values.yaml and redeploy:
|
||||
|
||||
```yaml
|
||||
pvc:
|
||||
size: 120Gi # increase only
|
||||
```
|
||||
|
||||
```bash
|
||||
vsource .env && helmfile -f helmfile.yaml.gotmpl -l name=ollama apply
|
||||
```
|
||||
|
||||
## Networking
|
||||
|
||||
### NetworkPolicy
|
||||
- Default-deny ingress on Ollama pods
|
||||
- Allow from pods labeled `app.kubernetes.io/name: llm-worker` (port 11434)
|
||||
- Allow from pods labeled `app.kubernetes.io/role: llm-debug` (port 11434)
|
||||
|
||||
View policy:
|
||||
```bash
|
||||
kubectl -n llm get networkpolicy ollama
|
||||
```
|
||||
|
||||
Test access from external pod (should fail):
|
||||
```bash
|
||||
kubectl run test --rm -it --image=curlimages/curl -- \
|
||||
curl http://ollama.llm.svc.cluster.local:11434/
|
||||
# Connection timeout (correct)
|
||||
```
|
||||
|
||||
Test access from debug pod (should succeed):
|
||||
```bash
|
||||
kubectl -n llm logs job/bootstrap-models # verify bootstrap completed
|
||||
# Then run debug pod as shown above
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
### Helm values (`k8s/llm/charts/ollama/values.yaml`)
|
||||
|
||||
```yaml
|
||||
resources:
|
||||
requests:
|
||||
cpu: 8
|
||||
memory: 50Gi
|
||||
limits:
|
||||
cpu: 16
|
||||
memory: 70Gi
|
||||
|
||||
env:
|
||||
OLLAMA_MAX_LOADED_MODELS: "1"
|
||||
OLLAMA_NUM_PARALLEL: "1"
|
||||
OLLAMA_MAX_QUEUE: "32"
|
||||
OLLAMA_KEEP_ALIVE: "-1"
|
||||
OLLAMA_HOST: "0.0.0.0:11434"
|
||||
|
||||
preloadJob:
|
||||
enabled: true
|
||||
hotModels:
|
||||
- deepseek-r1:70b
|
||||
```
|
||||
|
||||
### Environment variables
|
||||
|
||||
| Variable | Value | Purpose |
|
||||
|----------|-------|---------|
|
||||
| `OLLAMA_MODELS` | `/root/.ollama/models` | Model cache dir |
|
||||
| `OLLAMA_MAX_LOADED_MODELS` | `1` | Max concurrent models in RAM |
|
||||
| `OLLAMA_NUM_PARALLEL` | `1` | Parallel request threads |
|
||||
| `OLLAMA_MAX_QUEUE` | `32` | Request queue depth |
|
||||
| `OLLAMA_KEEP_ALIVE` | `-1` | Keep model resident (never unload) |
|
||||
| `OLLAMA_HOST` | `0.0.0.0:11434` | Bind address |
|
||||
|
||||
Tune `OLLAMA_NUM_PARALLEL` based on CPU cores. Current: 1 (conservative, CPU bottleneck).
|
||||
|
||||
## Model Management
|
||||
|
||||
### Current model
|
||||
- **Name:** `deepseek-r1:70b`
|
||||
- **Size:** ~42GB
|
||||
- **Quantization:** Default Ollama quant
|
||||
- **Status:** Downloaded during pod init via bootstrap job
|
||||
|
||||
### Change model
|
||||
|
||||
1. Edit `values.yaml`:
|
||||
```yaml
|
||||
preloadJob:
|
||||
hotModels:
|
||||
- deepseek-r1:32b # or any available model
|
||||
```
|
||||
|
||||
2. Redeploy:
|
||||
```bash
|
||||
kubectl -n llm delete job bootstrap-models --ignore-not-found
|
||||
vsource .env && helmfile -f helmfile.yaml.gotmpl -l name=ollama apply
|
||||
```
|
||||
|
||||
3. Monitor:
|
||||
```bash
|
||||
kubectl -n llm logs -f job/bootstrap-models -c model-download
|
||||
```
|
||||
|
||||
### Available models
|
||||
Ollama registry: https://ollama.com/library
|
||||
|
||||
Examples:
|
||||
- `deepseek-r1:70b` (reasoning, 42GB)
|
||||
- `deepseek-r1:32b` (faster, 20GB)
|
||||
- `llama3.1:70b` (general, 41GB)
|
||||
- `mistral:large` (26GB)
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Pod stuck in `ContainerCreating`
|
||||
```bash
|
||||
kubectl -n llm describe pod -l app.kubernetes.io/name=ollama
|
||||
# Check Events section for PVC/image pull issues
|
||||
```
|
||||
|
||||
### Bootstrap job failing
|
||||
```bash
|
||||
kubectl -n llm logs job/bootstrap-models -c model-download --tail=50
|
||||
# Common: model not found in registry, disk full, network timeout
|
||||
```
|
||||
|
||||
### Model pull timeout
|
||||
```bash
|
||||
# Increase pod timeout (edit deployment directly)
|
||||
kubectl -n llm edit deployment ollama
|
||||
# Change readinessProbe.initialDelaySeconds, livenessProbe.periodSeconds
|
||||
```
|
||||
|
||||
### Out of memory
|
||||
Model size exceeds limit. Reduce `memory.limits` or choose smaller model.
|
||||
|
||||
```bash
|
||||
kubectl top pod -n llm # check actual usage
|
||||
```
|
||||
|
||||
### Cannot connect from other pods
|
||||
Verify NetworkPolicy:
|
||||
```bash
|
||||
kubectl -n llm get networkpolicy
|
||||
kubectl -n llm describe networkpolicy ollama
|
||||
# Add pod label: app.kubernetes.io/name: llm-worker or app.kubernetes.io/role: llm-debug
|
||||
```
|
||||
|
||||
## Secrets
|
||||
|
||||
Ollama pod receives MinIO credentials via Secret `ollama-minio` (created by helmfile presync):
|
||||
|
||||
```bash
|
||||
kubectl -n llm get secret ollama-minio -o jsonpath='{.data}' | jq
|
||||
```
|
||||
|
||||
Keys: `endpoint`, `bucket`, `access_key`, `secret_key`
|
||||
|
||||
Used by bootstrap job to upload model blobs to MinIO (future: auto-backup).
|
||||
|
||||
## Metrics & Observability
|
||||
|
||||
### Prometheus scrape (if enabled)
|
||||
ServiceMonitor: Not yet configured (see `k8s/monitoring/dashboards/services/`)
|
||||
|
||||
Metrics to add:
|
||||
- `ollama_requests_total` (counter)
|
||||
- `ollama_request_duration_seconds` (histogram)
|
||||
- `ollama_loaded_models` (gauge)
|
||||
|
||||
### Logs
|
||||
Pod logs via kubectl. No log aggregation to Loki yet.
|
||||
|
||||
```bash
|
||||
kubectl -n llm logs deployment/ollama -f --timestamps
|
||||
```
|
||||
|
||||
## Cleanup
|
||||
|
||||
### Delete Ollama completely
|
||||
```bash
|
||||
vsource .env && helmfile -f helmfile.yaml.gotmpl -l name=ollama destroy
|
||||
# Keeps PVC (data safety). To delete: kubectl -n llm delete pvc ollama-models-cache
|
||||
```
|
||||
|
||||
### Delete just the model cache (keep deployment)
|
||||
```bash
|
||||
kubectl -n llm delete pvc ollama-models-cache
|
||||
# Recreate: kubectl -n llm patch deployment ollama -p '{"spec":{"template":{"metadata":{"annotations":{"restart":"now"}}}}}'
|
||||
```
|
||||
|
||||
## See Also
|
||||
|
||||
- Helmfile: `helmfile.yaml.gotmpl` (llm release block)
|
||||
- Chart: `k8s/llm/charts/ollama/`
|
||||
- Namespace: `llm`
|
||||
- Bootstrap: `k8s/llm/bootstrap-models-job.yaml` (manual preload fallback)
|
||||
@@ -1,6 +0,0 @@
|
||||
apiVersion: v2
|
||||
name: ollama
|
||||
description: CPU-only Ollama LLM server with MinIO model registry
|
||||
type: application
|
||||
version: 0.1.0
|
||||
appVersion: "latest"
|
||||
@@ -1,126 +0,0 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: ollama
|
||||
namespace: llm
|
||||
labels:
|
||||
app.kubernetes.io/name: ollama
|
||||
app.kubernetes.io/part-of: llm
|
||||
spec:
|
||||
replicas: {{ .Values.replicaCount }}
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: ollama
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: ollama
|
||||
app.kubernetes.io/part-of: llm
|
||||
spec:
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: topology.kubernetes.io/zone
|
||||
operator: In
|
||||
values:
|
||||
- {{ .Values.nodeAffinity.zone }}
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Equal
|
||||
value: ""
|
||||
effect: NoSchedule
|
||||
|
||||
initContainers:
|
||||
- name: preload-model
|
||||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -e
|
||||
echo "Starting Ollama server for model preload..."
|
||||
ollama serve &
|
||||
OLLAMA_PID=$!
|
||||
sleep 10
|
||||
{{- range .Values.preloadJob.hotModels }}
|
||||
echo "Preloading {{ . }}..."
|
||||
if ollama ls | grep -q "{{ . }}"; then
|
||||
echo "✓ {{ . }} already cached"
|
||||
else
|
||||
ollama pull {{ . }}
|
||||
fi
|
||||
{{- end }}
|
||||
echo "Model preload complete"
|
||||
kill $OLLAMA_PID || true
|
||||
wait $OLLAMA_PID 2>/dev/null || true
|
||||
volumeMounts:
|
||||
- name: models-cache
|
||||
mountPath: /root/.ollama/models
|
||||
env:
|
||||
- name: OLLAMA_HOST
|
||||
value: "127.0.0.1:11434"
|
||||
|
||||
containers:
|
||||
- name: ollama
|
||||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
ports:
|
||||
- containerPort: 11434
|
||||
name: http
|
||||
env:
|
||||
{{- range $key, $value := .Values.env }}
|
||||
- name: {{ $key }}
|
||||
value: "{{ $value }}"
|
||||
{{- end }}
|
||||
- name: OLLAMA_MODELS_MINIO_ENDPOINT
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: ollama-minio
|
||||
key: endpoint
|
||||
- name: OLLAMA_MODELS_MINIO_BUCKET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: ollama-minio
|
||||
key: bucket
|
||||
- name: OLLAMA_MODELS_MINIO_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: ollama-minio
|
||||
key: access_key
|
||||
- name: OLLAMA_MODELS_MINIO_SECRET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: ollama-minio
|
||||
key: secret_key
|
||||
resources:
|
||||
requests:
|
||||
cpu: {{ .Values.resources.requests.cpu }}
|
||||
memory: {{ .Values.resources.requests.memory }}
|
||||
limits:
|
||||
cpu: {{ .Values.resources.limits.cpu }}
|
||||
memory: {{ .Values.resources.limits.memory }}
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: 11434
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: 11434
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 5
|
||||
volumeMounts:
|
||||
- name: models-cache
|
||||
mountPath: /root/.ollama/models
|
||||
|
||||
volumes:
|
||||
- name: models-cache
|
||||
persistentVolumeClaim:
|
||||
claimName: ollama-models-cache
|
||||
@@ -1,28 +0,0 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: ollama-access
|
||||
namespace: llm
|
||||
labels:
|
||||
app.kubernetes.io/name: ollama
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: ollama
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: llm-worker
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 11434
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/role: llm-debug
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 11434
|
||||
@@ -1,92 +0,0 @@
|
||||
{{- if .Values.preloadJob.enabled }}
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: ollama-preload
|
||||
namespace: llm
|
||||
labels:
|
||||
app.kubernetes.io/name: ollama-preload
|
||||
spec:
|
||||
backoffLimit: 3
|
||||
template:
|
||||
spec:
|
||||
serviceAccountName: default
|
||||
restartPolicy: Never
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: topology.kubernetes.io/zone
|
||||
operator: In
|
||||
values:
|
||||
- az-a
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Equal
|
||||
effect: NoSchedule
|
||||
initContainers:
|
||||
- name: model-cache-init
|
||||
image: ollama/ollama:latest
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -e
|
||||
echo "Starting Ollama server to cache models..."
|
||||
ollama serve &
|
||||
OLLAMA_PID=$!
|
||||
sleep 10
|
||||
echo "Caching hot-tier models..."
|
||||
{{- range .Values.preloadJob.hotModels }}
|
||||
echo "Checking if {{ . }} is cached..."
|
||||
if ollama ls | grep -q "{{ . }}"; then
|
||||
echo "✓ {{ . }} already cached, skipping"
|
||||
else
|
||||
echo "Pulling {{ . }}..."
|
||||
ollama pull {{ . }}
|
||||
fi
|
||||
{{- end }}
|
||||
echo "Model cache initialization complete"
|
||||
kill $OLLAMA_PID || true
|
||||
wait $OLLAMA_PID 2>/dev/null || true
|
||||
volumeMounts:
|
||||
- name: models
|
||||
mountPath: /root/.ollama
|
||||
env:
|
||||
- name: OLLAMA_HOST
|
||||
value: "127.0.0.1:11434"
|
||||
|
||||
containers:
|
||||
- name: cache-populate
|
||||
image: curlimages/curl:latest
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -e
|
||||
echo "Waiting for Ollama pod to be ready..."
|
||||
until curl -f http://ollama.llm.svc.cluster.local:11434/api/tags 2>/dev/null; do
|
||||
echo "Ollama not ready, waiting..."
|
||||
sleep 5
|
||||
done
|
||||
echo "Ollama is ready, populating local cache..."
|
||||
{{- range .Values.preloadJob.hotModels }}
|
||||
echo "Checking if {{ . }} is already cached..."
|
||||
if curl -s http://ollama.llm.svc.cluster.local:11434/api/tags | grep -q "{{ . }}"; then
|
||||
echo "✓ {{ . }} already cached, skipping"
|
||||
else
|
||||
echo "Caching {{ . }} locally..."
|
||||
curl -X POST http://ollama.llm.svc.cluster.local:11434/api/pull \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"name":"{{ . }}","stream":false}'
|
||||
fi
|
||||
{{- end }}
|
||||
echo "Local cache population complete"
|
||||
|
||||
volumes:
|
||||
- name: models
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
@@ -1,14 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: ollama-models-cache
|
||||
namespace: llm
|
||||
labels:
|
||||
app.kubernetes.io/name: ollama
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: {{ .Values.pvc.storageClassName }}
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.pvc.size }}
|
||||
@@ -1,16 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: ollama
|
||||
namespace: llm
|
||||
labels:
|
||||
app.kubernetes.io/name: ollama
|
||||
spec:
|
||||
type: {{ .Values.service.type }}
|
||||
ports:
|
||||
- port: {{ .Values.service.port }}
|
||||
targetPort: http
|
||||
protocol: TCP
|
||||
name: http
|
||||
selector:
|
||||
app.kubernetes.io/name: ollama
|
||||
@@ -1,12 +0,0 @@
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
name: longhorn-llm
|
||||
labels:
|
||||
app.kubernetes.io/name: ollama
|
||||
provisioner: driver.longhorn.io
|
||||
parameters:
|
||||
numberOfReplicas: "1"
|
||||
staleReplicaTimeout: "2880"
|
||||
reclaimPolicy: Retain
|
||||
allowVolumeExpansion: true
|
||||
@@ -1,40 +0,0 @@
|
||||
replicaCount: 1
|
||||
|
||||
image:
|
||||
repository: ollama/ollama
|
||||
pullPolicy: IfNotPresent
|
||||
tag: "latest"
|
||||
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 11434
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 8
|
||||
memory: 60Gi
|
||||
limits:
|
||||
cpu: 16
|
||||
memory: 100Gi
|
||||
|
||||
pvc:
|
||||
enabled: true
|
||||
size: 115Gi
|
||||
storageClassName: longhorn-llm
|
||||
|
||||
nodeAffinity:
|
||||
zone: az-a
|
||||
|
||||
env:
|
||||
OLLAMA_MODELS: /root/.ollama/models
|
||||
OLLAMA_MAX_LOADED_MODELS: "2"
|
||||
OLLAMA_NUM_PARALLEL: "2"
|
||||
OLLAMA_MAX_QUEUE: "64"
|
||||
OLLAMA_KEEP_ALIVE: "-1"
|
||||
OLLAMA_HOST: "0.0.0.0:11434"
|
||||
|
||||
preloadJob:
|
||||
enabled: false
|
||||
hotModels:
|
||||
- ornith:35b
|
||||
- deepseek-r1:70b
|
||||
@@ -1,6 +0,0 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: llm
|
||||
resources:
|
||||
- scripts/
|
||||
# Helm charts deployed via ArgoCD Helm source
|
||||
@@ -1,5 +0,0 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
resources: []
|
||||
# Shell scripts, not K8s manifests
|
||||
@@ -1,71 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# setup-minio-bucket.sh
|
||||
# Creates MinIO bucket and Kubernetes secrets for Ollama LLM server
|
||||
# Runs as helmfile presync hook; all commands are idempotent
|
||||
# Dependencies: kubectl, access to minio-az-a pod in storage namespace
|
||||
# Environment: MINIO_ROOT_USER, MINIO_ROOT_PASSWORD (from Vault), AUTHENTIK_OLLAMA_CLIENT_ID, AUTHENTIK_OLLAMA_CLIENT_SECRET
|
||||
|
||||
set -e
|
||||
|
||||
echo "=== Step 1: Create and label llm namespace ==="
|
||||
kubectl create namespace llm --dry-run=client -o yaml | kubectl apply -f -
|
||||
kubectl label namespace llm \
|
||||
pod-security.kubernetes.io/enforce=baseline \
|
||||
pod-security.kubernetes.io/enforce-version=latest \
|
||||
--overwrite
|
||||
|
||||
echo "✓ llm namespace created/labeled"
|
||||
|
||||
echo ""
|
||||
echo "=== Step 2: Create MinIO bucket riotpiao-models ==="
|
||||
|
||||
# Configure mc host inside MinIO pod
|
||||
kubectl -n storage exec deployment/minio-az-a -- \
|
||||
mc config host add local http://localhost:9000 \
|
||||
"${MINIO_ROOT_USER}" "${MINIO_ROOT_PASSWORD}"
|
||||
|
||||
echo "✓ mc host configured"
|
||||
|
||||
# Create bucket (idempotent)
|
||||
kubectl -n storage exec deployment/minio-az-a -- \
|
||||
mc mb --ignore-existing local/riotpiao-models
|
||||
|
||||
echo "✓ MinIO bucket riotpiao-models created (or already exists)"
|
||||
|
||||
# Enable versioning for model rollback safety
|
||||
kubectl -n storage exec deployment/minio-az-a -- \
|
||||
mc version enable local/riotpiao-models
|
||||
|
||||
echo "✓ Versioning enabled on riotpiao-models bucket"
|
||||
|
||||
echo ""
|
||||
echo "=== Step 3: Create ollama-minio Secret (MinIO credentials) ==="
|
||||
|
||||
kubectl create secret generic ollama-minio -n llm \
|
||||
--from-literal=endpoint="http://minio-az-a.storage:9000" \
|
||||
--from-literal=bucket="riotpiao-models" \
|
||||
--from-literal=access_key="${MINIO_ROOT_USER}" \
|
||||
--from-literal=secret_key="${MINIO_ROOT_PASSWORD}" \
|
||||
--dry-run=client -o yaml | kubectl apply -f -
|
||||
|
||||
echo "✓ Secret ollama-minio created/updated"
|
||||
|
||||
echo ""
|
||||
echo "=== Step 4: Create ollama-oidc Secret (Authentik credentials) ==="
|
||||
|
||||
kubectl create secret generic ollama-oidc -n llm \
|
||||
--from-literal=client_id="${AUTHENTIK_OLLAMA_CLIENT_ID}" \
|
||||
--from-literal=client_secret="${AUTHENTIK_OLLAMA_CLIENT_SECRET}" \
|
||||
--dry-run=client -o yaml | kubectl apply -f -
|
||||
|
||||
echo "✓ Secret ollama-oidc created/updated"
|
||||
|
||||
echo ""
|
||||
echo "=== Verification ==="
|
||||
echo ""
|
||||
echo "Run these commands to verify:"
|
||||
echo " kubectl -n llm get secret ollama-minio ollama-oidc"
|
||||
echo " kubectl -n storage exec deployment/minio-az-a -- mc ls local/riotpiao-models"
|
||||
echo ""
|
||||
echo "Setup complete!"
|
||||
@@ -1,54 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# portainer/bootstrap.sh
|
||||
# Deploys Portainer CE into the dashboard namespace.
|
||||
# No credentials needed — Portainer prompts you to create an admin account
|
||||
# on first browser visit.
|
||||
#
|
||||
# Prerequisites:
|
||||
# - kubectl configured (KUBECONFIG pointing to cluster-config/kubeconfig)
|
||||
# - helm >= 3.x
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
|
||||
KUBECONFIG="${KUBECONFIG:-${REPO_ROOT}/cluster-config/kubeconfig}"
|
||||
export KUBECONFIG
|
||||
|
||||
# ── Namespace ─────────────────────────────────────────────────────────────────
|
||||
echo "==> Creating dashboard namespace..."
|
||||
kubectl create namespace dashboard --dry-run=client -o yaml | kubectl apply -f -
|
||||
kubectl label namespace dashboard \
|
||||
pod-security.kubernetes.io/enforce=privileged \
|
||||
pod-security.kubernetes.io/enforce-version=latest \
|
||||
--overwrite
|
||||
|
||||
# ── Helm repo ─────────────────────────────────────────────────────────────────
|
||||
echo "==> Adding Portainer Helm repo..."
|
||||
helm repo add portainer https://portainer.github.io/k8s/
|
||||
helm repo update portainer
|
||||
|
||||
# ── Portainer ─────────────────────────────────────────────────────────────────
|
||||
echo "==> Installing Portainer..."
|
||||
helm upgrade --install portainer portainer/portainer \
|
||||
--namespace dashboard \
|
||||
--values "${SCRIPT_DIR}/portainer-values.yaml" \
|
||||
--wait \
|
||||
--timeout 5m
|
||||
|
||||
echo "==> Waiting for Portainer Deployment to be ready..."
|
||||
kubectl rollout status deployment/portainer -n dashboard --timeout=120s
|
||||
|
||||
# ── Done ──────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "==> Portainer is up."
|
||||
echo ""
|
||||
echo "Access Portainer UI:"
|
||||
echo " make pf-portainer"
|
||||
echo " http://localhost:9000"
|
||||
echo ""
|
||||
echo "First-time setup: Portainer will prompt you to create an admin account."
|
||||
echo "Choose 'Manage the local Kubernetes environment' when asked."
|
||||
echo ""
|
||||
echo "Node failure resilience tip:"
|
||||
echo " For faster PVC failover on hard node failure, enable in Longhorn UI → Settings:"
|
||||
echo " nodeDownPodDeletionPolicy = delete-deployment-pod"
|
||||
@@ -1,4 +0,0 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: shadowsocks
|
||||
resources: []
|
||||
@@ -1,159 +0,0 @@
|
||||
# k8s/shadowsocks/shadowsocks.yaml
|
||||
# Personal Shadowsocks proxy (for Shadowrocket/other SS clients) — an
|
||||
# alternative tunnel to the WireGuard setup in cluster-config/, useful when
|
||||
# a network blocks/throttles WireGuard but not generic TLS-looking traffic.
|
||||
#
|
||||
# Everything that varies between deployments (port, password, method) lives
|
||||
# in the shadowsocks-config Secret below — the Deployment/Service never
|
||||
# hardcode a value, so re-pointing this at a new port or rotating the
|
||||
# password is a Secret edit + rollout restart, no YAML edit.
|
||||
#
|
||||
# Prerequisites:
|
||||
# talos put cluster/SHADOWSOCKS_PASSWORD SHADOWSOCKS_PASSWORD="$(openssl rand -base64 24)"
|
||||
# talos put cluster/SHADOWSOCKS_PORT SHADOWSOCKS_PORT="8388"
|
||||
#
|
||||
# Apply:
|
||||
# kubectl create namespace vpn --dry-run=client -o yaml | kubectl apply -f -
|
||||
# kubectl -n vpn create secret generic shadowsocks-config \
|
||||
# --from-literal=SERVER_PORT="$(talos get cluster/SHADOWSOCKS_PORT --key SHADOWSOCKS_PORT)" \
|
||||
# --from-literal=PASSWORD="$(talos get cluster/SHADOWSOCKS_PASSWORD --key SHADOWSOCKS_PASSWORD)" \
|
||||
# --from-literal=METHOD="aes-256-gcm" \
|
||||
# --from-literal=TIMEOUT="300"
|
||||
# kubectl apply -f k8s/shadowsocks/shadowsocks.yaml
|
||||
#
|
||||
# Rotate password (or change port) later:
|
||||
# kubectl -n vpn delete secret shadowsocks-config && <recreate with new values>
|
||||
# kubectl -n vpn rollout restart deploy/shadowsocks
|
||||
#
|
||||
# Client config: SERVER_PORT/METHOD/PASSWORD above feed directly into the
|
||||
# Shadowrocket/SS client's server, method, and password fields. SERVER_ADDR
|
||||
# for the client is the LB IP below (192.168.1.166), or your router's WAN
|
||||
# address/DDNS hostname (riotpiao.duckdns.org) with port-forwarding to it —
|
||||
# same pattern as the wg1 WireGuard peer in cluster-config/phone_config.conf.
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: vpn
|
||||
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: shadowsocks
|
||||
namespace: vpn
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: shadowsocks
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: shadowsocks
|
||||
spec:
|
||||
containers:
|
||||
- name: shadowsocks
|
||||
image: shadowsocks/shadowsocks-libev:latest
|
||||
env:
|
||||
- name: SERVER_ADDR
|
||||
value: "0.0.0.0"
|
||||
- name: SERVER_PORT
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: shadowsocks-config
|
||||
key: SERVER_PORT
|
||||
- name: PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: shadowsocks-config
|
||||
key: PASSWORD
|
||||
- name: METHOD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: shadowsocks-config
|
||||
key: METHOD
|
||||
- name: TIMEOUT
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: shadowsocks-config
|
||||
key: TIMEOUT
|
||||
# containerPort is informational only (no portRange support for
|
||||
# env-driven SERVER_PORT) — the Service below is what actually
|
||||
# routes traffic, matched on the same Secret key via downward API
|
||||
# isn't available for Service ports, so targetPort uses the literal
|
||||
# port name instead; see Service ports comment.
|
||||
ports:
|
||||
- containerPort: 8388
|
||||
protocol: TCP
|
||||
- containerPort: 8388
|
||||
protocol: UDP
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 128Mi
|
||||
|
||||
---
|
||||
# LoadBalancer via Cilium LB-IPAM (see k8s/cilium/lb-ipam-pool.yaml) — pinned
|
||||
# to .166 so router port-forwarding and the DDNS hostname stay stable across
|
||||
# pod/service recreates, same pattern forgejo uses at .165.
|
||||
#
|
||||
# NOTE: SERVER_PORT in the Secret must match port/targetPort/nodePort here.
|
||||
# If you change the port, update both the Secret and this Service together.
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: shadowsocks
|
||||
namespace: vpn
|
||||
annotations:
|
||||
io.cilium/lb-ipam-ips: "192.168.1.166"
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
selector:
|
||||
app: shadowsocks
|
||||
ports:
|
||||
- name: tcp
|
||||
protocol: TCP
|
||||
port: 8388
|
||||
targetPort: 8388
|
||||
- name: udp
|
||||
protocol: UDP
|
||||
port: 8388
|
||||
targetPort: 8388
|
||||
|
||||
---
|
||||
# Restrict egress like the forgejo-runner pattern (k8s/forge/runner.yaml) —
|
||||
# a proxy server is, by design, an open relay to the internet for whoever
|
||||
# holds the password; LAN/pod-network egress is blocked so a compromised
|
||||
# password can't be used to pivot into the cluster or LAN. CoreDNS is
|
||||
# explicitly allowed — shadowsocks-libev resolves client-requested hostnames
|
||||
# itself, so blanket-blocking the service subnet would break that.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: shadowsocks-egress
|
||||
namespace: vpn
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: shadowsocks
|
||||
policyTypes: [Egress]
|
||||
egress:
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 53
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 0.0.0.0/0
|
||||
except:
|
||||
- 192.168.1.0/24
|
||||
- 10.244.0.0/16
|
||||
- 10.96.0.0/12
|
||||
@@ -1,25 +0,0 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: strimzi-operator
|
||||
namespace: cicd
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "0"
|
||||
spec:
|
||||
project: kmsvc
|
||||
source:
|
||||
repoURL: https://strimzi.io/charts/
|
||||
chart: strimzi-kafka-operator
|
||||
targetRevision: 0.46.0
|
||||
helm:
|
||||
values: |
|
||||
watchNamespaces: ["sqs"]
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: sqs
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
@@ -1,31 +0,0 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: kafka-cluster
|
||||
namespace: cicd
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "1"
|
||||
spec:
|
||||
project: kmsvc
|
||||
source:
|
||||
repoURL: https://forgejo.riotpiao.com/rock/kafaka-management-service.git
|
||||
targetRevision: main
|
||||
path: k8s/charts/kafka-cluster
|
||||
helm:
|
||||
values: |
|
||||
namespace: sqs
|
||||
nodePool:
|
||||
replicas: 3
|
||||
storage:
|
||||
class: longhorn
|
||||
sizeGi: 50
|
||||
resources:
|
||||
memory: 5Gi
|
||||
cpu: "2"
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: sqs
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
@@ -1,43 +0,0 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: kmsvc-redis
|
||||
namespace: cicd
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "1"
|
||||
spec:
|
||||
project: kmsvc
|
||||
source:
|
||||
repoURL: https://charts.bitnami.com/bitnami
|
||||
chart: redis
|
||||
targetRevision: 20.6.0
|
||||
helm:
|
||||
values: |
|
||||
architecture: standalone
|
||||
# docker.io/bitnami stopped publishing version-pinned tags; bitnamilegacy
|
||||
# mirrors them for free. allowInsecureImages silences the chart's
|
||||
# container-image allowlist check, which doesn't know about that mirror.
|
||||
global:
|
||||
security:
|
||||
allowInsecureImages: true
|
||||
image:
|
||||
repository: bitnamilegacy/redis
|
||||
auth:
|
||||
enabled: false
|
||||
master:
|
||||
persistence:
|
||||
enabled: true
|
||||
storageClass: longhorn
|
||||
size: 2Gi
|
||||
resources:
|
||||
limits:
|
||||
memory: 1Gi
|
||||
requests:
|
||||
memory: 1Gi
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: sqs
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
@@ -1,30 +0,0 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: queue-crd
|
||||
namespace: cicd
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "2"
|
||||
spec:
|
||||
project: kmsvc
|
||||
source:
|
||||
repoURL: https://forgejo.riotpiao.com/rock/kafaka-management-service.git
|
||||
targetRevision: main
|
||||
path: k8s/charts/queue-crd
|
||||
helm:
|
||||
values: |
|
||||
namespace: sqs
|
||||
kafkaBrokers: "kmsvc-kafka-bootstrap.sqs.svc.cluster.local:9092"
|
||||
redisAddr: "kmsvc-redis-master.sqs.svc.cluster.local:6379"
|
||||
image:
|
||||
repository: forgejo.riotpiao.com/rock/kafka-management-service-queue-operator
|
||||
# CI (.forgejo/workflows/release.yaml) writes the released git tag
|
||||
# here and pushes the commit -- ArgoCD picks it up on its next sync.
|
||||
tag: latest
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: sqs
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
@@ -1,37 +0,0 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: management-service
|
||||
namespace: cicd
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "2"
|
||||
spec:
|
||||
project: kmsvc
|
||||
source:
|
||||
repoURL: https://forgejo.riotpiao.com/rock/kafaka-management-service.git
|
||||
targetRevision: main
|
||||
path: k8s/charts/management-service
|
||||
helm:
|
||||
values: |
|
||||
namespace: sqs
|
||||
image:
|
||||
repository: forgejo.riotpiao.com/rock/kafka-management-service
|
||||
# CI (.forgejo/workflows/release.yaml) writes the released git tag
|
||||
# here and pushes the commit -- ArgoCD picks it up on its next sync.
|
||||
tag: latest
|
||||
env:
|
||||
kafkaBrokers: "kmsvc-kafka-bootstrap.sqs.svc.cluster.local:9092"
|
||||
redisAddr: "kmsvc-redis-master.sqs.svc.cluster.local:6379"
|
||||
authentikIssuerURL: "https://authentik.riotpiao.com/application/o/kafaka/"
|
||||
authentikAudience: "QI0gPtR99ar8VvhK8Tqox4SDkTKzbNU7lbgwBNSc"
|
||||
ingress:
|
||||
enabled: true
|
||||
host: kmsvc.riotpiao.com
|
||||
clusterIssuer: homelab-ca
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: sqs
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
@@ -1,7 +0,0 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: sqs
|
||||
resources:
|
||||
- 02-redis.yaml
|
||||
- 03-queue-crd.yaml
|
||||
- 04-management-service.yaml
|
||||
@@ -1,7 +0,0 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: sqs
|
||||
resources:
|
||||
- project.yaml
|
||||
- root.yaml
|
||||
- apps/
|
||||
@@ -1,23 +0,0 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: AppProject
|
||||
metadata:
|
||||
name: kmsvc
|
||||
namespace: cicd
|
||||
spec:
|
||||
description: Kafka Management Service (design.md) -- Strimzi/Kafka, Redis, queue-operator, message-plane server
|
||||
sourceRepos:
|
||||
- https://forgejo.riotpiao.com/rock/kafaka-management-service.git
|
||||
- https://strimzi.io/charts/
|
||||
- https://charts.bitnami.com/bitnami
|
||||
destinations:
|
||||
- namespace: sqs
|
||||
server: https://kubernetes.default.svc
|
||||
- namespace: cicd
|
||||
server: https://kubernetes.default.svc
|
||||
clusterResourceWhitelist:
|
||||
- group: "apiextensions.k8s.io"
|
||||
kind: CustomResourceDefinition
|
||||
- group: "rbac.authorization.k8s.io"
|
||||
kind: ClusterRole
|
||||
- group: "rbac.authorization.k8s.io"
|
||||
kind: ClusterRoleBinding
|
||||
@@ -1,22 +0,0 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: kmsvc-root
|
||||
namespace: cicd
|
||||
spec:
|
||||
project: kmsvc
|
||||
source:
|
||||
repoURL: https://forgejo.riotpiao.com/rock/kafaka-management-service.git
|
||||
targetRevision: main
|
||||
path: k8s/argocd/apps
|
||||
directory:
|
||||
recurse: false
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: cicd
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
@@ -1,29 +0,0 @@
|
||||
# design.md §7b: cluster-specific values for the homelab environment.
|
||||
# No secrets here — Authentik client secret etc. flow through the existing
|
||||
# Vault/talos-cli pattern, referenced at deploy time, not inlined.
|
||||
namespace: sqs
|
||||
|
||||
kafkaCluster:
|
||||
nodePool:
|
||||
replicas: 3
|
||||
storage:
|
||||
# Use unified longhorn StorageClass (3 replicas, Immediate binding)
|
||||
class: longhorn
|
||||
# Per-node headroom: with 3 nodes and existing PVCs, estimate ~100+ Gi total
|
||||
# available. Each node hosts one replica of all 3 broker volumes, so 3 *
|
||||
# sizeGi must fit. Monitor usage during Kafka deployment.
|
||||
sizeGi: 10
|
||||
resources:
|
||||
memory: 5Gi
|
||||
cpu: "2"
|
||||
|
||||
redis:
|
||||
storageClass: longhorn
|
||||
memoryLimit: 1Gi
|
||||
|
||||
managementService:
|
||||
ingress:
|
||||
host: kmsvc.riotpiao.com
|
||||
clusterIssuer: homelab-ca
|
||||
authentikIssuerURL: "https://authentik.riotpiao.com/application/o/kafaka/"
|
||||
authentikAudience: "QI0gPtR99ar8VvhK8Tqox4SDkTKzbNU7lbgwBNSc"
|
||||
@@ -1,5 +0,0 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: sqs
|
||||
resources: []
|
||||
# homelab.yaml is configuration, not a K8s manifest
|
||||
@@ -1,99 +0,0 @@
|
||||
environments:
|
||||
default:
|
||||
values:
|
||||
- environments/homelab.yaml
|
||||
homelab:
|
||||
values:
|
||||
- environments/homelab.yaml
|
||||
---
|
||||
helmDefaults:
|
||||
wait: true
|
||||
timeout: 600
|
||||
|
||||
repositories:
|
||||
- name: strimzi
|
||||
url: https://strimzi.io/charts/
|
||||
- name: bitnami
|
||||
url: https://charts.bitnami.com/bitnami
|
||||
|
||||
releases:
|
||||
- name: strimzi-operator
|
||||
namespace: {{ .Values.namespace }}
|
||||
chart: strimzi/strimzi-kafka-operator
|
||||
version: 0.46.0
|
||||
values:
|
||||
- watchNamespaces: ["{{ .Values.namespace }}"]
|
||||
|
||||
- name: kafka-cluster
|
||||
namespace: {{ .Values.namespace }}
|
||||
chart: charts/kafka-cluster
|
||||
needs:
|
||||
- {{ .Values.namespace }}/strimzi-operator
|
||||
values:
|
||||
- namespace: {{ .Values.namespace }}
|
||||
nodePool:
|
||||
replicas: {{ .Values.kafkaCluster.nodePool.replicas }}
|
||||
storage:
|
||||
class: {{ .Values.kafkaCluster.nodePool.storage.class }}
|
||||
sizeGi: {{ .Values.kafkaCluster.nodePool.storage.sizeGi }}
|
||||
resources:
|
||||
memory: {{ .Values.kafkaCluster.nodePool.resources.memory }}
|
||||
cpu: {{ .Values.kafkaCluster.nodePool.resources.cpu | quote }}
|
||||
|
||||
- name: kmsvc-redis
|
||||
namespace: {{ .Values.namespace }}
|
||||
chart: bitnami/redis
|
||||
version: 20.6.0
|
||||
values:
|
||||
- architecture: standalone
|
||||
# Bitnami stopped publishing version-pinned tags under docker.io/bitnami
|
||||
# (only `latest` remains there); bitnamilegacy/* mirrors the old
|
||||
# versioned tags for free, so pin there instead of floating on `latest`.
|
||||
# The chart's container-image allowlist check doesn't know about the
|
||||
# legacy mirror, hence allowInsecureImages.
|
||||
global:
|
||||
security:
|
||||
allowInsecureImages: true
|
||||
image:
|
||||
repository: bitnamilegacy/redis
|
||||
auth:
|
||||
enabled: false
|
||||
master:
|
||||
persistence:
|
||||
enabled: true
|
||||
storageClass: {{ .Values.redis.storageClass }}
|
||||
size: 2Gi
|
||||
resources:
|
||||
limits:
|
||||
memory: {{ .Values.redis.memoryLimit }}
|
||||
requests:
|
||||
memory: {{ .Values.redis.memoryLimit }}
|
||||
|
||||
- name: queue-crd
|
||||
namespace: {{ .Values.namespace }}
|
||||
chart: charts/queue-crd
|
||||
needs:
|
||||
- {{ .Values.namespace }}/kafka-cluster
|
||||
- {{ .Values.namespace }}/kmsvc-redis
|
||||
values:
|
||||
- namespace: {{ .Values.namespace }}
|
||||
kafkaBrokers: "kmsvc-kafka-bootstrap.{{ .Values.namespace }}.svc.cluster.local:9092"
|
||||
redisAddr: "kmsvc-redis-master.{{ .Values.namespace }}.svc.cluster.local:6379"
|
||||
|
||||
- name: management-service
|
||||
namespace: {{ .Values.namespace }}
|
||||
chart: charts/management-service
|
||||
needs:
|
||||
- {{ .Values.namespace }}/kafka-cluster
|
||||
- {{ .Values.namespace }}/kmsvc-redis
|
||||
values:
|
||||
- namespace: {{ .Values.namespace }}
|
||||
env:
|
||||
kafkaBrokers: "kmsvc-kafka-bootstrap.{{ .Values.namespace }}.svc.cluster.local:9092"
|
||||
redisAddr: "kmsvc-redis-master.{{ .Values.namespace }}.svc.cluster.local:6379"
|
||||
authentikIssuerURL: {{ .Values.managementService.authentikIssuerURL | quote }}
|
||||
authentikAudience: {{ .Values.managementService.authentikAudience | quote }}
|
||||
ingress:
|
||||
enabled: true
|
||||
host: {{ .Values.managementService.ingress.host | quote }}
|
||||
clusterIssuer: {{ .Values.managementService.ingress.clusterIssuer | quote }}
|
||||
@@ -1,8 +0,0 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: sqs
|
||||
resources:
|
||||
- argocd/
|
||||
- environments/
|
||||
- queues/
|
||||
# Helm charts deployed via ArgoCD Helm source
|
||||
@@ -1,5 +0,0 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
resources:
|
||||
- orders-fifo.yaml
|
||||
@@ -1,32 +0,0 @@
|
||||
apiVersion: kmsvc.io/v1
|
||||
kind: Queue
|
||||
metadata:
|
||||
name: orders-fifo
|
||||
namespace: sqs
|
||||
spec:
|
||||
fifoQueue: true
|
||||
visibilityTimeoutSeconds: 30
|
||||
messageRetentionPeriodSeconds: 345600
|
||||
maxReceiveCount: 5
|
||||
deadLetterTargetQueue: orders-fifo-dlq
|
||||
delaySeconds: 0
|
||||
partitionsPerShard: 6
|
||||
minShards: 1
|
||||
maxShards: 8
|
||||
shardSplitThresholdBytesPerSec: 5242880
|
||||
shardSplitCooldownSeconds: 300
|
||||
---
|
||||
apiVersion: kmsvc.io/v1
|
||||
kind: Queue
|
||||
metadata:
|
||||
name: orders-fifo-dlq
|
||||
namespace: sqs
|
||||
spec:
|
||||
fifoQueue: true
|
||||
isDLQ: true
|
||||
visibilityTimeoutSeconds: 30
|
||||
messageRetentionPeriodSeconds: 1209600
|
||||
maxReceiveCount: 5
|
||||
partitionsPerShard: 6
|
||||
minShards: 1
|
||||
maxShards: 1
|
||||
@@ -1,237 +0,0 @@
|
||||
# Phase 2: Namespace-Scoped Auto-Provisioning Testing Guide
|
||||
|
||||
## Overview
|
||||
|
||||
Phase 2 implements **namespace-scoped automatic TemporalWorker provisioning** (Kafka broker model).
|
||||
|
||||
One TemporalWorker per Temporal namespace processes ALL task queues in that namespace. When multiple Queues share the same `temporal.io/namespace` label, they trigger creation of a single TemporalWorker that handles all of them.
|
||||
|
||||
```
|
||||
Queues (labeled temporal.io/namespace: "production")
|
||||
├── orders-fifo
|
||||
├── payments
|
||||
└── notifications
|
||||
↓
|
||||
queue-operator creates 1 TemporalWorker (worker-production)
|
||||
↓
|
||||
TemporalWorker controller creates 1 Deployment
|
||||
↓
|
||||
Worker pod(s) connect to Temporal namespace "production"
|
||||
↓
|
||||
Process ALL task queues in that namespace (scale horizontally by replicas)
|
||||
```
|
||||
|
||||
## Implementation Changes
|
||||
|
||||
### 1. TemporalWorker CRD (`apis/kmsvc/v1/temporalworker_types.go`)
|
||||
- New Kubernetes resource type to manage namespace-scoped workers
|
||||
- Fields: Namespace (required), Image, Replicas, Resources, NodeSelector, Affinity, Tolerations
|
||||
- Status: Phase (Pending/Ready/Failed), Replicas, ReadyReplicas, Conditions
|
||||
- Model: 1 TemporalWorker per Temporal namespace (not per queue)
|
||||
|
||||
### 2. QueueReconciler Extension (`internal/operator/queue_controller.go`)
|
||||
- New method: `reconcileTemporalWorker()`
|
||||
- Logic: If Queue has `temporal.io/namespace` label, create TemporalWorker for that namespace
|
||||
- Idempotent: multiple queues with same namespace label create same TemporalWorker (no duplicates)
|
||||
|
||||
### 3. TemporalWorkerReconciler (`internal/operator/temporal_worker_controller.go`)
|
||||
- New controller watching TemporalWorker objects
|
||||
- Creates/updates Kubernetes Deployment with:
|
||||
- Pod spec: container image, env vars (TEMPORAL_FRONTEND_ADDRESS, TEMPORAL_TASK_QUEUE)
|
||||
- Replicas, resources, node selector, affinity, tolerations from TemporalWorker spec
|
||||
- Updates TemporalWorker status with deployment replica counts and phase
|
||||
|
||||
### 4. Operator Main (`cmd/queue-operator/main.go`)
|
||||
- Registers TemporalWorker CRD in scheme
|
||||
- Registers TemporalWorkerReconciler controller
|
||||
- Controller watches TemporalWorker objects; owns Deployment objects
|
||||
|
||||
## Testing Procedure
|
||||
|
||||
### Prerequisites
|
||||
- kmsvc queue-operator must be running (built and deployed)
|
||||
- Temporal cluster must be ready (temporal-frontend service available at `temporal-frontend.temporal.svc.cluster.local:7233`)
|
||||
- story-crater-backend Docker image must exist (used as default worker image)
|
||||
|
||||
### Step 1: Build and Deploy kmsvc Operator
|
||||
```bash
|
||||
cd /Users/rockliang/workplace/kmsvc-manage
|
||||
make build # builds queue-operator binary
|
||||
make docker-build # builds Docker image
|
||||
make deploy # deploys to cluster (requires Helm chart)
|
||||
```
|
||||
|
||||
Or manually:
|
||||
```bash
|
||||
cd /Users/rockliang/workplace/kmsvc-manage
|
||||
go build -o bin/queue-operator ./cmd/queue-operator
|
||||
kubectl apply -f k8s/queue-operator-rbac.yaml
|
||||
kubectl apply -f k8s/queue-operator-deployment.yaml
|
||||
```
|
||||
|
||||
### Step 2: Create Queues with Temporal Namespace Labels
|
||||
```bash
|
||||
kubectl apply -f /Users/rockliang/workplace/homelab/k8s/temporal/queues/example-queue.yaml
|
||||
```
|
||||
|
||||
Verify Queues are Ready:
|
||||
```bash
|
||||
kubectl get queue -n sqs -l temporal.io/namespace=production
|
||||
kubectl describe queue -n sqs story-crater-tasks
|
||||
```
|
||||
|
||||
Expected:
|
||||
```
|
||||
NAME FIFO PHASE AGE
|
||||
story-crater-tasks false Ready 5s
|
||||
story-crater-notifications false Ready 5s
|
||||
```
|
||||
|
||||
### Step 3: Verify TemporalWorker CRD Auto-Created (1 per namespace)
|
||||
```bash
|
||||
kubectl get temporalworker -n temporal
|
||||
kubectl describe temporalworker -n temporal worker-production
|
||||
```
|
||||
|
||||
Expected:
|
||||
```
|
||||
NAME PHASE READY DESIRED AGE
|
||||
worker-production Pending 0 1 5s
|
||||
```
|
||||
|
||||
Only ONE TemporalWorker for all queues in "production" namespace!
|
||||
|
||||
### Step 4: Verify Deployment Auto-Created
|
||||
```bash
|
||||
kubectl get deploy -n temporal -l app.kubernetes.io/managed-by=kmsvc-temporal-operator
|
||||
kubectl get pods -n temporal -l app.kubernetes.io/instance=worker-production
|
||||
```
|
||||
|
||||
Expected:
|
||||
```
|
||||
NAME READY UP-TO-DATE AVAILABLE AGE
|
||||
worker-production 1/1 1 1 10s
|
||||
|
||||
NAME READY STATUS RESTARTS AGE
|
||||
worker-production-5f8b4c... 1/1 Running 0 10s
|
||||
```
|
||||
|
||||
### Step 5: Verify Worker Connected to Temporal Namespace
|
||||
Check Temporal UI for namespace "production":
|
||||
```bash
|
||||
open https://temporal.riotpiao.com/namespaces/production/task-queues
|
||||
```
|
||||
|
||||
Look for all task queues with worker count > 0:
|
||||
- `story-crater-tasks`
|
||||
- `story-crater-notifications`
|
||||
- (worker processes all of them)
|
||||
|
||||
Or via CLI:
|
||||
```bash
|
||||
kubectl port-forward -n temporal svc/temporal-frontend 7233 &
|
||||
curl http://localhost:7233/api/v1/task-queues?namespace=production
|
||||
```
|
||||
|
||||
### Step 6: Verify TemporalWorker Status Updated
|
||||
```bash
|
||||
kubectl get temporalworker -n temporal
|
||||
kubectl describe temporalworker -n temporal worker-production
|
||||
```
|
||||
|
||||
Expected:
|
||||
```
|
||||
NAME PHASE READY DESIRED AGE
|
||||
worker-production Ready 1 1 15s
|
||||
|
||||
Status:
|
||||
Phase: Ready
|
||||
Ready Replicas: 1
|
||||
Replicas: 1
|
||||
```
|
||||
|
||||
### Step 7: Test Namespace-Level Scaling
|
||||
Create more queues in the same namespace:
|
||||
```yaml
|
||||
apiVersion: kmsvc.io/v1
|
||||
kind: Queue
|
||||
metadata:
|
||||
name: story-crater-llm-processing
|
||||
namespace: sqs
|
||||
labels:
|
||||
temporal.io/namespace: "production" # same namespace
|
||||
```
|
||||
|
||||
Verify: No new TemporalWorker created (same worker handles all 3 queues):
|
||||
```bash
|
||||
kubectl get temporalworker -n temporal # still just 1 worker-production
|
||||
kubectl get deploy -n temporal worker-production # same deployment
|
||||
```
|
||||
|
||||
Worker auto-discovers new task queue in namespace and processes it.
|
||||
|
||||
### Step 8: Test Cascading Deletion
|
||||
Delete a Queue; worker should remain (other queues still need it):
|
||||
```bash
|
||||
kubectl delete queue -n sqs story-crater-notifications
|
||||
```
|
||||
|
||||
Verify:
|
||||
```bash
|
||||
kubectl get temporalworker -n temporal # worker-production still exists
|
||||
kubectl get pods -n temporal worker-production # still running
|
||||
```
|
||||
|
||||
Delete all queues in namespace:
|
||||
```bash
|
||||
kubectl delete queue -n sqs -l temporal.io/namespace=production
|
||||
```
|
||||
|
||||
Verify: TemporalWorker now has no owner (not cascade-deleted; manual cleanup needed):
|
||||
```bash
|
||||
kubectl get temporalworker -n temporal # worker-production still there (manual cleanup)
|
||||
kubectl delete temporalworker -n temporal worker-production # cleanup manually
|
||||
```
|
||||
|
||||
## Debugging
|
||||
|
||||
### Queue stuck in Pending
|
||||
Check queue-operator logs:
|
||||
```bash
|
||||
kubectl logs -n sqs deploy/kmsvc-queue-operator -f
|
||||
kubectl logs -n sqs deploy/kmsvc-queue-operator --tail=50 | grep -i error
|
||||
```
|
||||
|
||||
### TemporalWorker not created
|
||||
- Verify Queue has the label: `kubectl get queue -o yaml | grep temporal.io`
|
||||
- Check queue-operator logs for "reconcileTemporalWorker" errors
|
||||
|
||||
### Deployment not created
|
||||
- Check TemporalWorker controller logs: `kubectl logs -n sqs deploy/kmsvc-queue-operator -f`
|
||||
- Verify TemporalWorker exists: `kubectl get temporalworker -n temporal`
|
||||
- Check Deployment errors: `kubectl describe deploy -n temporal worker-story-crater-tasks`
|
||||
|
||||
### Worker not showing in Temporal UI
|
||||
- Check pod logs: `kubectl logs -n temporal deploy/worker-story-crater-tasks`
|
||||
- Verify env vars: `kubectl set env pod -n temporal <pod-name> --list | grep TEMPORAL`
|
||||
- Test connectivity: `kubectl exec -n temporal <pod-name> -- nc -zv temporal-frontend.temporal.svc.cluster.local 7233`
|
||||
|
||||
## Next Steps
|
||||
|
||||
Once Phase 2 is working:
|
||||
1. **Phase 3 (Future):** Implement autoscaling based on queue depth metrics
|
||||
2. **Production Hardening:**
|
||||
- Add QueueRef validation (ensure Queue exists in sqs namespace)
|
||||
- Add image validation/defaults from ConfigMap
|
||||
- Add worker readiness probe configuration
|
||||
- Add graceful shutdown/drain behavior
|
||||
|
||||
## Files Modified/Created
|
||||
|
||||
| File | Change |
|
||||
|------|--------|
|
||||
| `apis/kmsvc/v1/temporalworker_types.go` | NEW: CRD type definitions |
|
||||
| `internal/operator/queue_controller.go` | MODIFIED: Added reconcileTemporalWorker() |
|
||||
| `internal/operator/temporal_worker_controller.go` | NEW: TemporalWorker → Deployment reconciler |
|
||||
| `cmd/queue-operator/main.go` | MODIFIED: Register TemporalWorker CRD + controller |
|
||||
| `k8s/temporal/queues/example-queue.yaml` | NEW: Example Queue with label |
|
||||
@@ -1,220 +0,0 @@
|
||||
# Temporal OAuth2-Proxy Setup (Authentik OIDC)
|
||||
|
||||
## Overview
|
||||
|
||||
Protects Temporal UI with Authentik OIDC authentication. Traffic flow:
|
||||
|
||||
```
|
||||
Browser → Ingress (TLS) → oauth2-proxy (OIDC check) → temporal-web (internal)
|
||||
↓
|
||||
Redirects to Authentik login
|
||||
↓
|
||||
JWT cookie issued
|
||||
↓
|
||||
Forwards to temporal-web
|
||||
```
|
||||
|
||||
## Prerequisites
|
||||
|
||||
✅ Authentik OIDC provider `temporal` already exists with:
|
||||
- Client ID: `temporal`
|
||||
- Client Secret: stored in Kubernetes secret `temporal-oidc` (key: `clientSecret`)
|
||||
- Redirect URI: `https://temporal.riotpiao.com/oauth2/callback`
|
||||
|
||||
## Secrets
|
||||
|
||||
The `temporal-oidc` secret must contain:
|
||||
|
||||
| Key | Value | Source |
|
||||
|-----|-------|--------|
|
||||
| `clientSecret` | OAuth2 client secret from Authentik | Authentik → Applications → temporal |
|
||||
| `cookieSecret` | Session encryption key (base64 32-byte) | Generate: `openssl rand -base64 32` |
|
||||
|
||||
### Check existing secret:
|
||||
|
||||
```bash
|
||||
kubectl get secret -n temporal temporal-oidc
|
||||
kubectl describe secret -n temporal temporal-oidc
|
||||
```
|
||||
|
||||
### If missing, create it:
|
||||
|
||||
```bash
|
||||
# Get client secret from Authentik UI
|
||||
# Applications → temporal → copy "Client Secret"
|
||||
CLIENT_SECRET="..."
|
||||
|
||||
# Generate cookie secret
|
||||
COOKIE_SECRET=$(openssl rand -base64 32)
|
||||
|
||||
# Create secret
|
||||
kubectl create secret generic temporal-oidc \
|
||||
-n temporal \
|
||||
--from-literal=clientSecret="${CLIENT_SECRET}" \
|
||||
--from-literal=cookieSecret="${COOKIE_SECRET}"
|
||||
```
|
||||
|
||||
## Deployment Steps
|
||||
|
||||
### Step 1: Apply OAuth2-Proxy Manifests
|
||||
```bash
|
||||
kubectl apply -f k8s/temporal/oauth2-proxy.yaml
|
||||
```
|
||||
|
||||
Verify:
|
||||
```bash
|
||||
kubectl get deploy -n temporal oauth2-proxy
|
||||
kubectl logs -n temporal deploy/oauth2-proxy
|
||||
```
|
||||
|
||||
Expected log:
|
||||
```
|
||||
[<timestamp>] [oauthproxy.go:...] Listening on 0.0.0.0:4180
|
||||
```
|
||||
|
||||
### Step 2: Apply OAuth2-Proxy Ingress
|
||||
```bash
|
||||
kubectl apply -f k8s/temporal/temporal-ingress-oauth2.yaml
|
||||
```
|
||||
|
||||
Verify:
|
||||
```bash
|
||||
kubectl get ingress -n temporal
|
||||
```
|
||||
|
||||
Expected:
|
||||
```
|
||||
NAME CLASS HOSTS ADDRESS PORTS AGE
|
||||
temporal nginx temporal.riotpiao.com ... 80, 443 10s
|
||||
```
|
||||
|
||||
### Step 3: Test Access
|
||||
|
||||
1. **Open Temporal UI (unauthenticated):**
|
||||
```bash
|
||||
open https://temporal.riotpiao.com
|
||||
```
|
||||
|
||||
Expected: Redirects to Authentik login page
|
||||
|
||||
2. **Login with Authentik credentials**
|
||||
- Username/email
|
||||
- Password
|
||||
- Should redirect back to `temporal.riotpiao.com` and display UI
|
||||
|
||||
3. **Verify auth:**
|
||||
```bash
|
||||
# Check for oauth2_proxy cookie
|
||||
curl -v https://temporal.riotpiao.com 2>&1 | grep -i cookie
|
||||
```
|
||||
|
||||
4. **Check oauth2-proxy logs:**
|
||||
```bash
|
||||
kubectl logs -n temporal deploy/oauth2-proxy -f
|
||||
```
|
||||
|
||||
Look for:
|
||||
```
|
||||
[timestamp] [auth_test.go:...] Authentication successful
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Redirect URI mismatch
|
||||
Error in oauth2-proxy logs:
|
||||
```
|
||||
redirect_uri_mismatch: The redirect_uri does not match the one registered in Authentik
|
||||
```
|
||||
|
||||
Fix:
|
||||
- Verify Authentik application (Applications → temporal) has redirect URI: `https://temporal.riotpiao.com/oauth2/callback`
|
||||
- Ensure HTTPS (not HTTP)
|
||||
|
||||
### Missing secret
|
||||
Error:
|
||||
```
|
||||
clientSecret: key not found in temporal-oidc secret
|
||||
```
|
||||
|
||||
Fix:
|
||||
```bash
|
||||
kubectl get secret -n temporal temporal-oidc -o yaml
|
||||
# If missing, create per "Secrets" section above
|
||||
```
|
||||
|
||||
### Cookie secret expiration
|
||||
OAuth2-Proxy won't start if `cookieSecret` is empty or invalid.
|
||||
|
||||
Fix:
|
||||
```bash
|
||||
COOKIE_SECRET=$(openssl rand -base64 32)
|
||||
kubectl patch secret temporal-oidc -n temporal \
|
||||
-p "{\"data\":{\"cookieSecret\":\"$(echo -n $COOKIE_SECRET | base64)\"}}}"
|
||||
kubectl rollout restart deploy/oauth2-proxy -n temporal
|
||||
```
|
||||
|
||||
### oauth2-proxy crashes with "connection refused"
|
||||
Error in logs:
|
||||
```
|
||||
upstream connect error or disconnect/reset before headers
|
||||
```
|
||||
|
||||
Likely cause: `temporal-web` service not accessible.
|
||||
|
||||
Check:
|
||||
```bash
|
||||
kubectl get svc -n temporal temporal-web
|
||||
kubectl exec -n temporal deploy/oauth2-proxy -- curl http://temporal-web:8080
|
||||
```
|
||||
|
||||
## File Structure
|
||||
|
||||
```
|
||||
k8s/temporal/
|
||||
├── oauth2-proxy.yaml # oauth2-proxy Deployment + Service + SA
|
||||
├── temporal-ingress-oauth2.yaml # Ingress routing to oauth2-proxy
|
||||
├── oauth2-proxy-values.yaml # Helm values (reference only)
|
||||
└── temporal-values.yaml # Modified: ingress.enabled=false
|
||||
```
|
||||
|
||||
## Next: Add to Helmfile
|
||||
|
||||
If integrating with helmfile.yaml.gotmpl:
|
||||
|
||||
```yaml
|
||||
releases:
|
||||
- name: temporal
|
||||
# ... existing config ...
|
||||
hooks:
|
||||
postSync:
|
||||
- events: ["success"]
|
||||
showlogs: true
|
||||
command: "sh"
|
||||
args:
|
||||
- -c
|
||||
- |
|
||||
kubectl apply -f k8s/temporal/oauth2-proxy.yaml
|
||||
kubectl apply -f k8s/temporal/temporal-ingress-oauth2.yaml
|
||||
```
|
||||
|
||||
Or add separate releases:
|
||||
|
||||
```yaml
|
||||
- name: oauth2-proxy-temporal
|
||||
namespace: temporal
|
||||
chart: oauth2-proxy/oauth2-proxy
|
||||
version: "6.x.x"
|
||||
values:
|
||||
- k8s/temporal/oauth2-proxy-values.yaml
|
||||
set:
|
||||
- name: config.clientSecret
|
||||
value: "{{ (env "TEMPORAL_OIDC_CLIENT_SECRET") }}"
|
||||
- name: config.cookieSecret
|
||||
value: "{{ (env "TEMPORAL_OIDC_COOKIE_SECRET") }}"
|
||||
```
|
||||
|
||||
Then add to `.env`:
|
||||
```bash
|
||||
TEMPORAL_OIDC_CLIENT_SECRET=<from Authentik>
|
||||
TEMPORAL_OIDC_COOKIE_SECRET=$(openssl rand -base64 32)
|
||||
```
|
||||
@@ -1,102 +0,0 @@
|
||||
# Elasticsearch 7.17.0 for Temporal visibility store
|
||||
# Deployed to worker nodes (not control plane to save CP resources for LLM work)
|
||||
# 2Gi heap + 4Gi memory limit for stable operation
|
||||
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: elasticsearch-config
|
||||
namespace: temporal
|
||||
data:
|
||||
elasticsearch.yml: |
|
||||
cluster.name: temporal-elasticsearch
|
||||
node.name: temporal-elasticsearch-0
|
||||
discovery.type: single-node
|
||||
network.host: 0.0.0.0
|
||||
http.host: 0.0.0.0
|
||||
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: temporal-elasticsearch
|
||||
namespace: temporal
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: temporal-elasticsearch
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: temporal-elasticsearch
|
||||
spec:
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: node-role.kubernetes.io/worker
|
||||
operator: Exists
|
||||
containers:
|
||||
- name: elasticsearch
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch:7.17.0
|
||||
env:
|
||||
- name: discovery.type
|
||||
value: single-node
|
||||
- name: "ES_JAVA_OPTS"
|
||||
value: "-Xms2g -Xmx2g"
|
||||
ports:
|
||||
- containerPort: 9200
|
||||
name: http
|
||||
- containerPort: 9300
|
||||
name: transport
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /_cluster/health
|
||||
port: 9200
|
||||
initialDelaySeconds: 180
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 10
|
||||
failureThreshold: 5
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /_cluster/health?local=true
|
||||
port: 9200
|
||||
initialDelaySeconds: 150
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 10
|
||||
failureThreshold: 5
|
||||
resources:
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 2Gi
|
||||
limits:
|
||||
cpu: 2000m
|
||||
memory: 4Gi
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /usr/share/elasticsearch/config/elasticsearch.yml
|
||||
subPath: elasticsearch.yml
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: elasticsearch-config
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: temporal-elasticsearch
|
||||
namespace: temporal
|
||||
spec:
|
||||
selector:
|
||||
app: temporal-elasticsearch
|
||||
ports:
|
||||
- port: 9200
|
||||
targetPort: 9200
|
||||
name: http
|
||||
- port: 9300
|
||||
targetPort: 9300
|
||||
name: transport
|
||||
type: ClusterIP
|
||||
@@ -1,8 +0,0 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: temporal
|
||||
resources:
|
||||
- elasticsearch.yaml
|
||||
- queues/
|
||||
# SOPS secrets (*.enc.yaml) handled by ArgoCD SOPS plugin at sync time
|
||||
# temporal deployed via ArgoCD Helm source
|
||||
@@ -1,38 +0,0 @@
|
||||
# Example Queues for "production" Temporal namespace
|
||||
# When applied, queue-operator creates 1 TemporalWorker (worker-production)
|
||||
# that processes ALL queues in the "production" namespace
|
||||
---
|
||||
apiVersion: kmsvc.io/v1
|
||||
kind: Queue
|
||||
metadata:
|
||||
name: story-crater-tasks
|
||||
namespace: sqs
|
||||
labels:
|
||||
temporal.io/namespace: "production"
|
||||
spec:
|
||||
fifoQueue: false
|
||||
visibilityTimeoutSeconds: 30
|
||||
messageRetentionPeriodSeconds: 345600
|
||||
maxReceiveCount: 5
|
||||
partitionsPerShard: 6
|
||||
minShards: 1
|
||||
maxShards: 8
|
||||
shardSplitThresholdBytesPerSec: 5242880
|
||||
|
||||
---
|
||||
apiVersion: kmsvc.io/v1
|
||||
kind: Queue
|
||||
metadata:
|
||||
name: story-crater-notifications
|
||||
namespace: sqs
|
||||
labels:
|
||||
temporal.io/namespace: "production"
|
||||
spec:
|
||||
fifoQueue: false
|
||||
visibilityTimeoutSeconds: 60
|
||||
messageRetentionPeriodSeconds: 345600
|
||||
maxReceiveCount: 3
|
||||
partitionsPerShard: 3
|
||||
minShards: 1
|
||||
maxShards: 4
|
||||
shardSplitThresholdBytesPerSec: 2621440
|
||||
@@ -1,5 +0,0 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
resources:
|
||||
- example-queue.yaml
|
||||
+12
-12
@@ -1,6 +1,6 @@
|
||||
# k8s/temporal/temporal-values.yaml
|
||||
# Temporal — workflow engine
|
||||
# Uses external CNPG PostgreSQL for persistence (ddb-cluster)
|
||||
# Uses external CNPG PostgreSQL for persistence (temporal-db)
|
||||
# Visibility via same PostgreSQL instance, separate database.
|
||||
#
|
||||
# IMPORTANT — chart schema note (root-caused after Postgres never actually
|
||||
@@ -43,11 +43,11 @@ grafana:
|
||||
enabled: false
|
||||
|
||||
# ── Schema setup/update Jobs ──────────────────────────────────────────────────
|
||||
# The `temporal` and `temporal_visibility` databases are provisioned
|
||||
# declaratively by CNPG Database CRs (k8s/data/temporal-database.yaml,
|
||||
# temporal-visibility-database.yaml), so createDatabase stays disabled (the
|
||||
# `temporal` role also lacks CREATEDB). setup/update run temporal-sql-tool as
|
||||
# the `temporal` owner against those existing DBs to install and migrate the
|
||||
# The `temporal` DB is created by the dedicated temporal-db cluster's initdb and
|
||||
# `temporal_visibility` by a CNPG Database CR — both in
|
||||
# k8s/infra/databases/temporal-db.yaml — so createDatabase stays disabled.
|
||||
# setup/update run temporal-sql-tool as the `app` owner against those existing
|
||||
# DBs to install and migrate the
|
||||
# Temporal server schema — without them both DBs have zero tables and the
|
||||
# server dies on "no usable database connection found" (no schema_version row).
|
||||
schema:
|
||||
@@ -92,7 +92,7 @@ server:
|
||||
driver: "sql"
|
||||
sql:
|
||||
driver: "postgres12"
|
||||
host: "ddb-cluster-rw.ddb.svc.cluster.local"
|
||||
host: "temporal-db-rw.temporal.svc.cluster.local"
|
||||
port: 5432
|
||||
database: "temporal"
|
||||
user: "app"
|
||||
@@ -102,8 +102,8 @@ server:
|
||||
# existingSecret is set the chart's own server-secret.yaml Secret
|
||||
# template is skipped entirely (see templates/server-secret.yaml:
|
||||
# `not $driverConfig.existingSecret` guards its creation).
|
||||
# Use unified ddb-cluster-app secret (copied to temporal namespace)
|
||||
existingSecret: "ddb-cluster-app"
|
||||
# Use unified temporal-db-app secret (generated in temporal namespace)
|
||||
existingSecret: "temporal-db-app"
|
||||
secretKey: "password"
|
||||
maxConns: 20
|
||||
maxIdleConns: 10
|
||||
@@ -118,12 +118,12 @@ server:
|
||||
driver: "sql"
|
||||
sql:
|
||||
driver: "postgres12"
|
||||
host: "ddb-cluster-rw.ddb.svc.cluster.local"
|
||||
host: "temporal-db-rw.temporal.svc.cluster.local"
|
||||
port: 5432
|
||||
database: "temporal_visibility"
|
||||
user: "app"
|
||||
# Use unified ddb-cluster-app secret (copied to temporal namespace)
|
||||
existingSecret: "ddb-cluster-app"
|
||||
# Use unified temporal-db-app secret (generated in temporal namespace)
|
||||
existingSecret: "temporal-db-app"
|
||||
secretKey: "password"
|
||||
maxConns: 20
|
||||
maxIdleConns: 10
|
||||
@@ -16,7 +16,7 @@ spec:
|
||||
targetRevision: "5.0.18"
|
||||
helm:
|
||||
valueFiles:
|
||||
- $values/k8s/infrastructure/minio/minio-operator-values.yaml
|
||||
- $values/k8s/infra/minio/minio-operator-values.yaml
|
||||
- repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git
|
||||
targetRevision: main
|
||||
ref: values
|
||||
@@ -43,7 +43,7 @@ spec:
|
||||
source:
|
||||
repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git
|
||||
targetRevision: main
|
||||
path: k8s/infrastructure/minio
|
||||
path: k8s/infra/minio
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: storage
|
||||
@@ -68,7 +68,7 @@ spec:
|
||||
source:
|
||||
repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git
|
||||
targetRevision: main
|
||||
path: k8s/infrastructure/longhorn
|
||||
path: k8s/infra/longhorn
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: longhorn-system
|
||||
@@ -93,7 +93,7 @@ spec:
|
||||
helm:
|
||||
skipCrds: true
|
||||
valueFiles:
|
||||
- $values/k8s/platform/monitoring/prometheus-values.yaml
|
||||
- $values/k8s/infra/monitoring/prometheus-values.yaml
|
||||
- repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git
|
||||
targetRevision: main
|
||||
ref: values
|
||||
@@ -122,7 +122,7 @@ spec:
|
||||
# metadata conflict since CRDs are cluster-scoped).
|
||||
---
|
||||
# CRDs only, extracted to plain YAML (`helm show crds kube-prometheus-stack`)
|
||||
# and committed to git under k8s/platform/monitoring/crds/, applied via Server-
|
||||
# and committed to git under k8s/infra/monitoring/crds/, applied via Server-
|
||||
# Side Apply to avoid the etcd 262144-byte last-applied-configuration
|
||||
# annotation limit that client-side apply hits on these very large CRDs
|
||||
# (prometheuses, alertmanagers, scrapeconfigs, etc). A plain git path source
|
||||
@@ -131,7 +131,7 @@ spec:
|
||||
# means from a Helm chart. Split out from the main `prometheus` Application
|
||||
# (helm.skipCrds: true there) because ServerSideApply conflicts with that
|
||||
# app's managedNamespaceMetadata.
|
||||
# NOTE: bump k8s/platform/monitoring/crds/kube-prometheus-stack-crds.yaml
|
||||
# NOTE: bump k8s/infra/monitoring/crds/kube-prometheus-stack-crds.yaml
|
||||
# whenever the kube-prometheus-stack chart version changes materially
|
||||
# (`helm show crds prometheus-community/kube-prometheus-stack > ...`).
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
@@ -146,7 +146,38 @@ spec:
|
||||
source:
|
||||
repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git
|
||||
targetRevision: main
|
||||
path: k8s/platform/monitoring/crds
|
||||
path: k8s/infra/monitoring/crds
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: monitoring
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
- ServerSideApply=true
|
||||
---
|
||||
# Cluster monitoring config: custom PrometheusRules (per-app namespaces),
|
||||
# ServiceMonitors (monitoring ns), and Grafana dashboard ConfigMaps (logging ns,
|
||||
# grafana sidecar-discovered). Single source = k8s/infra/monitoring (one
|
||||
# kustomization, no namespace transformer so per-app rule namespaces are kept).
|
||||
# Wave 2: after prometheus-operator CRDs (wave 0) + stack (wave 1) and grafana
|
||||
# (wave 2, logging). ServerSideApply avoids the etcd last-applied annotation
|
||||
# limit on the large dashboard ConfigMap JSON.
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: monitoring-config
|
||||
namespace: argocd
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "2"
|
||||
spec:
|
||||
project: homelab
|
||||
source:
|
||||
repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git
|
||||
targetRevision: main
|
||||
path: k8s/infra/monitoring
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: monitoring
|
||||
@@ -173,7 +204,7 @@ spec:
|
||||
targetRevision: "~11"
|
||||
helm:
|
||||
valueFiles:
|
||||
- $values/k8s/platform/monitoring/blackbox-exporter-values.yaml
|
||||
- $values/k8s/infra/monitoring/blackbox-exporter-values.yaml
|
||||
- repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git
|
||||
targetRevision: main
|
||||
ref: values
|
||||
|
||||
@@ -18,7 +18,7 @@ spec:
|
||||
targetRevision: "*"
|
||||
helm:
|
||||
valueFiles:
|
||||
- $values/k8s/platform/logging/loki-values.yaml
|
||||
- $values/k8s/infra/logging/loki-values.yaml
|
||||
- repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git
|
||||
targetRevision: main
|
||||
ref: values
|
||||
@@ -52,7 +52,7 @@ spec:
|
||||
targetRevision: "*"
|
||||
helm:
|
||||
valueFiles:
|
||||
- $values/k8s/platform/logging/grafana-values.yaml
|
||||
- $values/k8s/infra/logging/grafana-values.yaml
|
||||
- repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git
|
||||
targetRevision: main
|
||||
ref: values
|
||||
@@ -86,7 +86,7 @@ spec:
|
||||
targetRevision: "*"
|
||||
helm:
|
||||
valueFiles:
|
||||
- $values/k8s/platform/logging/promtail-values.yaml
|
||||
- $values/k8s/infra/logging/promtail-values.yaml
|
||||
- repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git
|
||||
targetRevision: main
|
||||
ref: values
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Wave 3 — Vault + Authentik (identity), plus IAM raw jobs and the Forgejo
|
||||
# runner. Authentik/Vault values reference SOPS-managed secrets (see *.enc.yaml
|
||||
# in k8s/security/iam) resolved by the ArgoCD SOPS plugin at sync time.
|
||||
# in k8s/infra/iam) resolved by the ArgoCD SOPS plugin at sync time.
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
@@ -16,7 +16,7 @@ spec:
|
||||
targetRevision: "*"
|
||||
helm:
|
||||
valueFiles:
|
||||
- $values/k8s/security/iam/vault-values.yaml
|
||||
- $values/k8s/infra/iam/vault-values.yaml
|
||||
- repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git
|
||||
targetRevision: main
|
||||
ref: values
|
||||
@@ -45,7 +45,7 @@ spec:
|
||||
targetRevision: "*"
|
||||
helm:
|
||||
valueFiles:
|
||||
- $values/k8s/security/iam/authentik-values.yaml
|
||||
- $values/k8s/infra/iam/authentik-values.yaml
|
||||
- repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git
|
||||
targetRevision: main
|
||||
ref: values
|
||||
@@ -70,7 +70,7 @@ spec:
|
||||
source:
|
||||
repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git
|
||||
targetRevision: main
|
||||
path: k8s/security/iam
|
||||
path: k8s/infra/iam
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: iam
|
||||
@@ -92,7 +92,7 @@ spec:
|
||||
source:
|
||||
repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git
|
||||
targetRevision: main
|
||||
path: k8s/security/ci-cd/charts/forgejo-runner
|
||||
path: k8s/infra/forgejo-runner
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: cicd
|
||||
|
||||
@@ -1,24 +1,28 @@
|
||||
# Wave 6 — Database schemas + init jobs.
|
||||
# CNPG operator and ddb-cluster are bootstrap-only (k8s/bootstrap-local/).
|
||||
# This app manages ONLY the per-app databases and schema initialization.
|
||||
# Dependencies: ddb-cluster (bootstrap wave 0), SOPS secrets (wave 4)
|
||||
# Wave 2 — dedicated per-app CNPG Postgres clusters (authentik-db → ns iam,
|
||||
# temporal-db + visibility → ns temporal). ONE App, ONE folder (k8s/infra/databases).
|
||||
# CNPG operator is Phase-0 bootstrap; these Cluster CRs are GitOps — no circular
|
||||
# dep (they run after ArgoCD is up, before their apps at w3/w8). CNPG generates
|
||||
# each cluster's `<name>-app` secret + `<name>-rw` service in-namespace; the apps
|
||||
# read them locally. Forgejo's DB stays separate (bootstrap/circular).
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: data-schemas
|
||||
name: databases
|
||||
namespace: argocd
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "6"
|
||||
argocd.argoproj.io/sync-wave: "2"
|
||||
spec:
|
||||
project: homelab
|
||||
source:
|
||||
repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git
|
||||
targetRevision: main
|
||||
path: k8s/data/schemas # CHANGED from k8s/data to avoid ddb-cluster duplication
|
||||
path: k8s/infra/databases
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: ddb
|
||||
namespace: default
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- ServerSideApply=true
|
||||
|
||||
@@ -70,7 +70,7 @@ spec:
|
||||
source:
|
||||
repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git
|
||||
targetRevision: main
|
||||
path: k8s/applications/sqs/charts/kafka-cluster
|
||||
path: k8s/apps/messaging/kafka-cluster
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: sqs
|
||||
@@ -91,7 +91,7 @@ spec:
|
||||
source:
|
||||
repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git
|
||||
targetRevision: main
|
||||
path: k8s/applications/sqs/charts/queue-crd
|
||||
path: k8s/apps/messaging/queue-crd
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: sqs
|
||||
@@ -112,7 +112,7 @@ spec:
|
||||
source:
|
||||
repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git
|
||||
targetRevision: main
|
||||
path: k8s/applications/sqs/charts/management-service
|
||||
path: k8s/apps/messaging/management-service
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: sqs
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# helpers (cloudflared tunnel, duckdns updater) that are already running.
|
||||
# Experimental dirs (llm, forge, dev-tools, shadowsocks) are intentionally
|
||||
# NOT included yet — add them here once they're production-ready.
|
||||
# temporal using unified CNPG pattern (app user, ddb-cluster-app secret)
|
||||
# temporal using unified CNPG pattern (app user, temporal-db-app secret)
|
||||
# Secret copied by bootstrap.sh (like cicd/iam namespaces)
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
@@ -19,7 +19,7 @@ spec:
|
||||
targetRevision: "0.74.0"
|
||||
helm:
|
||||
valueFiles:
|
||||
- $values/k8s/applications/temporal/temporal-values.yaml
|
||||
- $values/k8s/apps/temporal/temporal-values.yaml
|
||||
- repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git
|
||||
targetRevision: main
|
||||
ref: values
|
||||
@@ -48,7 +48,7 @@ spec:
|
||||
targetRevision: "*"
|
||||
helm:
|
||||
valueFiles:
|
||||
- $values/k8s/applications/portainer/portainer-values.yaml
|
||||
- $values/k8s/apps/portainer/portainer-values.yaml
|
||||
- repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git
|
||||
targetRevision: main
|
||||
ref: values
|
||||
@@ -74,7 +74,7 @@ spec:
|
||||
source:
|
||||
repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git
|
||||
targetRevision: main
|
||||
path: k8s/applications/cloudflared
|
||||
path: k8s/apps/cloudflared
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: cloudflared
|
||||
@@ -97,7 +97,7 @@ spec:
|
||||
source:
|
||||
repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git
|
||||
targetRevision: main
|
||||
path: k8s/applications/duckdns
|
||||
path: k8s/apps/duckdns
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: duckdns
|
||||
@@ -125,13 +125,13 @@ spec:
|
||||
targetRevision: "*"
|
||||
helm:
|
||||
valueFiles:
|
||||
- $values/k8s/applications/homarr/homarr-values.yaml
|
||||
- $values/k8s/apps/homarr/homarr-values.yaml
|
||||
- repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git
|
||||
targetRevision: main
|
||||
ref: values
|
||||
- repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git
|
||||
targetRevision: main
|
||||
path: k8s/applications/homarr # PostSync hook: fix-probes-job.yaml
|
||||
path: k8s/apps/homarr # PostSync hook: fix-probes-job.yaml
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: dashboard
|
||||
|
||||
@@ -115,7 +115,7 @@ git push forgejo main
|
||||
|
||||
- **Phase stuck?** Check `kubectl get events -n <namespace> --sort-by='.lastTimestamp'`
|
||||
- **ArgoCD duplicating?** Verify manifests match exactly (Helm values ↔ ArgoCD Application)
|
||||
- **Forgejo won't start?** Check CNPG cluster Ready: `kubectl get cluster forgejo-db -n forgejo`
|
||||
- **Forgejo won't start?** Check CNPG cluster Ready: `kubectl get cluster forgejo-db -n cicd`
|
||||
- **Can't push to Forgejo?** Verify ingress-nginx healthy, DNS resolves `forgejo.riotpiao.com`
|
||||
|
||||
## Migration from Old Bootstrap
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: forgejo
|
||||
name: cicd
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-options: Prune=false
|
||||
---
|
||||
@@ -13,7 +13,7 @@ apiVersion: postgresql.cnpg.io/v1
|
||||
kind: Cluster
|
||||
metadata:
|
||||
name: forgejo-db
|
||||
namespace: forgejo
|
||||
namespace: cicd
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-options: Prune=false # Let ArgoCD adopt, don't delete
|
||||
labels:
|
||||
@@ -71,15 +71,21 @@ spec:
|
||||
enablePodMonitor: true
|
||||
|
||||
affinity:
|
||||
podAntiAffinityType: required
|
||||
# preferred (not required) so it can't deadlock if fewer than 3 nodes are
|
||||
# schedulable; tolerations let CNPG pods land on control-plane nodes.
|
||||
podAntiAffinityType: preferred
|
||||
topologyKey: kubernetes.io/hostname
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
---
|
||||
# Forgejo Redis (cache, session, queue)
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: forgejo-redis
|
||||
namespace: forgejo
|
||||
namespace: cicd
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-options: Prune=false
|
||||
spec:
|
||||
@@ -95,7 +101,7 @@ apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: forgejo-redis
|
||||
namespace: forgejo
|
||||
namespace: cicd
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-options: Prune=false
|
||||
spec:
|
||||
@@ -108,6 +114,10 @@ spec:
|
||||
labels:
|
||||
app: forgejo-redis
|
||||
spec:
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
containers:
|
||||
- name: redis
|
||||
image: redis:7-alpine
|
||||
|
||||
@@ -16,7 +16,7 @@ gitea:
|
||||
|
||||
database:
|
||||
DB_TYPE: postgres
|
||||
HOST: forgejo-db-rw.forgejo.svc.cluster.local:5432
|
||||
HOST: forgejo-db-rw.cicd.svc.cluster.local:5432
|
||||
NAME: forgejo
|
||||
# User/password from CNPG-generated secret
|
||||
USER:
|
||||
@@ -32,15 +32,15 @@ gitea:
|
||||
|
||||
cache:
|
||||
ADAPTER: redis
|
||||
HOST: redis://forgejo-redis.forgejo.svc.cluster.local:6379/0
|
||||
HOST: redis://forgejo-redis.cicd.svc.cluster.local:6379/0
|
||||
|
||||
session:
|
||||
PROVIDER: redis
|
||||
PROVIDER_CONFIG: redis://forgejo-redis.forgejo.svc.cluster.local:6379/1
|
||||
PROVIDER_CONFIG: redis://forgejo-redis.cicd.svc.cluster.local:6379/1
|
||||
|
||||
queue:
|
||||
TYPE: redis
|
||||
CONN_STR: redis://forgejo-redis.forgejo.svc.cluster.local:6379/2
|
||||
CONN_STR: redis://forgejo-redis.cicd.svc.cluster.local:6379/2
|
||||
|
||||
# Persistence (shared storage for repos)
|
||||
persistence:
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: Cluster
|
||||
metadata:
|
||||
name: ddb-cluster
|
||||
namespace: ddb
|
||||
labels:
|
||||
app: postgresql
|
||||
layer: data
|
||||
spec:
|
||||
# 3-replica cluster — distributed across control-plane nodes (cp-1, cp-2, cp-3)
|
||||
# Provides HA for Forgejo and other stateful apps using shared DDB
|
||||
instances: 3
|
||||
|
||||
# PostgreSQL 16.2
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16.2
|
||||
|
||||
# Bootstrap: create app database + extensions
|
||||
bootstrap:
|
||||
initdb:
|
||||
database: app
|
||||
owner: app
|
||||
encoding: UTF8
|
||||
localeCollate: C
|
||||
localeCType: C
|
||||
postInitApplicationSQL:
|
||||
- CREATE EXTENSION IF NOT EXISTS vector;
|
||||
- CREATE EXTENSION IF NOT EXISTS pgcrypto;
|
||||
- CREATE EXTENSION IF NOT EXISTS pg_trgm;
|
||||
|
||||
# Simple ownership model: all apps use 'app' bootstrap user.
|
||||
# Isolation via separate database names, not separate roles.
|
||||
# Aligns with CNPG design (single cluster, multiple databases).
|
||||
# managed.roles removed - no per-app roles needed.
|
||||
|
||||
# Disable superuser (security)
|
||||
enableSuperuserAccess: false
|
||||
|
||||
# PostgreSQL configuration
|
||||
postgresql:
|
||||
parameters:
|
||||
shared_buffers: "256MB"
|
||||
max_parallel_workers: "4"
|
||||
max_parallel_workers_per_gather: "4"
|
||||
# WAL archiving for backups
|
||||
archive_mode: "on"
|
||||
archive_timeout: "5min"
|
||||
log_destination: "csvlog"
|
||||
log_directory: "/controller/log"
|
||||
log_filename: "postgres"
|
||||
log_rotation_age: "0"
|
||||
dynamic_shared_memory_type: "posix"
|
||||
|
||||
# Storage on Longhorn
|
||||
storage:
|
||||
size: 10Gi
|
||||
storageClass: longhorn
|
||||
|
||||
# Monitoring
|
||||
monitoring:
|
||||
enablePodMonitor: false
|
||||
disableDefaultQueries: false
|
||||
customQueriesConfigMap:
|
||||
- name: cnpg-default-monitoring
|
||||
key: queries
|
||||
|
||||
# Pod anti-affinity for spreading replicas
|
||||
affinity:
|
||||
podAntiAffinityType: preferred
|
||||
@@ -1,10 +0,0 @@
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: Database
|
||||
metadata:
|
||||
name: forgejo
|
||||
namespace: ddb
|
||||
spec:
|
||||
name: forgejo
|
||||
owner: app
|
||||
cluster:
|
||||
name: ddb-cluster
|
||||
@@ -1,17 +0,0 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: ddb-cluster-bootstrap
|
||||
annotations:
|
||||
description: |
|
||||
Bootstrap-only resources (NOT managed by ArgoCD GitOps).
|
||||
These are applied via k8s/bootstrap-local/ and never touched afterward.
|
||||
The actual deployment is in bootstrap-local/03-ddb-bootstrap.yaml.
|
||||
|
||||
namespace: ddb
|
||||
|
||||
# IMPORTANT: These files are duplicated in k8s/bootstrap-local/03-ddb-bootstrap.yaml
|
||||
# DO NOT reference this kustomization from any ArgoCD Application.
|
||||
resources:
|
||||
- ddb-cluster.yaml
|
||||
- forgejo-database.yaml
|
||||
@@ -1,49 +0,0 @@
|
||||
apiVersion: ENC[AES256_GCM,data:u74=,iv:vPf+N0mpxoTTHn8t3W6kiIbNrXGOFngfnc5vk3gzq+Y=,tag:OlJWpHdVqQHeA96OdJoA7A==,type:str]
|
||||
kind: ENC[AES256_GCM,data:9f6uPGLQ,iv:V5KjMaBQoQRwMxHRoStLIlSYW0zqgcORLX5GKYKZ7hY=,tag:zLozLmSNHOd2/lFxFfnxtw==,type:str]
|
||||
metadata:
|
||||
name: ENC[AES256_GCM,data:uNU+lZyIdW0bwLQHoecYqdU=,iv:ov6MQQ9e7s+dRoQbZQPbjtB5j42dckyg27nx+LWJw0I=,tag:74nJSTgy9pZ5LH/O3xpwSA==,type:str]
|
||||
namespace: ENC[AES256_GCM,data:xIre,iv:Gv//YoAeIoFgFbW9OJkHPv1Fz8NEAzB+qSpNYXm1008=,tag:Vsj/V3wLX8mUnHrMGZZxFw==,type:str]
|
||||
type: ENC[AES256_GCM,data:pkm2nd/50FTigAJyEW2mEcgKVR0Y61VA,iv:DQ3k8Xp54nySnVEJvbiMXYJ+0ssUC8oSDLsoRut+qMg=,tag:ZwVH9ivA5hBCCkiGjXPbUw==,type:str]
|
||||
stringData:
|
||||
username: ENC[AES256_GCM,data:TeIYIIDS+RWp,iv:feJ2lRnPWnDKIMgSEz4A070k2ZrDFz7VzF7vthxoQuM=,tag:QYeI+6qeuyL3hIZSH7eQWg==,type:str]
|
||||
password: ENC[AES256_GCM,data:ACIyp9eAiQtuzcN8piX5XX/UzHEjobxT1M6usFV6jwQ=,iv:AnmrpNo9JqtWYMwh4ghWXZfmeCTXFqzkfN69CPbWkpM=,tag:kZ2jfUAvtG39+Og/Km2fwA==,type:str]
|
||||
sops:
|
||||
age:
|
||||
- enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBSaFFueUY2MUZ1dzRkelRL
|
||||
aklvRytMSVBJQjA4UXdqSjlRQUo2R0lyTTFVCktUNWl1cWttZDdVd2ZyR0lPVERj
|
||||
T1ZpMkRrZGgvWFJVRnZOdlZJc3RNcTAKLS0tIHVwY0JNMXVWeTZnWW9BQnM0TCtK
|
||||
bVJ0eU05N3hIT3Z6TWdSOUxxeGh1b1UKuQ23KqCTNlEk3c8gbQsFGQwkN4gkI7Z2
|
||||
HauqbNHQoU64smns+HPDoHq7z87xRlQ8c8efNpIGhfYgS/nf1eGXbg==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
recipient: age1smu533f803gmd0jq60s2zaj9zlznajy0ca6rtewd4r37mr2hs3uqsrldfh
|
||||
lastmodified: "2026-07-21T06:35:41Z"
|
||||
mac: ENC[AES256_GCM,data:VeVffJWYwoqNe/irAsWrl1rAOr2xMnsPgSCqiszaoF1807iU7FSpuvSOCAbaOJXSKgJ/MBflnegCIsDb7a1W4BcwhwuMGoGl2C2vBs2YPZvqTzfnfXAGrWCbZdyCh16Dg8nZQlwRe2QwokqW+t3j1qAYfq4KVu8MLi648adRIt0=,iv:mCMQXdfVQNDgY1ZHEQGBC4YaHwkCsyo+oADMJ7p23B8=,tag:7c9VkImsgkjSxOkM1Q3qgg==,type:str]
|
||||
unencrypted_suffix: _unencrypted
|
||||
version: 3.13.2
|
||||
---
|
||||
apiVersion: ENC[AES256_GCM,data:DxA=,iv:eWizGuPMW/RfEkLvL4pei42sov2EUwSDHKrtdraSxWw=,tag:E1w/dzFGqUIr+W+wY4YKNA==,type:str]
|
||||
kind: ENC[AES256_GCM,data:amkGISvV,iv:13KTCVXg9EKQLfz4cCCPxfHAnPHZ0AheDmn8HIm87rw=,tag:wHfOOTJsrN2iiOTsSN507A==,type:str]
|
||||
metadata:
|
||||
name: ENC[AES256_GCM,data:lpkz7+GPScHzbUsBL276Gw==,iv:ce+pCqVqXZD7G6LZzFGMCK58k2icd07cE9DTJYOid5I=,tag:ug6gUmLuTkyy0eRsTGJqOw==,type:str]
|
||||
namespace: ENC[AES256_GCM,data:sCqW,iv:5rvJSGBshQsjrUkizUekMpS0Fpyfkf7IcNL8aZR3lks=,tag:cFtIZO6F9eAdzE+IVv1wyg==,type:str]
|
||||
type: ENC[AES256_GCM,data:aUYmzPoRlh2uDcjhZ5OD82ScAtBxd1Fm,iv:ovGv+qPiiLBC1mVSEX+QuJOkAjqPAUO/e+Nux2xuVKg=,tag:gaZwWSH7LF1FCKZr+P+R1A==,type:str]
|
||||
stringData:
|
||||
username: ENC[AES256_GCM,data:kvKdlAd1ox4=,iv:NcwxeRHzyRECQ7cLUwOeEiKkHMG7FhFMZRsfUfdtBmM=,tag:zfS+ZOFMHA6Q3LCkM2GSqA==,type:str]
|
||||
password: ENC[AES256_GCM,data:7/ZJxWr39GrKA+i50S7wLBjt2Kaz9aG1ML44VGyRWi0=,iv:y0KlJ5ogdBDGzL6IPIjqD6Ir/S6K6v1dGTg8wxxp8n4=,tag:zwnCvi7RPqQdLJvfk69KgA==,type:str]
|
||||
sops:
|
||||
age:
|
||||
- enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBSaFFueUY2MUZ1dzRkelRL
|
||||
aklvRytMSVBJQjA4UXdqSjlRQUo2R0lyTTFVCktUNWl1cWttZDdVd2ZyR0lPVERj
|
||||
T1ZpMkRrZGgvWFJVRnZOdlZJc3RNcTAKLS0tIHVwY0JNMXVWeTZnWW9BQnM0TCtK
|
||||
bVJ0eU05N3hIT3Z6TWdSOUxxeGh1b1UKuQ23KqCTNlEk3c8gbQsFGQwkN4gkI7Z2
|
||||
HauqbNHQoU64smns+HPDoHq7z87xRlQ8c8efNpIGhfYgS/nf1eGXbg==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
recipient: age1smu533f803gmd0jq60s2zaj9zlznajy0ca6rtewd4r37mr2hs3uqsrldfh
|
||||
lastmodified: "2026-07-21T06:35:41Z"
|
||||
mac: ENC[AES256_GCM,data:VeVffJWYwoqNe/irAsWrl1rAOr2xMnsPgSCqiszaoF1807iU7FSpuvSOCAbaOJXSKgJ/MBflnegCIsDb7a1W4BcwhwuMGoGl2C2vBs2YPZvqTzfnfXAGrWCbZdyCh16Dg8nZQlwRe2QwokqW+t3j1qAYfq4KVu8MLi648adRIt0=,iv:mCMQXdfVQNDgY1ZHEQGBC4YaHwkCsyo+oADMJ7p23B8=,tag:7c9VkImsgkjSxOkM1Q3qgg==,type:str]
|
||||
unencrypted_suffix: _unencrypted
|
||||
version: 3.13.2
|
||||
@@ -1,10 +0,0 @@
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: Database
|
||||
metadata:
|
||||
name: authentik
|
||||
namespace: ddb
|
||||
spec:
|
||||
name: authentik
|
||||
owner: app # All apps use shared 'app' user (CNPG design pattern)
|
||||
cluster:
|
||||
name: ddb-cluster
|
||||
@@ -1,67 +0,0 @@
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: db-schema-init
|
||||
namespace: ddb
|
||||
labels:
|
||||
app: postgresql-init
|
||||
layer: data
|
||||
spec:
|
||||
backoffLimit: 3
|
||||
ttlSecondsAfterFinished: 3600 # Keep Job for 1 hour after completion
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: postgresql-init
|
||||
spec:
|
||||
serviceAccountName: default
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: schema-init
|
||||
image: postgres:16.2-alpine
|
||||
env:
|
||||
- name: PGHOST
|
||||
value: ddb-cluster-rw.ddb.svc.cluster.local
|
||||
- name: PGPORT
|
||||
value: "5432"
|
||||
- name: PGDATABASE
|
||||
value: app
|
||||
- name: PGUSER
|
||||
value: app
|
||||
- name: PGPASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: ddb-cluster-app
|
||||
key: password
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
# Wait for cluster to be ready
|
||||
echo "Waiting for PostgreSQL to be ready..."
|
||||
until pg_isready -h $PGHOST -p $PGPORT -U $PGUSER; do
|
||||
echo "Waiting..."
|
||||
sleep 5
|
||||
done
|
||||
|
||||
echo "Creating schemas..."
|
||||
|
||||
# Create schemas from ConfigMap
|
||||
psql -h $PGHOST -p $PGPORT -U $PGUSER -d $PGDATABASE << 'EOF'
|
||||
CREATE SCHEMA IF NOT EXISTS authentik;
|
||||
GRANT USAGE ON SCHEMA authentik TO app;
|
||||
GRANT CREATE ON SCHEMA authentik TO app;
|
||||
|
||||
CREATE SCHEMA IF NOT EXISTS temporal;
|
||||
GRANT USAGE ON SCHEMA temporal TO app;
|
||||
GRANT CREATE ON SCHEMA temporal TO app;
|
||||
|
||||
CREATE SCHEMA IF NOT EXISTS vault;
|
||||
GRANT USAGE ON SCHEMA vault TO app;
|
||||
GRANT CREATE ON SCHEMA vault TO app;
|
||||
|
||||
GRANT USAGE ON SCHEMA public TO app;
|
||||
GRANT CREATE ON SCHEMA public TO app;
|
||||
EOF
|
||||
|
||||
echo "✓ Schema initialization complete"
|
||||
@@ -1,17 +0,0 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: data-schemas
|
||||
|
||||
namespace: ddb
|
||||
|
||||
# GitOps-managed database schemas (ArgoCD wave 6).
|
||||
# These depend on ddb-cluster existing (bootstrap wave 0).
|
||||
resources:
|
||||
- authentik-database.yaml
|
||||
- temporal-database.yaml
|
||||
- temporal-visibility-database.yaml
|
||||
- schemas.yaml
|
||||
- db-init-job.yaml
|
||||
|
||||
# db-role-secrets.enc.yaml handled by SOPS secrets Application (wave 4)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user