feat:Fix the bootstrap to be deploy key application

This commit is contained in:
Story Crater Bot
2026-07-23 19:07:39 -07:00
parent eba9f2144c
commit 1c7395d9e1
37 changed files with 1216 additions and 1202 deletions
@@ -1,140 +0,0 @@
# k8s/temporal/temporal-values.yaml
# Temporal — workflow engine
# Uses external CNPG PostgreSQL for persistence (ddb-cluster)
# Visibility via same PostgreSQL instance, separate database.
#
# IMPORTANT — chart schema note (root-caused after Postgres never actually
# taking effect despite looking configured):
# We're pinned to temporalio/helm-charts @ 0.74.0 (see targetRevision in
# k8s/argocd/apps/60-applications.yaml), which uses the OLD flat persistence
# schema:
# server.config.persistence.<default|visibility>.driver: "sql"|"cassandra"
# server.config.persistence.<default|visibility>.sql: {...}
# NOT the newer `datastores:`-wrapped schema
# (server.config.persistence.datastores.<store>.sql) shown in the current
# chart's values/values.postgresql.yaml example - that key was introduced in
# a later major version and doesn't exist in 0.74.0. Helm doesn't validate
# unknown keys, so a `datastores:` block here is silently a no-op: Temporal
# would keep defaulting to Cassandra (with empty hosts: []) regardless of
# anything nested inside it. Verified via `helm template` against the actual
# 0.74.0 chart before writing this file - see chat history for the
# side-by-side proof (rendered manifest showed CASSANDRA_HOST env vars and
# temporal-cassandra-tool commands using the old datastores:-based values).
#
# Likewise `schema.setup.enabled` / `schema.update.enabled` /
# `schema.createDatabase.enabled` are the real toggles for the schema-setup
# Job (all default true) - there is no `jobs.autoSetup` key in this chart.
# ── Disable every bundled/optional sub-chart ─────────────────────────────────
# postgresql/mysql: never enable - we never want the chart to deploy its own
# DB, only to know how to talk to our external CNPG instance (which happens
# via server.config.persistence.*.sql below, independent of these flags).
postgresql:
enabled: false
mysql:
enabled: false
cassandra:
enabled: false
elasticsearch:
enabled: false
prometheus:
enabled: false
grafana:
enabled: false
# ── Schema setup/update Jobs ──────────────────────────────────────────────────
# The `temporal` and `temporal_visibility` databases are provisioned
# declaratively by CNPG Database CRs (k8s/data/temporal-database.yaml,
# temporal-visibility-database.yaml), so createDatabase stays disabled (the
# `temporal` role also lacks CREATEDB). setup/update run temporal-sql-tool as
# the `temporal` owner against those existing DBs to install and migrate the
# Temporal server schema — without them both DBs have zero tables and the
# server dies on "no usable database connection found" (no schema_version row).
schema:
createDatabase:
enabled: false
setup:
enabled: true
update:
enabled: true
# ── Temporal server config (PostgreSQL persistence) ──────────────────────────
server:
replicaCount: 1
# temporalio/server:1.30.0+ dropped the `dockerize` binary and switched to
# built-in sprig config templating. The chart still defaults to the legacy
# configMapsToMount: "dockerize" + setConfigFilePath: false, which produces a
# config the 1.30 server never loads — it then falls back to its embedded
# env-only template (Cassandra default) and dies with
# "Persistence.DataStores[default](value).Cassandra.Hosts: zero value".
# Switch to the sprig ConfigMap and point the server at it (chart's own
# recommendation for 1.30.0+ images; sprig mode requires setConfigFilePath).
configMapsToMount: "sprig"
setConfigFilePath: true
jobService:
enabled: false
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchLabels:
app.kubernetes.io/instance: temporal
topologyKey: kubernetes.io/hostname
config:
logLevel: "info"
persistence:
defaultStore: default
visibilityStore: visibility
numHistoryShards: 512
default:
driver: "sql"
sql:
driver: "postgres12"
host: "ddb-cluster-rw.ddb.svc.cluster.local"
port: 5432
database: "temporal"
user: "temporal"
# existingSecret + secretKey: point directly at the CNPG-generated
# Secret (kubernetes.io/basic-auth, keys: username/password/...)
# rather than duplicating the password in git as plaintext. When
# existingSecret is set the chart's own server-secret.yaml Secret
# template is skipped entirely (see templates/server-secret.yaml:
# `not $driverConfig.existingSecret` guards its creation).
existingSecret: "temporal-db-role"
secretKey: "password"
maxConns: 20
maxIdleConns: 10
maxConnLifetime: "1h"
# NOTE: no `connectAttributes: { tx_isolation: ... }` here — tx_isolation
# is a MySQL-only connection parameter. The Postgres `pq` driver rejects
# it ("unrecognized configuration parameter"), which killed every DB
# connection (schema-setup job AND server) with the misleading
# "no usable database connection found". Postgres defaults to READ
# COMMITTED isolation anyway, so nothing is lost by omitting it.
visibility:
driver: "sql"
sql:
driver: "postgres12"
host: "ddb-cluster-rw.ddb.svc.cluster.local"
port: 5432
database: "temporal_visibility"
user: "temporal"
existingSecret: "temporal-db-role"
secretKey: "password"
maxConns: 20
maxIdleConns: 10
maxConnLifetime: "1h"
service:
type: ClusterIP
# ── Temporal Web UI ────────────────────────────────────────────────────────
web:
replicaCount: 1
service:
type: ClusterIP
# ── Ingress ────────────────────────────────────────────────────────
ingress:
enabled: false
-119
View File
@@ -1,119 +0,0 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: argocd
labels:
name: argocd
---
apiVersion: v1
kind: Namespace
metadata:
name: ddb
labels:
name: ddb
---
apiVersion: v1
kind: Namespace
metadata:
name: cicd
labels:
name: cicd
# REQUIRED: Forgejo runner needs privileged (DinD, hostPath, securityContext.privileged)
pod-security.kubernetes.io/enforce: privileged
pod-security.kubernetes.io/audit: privileged
pod-security.kubernetes.io/warn: privileged
---
apiVersion: v1
kind: Namespace
metadata:
name: cert-manager
labels:
name: cert-manager
---
apiVersion: v1
kind: Namespace
metadata:
name: ingress-nginx
labels:
name: ingress-nginx
# REQUIRED: nginx controller needs hostPort 80/443
pod-security.kubernetes.io/enforce: privileged
---
apiVersion: v1
kind: Namespace
metadata:
name: reloader
labels:
name: reloader
---
apiVersion: v1
kind: Namespace
metadata:
name: storage
labels:
name: storage
# REQUIRED: minio operator needs privileged securityContext
pod-security.kubernetes.io/enforce: privileged
pod-security.kubernetes.io/audit: privileged
pod-security.kubernetes.io/warn: privileged
---
apiVersion: v1
kind: Namespace
metadata:
name: monitoring
labels:
name: monitoring
# REQUIRED: node-exporter needs hostNetwork/hostPID/hostPath/hostPort
pod-security.kubernetes.io/enforce: privileged
---
apiVersion: v1
kind: Namespace
metadata:
name: logging
labels:
name: logging
# REQUIRED: promtail needs hostPath, DAC_READ_SEARCH, privileged:true
pod-security.kubernetes.io/enforce: privileged
---
apiVersion: v1
kind: Namespace
metadata:
name: iam
labels:
name: iam
---
apiVersion: v1
kind: Namespace
metadata:
name: sqs
labels:
name: sqs
---
apiVersion: v1
kind: Namespace
metadata:
name: temporal
labels:
name: temporal
---
apiVersion: v1
kind: Namespace
metadata:
name: dashboard
labels:
name: dashboard
---
apiVersion: v1
kind: Namespace
metadata:
name: cloudflared
labels:
name: cloudflared
---
apiVersion: v1
kind: Namespace
metadata:
name: duckdns
labels:
name: duckdns
-47
View File
@@ -1,47 +0,0 @@
# ArgoCD installation - NOT managed by ArgoCD itself (bootstrap only).
# Install via: kubectl apply -k k8s/bootstrap-local/
# Or manually: kubectl create namespace argocd
# kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml
---
apiVersion: v1
kind: ConfigMap
metadata:
name: argocd-cm
namespace: argocd
labels:
app.kubernetes.io/name: argocd-cm
app.kubernetes.io/part-of: argocd
data:
# Point at Forgejo (will be available after 04-forgejo.yaml completes)
repositories: |
- url: https://forgejo.riotpiao.com/riotpiao.com/homelab.git
name: homelab
type: git
# Reconciliation timeout (default 180s)
timeout.reconciliation: "300"
# Resource exclusions (prevent ArgoCD from managing certain resources)
resource.exclusions: |
- apiGroups:
- cilium.io
kinds:
- CiliumIdentity
clusters:
- "*"
---
apiVersion: v1
kind: ConfigMap
metadata:
name: argocd-rbac-cm
namespace: argocd
data:
# Admin policy (adjust as needed)
policy.default: role:readonly
policy.csv: |
g, admin, role:admin
---
# NOTE: ArgoCD installation itself not included here - apply it separately:
# kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml
# Or use Helm chart (recommended for production):
# helm install argocd argo/argo-cd -n argocd --version 7.x.x
-30
View File
@@ -1,30 +0,0 @@
# CloudNativePG operator - deployed as Helm chart via kubectl/ArgoCD.
# This file is a placeholder - actual install via Helm:
# helm repo add cnpg https://cloudnative-pg.github.io/charts
# helm install cnpg cnpg/cloudnative-pg -n ddb --create-namespace --version ~0.20
#
# Or create an ArgoCD Application (recommended):
---
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: cnpg-operator
namespace: argocd
annotations:
argocd.argoproj.io/sync-wave: "-1" # Bootstrap wave (before everything)
spec:
project: homelab
source:
repoURL: https://cloudnative-pg.github.io/charts
chart: cloudnative-pg
targetRevision: "~0.20"
destination:
server: https://kubernetes.default.svc
namespace: ddb
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true
- ServerSideApply=true
-152
View File
@@ -1,152 +0,0 @@
# PostgreSQL cluster + Forgejo dependencies (bootstrap only, not GitOps-managed).
# These resources MUST exist before Forgejo can start, and Forgejo MUST exist
# before ArgoCD can sync from the git repo it hosts → circular dependency.
# Apply once via bootstrap.sh, never touched by ArgoCD afterward.
---
apiVersion: postgresql.cnpg.io/v1
kind: Cluster
metadata:
name: ddb-cluster
namespace: ddb
labels:
app: postgresql
layer: data
bootstrap-phase: "0"
spec:
instances: 3 # HA across 3 control-plane nodes
imageName: ghcr.io/cloudnative-pg/postgresql:16.2
bootstrap:
initdb:
database: app
owner: app
encoding: UTF8
localeCollate: C
localeCType: C
postInitApplicationSQL:
- CREATE EXTENSION IF NOT EXISTS vector;
- CREATE EXTENSION IF NOT EXISTS pgcrypto;
- CREATE EXTENSION IF NOT EXISTS pg_trgm;
# Role management: passwords from secrets, databases from separate Database CRs
managed:
roles:
- name: authentik
ensure: present
login: true
passwordSecret:
name: authentik-db-role
- name: temporal
ensure: present
login: true
passwordSecret:
name: temporal-db-role
enableSuperuserAccess: false
postgresql:
parameters:
shared_buffers: "256MB"
max_parallel_workers: "4"
max_parallel_workers_per_gather: "4"
archive_mode: "on"
archive_timeout: "5min"
log_destination: "csvlog"
log_directory: "/controller/log"
log_filename: "postgres"
log_rotation_age: "0"
dynamic_shared_memory_type: "posix"
storage:
size: 10Gi
storageClass: longhorn
monitoring:
enablePodMonitor: false
disableDefaultQueries: false
customQueriesConfigMap:
- name: cnpg-default-monitoring
key: queries
affinity:
podAntiAffinityType: preferred
---
# Forgejo database (depends on ddb-cluster being ready)
apiVersion: postgresql.cnpg.io/v1
kind: Database
metadata:
name: forgejo
namespace: ddb
labels:
bootstrap-phase: "0"
spec:
name: forgejo
owner: app
cluster:
name: ddb-cluster
---
# Forgejo Redis (cache, session, queue)
apiVersion: v1
kind: Service
metadata:
name: forgejo-redis
namespace: cicd
labels:
app: forgejo-redis
bootstrap-phase: "0"
spec:
ports:
- port: 6379
targetPort: 6379
protocol: TCP
name: redis
selector:
app: forgejo-redis
type: ClusterIP
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: forgejo-redis
namespace: cicd
labels:
app: forgejo-redis
bootstrap-phase: "0"
spec:
replicas: 1
selector:
matchLabels:
app: forgejo-redis
template:
metadata:
labels:
app: forgejo-redis
spec:
containers:
- name: redis
image: redis:7-alpine
ports:
- containerPort: 6379
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 200m
memory: 256Mi
livenessProbe:
tcpSocket:
port: 6379
initialDelaySeconds: 30
periodSeconds: 10
readinessProbe:
exec:
command:
- redis-cli
- ping
initialDelaySeconds: 5
periodSeconds: 5
tolerations:
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
-151
View File
@@ -1,151 +0,0 @@
# Forgejo - Git server hosting the GitOps repo (bootstrap only, manual sync).
# ArgoCD cannot auto-sync Forgejo because Forgejo hosts the repo ArgoCD syncs
# from → circular dependency. Apply once via bootstrap, manual sync only afterward.
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: forgejo
namespace: argocd
annotations:
argocd.argoproj.io/sync-wave: "0" # Bootstrap wave
bootstrap-phase: "0"
description: "Bootstrap-only: Forgejo hosts the GitOps repo"
spec:
project: homelab
source:
repoURL: https://dl.gitea.com/charts/
chart: gitea
targetRevision: "~10"
helm:
# Inline values (git-independent) - keep in sync with k8s/security/ci-cd/forgejo-values.yaml
valuesObject:
image:
repository: codeberg.org/forgejo/forgejo
tag: "13"
pullPolicy: IfNotPresent
gitea:
admin:
username: rock
email: [email protected]
config:
server:
PROTOCOL: http
DOMAIN: forgejo.riotpiao.com
ROOT_URL: https://forgejo.riotpiao.com/
HTTP_PORT: 3000
START_SSH_SERVER: true
SSH_DOMAIN: forgejo.riotpiao.com
SSH_PORT: 2222
SSH_LISTEN_PORT: 2222
database:
DB_TYPE: postgres
HOST: ddb-cluster-rw.ddb.svc:5432
NAME: forgejo
USER: app
repository:
ROOT: /data/git
actions:
ENABLED: true
packages:
ENABLED: true
metrics:
ENABLED: true
service:
DISABLE_REGISTRATION: true
oauth2:
ENABLED: true
PROVIDER: openidconnect
OPENID_CONNECT_DISCOVERY_URL: https://authentik.riotpiao.com/application/o/forgejo/.well-known/openid-configuration
CLIENT_ID: forgejo
AUTO_DISCOVER_URL: https://authentik.riotpiao.com/application/o/forgejo/.well-known/openid-configuration
cache:
ADAPTER: redis
HOST: "redis://forgejo-redis.cicd.svc:6379/0"
session:
PROVIDER: redis
PROVIDER_CONFIG: "redis://forgejo-redis.cicd.svc:6379/1"
queue:
TYPE: redis
CONN_STR: "redis://forgejo-redis.cicd.svc:6379/2"
metrics:
enabled: true
serviceMonitor:
enabled: false
persistence:
enabled: true
storageClass: longhorn
size: 20Gi
accessModes:
- ReadWriteMany
replicaCount: 2
deployment:
strategy:
type: RollingUpdate
env:
- name: SSL_CERT_DIR
value: /homelab-ca
- name: GITEA__database__PASSWD
valueFrom:
secretKeyRef:
name: ddb-cluster-app
key: password
- name: GITEA__oauth2__CLIENT_SECRET
valueFrom:
secretKeyRef:
name: forgejo-oidc
key: CLIENT_SECRET
podAnnotations:
configmap.reloader.stakater.com/reload: "homelab-ca"
service:
http:
type: LoadBalancer
port: 3000
targetPort: 3000
annotations:
io.cilium/lb-ipam-ips: "192.168.1.165"
io.cilium/lb-ipam-sharing-key: "forgejo"
ssh:
type: LoadBalancer
port: 2222
targetPort: 2222
annotations:
io.cilium/lb-ipam-ips: "192.168.1.165"
io.cilium/lb-ipam-sharing-key: "forgejo"
resources:
requests:
cpu: 250m
memory: 512Mi
limits:
cpu: "1"
memory: 1Gi
tolerations:
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
extraVolumes:
- name: homelab-ca
configMap:
name: homelab-ca
extraVolumeMounts:
- name: homelab-ca
mountPath: /homelab-ca
readOnly: true
ingress:
enabled: false
postgresql:
enabled: false
postgresql-ha:
enabled: false
mysql:
enabled: false
redis-cluster:
enabled: false
act_runner:
enabled: false
destination:
server: https://kubernetes.default.svc
namespace: cicd
syncPolicy:
# NO automated sync - Forgejo hosts the repo; auto-sync would let a bad
# CI commit break the system CI depends on. Manual sync only.
syncOptions: []
@@ -1,89 +0,0 @@
# Wait-for-databases Job - ensures Database CRs are reconciled before apps start
# This solves the race condition where Forgejo starts before CNPG creates the database
apiVersion: batch/v1
kind: Job
metadata:
name: wait-for-databases
namespace: ddb
annotations:
description: "Waits for CNPG to reconcile Database CRs and create databases in PostgreSQL"
spec:
backoffLimit: 5
template:
metadata:
name: wait-for-databases
spec:
restartPolicy: Never
serviceAccountName: wait-for-databases
containers:
- name: wait
image: bitnami/kubectl:latest
command:
- /bin/bash
- -c
- |
set -euo pipefail
echo "==> Waiting for CNPG Database CRs to be reconciled..."
DATABASES="forgejo authentik temporal temporal-visibility"
for db in $DATABASES; do
echo "Checking database: $db"
for i in {1..60}; do
# Check if Database CR exists and is ready
READY=$(kubectl get database $db -n ddb -o jsonpath='{.status.ready}' 2>/dev/null || echo "false")
if [ "$READY" = "true" ]; then
echo " ✓ $db is ready"
break
fi
echo " Waiting for $db to be ready... ($i/60)"
sleep 5
if [ $i -eq 60 ]; then
echo " ✗ Timeout waiting for $db"
exit 1
fi
done
done
echo ""
echo "==> All databases are ready!"
echo "CNPG has created the following databases:"
kubectl get databases -n ddb
echo ""
echo "✅ Safe to deploy applications now"
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: wait-for-databases
namespace: ddb
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: wait-for-databases
namespace: ddb
rules:
- apiGroups: ["postgresql.cnpg.io"]
resources: ["databases"]
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: wait-for-databases
namespace: ddb
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: wait-for-databases
subjects:
- kind: ServiceAccount
name: wait-for-databases
namespace: ddb
-56
View File
@@ -1,56 +0,0 @@
# ingress-nginx - required for Forgejo domain access before ArgoCD can sync
# This breaks the circular dependency: ArgoCD needs https://forgejo.riotpiao.com
# but that domain requires ingress-nginx to route traffic.
apiVersion: v1
kind: Namespace
metadata:
name: ingress-nginx
labels:
pod-security.kubernetes.io/enforce: privileged
pod-security.kubernetes.io/audit: privileged
pod-security.kubernetes.io/warn: privileged
---
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: ingress-nginx-bootstrap
namespace: argocd
annotations:
argocd.argoproj.io/sync-wave: "0"
description: "Bootstrap ingress-nginx to enable Forgejo domain access"
spec:
project: homelab
source:
repoURL: https://kubernetes.github.io/ingress-nginx
chart: ingress-nginx
targetRevision: "4.15.1"
helm:
values: |
controller:
kind: DaemonSet
service:
type: LoadBalancer
annotations:
io.cilium/lb-ipam-ips: "192.168.1.160"
hostPort:
enabled: false
tolerations:
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: 500m
memory: 512Mi
destination:
server: https://kubernetes.default.svc
namespace: ingress-nginx
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true
-24
View File
@@ -1,24 +0,0 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
metadata:
name: bootstrap-local
annotations:
description: |
Phase 0 bootstrap bundle - apply once from local checkout on a fresh cluster.
Contains only resources that have circular git dependencies (Forgejo hosts
the repo ArgoCD syncs from). Everything else is GitOps-managed via ArgoCD.
# Resources in strict dependency order
resources:
- 00-namespaces.yaml # Pre-create with PodSecurity labels
- 01-argocd.yaml # ArgoCD + SOPS plugin ConfigMap
- 02-cnpg-operator.yaml # CloudNativePG operator + CRDs
- 03-ddb-bootstrap.yaml # PostgreSQL cluster + Forgejo DB + Redis
- 05-wait-for-databases.yaml # Wait for CNPG to create databases
- 04-forgejo.yaml # Forgejo Helm chart (inline values)
- 06-ingress-nginx.yaml # Ingress for Forgejo domain access
# Notes:
# - SOPS age secret created via bootstrap.sh (not in git)
# - After bootstrap: git push → kubectl apply -k k8s/argocd/root → done
# - ALL future changes via git push (ArgoCD auto-syncs)
+128
View File
@@ -0,0 +1,128 @@
# Homelab Bootstrap — Single-Cluster, GitOps-Ready
**Run once manually, GitOps forever after.**
This bootstrap breaks the ArgoCD ↔ Forgejo circular dependency by:
1. Installing infrastructure in correct dependency order
2. Pointing ArgoCD at a GitHub mirror initially
3. Cutting over to Forgejo once healthy
4. Using Helm for reproducible installs
5. Ensuring ArgoCD adopts (not duplicates) bootstrap resources
## Prerequisites
- Talos cluster running (terraform applied)
- kubectl configured (`KUBECONFIG` points at cluster)
- Helm 3 installed
- SOPS age key at `~/.sops/homelab-age.key`
- GitHub mirror of this repo (for initial ArgoCD source)
## Directory Structure
```
bootstrap/
├── phase1-storage/ # Longhorn via Helm
├── phase2-cnpg/ # CNPG operator via Helm
├── phase3-forgejo/ # Forgejo DB + Forgejo via Helm
├── phase4-argocd/ # ArgoCD via Helm → GitHub initially
└── phase5-cutover/ # Switch ArgoCD source to Forgejo
```
## Usage
```bash
# From repo root:
./bootstrap.sh
# Or step-by-step:
./bootstrap.sh phase1 # Storage
./bootstrap.sh phase2 # CNPG
./bootstrap.sh phase3 # Forgejo
./bootstrap.sh phase4 # ArgoCD (GitHub mirror)
./bootstrap.sh phase5 # Cut over to Forgejo
```
## Design Principles
1. **DRY**: Helm values used by both bootstrap and ArgoCD
2. **Single Source of Truth**: Manifests match what ArgoCD will manage
3. **Idempotent**: Can re-run phases safely
4. **Adoption Ready**: Resources have `argocd.argoproj.io/sync-options: Prune=false`
5. **Dependency Ordered**: Each phase waits for previous to be Ready
## Phase Details
### Phase 1: Storage (Longhorn)
Installs Longhorn with:
- 3-node HA configuration
- Unified `longhorn` StorageClass (default)
- Special `longhorn-cnpg` StorageClass with postgres UID/GID mount options
- CSI plugin tolerations for control-plane nodes
**Source of Truth**: `phase1-storage/longhorn-values.yaml`
### Phase 2: CNPG Operator
Installs CloudNativePG operator with:
- CRD registration (blocks until CRD available)
- Webhook configuration
- Monitoring enabled
**Source of Truth**: `phase2-cnpg/cnpg-values.yaml`
### Phase 3: Forgejo Database + Forgejo
1. Creates `forgejo-db` CNPG Cluster
2. Waits for cluster Ready (PostgreSQL accepting connections)
3. Installs Forgejo via Helm pointing at `forgejo-db-rw` service
4. Waits for Forgejo healthy
**Source of Truth**:
- `phase3-forgejo/forgejo-db.yaml` (CNPG Cluster CR)
- `phase3-forgejo/forgejo-values.yaml` (Helm values)
### Phase 4: ArgoCD (GitHub Mirror)
Installs ArgoCD via Helm, then applies root app-of-apps pointing at **GitHub mirror**.
This is the circle-breaker: ArgoCD syncs from GitHub (not Forgejo) initially.
**Source of Truth**:
- `phase4-argocd/argocd-values.yaml`
- `phase4-argocd/root-app-github.yaml` (repoURL = GitHub)
ArgoCD **adopts** Phases 1-3 resources (no duplication) because manifests match.
### Phase 5: Cut Over to Forgejo
1. Push repo to Forgejo
2. Update root app `repoURL` from GitHub → Forgejo
3. ArgoCD re-syncs from Forgejo
**The circle is broken. GitHub mirror is now disaster recovery only.**
## Post-Bootstrap
All changes via Git:
```bash
git commit -m "feat(app): add new service"
git push forgejo main
# ArgoCD auto-syncs
```
## Troubleshooting
- **Phase stuck?** Check `kubectl get events -n <namespace> --sort-by='.lastTimestamp'`
- **ArgoCD duplicating?** Verify manifests match exactly (Helm values ↔ ArgoCD Application)
- **Forgejo won't start?** Check CNPG cluster Ready: `kubectl get cluster forgejo-db -n forgejo`
- **Can't push to Forgejo?** Verify ingress-nginx healthy, DNS resolves `forgejo.riotpiao.com`
## Migration from Old Bootstrap
If you have existing `k8s/bootstrap-local/`:
1. **DO NOT delete** existing resources (Longhorn data!)
2. Run refined bootstrap in "adoption mode" (no delete, just apply)
3. Verify ArgoCD shows "Synced" for all apps
4. Archive old bootstrap: `git mv k8s/bootstrap-local k8s/archive/bootstrap-local-v1`
@@ -0,0 +1,56 @@
# Longhorn Helm Values — Single Source of Truth
# Used by both bootstrap.sh (Helm install) and ArgoCD (adoption)
# Chart: https://github.com/longhorn/charts
defaultSettings:
# 3-node HA configuration
replicaReplenishmentWaitInterval: 600 # 10min before auto-repair
replicaSoftAntiAffinity: false # REQUIRED for true HA
replicaAutoBalance: best-effort
storageMinimalAvailablePercentage: 10
# Performance tuning
defaultDataPath: /var/lib/longhorn
defaultDataLocality: best-effort
backupTarget: "" # TODO: Add MinIO backup target later
# Monitoring
guaranteedEngineManagerCPU: 12 # mCPU
guaranteedReplicaManagerCPU: 12
persistence:
defaultClass: true # Make 'longhorn' the default StorageClass
defaultClassReplicaCount: 3
defaultFsType: ext4
reclaimPolicy: Delete
# CSI plugin must tolerate control-plane taints
csi:
kubeletRootDir: /var/lib/kubelet
attacherReplicaCount: 3
provisionerReplicaCount: 3
resizerReplicaCount: 3
snapshotterReplicaCount: 3
# Longhorn manager on all nodes
longhornManager:
tolerations:
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
# Driver deployer tolerations
longhornDriver:
tolerations:
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
# UI for debugging
longhornUI:
replicas: 1
# Monitoring (Prometheus ServiceMonitor)
metrics:
serviceMonitor:
enabled: true
@@ -0,0 +1,27 @@
# StorageClasses — Applied after Longhorn installation
# The default 'longhorn' SC is created by Helm chart
# These are additional specialized classes
---
# CNPG-specific StorageClass with postgres UID/GID mount options
# Fixes "read-only filesystem" error when PostgreSQL (UID 26) tries to write
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: longhorn-cnpg
annotations:
storageclass.kubernetes.io/is-default-class: "false"
argocd.argoproj.io/sync-options: Prune=false # Allow ArgoCD adoption
provisioner: driver.longhorn.io
allowVolumeExpansion: true
parameters:
numberOfReplicas: "3"
staleReplicaTimeout: "30"
fromBackup: ""
dataLocality: "best-effort"
fsType: "ext4"
mountOptions:
- "noatime"
- "uid=26" # postgres user
- "gid=26" # postgres group
reclaimPolicy: Delete
volumeBindingMode: Immediate
@@ -0,0 +1,55 @@
# CloudNativePG Operator Helm Values — Single Source of Truth
# Chart: https://github.com/cloudnative-pg/charts
# Basic operator configuration
replicaCount: 1
# CRDs must be installed (Helm default behavior)
crds:
create: true
# Webhook configuration
webhook:
port: 9443
mutating:
create: true
failurePolicy: Fail
validating:
create: true
failurePolicy: Fail
# Monitoring
monitoring:
podMonitorEnabled: true
grafanaDashboard:
create: false # We'll manage dashboards via ArgoCD later
# Resource limits for operator
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: 500m
memory: 512Mi
# Security context
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 1000
capabilities:
drop:
- ALL
# Tolerate control-plane taints
tolerations:
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
# Additional labels for ArgoCD adoption
commonLabels:
app.kubernetes.io/managed-by: Helm
argocd.argoproj.io/instance: cnpg-operator
@@ -0,0 +1,122 @@
# Forgejo PostgreSQL Database — CNPG Cluster CR
# This is the source of truth for Forgejo's database
# ArgoCD will adopt this (not recreate it)
---
apiVersion: v1
kind: Namespace
metadata:
name: forgejo
annotations:
argocd.argoproj.io/sync-options: Prune=false
---
apiVersion: postgresql.cnpg.io/v1
kind: Cluster
metadata:
name: forgejo-db
namespace: forgejo
annotations:
argocd.argoproj.io/sync-options: Prune=false # Let ArgoCD adopt, don't delete
labels:
app: forgejo-db
layer: data
spec:
instances: 3 # HA configuration
imageName: ghcr.io/cloudnative-pg/postgresql:16.2
bootstrap:
initdb:
database: forgejo
owner: forgejo
encoding: UTF8
localeCollate: C
localeCType: C
enableSuperuserAccess: false
# Resource limits per best practices
resources:
requests:
memory: "4Gi"
cpu: "1"
limits:
memory: "8Gi"
cpu: "2"
postgresql:
parameters:
# Tuned for 4-8GB RAM
shared_buffers: "1GB"
effective_cache_size: "3GB"
maintenance_work_mem: "512MB"
work_mem: "64MB"
max_connections: "100"
max_parallel_workers: "2"
max_parallel_workers_per_gather: "1"
# WAL
wal_buffers: "16MB"
checkpoint_completion_target: "0.9"
min_wal_size: "512MB"
max_wal_size: "2GB"
# Logging
log_destination: "csvlog"
log_directory: "/controller/log"
log_filename: "postgres"
storage:
size: 50Gi
storageClass: longhorn-cnpg # Uses postgres UID/GID mount options
monitoring:
enablePodMonitor: true
affinity:
podAntiAffinityType: required
topologyKey: kubernetes.io/hostname
---
# Forgejo Redis (cache, session, queue)
apiVersion: v1
kind: Service
metadata:
name: forgejo-redis
namespace: forgejo
annotations:
argocd.argoproj.io/sync-options: Prune=false
spec:
ports:
- port: 6379
targetPort: 6379
protocol: TCP
selector:
app: forgejo-redis
type: ClusterIP
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: forgejo-redis
namespace: forgejo
annotations:
argocd.argoproj.io/sync-options: Prune=false
spec:
replicas: 1
selector:
matchLabels:
app: forgejo-redis
template:
metadata:
labels:
app: forgejo-redis
spec:
containers:
- name: redis
image: redis:7-alpine
ports:
- containerPort: 6379
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 200m
memory: 256Mi
@@ -0,0 +1,86 @@
# Forgejo Helm Values — Single Source of Truth
# Chart: https://codeberg.org/forgejo-contrib/forgejo-helm
gitea:
admin:
username: "admin"
email: "[email protected]"
# Password set via secret (not in values)
config:
server:
DOMAIN: forgejo.riotpiao.com
ROOT_URL: https://forgejo.riotpiao.com
SSH_DOMAIN: forgejo.riotpiao.com
SSH_PORT: 22
database:
DB_TYPE: postgres
HOST: forgejo-db-rw.forgejo.svc.cluster.local:5432
NAME: forgejo
# User/password from CNPG-generated secret
USER:
valueFrom:
secretKeyRef:
name: forgejo-db-app
key: username
PASSWD:
valueFrom:
secretKeyRef:
name: forgejo-db-app
key: password
cache:
ADAPTER: redis
HOST: redis://forgejo-redis.forgejo.svc.cluster.local:6379/0
session:
PROVIDER: redis
PROVIDER_CONFIG: redis://forgejo-redis.forgejo.svc.cluster.local:6379/1
queue:
TYPE: redis
CONN_STR: redis://forgejo-redis.forgejo.svc.cluster.local:6379/2
# Persistence (shared storage for repos)
persistence:
enabled: true
storageClass: longhorn
size: 20Gi
accessModes:
- ReadWriteOnce
# Ingress
ingress:
enabled: true
className: nginx
annotations:
cert-manager.io/cluster-issuer: letsencrypt-prod
hosts:
- host: forgejo.riotpiao.com
paths:
- path: /
pathType: Prefix
tls:
- secretName: forgejo-tls
hosts:
- forgejo.riotpiao.com
# Resources
resources:
requests:
cpu: 200m
memory: 512Mi
limits:
cpu: 1000m
memory: 2Gi
# Tolerations for control-plane
tolerations:
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
# ArgoCD adoption labels
labels:
argocd.argoproj.io/instance: forgejo
@@ -0,0 +1,124 @@
# ArgoCD Helm Values — Single Source of Truth
# Chart: https://github.com/argoproj/argo-helm
global:
domain: argocd.riotpiao.com
# Server configuration
server:
ingress:
enabled: true
ingressClassName: nginx
annotations:
cert-manager.io/cluster-issuer: letsencrypt-prod
nginx.ingress.kubernetes.io/ssl-passthrough: "true"
nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
hosts:
- argocd.riotpiao.com
tls:
- secretName: argocd-server-tls
hosts:
- argocd.riotpiao.com
# Allow insecure mode (terminate TLS at ingress)
extraArgs:
- --insecure
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: 500m
memory: 1Gi
# Repo server configuration
repoServer:
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: 500m
memory: 1Gi
# SOPS plugin for encrypted secrets
volumes:
- name: sops-age
secret:
secretName: sops-age
optional: true
volumeMounts:
- name: sops-age
mountPath: /home/argocd/.config/sops/age
readOnly: true
# Environment for SOPS
env:
- name: SOPS_AGE_KEY_FILE
value: /home/argocd/.config/sops/age/keys.txt
# Controller configuration
controller:
resources:
requests:
cpu: 200m
memory: 512Mi
limits:
cpu: 1000m
memory: 2Gi
# Application controller configuration
applicationSet:
enabled: true
# Notifications (optional, for Slack/Discord alerts)
notifications:
enabled: false
# Redis for caching
redis:
enabled: true
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 200m
memory: 256Mi
# Tolerations for control-plane
server:
tolerations:
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
repoServer:
tolerations:
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
controller:
tolerations:
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
# ArgoCD projects
configs:
# Default project allows all repos
cm:
admin.enabled: "true"
application.instanceLabelKey: argocd.argoproj.io/instance
params:
server.insecure: true
# RBAC (allow admin full access)
configs:
rbac:
policy.default: role:readonly
policy.csv: |
g, admin, role:admin
@@ -0,0 +1,66 @@
# ArgoCD Root App-of-Apps — GitHub Mirror Source
# This is the initial configuration that breaks the circular dependency
# Points at GitHub mirror, not Forgejo (Forgejo isn't ready yet)
#
# After Forgejo is healthy and repo is pushed, use phase5-cutover/root-app-forgejo.yaml
---
apiVersion: v1
kind: Namespace
metadata:
name: argocd
---
apiVersion: argoproj.io/v1alpha1
kind: AppProject
metadata:
name: homelab
namespace: argocd
spec:
description: Homelab infrastructure and applications
sourceRepos:
- 'https://github.com/YOUR-ORG/homelab.git' # ← REPLACE with your GitHub mirror
- 'https://forgejo.riotpiao.com/YOUR-ORG/homelab.git'
- 'https://*.github.io/*' # Helm charts from GitHub Pages
- 'https://charts.*' # Public Helm repos
destinations:
- namespace: '*'
server: 'https://kubernetes.default.svc'
clusterResourceWhitelist:
- group: '*'
kind: '*'
namespaceResourceWhitelist:
- group: '*'
kind: '*'
---
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: homelab-root
namespace: argocd
finalizers:
- resources-finalizer.argocd.argoproj.io
spec:
project: homelab
source:
repoURL: https://github.com/YOUR-ORG/homelab.git # ← REPLACE with your GitHub mirror
targetRevision: main
path: k8s/argocd/apps
destination:
server: https://kubernetes.default.svc
namespace: argocd
syncPolicy:
automated:
prune: true
selfHeal: true
allowEmpty: false
syncOptions:
- CreateNamespace=true
- ServerSideApply=true
retry:
limit: 5
backoff:
duration: 5s
factor: 2
maxDuration: 3m
@@ -0,0 +1,39 @@
# ArgoCD Root App-of-Apps — Forgejo Source (Final State)
# This replaces the GitHub mirror with Forgejo as the source of truth
# Apply this AFTER Forgejo is healthy and you've pushed the repo
#
# The circle is broken: ArgoCD → Forgejo works because both already exist
---
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: homelab-root
namespace: argocd
finalizers:
- resources-finalizer.argocd.argoproj.io
spec:
project: homelab
source:
repoURL: https://forgejo.riotpiao.com/YOUR-ORG/homelab.git # ← REPLACE with your Forgejo URL
targetRevision: main
path: k8s/argocd/apps
destination:
server: https://kubernetes.default.svc
namespace: argocd
syncPolicy:
automated:
prune: true
selfHeal: true
allowEmpty: false
syncOptions:
- CreateNamespace=true
- ServerSideApply=true
retry:
limit: 5
backoff:
duration: 5s
factor: 2
maxDuration: 3m
@@ -3,6 +3,7 @@ kind: Kustomization
namespace: longhorn-system
resources:
- longhorn-storageclass.yaml
- longhorn-cnpg-storageclass.yaml # CNPG-specific with postgres UID/GID
- longhorn-servicemonitor.yaml
- longhorn-taint-toleration.yaml
- longhorn-nodes.yaml
@@ -0,0 +1,26 @@
# StorageClass specifically for CNPG (CloudNativePG) PostgreSQL clusters
# Fixes the "read-only filesystem" issue by mounting with postgres UID/GID
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: longhorn-cnpg
namespace: longhorn-system
annotations:
storageclass.kubernetes.io/is-default-class: "false"
provisioner: driver.longhorn.io
allowVolumeExpansion: true
parameters:
numberOfReplicas: "3"
staleReplicaTimeout: "30"
fromBackup: ""
dataLocality: "best-effort"
fsType: "ext4"
# Mount options to ensure PostgreSQL can write
mkfsParams: "-O ^64bit,^metadata_csum"
mountOptions:
- "noatime"
# Critical: mount with postgres UID/GID (26:26) to avoid permission issues
- "uid=26"
- "gid=26"
reclaimPolicy: Delete
volumeBindingMode: Immediate
+55
View File
@@ -0,0 +1,55 @@
# Continuous smoke — runs the full WebKit suite every 15 min. A failed run means
# something users touch broke; alert on it (Job failure → kube-state-metrics
# kube_job_status_failed → Alertmanager). Same image/env as the deploy-gate Job.
apiVersion: batch/v1
kind: CronJob
metadata:
name: e2e-smoke
namespace: platform
annotations:
argocd.argoproj.io/sync-options: Prune=false
spec:
schedule: "*/15 * * * *"
concurrencyPolicy: Forbid
successfulJobsHistoryLimit: 3
failedJobsHistoryLimit: 5
jobTemplate:
spec:
backoffLimit: 1
ttlSecondsAfterFinished: 86400
template:
spec:
restartPolicy: Never
tolerations:
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
containers:
- name: e2e
image: forgejo-gitea-http.cicd.svc.cluster.local:3000/riotpiao.com/homelab-e2e:latest
imagePullPolicy: Always
env:
- name: BASE_DOMAIN
value: riotpiao.com
- name: E2E_IGNORE_TLS
value: "0"
- name: CI
value: "1"
- name: AK_ADMIN_USER
value: akadmin
- name: AK_ADMIN_PASSWORD
valueFrom:
secretKeyRef: { name: e2e-credentials, key: authentik-admin-password }
- name: MINIO_ENDPOINT
value: http://minio.storage.svc.cluster.local:9000
- name: MINIO_BUCKET
value: e2e-artifacts
- name: MINIO_ACCESS_KEY
valueFrom:
secretKeyRef: { name: e2e-credentials, key: minio-access-key }
- name: MINIO_SECRET_KEY
valueFrom:
secretKeyRef: { name: e2e-credentials, key: minio-secret-key }
resources:
requests: { cpu: 200m, memory: 512Mi }
limits: { cpu: "1", memory: 2Gi }
+59
View File
@@ -0,0 +1,59 @@
# One-shot E2E smoke — run as a deploy verification gate.
# Wire as an ArgoCD PostSync hook (annotation below) OR call from an Argo Rollouts
# AnalysisTemplate. Job success == the app is actually viewable in Safari's engine;
# failure fails the sync/rollout.
#
# NOTE: not yet added to any kustomization / app-of-apps. Wiring = migration phase 7.
apiVersion: batch/v1
kind: Job
metadata:
name: e2e-smoke
namespace: platform
annotations:
# Uncomment to make this a deploy gate on the owning Application:
# argocd.argoproj.io/hook: PostSync
# argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
argocd.argoproj.io/sync-options: Prune=false
spec:
backoffLimit: 1
ttlSecondsAfterFinished: 86400
template:
spec:
restartPolicy: Never
tolerations:
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
containers:
- name: e2e
# Built + pushed by CI from tests/e2e/Dockerfile. Pin a digest in prod.
image: forgejo-gitea-http.cicd.svc.cluster.local:3000/riotpiao.com/homelab-e2e:latest
imagePullPolicy: Always
env:
- name: BASE_DOMAIN
value: riotpiao.com
# strict TLS by default; set "1" only during staging-cert bootstrap
- name: E2E_IGNORE_TLS
value: "0"
- name: CI
value: "1"
- name: AK_ADMIN_USER
value: akadmin
- name: AK_ADMIN_PASSWORD
valueFrom:
secretKeyRef:
name: e2e-credentials
key: authentik-admin-password
- name: MINIO_ENDPOINT
value: http://minio.storage.svc.cluster.local:9000
- name: MINIO_BUCKET
value: e2e-artifacts
- name: MINIO_ACCESS_KEY
valueFrom:
secretKeyRef: { name: e2e-credentials, key: minio-access-key }
- name: MINIO_SECRET_KEY
valueFrom:
secretKeyRef: { name: e2e-credentials, key: minio-secret-key }
resources:
requests: { cpu: 200m, memory: 512Mi }
limits: { cpu: "1", memory: 2Gi }