diff --git a/.gitignore b/.gitignore index 5e8863a..6ae447d 100644 --- a/.gitignore +++ b/.gitignore @@ -50,4 +50,6 @@ terraform/*.tfstate.* terraform.tfvars.local skills-lock.json secrets-plaintext.yaml -skills-lock.json \ No newline at end of file +skills-lock.json + +CLAUDE.md \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index a2945b7..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1,251 +0,0 @@ -# CLAUDE.md — Homelab Project Reference - -## Cluster Topology (3 control-plane HA) - -| Node | IP | Zone | Scheduling | Storage | -|------|----|----|-----------|---------| -| `talos-cp-1` | .213 | az-a | schedulable (all workloads) | Longhorn replica | -| `talos-cp-2` | .163 | az-b | dedicated (`NoSchedule`) | Longhorn replica | -| `talos-cp-3` | .166 | az-c | dedicated (`NoSchedule`) | Longhorn replica | - -3 voting etcd members peering on the LAN. All 3 nodes run Longhorn with 3-replica -HA storage (verified: all 17 PVCs have replicas across all nodes). Only `talos-cp-1` -is schedulable for workloads (cp-2/cp-3 are control-plane only). Full detail + gotchas -in `USAGE.md` and memory `reference_talos_etcd_and_ca_gotchas`. - -## Deployment Model: ArgoCD GitOps (helmfile is retired) - -**helmfile.yaml.gotmpl and the `core` CLI's helmfile-era workflows described in -`USAGE.md`/`project-usage/*.md` are STALE.** Actual practice, 100% of the time: - -``` -git commit → git push (Forgejo) → ArgoCD auto-sync → cluster -``` - -App-of-apps structure: `k8s/argocd/root/homelab-root.yaml` (root Application) -→ `k8s/argocd/apps/*.yaml` (one file per "wave" of Applications, each -`argocd.argoproj.io/sync-wave` annotated) → each Application points at either -a remote Helm chart (+ a second `ref: values` git source for the values file) -or a plain git directory of raw manifests. - -**Never `kubectl apply`/`patch`/`delete` a resource ArgoCD manages** except: -- Pure cleanup of stuck/dead state (delete a failed hook Job so the *next* - legitimate sync creates a fresh one — not a config change, just clearing - wreckage). Confirm with the user first if in doubt. -- One-time bootstrap actions with a genuine circular dependency (Vault - `operator init`/unseal — see `k8s/security/iam/VAULT-BOOTSTRAP-README.md` - equivalent scripts). - -## Hard Rules (Never Violate) - -🔴 **NEVER rename or wipe `talos-cp-1` (.213).** It is the sole Longhorn storage -node — all replicas are pinned to that node name. Renaming orphans its Longhorn -node CR and faults every volume (permanent data loss). Rename/reprovision only -the dedicated CPs (.163/.166), never the data node. - -🔴 **Control-plane etcd must advertise on the LAN.** Keep -`cluster.etcd.advertisedSubnets: ["192.168.1.0/24"]` in the controlplane -template — without it Talos advertises on the WireGuard IP and new members hang -as non-promoting etcd learners. - -🔴 **ALWAYS run `terraform fmt` after any terraform code changes.** Before commit: -```bash -terraform fmt -recursive terraform/ -``` -Verify no changes (clean output = formatted correctly). If files change, review diffs, commit fmt changes separately. Workflow's "Terraform Format Check" step will fail otherwise. - -🔴 **NEVER manually kubectl delete/patch resources managed by Terraform.** Terraform is the source of truth for IaC-managed resources (deployments, PVCs, services in Terraform-controlled namespaces). Manual edits create state drift. If a resource is stuck: -1. Update Terraform config (variables.tf, *.tf files) -2. Run `terraform apply` (locally or via CI/CD) -3. Never bypass with manual kubectl operations - -🔴 **NEVER delete a PVC unless there are replicas or backups.** A PVC deletion = permanent data loss. Verify replication status first. - -🔴 **NO co-authored commit messages.** All commits are solo work. Never append `Co-Authored-By:` footer. - -🔴 **Commit message format:** First line must capture what is added, what is fixed, what is removed, and why in one sentence. Example: `fix(kubernetes): use direct in-cluster auth in providers — removes kubeconfig file dependency in CI runner`. No multi-paragraph messages. - -🔴 **Git workflow: rebase only, no pull/merge.** Always rebase when pulling. Use `git pull --rebase` or `git rebase main` before pushing. Keep history linear. - -🔴 **Long-running commands (>10s) must run async.** Use `run_in_background: true` for Bash or spawn Agent. Don't actively wait. Prevents blocking on terraform plan, kubectl apply, downloads. - -🔴 **Infrastructure changes should flow through GitOps when possible:** git commit → push → CI/CD runner (terraform apply) → ArgoCD sync. Local `terraform apply` is permitted (e.g. for local iteration, config regeneration, or when CI/CD isn't wired up for a given module) — still commit + push the resulting state/config afterward so git remains the record of truth. Manual `kubectl apply` remains disallowed for Terraform-managed resources. - -## CloudNativePG (CNPG) Database Pattern - -**Simple ownership model — all apps use shared 'app' user:** - -```yaml -# CNPG Cluster (bootstrap, wave 0) -bootstrap: - initdb: - database: app # Bootstrap database - owner: app # Bootstrap user (owns all databases) - -# Database CR (per-app, wave 6) -spec: - name: authentik # Database name - owner: app # ← All apps use 'app' (not per-app roles) - cluster: - name: ddb-cluster -``` - -**Credential Distribution (bootstrap.sh pattern):** - -1. **Source of truth:** CNPG creates `ddb-cluster-app` secret in `ddb` namespace -2. **Distribution:** `bootstrap.sh` copies secret to app namespaces: - ```bash - # For each app namespace (cicd, iam, etc): - kubectl get secret ddb-cluster-app -n ddb -o yaml \ - | sed 's/namespace: ddb/namespace: /' \ - | kubectl apply -f - - ``` -3. **Apps reference local copy:** - ```yaml - env: - - name: DB_USER - valueFrom: - secretKeyRef: - name: ddb-cluster-app # Local copy in app's namespace - key: username # Always "app" - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: ddb-cluster-app - key: password - ``` - -**Key Points:** - -✅ **DO:** -- All apps connect as `app` user -- Database CRs specify `owner: app` -- Isolation via separate database names (not roles) -- Copy credentials to app namespace via `bootstrap.sh` -- Reference local secret copy via `secretKeyRef` - -❌ **DON'T:** -- Create per-app roles in `managed.roles` (CNPG doesn't transfer ownership properly) -- Grant permissions via PostSync Jobs (owner already has full rights) -- Use cross-namespace `secretKeyRef` (not supported) -- Manually patch secrets (ArgoCD will revert) - -**Working Examples:** -- Forgejo: `k8s/bootstrap-local/04-forgejo.yaml` (references `ddb-cluster-app` in cicd namespace) -- Authentik: `k8s/security/iam/authentik-values.yaml` (references `ddb-cluster-app` in iam namespace) - -**For New Apps:** -1. Add Database CR to `k8s/data/schemas/` with `owner: app` -2. Add secret copy to `bootstrap.sh` (like cicd/iam examples) -3. Reference `ddb-cluster-app` via `secretKeyRef` in app's namespace -4. No permission grants needed — app user owns the database - -## GitOps / ArgoCD Gotchas (hard-won, all confirmed live in this cluster) - -🟠 **`kustomization.yaml` with an explicit `resources:` allowlist silently -drops anything not listed — no error, no drift shown.** ArgoCD reports -`Synced/Healthy` against a manifest set that never included the missing -file at all. Symptom: you commit+push a new manifest, ArgoCD says -"Synced", but the resource never appears in-cluster. Fix: check every -`kustomization.yaml` along the app's source path actually lists your new -file. `kubectl kustomize /` locally reproduces exactly what ArgoCD -will apply — always verify with it before pushing. - -🟠 **A `kustomization.yaml` top-level `namespace:` transformer rewrites -`metadata.namespace` on *every* resource it builds — including RBAC -RoleBindings deliberately targeting a *different* namespace.** If any -manifest in that directory needs cross-namespace resources (e.g. a -RoleBinding granting access to Secrets in another namespace), either drop -the transformer (safe if every resource already sets its own explicit -namespace) or move that manifest to its own directory/Application. - -🟠 **PreSync hooks run *before* an Application's own normal (non-hook) -resources are synced.** A PreSync-hooked Job that depends on a -ServiceAccount/RBAC defined as plain resources in the *same* Application -deadlocks: the Job tries to start before its own ServiceAccount exists. -Confirmed live — Job sat "Running" for 14+ minutes producing zero pods, -`job-controller` event log showed `serviceaccount ... not found` on every -retry. Fix: use PostSync instead (runs after that app's own resources are -applied), or split the hook into its own earlier-sync-wave Application. - -🟠 **ArgoCD hooks (PreSync/PostSync) are NOT continuously reconciled by -`selfHeal` the way normal resources are.** Once a hook Job completes -(success or exhausts `backoffLimit` into Failed), it only gets -deleted+recreated (per `hook-delete-policy: BeforeHookCreation`) during an -**actual new Sync operation** — not from passive drift detection, even -with `automated.selfHeal: true`. If you fix a hook Job's spec (image, -command, RBAC) and push, `status.sync.revision` may show "caught up" while -the *live* hook resource is still running the old, broken spec — because -no new operation actually re-ran it. To force a real resync: delete the -stuck Job (clear the `argocd.argoproj.io/hook-finalizer` if it's stuck -`Terminating`), and if that alone doesn't trigger a fresh full sync, delete -+ `kubectl apply -f` the Application object itself (re-reads current git -HEAD, starts a genuinely new operation, no cascade-delete of underlying -resources since Applications don't carry a cascade finalizer by default — -confirm with `kubectl get app -o jsonpath='{.metadata.finalizers}'` -first). - -🟠 **ArgoCD's repo-server caches rendered manifests (~120s TTL by default, -`argocd-cm` → `timeout.reconciliation`).** If you change a values file and -`Application` fields both, sometimes the old rendering wins on the next -sync. Restart `argocd-repo-server` after big multi-source/values changes if -sync behavior looks stale. - -🟠 **ArgoCD `repoURL` pointing at a hostname that CoreDNS rewrites to the -nginx ingress controller (for TLS termination) breaks if the URL includes -a non-standard port.** nginx only listens on 80/443 — `http://host:3000/...` -silently times out (`context deadline exceeded`) if `host` resolves to the -ingress controller, not the actual backend service. This blocked **every -single Application's sync** cluster-wide simultaneously (all showed -`Unknown` sync status) because the repo-server couldn't fetch git refs at -all. Use `https://host/...` (no port) so nginx's default TLS cert + normal -443 routing handles it. - -🟠 **Bitnami Docker Hub images no longer publish versioned tags (2025 -policy change) — only `latest` and sha256-pinned digests remain for their -free tier.** A pinned tag like `bitnami/kubectl:1.30` will 404/ImagePullBackOff -forever. Verify tags exist first: `curl -s "https://hub.docker.com/v2/repositories///tags?page_size=25" | jq -r '.results[].name'`. -Prefer avoiding third-party utility images entirely where possible — e.g. -`python:3.12-alpine` + stdlib `urllib.request` to fetch a static binary -(kubectl) avoids depending on any registry's tagging policy at all. - -🟠 **Non-root containers (`runAsNonRoot: true`, non-zero UID) can't `apk -add` in Alpine-based images** — apk's working directories and most of -`/usr/local/bin` are root-owned. Symptom: `ERROR: Unable to open log: -Permission denied`. Use `/tmp` (world-writable) for any binary you need to -download/install at runtime, and extend `PATH` rather than writing to -`/usr/local/bin`. - -🟠 **Helm does not validate unknown `values.yaml` keys — a typo'd or -wrong-schema key is silently a no-op, not an error.** Confirmed root cause -of a multi-week "Temporal doesn't support PostgreSQL" belief: the actual -chart version pinned (`temporalio/helm-charts@0.74.0`) uses a flat -`server.config.persistence..driver/.sql` schema, but the values file -used the *newer* chart's `datastores:`-wrapped schema (introduced in a -later major version) — silently ignored, so persistence stayed on the -chart's Cassandra default the entire time. **Before assuming "this chart -doesn't support X," clone the chart at the exact pinned tag/version and run -`helm template` with your real values — diff the rendered output, don't -trust values.yaml comments/examples from the chart's current `main` -branch, which may not match your pinned version's schema at all.** - -## Workflow - -- **Modify** → **Format** (`terraform fmt`) → **Apply** (locally or let CI/CD do it) → **Commit** → **Push** -- If fmt check fails in CI, fix locally, commit fmt changes, push again -- For k8s/ManifestsChanges: **Modify** → **validate** (`kubectl apply --dry-run=client -f`, or `kubectl kustomize /` if a `kustomization.yaml` is involved, or `helm template` against the exact pinned chart version for Helm-sourced Applications) → **Commit** → **Push** → confirm ArgoCD picked it up (check `status.sync.revision` matches your commit, not just `status.sync.status`) - -## Documentation Map - -- `CLAUDE.md` (this file) — private, cluster-specific, always current -- `CLAUDE.example.md` — sanitized, hardware-generic template for reuse on other 3-node Talos clusters; update alongside this file when a lesson is genuinely hardware/topology-generic (not homelab-specific secrets/IPs) -- `USAGE.md` / `project-usage/*.md` — **STALE, helmfile-era.** Written for a - deprecated `helmfile apply` + `core iam`/`core secrets` CLI workflow that - does not reflect actual current practice (100% ArgoCD GitOps as of this - writing). Treat as historical reference only until rewritten; do not - follow their deployment procedures literally. -- `TROUBLESHOOTING.md` — generic Kubernetes SRE layer-before-tool methodology, still broadly applicable regardless of deployment mechanism - ---- - -**Last updated:** 2026-07-23 diff --git a/bootstrap.sh b/bootstrap.sh deleted file mode 100755 index 0e429fd..0000000 --- a/bootstrap.sh +++ /dev/null @@ -1,142 +0,0 @@ -#!/usr/bin/env bash -# -# Homelab Cluster Bootstrap (Phase 0) -# Bootstrap a fresh Talos cluster to GitOps-ready state (ArgoCD + Forgejo). -# Run once from local checkout, then all future changes via git push. -# -# Prerequisites: -# - Talos cluster up (terraform apply completed) -# - kubectl configured (KUBECONFIG points at cluster) -# - SOPS age key at ~/.sops/homelab-age.key -# - ArgoCD CLI installed (for final sync) -# -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -K8S_DIR="$SCRIPT_DIR/k8s" -BOOTSTRAP_DIR="$K8S_DIR/bootstrap-local" -SOPS_KEY="${SOPS_KEY:-$HOME/.sops/homelab-age.key}" - -# Colors -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -log() { echo -e "${GREEN}[$(date +'%H:%M:%S')]${NC} $*"; } -warn() { echo -e "${YELLOW}[$(date +'%H:%M:%S')]${NC} $*"; } -error() { echo -e "${RED}[$(date +'%H:%M:%S')]${NC} $*"; exit 1; } - -# Preflight checks -log "Running preflight checks..." -kubectl cluster-info > /dev/null || error "kubectl not configured or cluster unreachable" -[[ -f "$SOPS_KEY" ]] || error "SOPS age key not found at $SOPS_KEY" -command -v argocd > /dev/null || warn "ArgoCD CLI not found - manual sync required at end" - -# 1. Install ArgoCD itself (if not already present) -if ! kubectl get namespace argocd &>/dev/null; then - log "Installing ArgoCD..." - kubectl create namespace argocd - kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml - log "Waiting for ArgoCD to be ready..." - kubectl wait --for=condition=available --timeout=300s deployment/argocd-server -n argocd -else - log "ArgoCD already installed, skipping..." -fi - -# 2. Create SOPS age secret (NEVER commit this to git) -log "Creating SOPS age secret..." -kubectl create namespace argocd --dry-run=client -o yaml | kubectl apply -f - -kubectl create secret generic sops-age \ - -n argocd \ - --from-file=keys.txt="$SOPS_KEY" \ - --dry-run=client -o yaml | kubectl apply -f - - -# 3. Apply bootstrap bundle (namespaces, CNPG, DDB, Forgejo) -log "Applying bootstrap bundle..." -kubectl apply -k "$BOOTSTRAP_DIR" --server-side - -# 4. Wait for CNPG operator -log "Waiting for CNPG operator..." -kubectl wait --for=condition=available --timeout=300s \ - deployment/cnpg-controller-manager -n ddb 2>/dev/null || { - warn "CNPG operator not found - checking if it exists as different deployment name..." - kubectl get deployments -n ddb -} - -# 5. Wait for DDB cluster -log "Waiting for PostgreSQL cluster (ddb-cluster) to be ready..." -for i in {1..60}; do - STATUS=$(kubectl get cluster ddb-cluster -n ddb -o jsonpath='{.status.phase}' 2>/dev/null || echo "NotFound") - if [[ "$STATUS" == "Cluster in healthy state" ]]; then - log "DDB cluster is ready!" - break - fi - [[ $i -eq 60 ]] && error "Timeout waiting for ddb-cluster" - sleep 5 -done - -# 6. Copy DB secret from ddb to cicd namespace -log "Copying ddb-cluster-app secret to cicd namespace..." -kubectl get secret ddb-cluster-app -n ddb -o yaml \ - | sed 's/namespace: ddb/namespace: cicd/' \ - | kubectl apply -f - - -# Copy DB secret to iam namespace (for authentik) -log "Copying ddb-cluster-app secret to iam namespace..." -kubectl get secret ddb-cluster-app -n ddb -o yaml \ - | sed 's/namespace: ddb/namespace: iam/' \ - | kubectl apply -f - - - -# Copy DB secret to temporal namespace (for temporal) -log "Copying ddb-cluster-app secret to temporal namespace..." -kubectl get secret ddb-cluster-app -n ddb -o yaml \ - | sed 's/namespace: ddb/namespace: temporal/' \ - | kubectl apply -f - -# 7. Wait for Forgejo -log "Waiting for Forgejo to be ready..." -kubectl wait --for=condition=available --timeout=600s \ - deployment/forgejo -n cicd 2>/dev/null || { - warn "Forgejo not found as deployment - checking StatefulSet..." - kubectl wait --for=condition=available --timeout=600s \ - statefulset/forgejo -n cicd || warn "Could not find Forgejo - check manually" -} - -# 8. Apply Longhorn 3-node configuration (if it exists) -if [[ -d "$K8S_DIR/infrastructure/longhorn" ]]; then - log "Applying Longhorn 3-node HA configuration..." - kubectl apply -k "$K8S_DIR/infrastructure/longhorn/" || warn "Longhorn config failed - may need manual intervention" -else - warn "Longhorn config not found at k8s/infrastructure/longhorn/ - storage may be single-node only!" -fi - -# 9. Get Forgejo LoadBalancer IP -FORGEJO_IP=$(kubectl get svc forgejo-http -n cicd -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null || echo "unknown") -log "Forgejo available at: http://$FORGEJO_IP:3000 (or https://forgejo.riotpiao.com)" - -echo "" -log "${GREEN}========================================${NC}" -log "${GREEN}✅ Bootstrap Complete!${NC}" -log "${GREEN}========================================${NC}" -echo "" -echo "Next steps:" -echo "" -echo " 1. Push this repo to Forgejo:" -echo " git remote add forgejo https://forgejo.riotpiao.com/riotpiao.com/homelab.git" -echo " git push forgejo main" -echo "" -echo " 2. Apply app-of-apps root:" -echo " kubectl apply -k k8s/argocd/root" -echo "" -echo " 3. VERIFY STORAGE REPLICATION (CRITICAL!):" -echo " See STORAGE-ARCHITECTURE-CLARIFICATION.md" -echo " kubectl get nodes.longhorn.io -n longhorn-system" -echo " kubectl get volumes.longhorn.io -n longhorn-system -o wide" -echo "" -echo " 4. Sync all applications:" -echo " argocd app sync homelab-root" -echo " # Or via UI: https://argocd.riotpiao.com" -echo "" -echo " 5. All future changes: git commit → git push (ArgoCD auto-syncs)" -echo "" diff --git a/k8s/applications/temporal/temporal-values.yaml.bak b/k8s/applications/temporal/temporal-values.yaml.bak deleted file mode 100644 index c48acae..0000000 --- a/k8s/applications/temporal/temporal-values.yaml.bak +++ /dev/null @@ -1,140 +0,0 @@ -# k8s/temporal/temporal-values.yaml -# Temporal — workflow engine -# Uses external CNPG PostgreSQL for persistence (ddb-cluster) -# Visibility via same PostgreSQL instance, separate database. -# -# IMPORTANT — chart schema note (root-caused after Postgres never actually -# taking effect despite looking configured): -# We're pinned to temporalio/helm-charts @ 0.74.0 (see targetRevision in -# k8s/argocd/apps/60-applications.yaml), which uses the OLD flat persistence -# schema: -# server.config.persistence..driver: "sql"|"cassandra" -# server.config.persistence..sql: {...} -# NOT the newer `datastores:`-wrapped schema -# (server.config.persistence.datastores..sql) shown in the current -# chart's values/values.postgresql.yaml example - that key was introduced in -# a later major version and doesn't exist in 0.74.0. Helm doesn't validate -# unknown keys, so a `datastores:` block here is silently a no-op: Temporal -# would keep defaulting to Cassandra (with empty hosts: []) regardless of -# anything nested inside it. Verified via `helm template` against the actual -# 0.74.0 chart before writing this file - see chat history for the -# side-by-side proof (rendered manifest showed CASSANDRA_HOST env vars and -# temporal-cassandra-tool commands using the old datastores:-based values). -# -# Likewise `schema.setup.enabled` / `schema.update.enabled` / -# `schema.createDatabase.enabled` are the real toggles for the schema-setup -# Job (all default true) - there is no `jobs.autoSetup` key in this chart. - -# ── Disable every bundled/optional sub-chart ───────────────────────────────── -# postgresql/mysql: never enable - we never want the chart to deploy its own -# DB, only to know how to talk to our external CNPG instance (which happens -# via server.config.persistence.*.sql below, independent of these flags). -postgresql: - enabled: false -mysql: - enabled: false -cassandra: - enabled: false -elasticsearch: - enabled: false -prometheus: - enabled: false -grafana: - enabled: false - -# ── Schema setup/update Jobs ────────────────────────────────────────────────── -# The `temporal` and `temporal_visibility` databases are provisioned -# declaratively by CNPG Database CRs (k8s/data/temporal-database.yaml, -# temporal-visibility-database.yaml), so createDatabase stays disabled (the -# `temporal` role also lacks CREATEDB). setup/update run temporal-sql-tool as -# the `temporal` owner against those existing DBs to install and migrate the -# Temporal server schema — without them both DBs have zero tables and the -# server dies on "no usable database connection found" (no schema_version row). -schema: - createDatabase: - enabled: false - setup: - enabled: true - update: - enabled: true - -# ── Temporal server config (PostgreSQL persistence) ────────────────────────── -server: - replicaCount: 1 - # temporalio/server:1.30.0+ dropped the `dockerize` binary and switched to - # built-in sprig config templating. The chart still defaults to the legacy - # configMapsToMount: "dockerize" + setConfigFilePath: false, which produces a - # config the 1.30 server never loads — it then falls back to its embedded - # env-only template (Cassandra default) and dies with - # "Persistence.DataStores[default](value).Cassandra.Hosts: zero value". - # Switch to the sprig ConfigMap and point the server at it (chart's own - # recommendation for 1.30.0+ images; sprig mode requires setConfigFilePath). - configMapsToMount: "sprig" - setConfigFilePath: true - jobService: - enabled: false - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchLabels: - app.kubernetes.io/instance: temporal - topologyKey: kubernetes.io/hostname - config: - logLevel: "info" - persistence: - defaultStore: default - visibilityStore: visibility - numHistoryShards: 512 - default: - driver: "sql" - sql: - driver: "postgres12" - host: "ddb-cluster-rw.ddb.svc.cluster.local" - port: 5432 - database: "temporal" - user: "temporal" - # existingSecret + secretKey: point directly at the CNPG-generated - # Secret (kubernetes.io/basic-auth, keys: username/password/...) - # rather than duplicating the password in git as plaintext. When - # existingSecret is set the chart's own server-secret.yaml Secret - # template is skipped entirely (see templates/server-secret.yaml: - # `not $driverConfig.existingSecret` guards its creation). - existingSecret: "temporal-db-role" - secretKey: "password" - maxConns: 20 - maxIdleConns: 10 - maxConnLifetime: "1h" - # NOTE: no `connectAttributes: { tx_isolation: ... }` here — tx_isolation - # is a MySQL-only connection parameter. The Postgres `pq` driver rejects - # it ("unrecognized configuration parameter"), which killed every DB - # connection (schema-setup job AND server) with the misleading - # "no usable database connection found". Postgres defaults to READ - # COMMITTED isolation anyway, so nothing is lost by omitting it. - visibility: - driver: "sql" - sql: - driver: "postgres12" - host: "ddb-cluster-rw.ddb.svc.cluster.local" - port: 5432 - database: "temporal_visibility" - user: "temporal" - existingSecret: "temporal-db-role" - secretKey: "password" - maxConns: 20 - maxIdleConns: 10 - maxConnLifetime: "1h" - service: - type: ClusterIP - -# ── Temporal Web UI ──────────────────────────────────────────────────────── -web: - replicaCount: 1 - service: - type: ClusterIP - -# ── Ingress ──────────────────────────────────────────────────────── -ingress: - enabled: false diff --git a/k8s/bootstrap-local/00-namespaces.yaml b/k8s/bootstrap-local/00-namespaces.yaml deleted file mode 100644 index a0c8f0d..0000000 --- a/k8s/bootstrap-local/00-namespaces.yaml +++ /dev/null @@ -1,119 +0,0 @@ ---- -apiVersion: v1 -kind: Namespace -metadata: - name: argocd - labels: - name: argocd ---- -apiVersion: v1 -kind: Namespace -metadata: - name: ddb - labels: - name: ddb ---- -apiVersion: v1 -kind: Namespace -metadata: - name: cicd - labels: - name: cicd - # REQUIRED: Forgejo runner needs privileged (DinD, hostPath, securityContext.privileged) - pod-security.kubernetes.io/enforce: privileged - pod-security.kubernetes.io/audit: privileged - pod-security.kubernetes.io/warn: privileged ---- -apiVersion: v1 -kind: Namespace -metadata: - name: cert-manager - labels: - name: cert-manager ---- -apiVersion: v1 -kind: Namespace -metadata: - name: ingress-nginx - labels: - name: ingress-nginx - # REQUIRED: nginx controller needs hostPort 80/443 - pod-security.kubernetes.io/enforce: privileged ---- -apiVersion: v1 -kind: Namespace -metadata: - name: reloader - labels: - name: reloader ---- -apiVersion: v1 -kind: Namespace -metadata: - name: storage - labels: - name: storage - # REQUIRED: minio operator needs privileged securityContext - pod-security.kubernetes.io/enforce: privileged - pod-security.kubernetes.io/audit: privileged - pod-security.kubernetes.io/warn: privileged ---- -apiVersion: v1 -kind: Namespace -metadata: - name: monitoring - labels: - name: monitoring - # REQUIRED: node-exporter needs hostNetwork/hostPID/hostPath/hostPort - pod-security.kubernetes.io/enforce: privileged ---- -apiVersion: v1 -kind: Namespace -metadata: - name: logging - labels: - name: logging - # REQUIRED: promtail needs hostPath, DAC_READ_SEARCH, privileged:true - pod-security.kubernetes.io/enforce: privileged ---- -apiVersion: v1 -kind: Namespace -metadata: - name: iam - labels: - name: iam ---- -apiVersion: v1 -kind: Namespace -metadata: - name: sqs - labels: - name: sqs ---- -apiVersion: v1 -kind: Namespace -metadata: - name: temporal - labels: - name: temporal ---- -apiVersion: v1 -kind: Namespace -metadata: - name: dashboard - labels: - name: dashboard ---- -apiVersion: v1 -kind: Namespace -metadata: - name: cloudflared - labels: - name: cloudflared ---- -apiVersion: v1 -kind: Namespace -metadata: - name: duckdns - labels: - name: duckdns diff --git a/k8s/bootstrap-local/01-argocd.yaml b/k8s/bootstrap-local/01-argocd.yaml deleted file mode 100644 index 8ad7864..0000000 --- a/k8s/bootstrap-local/01-argocd.yaml +++ /dev/null @@ -1,47 +0,0 @@ -# ArgoCD installation - NOT managed by ArgoCD itself (bootstrap only). -# Install via: kubectl apply -k k8s/bootstrap-local/ -# Or manually: kubectl create namespace argocd -# kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: argocd-cm - namespace: argocd - labels: - app.kubernetes.io/name: argocd-cm - app.kubernetes.io/part-of: argocd -data: - # Point at Forgejo (will be available after 04-forgejo.yaml completes) - repositories: | - - url: https://forgejo.riotpiao.com/riotpiao.com/homelab.git - name: homelab - type: git - - # Reconciliation timeout (default 180s) - timeout.reconciliation: "300" - - # Resource exclusions (prevent ArgoCD from managing certain resources) - resource.exclusions: | - - apiGroups: - - cilium.io - kinds: - - CiliumIdentity - clusters: - - "*" ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: argocd-rbac-cm - namespace: argocd -data: - # Admin policy (adjust as needed) - policy.default: role:readonly - policy.csv: | - g, admin, role:admin ---- -# NOTE: ArgoCD installation itself not included here - apply it separately: -# kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml -# Or use Helm chart (recommended for production): -# helm install argocd argo/argo-cd -n argocd --version 7.x.x diff --git a/k8s/bootstrap-local/02-cnpg-operator.yaml b/k8s/bootstrap-local/02-cnpg-operator.yaml deleted file mode 100644 index 320d2e0..0000000 --- a/k8s/bootstrap-local/02-cnpg-operator.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# CloudNativePG operator - deployed as Helm chart via kubectl/ArgoCD. -# This file is a placeholder - actual install via Helm: -# helm repo add cnpg https://cloudnative-pg.github.io/charts -# helm install cnpg cnpg/cloudnative-pg -n ddb --create-namespace --version ~0.20 -# -# Or create an ArgoCD Application (recommended): ---- -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: cnpg-operator - namespace: argocd - annotations: - argocd.argoproj.io/sync-wave: "-1" # Bootstrap wave (before everything) -spec: - project: homelab - source: - repoURL: https://cloudnative-pg.github.io/charts - chart: cloudnative-pg - targetRevision: "~0.20" - destination: - server: https://kubernetes.default.svc - namespace: ddb - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true - - ServerSideApply=true diff --git a/k8s/bootstrap-local/03-ddb-bootstrap.yaml b/k8s/bootstrap-local/03-ddb-bootstrap.yaml deleted file mode 100644 index f611b79..0000000 --- a/k8s/bootstrap-local/03-ddb-bootstrap.yaml +++ /dev/null @@ -1,152 +0,0 @@ -# PostgreSQL cluster + Forgejo dependencies (bootstrap only, not GitOps-managed). -# These resources MUST exist before Forgejo can start, and Forgejo MUST exist -# before ArgoCD can sync from the git repo it hosts → circular dependency. -# Apply once via bootstrap.sh, never touched by ArgoCD afterward. ---- -apiVersion: postgresql.cnpg.io/v1 -kind: Cluster -metadata: - name: ddb-cluster - namespace: ddb - labels: - app: postgresql - layer: data - bootstrap-phase: "0" -spec: - instances: 3 # HA across 3 control-plane nodes - imageName: ghcr.io/cloudnative-pg/postgresql:16.2 - - bootstrap: - initdb: - database: app - owner: app - encoding: UTF8 - localeCollate: C - localeCType: C - postInitApplicationSQL: - - CREATE EXTENSION IF NOT EXISTS vector; - - CREATE EXTENSION IF NOT EXISTS pgcrypto; - - CREATE EXTENSION IF NOT EXISTS pg_trgm; - - # Role management: passwords from secrets, databases from separate Database CRs - managed: - roles: - - name: authentik - ensure: present - login: true - passwordSecret: - name: authentik-db-role - - name: temporal - ensure: present - login: true - passwordSecret: - name: temporal-db-role - - enableSuperuserAccess: false - - postgresql: - parameters: - shared_buffers: "256MB" - max_parallel_workers: "4" - max_parallel_workers_per_gather: "4" - archive_mode: "on" - archive_timeout: "5min" - log_destination: "csvlog" - log_directory: "/controller/log" - log_filename: "postgres" - log_rotation_age: "0" - dynamic_shared_memory_type: "posix" - - storage: - size: 10Gi - storageClass: longhorn - - monitoring: - enablePodMonitor: false - disableDefaultQueries: false - customQueriesConfigMap: - - name: cnpg-default-monitoring - key: queries - - affinity: - podAntiAffinityType: preferred ---- -# Forgejo database (depends on ddb-cluster being ready) -apiVersion: postgresql.cnpg.io/v1 -kind: Database -metadata: - name: forgejo - namespace: ddb - labels: - bootstrap-phase: "0" -spec: - name: forgejo - owner: app - cluster: - name: ddb-cluster ---- -# Forgejo Redis (cache, session, queue) -apiVersion: v1 -kind: Service -metadata: - name: forgejo-redis - namespace: cicd - labels: - app: forgejo-redis - bootstrap-phase: "0" -spec: - ports: - - port: 6379 - targetPort: 6379 - protocol: TCP - name: redis - selector: - app: forgejo-redis - type: ClusterIP ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: forgejo-redis - namespace: cicd - labels: - app: forgejo-redis - bootstrap-phase: "0" -spec: - replicas: 1 - selector: - matchLabels: - app: forgejo-redis - template: - metadata: - labels: - app: forgejo-redis - spec: - containers: - - name: redis - image: redis:7-alpine - ports: - - containerPort: 6379 - resources: - requests: - cpu: 50m - memory: 64Mi - limits: - cpu: 200m - memory: 256Mi - livenessProbe: - tcpSocket: - port: 6379 - initialDelaySeconds: 30 - periodSeconds: 10 - readinessProbe: - exec: - command: - - redis-cli - - ping - initialDelaySeconds: 5 - periodSeconds: 5 - tolerations: - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoSchedule diff --git a/k8s/bootstrap-local/04-forgejo.yaml b/k8s/bootstrap-local/04-forgejo.yaml deleted file mode 100644 index ac7d37f..0000000 --- a/k8s/bootstrap-local/04-forgejo.yaml +++ /dev/null @@ -1,151 +0,0 @@ -# Forgejo - Git server hosting the GitOps repo (bootstrap only, manual sync). -# ArgoCD cannot auto-sync Forgejo because Forgejo hosts the repo ArgoCD syncs -# from → circular dependency. Apply once via bootstrap, manual sync only afterward. -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: forgejo - namespace: argocd - annotations: - argocd.argoproj.io/sync-wave: "0" # Bootstrap wave - bootstrap-phase: "0" - description: "Bootstrap-only: Forgejo hosts the GitOps repo" -spec: - project: homelab - source: - repoURL: https://dl.gitea.com/charts/ - chart: gitea - targetRevision: "~10" - helm: - # Inline values (git-independent) - keep in sync with k8s/security/ci-cd/forgejo-values.yaml - valuesObject: - image: - repository: codeberg.org/forgejo/forgejo - tag: "13" - pullPolicy: IfNotPresent - gitea: - admin: - username: rock - email: locartrock@gmail.com - config: - server: - PROTOCOL: http - DOMAIN: forgejo.riotpiao.com - ROOT_URL: https://forgejo.riotpiao.com/ - HTTP_PORT: 3000 - START_SSH_SERVER: true - SSH_DOMAIN: forgejo.riotpiao.com - SSH_PORT: 2222 - SSH_LISTEN_PORT: 2222 - database: - DB_TYPE: postgres - HOST: ddb-cluster-rw.ddb.svc:5432 - NAME: forgejo - USER: app - repository: - ROOT: /data/git - actions: - ENABLED: true - packages: - ENABLED: true - metrics: - ENABLED: true - service: - DISABLE_REGISTRATION: true - oauth2: - ENABLED: true - PROVIDER: openidconnect - OPENID_CONNECT_DISCOVERY_URL: https://authentik.riotpiao.com/application/o/forgejo/.well-known/openid-configuration - CLIENT_ID: forgejo - AUTO_DISCOVER_URL: https://authentik.riotpiao.com/application/o/forgejo/.well-known/openid-configuration - cache: - ADAPTER: redis - HOST: "redis://forgejo-redis.cicd.svc:6379/0" - session: - PROVIDER: redis - PROVIDER_CONFIG: "redis://forgejo-redis.cicd.svc:6379/1" - queue: - TYPE: redis - CONN_STR: "redis://forgejo-redis.cicd.svc:6379/2" - metrics: - enabled: true - serviceMonitor: - enabled: false - persistence: - enabled: true - storageClass: longhorn - size: 20Gi - accessModes: - - ReadWriteMany - replicaCount: 2 - deployment: - strategy: - type: RollingUpdate - env: - - name: SSL_CERT_DIR - value: /homelab-ca - - name: GITEA__database__PASSWD - valueFrom: - secretKeyRef: - name: ddb-cluster-app - key: password - - name: GITEA__oauth2__CLIENT_SECRET - valueFrom: - secretKeyRef: - name: forgejo-oidc - key: CLIENT_SECRET - podAnnotations: - configmap.reloader.stakater.com/reload: "homelab-ca" - service: - http: - type: LoadBalancer - port: 3000 - targetPort: 3000 - annotations: - io.cilium/lb-ipam-ips: "192.168.1.165" - io.cilium/lb-ipam-sharing-key: "forgejo" - ssh: - type: LoadBalancer - port: 2222 - targetPort: 2222 - annotations: - io.cilium/lb-ipam-ips: "192.168.1.165" - io.cilium/lb-ipam-sharing-key: "forgejo" - resources: - requests: - cpu: 250m - memory: 512Mi - limits: - cpu: "1" - memory: 1Gi - tolerations: - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoSchedule - extraVolumes: - - name: homelab-ca - configMap: - name: homelab-ca - extraVolumeMounts: - - name: homelab-ca - mountPath: /homelab-ca - readOnly: true - ingress: - enabled: false - postgresql: - enabled: false - postgresql-ha: - enabled: false - mysql: - enabled: false - redis-cluster: - enabled: false - act_runner: - enabled: false - destination: - server: https://kubernetes.default.svc - namespace: cicd - syncPolicy: - # NO automated sync - Forgejo hosts the repo; auto-sync would let a bad - # CI commit break the system CI depends on. Manual sync only. - syncOptions: [] diff --git a/k8s/bootstrap-local/05-wait-for-databases.yaml b/k8s/bootstrap-local/05-wait-for-databases.yaml deleted file mode 100644 index 6da62f8..0000000 --- a/k8s/bootstrap-local/05-wait-for-databases.yaml +++ /dev/null @@ -1,89 +0,0 @@ -# Wait-for-databases Job - ensures Database CRs are reconciled before apps start -# This solves the race condition where Forgejo starts before CNPG creates the database -apiVersion: batch/v1 -kind: Job -metadata: - name: wait-for-databases - namespace: ddb - annotations: - description: "Waits for CNPG to reconcile Database CRs and create databases in PostgreSQL" -spec: - backoffLimit: 5 - template: - metadata: - name: wait-for-databases - spec: - restartPolicy: Never - serviceAccountName: wait-for-databases - containers: - - name: wait - image: bitnami/kubectl:latest - command: - - /bin/bash - - -c - - | - set -euo pipefail - - echo "==> Waiting for CNPG Database CRs to be reconciled..." - - DATABASES="forgejo authentik temporal temporal-visibility" - - for db in $DATABASES; do - echo "Checking database: $db" - - for i in {1..60}; do - # Check if Database CR exists and is ready - READY=$(kubectl get database $db -n ddb -o jsonpath='{.status.ready}' 2>/dev/null || echo "false") - - if [ "$READY" = "true" ]; then - echo " ✓ $db is ready" - break - fi - - echo " Waiting for $db to be ready... ($i/60)" - sleep 5 - - if [ $i -eq 60 ]; then - echo " ✗ Timeout waiting for $db" - exit 1 - fi - done - done - - echo "" - echo "==> All databases are ready!" - echo "CNPG has created the following databases:" - kubectl get databases -n ddb - - echo "" - echo "✅ Safe to deploy applications now" ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: wait-for-databases - namespace: ddb ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: wait-for-databases - namespace: ddb -rules: -- apiGroups: ["postgresql.cnpg.io"] - resources: ["databases"] - verbs: ["get", "list", "watch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: wait-for-databases - namespace: ddb -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: wait-for-databases -subjects: -- kind: ServiceAccount - name: wait-for-databases - namespace: ddb diff --git a/k8s/bootstrap-local/06-ingress-nginx.yaml b/k8s/bootstrap-local/06-ingress-nginx.yaml deleted file mode 100644 index 670d68c..0000000 --- a/k8s/bootstrap-local/06-ingress-nginx.yaml +++ /dev/null @@ -1,56 +0,0 @@ -# ingress-nginx - required for Forgejo domain access before ArgoCD can sync -# This breaks the circular dependency: ArgoCD needs https://forgejo.riotpiao.com -# but that domain requires ingress-nginx to route traffic. -apiVersion: v1 -kind: Namespace -metadata: - name: ingress-nginx - labels: - pod-security.kubernetes.io/enforce: privileged - pod-security.kubernetes.io/audit: privileged - pod-security.kubernetes.io/warn: privileged ---- -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: ingress-nginx-bootstrap - namespace: argocd - annotations: - argocd.argoproj.io/sync-wave: "0" - description: "Bootstrap ingress-nginx to enable Forgejo domain access" -spec: - project: homelab - source: - repoURL: https://kubernetes.github.io/ingress-nginx - chart: ingress-nginx - targetRevision: "4.15.1" - helm: - values: | - controller: - kind: DaemonSet - service: - type: LoadBalancer - annotations: - io.cilium/lb-ipam-ips: "192.168.1.160" - hostPort: - enabled: false - tolerations: - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoSchedule - resources: - requests: - cpu: 100m - memory: 128Mi - limits: - cpu: 500m - memory: 512Mi - destination: - server: https://kubernetes.default.svc - namespace: ingress-nginx - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true diff --git a/k8s/bootstrap-local/kustomization.yaml b/k8s/bootstrap-local/kustomization.yaml deleted file mode 100644 index 6b7e78f..0000000 --- a/k8s/bootstrap-local/kustomization.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -metadata: - name: bootstrap-local - annotations: - description: | - Phase 0 bootstrap bundle - apply once from local checkout on a fresh cluster. - Contains only resources that have circular git dependencies (Forgejo hosts - the repo ArgoCD syncs from). Everything else is GitOps-managed via ArgoCD. - -# Resources in strict dependency order -resources: - - 00-namespaces.yaml # Pre-create with PodSecurity labels - - 01-argocd.yaml # ArgoCD + SOPS plugin ConfigMap - - 02-cnpg-operator.yaml # CloudNativePG operator + CRDs - - 03-ddb-bootstrap.yaml # PostgreSQL cluster + Forgejo DB + Redis - - 05-wait-for-databases.yaml # Wait for CNPG to create databases - - 04-forgejo.yaml # Forgejo Helm chart (inline values) - - 06-ingress-nginx.yaml # Ingress for Forgejo domain access - -# Notes: -# - SOPS age secret created via bootstrap.sh (not in git) -# - After bootstrap: git push → kubectl apply -k k8s/argocd/root → done -# - ALL future changes via git push (ArgoCD auto-syncs) diff --git a/k8s/bootstrap/README.md b/k8s/bootstrap/README.md new file mode 100644 index 0000000..fac7724 --- /dev/null +++ b/k8s/bootstrap/README.md @@ -0,0 +1,128 @@ +# Homelab Bootstrap — Single-Cluster, GitOps-Ready + +**Run once manually, GitOps forever after.** + +This bootstrap breaks the ArgoCD ↔ Forgejo circular dependency by: +1. Installing infrastructure in correct dependency order +2. Pointing ArgoCD at a GitHub mirror initially +3. Cutting over to Forgejo once healthy +4. Using Helm for reproducible installs +5. Ensuring ArgoCD adopts (not duplicates) bootstrap resources + +## Prerequisites + +- Talos cluster running (terraform applied) +- kubectl configured (`KUBECONFIG` points at cluster) +- Helm 3 installed +- SOPS age key at `~/.sops/homelab-age.key` +- GitHub mirror of this repo (for initial ArgoCD source) + +## Directory Structure + +``` +bootstrap/ +├── phase1-storage/ # Longhorn via Helm +├── phase2-cnpg/ # CNPG operator via Helm +├── phase3-forgejo/ # Forgejo DB + Forgejo via Helm +├── phase4-argocd/ # ArgoCD via Helm → GitHub initially +└── phase5-cutover/ # Switch ArgoCD source to Forgejo +``` + +## Usage + +```bash +# From repo root: +./bootstrap.sh + +# Or step-by-step: +./bootstrap.sh phase1 # Storage +./bootstrap.sh phase2 # CNPG +./bootstrap.sh phase3 # Forgejo +./bootstrap.sh phase4 # ArgoCD (GitHub mirror) +./bootstrap.sh phase5 # Cut over to Forgejo +``` + +## Design Principles + +1. **DRY**: Helm values used by both bootstrap and ArgoCD +2. **Single Source of Truth**: Manifests match what ArgoCD will manage +3. **Idempotent**: Can re-run phases safely +4. **Adoption Ready**: Resources have `argocd.argoproj.io/sync-options: Prune=false` +5. **Dependency Ordered**: Each phase waits for previous to be Ready + +## Phase Details + +### Phase 1: Storage (Longhorn) + +Installs Longhorn with: +- 3-node HA configuration +- Unified `longhorn` StorageClass (default) +- Special `longhorn-cnpg` StorageClass with postgres UID/GID mount options +- CSI plugin tolerations for control-plane nodes + +**Source of Truth**: `phase1-storage/longhorn-values.yaml` + +### Phase 2: CNPG Operator + +Installs CloudNativePG operator with: +- CRD registration (blocks until CRD available) +- Webhook configuration +- Monitoring enabled + +**Source of Truth**: `phase2-cnpg/cnpg-values.yaml` + +### Phase 3: Forgejo Database + Forgejo + +1. Creates `forgejo-db` CNPG Cluster +2. Waits for cluster Ready (PostgreSQL accepting connections) +3. Installs Forgejo via Helm pointing at `forgejo-db-rw` service +4. Waits for Forgejo healthy + +**Source of Truth**: +- `phase3-forgejo/forgejo-db.yaml` (CNPG Cluster CR) +- `phase3-forgejo/forgejo-values.yaml` (Helm values) + +### Phase 4: ArgoCD (GitHub Mirror) + +Installs ArgoCD via Helm, then applies root app-of-apps pointing at **GitHub mirror**. + +This is the circle-breaker: ArgoCD syncs from GitHub (not Forgejo) initially. + +**Source of Truth**: +- `phase4-argocd/argocd-values.yaml` +- `phase4-argocd/root-app-github.yaml` (repoURL = GitHub) + +ArgoCD **adopts** Phases 1-3 resources (no duplication) because manifests match. + +### Phase 5: Cut Over to Forgejo + +1. Push repo to Forgejo +2. Update root app `repoURL` from GitHub → Forgejo +3. ArgoCD re-syncs from Forgejo + +**The circle is broken. GitHub mirror is now disaster recovery only.** + +## Post-Bootstrap + +All changes via Git: +```bash +git commit -m "feat(app): add new service" +git push forgejo main +# ArgoCD auto-syncs +``` + +## Troubleshooting + +- **Phase stuck?** Check `kubectl get events -n --sort-by='.lastTimestamp'` +- **ArgoCD duplicating?** Verify manifests match exactly (Helm values ↔ ArgoCD Application) +- **Forgejo won't start?** Check CNPG cluster Ready: `kubectl get cluster forgejo-db -n forgejo` +- **Can't push to Forgejo?** Verify ingress-nginx healthy, DNS resolves `forgejo.riotpiao.com` + +## Migration from Old Bootstrap + +If you have existing `k8s/bootstrap-local/`: + +1. **DO NOT delete** existing resources (Longhorn data!) +2. Run refined bootstrap in "adoption mode" (no delete, just apply) +3. Verify ArgoCD shows "Synced" for all apps +4. Archive old bootstrap: `git mv k8s/bootstrap-local k8s/archive/bootstrap-local-v1` diff --git a/k8s/bootstrap/phase1-storage/longhorn-values.yaml b/k8s/bootstrap/phase1-storage/longhorn-values.yaml new file mode 100644 index 0000000..10cad85 --- /dev/null +++ b/k8s/bootstrap/phase1-storage/longhorn-values.yaml @@ -0,0 +1,56 @@ +# Longhorn Helm Values — Single Source of Truth +# Used by both bootstrap.sh (Helm install) and ArgoCD (adoption) +# Chart: https://github.com/longhorn/charts + +defaultSettings: + # 3-node HA configuration + replicaReplenishmentWaitInterval: 600 # 10min before auto-repair + replicaSoftAntiAffinity: false # REQUIRED for true HA + replicaAutoBalance: best-effort + storageMinimalAvailablePercentage: 10 + + # Performance tuning + defaultDataPath: /var/lib/longhorn + defaultDataLocality: best-effort + backupTarget: "" # TODO: Add MinIO backup target later + + # Monitoring + guaranteedEngineManagerCPU: 12 # mCPU + guaranteedReplicaManagerCPU: 12 + +persistence: + defaultClass: true # Make 'longhorn' the default StorageClass + defaultClassReplicaCount: 3 + defaultFsType: ext4 + reclaimPolicy: Delete + +# CSI plugin must tolerate control-plane taints +csi: + kubeletRootDir: /var/lib/kubelet + attacherReplicaCount: 3 + provisionerReplicaCount: 3 + resizerReplicaCount: 3 + snapshotterReplicaCount: 3 + +# Longhorn manager on all nodes +longhornManager: + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + +# Driver deployer tolerations +longhornDriver: + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + +# UI for debugging +longhornUI: + replicas: 1 + +# Monitoring (Prometheus ServiceMonitor) +metrics: + serviceMonitor: + enabled: true diff --git a/k8s/bootstrap/phase1-storage/storageclasses.yaml b/k8s/bootstrap/phase1-storage/storageclasses.yaml new file mode 100644 index 0000000..61f72b7 --- /dev/null +++ b/k8s/bootstrap/phase1-storage/storageclasses.yaml @@ -0,0 +1,27 @@ +# StorageClasses — Applied after Longhorn installation +# The default 'longhorn' SC is created by Helm chart +# These are additional specialized classes +--- +# CNPG-specific StorageClass with postgres UID/GID mount options +# Fixes "read-only filesystem" error when PostgreSQL (UID 26) tries to write +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: longhorn-cnpg + annotations: + storageclass.kubernetes.io/is-default-class: "false" + argocd.argoproj.io/sync-options: Prune=false # Allow ArgoCD adoption +provisioner: driver.longhorn.io +allowVolumeExpansion: true +parameters: + numberOfReplicas: "3" + staleReplicaTimeout: "30" + fromBackup: "" + dataLocality: "best-effort" + fsType: "ext4" +mountOptions: + - "noatime" + - "uid=26" # postgres user + - "gid=26" # postgres group +reclaimPolicy: Delete +volumeBindingMode: Immediate diff --git a/k8s/bootstrap/phase2-cnpg/cnpg-values.yaml b/k8s/bootstrap/phase2-cnpg/cnpg-values.yaml new file mode 100644 index 0000000..ed82ecc --- /dev/null +++ b/k8s/bootstrap/phase2-cnpg/cnpg-values.yaml @@ -0,0 +1,55 @@ +# CloudNativePG Operator Helm Values — Single Source of Truth +# Chart: https://github.com/cloudnative-pg/charts + +# Basic operator configuration +replicaCount: 1 + +# CRDs must be installed (Helm default behavior) +crds: + create: true + +# Webhook configuration +webhook: + port: 9443 + mutating: + create: true + failurePolicy: Fail + validating: + create: true + failurePolicy: Fail + +# Monitoring +monitoring: + podMonitorEnabled: true + grafanaDashboard: + create: false # We'll manage dashboards via ArgoCD later + +# Resource limits for operator +resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + +# Security context +securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + +# Tolerate control-plane taints +tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + +# Additional labels for ArgoCD adoption +commonLabels: + app.kubernetes.io/managed-by: Helm + argocd.argoproj.io/instance: cnpg-operator diff --git a/k8s/bootstrap/phase3-forgejo/forgejo-db.yaml b/k8s/bootstrap/phase3-forgejo/forgejo-db.yaml new file mode 100644 index 0000000..c2c4060 --- /dev/null +++ b/k8s/bootstrap/phase3-forgejo/forgejo-db.yaml @@ -0,0 +1,122 @@ +# Forgejo PostgreSQL Database — CNPG Cluster CR +# This is the source of truth for Forgejo's database +# ArgoCD will adopt this (not recreate it) +--- +apiVersion: v1 +kind: Namespace +metadata: + name: forgejo + annotations: + argocd.argoproj.io/sync-options: Prune=false +--- +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: forgejo-db + namespace: forgejo + annotations: + argocd.argoproj.io/sync-options: Prune=false # Let ArgoCD adopt, don't delete + labels: + app: forgejo-db + layer: data +spec: + instances: 3 # HA configuration + + imageName: ghcr.io/cloudnative-pg/postgresql:16.2 + + bootstrap: + initdb: + database: forgejo + owner: forgejo + encoding: UTF8 + localeCollate: C + localeCType: C + + enableSuperuserAccess: false + + # Resource limits per best practices + resources: + requests: + memory: "4Gi" + cpu: "1" + limits: + memory: "8Gi" + cpu: "2" + + postgresql: + parameters: + # Tuned for 4-8GB RAM + shared_buffers: "1GB" + effective_cache_size: "3GB" + maintenance_work_mem: "512MB" + work_mem: "64MB" + max_connections: "100" + max_parallel_workers: "2" + max_parallel_workers_per_gather: "1" + # WAL + wal_buffers: "16MB" + checkpoint_completion_target: "0.9" + min_wal_size: "512MB" + max_wal_size: "2GB" + # Logging + log_destination: "csvlog" + log_directory: "/controller/log" + log_filename: "postgres" + + storage: + size: 50Gi + storageClass: longhorn-cnpg # Uses postgres UID/GID mount options + + monitoring: + enablePodMonitor: true + + affinity: + podAntiAffinityType: required + topologyKey: kubernetes.io/hostname +--- +# Forgejo Redis (cache, session, queue) +apiVersion: v1 +kind: Service +metadata: + name: forgejo-redis + namespace: forgejo + annotations: + argocd.argoproj.io/sync-options: Prune=false +spec: + ports: + - port: 6379 + targetPort: 6379 + protocol: TCP + selector: + app: forgejo-redis + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: forgejo-redis + namespace: forgejo + annotations: + argocd.argoproj.io/sync-options: Prune=false +spec: + replicas: 1 + selector: + matchLabels: + app: forgejo-redis + template: + metadata: + labels: + app: forgejo-redis + spec: + containers: + - name: redis + image: redis:7-alpine + ports: + - containerPort: 6379 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 256Mi diff --git a/k8s/bootstrap/phase3-forgejo/forgejo-values.yaml b/k8s/bootstrap/phase3-forgejo/forgejo-values.yaml new file mode 100644 index 0000000..3303699 --- /dev/null +++ b/k8s/bootstrap/phase3-forgejo/forgejo-values.yaml @@ -0,0 +1,86 @@ +# Forgejo Helm Values — Single Source of Truth +# Chart: https://codeberg.org/forgejo-contrib/forgejo-helm + +gitea: + admin: + username: "admin" + email: "admin@riotpiao.com" + # Password set via secret (not in values) + + config: + server: + DOMAIN: forgejo.riotpiao.com + ROOT_URL: https://forgejo.riotpiao.com + SSH_DOMAIN: forgejo.riotpiao.com + SSH_PORT: 22 + + database: + DB_TYPE: postgres + HOST: forgejo-db-rw.forgejo.svc.cluster.local:5432 + NAME: forgejo + # User/password from CNPG-generated secret + USER: + valueFrom: + secretKeyRef: + name: forgejo-db-app + key: username + PASSWD: + valueFrom: + secretKeyRef: + name: forgejo-db-app + key: password + + cache: + ADAPTER: redis + HOST: redis://forgejo-redis.forgejo.svc.cluster.local:6379/0 + + session: + PROVIDER: redis + PROVIDER_CONFIG: redis://forgejo-redis.forgejo.svc.cluster.local:6379/1 + + queue: + TYPE: redis + CONN_STR: redis://forgejo-redis.forgejo.svc.cluster.local:6379/2 + +# Persistence (shared storage for repos) +persistence: + enabled: true + storageClass: longhorn + size: 20Gi + accessModes: + - ReadWriteOnce + +# Ingress +ingress: + enabled: true + className: nginx + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod + hosts: + - host: forgejo.riotpiao.com + paths: + - path: / + pathType: Prefix + tls: + - secretName: forgejo-tls + hosts: + - forgejo.riotpiao.com + +# Resources +resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: 1000m + memory: 2Gi + +# Tolerations for control-plane +tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + +# ArgoCD adoption labels +labels: + argocd.argoproj.io/instance: forgejo diff --git a/k8s/bootstrap/phase4-argocd/argocd-values.yaml b/k8s/bootstrap/phase4-argocd/argocd-values.yaml new file mode 100644 index 0000000..46d28fd --- /dev/null +++ b/k8s/bootstrap/phase4-argocd/argocd-values.yaml @@ -0,0 +1,124 @@ +# ArgoCD Helm Values — Single Source of Truth +# Chart: https://github.com/argoproj/argo-helm + +global: + domain: argocd.riotpiao.com + +# Server configuration +server: + ingress: + enabled: true + ingressClassName: nginx + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" + hosts: + - argocd.riotpiao.com + tls: + - secretName: argocd-server-tls + hosts: + - argocd.riotpiao.com + + # Allow insecure mode (terminate TLS at ingress) + extraArgs: + - --insecure + + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 1Gi + +# Repo server configuration +repoServer: + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 1Gi + + # SOPS plugin for encrypted secrets + volumes: + - name: sops-age + secret: + secretName: sops-age + optional: true + volumeMounts: + - name: sops-age + mountPath: /home/argocd/.config/sops/age + readOnly: true + + # Environment for SOPS + env: + - name: SOPS_AGE_KEY_FILE + value: /home/argocd/.config/sops/age/keys.txt + +# Controller configuration +controller: + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: 1000m + memory: 2Gi + +# Application controller configuration +applicationSet: + enabled: true + +# Notifications (optional, for Slack/Discord alerts) +notifications: + enabled: false + +# Redis for caching +redis: + enabled: true + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 256Mi + +# Tolerations for control-plane +server: + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + +repoServer: + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + +controller: + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + +# ArgoCD projects +configs: + # Default project allows all repos + cm: + admin.enabled: "true" + application.instanceLabelKey: argocd.argoproj.io/instance + + params: + server.insecure: true + +# RBAC (allow admin full access) +configs: + rbac: + policy.default: role:readonly + policy.csv: | + g, admin, role:admin diff --git a/k8s/bootstrap/phase4-argocd/root-app-github.yaml b/k8s/bootstrap/phase4-argocd/root-app-github.yaml new file mode 100644 index 0000000..61f40fd --- /dev/null +++ b/k8s/bootstrap/phase4-argocd/root-app-github.yaml @@ -0,0 +1,66 @@ +# ArgoCD Root App-of-Apps — GitHub Mirror Source +# This is the initial configuration that breaks the circular dependency +# Points at GitHub mirror, not Forgejo (Forgejo isn't ready yet) +# +# After Forgejo is healthy and repo is pushed, use phase5-cutover/root-app-forgejo.yaml +--- +apiVersion: v1 +kind: Namespace +metadata: + name: argocd +--- +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: homelab + namespace: argocd +spec: + description: Homelab infrastructure and applications + sourceRepos: + - 'https://github.com/YOUR-ORG/homelab.git' # ← REPLACE with your GitHub mirror + - 'https://forgejo.riotpiao.com/YOUR-ORG/homelab.git' + - 'https://*.github.io/*' # Helm charts from GitHub Pages + - 'https://charts.*' # Public Helm repos + destinations: + - namespace: '*' + server: 'https://kubernetes.default.svc' + clusterResourceWhitelist: + - group: '*' + kind: '*' + namespaceResourceWhitelist: + - group: '*' + kind: '*' +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: homelab-root + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: homelab + + source: + repoURL: https://github.com/YOUR-ORG/homelab.git # ← REPLACE with your GitHub mirror + targetRevision: main + path: k8s/argocd/apps + + destination: + server: https://kubernetes.default.svc + namespace: argocd + + syncPolicy: + automated: + prune: true + selfHeal: true + allowEmpty: false + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + retry: + limit: 5 + backoff: + duration: 5s + factor: 2 + maxDuration: 3m diff --git a/k8s/bootstrap/phase5-cutover/root-app-forgejo.yaml b/k8s/bootstrap/phase5-cutover/root-app-forgejo.yaml new file mode 100644 index 0000000..c6ba0a1 --- /dev/null +++ b/k8s/bootstrap/phase5-cutover/root-app-forgejo.yaml @@ -0,0 +1,39 @@ +# ArgoCD Root App-of-Apps — Forgejo Source (Final State) +# This replaces the GitHub mirror with Forgejo as the source of truth +# Apply this AFTER Forgejo is healthy and you've pushed the repo +# +# The circle is broken: ArgoCD → Forgejo works because both already exist +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: homelab-root + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: homelab + + source: + repoURL: https://forgejo.riotpiao.com/YOUR-ORG/homelab.git # ← REPLACE with your Forgejo URL + targetRevision: main + path: k8s/argocd/apps + + destination: + server: https://kubernetes.default.svc + namespace: argocd + + syncPolicy: + automated: + prune: true + selfHeal: true + allowEmpty: false + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + retry: + limit: 5 + backoff: + duration: 5s + factor: 2 + maxDuration: 3m diff --git a/k8s/infrastructure/longhorn/kustomization.yaml b/k8s/infrastructure/longhorn/kustomization.yaml index 84ac376..1e11e99 100644 --- a/k8s/infrastructure/longhorn/kustomization.yaml +++ b/k8s/infrastructure/longhorn/kustomization.yaml @@ -3,6 +3,7 @@ kind: Kustomization namespace: longhorn-system resources: - longhorn-storageclass.yaml + - longhorn-cnpg-storageclass.yaml # CNPG-specific with postgres UID/GID - longhorn-servicemonitor.yaml - longhorn-taint-toleration.yaml - longhorn-nodes.yaml diff --git a/k8s/infrastructure/longhorn/longhorn-cnpg-storageclass.yaml b/k8s/infrastructure/longhorn/longhorn-cnpg-storageclass.yaml new file mode 100644 index 0000000..580d5ab --- /dev/null +++ b/k8s/infrastructure/longhorn/longhorn-cnpg-storageclass.yaml @@ -0,0 +1,26 @@ +# StorageClass specifically for CNPG (CloudNativePG) PostgreSQL clusters +# Fixes the "read-only filesystem" issue by mounting with postgres UID/GID +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: longhorn-cnpg + namespace: longhorn-system + annotations: + storageclass.kubernetes.io/is-default-class: "false" +provisioner: driver.longhorn.io +allowVolumeExpansion: true +parameters: + numberOfReplicas: "3" + staleReplicaTimeout: "30" + fromBackup: "" + dataLocality: "best-effort" + fsType: "ext4" + # Mount options to ensure PostgreSQL can write + mkfsParams: "-O ^64bit,^metadata_csum" +mountOptions: + - "noatime" + # Critical: mount with postgres UID/GID (26:26) to avoid permission issues + - "uid=26" + - "gid=26" +reclaimPolicy: Delete +volumeBindingMode: Immediate \ No newline at end of file diff --git a/k8s/platform/e2e/cronjob-smoke.yaml b/k8s/platform/e2e/cronjob-smoke.yaml new file mode 100644 index 0000000..8b4e07d --- /dev/null +++ b/k8s/platform/e2e/cronjob-smoke.yaml @@ -0,0 +1,55 @@ +# Continuous smoke — runs the full WebKit suite every 15 min. A failed run means +# something users touch broke; alert on it (Job failure → kube-state-metrics +# kube_job_status_failed → Alertmanager). Same image/env as the deploy-gate Job. +apiVersion: batch/v1 +kind: CronJob +metadata: + name: e2e-smoke + namespace: platform + annotations: + argocd.argoproj.io/sync-options: Prune=false +spec: + schedule: "*/15 * * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 5 + jobTemplate: + spec: + backoffLimit: 1 + ttlSecondsAfterFinished: 86400 + template: + spec: + restartPolicy: Never + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + containers: + - name: e2e + image: forgejo-gitea-http.cicd.svc.cluster.local:3000/riotpiao.com/homelab-e2e:latest + imagePullPolicy: Always + env: + - name: BASE_DOMAIN + value: riotpiao.com + - name: E2E_IGNORE_TLS + value: "0" + - name: CI + value: "1" + - name: AK_ADMIN_USER + value: akadmin + - name: AK_ADMIN_PASSWORD + valueFrom: + secretKeyRef: { name: e2e-credentials, key: authentik-admin-password } + - name: MINIO_ENDPOINT + value: http://minio.storage.svc.cluster.local:9000 + - name: MINIO_BUCKET + value: e2e-artifacts + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: { name: e2e-credentials, key: minio-access-key } + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: { name: e2e-credentials, key: minio-secret-key } + resources: + requests: { cpu: 200m, memory: 512Mi } + limits: { cpu: "1", memory: 2Gi } diff --git a/k8s/platform/e2e/job-smoke.yaml b/k8s/platform/e2e/job-smoke.yaml new file mode 100644 index 0000000..bfb820b --- /dev/null +++ b/k8s/platform/e2e/job-smoke.yaml @@ -0,0 +1,59 @@ +# One-shot E2E smoke — run as a deploy verification gate. +# Wire as an ArgoCD PostSync hook (annotation below) OR call from an Argo Rollouts +# AnalysisTemplate. Job success == the app is actually viewable in Safari's engine; +# failure fails the sync/rollout. +# +# NOTE: not yet added to any kustomization / app-of-apps. Wiring = migration phase 7. +apiVersion: batch/v1 +kind: Job +metadata: + name: e2e-smoke + namespace: platform + annotations: + # Uncomment to make this a deploy gate on the owning Application: + # argocd.argoproj.io/hook: PostSync + # argocd.argoproj.io/hook-delete-policy: BeforeHookCreation + argocd.argoproj.io/sync-options: Prune=false +spec: + backoffLimit: 1 + ttlSecondsAfterFinished: 86400 + template: + spec: + restartPolicy: Never + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + containers: + - name: e2e + # Built + pushed by CI from tests/e2e/Dockerfile. Pin a digest in prod. + image: forgejo-gitea-http.cicd.svc.cluster.local:3000/riotpiao.com/homelab-e2e:latest + imagePullPolicy: Always + env: + - name: BASE_DOMAIN + value: riotpiao.com + # strict TLS by default; set "1" only during staging-cert bootstrap + - name: E2E_IGNORE_TLS + value: "0" + - name: CI + value: "1" + - name: AK_ADMIN_USER + value: akadmin + - name: AK_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: e2e-credentials + key: authentik-admin-password + - name: MINIO_ENDPOINT + value: http://minio.storage.svc.cluster.local:9000 + - name: MINIO_BUCKET + value: e2e-artifacts + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: { name: e2e-credentials, key: minio-access-key } + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: { name: e2e-credentials, key: minio-secret-key } + resources: + requests: { cpu: 200m, memory: 512Mi } + limits: { cpu: "1", memory: 2Gi } diff --git a/tests/e2e/.dockerignore b/tests/e2e/.dockerignore new file mode 100644 index 0000000..bf771ee --- /dev/null +++ b/tests/e2e/.dockerignore @@ -0,0 +1,4 @@ +node_modules/ +test-results/ +playwright-report/ +.env diff --git a/tests/e2e/Dockerfile b/tests/e2e/Dockerfile new file mode 100644 index 0000000..3040d6f --- /dev/null +++ b/tests/e2e/Dockerfile @@ -0,0 +1,18 @@ +# Playwright base ships WebKit + all OS deps. Pin must match @playwright/test in +# package.json. Verify tag exists: https://mcr.microsoft.com/en-us/product/playwright/tags +FROM mcr.microsoft.com/playwright:v1.50.0-jammy + +WORKDIR /e2e + +# minio client for artifact upload (video/trace → MinIO bucket) +RUN curl -sSLo /usr/local/bin/mc https://dl.min.io/client/mc/release/linux-amd64/mc \ + && chmod +x /usr/local/bin/mc + +COPY package.json ./ +RUN npm install --no-audit --no-fund + +COPY . . + +# run-and-upload.sh runs the suite, uploads artifacts regardless of result, +# then exits with the suite's real exit code so the Job/gate reflects pass/fail. +ENTRYPOINT ["/e2e/run-and-upload.sh"] diff --git a/tests/e2e/README.md b/tests/e2e/README.md new file mode 100644 index 0000000..e600f14 --- /dev/null +++ b/tests/e2e/README.md @@ -0,0 +1,44 @@ +# Homelab E2E smoke (Playwright · WebKit / Safari engine) + +Proves apps are **viewable**, not just that a pod is `Running`. Verifies real +ingress + TLS + rendered UI through WebKit — Safari's engine (hard constraint). + +## Layout +- `playwright.config.ts` — WebKit-only project, video + trace + screenshot always on. +- `targets.ts` — the app list (`NAME-APP.`) + per-app "ready" selector. **Edit to match your ingress hosts.** +- `tests/reachability.spec.ts` — every non-auth app: 2xx/3xx + own-UI rendered (not nginx default backend). +- `tests/portainer.spec.ts` — login page renders. +- `tests/authentik.spec.ts` — full admin sign-in (needs `AK_ADMIN_PASSWORD`). +- `tests/oauth.spec.ts` — OIDC/OAuth **federation** flow per app: app → Authentik → back, authenticated. Driven by `OAUTH_TARGETS` in `targets.ts`; uses `helpers/authentik.ts`. +- `Dockerfile` / `run-and-upload.sh` — container image; runs suite, uploads artifacts to MinIO, exits with the suite's real code. + +## Run locally +```bash +cd tests/e2e +npm install +npx playwright install webkit +BASE_DOMAIN=riotpiao.com npm test # add E2E_IGNORE_TLS=1 for staging certs +AK_ADMIN_PASSWORD=… npm test # to exercise the authentik login +npm run report # open HTML report (video/trace) +``` + +## Env +| Var | Default | Purpose | +|---|---|---| +| `BASE_DOMAIN` | `riotpiao.com` | domain suffix for `NAME.` | +| `E2E_IGNORE_TLS` | `0` | `1` = accept staging/self-signed certs (bootstrap only) | +| `AK_ADMIN_USER` / `AK_ADMIN_PASSWORD` | `akadmin` / — | authentik login (password from secret store) | +| `MINIO_ENDPOINT` / `MINIO_BUCKET` / `MINIO_ACCESS_KEY` / `MINIO_SECRET_KEY` | — | artifact upload target | + +## In-cluster +`k8s/platform/e2e/` has a **Job** (deploy gate — attach as ArgoCD PostSync hook or +Rollouts AnalysisTemplate) and a **CronJob** (continuous smoke, alert on failure). +Both need a `e2e-credentials` Secret (authentik password + MinIO keys) — create it +SOPS-encrypted. **Not yet wired into any kustomization / app-of-apps** — that is +ADR 0001 migration phase 7. + +## Build image +```bash +docker build -t /riotpiao.com/homelab-e2e:$(git rev-parse --short HEAD) tests/e2e +docker push /riotpiao.com/homelab-e2e:… +``` diff --git a/tests/e2e/helpers/authentik.ts b/tests/e2e/helpers/authentik.ts new file mode 100644 index 0000000..f6eddf4 --- /dev/null +++ b/tests/e2e/helpers/authentik.ts @@ -0,0 +1,27 @@ +import { Page, expect } from '@playwright/test'; + +// Complete Authentik's identification → password flow on whatever page is +// currently showing it (used both for direct login and mid-OAuth redirect). +// Idempotent: if Authentik already has a session and skipped straight through, +// the fields won't appear and this returns without error. +export async function completeAuthentikLogin(page: Page, user: string, pass: string): Promise { + const uid = page.locator('input[name="uidField"], input[type="email"], input[type="text"]').first(); + if (await uid.isVisible({ timeout: 8_000 }).catch(() => false)) { + await uid.fill(user); + await page.keyboard.press('Enter'); + } + + const pw = page.locator('input[type="password"]').first(); + if (await pw.isVisible({ timeout: 8_000 }).catch(() => false)) { + await pw.fill(pass); + await page.keyboard.press('Enter'); + } + + // Authentik may show a consent/authorize step on first SSO to an app. + const authorize = page.getByRole('button', { name: /authorize|continue|allow/i }).first(); + if (await authorize.isVisible({ timeout: 5_000 }).catch(() => false)) { + await authorize.click(); + } + + await expect(page.getByText(/invalid|incorrect|failed to/i)).toHaveCount(0); +} diff --git a/tests/e2e/package.json b/tests/e2e/package.json new file mode 100644 index 0000000..5cbdfc6 --- /dev/null +++ b/tests/e2e/package.json @@ -0,0 +1,12 @@ +{ + "name": "homelab-e2e", + "private": true, + "description": "Playwright WebKit (Safari-engine) synthetic E2E smoke suite for homelab ingress apps", + "scripts": { + "test": "playwright test --project=webkit", + "report": "playwright show-report" + }, + "devDependencies": { + "@playwright/test": "1.50.0" + } +} diff --git a/tests/e2e/playwright.config.ts b/tests/e2e/playwright.config.ts new file mode 100644 index 0000000..3787e47 --- /dev/null +++ b/tests/e2e/playwright.config.ts @@ -0,0 +1,40 @@ +import { defineConfig, devices } from '@playwright/test'; + +// WebKit only — hard constraint: verify apps in Safari's rendering engine. +// Video + trace recorded for every test so a failed deploy has a replay. +const BASE_DOMAIN = process.env.BASE_DOMAIN ?? 'riotpiao.com'; + +// Set E2E_IGNORE_TLS=1 only during early bootstrap (staging certs). Default is +// strict so a broken TLS chain FAILS the suite — that is a real deploy failure. +const ignoreHTTPSErrors = process.env.E2E_IGNORE_TLS === '1'; + +export default defineConfig({ + testDir: './tests', + outputDir: './test-results', + // Serial + retries: this is a smoke gate, not a load test. + fullyParallel: false, + workers: 1, + retries: process.env.CI ? 1 : 0, + timeout: 60_000, + expect: { timeout: 15_000 }, + + reporter: [ + ['list'], + ['html', { outputFolder: 'playwright-report', open: 'never' }], + ['json', { outputFile: 'test-results/results.json' }], + ], + + use: { + baseURL: `https://${BASE_DOMAIN}`, + ignoreHTTPSErrors, + video: 'on', + trace: 'on', + screenshot: 'on', + navigationTimeout: 30_000, + actionTimeout: 15_000, + }, + + projects: [ + { name: 'webkit', use: { ...devices['Desktop Safari'] } }, + ], +}); diff --git a/tests/e2e/run-and-upload.sh b/tests/e2e/run-and-upload.sh new file mode 100755 index 0000000..ef15b28 --- /dev/null +++ b/tests/e2e/run-and-upload.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Run the WebKit smoke suite, upload artifacts (video/trace/screenshots/report) +# to MinIO, then exit with the suite's real result so a CD gate can act on it. +set -uo pipefail + +STAMP="${RUN_ID:-$(date +%Y%m%d-%H%M%S)}" +DEST="e2e/${STAMP}" + +echo "▶ Playwright WebKit smoke — BASE_DOMAIN=${BASE_DOMAIN:-riotpiao.com}" +npm test +RESULT=$? +echo "▶ suite exit=${RESULT}" + +# Upload artifacts even on failure — a failed deploy is exactly when you want the video. +if [[ -n "${MINIO_ENDPOINT:-}" && -n "${MINIO_ACCESS_KEY:-}" ]]; then + echo "▶ uploading artifacts → ${MINIO_ENDPOINT}/${MINIO_BUCKET:-e2e-artifacts}/${DEST}" + mc alias set store "${MINIO_ENDPOINT}" "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}" >/dev/null 2>&1 + mc mb --ignore-existing "store/${MINIO_BUCKET:-e2e-artifacts}" >/dev/null 2>&1 + mc cp --recursive test-results/ "store/${MINIO_BUCKET:-e2e-artifacts}/${DEST}/test-results/" >/dev/null 2>&1 || true + mc cp --recursive playwright-report/ "store/${MINIO_BUCKET:-e2e-artifacts}/${DEST}/playwright-report/" >/dev/null 2>&1 || true + echo "▶ artifacts uploaded under ${DEST}/" +else + echo "▶ MINIO_ENDPOINT unset — skipping upload (artifacts in ./test-results)" +fi + +exit ${RESULT} diff --git a/tests/e2e/targets.ts b/tests/e2e/targets.ts new file mode 100644 index 0000000..4722634 --- /dev/null +++ b/tests/e2e/targets.ts @@ -0,0 +1,72 @@ +// Ingress apps to smoke-test. Each entry = NAME-APP.. +// `ready` is a selector or text that only appears once the app actually rendered +// (not just a 200 from nginx's default backend). +// +// EDIT THIS LIST to match what you actually expose. Confirmed hosts are marked; +// others are best-guess — fix the subdomain if yours differs. + +export type Target = { + name: string; + subdomain: string; + path?: string; + // A locator that proves the app's own UI rendered. + ready: { role?: string; name?: RegExp; text?: RegExp; selector?: string }; + // Skip in the default reachability run (has a dedicated spec, e.g. auth flow). + dedicated?: boolean; +}; + +export const BASE_DOMAIN = process.env.BASE_DOMAIN ?? 'riotpiao.com'; + +export function url(t: Pick): string { + return `https://${t.subdomain}.${BASE_DOMAIN}${t.path ?? '/'}`; +} + +// Apps that federate login through Authentik (OIDC/OAuth2). The OAuth spec drives +// the full chain: open the app, click its "sign in via SSO" control, complete the +// Authentik login on redirect, and assert we land back in the app authenticated. +// EDIT selectors to match your apps — the SSO button text/label varies per app. +export type OAuthTarget = { + name: string; + subdomain: string; + path?: string; + // The app's "log in with Authentik/SSO" control. + ssoButton: { role?: string; name?: RegExp; selector?: string }; + // Proof we are authenticated back inside the app after the round-trip. + success: { urlRe?: RegExp; text?: RegExp; selector?: string }; +}; + +export const OAUTH_TARGETS: OAuthTarget[] = [ + { + name: 'argocd', + subdomain: 'argocd', + ssoButton: { role: 'link', name: /log ?in via|authentik|sso|oidc/i }, + success: { urlRe: /\/applications/i, text: /Applications/i }, + }, + { + name: 'forgejo', + subdomain: 'forgejo', + path: '/user/login', + ssoButton: { role: 'link', name: /authentik|oauth|sign in with|openid/i }, + success: { selector: 'a[href="/notifications"], .avatar, nav .user' }, + }, + { + name: 'grafana', + subdomain: 'grafana', + path: '/login', + ssoButton: { role: 'link', name: /sign in with|authentik|oauth/i }, + success: { urlRe: /\/(\?|$)|\/d\//, text: /Welcome to Grafana|Dashboards|Home/i }, + }, +]; + +export const TARGETS: Target[] = [ + // confirmed hosts (from bootstrap values) + { name: 'argocd', subdomain: 'argocd', ready: { text: /Argo\s*CD|Let's get started|Login/i } }, + { name: 'forgejo', subdomain: 'forgejo', ready: { text: /Forgejo|Sign In|Explore/i } }, + + // dashboards / apps — adjust subdomain to your actual ingress host + { name: 'portainer', subdomain: 'portainer', dedicated: true, ready: { selector: 'input[type="password"]' } }, + { name: 'authentik', subdomain: 'authentik', dedicated: true, ready: { selector: 'input[name="uidField"], input[type="password"]' } }, + { name: 'homarr', subdomain: 'homarr', ready: { text: /Homarr|Dashboard/i } }, + { name: 'grafana', subdomain: 'grafana', ready: { text: /Grafana|Welcome/i } }, + { name: 'minio', subdomain: 'minio', ready: { selector: 'input, button' } }, +]; diff --git a/tests/e2e/tests/authentik.spec.ts b/tests/e2e/tests/authentik.spec.ts new file mode 100644 index 0000000..dfd6c0e --- /dev/null +++ b/tests/e2e/tests/authentik.spec.ts @@ -0,0 +1,32 @@ +import { test, expect } from '@playwright/test'; +import { TARGETS, url } from '../targets'; + +const t = TARGETS.find((x) => x.name === 'authentik')!; + +// Authentik: the real signal. A healthy pod does NOT prove OIDC/redirect/session +// work — only a completed sign-in does. Credentials come from env (injected from +// the secret store in-cluster), never hardcoded. +const USER = process.env.AK_ADMIN_USER ?? 'akadmin'; +const PASS = process.env.AK_ADMIN_PASSWORD; + +test('authentik admin can sign in', async ({ page }) => { + test.skip(!PASS, 'AK_ADMIN_PASSWORD not set — provide via secret to run the login flow'); + + await page.goto(url(t), { waitUntil: 'domcontentloaded' }); + + // Authentik identification stage: username, Enter/continue, then password. + const uid = page.locator('input[name="uidField"], input[type="text"], input[type="email"]').first(); + await expect(uid).toBeVisible(); + await uid.fill(USER); + await page.keyboard.press('Enter'); + + const pw = page.locator('input[type="password"]').first(); + await expect(pw).toBeVisible(); + await pw.fill(PASS!); + await page.keyboard.press('Enter'); + + // Landed on the user dashboard — no auth error banner. + await expect(page).toHaveURL(/\/if\/user|\/if\/admin|\/library/i, { timeout: 20_000 }); + await expect(page.getByText(/invalid|incorrect|failed/i)).toHaveCount(0); + await page.screenshot({ path: 'test-results/authentik-dashboard.png', fullPage: true }); +}); diff --git a/tests/e2e/tests/oauth.spec.ts b/tests/e2e/tests/oauth.spec.ts new file mode 100644 index 0000000..814203b --- /dev/null +++ b/tests/e2e/tests/oauth.spec.ts @@ -0,0 +1,47 @@ +import { test, expect } from '@playwright/test'; +import { OAUTH_TARGETS, url, BASE_DOMAIN } from '../targets'; +import { completeAuthentikLogin } from '../helpers/authentik'; + +// OIDC/OAuth federation flow — the real SSO integration signal. A healthy app pod +// does NOT prove SSO works: client mis-registration, redirect-URI mismatch, issuer +// cert, or a broken Authentik provider all fail HERE, not at the pod. Each test: +// 1. open the app, +// 2. click its "sign in via Authentik" control, +// 3. complete the Authentik login on the redirect, +// 4. assert we return to the app authenticated. +const USER = process.env.AK_ADMIN_USER ?? 'akadmin'; +const PASS = process.env.AK_ADMIN_PASSWORD; + +for (const t of OAUTH_TARGETS) { + test(`${t.name} SSO login via Authentik`, async ({ page }) => { + test.skip(!PASS, 'AK_ADMIN_PASSWORD not set — provide via secret to run OAuth flows'); + + await page.goto(url(t), { waitUntil: 'domcontentloaded' }); + + // Click the SSO control. + const b = t.ssoButton; + const btn = b.selector + ? page.locator(b.selector).first() + : page.getByRole((b.role as any) ?? 'link', { name: b.name! }).first(); + await expect(btn, `${t.name}: SSO login control not found`).toBeVisible(); + await btn.click(); + + // We should be redirected to the Authentik domain (or already have a session). + await page.waitForLoadState('domcontentloaded'); + if (page.url().includes(`.${BASE_DOMAIN}`) && /authentik|\/if\/flow/i.test(page.url())) { + await completeAuthentikLogin(page, USER, PASS!); + } else { + // Not obviously on Authentik — still attempt, in case creds render inline. + await completeAuthentikLogin(page, USER, PASS!).catch(() => {}); + } + + // Back in the app, authenticated. + const s = t.success; + if (s.urlRe) await expect(page).toHaveURL(s.urlRe, { timeout: 25_000 }); + if (s.selector) await expect(page.locator(s.selector).first()).toBeVisible({ timeout: 25_000 }); + if (s.text) await expect(page.getByText(s.text).first()).toBeVisible({ timeout: 25_000 }); + + await expect(page.getByText(/invalid|unauthorized|access denied|redirect_uri/i)).toHaveCount(0); + await page.screenshot({ path: `test-results/oauth-${t.name}.png`, fullPage: true }); + }); +} diff --git a/tests/e2e/tests/portainer.spec.ts b/tests/e2e/tests/portainer.spec.ts new file mode 100644 index 0000000..a90a84f --- /dev/null +++ b/tests/e2e/tests/portainer.spec.ts @@ -0,0 +1,14 @@ +import { test, expect } from '@playwright/test'; +import { TARGETS, url } from '../targets'; + +const t = TARGETS.find((x) => x.name === 'portainer')!; + +// Portainer: prove the login UI renders over real ingress+TLS in Safari's engine. +// (Full authenticated flow needs an initial-admin password; add once seeded.) +test('portainer login page renders', async ({ page }) => { + const resp = await page.goto(url(t), { waitUntil: 'domcontentloaded' }); + expect(resp?.status(), 'ingress did not serve portainer').toBeLessThan(400); + + await expect(page.locator('input[type="password"]').first()).toBeVisible(); + await page.screenshot({ path: 'test-results/portainer-login.png', fullPage: true }); +}); diff --git a/tests/e2e/tests/reachability.spec.ts b/tests/e2e/tests/reachability.spec.ts new file mode 100644 index 0000000..0fe6148 --- /dev/null +++ b/tests/e2e/tests/reachability.spec.ts @@ -0,0 +1,33 @@ +import { test, expect } from '@playwright/test'; +import { TARGETS, url } from '../targets'; + +// Core assertion the user asked for: once ingress-nginx is up and the app is +// ready, NAME-APP. must be VIEWABLE in Safari's engine — not just return +// a status code. We navigate, assert a real 2xx/3xx (not nginx 404/503 default +// backend), and assert the app's own UI actually rendered. + +// Only apps without a dedicated flow spec (auth logins live in their own file). +const smokeTargets = TARGETS.filter((t) => !t.dedicated); + +for (const t of smokeTargets) { + test(`${t.name} is viewable at ${url(t)}`, async ({ page }) => { + const resp = await page.goto(url(t), { waitUntil: 'domcontentloaded' }); + + expect(resp, 'no response from ingress').toBeTruthy(); + const status = resp!.status(); + expect(status, `unexpected HTTP status ${status}`).toBeLessThan(400); + + // Not the nginx default backend / error page. + await expect(page.locator('body')).not.toContainText(/default backend - 404|503 Service Temporarily/i); + + // App's own UI rendered. + const r = t.ready; + if (r.selector) { + await expect(page.locator(r.selector).first()).toBeVisible(); + } else if (r.role && r.name) { + await expect(page.getByRole(r.role as any, { name: r.name }).first()).toBeVisible(); + } else if (r.text) { + await expect(page.getByText(r.text).first()).toBeVisible(); + } + }); +}