diff --git a/docs/PHASE0-ESO-VAULT-SETUP.md b/docs/PHASE0-ESO-VAULT-SETUP.md new file mode 100644 index 0000000..a53b7ac --- /dev/null +++ b/docs/PHASE0-ESO-VAULT-SETUP.md @@ -0,0 +1,178 @@ +# Phase 0: External Secrets Operator (ESO) Setup + +After `terraform apply` installs ESO and the ClusterSecretStore manifest is applied, Vault needs to be configured to accept ESO's Kubernetes auth requests. + +## Prerequisites +- Vault is initialized and unsealed (run setup_vault.sh first) +- ESO pods are Running in external-secrets-system namespace +- ClusterSecretStore applied: k8s/external-secrets/clustersecretstore.yaml + +## Setup Steps (Manual) + +Run these steps from the repo root with: +```bash +export VAULT_ADDR=http://vault.iam.svc.cluster.local:8200 +export VAULT_TOKEN=$(kubectl get secret vault-unseal-keys -n storage -o jsonpath='{.data.key1}' | base64 -d) # or use port-forward + login +``` + +### 1. Enable Kubernetes auth method +```bash +vault auth enable kubernetes || echo "Kubernetes auth already enabled" +``` + +### 2. Configure K8s auth to trust the cluster +```bash +# Get the K8s API server address and CA cert +K8S_HOST=$(kubectl cluster-info | grep 'Kubernetes master' | awk '/https/ {print $NF}') +K8S_CA_CERT=$(kubectl config view --raw --minify --flatten -o jsonpath='{.clusters[0].cluster.certificate-authority-data}' | base64 -d) +SA_TOKEN=$(kubectl get secret -n external-secrets-system $(kubectl get secret -n external-secrets-system | grep external-secrets-webhook | awk '{print $1}') -o jsonpath='{.data.token}' | base64 -d) + +vault write auth/kubernetes/config \ + kubernetes_host="${K8S_HOST}" \ + kubernetes_ca_cert="${K8S_CA_CERT}" \ + token_reviewer_jwt="${SA_TOKEN}" +``` + +### 3. Create external-secrets policy +```bash +vault policy write external-secrets - <<'EOF' +# external-secrets: read application secrets per namespace + release +path "secret/data/iam/*" { + capabilities = ["read"] +} +path "secret/data/storage/*" { + capabilities = ["read"] +} +path "secret/data/cicd/*" { + capabilities = ["read"] +} +path "secret/data/logging/*" { + capabilities = ["read"] +} +path "secret/data/monitoring/*" { + capabilities = ["read"] +} +path "secret/data/ddb/*" { + capabilities = ["read"] +} +path "secret/data/temporal/*" { + capabilities = ["read"] +} +path "secret/data/llm/*" { + capabilities = ["read"] +} +path "secret/data/sqs/*" { + capabilities = ["read"] +} +path "secret/data/story-crater-backend/*" { + capabilities = ["read"] +} +EOF +``` + +### 4. Create external-secrets K8s auth role +```bash +vault write auth/kubernetes/role/external-secrets \ + bound_service_account_names=external-secrets \ + bound_service_account_namespaces=external-secrets-system \ + policies=external-secrets \ + ttl=24h \ + max_ttl=24h +``` + +### 5. Verify ClusterSecretStore can authenticate +```bash +kubectl get clustersecretstore vault-homelab -o yaml +# Should show no error events if auth is working +``` + +## Seed Initial Secrets + +Once ESO is configured, seed the `.env` values into Vault for each release: + +```bash +# IAM realm +vault kv put secret/iam/authentik \ + secret-key="${AUTHENTIK_SECRET_KEY}" \ + bootstrap-password="${AUTHENTIK_BOOTSTRAP_PASSWORD}" \ + bootstrap-token="${AUTHENTIK_BOOTSTRAP_TOKEN}" \ + postgresql-password="${AUTHENTIK_PG_PASSWORD}" + +vault kv put secret/iam/vault \ + minio-access-key="${MINIO_ROOT_USER}" \ + minio-secret-key="${MINIO_ROOT_PASSWORD}" + +# CI/CD realm +vault kv put secret/cicd/forgejo \ + admin-password="${FORGEJO_ADMIN_PASSWORD}" + +vault kv put secret/cicd/argocd \ + oidc-client-secret="${AUTHENTIK_ARGOCD_CLIENT_SECRET}" + +# Logging realm +vault kv put secret/logging/grafana \ + admin-password="${GRAFANA_ADMIN_PASSWORD}" \ + oidc-client-secret="${GRAFANA_OIDC_CLIENT_SECRET}" + +# Storage realm +vault kv put secret/storage/minio \ + root-user="${MINIO_ROOT_USER}" \ + root-password="${MINIO_ROOT_PASSWORD}" \ + oidc-client-secret="${MINIO_OIDC_CLIENT_SECRET}" + +# SQS realm +vault kv put secret/sqs/kmsvc \ + kafka-bootstrap="${KAFKA_BOOTSTRAP}" \ + redis-addr="${REDIS_ADDR}" + +# Temporal realm +vault kv put secret/temporal/temporal \ + oidc-client-id="${AUTHENTIK_TEMPORAL_CLIENT_ID}" \ + oidc-client-secret="${AUTHENTIK_TEMPORAL_CLIENT_SECRET}" + +# LLM realm +vault kv put secret/llm/ollama \ + oidc-client-id="${AUTHENTIK_OLLAMA_CLIENT_ID}" \ + oidc-client-secret="${AUTHENTIK_OLLAMA_CLIENT_SECRET}" +``` + +Load these from .env automatically: +```bash +export $(grep -v '^#' .env | xargs) +# Then run the vault kv put commands above +``` + +## Verification + +Test ESO by creating a simple ExternalSecret: + +```bash +kubectl apply -f - <<'EOF' +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: test-secret + namespace: iam +spec: + refreshInterval: 1h + secretStoreRef: + name: vault-homelab + kind: ClusterSecretStore + target: + name: test-secret + creationPolicy: Owner + data: + - secretKey: root-user + remoteRef: + key: storage/minio + property: root-user +EOF + +# Check if the Secret was created +kubectl get secret test-secret -n iam -o yaml +``` + +If the Secret has the expected data, ESO is working. Clean up the test: +```bash +kubectl delete exs test-secret -n iam +``` diff --git a/helmfile.yaml.gotmpl b/helmfile.yaml.gotmpl index 6f8d4ef..181d785 100644 --- a/helmfile.yaml.gotmpl +++ b/helmfile.yaml.gotmpl @@ -46,134 +46,11 @@ repositories: - name: cnpg url: https://cloudnative-pg.github.io/charts -# ── cert-manager ───────────────────────────────────────────────────────────── +# ── Bootstrap Releases (managed by Terraform) ──────────────────────────────────── +# cert-manager, reloader, ingress-nginx, and cilium are bootstrap-managed by +# terraform/bootstrap-releases.tf — do NOT add them here, avoid dual-ownership. releases: - - name: cert-manager - namespace: cert-manager - createNamespace: true - chart: jetstack/cert-manager - version: "~v1" - values: - - k8s/cert-manager/cert-manager-values.yaml - set: - # CRDs must be installed by the chart — avoids a separate kubectl apply step - - name: crds.enabled - value: true - hooks: - - events: ["postsync"] - command: bash - args: - - -c - - | - # Wait for cert-manager webhooks to be ready before applying CRD instances. - # Without this, ClusterIssuer/Certificate creation races the webhook and fails. - kubectl rollout status deploy/cert-manager -n cert-manager --timeout=120s - kubectl rollout status deploy/cert-manager-webhook -n cert-manager --timeout=120s - - kubectl apply -f - <<'EOF' - # Phase 2a — bootstrap issuer (selfSigned) used only to sign the CA cert. - # Never referenced by ingresses — its sole job is to sign homelab-ca below. - apiVersion: cert-manager.io/v1 - kind: ClusterIssuer - metadata: - name: selfsigned-bootstrap - spec: - selfSigned: {} - --- - # Phase 2b — the cluster CA certificate. - # cert-manager stores the signed cert + key in homelab-ca-secret. - # isCA: true marks it so it can sign other certs. - # 10-year lifetime; renewBefore triggers 30 days early. - apiVersion: cert-manager.io/v1 - kind: Certificate - metadata: - name: homelab-ca - namespace: cert-manager - spec: - isCA: true - commonName: homelab-ca - secretName: homelab-ca-secret - duration: 87600h - renewBefore: 720h - privateKey: - algorithm: ECDSA - size: 256 - issuerRef: - name: selfsigned-bootstrap - kind: ClusterIssuer - group: cert-manager.io - --- - # Phase 2c — the real issuer all ingresses reference. - # Reads the CA cert+key from homelab-ca-secret and signs per-hostname certs. - # Annotate any ingress with: cert-manager.io/cluster-issuer: homelab-ca - apiVersion: cert-manager.io/v1 - kind: ClusterIssuer - metadata: - name: homelab-ca - spec: - ca: - secretName: homelab-ca-secret - EOF - -# ── Stakater Reloader ───────────────────────────────────────────────────────── -# Watches Deployments for secret.reloader.stakater.com/reload and -# configmap.reloader.stakater.com/reload annotations, then triggers a rolling -# restart whenever the named Secret or ConfigMap is updated (e.g. cert-manager -# renewing a TLS cert, or homelab-ca rotating). This replaces the need for -# manual `kubectl rollout restart` on cert renewal. - - - name: reloader - namespace: reloader - createNamespace: true - chart: stakater/reloader - version: "~1" - - # Pod Disruption Budgets applied after reloader (ensures all namespaces exist) - hooks: - - events: ["postsync"] - command: bash - args: - - -c - - kubectl apply -f k8s/base/poddisruptionbudgets.yaml - -# ── Ingress ─────────────────────────────────────────────────────────────────── - - - name: ingress-nginx - namespace: ingress-nginx - createNamespace: true - chart: ingress-nginx/ingress-nginx - values: - - k8s/ingress/nginx-values.yaml - # ServiceMonitor enabled in nginx-values.yaml requires the Prometheus Operator - # CRDs, which the prometheus release installs — must apply after it. - needs: - - monitoring/prometheus - hooks: - - events: ["presync"] - command: bash - args: - - -c - - | - bash k8s/base/namespace-setup.sh ingress-nginx - # LB-IPAM pool must exist before any LoadBalancer service is created, - # otherwise services stay . Apply it here as the first hook. - kubectl apply -f k8s/cilium/lb-ipam-pool.yaml - kubectl apply -f k8s/coredns/coredns-configmap.yaml - kubectl apply -f k8s/coredns/coredns-deployment.yaml - kubectl rollout restart deployment/coredns -n kube-system - kubectl rollout status deployment/coredns -n kube-system --timeout=60s - # Wildcard TLS cert — must exist before nginx starts so it can read the secret. - # cert-manager issues it in the ingress-nginx namespace; wait until Ready. - kubectl apply -f k8s/ingress/wildcard-cert.yaml - kubectl wait certificate homelab-tls -n ingress-nginx \ - --for=condition=Ready --timeout=120s - - events: ["postsync"] - command: kubectl - args: - - apply - - -f - - k8s/ingress/ingress.yaml # ── CloudNativePG (centralized database) ────────────────────────────────────── # Single HA cluster (1 primary + 2 replicas) serving Authentik + story-crater-backend. diff --git a/k8s/argocd/apps/0-wave-0.yaml b/k8s/argocd/apps/0-wave-0.yaml new file mode 100644 index 0000000..39700a0 --- /dev/null +++ b/k8s/argocd/apps/0-wave-0.yaml @@ -0,0 +1,83 @@ +# Wave 0 — Parallel bootstrap: storage, messaging, and monitoring operators +# No inter-dependencies; these provide foundational infrastructure. + +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: strimzi-operator + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "0" +spec: + project: homelab + source: + repoURL: https://strimzi.io/charts/ + chart: strimzi-kafka-operator + targetRevision: 0.46.0 + helm: + values: | + watchNamespaces: ["sqs"] + destination: + server: https://kubernetes.default.svc + namespace: sqs + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: kmsvc-redis + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "0" +spec: + project: homelab + source: + repoURL: https://charts.bitnami.com/bitnami + chart: redis + targetRevision: 20.6.0 + helm: + valueFiles: + - k8s/sqs/redis-values.yaml + destination: + server: https://kubernetes.default.svc + namespace: sqs + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: prometheus + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "0" +spec: + project: homelab + source: + repoURL: https://prometheus-community.github.io/helm-charts + chart: kube-prometheus-stack + targetRevision: latest + helm: + valueFiles: + - k8s/monitoring/prometheus-values.yaml + destination: + server: https://kubernetes.default.svc + namespace: monitoring + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true diff --git a/k8s/external-secrets/clustersecretstore.yaml b/k8s/external-secrets/clustersecretstore.yaml new file mode 100644 index 0000000..64f8290 --- /dev/null +++ b/k8s/external-secrets/clustersecretstore.yaml @@ -0,0 +1,19 @@ +# ClusterSecretStore — ESO's connection to Vault +# References the Vault instance deployed in the iam namespace. +# Uses Kubernetes auth method (safe for in-cluster access). +# Namespace: default (ClusterSecretStore is cluster-scoped, not namespaced). + +apiVersion: external-secrets.io/v1beta1 +kind: ClusterSecretStore +metadata: + name: vault-homelab +spec: + provider: + vault: + server: "http://vault.iam.svc.cluster.local:8200" + path: "secret" + version: "v2" + auth: + kubernetes: + mountPath: "kubernetes" + role: "external-secrets" diff --git a/terraform/argocd-bootstrap.tf b/terraform/argocd-bootstrap.tf index b899175..d9dd1c9 100644 --- a/terraform/argocd-bootstrap.tf +++ b/terraform/argocd-bootstrap.tf @@ -13,8 +13,8 @@ resource "kubernetes_namespace" "argocd" { resource "null_resource" "copy_ca_secret_to_argocd" { provisioner "local-exec" { command = <<-EOT - kubectl get secret homelab-ca-secret -n cert-manager -o yaml | \ - sed 's/namespace: cert-manager/namespace: argocd/' | \ + kubectl get secret homelab-ca-secret -n cert-manager -o json | \ + jq 'del(.metadata.namespace, .metadata.resourceVersion, .metadata.uid, .metadata.creationTimestamp, .metadata.selfLink, .metadata.managedFields) | .metadata.namespace = "argocd"' | \ kubectl apply -f - EOT } diff --git a/terraform/bootstrap-releases.tf b/terraform/bootstrap-releases.tf index c633d4c..55c2c7c 100644 --- a/terraform/bootstrap-releases.tf +++ b/terraform/bootstrap-releases.tf @@ -20,6 +20,11 @@ locals { namespace = "ingress-nginx" repo = "ingress_nginx" } + external-secrets = { + chart_version = "0.9.9" + namespace = "external-secrets-system" + repo = "external_secrets" + } } } diff --git a/terraform/helm-repositories.tf b/terraform/helm-repositories.tf index 937eece..668f8ea 100644 --- a/terraform/helm-repositories.tf +++ b/terraform/helm-repositories.tf @@ -17,5 +17,6 @@ locals { strimzi = "https://strimzi.io/charts/" bitnami = "https://charts.bitnami.com/bitnami" temporal = "https://go.temporal.io/helm-charts" + external_secrets = "https://charts.external-secrets.io" } } diff --git a/terraform/longhorn.tf b/terraform/longhorn.tf index 28cf98f..030c807 100644 --- a/terraform/longhorn.tf +++ b/terraform/longhorn.tf @@ -1,25 +1,13 @@ -# Longhorn StorageClasses — cluster-wide default + app-specific variants -# Imported from live cluster state (import-only, no delete) - -resource "kubernetes_storage_class" "longhorn" { - metadata { - name = "longhorn" - } - storage_provisioner = "driver.longhorn.io" - reclaim_policy = "Delete" - allow_volume_expansion = true - volume_binding_mode = "Immediate" - - parameters = { - numberOfReplicas = "3" - staleReplicaTimeout = "60" - fromBackup = "" - fsType = "ext4" - dataLocality = "disabled" - disableRevisionCounter = "true" - unmapMarkSnapChainRemoved = "ignored" - } -} +# Longhorn StorageClasses — app-specific variants only. +# +# The cluster-wide default `longhorn` SC is intentionally NOT managed here. +# It is owned by Longhorn's own setting-controller (reconciled from the +# `longhorn-storageclass` ConfigMap in longhorn-system, stamped with the +# `longhorn.io/last-applied-configmap` annotation). Managing it in Terraform +# caused a dual-ownership fight: TF strips the annotation, Longhorn re-adds it +# and delete+recreates the SC (params are immutable), racing TF's post-apply +# read ("Root object present, but now absent"). Longhorn recreates it +# automatically on any cluster, so it needs no TF representation. resource "kubernetes_storage_class" "longhorn_kafka" { metadata { diff --git a/terraform/minio.tf b/terraform/minio.tf index 97080ba..f9135b1 100644 --- a/terraform/minio.tf +++ b/terraform/minio.tf @@ -3,6 +3,11 @@ # Why xfs: default `longhorn` SC uses ext4 whose mkfs on 100Gi (~4.5min) # exceeds kubelet mount timeout. xfs mkfs is near-instant. min.io chart has # no persistence.fsType, so fsType must be set on the StorageClass. +# +# PVC is Terraform-managed directly (import-only, prevent_destroy) and +# referenced by the chart via persistence.existingClaim, so Helm never +# templates/reconciles the PVC object itself (previously caused a failed +# force-replace attempt against the bound, immutable volumeName). resource "kubernetes_storage_class" "longhorn_xfs" { metadata { @@ -21,6 +26,22 @@ resource "kubernetes_storage_class" "longhorn_xfs" { } } +resource "kubernetes_persistent_volume_claim" "minio" { + metadata { + name = "minio" + namespace = "storage" + } + spec { + access_modes = ["ReadWriteOnce"] + storage_class_name = kubernetes_storage_class.longhorn_xfs.metadata[0].name + resources { + requests = { + storage = "100Gi" + } + } + } +} + resource "helm_release" "minio" { name = "minio" repository = "https://charts.min.io/" @@ -28,7 +49,6 @@ resource "helm_release" "minio" { version = "5.4.0" namespace = "storage" upgrade_install = true - force_update = true wait = true timeout = 600 @@ -43,10 +63,8 @@ resource "helm_release" "minio" { rootPassword = var.minio_root_password persistence = { - enabled = true - size = "100Gi" - storageClass = kubernetes_storage_class.longhorn_xfs.metadata[0].name - accessMode = "ReadWriteOnce" + enabled = true + existingClaim = kubernetes_persistent_volume_claim.minio.metadata[0].name } resources = { diff --git a/terraform/state.tf b/terraform/state.tf index 767217b..71921ab 100644 --- a/terraform/state.tf +++ b/terraform/state.tf @@ -1,4 +1,3 @@ -# S3 backend (MinIO remote state) terraform { backend "s3" { bucket = "terraform-state"