diff --git a/docs/PHASE0-ESO-VAULT-SETUP.md b/docs/PHASE0-ESO-VAULT-SETUP.md deleted file mode 100644 index a53b7ac..0000000 --- a/docs/PHASE0-ESO-VAULT-SETUP.md +++ /dev/null @@ -1,178 +0,0 @@ -# Phase 0: External Secrets Operator (ESO) Setup - -After `terraform apply` installs ESO and the ClusterSecretStore manifest is applied, Vault needs to be configured to accept ESO's Kubernetes auth requests. - -## Prerequisites -- Vault is initialized and unsealed (run setup_vault.sh first) -- ESO pods are Running in external-secrets-system namespace -- ClusterSecretStore applied: k8s/external-secrets/clustersecretstore.yaml - -## Setup Steps (Manual) - -Run these steps from the repo root with: -```bash -export VAULT_ADDR=http://vault.iam.svc.cluster.local:8200 -export VAULT_TOKEN=$(kubectl get secret vault-unseal-keys -n storage -o jsonpath='{.data.key1}' | base64 -d) # or use port-forward + login -``` - -### 1. Enable Kubernetes auth method -```bash -vault auth enable kubernetes || echo "Kubernetes auth already enabled" -``` - -### 2. Configure K8s auth to trust the cluster -```bash -# Get the K8s API server address and CA cert -K8S_HOST=$(kubectl cluster-info | grep 'Kubernetes master' | awk '/https/ {print $NF}') -K8S_CA_CERT=$(kubectl config view --raw --minify --flatten -o jsonpath='{.clusters[0].cluster.certificate-authority-data}' | base64 -d) -SA_TOKEN=$(kubectl get secret -n external-secrets-system $(kubectl get secret -n external-secrets-system | grep external-secrets-webhook | awk '{print $1}') -o jsonpath='{.data.token}' | base64 -d) - -vault write auth/kubernetes/config \ - kubernetes_host="${K8S_HOST}" \ - kubernetes_ca_cert="${K8S_CA_CERT}" \ - token_reviewer_jwt="${SA_TOKEN}" -``` - -### 3. Create external-secrets policy -```bash -vault policy write external-secrets - <<'EOF' -# external-secrets: read application secrets per namespace + release -path "secret/data/iam/*" { - capabilities = ["read"] -} -path "secret/data/storage/*" { - capabilities = ["read"] -} -path "secret/data/cicd/*" { - capabilities = ["read"] -} -path "secret/data/logging/*" { - capabilities = ["read"] -} -path "secret/data/monitoring/*" { - capabilities = ["read"] -} -path "secret/data/ddb/*" { - capabilities = ["read"] -} -path "secret/data/temporal/*" { - capabilities = ["read"] -} -path "secret/data/llm/*" { - capabilities = ["read"] -} -path "secret/data/sqs/*" { - capabilities = ["read"] -} -path "secret/data/story-crater-backend/*" { - capabilities = ["read"] -} -EOF -``` - -### 4. Create external-secrets K8s auth role -```bash -vault write auth/kubernetes/role/external-secrets \ - bound_service_account_names=external-secrets \ - bound_service_account_namespaces=external-secrets-system \ - policies=external-secrets \ - ttl=24h \ - max_ttl=24h -``` - -### 5. Verify ClusterSecretStore can authenticate -```bash -kubectl get clustersecretstore vault-homelab -o yaml -# Should show no error events if auth is working -``` - -## Seed Initial Secrets - -Once ESO is configured, seed the `.env` values into Vault for each release: - -```bash -# IAM realm -vault kv put secret/iam/authentik \ - secret-key="${AUTHENTIK_SECRET_KEY}" \ - bootstrap-password="${AUTHENTIK_BOOTSTRAP_PASSWORD}" \ - bootstrap-token="${AUTHENTIK_BOOTSTRAP_TOKEN}" \ - postgresql-password="${AUTHENTIK_PG_PASSWORD}" - -vault kv put secret/iam/vault \ - minio-access-key="${MINIO_ROOT_USER}" \ - minio-secret-key="${MINIO_ROOT_PASSWORD}" - -# CI/CD realm -vault kv put secret/cicd/forgejo \ - admin-password="${FORGEJO_ADMIN_PASSWORD}" - -vault kv put secret/cicd/argocd \ - oidc-client-secret="${AUTHENTIK_ARGOCD_CLIENT_SECRET}" - -# Logging realm -vault kv put secret/logging/grafana \ - admin-password="${GRAFANA_ADMIN_PASSWORD}" \ - oidc-client-secret="${GRAFANA_OIDC_CLIENT_SECRET}" - -# Storage realm -vault kv put secret/storage/minio \ - root-user="${MINIO_ROOT_USER}" \ - root-password="${MINIO_ROOT_PASSWORD}" \ - oidc-client-secret="${MINIO_OIDC_CLIENT_SECRET}" - -# SQS realm -vault kv put secret/sqs/kmsvc \ - kafka-bootstrap="${KAFKA_BOOTSTRAP}" \ - redis-addr="${REDIS_ADDR}" - -# Temporal realm -vault kv put secret/temporal/temporal \ - oidc-client-id="${AUTHENTIK_TEMPORAL_CLIENT_ID}" \ - oidc-client-secret="${AUTHENTIK_TEMPORAL_CLIENT_SECRET}" - -# LLM realm -vault kv put secret/llm/ollama \ - oidc-client-id="${AUTHENTIK_OLLAMA_CLIENT_ID}" \ - oidc-client-secret="${AUTHENTIK_OLLAMA_CLIENT_SECRET}" -``` - -Load these from .env automatically: -```bash -export $(grep -v '^#' .env | xargs) -# Then run the vault kv put commands above -``` - -## Verification - -Test ESO by creating a simple ExternalSecret: - -```bash -kubectl apply -f - <<'EOF' -apiVersion: external-secrets.io/v1beta1 -kind: ExternalSecret -metadata: - name: test-secret - namespace: iam -spec: - refreshInterval: 1h - secretStoreRef: - name: vault-homelab - kind: ClusterSecretStore - target: - name: test-secret - creationPolicy: Owner - data: - - secretKey: root-user - remoteRef: - key: storage/minio - property: root-user -EOF - -# Check if the Secret was created -kubectl get secret test-secret -n iam -o yaml -``` - -If the Secret has the expected data, ESO is working. Clean up the test: -```bash -kubectl delete exs test-secret -n iam -``` diff --git a/helmfile.yaml.gotmpl b/helmfile.yaml.gotmpl index 181d785..6f8d4ef 100644 --- a/helmfile.yaml.gotmpl +++ b/helmfile.yaml.gotmpl @@ -46,11 +46,134 @@ repositories: - name: cnpg url: https://cloudnative-pg.github.io/charts -# ── Bootstrap Releases (managed by Terraform) ──────────────────────────────────── -# cert-manager, reloader, ingress-nginx, and cilium are bootstrap-managed by -# terraform/bootstrap-releases.tf — do NOT add them here, avoid dual-ownership. +# ── cert-manager ───────────────────────────────────────────────────────────── releases: + - name: cert-manager + namespace: cert-manager + createNamespace: true + chart: jetstack/cert-manager + version: "~v1" + values: + - k8s/cert-manager/cert-manager-values.yaml + set: + # CRDs must be installed by the chart — avoids a separate kubectl apply step + - name: crds.enabled + value: true + hooks: + - events: ["postsync"] + command: bash + args: + - -c + - | + # Wait for cert-manager webhooks to be ready before applying CRD instances. + # Without this, ClusterIssuer/Certificate creation races the webhook and fails. + kubectl rollout status deploy/cert-manager -n cert-manager --timeout=120s + kubectl rollout status deploy/cert-manager-webhook -n cert-manager --timeout=120s + + kubectl apply -f - <<'EOF' + # Phase 2a — bootstrap issuer (selfSigned) used only to sign the CA cert. + # Never referenced by ingresses — its sole job is to sign homelab-ca below. + apiVersion: cert-manager.io/v1 + kind: ClusterIssuer + metadata: + name: selfsigned-bootstrap + spec: + selfSigned: {} + --- + # Phase 2b — the cluster CA certificate. + # cert-manager stores the signed cert + key in homelab-ca-secret. + # isCA: true marks it so it can sign other certs. + # 10-year lifetime; renewBefore triggers 30 days early. + apiVersion: cert-manager.io/v1 + kind: Certificate + metadata: + name: homelab-ca + namespace: cert-manager + spec: + isCA: true + commonName: homelab-ca + secretName: homelab-ca-secret + duration: 87600h + renewBefore: 720h + privateKey: + algorithm: ECDSA + size: 256 + issuerRef: + name: selfsigned-bootstrap + kind: ClusterIssuer + group: cert-manager.io + --- + # Phase 2c — the real issuer all ingresses reference. + # Reads the CA cert+key from homelab-ca-secret and signs per-hostname certs. + # Annotate any ingress with: cert-manager.io/cluster-issuer: homelab-ca + apiVersion: cert-manager.io/v1 + kind: ClusterIssuer + metadata: + name: homelab-ca + spec: + ca: + secretName: homelab-ca-secret + EOF + +# ── Stakater Reloader ───────────────────────────────────────────────────────── +# Watches Deployments for secret.reloader.stakater.com/reload and +# configmap.reloader.stakater.com/reload annotations, then triggers a rolling +# restart whenever the named Secret or ConfigMap is updated (e.g. cert-manager +# renewing a TLS cert, or homelab-ca rotating). This replaces the need for +# manual `kubectl rollout restart` on cert renewal. + + - name: reloader + namespace: reloader + createNamespace: true + chart: stakater/reloader + version: "~1" + + # Pod Disruption Budgets applied after reloader (ensures all namespaces exist) + hooks: + - events: ["postsync"] + command: bash + args: + - -c + - kubectl apply -f k8s/base/poddisruptionbudgets.yaml + +# ── Ingress ─────────────────────────────────────────────────────────────────── + + - name: ingress-nginx + namespace: ingress-nginx + createNamespace: true + chart: ingress-nginx/ingress-nginx + values: + - k8s/ingress/nginx-values.yaml + # ServiceMonitor enabled in nginx-values.yaml requires the Prometheus Operator + # CRDs, which the prometheus release installs — must apply after it. + needs: + - monitoring/prometheus + hooks: + - events: ["presync"] + command: bash + args: + - -c + - | + bash k8s/base/namespace-setup.sh ingress-nginx + # LB-IPAM pool must exist before any LoadBalancer service is created, + # otherwise services stay . Apply it here as the first hook. + kubectl apply -f k8s/cilium/lb-ipam-pool.yaml + kubectl apply -f k8s/coredns/coredns-configmap.yaml + kubectl apply -f k8s/coredns/coredns-deployment.yaml + kubectl rollout restart deployment/coredns -n kube-system + kubectl rollout status deployment/coredns -n kube-system --timeout=60s + # Wildcard TLS cert — must exist before nginx starts so it can read the secret. + # cert-manager issues it in the ingress-nginx namespace; wait until Ready. + kubectl apply -f k8s/ingress/wildcard-cert.yaml + kubectl wait certificate homelab-tls -n ingress-nginx \ + --for=condition=Ready --timeout=120s + - events: ["postsync"] + command: kubectl + args: + - apply + - -f + - k8s/ingress/ingress.yaml # ── CloudNativePG (centralized database) ────────────────────────────────────── # Single HA cluster (1 primary + 2 replicas) serving Authentik + story-crater-backend. diff --git a/k8s/argocd/apps/0-wave-0.yaml b/k8s/argocd/apps/0-wave-0.yaml deleted file mode 100644 index 39700a0..0000000 --- a/k8s/argocd/apps/0-wave-0.yaml +++ /dev/null @@ -1,83 +0,0 @@ -# Wave 0 — Parallel bootstrap: storage, messaging, and monitoring operators -# No inter-dependencies; these provide foundational infrastructure. - ---- -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: strimzi-operator - namespace: argocd - annotations: - argocd.argoproj.io/sync-wave: "0" -spec: - project: homelab - source: - repoURL: https://strimzi.io/charts/ - chart: strimzi-kafka-operator - targetRevision: 0.46.0 - helm: - values: | - watchNamespaces: ["sqs"] - destination: - server: https://kubernetes.default.svc - namespace: sqs - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true - ---- -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: kmsvc-redis - namespace: argocd - annotations: - argocd.argoproj.io/sync-wave: "0" -spec: - project: homelab - source: - repoURL: https://charts.bitnami.com/bitnami - chart: redis - targetRevision: 20.6.0 - helm: - valueFiles: - - k8s/sqs/redis-values.yaml - destination: - server: https://kubernetes.default.svc - namespace: sqs - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true - ---- -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: prometheus - namespace: argocd - annotations: - argocd.argoproj.io/sync-wave: "0" -spec: - project: homelab - source: - repoURL: https://prometheus-community.github.io/helm-charts - chart: kube-prometheus-stack - targetRevision: latest - helm: - valueFiles: - - k8s/monitoring/prometheus-values.yaml - destination: - server: https://kubernetes.default.svc - namespace: monitoring - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true diff --git a/k8s/external-secrets/clustersecretstore.yaml b/k8s/external-secrets/clustersecretstore.yaml deleted file mode 100644 index 64f8290..0000000 --- a/k8s/external-secrets/clustersecretstore.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# ClusterSecretStore — ESO's connection to Vault -# References the Vault instance deployed in the iam namespace. -# Uses Kubernetes auth method (safe for in-cluster access). -# Namespace: default (ClusterSecretStore is cluster-scoped, not namespaced). - -apiVersion: external-secrets.io/v1beta1 -kind: ClusterSecretStore -metadata: - name: vault-homelab -spec: - provider: - vault: - server: "http://vault.iam.svc.cluster.local:8200" - path: "secret" - version: "v2" - auth: - kubernetes: - mountPath: "kubernetes" - role: "external-secrets" diff --git a/terraform/argocd-bootstrap.tf b/terraform/argocd-bootstrap.tf index d9dd1c9..b899175 100644 --- a/terraform/argocd-bootstrap.tf +++ b/terraform/argocd-bootstrap.tf @@ -13,8 +13,8 @@ resource "kubernetes_namespace" "argocd" { resource "null_resource" "copy_ca_secret_to_argocd" { provisioner "local-exec" { command = <<-EOT - kubectl get secret homelab-ca-secret -n cert-manager -o json | \ - jq 'del(.metadata.namespace, .metadata.resourceVersion, .metadata.uid, .metadata.creationTimestamp, .metadata.selfLink, .metadata.managedFields) | .metadata.namespace = "argocd"' | \ + kubectl get secret homelab-ca-secret -n cert-manager -o yaml | \ + sed 's/namespace: cert-manager/namespace: argocd/' | \ kubectl apply -f - EOT } diff --git a/terraform/bootstrap-releases.tf b/terraform/bootstrap-releases.tf index 55c2c7c..c633d4c 100644 --- a/terraform/bootstrap-releases.tf +++ b/terraform/bootstrap-releases.tf @@ -20,11 +20,6 @@ locals { namespace = "ingress-nginx" repo = "ingress_nginx" } - external-secrets = { - chart_version = "0.9.9" - namespace = "external-secrets-system" - repo = "external_secrets" - } } } diff --git a/terraform/helm-repositories.tf b/terraform/helm-repositories.tf index 668f8ea..937eece 100644 --- a/terraform/helm-repositories.tf +++ b/terraform/helm-repositories.tf @@ -17,6 +17,5 @@ locals { strimzi = "https://strimzi.io/charts/" bitnami = "https://charts.bitnami.com/bitnami" temporal = "https://go.temporal.io/helm-charts" - external_secrets = "https://charts.external-secrets.io" } } diff --git a/terraform/longhorn.tf b/terraform/longhorn.tf index 030c807..28cf98f 100644 --- a/terraform/longhorn.tf +++ b/terraform/longhorn.tf @@ -1,13 +1,25 @@ -# Longhorn StorageClasses — app-specific variants only. -# -# The cluster-wide default `longhorn` SC is intentionally NOT managed here. -# It is owned by Longhorn's own setting-controller (reconciled from the -# `longhorn-storageclass` ConfigMap in longhorn-system, stamped with the -# `longhorn.io/last-applied-configmap` annotation). Managing it in Terraform -# caused a dual-ownership fight: TF strips the annotation, Longhorn re-adds it -# and delete+recreates the SC (params are immutable), racing TF's post-apply -# read ("Root object present, but now absent"). Longhorn recreates it -# automatically on any cluster, so it needs no TF representation. +# Longhorn StorageClasses — cluster-wide default + app-specific variants +# Imported from live cluster state (import-only, no delete) + +resource "kubernetes_storage_class" "longhorn" { + metadata { + name = "longhorn" + } + storage_provisioner = "driver.longhorn.io" + reclaim_policy = "Delete" + allow_volume_expansion = true + volume_binding_mode = "Immediate" + + parameters = { + numberOfReplicas = "3" + staleReplicaTimeout = "60" + fromBackup = "" + fsType = "ext4" + dataLocality = "disabled" + disableRevisionCounter = "true" + unmapMarkSnapChainRemoved = "ignored" + } +} resource "kubernetes_storage_class" "longhorn_kafka" { metadata { diff --git a/terraform/minio.tf b/terraform/minio.tf index f9135b1..97080ba 100644 --- a/terraform/minio.tf +++ b/terraform/minio.tf @@ -3,11 +3,6 @@ # Why xfs: default `longhorn` SC uses ext4 whose mkfs on 100Gi (~4.5min) # exceeds kubelet mount timeout. xfs mkfs is near-instant. min.io chart has # no persistence.fsType, so fsType must be set on the StorageClass. -# -# PVC is Terraform-managed directly (import-only, prevent_destroy) and -# referenced by the chart via persistence.existingClaim, so Helm never -# templates/reconciles the PVC object itself (previously caused a failed -# force-replace attempt against the bound, immutable volumeName). resource "kubernetes_storage_class" "longhorn_xfs" { metadata { @@ -26,22 +21,6 @@ resource "kubernetes_storage_class" "longhorn_xfs" { } } -resource "kubernetes_persistent_volume_claim" "minio" { - metadata { - name = "minio" - namespace = "storage" - } - spec { - access_modes = ["ReadWriteOnce"] - storage_class_name = kubernetes_storage_class.longhorn_xfs.metadata[0].name - resources { - requests = { - storage = "100Gi" - } - } - } -} - resource "helm_release" "minio" { name = "minio" repository = "https://charts.min.io/" @@ -49,6 +28,7 @@ resource "helm_release" "minio" { version = "5.4.0" namespace = "storage" upgrade_install = true + force_update = true wait = true timeout = 600 @@ -63,8 +43,10 @@ resource "helm_release" "minio" { rootPassword = var.minio_root_password persistence = { - enabled = true - existingClaim = kubernetes_persistent_volume_claim.minio.metadata[0].name + enabled = true + size = "100Gi" + storageClass = kubernetes_storage_class.longhorn_xfs.metadata[0].name + accessMode = "ReadWriteOnce" } resources = { diff --git a/terraform/state.tf b/terraform/state.tf index 71921ab..767217b 100644 --- a/terraform/state.tf +++ b/terraform/state.tf @@ -1,3 +1,4 @@ +# S3 backend (MinIO remote state) terraform { backend "s3" { bucket = "terraform-state"