Revert "feat(phase0): bootstrap External Secrets Operator and fix helmfile dual-ownership"

This reverts commit e7f3409d0f.

docs/PHASE0-ESO-VAULT-SETUP.md

@@ -1,178 +0,0 @@
-# Phase 0: External Secrets Operator (ESO) Setup
-
-After `terraform apply` installs ESO and the ClusterSecretStore manifest is applied, Vault needs to be configured to accept ESO's Kubernetes auth requests.
-
-## Prerequisites
-- Vault is initialized and unsealed (run setup_vault.sh first)
-- ESO pods are Running in external-secrets-system namespace
-- ClusterSecretStore applied: k8s/external-secrets/clustersecretstore.yaml
-
-## Setup Steps (Manual)
-
-Run these steps from the repo root with:
-```bash
-export VAULT_ADDR=http://vault.iam.svc.cluster.local:8200
-export VAULT_TOKEN=$(kubectl get secret vault-unseal-keys -n storage -o jsonpath='{.data.key1}' | base64 -d)  # or use port-forward + login
-```
-
-### 1. Enable Kubernetes auth method
-```bash
-vault auth enable kubernetes || echo "Kubernetes auth already enabled"
-```
-
-### 2. Configure K8s auth to trust the cluster
-```bash
-# Get the K8s API server address and CA cert
-K8S_HOST=$(kubectl cluster-info | grep 'Kubernetes master' | awk '/https/ {print $NF}')
-K8S_CA_CERT=$(kubectl config view --raw --minify --flatten -o jsonpath='{.clusters[0].cluster.certificate-authority-data}' | base64 -d)
-SA_TOKEN=$(kubectl get secret -n external-secrets-system $(kubectl get secret -n external-secrets-system | grep external-secrets-webhook | awk '{print $1}') -o jsonpath='{.data.token}' | base64 -d)
-
-vault write auth/kubernetes/config \
-  kubernetes_host="${K8S_HOST}" \
-  kubernetes_ca_cert="${K8S_CA_CERT}" \
-  token_reviewer_jwt="${SA_TOKEN}"
-```
-
-### 3. Create external-secrets policy
-```bash
-vault policy write external-secrets - <<'EOF'
-# external-secrets: read application secrets per namespace + release
-path "secret/data/iam/*" {
-  capabilities = ["read"]
-}
-path "secret/data/storage/*" {
-  capabilities = ["read"]
-}
-path "secret/data/cicd/*" {
-  capabilities = ["read"]
-}
-path "secret/data/logging/*" {
-  capabilities = ["read"]
-}
-path "secret/data/monitoring/*" {
-  capabilities = ["read"]
-}
-path "secret/data/ddb/*" {
-  capabilities = ["read"]
-}
-path "secret/data/temporal/*" {
-  capabilities = ["read"]
-}
-path "secret/data/llm/*" {
-  capabilities = ["read"]
-}
-path "secret/data/sqs/*" {
-  capabilities = ["read"]
-}
-path "secret/data/story-crater-backend/*" {
-  capabilities = ["read"]
-}
-EOF
-```
-
-### 4. Create external-secrets K8s auth role
-```bash
-vault write auth/kubernetes/role/external-secrets \
-  bound_service_account_names=external-secrets \
-  bound_service_account_namespaces=external-secrets-system \
-  policies=external-secrets \
-  ttl=24h \
-  max_ttl=24h
-```
-
-### 5. Verify ClusterSecretStore can authenticate
-```bash
-kubectl get clustersecretstore vault-homelab -o yaml
-# Should show no error events if auth is working
-```
-
-## Seed Initial Secrets
-
-Once ESO is configured, seed the `.env` values into Vault for each release:
-
-```bash
-# IAM realm
-vault kv put secret/iam/authentik \
-  secret-key="${AUTHENTIK_SECRET_KEY}" \
-  bootstrap-password="${AUTHENTIK_BOOTSTRAP_PASSWORD}" \
-  bootstrap-token="${AUTHENTIK_BOOTSTRAP_TOKEN}" \
-  postgresql-password="${AUTHENTIK_PG_PASSWORD}"
-
-vault kv put secret/iam/vault \
-  minio-access-key="${MINIO_ROOT_USER}" \
-  minio-secret-key="${MINIO_ROOT_PASSWORD}"
-
-# CI/CD realm
-vault kv put secret/cicd/forgejo \
-  admin-password="${FORGEJO_ADMIN_PASSWORD}"
-
-vault kv put secret/cicd/argocd \
-  oidc-client-secret="${AUTHENTIK_ARGOCD_CLIENT_SECRET}"
-
-# Logging realm
-vault kv put secret/logging/grafana \
-  admin-password="${GRAFANA_ADMIN_PASSWORD}" \
-  oidc-client-secret="${GRAFANA_OIDC_CLIENT_SECRET}"
-
-# Storage realm
-vault kv put secret/storage/minio \
-  root-user="${MINIO_ROOT_USER}" \
-  root-password="${MINIO_ROOT_PASSWORD}" \
-  oidc-client-secret="${MINIO_OIDC_CLIENT_SECRET}"
-
-# SQS realm
-vault kv put secret/sqs/kmsvc \
-  kafka-bootstrap="${KAFKA_BOOTSTRAP}" \
-  redis-addr="${REDIS_ADDR}"
-
-# Temporal realm
-vault kv put secret/temporal/temporal \
-  oidc-client-id="${AUTHENTIK_TEMPORAL_CLIENT_ID}" \
-  oidc-client-secret="${AUTHENTIK_TEMPORAL_CLIENT_SECRET}"
-
-# LLM realm
-vault kv put secret/llm/ollama \
-  oidc-client-id="${AUTHENTIK_OLLAMA_CLIENT_ID}" \
-  oidc-client-secret="${AUTHENTIK_OLLAMA_CLIENT_SECRET}"
-```
-
-Load these from .env automatically:
-```bash
-export $(grep -v '^#' .env | xargs)
-# Then run the vault kv put commands above
-```
-
-## Verification
-
-Test ESO by creating a simple ExternalSecret:
-
-```bash
-kubectl apply -f - <<'EOF'
-apiVersion: external-secrets.io/v1beta1
-kind: ExternalSecret
-metadata:
-  name: test-secret
-  namespace: iam
-spec:
-  refreshInterval: 1h
-  secretStoreRef:
-    name: vault-homelab
-    kind: ClusterSecretStore
-  target:
-    name: test-secret
-    creationPolicy: Owner
-  data:
-    - secretKey: root-user
-      remoteRef:
-        key: storage/minio
-        property: root-user
-EOF
-
-# Check if the Secret was created
-kubectl get secret test-secret -n iam -o yaml
-```
-
-If the Secret has the expected data, ESO is working. Clean up the test:
-```bash
-kubectl delete exs test-secret -n iam
-```

helmfile.yaml.gotmpl

@@ -46,11 +46,134 @@ repositories:
   - name: cnpg
     url: https://cloudnative-pg.github.io/charts
 
-# ── Bootstrap Releases (managed by Terraform) ────────────────────────────────────
+# ── cert-manager ─────────────────────────────────────────────────────────────
-# cert-manager, reloader, ingress-nginx, and cilium are bootstrap-managed by
-# terraform/bootstrap-releases.tf — do NOT add them here, avoid dual-ownership.
 
 releases:
+  - name: cert-manager
+    namespace: cert-manager
+    createNamespace: true
+    chart: jetstack/cert-manager
+    version: "~v1"
+    values:
+      - k8s/cert-manager/cert-manager-values.yaml
+    set:
+      # CRDs must be installed by the chart — avoids a separate kubectl apply step
+      - name: crds.enabled
+        value: true
+    hooks:
+      - events: ["postsync"]
+        command: bash
+        args:
+          - -c
+          - |
+            # Wait for cert-manager webhooks to be ready before applying CRD instances.
+            # Without this, ClusterIssuer/Certificate creation races the webhook and fails.
+            kubectl rollout status deploy/cert-manager -n cert-manager --timeout=120s
+            kubectl rollout status deploy/cert-manager-webhook -n cert-manager --timeout=120s
+
+            kubectl apply -f - <<'EOF'
+            # Phase 2a — bootstrap issuer (selfSigned) used only to sign the CA cert.
+            # Never referenced by ingresses — its sole job is to sign homelab-ca below.
+            apiVersion: cert-manager.io/v1
+            kind: ClusterIssuer
+            metadata:
+              name: selfsigned-bootstrap
+            spec:
+              selfSigned: {}
+            ---
+            # Phase 2b — the cluster CA certificate.
+            # cert-manager stores the signed cert + key in homelab-ca-secret.
+            # isCA: true marks it so it can sign other certs.
+            # 10-year lifetime; renewBefore triggers 30 days early.
+            apiVersion: cert-manager.io/v1
+            kind: Certificate
+            metadata:
+              name: homelab-ca
+              namespace: cert-manager
+            spec:
+              isCA: true
+              commonName: homelab-ca
+              secretName: homelab-ca-secret
+              duration: 87600h
+              renewBefore: 720h
+              privateKey:
+                algorithm: ECDSA
+                size: 256
+              issuerRef:
+                name: selfsigned-bootstrap
+                kind: ClusterIssuer
+                group: cert-manager.io
+            ---
+            # Phase 2c — the real issuer all ingresses reference.
+            # Reads the CA cert+key from homelab-ca-secret and signs per-hostname certs.
+            # Annotate any ingress with: cert-manager.io/cluster-issuer: homelab-ca
+            apiVersion: cert-manager.io/v1
+            kind: ClusterIssuer
+            metadata:
+              name: homelab-ca
+            spec:
+              ca:
+                secretName: homelab-ca-secret
+            EOF
+
+# ── Stakater Reloader ─────────────────────────────────────────────────────────
+# Watches Deployments for secret.reloader.stakater.com/reload and
+# configmap.reloader.stakater.com/reload annotations, then triggers a rolling
+# restart whenever the named Secret or ConfigMap is updated (e.g. cert-manager
+# renewing a TLS cert, or homelab-ca rotating). This replaces the need for
+# manual `kubectl rollout restart` on cert renewal.
+
+  - name: reloader
+    namespace: reloader
+    createNamespace: true
+    chart: stakater/reloader
+    version: "~1"
+
+  # Pod Disruption Budgets applied after reloader (ensures all namespaces exist)
+    hooks:
+      - events: ["postsync"]
+        command: bash
+        args:
+          - -c
+          - kubectl apply -f k8s/base/poddisruptionbudgets.yaml
+
+# ── Ingress ───────────────────────────────────────────────────────────────────
+
+  - name: ingress-nginx
+    namespace: ingress-nginx
+    createNamespace: true
+    chart: ingress-nginx/ingress-nginx
+    values:
+      - k8s/ingress/nginx-values.yaml
+    # ServiceMonitor enabled in nginx-values.yaml requires the Prometheus Operator
+    # CRDs, which the prometheus release installs — must apply after it.
+    needs:
+      - monitoring/prometheus
+    hooks:
+      - events: ["presync"]
+        command: bash
+        args:
+          - -c
+          - |
+            bash k8s/base/namespace-setup.sh ingress-nginx
+            # LB-IPAM pool must exist before any LoadBalancer service is created,
+            # otherwise services stay <pending>. Apply it here as the first hook.
+            kubectl apply -f k8s/cilium/lb-ipam-pool.yaml
+            kubectl apply -f k8s/coredns/coredns-configmap.yaml
+            kubectl apply -f k8s/coredns/coredns-deployment.yaml
+            kubectl rollout restart deployment/coredns -n kube-system
+            kubectl rollout status deployment/coredns -n kube-system --timeout=60s
+            # Wildcard TLS cert — must exist before nginx starts so it can read the secret.
+            # cert-manager issues it in the ingress-nginx namespace; wait until Ready.
+            kubectl apply -f k8s/ingress/wildcard-cert.yaml
+            kubectl wait certificate homelab-tls -n ingress-nginx \
+              --for=condition=Ready --timeout=120s
+      - events: ["postsync"]
+        command: kubectl
+        args:
+          - apply
+          - -f
+          - k8s/ingress/ingress.yaml
 
 # ── CloudNativePG (centralized database) ──────────────────────────────────────
 # Single HA cluster (1 primary + 2 replicas) serving Authentik + story-crater-backend.

k8s/argocd/apps/0-wave-0.yaml

@@ -1,83 +0,0 @@
-# Wave 0 — Parallel bootstrap: storage, messaging, and monitoring operators
-# No inter-dependencies; these provide foundational infrastructure.
-
----
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: strimzi-operator
-  namespace: argocd
-  annotations:
-    argocd.argoproj.io/sync-wave: "0"
-spec:
-  project: homelab
-  source:
-    repoURL: https://strimzi.io/charts/
-    chart: strimzi-kafka-operator
-    targetRevision: 0.46.0
-    helm:
-      values: |
-        watchNamespaces: ["sqs"]
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: sqs
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
-    syncOptions:
-      - CreateNamespace=true
-
----
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: kmsvc-redis
-  namespace: argocd
-  annotations:
-    argocd.argoproj.io/sync-wave: "0"
-spec:
-  project: homelab
-  source:
-    repoURL: https://charts.bitnami.com/bitnami
-    chart: redis
-    targetRevision: 20.6.0
-    helm:
-      valueFiles:
-        - k8s/sqs/redis-values.yaml
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: sqs
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
-    syncOptions:
-      - CreateNamespace=true
-
----
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: prometheus
-  namespace: argocd
-  annotations:
-    argocd.argoproj.io/sync-wave: "0"
-spec:
-  project: homelab
-  source:
-    repoURL: https://prometheus-community.github.io/helm-charts
-    chart: kube-prometheus-stack
-    targetRevision: latest
-    helm:
-      valueFiles:
-        - k8s/monitoring/prometheus-values.yaml
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: monitoring
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
-    syncOptions:
-      - CreateNamespace=true

k8s/external-secrets/clustersecretstore.yaml

@@ -1,19 +0,0 @@
-# ClusterSecretStore — ESO's connection to Vault
-# References the Vault instance deployed in the iam namespace.
-# Uses Kubernetes auth method (safe for in-cluster access).
-# Namespace: default (ClusterSecretStore is cluster-scoped, not namespaced).
-
-apiVersion: external-secrets.io/v1beta1
-kind: ClusterSecretStore
-metadata:
-  name: vault-homelab
-spec:
-  provider:
-    vault:
-      server: "http://vault.iam.svc.cluster.local:8200"
-      path: "secret"
-      version: "v2"
-      auth:
-        kubernetes:
-          mountPath: "kubernetes"
-          role: "external-secrets"

terraform/argocd-bootstrap.tf

@@ -13,8 +13,8 @@ resource "kubernetes_namespace" "argocd" {
 resource "null_resource" "copy_ca_secret_to_argocd" {
   provisioner "local-exec" {
     command = <<-EOT
-      kubectl get secret homelab-ca-secret -n cert-manager -o json | \
+      kubectl get secret homelab-ca-secret -n cert-manager -o yaml | \
-        jq 'del(.metadata.namespace, .metadata.resourceVersion, .metadata.uid, .metadata.creationTimestamp, .metadata.selfLink, .metadata.managedFields) | .metadata.namespace = "argocd"' | \
+        sed 's/namespace: cert-manager/namespace: argocd/' | \
         kubectl apply -f -
     EOT
   }

terraform/bootstrap-releases.tf

@@ -20,11 +20,6 @@ locals {
       namespace     = "ingress-nginx"
       repo          = "ingress_nginx"
     }
-    external-secrets = {
-      chart_version = "0.9.9"
-      namespace     = "external-secrets-system"
-      repo          = "external_secrets"
-    }
   }
 }
 

terraform/helm-repositories.tf

@@ -17,6 +17,5 @@ locals {
     strimzi              = "https://strimzi.io/charts/"
     bitnami              = "https://charts.bitnami.com/bitnami"
     temporal             = "https://go.temporal.io/helm-charts"
-    external_secrets     = "https://charts.external-secrets.io"
   }
 }

terraform/longhorn.tf

@@ -1,13 +1,25 @@
-# Longhorn StorageClasses — app-specific variants only.
+# Longhorn StorageClasses — cluster-wide default + app-specific variants
-#
+# Imported from live cluster state (import-only, no delete)
-# The cluster-wide default `longhorn` SC is intentionally NOT managed here.
+
-# It is owned by Longhorn's own setting-controller (reconciled from the
+resource "kubernetes_storage_class" "longhorn" {
-# `longhorn-storageclass` ConfigMap in longhorn-system, stamped with the
+  metadata {
-# `longhorn.io/last-applied-configmap` annotation). Managing it in Terraform
+    name = "longhorn"
-# caused a dual-ownership fight: TF strips the annotation, Longhorn re-adds it
+  }
-# and delete+recreates the SC (params are immutable), racing TF's post-apply
+  storage_provisioner    = "driver.longhorn.io"
-# read ("Root object present, but now absent"). Longhorn recreates it
+  reclaim_policy         = "Delete"
-# automatically on any cluster, so it needs no TF representation.
+  allow_volume_expansion = true
+  volume_binding_mode    = "Immediate"
+
+  parameters = {
+    numberOfReplicas          = "3"
+    staleReplicaTimeout       = "60"
+    fromBackup                = ""
+    fsType                    = "ext4"
+    dataLocality              = "disabled"
+    disableRevisionCounter    = "true"
+    unmapMarkSnapChainRemoved = "ignored"
+  }
+}
 
 resource "kubernetes_storage_class" "longhorn_kafka" {
   metadata {

terraform/minio.tf

@@ -3,11 +3,6 @@
 # Why xfs: default `longhorn` SC uses ext4 whose mkfs on 100Gi (~4.5min)
 # exceeds kubelet mount timeout. xfs mkfs is near-instant. min.io chart has
 # no persistence.fsType, so fsType must be set on the StorageClass.
-#
-# PVC is Terraform-managed directly (import-only, prevent_destroy) and
-# referenced by the chart via persistence.existingClaim, so Helm never
-# templates/reconciles the PVC object itself (previously caused a failed
-# force-replace attempt against the bound, immutable volumeName).
 
 resource "kubernetes_storage_class" "longhorn_xfs" {
   metadata {
@@ -26,22 +21,6 @@ resource "kubernetes_storage_class" "longhorn_xfs" {
   }
 }
 
-resource "kubernetes_persistent_volume_claim" "minio" {
-  metadata {
-    name      = "minio"
-    namespace = "storage"
-  }
-  spec {
-    access_modes       = ["ReadWriteOnce"]
-    storage_class_name = kubernetes_storage_class.longhorn_xfs.metadata[0].name
-    resources {
-      requests = {
-        storage = "100Gi"
-      }
-    }
-  }
-}
-
 resource "helm_release" "minio" {
   name            = "minio"
   repository      = "https://charts.min.io/"
@@ -49,6 +28,7 @@ resource "helm_release" "minio" {
   version         = "5.4.0"
   namespace       = "storage"
   upgrade_install = true
+  force_update    = true
   wait            = true
   timeout         = 600
 
@@ -63,8 +43,10 @@ resource "helm_release" "minio" {
       rootPassword = var.minio_root_password
 
       persistence = {
-        enabled       = true
+        enabled      = true
-        existingClaim = kubernetes_persistent_volume_claim.minio.metadata[0].name
+        size         = "100Gi"
+        storageClass = kubernetes_storage_class.longhorn_xfs.metadata[0].name
+        accessMode   = "ReadWriteOnce"
       }
 
       resources = {

terraform/state.tf

@@ -1,3 +1,4 @@
+# S3 backend (MinIO remote state)
 terraform {
   backend "s3" {
     bucket                      = "terraform-state"