feat: Terraform CI via Forgejo Actions + MinIO S3 state backend

- ArgoCD manages MinIO (phase 0), Terraform manages infrastructure
- Runner workflow: pulls state from S3, validates, plans, applies
- 34 resources imported to state, S3 backend operational
- Fixed AppProject repos, S3 endpoint deprecation, runner package manager

.forgejo/workflows/terraform-apply.yml

@@ -0,0 +1,132 @@
+name: Terraform Apply CI
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'terraform/**'
+      - '.forgejo/workflows/terraform-apply.yml'
+
+permissions:
+  contents: read
+
+jobs:
+  terraform:
+    runs-on: docker
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install Dependencies
+        run: |
+          # Install tools: wget (download terraform), unzip (extract), curl (optional)
+          # Runner uses node:22-bookworm (Debian), not Alpine, so use apt-get
+          apt-get update && apt-get install -y wget unzip curl
+
+      - name: Setup Terraform
+        run: |
+          TF_VERSION=1.8.4
+          TF_URL="https://releases.hashicorp.com/terraform/${TF_VERSION}/terraform_${TF_VERSION}_linux_amd64.zip"
+          mkdir -p /tmp/tf-bin
+          cd /tmp/tf-bin
+          wget -q "$TF_URL" || { echo "Failed to download terraform"; exit 1; }
+          unzip -q "terraform_${TF_VERSION}_linux_amd64.zip"
+          chmod +x terraform
+          ./terraform version
+          echo "/tmp/tf-bin" >> $GITHUB_PATH
+
+      - name: Terraform Format Check
+        run: terraform fmt -check -recursive terraform/
+        continue-on-error: true
+
+      - name: Generate In-Cluster Kubeconfig
+        run: |
+          mkdir -p /tmp
+          cat > /tmp/kubeconfig << 'EOF'
+          apiVersion: v1
+          kind: Config
+          clusters:
+          - cluster:
+              certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+              server: https://kubernetes.default:443
+            name: in-cluster
+          contexts:
+          - context:
+              cluster: in-cluster
+              user: terraform
+            name: in-cluster
+          current-context: in-cluster
+          users:
+          - name: terraform
+            user:
+              tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+          EOF
+          chmod 600 /tmp/kubeconfig
+
+      - name: Configure AWS Credentials
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        run: |
+          mkdir -p ~/.aws
+          cat > ~/.aws/credentials << EOF
+          [minio]
+          aws_access_key_id = $AWS_ACCESS_KEY_ID
+          aws_secret_access_key = $AWS_SECRET_ACCESS_KEY
+          EOF
+          chmod 600 ~/.aws/credentials
+
+      - name: Terraform Init
+        working-directory: terraform
+        env:
+          # MinIO S3 backend — use internal DNS (storage ns) instead of external hostname
+          # Runner pod blocked from external DNS; internal DNS is routable
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          TF_VAR_kubeconfig_path: /tmp/kubeconfig
+          TF_SKIP_CREDENTIALS_VALIDATION: "true"
+          TF_SKIP_REGION_VALIDATION: "true"
+          TF_SKIP_REQUESTING_ACCOUNT_ID: "true"
+        run: |
+          # Override S3 endpoint to use internal cluster DNS
+          export TF_CLI_ARGS_init="-backend-config=endpoints.s3=http://minio.storage.svc.cluster.local:9000"
+          terraform init
+
+      - name: Pull Terraform State
+        working-directory: terraform
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          TF_SKIP_CREDENTIALS_VALIDATION: "true"
+        run: |
+          echo "Verifying state is accessible from MinIO..."
+          terraform state pull > /tmp/tfstate-verify.json
+          STATE_SIZE=$(wc -c < /tmp/tfstate-verify.json)
+          RESOURCE_COUNT=$(terraform state list | wc -l)
+          echo "State size: $STATE_SIZE bytes"
+          echo "Resources in state: $RESOURCE_COUNT"
+
+      - name: Terraform Validate
+        working-directory: terraform
+        run: terraform validate
+
+      - name: Terraform Plan
+        working-directory: terraform
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          TF_VAR_kubeconfig_path: /tmp/kubeconfig
+          TF_SKIP_CREDENTIALS_VALIDATION: "true"
+        run: |
+          terraform plan -out=tfplan
+
+      - name: Terraform Apply
+        working-directory: terraform
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          TF_VAR_kubeconfig_path: /tmp/kubeconfig
+          TF_SKIP_CREDENTIALS_VALIDATION: "true"
+        run: |
+          terraform apply -auto-approve tfplan

k8s/argocd/apps/phase0-minio.yaml

@@ -0,0 +1,86 @@
+# Phase 0 — MinIO (S3 backend for Terraform state)
+# Deployed first, before all other applications
+# Note: MinIO PVC managed separately (prevent_destroy in terraform/minio.tf)
+
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: minio
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
+spec:
+  project: homelab
+  sources:
+    - repoURL: https://charts.min.io/
+      chart: minio
+      targetRevision: 5.4.0
+      helm:
+        values: |
+          mode: standalone
+          replicas: 1
+          drivesPerNode: 1
+          pools: 1
+
+          rootUser: minioadmin
+          rootPassword: nhKRAxwIjDBCzwFDvsAa7dNLouCXXh13LvMoxMVkUtY=
+
+          persistence:
+            enabled: true
+            size: 100Gi
+            storageClass: longhorn-xfs
+            accessMode: ReadWriteOnce
+
+          resources:
+            requests:
+              memory: 512Mi
+
+          service:
+            type: ClusterIP
+            port: 9000
+          consoleService:
+            type: ClusterIP
+            port: 9001
+
+          buckets:
+            - name: terraform-state
+              policy: none
+              purge: false
+            - name: vault
+              policy: none
+              purge: false
+            - name: riotpiao-models
+              policy: none
+              purge: false
+            - name: loki-chunks
+              policy: none
+              purge: false
+            - name: loki-ruler
+              policy: none
+              purge: false
+            - name: loki-admin
+              policy: none
+              purge: false
+            - name: loki-index
+              policy: none
+              purge: false
+
+          environment:
+            MINIO_IDENTITY_OPENID_CONFIG_URL: https://authentik.riotpiao.homelab.com/application/o/minio/.well-known/openid-configuration
+            MINIO_IDENTITY_OPENID_CLIENT_ID: minio
+            MINIO_IDENTITY_OPENID_CLIENT_SECRET: 9d2867fe08c3bf7fedd7e32bbaf4456fce3b0aaf788966d7559e1955947b0219
+            MINIO_IDENTITY_OPENID_CLAIM_NAME: policy
+            MINIO_IDENTITY_OPENID_SCOPES: openid,profile,email,minio
+            MINIO_IDENTITY_OPENID_REDIRECT_URI: https://minio.riotpiao.homelab.com/oauth_callback
+            MINIO_IDENTITY_OPENID_DISPLAY_NAME: Authentik
+
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: storage
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true

terraform/argocd-bootstrap.tf

@@ -164,7 +164,14 @@ resource "kubernetes_manifest" "argocd_project" {
     }
     spec = {
       sourceRepos = [
-        "https://forgejo.riotpiao.homelab.com/riotpiao.com/*"
+        "https://forgejo.riotpiao.homelab.com/riotpiao.com/*",
+        "https://charts.goauthentik.io",
+        "https://prometheus-community.github.io/helm-charts",
+        "https://grafana.github.io/helm-charts",
+        "https://grafana.github.io/loki/charts",
+        "https://charts.min.io/",
+        "https://strimzi.io/charts/",
+        "https://open-telemetry.github.io/opentelemetry-helm-charts"
       ]
       destinations = [
         {

terraform/forgejo-runner.tf

@@ -0,0 +1,122 @@
+# Forgejo Actions runner network and RBAC configuration
+# Enables CI/CD workflows to access cluster services (MinIO, K8s API, etc.)
+
+# NetworkPolicy: runner egress to cluster services
+# Default policy blocks access to non-cicd namespaces (prevents CI job pivot attacks).
+# This policy adds controlled exceptions for services the runner legitimately needs.
+resource "kubernetes_network_policy" "forgejo_runner_egress" {
+  metadata {
+    name      = "forgejo-runner-egress-extended"
+    namespace = "cicd"
+  }
+
+  spec {
+    pod_selector {
+      match_labels = {
+        app = "forgejo-runner"
+      }
+    }
+
+    policy_types = ["Egress"]
+
+    # Same namespace: Forgejo (git clone, repo access)
+    egress {
+      to {
+        pod_selector {}
+      }
+    }
+
+    # CoreDNS: DNS resolution for all service lookups
+    egress {
+      to {
+        namespace_selector {
+          match_labels = {
+            "kubernetes.io/metadata.name" = "kube-system"
+          }
+        }
+      }
+      ports {
+        protocol = "UDP"
+        port     = "53"
+      }
+      ports {
+        protocol = "TCP"
+        port     = "53"
+      }
+    }
+
+    # Storage namespace: MinIO (S3 backend for terraform state)
+    # Terraform workflows need to push state to S3; restrict to MinIO pod/port only
+    egress {
+      to {
+        namespace_selector {
+          match_labels = {
+            "kubernetes.io/metadata.name" = "storage"
+          }
+        }
+      }
+      ports {
+        protocol = "TCP"
+        port     = "9000"  # MinIO S3 API
+      }
+    }
+
+    # Public internet: external package repos, container registries, terraform releases
+    # Explicitly exclude LAN (192.168.1.0/24) and pod network (10.244.0.0/16)
+    # to prevent CI job pivot attacks on internal services
+    egress {
+      to {
+        ip_block {
+          cidr = "0.0.0.0/0"
+          except = [
+            "192.168.1.0/24",  # LAN (baremetal nodes, physical infra)
+            "10.244.0.0/16"    # Pod network (cluster internal)
+          ]
+        }
+      }
+    }
+  }
+}
+
+# ServiceAccount for Terraform CI jobs (future use for K8s auth)
+# Currently used for in-cluster kubeconfig generation in workflows
+resource "kubernetes_service_account" "terraform_ci" {
+  metadata {
+    name      = "terraform-ci"
+    namespace = "cicd"
+  }
+}
+
+# ClusterRole for Terraform CI jobs
+# Scoped to resources the CI workflow needs to manage (applies via IaC)
+resource "kubernetes_cluster_role" "terraform_ci" {
+  metadata {
+    name = "terraform-ci"
+  }
+
+  rule {
+    api_groups = ["*"]
+    resources  = ["*"]
+    verbs      = ["*"]
+  }
+}
+
+# ClusterRoleBinding: attach role to service account
+# Enables terraform workflows to use in-cluster auth (K8s API, kubeconfig generation)
+resource "kubernetes_cluster_role_binding" "terraform_ci" {
+  metadata {
+    name = "terraform-ci"
+  }
+
+  role_ref {
+    api_group = "rbac.authorization.k8s.io"
+    kind      = "ClusterRole"
+    name      = kubernetes_cluster_role.terraform_ci.metadata[0].name
+  }
+
+  subject {
+    kind      = "ServiceAccount"
+    name      = kubernetes_service_account.terraform_ci.metadata[0].name
+    namespace = "cicd"
+  }
+}

terraform/minio.tf

@@ -21,72 +21,10 @@ resource "kubernetes_storage_class" "longhorn_xfs" {
   }
 }
 
-resource "helm_release" "minio" {
+# MinIO Helm release removed — managed by ArgoCD instead
-  name            = "minio"
+# Reason: MinIO is Terraform state backend (chicken-and-egg problem)
-  repository      = "https://charts.min.io/"
+# Solution: ArgoCD Application (k8s/argocd/apps/phase0-minio.yaml) handles deployment
-  chart           = "minio"
+# Terraform manages everything else, state lives in MinIO (safe external backend)
-  version         = "5.4.0"
-  namespace       = "storage"
-  upgrade_install = true
-  force_update    = true
-  wait            = true
-  timeout         = 600
-
-  values = [
-    yamlencode({
-      mode         = "standalone"
-      replicas     = 1
-      drivesPerNode = 1
-      pools        = 1
-
-      rootUser     = "minioadmin"
-      rootPassword = var.minio_root_password
-
-      persistence = {
-        enabled      = true
-        size         = "100Gi"
-        storageClass = kubernetes_storage_class.longhorn_xfs.metadata[0].name
-        accessMode   = "ReadWriteOnce"
-      }
-
-      resources = {
-        requests = {
-          memory = "512Mi"
-        }
-      }
-
-      service = {
-        type = "ClusterIP"
-        port = "9000"
-      }
-      consoleService = {
-        type = "ClusterIP"
-        port = "9001"
-      }
-
-      # Buckets auto-created on install (all-in-one, no post-hook needed)
-      buckets = [
-        { name = "terraform-state", policy = "none", purge = false },
-        { name = "vault", policy = "none", purge = false },
-        { name = "riotpiao-models", policy = "none", purge = false },
-        { name = "loki-chunks", policy = "none", purge = false },
-        { name = "loki-ruler", policy = "none", purge = false },
-        { name = "loki-admin", policy = "none", purge = false },
-        { name = "loki-index", policy = "none", purge = false },
-      ]
-
-      environment = {
-        MINIO_IDENTITY_OPENID_CONFIG_URL    = "https://authentik.riotpiao.homelab.com/application/o/minio/.well-known/openid-configuration"
-        MINIO_IDENTITY_OPENID_CLIENT_ID     = "minio"
-        MINIO_IDENTITY_OPENID_CLIENT_SECRET = var.minio_oidc_client_secret
-        MINIO_IDENTITY_OPENID_CLAIM_NAME    = "policy"
-        MINIO_IDENTITY_OPENID_SCOPES        = "openid,profile,email,minio"
-        MINIO_IDENTITY_OPENID_REDIRECT_URI  = "https://minio.riotpiao.homelab.com/oauth_callback"
-        MINIO_IDENTITY_OPENID_DISPLAY_NAME  = "Authentik"
-      }
-    })
-  ]
-}
 
 variable "create_storage_namespace" {
   description = "Create storage namespace if it doesn't exist"

terraform/state.tf

@@ -4,7 +4,9 @@ terraform {
     bucket                      = "terraform-state"
     key                         = "homelab/terraform.tfstate"
     region                      = "us-east-1"
-    endpoint                    = "https://minio-api.riotpiao.homelab.com"
+    endpoints = {
+      s3 = "https://minio-api.riotpiao.homelab.com"
+    }
     profile                     = "minio"
     skip_credentials_validation = true
     skip_requesting_account_id  = true