fix(bootstrap): complete Phase 4 ArgoCD bootstrap with all permanent fixes

- Fix ArgoCD Application schema: move syncOptions under syncPolicy (00-secrets.yaml) - Remove helm install --wait flag (talos-cp-2 slow node timeout issue) - Add comprehensive progress logging with timestamps to bootstrap.sh - Fix SOPS key path (/Users/rockliang/.sops/key.txt, not homelab-age.key) - Add local SOPS decryption for bootstrap secrets - Add CNPG NetworkPolicy allowing app→database connectivity - Disable Forgejo bundled dependencies (saves 66Gi storage) - Inject database credentials via deployment.env (GITEA__DATABASE__*) - Remove invalid ext4 mount options from StorageClass - Add namespace manifests with PodSecurity labels - Add encrypted forgejo-admin secret (SOPS) - Reduce forgejo-db size 50Gi→25Gi per instance - Prepare ArgoCD SOPS CMP plugin (for post-bootstrap)
2026-07-25 12:24:30 -07:00
parent bf67d2d9de
commit 8d63db9f3b
13 changed files with 350 additions and 97 deletions
@@ -1,6 +1,10 @@
 # Longhorn Helm Values — Single Source of Truth
 # Used by both bootstrap.sh (Helm install) and ArgoCD (adoption)
 # Chart: https://github.com/longhorn/charts
+#
+# NOTE: Namespace requires PodSecurity=privileged labels (Longhorn needs host access).
+# bootstrap.sh applies namespace.yaml automatically. For manual install:
+#   kubectl apply -f k8s/bootstrap/phase1-storage/namespace.yaml

 defaultSettings:
  # 3-node HA configuration
@@ -52,6 +56,8 @@ longhornUI:
  replicas: 1

 # Monitoring (Prometheus ServiceMonitor)
+# Disabled during bootstrap (Prometheus CRDs not installed yet)
+# Re-enable via ArgoCD after Prometheus stack is deployed
 metrics:
  serviceMonitor:
-    enabled: true
+    enabled: false
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: longhorn-system
+  labels:
+    pod-security.kubernetes.io/enforce: privileged
+    pod-security.kubernetes.io/audit: privileged
+    pod-security.kubernetes.io/warn: privileged
@@ -2,8 +2,9 @@
 # The default 'longhorn' SC is created by Helm chart
 # These are additional specialized classes
 ---
-# CNPG-specific StorageClass with postgres UID/GID mount options
-# Fixes "read-only filesystem" error when PostgreSQL (UID 26) tries to write
+# CNPG-specific StorageClass for PostgreSQL volumes
+# CNPG handles filesystem ownership via securityContext.fsGroup (UID/GID 26)
+# Separate from default 'longhorn' to allow CNPG-specific tuning
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:
@@ -19,9 +20,7 @@ parameters:
  fromBackup: ""
  dataLocality: "best-effort"
  fsType: "ext4"
-mountOptions:
-  - "noatime"
-  - "uid=26"   # postgres user
-  - "gid=26"   # postgres group
+# mountOptions removed - uid/gid are NOT valid for ext4 (only for FAT/VFAT)
+# CNPG handles ownership via securityContext.fsGroup automatically
 reclaimPolicy: Delete
 volumeBindingMode: Immediate
@@ -19,8 +19,10 @@ webhook:
    failurePolicy: Fail

 # Monitoring
+# PodMonitor disabled during bootstrap (Prometheus CRDs not installed yet)
+# Re-enable via ArgoCD after Prometheus stack is deployed
 monitoring:
-  podMonitorEnabled: true
+  podMonitorEnabled: false
  grafanaDashboard:
    create: false  # We'll manage dashboards via ArgoCD later

@@ -0,0 +1,49 @@
+# NetworkPolicy for CNPG pods - allow pod-to-pod replication traffic
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: forgejo-db-allow-replication
+  namespace: cicd
+spec:
+  podSelector:
+    matchLabels:
+      cnpg.io/cluster: forgejo-db
+  policyTypes:
+    - Ingress
+    - Egress
+  ingress:
+    # Allow CNPG operator to reach instance status endpoints (port 8000)
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: cnpg-system
+      ports:
+        - protocol: TCP
+          port: 8000
+    # Allow PostgreSQL connections from application pods (Forgejo)
+    - from:
+        - podSelector: {}
+          namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: cicd
+      ports:
+        - protocol: TCP
+          port: 5432
+    # Allow PostgreSQL replication from other CNPG pods
+    - from:
+        - podSelector:
+            matchLabels:
+              cnpg.io/cluster: forgejo-db
+      ports:
+        - protocol: TCP
+          port: 5432
+    # Allow metrics scraping
+    - from:
+        - namespaceSelector: {}
+      ports:
+        - protocol: TCP
+          port: 9187
+  egress:
+    # Allow all egress (CNPG needs to reach services, API server, DNS, etc.)
+    # Restricting egress breaks replication through services
+    - {}
@@ -64,8 +64,8 @@ spec:
      log_filename: "postgres"

  storage:
-    size: 50Gi
-    storageClass: longhorn-cnpg  # Uses postgres UID/GID mount options
+    size: 25Gi
+    storageClass: longhorn-cnpg

  monitoring:
    enablePodMonitor: true
@@ -0,0 +1,26 @@
+#ENC[AES256_GCM,data:85lNBL02TeroUW8dissgxEwjnOZnb0fg7jndGim3/QSi3/hFEF8FQa9nrQ==,iv:lM1jBHIxFkQriZ6BjGRXAlymUs5nX4Kvt1WNxUeZlkU=,tag:wm/D5qVcrcu3VXcZkEj70A==,type:comment]
+apiVersion: ENC[AES256_GCM,data:Vrs=,iv:0TzPcIoozs2MXJNXkzgcVtjjBUgfOHaSXQZiD37fb+Q=,tag:CabTlLwtz6RcBF/gr4Ri4g==,type:str]
+kind: ENC[AES256_GCM,data:22Y5w+Df,iv:Mf2s3h8++Vxqb4JoymHXY4/WAknDZ2GGrVVtKK51JxI=,tag:k5DHUqWGMBPpLkQXADaMlw==,type:str]
+metadata:
+    name: ENC[AES256_GCM,data:ItXKquY5a7gu8CJZog==,iv:jxbj7Qtv+DRbhzTdvtv+eJuTQPNIf497NZPYA6ld4s0=,tag:XdaaSavFBxOjvxna2kr/Tg==,type:str]
+    namespace: ENC[AES256_GCM,data:VH6NMg==,iv:4PfWZu5qVGXP3ZzRHMrh5N9dzJ3SoUPPo58ppcDTnpk=,tag:Vr8E+WF0Z9ym3GyTGbdi2g==,type:str]
+type: ENC[AES256_GCM,data:DmbHZRIk,iv:EZHnf1h1L29G1HOBYBSBeydNe4nC8XiBOw8YEL3kxrY=,tag:pFhUSMAIEqJET3NmaajO1g==,type:str]
+stringData:
+    username: ENC[AES256_GCM,data:ZTrmFA==,iv:1+tLTAxrDitXJwCAEccaVQzc9I9lNRgT3FsxO2NPDDc=,tag:69Wn4a8pjhdUycc/MXsn3Q==,type:str]
+    password: ENC[AES256_GCM,data:w7Vn8XaC1ykNrwPpJjVYg8J5KXkUbaPspu0CoceqHVdai6BFNW5rtA==,iv:jAwDFvJfQ1GkeU/qpEVUAQ6cWqxYE8nrgs+/RouyUxg=,tag:foGmYika/9uNBdsKVrspEA==,type:str]
+    email: ENC[AES256_GCM,data:pIOq80OjVERFCXJmx8+qTJEw,iv:mfkMj3u8W2ZX4N4IH39mZXfEp+xphS4shaJcFOD/LEE=,tag:FOSq0E7VzzOM69Bn77v2Dw==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwSi84U3FQZXJrYzA0WCtS
+            c1g1Sm04eFhha2huSEVCd2x3OGVUT08yejFjCmpPV0NsTVRyWTRVUkx6WktKaU1k
+            Vk12VGRIejRWbjQxeDZxdFUxWkgzaVUKLS0tIDRZdFNxY1AvcFhOREMxU01zTzhX
+            RzFmMjdjOFd4ZkxscnBIa1E2NWtRaVEK7qZXtq2VwZBwsLAulRh93TpCXCos0Vu7
+            fX+/oMEN6gF5VxDJ/e5C644EKUY+tSLAoh75xA5DAytOxEVdhe2ozA==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1smu533f803gmd0jq60s2zaj9zlznajy0ca6rtewd4r37mr2hs3uqsrldfh
+    lastmodified: "2026-07-25T17:45:22Z"
+    mac: ENC[AES256_GCM,data:gi8smimG00EQbF492BVzsWOEubGdT0zSG39D+r3fgLL/wgtDqF2mgtFA0iDiy1pInt1Viu8yYpu1l/eBSD8BNW4bfrglIHOJdb1I5+19u8SMebhWCwbEcbiHaXNmVUKKapGsNjbApCApq00NXoOZuOUSUzZlh19JPvg/vd7nl2M=,iv:n8Kni4RQhj/VgIruqptVa6m0YwFJF6v2+p2boJsLT5c=,tag:Tc4gkhH7hSrJKEOJ3X3SjQ==,type:str]
+    unencrypted_suffix: _unencrypted
+    version: 3.13.2
@@ -1,11 +1,22 @@
 # Forgejo Helm Values — Single Source of Truth
 # Chart: https://codeberg.org/forgejo-contrib/forgejo-helm

+# Disable bundled dependencies (use external CNPG + Redis instead)
+postgresql-ha:
+  enabled: false
+
+valkey:
+  enabled: false
+
+valkey-cluster:
+  enabled: false
+
+redis:
+  enabled: false
+
 gitea:
  admin:
-    username: "admin"
-    email: "[email protected]"
-    # Password set via secret (not in values)
+    existingSecret: forgejo-admin

  config:
    server:
@@ -18,17 +29,7 @@ gitea:
      DB_TYPE: postgres
      HOST: forgejo-db-rw.cicd.svc.cluster.local:5432
      NAME: forgejo
-      # User/password from CNPG-generated secret
-      USER:
-        valueFrom:
-          secretKeyRef:
-            name: forgejo-db-app
-            key: username
-      PASSWD:
-        valueFrom:
-          secretKeyRef:
-            name: forgejo-db-app
-            key: password
+      # User/password injected via extraEnv (secretKeyRef doesn't work in config)
      
    cache:
      ADAPTER: redis
@@ -84,3 +85,17 @@ tolerations:
 # ArgoCD adoption labels
 labels:
  argocd.argoproj.io/instance: forgejo
+
+# Inject database credentials via environment variables (overrides app.ini)
+deployment:
+  env:
+    - name: GITEA__DATABASE__USER
+      valueFrom:
+        secretKeyRef:
+          name: forgejo-db-app
+          key: username
+    - name: GITEA__DATABASE__PASSWD
+      valueFrom:
+        secretKeyRef:
+          name: forgejo-db-app
+          key: password
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: cicd
+  labels:
+    # Baseline allows most workloads while blocking clearly dangerous configurations
+    # Redis needs some relaxed settings but doesn't need full privileged access
+    pod-security.kubernetes.io/enforce: baseline
+    pod-security.kubernetes.io/audit: baseline
+    pod-security.kubernetes.io/warn: baseline
@@ -0,0 +1,34 @@
+# ArgoCD CMP plugin for SOPS secret decryption
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: argocd-cmp-cm
+  namespace: argocd
+data:
+  sops-secrets-v1.0.yaml: |
+    apiVersion: argoproj.io/v1alpha1
+    kind: ConfigManagementPlugin
+    metadata:
+      name: sops-secrets-v1.0
+    spec:
+      version: v1.0
+      init:
+        command: [sh, -c]
+        args:
+          - |
+            # Install sops if not present
+            if ! command -v sops &> /dev/null; then
+              wget -qO- https://github.com/getsops/sops/releases/download/v3.9.3/sops-v3.9.3.linux.amd64 > /usr/local/bin/sops
+              chmod +x /usr/local/bin/sops
+            fi
+      generate:
+        command: [sh, -c]
+        args:
+          - |
+            # Find all .enc.yaml files and decrypt them
+            find . -name '*.enc.yaml' -type f | while read -r file; do
+              sops -d "$file"
+            done
+      discover:
+        find:
+          glob: "**/*.enc.yaml"
@@ -1,4 +1,4 @@
-# ArgoCD Helm Values — Single Source of Truth
+# ArgoCD Helm Values — Bootstrap Mode (SOPS plugin added post-bootstrap)
 # Chart: https://github.com/argoproj/argo-helm

 global:
@@ -32,6 +32,12 @@ server:
      cpu: 500m
      memory: 1Gi

+  # Tolerations for control-plane
+  tolerations:
+    - key: node-role.kubernetes.io/control-plane
+      operator: Exists
+      effect: NoSchedule
+
 # Repo server configuration
 repoServer:
  resources:
@@ -42,21 +48,11 @@ repoServer:
      cpu: 500m
      memory: 1Gi

-  # SOPS plugin for encrypted secrets
-  volumes:
-    - name: sops-age
-      secret:
-        secretName: sops-age
-        optional: true
-  volumeMounts:
-    - name: sops-age
-      mountPath: /home/argocd/.config/sops/age
-      readOnly: true
-
-  # Environment for SOPS
-  env:
-    - name: SOPS_AGE_KEY_FILE
-      value: /home/argocd/.config/sops/age/keys.txt
+  # Tolerations for control-plane
+  tolerations:
+    - key: node-role.kubernetes.io/control-plane
+      operator: Exists
+      effect: NoSchedule

 # Controller configuration
 controller:
@@ -68,6 +64,12 @@ controller:
      cpu: 1000m
      memory: 2Gi

+  # Tolerations for control-plane
+  tolerations:
+    - key: node-role.kubernetes.io/control-plane
+      operator: Exists
+      effect: NoSchedule
+
 # Application controller configuration
 applicationSet:
  enabled: true
@@ -87,26 +89,7 @@ redis:
      cpu: 200m
      memory: 256Mi

-# Tolerations for control-plane
-server:
-  tolerations:
-    - key: node-role.kubernetes.io/control-plane
-      operator: Exists
-      effect: NoSchedule
-
-repoServer:
-  tolerations:
-    - key: node-role.kubernetes.io/control-plane
-      operator: Exists
-      effect: NoSchedule
-
-controller:
-  tolerations:
-    - key: node-role.kubernetes.io/control-plane
-      operator: Exists
-      effect: NoSchedule
-
-# ArgoCD projects
+# ArgoCD configuration
 configs:
  # Default project allows all repos
  cm:
@@ -116,8 +99,7 @@ configs:
  params:
    server.insecure: true

-# RBAC (allow admin full access)
-configs:
+  # RBAC (allow admin full access)
  rbac:
    policy.default: role:readonly
    policy.csv: |