feat(argocd): wire SOPS CMP sidecar + fix loki/grafana/authentik secret resolution

k8s/argocd/apps/00-secrets.yaml

@@ -19,7 +19,8 @@ spec:
   source:
     repoURL: http://forgejo.riotpiao.com:3000/riotpiao.com/homelab.git
     targetRevision: main
-    path: k8s/security/sops-secrets
-    directory: {}
+    path: .
+    plugin:
+      name: sops-secrets-v1.0
   destination:
     server: https://kubernetes.default.svc

k8s/platform/logging/grafana-values.yaml

@@ -157,6 +157,18 @@ tolerations:
     operator: Exists
     effect: NoSchedule
 
+# Pin to az-a (talos-cp-1) — the only node running Longhorn. Without this the
+# scheduler may place grafana on cp-2/cp-3, where its Longhorn PVC cannot attach
+# ("CSINode does not contain driver driver.longhorn.io").
+affinity:
+  nodeAffinity:
+    requiredDuringSchedulingIgnoredDuringExecution:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: topology.kubernetes.io/zone
+              operator: In
+              values: [az-a]
+
 # ClusterIP only — access via ingress (grafana.riotpiao.com) or port-forward.
 service:
   type: ClusterIP

k8s/platform/logging/loki-values.yaml

@@ -37,8 +37,10 @@ loki:
       region: us-east-1          # MinIO ignores region but Loki's S3 client requires it
       s3ForcePathStyle: true
       insecure: true
-      access_key_id: ""          # overridden by helmfile --set (MINIO_ROOT_USER)
-      secret_access_key: ""      # overridden by helmfile --set (MINIO_ROOT_PASSWORD)
+      # Interpolated at runtime from the loki-s3-creds Secret (SOPS-CMP emitted),
+      # injected as env vars via extraEnvFrom below. Requires -config.expand-env.
+      access_key_id: "${access_key_id}"
+      secret_access_key: "${secret_access_key}"
     bucketNames:
       chunks: loki-chunks
       ruler: loki-ruler
@@ -110,6 +112,14 @@ loki:
 singleBinary:
   replicas: 1
 
+  # Interpolate ${access_key_id}/${secret_access_key} in the S3 config from the
+  # loki-s3-creds Secret (SOPS-CMP emitted).
+  extraArgs:
+    - "-config.expand-env=true"
+  extraEnvFrom:
+    - secretRef:
+        name: loki-s3-creds
+
   resources:
     requests:
       cpu: 100m
@@ -124,6 +134,11 @@ singleBinary:
     storageClassName: longhorn
     size: 5Gi
 
+  # Pin to az-a (talos-cp-1) — sole Longhorn node; otherwise the PVC fails to
+  # attach on cp-2/cp-3.
+  nodeSelector:
+    topology.kubernetes.io/zone: az-a
+
 # ── Disable micro-service replicas ───────────────────────────────────────────
 # In SingleBinary mode the chart still templates read/write/backend Deployments
 # unless explicitly set to 0. Setting replicas: 0 keeps them out of the cluster.

k8s/security/ci-cd/argocd-values.yaml

@@ -91,6 +91,69 @@ repoServer:
     serviceMonitor:
       enabled: true
 
+  # ── SOPS ConfigManagementPlugin ─────────────────────────────────────────────
+  # initContainer fetches sops+yq into a shared volume; the sidecar runs
+  # argocd-cmp-server with the plugin.yaml from the sops-cmp-plugin ConfigMap and
+  # decrypts *.enc.yaml with the age key from the sops-age Secret.
+  initContainers:
+    - name: install-sops-tools
+      image: alpine:3.20
+      command: [sh, -c]
+      args:
+        - |
+          set -e
+          apk add --no-cache curl
+          curl -sSfL https://github.com/getsops/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 -o /custom-tools/sops
+          curl -sSfL https://github.com/mikefarah/yq/releases/download/v4.44.3/yq_linux_amd64 -o /custom-tools/yq
+          chmod +x /custom-tools/sops /custom-tools/yq
+      volumeMounts:
+        - name: custom-tools
+          mountPath: /custom-tools
+  extraContainers:
+    - name: sops-cmp
+      image: quay.io/argoproj/argocd:v3.4.5
+      command: [/var/run/argocd/argocd-cmp-server]
+      env:
+        - name: PATH
+          value: /custom-tools:/usr/local/bin:/usr/bin:/bin
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 999
+        allowPrivilegeEscalation: false
+        capabilities:
+          drop: ["ALL"]
+        seccompProfile:
+          type: RuntimeDefault
+      volumeMounts:
+        - mountPath: /var/run/argocd
+          name: var-files
+        - mountPath: /home/argocd/cmp-server/plugins
+          name: plugins
+        - mountPath: /home/argocd/cmp-server/config/plugin.yaml
+          subPath: plugin.yaml
+          name: sops-cmp-plugin
+        - mountPath: /home/argocd/plugins/generate.sh
+          subPath: generate.sh
+          name: sops-cmp-plugin
+        - mountPath: /custom-tools
+          name: custom-tools
+        - mountPath: /sops-age
+          name: sops-age
+        - mountPath: /tmp
+          name: cmp-tmp
+  volumes:
+    - name: custom-tools
+      emptyDir: {}
+    - name: cmp-tmp
+      emptyDir: {}
+    - name: sops-cmp-plugin
+      configMap:
+        name: sops-cmp-plugin
+        defaultMode: 0555
+    - name: sops-age
+      secret:
+        secretName: sops-age
+
 applicationSet:
   replicas: 1
   resources:

k8s/security/ci-cd/sops-cmp-plugin.yaml

@@ -0,0 +1,68 @@
+# ConfigMap holding the SOPS CMP plugin spec + generate script. Mounted into the
+# repo-server sidecar at /home/argocd/cmp-server/config/plugin.yaml (the path the
+# argocd-cmp-server binary reads) and /home/argocd/plugins/generate.sh.
+#
+# The plugin decrypts every k8s/**/*secrets*.enc.yaml Helm-values fragment and
+# emits correctly-keyed Kubernetes Secrets to stdout — no helm template inside
+# the plugin. Applied to the argocd namespace (bootstrap resource).
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: sops-cmp-plugin
+  namespace: argocd
+data:
+  plugin.yaml: |
+    apiVersion: argoproj.io/v1alpha1
+    kind: ConfigManagementPlugin
+    metadata:
+      name: sops-secrets
+    spec:
+      version: v1.0
+      generate:
+        command: [sh, -c]
+        args:
+          - /home/argocd/plugins/generate.sh
+  generate.sh: |
+    #!/bin/sh
+    set -eu
+    export SOPS_AGE_KEY_FILE=/sops-age/keys.txt
+    # CMP runs with cwd = the app source path; sops-secrets app points at repo
+    # root, so enc files resolve from the current directory.
+    REPO_ROOT="$(pwd)"
+
+    emit_secret() {
+      # $1 ns  $2 name  then key=jqpath pairs read from decrypted $ENC
+      ns="$1"; name="$2"; shift 2
+      printf 'apiVersion: v1\nkind: Secret\nmetadata:\n  name: %s\n  namespace: %s\ntype: Opaque\ndata:\n' "$name" "$ns"
+      for kv in "$@"; do
+        k="${kv%%=*}"; path="${kv#*=}"
+        val="$(echo "$DEC" | yq -r "$path")"
+        printf '  %s: %s\n' "$k" "$(printf '%s' "$val" | base64 -w0)"
+      done
+      printf -- '---\n'
+    }
+
+    # ── authentik (iam) ────────────────────────────────────────────────
+    # Separate secret (authentik-secrets) merged via envFrom AFTER the chart's
+    # own `authentik` config secret — avoids two ArgoCD apps owning `authentik`.
+    DEC="$(sops -d "$REPO_ROOT/k8s/security/iam/authentik-secrets.enc.yaml")"
+    emit_secret iam authentik-secrets \
+      AUTHENTIK_SECRET_KEY=.authentik.secret_key \
+      AUTHENTIK_BOOTSTRAP_PASSWORD=.authentik.bootstrap_password \
+      AUTHENTIK_BOOTSTRAP_TOKEN=.authentik.bootstrap_token \
+      AUTHENTIK_POSTGRESQL__PASSWORD=.authentik.postgresql_password
+
+    # ── loki S3 (logging) ──────────────────────────────────────────────
+    DEC="$(sops -d "$REPO_ROOT/k8s/platform/logging/loki-secrets.enc.yaml")"
+    emit_secret logging loki-s3-creds \
+      access_key_id=.loki.storage.s3.accessKeyId \
+      secret_access_key=.loki.storage.s3.secretAccessKey
+
+    # ── grafana (logging) ──────────────────────────────────────────────
+    DEC="$(sops -d "$REPO_ROOT/k8s/platform/logging/grafana-secrets.enc.yaml")"
+    # grafana chart's admin.existingSecret needs BOTH admin-user and admin-password.
+    printf 'apiVersion: v1\nkind: Secret\nmetadata:\n  name: grafana-admin\n  namespace: logging\ntype: Opaque\ndata:\n  admin-user: %s\n  admin-password: %s\n---\n' \
+      "$(printf 'admin' | base64 -w0)" \
+      "$(echo "$DEC" | yq -r '.adminPassword' | base64 -w0)"
+    emit_secret logging grafana-oidc \
+      GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=.env.GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET

k8s/security/iam/authentik-values.yaml

@@ -92,6 +92,10 @@ _caInitContainers: &caInitContainers
     envFrom:
       - secretRef:
           name: authentik
+      # SOPS-CMP-emitted secret values (SECRET_KEY, PG password, bootstrap creds)
+      # merged after the chart config secret — see sops-cmp-plugin.yaml.
+      - secretRef:
+          name: authentik-secrets
     volumeMounts: *caVolumeMounts
 
 _caEnv: &caEnv
@@ -107,6 +111,10 @@ _caEnv: &caEnv
 # Recreate: single replica + RWO-adjacent state — avoids split-brain on redeploy.
 server:
   replicas: 1
+  # Merge SOPS-CMP-emitted secret values after the chart's own `authentik` secret.
+  envFrom:
+    - secretRef:
+        name: authentik-secrets
   deploymentStrategy:
     type: Recreate
   service:
@@ -145,6 +153,9 @@ server:
 # into the server process but the worker pod is still required.
 worker:
   replicas: 1
+  envFrom:
+    - secretRef:
+        name: authentik-secrets
   deploymentStrategy:
     type: Recreate
   resources: