refactor(k8s): consolidate to infra/+apps/ single-source tree, dedicated per-app CNPG (authentik-db/temporal-db), wire monitoring-config, forgejo→cicd ns, drop orphan/stale (data-schemas, ollama, story-crater, sqs/argocd, key-rotation)

2026-08-18 15:08:03 -07:00
parent 54fa540b33
commit 41f5b05395
241 changed files with 467 additions and 10416 deletions
@@ -0,0 +1,96 @@
+# PostSync hook Job that expands all Longhorn volumes to 3 replicas.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: longhorn-expand-replicas
+  namespace: longhorn-system
+  annotations:
+    argocd.argoproj.io/hook: PostSync
+    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+spec:
+  backoffLimit: 3
+  template:
+    metadata:
+      name: longhorn-expand-replicas
+    spec:
+      restartPolicy: Never
+      serviceAccountName: longhorn-expand-replicas
+      containers:
+      - name: expand
+        image: bitnami/kubectl:latest
+        command:
+        - /bin/bash
+        - -c
+        - |
+          set -euo pipefail
+          
+          echo "Waiting for all 3 Longhorn nodes to be Ready..."
+          for i in {1..30}; do
+            READY_COUNT=$(kubectl -n longhorn-system get nodes.longhorn.io -o json | \
+              jq -r '[.items[] | select(.status.conditions[] | select(.type=="Ready" and .status=="True"))] | length')
+            
+            if [ "$READY_COUNT" -ge 3 ]; then
+              echo "✓ All 3 nodes Ready"
+              break
+            fi
+            
+            echo "  $READY_COUNT/3 nodes ready, waiting..."
+            sleep 10
+            
+            if [ $i -eq 30 ]; then
+              echo "✗ Timeout waiting for 3 nodes"
+              exit 1
+            fi
+          done
+          
+          echo
+          echo "Expanding volumes with < 3 replicas..."
+          
+          VOLUMES=$(kubectl -n longhorn-system get volumes.longhorn.io -o json | \
+            jq -r '.items[] | select(.spec.numberOfReplicas < 3) | .metadata.name')
+          
+          if [ -z "$VOLUMES" ]; then
+            echo "  No volumes need expansion"
+          else
+            echo "$VOLUMES" | while read -r vol; do
+              CURRENT=$(kubectl -n longhorn-system get volume "$vol" -o jsonpath='{.spec.numberOfReplicas}')
+              echo "  $vol: $CURRENT → 3 replicas"
+              kubectl -n longhorn-system patch volume "$vol" --type merge \
+                -p '{"spec":{"numberOfReplicas":3}}'
+            done
+          fi
+          
+          echo
+          echo "Done. Final replica counts:"
+          kubectl -n longhorn-system get volumes.longhorn.io \
+            -o custom-columns='NAME:.metadata.name,REPLICAS:.spec.numberOfReplicas' | head -20
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: longhorn-expand-replicas
+  namespace: longhorn-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: longhorn-expand-replicas
+  namespace: longhorn-system
+rules:
+- apiGroups: ["longhorn.io"]
+  resources: ["volumes", "nodes"]
+  verbs: ["get", "list", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: longhorn-expand-replicas
+  namespace: longhorn-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: longhorn-expand-replicas
+subjects:
+- kind: ServiceAccount
+  name: longhorn-expand-replicas
+  namespace: longhorn-system
@@ -0,0 +1,16 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: longhorn-system
+resources:
+  - longhorn-storageclass.yaml
+  - longhorn-cnpg-storageclass.yaml  # CNPG-specific with postgres UID/GID
+  - longhorn-servicemonitor.yaml
+  - longhorn-taint-toleration.yaml
+  - longhorn-nodes.yaml
+  - expand-replicas-job.yaml
+  - patch-csi-tolerations-job.yaml
+# Longhorn deployed via bootstrap script or Helm.
+# These manifests configure it: unified StorageClass (default, 3 replicas),
+# Prometheus ServiceMonitor, taint toleration for control-plane nodes, explicit
+# Node CRDs for cp-2/cp-3, CSI plugin tolerations, and a PostSync hook Job
+# that ensures all existing volumes have 3 replicas.
@@ -0,0 +1,26 @@
+# StorageClass specifically for CNPG (CloudNativePG) PostgreSQL clusters
+# Fixes the "read-only filesystem" issue by mounting with postgres UID/GID
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: longhorn-cnpg
+  namespace: longhorn-system
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "false"
+provisioner: driver.longhorn.io
+allowVolumeExpansion: true
+parameters:
+  numberOfReplicas: "3"
+  staleReplicaTimeout: "30"
+  fromBackup: ""
+  dataLocality: "best-effort"
+  fsType: "ext4"
+  # Mount options to ensure PostgreSQL can write
+  mkfsParams: "-O ^64bit,^metadata_csum"
+mountOptions:
+  - "noatime"
+  # Critical: mount with postgres UID/GID (26:26) to avoid permission issues
+  - "uid=26"
+  - "gid=26"
+reclaimPolicy: Delete
+volumeBindingMode: Immediate
@@ -0,0 +1,35 @@
+# Longhorn Node CRDs for cp-2 and cp-3.
+# These nodes have the control-plane taint, so Longhorn doesn't auto-discover them.
+# Explicit Node CRDs + the taint-toleration setting enable storage across all 3 nodes.
+---
+apiVersion: longhorn.io/v1beta2
+kind: Node
+metadata:
+  name: talos-cp-2
+  namespace: longhorn-system
+spec:
+  name: talos-cp-2
+  allowScheduling: true
+  disks:
+    default-disk:
+      allowScheduling: true
+      path: /var/lib/longhorn
+      storageReserved: 0
+      tags: []
+  tags: []
+---
+apiVersion: longhorn.io/v1beta2
+kind: Node
+metadata:
+  name: talos-cp-3
+  namespace: longhorn-system
+spec:
+  name: talos-cp-3
+  allowScheduling: true
+  disks:
+    default-disk:
+      allowScheduling: true
+      path: /var/lib/longhorn
+      storageReserved: 0
+      tags: []
+  tags: []
@@ -0,0 +1,23 @@
+# k8s/longhorn/longhorn-servicemonitor.yaml
+# Longhorn is deployed via raw manifest (cluster-config/longhorn_bootstrap.sh), not
+# Helm, so there's no chart values toggle for metrics — the manager already exposes
+# them on the existing longhorn-backend service (port 9500); this just wires Prometheus
+# to it. longhorn_volume_robustness is the availability signal (degraded/faulted).
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: longhorn-manager
+  namespace: longhorn-system
+  labels:
+    app: longhorn-manager
+spec:
+  selector:
+    matchLabels:
+      app: longhorn-manager
+  namespaceSelector:
+    matchNames:
+      - longhorn-system
+  endpoints:
+    - port: manager
+      interval: 30s
+      scrapeTimeout: 10s
@@ -0,0 +1,21 @@
+# Longhorn StorageClass - single unified storage class for the entire cluster
+# Replaces: longhorn-wffc, longhorn-kafka, longhorn-static (all deprecated)
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: longhorn
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"
+    description: "Longhorn distributed storage - 3 replicas, WaitForFirstConsumer"
+provisioner: driver.longhorn.io
+allowVolumeExpansion: true
+reclaimPolicy: Delete
+volumeBindingMode: Immediate  # Immediate binding for StatefulSets (matches deployed config)
+parameters:
+  numberOfReplicas: "3"           # HA across all 3 nodes
+  staleReplicaTimeout: "30"
+  fromBackup: ""
+  dataLocality: "disabled"        # Match deployed config (not best-effort)
+  fsType: "ext4"
+  disableRevisionCounter: "true"  # Performance optimization
+  unmapMarkSnapChainRemoved: "ignored"
@@ -0,0 +1,9 @@
+# Longhorn taint-toleration setting - allows Longhorn to run on all control-plane nodes.
+# Without this, Longhorn DaemonSet only runs on nodes without the control-plane taint,
+# which in a 3-CP cluster with dedicated cp-2/cp-3 means only cp-1 gets storage.
+apiVersion: longhorn.io/v1beta2
+kind: Setting
+metadata:
+  name: taint-toleration
+  namespace: longhorn-system
+value: "node-role.kubernetes.io/control-plane:NoSchedule"
@@ -0,0 +1,84 @@
+# PostSync hook to patch longhorn-csi-plugin DaemonSet with control-plane tolerations
+# This runs after longhorn-config Application syncs, ensuring CSI plugin can run on all nodes
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: longhorn-patch-csi-tolerations
+  namespace: longhorn-system
+  annotations:
+    argocd.argoproj.io/hook: PostSync
+    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+spec:
+  backoffLimit: 3
+  template:
+    metadata:
+      name: patch-csi-tolerations
+    spec:
+      restartPolicy: Never
+      serviceAccountName: longhorn-patch-csi-tolerations
+      containers:
+      - name: patch
+        image: bitnami/kubectl:latest
+        command:
+        - /bin/bash
+        - -c
+        - |
+          set -euo pipefail
+          
+          echo "Patching longhorn-csi-plugin DaemonSet with control-plane tolerations..."
+          
+          kubectl patch daemonset longhorn-csi-plugin -n longhorn-system --type=json -p='[
+            {
+              "op": "add",
+              "path": "/spec/template/spec/tolerations/-",
+              "value": {
+                "key": "node-role.kubernetes.io/control-plane",
+                "operator": "Exists",
+                "effect": "NoSchedule"
+              }
+            }
+          ]'
+          
+          echo "✓ Patch applied successfully"
+          
+          echo ""
+          echo "Waiting for CSI plugin pods to roll out to all nodes..."
+          kubectl rollout status daemonset/longhorn-csi-plugin -n longhorn-system --timeout=120s
+          
+          echo ""
+          echo "Final status:"
+          kubectl get daemonset longhorn-csi-plugin -n longhorn-system
+          kubectl get pods -n longhorn-system -l app=longhorn-csi-plugin -o wide
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: longhorn-patch-csi-tolerations
+  namespace: longhorn-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: longhorn-patch-csi-tolerations
+  namespace: longhorn-system
+rules:
+- apiGroups: ["apps"]
+  resources: ["daemonsets"]
+  verbs: ["get", "patch"]
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["list", "get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: longhorn-patch-csi-tolerations
+  namespace: longhorn-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: longhorn-patch-csi-tolerations
+subjects:
+- kind: ServiceAccount
+  name: longhorn-patch-csi-tolerations
+  namespace: longhorn-system