refactor(k8s): Reorganize into 5-layer structure with production kustomizations

2026-07-16 14:28:19 -07:00
parent a81b9b6169
commit 2d7330798b
214 changed files with 405 additions and 885 deletions
@@ -0,0 +1,109 @@
+# k8s/monitoring/dashboards/service-internals.yaml
+# Native per-service metrics — the "why" layer behind the ingress RED/uptime
+# dashboards (e.g. ingress shows MinIO is slow; this shows disk offline).
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: service-internals-dashboard
+  namespace: logging
+  labels:
+    grafana_dashboard: "1"
+data:
+  service-internals.json: |
+    {
+      "title": "Service Internals (MinIO / Forgejo / Argo CD / cert-manager / Vault / Longhorn)",
+      "uid": "svc-internals",
+      "schemaVersion": 39,
+      "timezone": "browser",
+      "time": { "from": "now-6h", "to": "now" },
+      "refresh": "30s",
+      "panels": [
+        { "id": 1, "title": "MinIO — disk/node offline", "type": "timeseries",
+          "gridPos": { "h": 6, "w": 12, "x": 0, "y": 0 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "targets": [
+            { "expr": "minio_cluster_disk_offline_total", "legendFormat": "disks offline" },
+            { "expr": "minio_cluster_nodes_offline_total", "legendFormat": "nodes offline" }
+          ]
+        },
+        { "id": 2, "title": "MinIO — S3 request errors", "type": "timeseries",
+          "gridPos": { "h": 6, "w": 12, "x": 12, "y": 0 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "targets": [
+            { "expr": "sum(rate(minio_s3_requests_errors_total[5m])) by (api)", "legendFormat": "{{api}}" }
+          ]
+        },
+        { "id": 3, "title": "MinIO — S3 TTFB latency", "type": "timeseries",
+          "gridPos": { "h": 6, "w": 12, "x": 0, "y": 6 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": { "defaults": { "unit": "s" } },
+          "targets": [
+            { "expr": "minio_s3_time_ttfb_seconds_distribution", "legendFormat": "{{api}}" }
+          ]
+        },
+        { "id": 4, "title": "Forgejo — repos / orgs", "type": "stat",
+          "gridPos": { "h": 6, "w": 12, "x": 12, "y": 6 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "targets": [
+            { "expr": "gitea_repositories", "legendFormat": "repos" },
+            { "expr": "gitea_organizations", "legendFormat": "orgs" }
+          ]
+        },
+        { "id": 5, "title": "Forgejo — process health (CPU/mem)", "type": "timeseries",
+          "gridPos": { "h": 6, "w": 12, "x": 0, "y": 12 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "targets": [
+            { "expr": "rate(process_cpu_seconds_total{job=~\".*forgejo.*|.*gitea.*\"}[5m])", "legendFormat": "cpu" },
+            { "expr": "process_resident_memory_bytes{job=~\".*forgejo.*|.*gitea.*\"}", "legendFormat": "mem" }
+          ]
+        },
+        { "id": 6, "title": "Argo CD — app sync/health status", "type": "table",
+          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "targets": [
+            { "expr": "argocd_app_info", "format": "table", "instant": true }
+          ]
+        },
+        { "id": 7, "title": "cert-manager — days to cert expiry", "type": "stat",
+          "gridPos": { "h": 6, "w": 12, "x": 0, "y": 18 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  { "value": 0, "color": "red" },
+                  { "value": 14, "color": "yellow" },
+                  { "value": 30, "color": "green" }
+                ]
+              }
+            }
+          },
+          "targets": [
+            { "expr": "(certmanager_certificate_expiration_timestamp_seconds - time()) / 86400", "legendFormat": "{{name}}" }
+          ]
+        },
+        { "id": 8, "title": "Vault — sealed/unsealed", "type": "stat",
+          "gridPos": { "h": 6, "w": 6, "x": 12, "y": 20 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "mappings": [
+                { "type": "value", "options": { "0": { "text": "SEALED", "color": "red" } } },
+                { "type": "value", "options": { "1": { "text": "UNSEALED", "color": "green" } } }
+              ]
+            }
+          },
+          "targets": [
+            { "expr": "vault_core_unsealed", "legendFormat": "vault" }
+          ]
+        },
+        { "id": 9, "title": "Longhorn — volume robustness", "type": "table",
+          "gridPos": { "h": 6, "w": 6, "x": 18, "y": 20 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "targets": [
+            { "expr": "longhorn_volume_robustness", "format": "table", "instant": true }
+          ]
+        }
+      ]
+    }