refactor(k8s): consolidate to infra/+apps/ single-source tree, dedicated per-app CNPG (authentik-db/temporal-db), wire monitoring-config, forgejo→cicd ns, drop orphan/stale (data-schemas, ollama, story-crater, sqs/argocd, key-rotation)
This commit is contained in:
Story Crater Bot
@@ -0,0 +1,38 @@
|
||||
# Dedicated CNPG Postgres for Authentik (GitOps, wave 2 — before authentik w3).
|
||||
# CNPG generates secret `authentik-db-app` + service `authentik-db-rw` in ns iam;
|
||||
# authentik-values reads them locally (no cross-ns secret copy).
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: Cluster
|
||||
metadata:
|
||||
name: authentik-db
|
||||
namespace: iam
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
|
||||
spec:
|
||||
instances: 3
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16.2
|
||||
bootstrap:
|
||||
initdb:
|
||||
database: authentik
|
||||
owner: app
|
||||
encoding: UTF8
|
||||
localeCollate: C
|
||||
localeCType: C
|
||||
enableSuperuserAccess: false
|
||||
resources:
|
||||
requests: { memory: "512Mi", cpu: "250m" }
|
||||
limits: { memory: "2Gi", cpu: "1" }
|
||||
storage:
|
||||
size: 10Gi
|
||||
storageClass: longhorn-cnpg
|
||||
monitoring:
|
||||
enablePodMonitor: true
|
||||
affinity:
|
||||
# preferred + tolerations: HA across nodes without deadlocking a partly-
|
||||
# schedulable 3-CP (same as forgejo-db).
|
||||
podAntiAffinityType: preferred
|
||||
topologyKey: kubernetes.io/hostname
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
@@ -0,0 +1,8 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
# Dedicated per-app CNPG clusters. NO top-level `namespace:` — each Cluster
|
||||
# carries its own ns (iam / temporal); a transformer would wrongly collapse them.
|
||||
resources:
|
||||
- namespaces.yaml
|
||||
- authentik-db.yaml
|
||||
- temporal-db.yaml
|
||||
@@ -0,0 +1,12 @@
|
||||
# DB clusters are wave 2 — their namespaces must exist first (their apps that
|
||||
# would CreateNamespace run later, w3/w8). Declared here so the databases App
|
||||
# creates them. authentik/vault/temporal CreateNamespace=true then no-ops.
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: iam
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: temporal
|
||||
@@ -0,0 +1,49 @@
|
||||
# Dedicated CNPG Postgres for Temporal (GitOps, wave 2 — before temporal w8).
|
||||
# Temporal needs TWO databases: `temporal` (initdb) + `temporal_visibility`
|
||||
# (Database CR). CNPG generates secret `temporal-db-app` + service
|
||||
# `temporal-db-rw` in ns temporal; temporal-values reads them locally.
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: Cluster
|
||||
metadata:
|
||||
name: temporal-db
|
||||
namespace: temporal
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
|
||||
spec:
|
||||
instances: 3
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16.2
|
||||
bootstrap:
|
||||
initdb:
|
||||
database: temporal
|
||||
owner: app
|
||||
encoding: UTF8
|
||||
localeCollate: C
|
||||
localeCType: C
|
||||
enableSuperuserAccess: false
|
||||
resources:
|
||||
requests: { memory: "512Mi", cpu: "250m" }
|
||||
limits: { memory: "2Gi", cpu: "1" }
|
||||
storage:
|
||||
size: 10Gi
|
||||
storageClass: longhorn-cnpg
|
||||
monitoring:
|
||||
enablePodMonitor: true
|
||||
affinity:
|
||||
podAntiAffinityType: preferred
|
||||
topologyKey: kubernetes.io/hostname
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
---
|
||||
# Second database Temporal requires (visibility store), on the same cluster.
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: Database
|
||||
metadata:
|
||||
name: temporal-visibility
|
||||
namespace: temporal
|
||||
spec:
|
||||
name: temporal_visibility
|
||||
owner: app
|
||||
cluster:
|
||||
name: temporal-db
|
||||
@@ -0,0 +1,6 @@
|
||||
apiVersion: v2
|
||||
name: forgejo-runner
|
||||
description: Forgejo Actions runner with Docker-in-Docker sidecar for homelab CI/CD
|
||||
type: application
|
||||
version: 0.1.0
|
||||
appVersion: "6"
|
||||
@@ -0,0 +1,108 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ .Release.Name }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
app: {{ .Release.Name }}
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate # RWO PVCs — old pod must terminate before new one mounts them
|
||||
selector:
|
||||
matchLabels:
|
||||
app: {{ .Release.Name }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: {{ .Release.Name }}
|
||||
spec:
|
||||
# runner image runs as UID 1000; fsGroup makes the Longhorn /data PVC
|
||||
# group-writable so `register` can write /data/.runner (else permission denied).
|
||||
securityContext:
|
||||
fsGroup: 1000
|
||||
tolerations:
|
||||
{{- toYaml .Values.tolerations | nindent 8 }}
|
||||
{{- with .Values.nodeSelector }}
|
||||
nodeSelector:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
|
||||
initContainers:
|
||||
- name: register
|
||||
image: {{ .Values.runner.image.repository }}:{{ .Values.runner.image.tag }}
|
||||
command: ["sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
test -f /data/.runner || forgejo-runner register --no-interactive \
|
||||
--instance {{ .Values.runner.forgejoUrl }} \
|
||||
--token $(RUNNER_TOKEN) \
|
||||
--name {{ .Values.runner.name }} \
|
||||
--labels "{{ .Values.runner.labels }}"
|
||||
env:
|
||||
- name: RUNNER_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ .Values.runner.tokenSecret }}
|
||||
key: token
|
||||
volumeMounts:
|
||||
- name: runner-data
|
||||
mountPath: /data
|
||||
- name: homelab-ca
|
||||
mountPath: /etc/ssl/certs/homelab-ca.pem
|
||||
subPath: ca.crt
|
||||
workingDir: /data
|
||||
|
||||
containers:
|
||||
- name: runner
|
||||
image: {{ .Values.runner.image.repository }}:{{ .Values.runner.image.tag }}
|
||||
command: ["sh", "-c", "forgejo-runner daemon"]
|
||||
workingDir: /data
|
||||
env:
|
||||
- name: DOCKER_HOST
|
||||
value: tcp://localhost:2376
|
||||
- name: DOCKER_TLS_VERIFY
|
||||
value: "1"
|
||||
- name: DOCKER_CERT_PATH
|
||||
value: /docker-certs/client
|
||||
volumeMounts:
|
||||
- name: runner-data
|
||||
mountPath: /data
|
||||
- name: docker-certs
|
||||
mountPath: /docker-certs
|
||||
- name: homelab-ca
|
||||
mountPath: /etc/ssl/certs/homelab-ca.pem
|
||||
subPath: ca.crt
|
||||
resources:
|
||||
{{- toYaml .Values.runner.resources | nindent 12 }}
|
||||
|
||||
- name: dind
|
||||
image: {{ .Values.dind.image.repository }}:{{ .Values.dind.image.tag }}
|
||||
securityContext:
|
||||
privileged: true # required for DinD; cicd namespace is labelled privileged
|
||||
env:
|
||||
- name: DOCKER_TLS_CERTDIR
|
||||
value: /docker-certs
|
||||
volumeMounts:
|
||||
- name: docker-certs
|
||||
mountPath: /docker-certs
|
||||
- name: dind-storage
|
||||
mountPath: /var/lib/docker
|
||||
- name: homelab-ca
|
||||
mountPath: /etc/ssl/certs/homelab-ca.pem
|
||||
subPath: ca.crt
|
||||
resources:
|
||||
{{- toYaml .Values.dind.resources | nindent 12 }}
|
||||
|
||||
volumes:
|
||||
- name: runner-data
|
||||
persistentVolumeClaim:
|
||||
claimName: runner-reg
|
||||
- name: dind-storage
|
||||
persistentVolumeClaim:
|
||||
claimName: runner-dind
|
||||
- name: docker-certs
|
||||
emptyDir: {} # DinD regenerates mTLS certs on each start
|
||||
- name: homelab-ca
|
||||
secret:
|
||||
secretName: homelab-ca
|
||||
@@ -0,0 +1,31 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-egress
|
||||
namespace: {{ .Release.Namespace }}
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: {{ .Release.Name }}
|
||||
policyTypes: [Egress]
|
||||
egress:
|
||||
# Forgejo — same cicd namespace (git push, registry push/pull)
|
||||
- to:
|
||||
- podSelector: {}
|
||||
# CoreDNS
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 53
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
# Internet (action deps, base images) — never LAN or pod network
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 0.0.0.0/0
|
||||
except:
|
||||
- 192.168.1.0/24
|
||||
- 10.244.0.0/16
|
||||
@@ -0,0 +1,27 @@
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: runner-reg
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
app: {{ .Release.Name }}
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
storageClassName: {{ .Values.persistence.reg.storageClass }}
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.persistence.reg.size }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: runner-dind
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
app: {{ .Release.Name }}
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
storageClassName: {{ .Values.persistence.dind.storageClass }}
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.persistence.dind.size }}
|
||||
@@ -0,0 +1,49 @@
|
||||
runner:
|
||||
image:
|
||||
repository: code.forgejo.org/forgejo/runner
|
||||
tag: "6" # pin exact release before apply
|
||||
name: talos-runner
|
||||
labels: "docker:docker://node:22-bookworm"
|
||||
# In-cluster Service (:3000) — direct, avoids the ingress/public-hostname hop
|
||||
# (the public URL is :443 which forgejo doesn't serve; runner got i/o timeout).
|
||||
forgejoUrl: http://forgejo-gitea-http.cicd.svc.cluster.local:3000
|
||||
# tokenSecret: name of the K8s Secret that holds the runner registration token
|
||||
# created automatically by the helmfile presync hook (see helmfile.yaml.gotmpl)
|
||||
tokenSecret: runner-token
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 4Gi
|
||||
|
||||
dind:
|
||||
image:
|
||||
repository: docker
|
||||
tag: "27-dind" # pin exact release before apply
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 4Gi
|
||||
|
||||
persistence:
|
||||
reg:
|
||||
storageClass: longhorn # Unified StorageClass (3 replicas)
|
||||
size: 1Gi # .runner registration file + config — survives pod restarts
|
||||
dind:
|
||||
storageClass: longhorn # Unified StorageClass (3 replicas)
|
||||
size: 30Gi # docker layer cache — keeps rebuilds fast across restarts
|
||||
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
|
||||
# Pin to az-a (talos-cp-1) — sole Longhorn node; RWO PVCs (reg/dind cache) only
|
||||
# attach there.
|
||||
nodeSelector:
|
||||
topology.kubernetes.io/zone: az-a
|
||||
@@ -0,0 +1,167 @@
|
||||
# Authentik OAuth provisioning — PostSync hook, reruns on every ArgoCD sync
|
||||
# (hook-delete-policy: BeforeHookCreation deletes the previous run's Job before
|
||||
# creating a new one, so this stays reconciled the same way the rest of the
|
||||
# cluster does — no separate manual bootstrap step like setup_talos_iam.sh /
|
||||
# provision_oidc.py, which never got migrated off the old helmfile workflow).
|
||||
#
|
||||
# What it does (see scripts/authentik-provision.py docstring): creates the
|
||||
# "groups" scope mapping, homelab-admins / grafana-admins groups, the "rock"
|
||||
# admin user, OAuth2 providers + Applications for grafana/minio/forgejo/argocd,
|
||||
# and binds homelab-admins to all of them. The script is generated into the
|
||||
# authentik-provision-script ConfigMap by kustomize configMapGenerator (see
|
||||
# kustomization.yaml), not embedded here.
|
||||
#
|
||||
# RBAC: this Job only touches Secrets (get existing client secrets, create new
|
||||
# ones for forgejo/argocd/rock) across the namespaces those services live in.
|
||||
# It never touches any other resource type.
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: authentik-provisioner
|
||||
namespace: iam
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: authentik-provisioner
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["get", "list", "create", "update", "patch"]
|
||||
---
|
||||
# One RoleBinding per namespace the script touches (least-privilege: Secrets
|
||||
# only, and only in these 5 namespaces — not a cluster-wide ClusterRoleBinding).
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: authentik-provisioner
|
||||
namespace: iam
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: authentik-provisioner
|
||||
namespace: iam
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: authentik-provisioner
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: authentik-provisioner
|
||||
namespace: cicd
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: authentik-provisioner
|
||||
namespace: iam
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: authentik-provisioner
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: authentik-provisioner
|
||||
namespace: argocd
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: authentik-provisioner
|
||||
namespace: iam
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: authentik-provisioner
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: authentik-provisioner
|
||||
namespace: logging
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: authentik-provisioner
|
||||
namespace: iam
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: authentik-provisioner
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: authentik-provisioner
|
||||
namespace: storage
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: authentik-provisioner
|
||||
namespace: iam
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: authentik-provisioner
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: authentik-provision
|
||||
namespace: iam
|
||||
annotations:
|
||||
argocd.argoproj.io/hook: PostSync
|
||||
argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
|
||||
spec:
|
||||
ttlSecondsAfterFinished: 600
|
||||
backoffLimit: 3
|
||||
template:
|
||||
spec:
|
||||
serviceAccountName: authentik-provisioner
|
||||
restartPolicy: Never
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: provision
|
||||
image: python:3.12-alpine
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
env:
|
||||
- name: AUTHENTIK_BOOTSTRAP_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: authentik-secrets
|
||||
key: AUTHENTIK_BOOTSTRAP_TOKEN
|
||||
volumeMounts:
|
||||
- name: script
|
||||
mountPath: /script
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
set -e
|
||||
echo "waiting for authentik-server..."
|
||||
until wget -q -O /dev/null http://authentik-server.iam.svc.cluster.local/-/health/ready/ 2>/dev/null; do
|
||||
sleep 5
|
||||
done
|
||||
echo "installing kubectl (via python urllib - no apk/curl: this"
|
||||
echo "container runs as non-root UID 1000 and can't write to"
|
||||
echo "apk's directories or /usr/local/bin, both root-owned in"
|
||||
echo "the python:3.12-alpine image; /tmp is world-writable)..."
|
||||
python3 -c "
|
||||
import urllib.request, os, stat
|
||||
kver = urllib.request.urlopen('https://dl.k8s.io/release/stable.txt').read().decode().strip()
|
||||
url = f'https://dl.k8s.io/release/{kver}/bin/linux/amd64/kubectl'
|
||||
urllib.request.urlretrieve(url, '/tmp/kubectl')
|
||||
st = os.stat('/tmp/kubectl')
|
||||
os.chmod('/tmp/kubectl', st.st_mode | stat.S_IEXEC)
|
||||
"
|
||||
export PATH="/tmp:$PATH"
|
||||
echo "running provisioning script..."
|
||||
python3 /script/authentik-provision.py
|
||||
volumes:
|
||||
- name: script
|
||||
configMap:
|
||||
name: authentik-provision-script
|
||||
@@ -0,0 +1,20 @@
|
||||
authentik:
|
||||
secret_key: ENC[AES256_GCM,data:55ne/khf01ZD3FP2Zek+0Ar9C+GP/GPf33PwOwgXdnOXF4kLrEwwJ/E7yrH0nkBKGO221ClGBjlo7kb5vGMmmmg8TVBa1upQbk8b0s723n8=,iv:ZkDp4gOHY1eH44cDhGaZILi8dFDTNuYf1CVE+qrapzE=,tag:JTE4tfMJSowsn9YZ0xKFng==,type:str]
|
||||
bootstrap_password: ENC[AES256_GCM,data:vBk/ivCG4x2TlvURrDxHAiaZlxrF+PeHBAuoZR6muWw=,iv:ccA703HMomR58cD2/6wK1W0IKJb7U1hNws6fmwG15E8=,tag:MdhTx9ebXLjRCnDpUxpJ0g==,type:str]
|
||||
bootstrap_token: ENC[AES256_GCM,data:NwGZzaL8JufXYp6sjeeN1etdgaHT8IuaZFLZJegzs+fPXMvVCGPKKlStythIOa3gwgCXkhA+kV8+jyxYKr1UMw==,iv:EJCetXnkNe+UuhWk4f3vr3kIEbCuVplVKHPne3aVIQ8=,tag:kwlE+9O6DczO+mn9Rqx+Rw==,type:str]
|
||||
postgresql_password: ENC[AES256_GCM,data:+WeZKo+24awwWfTAt5q6KJRkqaHD7YmlnKm4oGaUNvw=,iv:VSJ3d8p4f5SQM2lVdhSLY5dqdQTKwcfzOCATIM5M/cw=,tag:6K8mQmyLEg7KJi3bg+zPig==,type:str]
|
||||
sops:
|
||||
age:
|
||||
- enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBJSDJSYmRQNDdjWjlORmJX
|
||||
Zm9yd2xRNi9XKy95Y2NQWUMyTGxjb1NjRTNzCnlNbko5NVJVSitiaE1DQ2QvMVoy
|
||||
czRya01EaFZmZzAzcjRESG5NY1JIVTAKLS0tIFZ1UkREaGc4VUczMjFITWR6N0Ez
|
||||
NStJZmpuWEw1dlRFcExZUlFLWkpBSFkK9r0NG3IKV7+AU00VXVCuHu+aBOOLydD0
|
||||
ncioyDrJWgkyDxn9+BNZh4vX8vEERANYh+1/3P99Ubz28tifhmi2vg==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
recipient: age1smu533f803gmd0jq60s2zaj9zlznajy0ca6rtewd4r37mr2hs3uqsrldfh
|
||||
lastmodified: "2026-07-15T23:22:55Z"
|
||||
mac: ENC[AES256_GCM,data:+MqbhlVnhriigDEj/AMJj4yLjcIjxkQc4QANcVeAXefcljHS4yOVtrwh2js3l9WVyIgVdl0MdlW+ycykBU4Pi3u4sdJoiJd945wZDsHvFCREYZIZdsVW/EHmKjIroqxQfeEtbWDStkaE+WTF/yGrZ2HLLztrXIafxZq7KhbXlo0=,iv:PCzhd8j539BxvpQQCds1vMXm3gfmlvSnrbtfOpFaKrY=,tag:NiHwSfQRvf//lpjFAKA1Jg==,type:str]
|
||||
unencrypted_suffix: _unencrypted
|
||||
version: 3.13.2
|
||||
@@ -0,0 +1,275 @@
|
||||
# k8s/talos-iam/authentik-values.yaml
|
||||
# Authentik — SSO Identity Provider for the homelab.
|
||||
# Provides OAuth2/OIDC login for Grafana, MinIO, Forgejo, and Argo CD.
|
||||
# Chart: authentik/authentik from https://charts.goauthentik.io
|
||||
#
|
||||
# Architecture: server (UI+API) + worker (background tasks) + PostgreSQL + Redis.
|
||||
# PostgreSQL is the system of record — must persist. Redis is ephemeral cache/queue.
|
||||
#
|
||||
# Secrets injected via helmfile --set (from .env / vsource):
|
||||
# AUTHENTIK_SECRET_KEY — signs sessions and tokens; set once, never rotate casually
|
||||
# AUTHENTIK_BOOTSTRAP_PASSWORD — initial akadmin password (used once at first login)
|
||||
# AUTHENTIK_BOOTSTRAP_TOKEN — API token for the setup_talos_iam.sh bootstrap script
|
||||
# AUTHENTIK_PG_PASSWORD — PostgreSQL user password
|
||||
|
||||
authentik:
|
||||
# host: the external URL Authentik uses to build redirect URIs in OAuth2 flows.
|
||||
# Must match what the browser sees — if it returns an internal svc URL,
|
||||
# the browser's redirect after login will fail (can't reach svc DNS externally).
|
||||
# HTTP (not HTTPS) because the Authentik ingress has no TLS cert configured.
|
||||
host: "https://authentik.riotpiao.com"
|
||||
|
||||
error_reporting:
|
||||
enabled: false # do not phone home to Sentry
|
||||
|
||||
# PostgreSQL connection — points at CloudNativePG cluster in ddb namespace.
|
||||
# Uses 'app' bootstrap user (CNPG simple pattern, same as Forgejo).
|
||||
# Credentials injected from authentik-db-app secret via env vars below.
|
||||
postgresql:
|
||||
host: authentik-db-rw.iam.svc.cluster.local
|
||||
port: 5432
|
||||
name: authentik
|
||||
user: app # All apps use shared 'app' user (CNPG design pattern)
|
||||
password: "" # overridden by AUTHENTIK_POSTGRESQL__PASSWORD env var
|
||||
|
||||
# Redis connection — bundled subchart, standalone mode (no sentinel/cluster).
|
||||
redis:
|
||||
host: authentik-redis-master
|
||||
|
||||
# ── HTTP client timeouts ──────────────────────────────────────────────────────
|
||||
# Increased to tolerate 5+ second pod-to-pod network latency.
|
||||
# Affects webhooks, outpost management, SCIM, LDAP sync.
|
||||
# Default: ~30s — too aggressive when latency spikes hit 5-10s.
|
||||
log_level: debug # enable debug logging to monitor connection issues
|
||||
|
||||
# ── CA trust (shared by server and worker) ────────────────────────────────────
|
||||
# Authentik (Python/Debian) uses requests + httpx for outgoing HTTPS — webhooks,
|
||||
# outpost management, SCIM. Both libraries need REQUESTS_CA_BUNDLE / SSL_CERT_FILE
|
||||
# to point to a bundle that includes homelab-ca, otherwise connections to other
|
||||
# homelab services fail with "certificate signed by unknown authority".
|
||||
#
|
||||
# Strategy: a debian:12-slim init container (run as root) concatenates the
|
||||
# Debian system Mozilla bundle with homelab-ca.crt into an emptyDir. The main
|
||||
# container then references /merged/ca-bundle.crt via two env vars that cover
|
||||
# every Python HTTP library.
|
||||
_caVolumes: &caVolumes
|
||||
- name: homelab-ca
|
||||
configMap:
|
||||
name: homelab-ca
|
||||
- name: merged-ca
|
||||
emptyDir: {}
|
||||
|
||||
_caVolumeMounts: &caVolumeMounts
|
||||
- name: homelab-ca
|
||||
mountPath: /homelab-ca
|
||||
readOnly: true
|
||||
- name: merged-ca
|
||||
mountPath: /merged
|
||||
readOnly: true
|
||||
|
||||
_caInitContainers: &caInitContainers
|
||||
- name: merge-ca-certs
|
||||
image: debian:bookworm
|
||||
imagePullPolicy: IfNotPresent
|
||||
securityContext:
|
||||
runAsUser: 0
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- (cat /etc/ssl/certs/ca-certificates.crt 2>/dev/null; cat /homelab-ca/homelab-ca.crt) > /merged/ca-bundle.crt
|
||||
volumeMounts:
|
||||
- name: homelab-ca
|
||||
mountPath: /homelab-ca
|
||||
readOnly: true
|
||||
- name: merged-ca
|
||||
mountPath: /merged
|
||||
# NOTE: no authentik-migrate init container — the authentik `server` entrypoint
|
||||
# runs migrations itself on startup. A separate `manage migrate` init pinned to
|
||||
# an older image tripped a version-history precheck on an empty DB
|
||||
# (relation "authentik_version_history" does not exist) and blocked boot.
|
||||
|
||||
_caEnv: &caEnv
|
||||
- name: REQUESTS_CA_BUNDLE
|
||||
value: /merged/ca-bundle.crt
|
||||
- name: SSL_CERT_FILE
|
||||
value: /merged/ca-bundle.crt
|
||||
|
||||
# ── Authentik server (UI + API) ───────────────────────────────────────────────
|
||||
# Handles all browser traffic: login flows, admin UI, OAuth2 authorize/token endpoints.
|
||||
# NodePort 32172 is a fallback for direct node access during troubleshooting;
|
||||
# normal access is via nginx ingress (authentik.riotpiao.com → svc:80).
|
||||
# Recreate: single replica + RWO-adjacent state — avoids split-brain on redeploy.
|
||||
server:
|
||||
replicas: 1
|
||||
# Merge SOPS-CMP-emitted secret values after the chart's own `authentik` secret.
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: authentik-secrets
|
||||
deploymentStrategy:
|
||||
type: Recreate
|
||||
service:
|
||||
type: NodePort
|
||||
nodePort: 32172
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: 1000m
|
||||
memory: 1Gi
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
volumes: *caVolumes
|
||||
volumeMounts: *caVolumeMounts
|
||||
initContainers: *caInitContainers
|
||||
env:
|
||||
# Merge CA trust env vars
|
||||
- name: REQUESTS_CA_BUNDLE
|
||||
value: /merged/ca-bundle.crt
|
||||
- name: SSL_CERT_FILE
|
||||
value: /merged/ca-bundle.crt
|
||||
# Override database credentials to use 'app' from authentik-db-app
|
||||
- name: AUTHENTIK_POSTGRESQL__USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: authentik-db-app
|
||||
key: username
|
||||
- name: AUTHENTIK_POSTGRESQL__PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: authentik-db-app
|
||||
key: password
|
||||
podAnnotations:
|
||||
configmap.reloader.stakater.com/reload: "homelab-ca"
|
||||
homelab.io/restart-at: "2026-06-21T13-40"
|
||||
# The /-/health/{live,ready}/ endpoints do a DB round-trip; under transient
|
||||
# CNPG contention they respond in 5-6s while still returning 200. The chart's
|
||||
# default 3s liveness timeout then flags a working backend as dead and kubelet
|
||||
# kills it in a restart loop — the pod never stays Ready, gets dropped from the
|
||||
# authentik-server Service endpoints, and the OAuth-provisioning PostSync hook
|
||||
# fails with "Host is unreachable". Widen the timeouts so slow-but-healthy
|
||||
# checks aren't treated as failures. (Only these fields are overridden; the
|
||||
# chart deep-merges the rest of each probe, incl. the httpGet path.)
|
||||
livenessProbe:
|
||||
timeoutSeconds: 15
|
||||
failureThreshold: 6
|
||||
readinessProbe:
|
||||
timeoutSeconds: 15
|
||||
failureThreshold: 6
|
||||
startupProbe:
|
||||
timeoutSeconds: 15
|
||||
failureThreshold: 120 # 120 × 10s = 20min for fresh DB migrations
|
||||
# Every OIDC login (Grafana, Argo CD, MinIO, Forgejo) depends on this server —
|
||||
# its request latency/error rate explains SSO-driven slowness on those services.
|
||||
metrics:
|
||||
enabled: true
|
||||
serviceMonitor:
|
||||
enabled: true
|
||||
scrapeTimeout: 30s
|
||||
|
||||
# ── Authentik worker ──────────────────────────────────────────────────────────
|
||||
# Runs background tasks: email delivery, LDAP sync, flow policy evaluation,
|
||||
# event log cleanup, and managed outpost updates. Stateless — no PVC needed.
|
||||
# Same resource profile as server; Authentik 2023+ merged some worker duties
|
||||
# into the server process but the worker pod is still required.
|
||||
worker:
|
||||
replicas: 1
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: authentik-secrets
|
||||
deploymentStrategy:
|
||||
type: Recreate
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: 1000m
|
||||
memory: 1Gi
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
volumes: *caVolumes
|
||||
volumeMounts: *caVolumeMounts
|
||||
initContainers: *caInitContainers
|
||||
env:
|
||||
# Merge CA trust env vars
|
||||
- name: REQUESTS_CA_BUNDLE
|
||||
value: /merged/ca-bundle.crt
|
||||
- name: SSL_CERT_FILE
|
||||
value: /merged/ca-bundle.crt
|
||||
# Override database credentials to use 'app' from authentik-db-app
|
||||
- name: AUTHENTIK_POSTGRESQL__USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: authentik-db-app
|
||||
key: username
|
||||
- name: AUTHENTIK_POSTGRESQL__PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: authentik-db-app
|
||||
key: password
|
||||
podAnnotations:
|
||||
configmap.reloader.stakater.com/reload: "homelab-ca"
|
||||
homelab.io/restart-at: "2026-06-21T13-40"
|
||||
metrics:
|
||||
enabled: true
|
||||
serviceMonitor:
|
||||
enabled: true
|
||||
scrapeTimeout: 30s
|
||||
|
||||
# ── PostgreSQL (external: CloudNativePG cluster in ddb namespace) ─────────────
|
||||
# Authentik connects to the dedicated authentik-db (1 primary + 2 replicas).
|
||||
# Do not use the bundled Bitnami subchart — CNPG is already running.
|
||||
postgresql:
|
||||
enabled: false
|
||||
primary:
|
||||
persistence:
|
||||
enabled: true
|
||||
storageClass: longhorn
|
||||
size: 8Gi
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- weight: 100
|
||||
preference:
|
||||
matchExpressions:
|
||||
- key: node-role.kubernetes.io/worker
|
||||
operator: Exists
|
||||
|
||||
# ── Bundled Redis ─────────────────────────────────────────────────────────────
|
||||
# Cache and async task queue only — no durable data. If Redis restarts, in-flight
|
||||
# background tasks are retried and cached tokens are recomputed. Losing Redis
|
||||
# data does not lose user accounts or flow configuration (that's in PostgreSQL).
|
||||
# persistence: false saves a PVC and makes restarts faster.
|
||||
#
|
||||
# Same prefer-worker / fallback-to-cp scheduling as PostgreSQL.
|
||||
# architecture: standalone — no Sentinel/cluster overhead for a 3-node homelab.
|
||||
redis:
|
||||
enabled: true
|
||||
master:
|
||||
persistence:
|
||||
enabled: false
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- weight: 100
|
||||
preference:
|
||||
matchExpressions:
|
||||
- key: node-role.kubernetes.io/worker
|
||||
operator: Exists
|
||||
architecture: standalone
|
||||
|
||||
# Ingress disabled — rule lives in k8s/ingress/ingress.yaml (authentik.riotpiao.com).
|
||||
# For direct access during bootstrap: kubectl -n iam port-forward svc/authentik-server 7000:80
|
||||
@@ -0,0 +1,35 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
# NOTE: no top-level `namespace:` transformer here (removed) - it used to
|
||||
# force-rewrite metadata.namespace to "iam" on every resource in this
|
||||
# kustomization, which was harmless while every manifest here only ever
|
||||
# targeted the iam namespace itself. authentik-provision-job.yaml's
|
||||
# RoleBindings deliberately target cicd/argocd/logging/storage (least-
|
||||
# privilege access for the authentik-provisioner ServiceAccount to touch
|
||||
# Secrets in those namespaces) - the namespace transformer would have
|
||||
# silently rewritten all of them back to iam, breaking the RBAC. Every
|
||||
# manifest in this directory already sets its own explicit
|
||||
# metadata.namespace, so dropping the transformer changes nothing for the
|
||||
# existing resources/.
|
||||
resources:
|
||||
- authentik-provision-job.yaml
|
||||
- rbac-dashboard-rolebinding.yaml
|
||||
|
||||
# Provisioning/verification python lives in scripts/*.py (real files, linted +
|
||||
# diff-friendly) and is generated into ConfigMaps here rather than embedded in
|
||||
# the job YAML. disableNameSuffixHash keeps the names stable so the Jobs'
|
||||
# configMap volume refs and PostSync hook-delete semantics keep working; each
|
||||
# hook Job is recreated per sync so it always mounts the latest script.
|
||||
configMapGenerator:
|
||||
- name: authentik-provision-script
|
||||
namespace: iam
|
||||
files:
|
||||
- authentik-provision.py=scripts/authentik-provision.py
|
||||
|
||||
generatorOptions:
|
||||
disableNameSuffixHash: true
|
||||
# authentik-migrations-job.yaml removed — redundant + broken. The authentik
|
||||
# `server` entrypoint runs migrations itself; this standalone job lacked the
|
||||
# authentik-secrets envFrom (Secret key missing) and always failed.
|
||||
# SOPS secrets (*.enc.yaml) handled by ArgoCD SOPS plugin at sync time
|
||||
# authentik/vault deployed via ArgoCD Helm source
|
||||
@@ -0,0 +1,13 @@
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: authentik-provisioner
|
||||
namespace: dashboard
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: authentik-provisioner
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: authentik-provisioner
|
||||
namespace: iam
|
||||
@@ -0,0 +1,359 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Authentik OAuth provisioning - idempotent, safe to re-run (ArgoCD PostSync hook).
|
||||
|
||||
Creates/updates, in order:
|
||||
1. A custom "groups" OAuth2 scope mapping (Authentik ships openid/email/profile
|
||||
by default but NOT groups - required for ArgoCD RBAC group mapping and
|
||||
Grafana's role_attribute_path, both of which read a `groups` claim).
|
||||
2. Groups: homelab-admins (is_superuser=true), grafana-admins.
|
||||
3. User "rock": created if missing, always (re-)synced into both groups above.
|
||||
Password is generated once and only written to the k8s Secret
|
||||
rock-credentials (iam ns) the first time the user is created - re-runs
|
||||
never rotate an existing password.
|
||||
4. OAuth2/OIDC providers + Applications for: grafana, minio, forgejo, argocd.
|
||||
Client secrets are read from existing k8s Secrets (grafana-oidc, minio-oidc)
|
||||
if present, or generated once and written out (forgejo-oidc, oidc-secret)
|
||||
the first time.
|
||||
5. PolicyBinding of homelab-admins -> every Application above, so "rock" (and
|
||||
anyone else in that group) has guaranteed access regardless of each app's
|
||||
default visibility.
|
||||
|
||||
Talks to Authentik over the in-cluster Service (authentik-server.iam.svc:80),
|
||||
authenticating with the bootstrap token. Everything is done with GET-then-
|
||||
create-or-patch so this can be re-run on every ArgoCD sync without duplicating
|
||||
or clobbering objects (PostSync hook, not a one-shot Job with hook-delete).
|
||||
|
||||
kubectl is used only to read/write the small set of Secrets this script
|
||||
touches - it shells out rather than using the Python k8s client to keep the
|
||||
container image to stdlib Python + the kubectl binary, no pip installs.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import string
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
AUTHENTIK_URL = "http://authentik-server.iam.svc.cluster.local"
|
||||
TOKEN = os.environ["AUTHENTIK_BOOTSTRAP_TOKEN"]
|
||||
|
||||
|
||||
def api(method, path, data=None):
|
||||
url = f"{AUTHENTIK_URL}{path}"
|
||||
body = json.dumps(data).encode() if data is not None else None
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
data=body,
|
||||
method=method,
|
||||
headers={
|
||||
"Authorization": f"Bearer {TOKEN}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
raw = resp.read()
|
||||
return resp.status, (json.loads(raw) if raw else {})
|
||||
except urllib.error.HTTPError as e:
|
||||
raw = e.read()
|
||||
try:
|
||||
parsed = json.loads(raw) if raw else {}
|
||||
except json.JSONDecodeError:
|
||||
parsed = {"raw": raw.decode(errors="replace")}
|
||||
return e.code, parsed
|
||||
|
||||
|
||||
def die(msg):
|
||||
print(f"FATAL: {msg}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def gen_secret(n=40):
|
||||
alphabet = string.ascii_letters + string.digits
|
||||
return "".join(secrets.choice(alphabet) for _ in range(n))
|
||||
|
||||
|
||||
def kubectl_get_secret_key(namespace, name, key):
|
||||
"""Returns decoded value, or None if the secret/key doesn't exist."""
|
||||
p = subprocess.run(
|
||||
["kubectl", "-n", namespace, "get", "secret", name, "-o", f"jsonpath={{.data.{key}}}"],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
if p.returncode != 0 or not p.stdout.strip():
|
||||
return None
|
||||
import base64
|
||||
return base64.b64decode(p.stdout).decode()
|
||||
|
||||
|
||||
def kubectl_create_secret(namespace, name, literals: dict):
|
||||
"""Idempotent: create-or-update via dry-run|apply, same pattern used
|
||||
elsewhere in this repo (setup_vault.sh, apply-vault-secrets.sh)."""
|
||||
args = ["kubectl", "-n", namespace, "create", "secret", "generic", name]
|
||||
for k, v in literals.items():
|
||||
args += [f"--from-literal={k}={v}"]
|
||||
args += ["--dry-run=client", "-o", "yaml"]
|
||||
render = subprocess.run(args, capture_output=True, text=True)
|
||||
if render.returncode != 0:
|
||||
die(f"rendering secret {namespace}/{name}: {render.stderr}")
|
||||
apply = subprocess.run(["kubectl", "apply", "-f", "-"], input=render.stdout,
|
||||
capture_output=True, text=True)
|
||||
if apply.returncode != 0:
|
||||
die(f"applying secret {namespace}/{name}: {apply.stderr}")
|
||||
print(f" secret {namespace}/{name}: {apply.stdout.strip()}")
|
||||
|
||||
|
||||
def get_or_create(list_path, create_path, query, payload, patch_existing=None):
|
||||
status, res = api("GET", f"{list_path}?{query}")
|
||||
if status != 200:
|
||||
die(f"GET {list_path}?{query} -> {status} {res}")
|
||||
results = res.get("results", [])
|
||||
if results:
|
||||
obj = results[0]
|
||||
if patch_existing:
|
||||
status, obj2 = api("PATCH", f"{create_path}{obj['pk']}/", patch_existing)
|
||||
if status not in (200, 201):
|
||||
die(f"PATCH {create_path}{obj['pk']}/ -> {status} {obj2}")
|
||||
return obj2
|
||||
return obj
|
||||
status, obj = api("POST", create_path, payload)
|
||||
if status not in (200, 201):
|
||||
die(f"POST {create_path} -> {status} {obj}")
|
||||
return obj
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
print("[1/5] Ensuring custom 'groups' scope mapping exists...")
|
||||
groups_mapping = get_or_create(
|
||||
"/api/v3/propertymappings/provider/scope/",
|
||||
"/api/v3/propertymappings/provider/scope/",
|
||||
"scope_name=groups",
|
||||
{
|
||||
"name": "homelab: groups claim",
|
||||
"scope_name": "groups",
|
||||
# request.user.ak_groups is deprecated in authentik 2026.x (logs a
|
||||
# deprecation warning on every token issue) -> use request.user.groups.
|
||||
"expression": (
|
||||
"return {\"groups\": [group.name for group in request.user.groups.all()]}"
|
||||
),
|
||||
},
|
||||
# Force the expression onto the already-created mapping on re-run.
|
||||
patch_existing={
|
||||
"expression": (
|
||||
"return {\"groups\": [group.name for group in request.user.groups.all()]}"
|
||||
),
|
||||
},
|
||||
)
|
||||
GROUPS_MAPPING_PK = groups_mapping["pk"]
|
||||
|
||||
# Fetch the standard openid/email/profile mapping pks (shipped by default).
|
||||
status, res = api("GET", "/api/v3/propertymappings/provider/scope/")
|
||||
by_scope = {m["scope_name"]: m["pk"] for m in res["results"]}
|
||||
SCOPE_PKS = [by_scope["openid"], by_scope["email"], by_scope["profile"], GROUPS_MAPPING_PK]
|
||||
|
||||
status, res = api("GET", "/api/v3/flows/instances/?slug=default-provider-authorization-implicit-consent")
|
||||
AUTHORIZATION_FLOW_PK = res["results"][0]["pk"]
|
||||
status, res = api("GET", "/api/v3/flows/instances/?slug=default-provider-invalidation-flow")
|
||||
INVALIDATION_FLOW_PK = res["results"][0]["pk"]
|
||||
status, res = api("GET", "/api/v3/crypto/certificatekeypairs/?has_key=true")
|
||||
SIGNING_KEY_PK = res["results"][0]["pk"]
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
print("[2/5] Ensuring groups homelab-admins / grafana-admins exist...")
|
||||
homelab_admins = get_or_create(
|
||||
"/api/v3/core/groups/", "/api/v3/core/groups/",
|
||||
"name=homelab-admins",
|
||||
{"name": "homelab-admins", "is_superuser": True},
|
||||
)
|
||||
grafana_admins = get_or_create(
|
||||
"/api/v3/core/groups/", "/api/v3/core/groups/",
|
||||
"name=grafana-admins",
|
||||
{"name": "grafana-admins", "is_superuser": False},
|
||||
)
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
print("[3/5] Ensuring user 'rock' exists with admin group membership...")
|
||||
status, res = api("GET", "/api/v3/core/users/?username=rock")
|
||||
rock_password = None
|
||||
if res.get("results"):
|
||||
rock = res["results"][0]
|
||||
status, rock = api("PATCH", f"/api/v3/core/users/{rock['pk']}/", {
|
||||
"groups": [homelab_admins["pk"], grafana_admins["pk"]],
|
||||
"is_active": True,
|
||||
})
|
||||
if status not in (200, 201):
|
||||
die(f"PATCH user rock -> {status} {rock}")
|
||||
print(" rock already exists, group membership synced (password unchanged)")
|
||||
else:
|
||||
rock_password = gen_secret(24)
|
||||
status, rock = api("POST", "/api/v3/core/users/", {
|
||||
"username": "rock",
|
||||
"name": "Rock",
|
||||
"is_active": True,
|
||||
"groups": [homelab_admins["pk"], grafana_admins["pk"]],
|
||||
"path": "users",
|
||||
"type": "internal",
|
||||
})
|
||||
if status not in (200, 201):
|
||||
die(f"POST user rock -> {status} {rock}")
|
||||
status, pw_res = api("POST", f"/api/v3/core/users/{rock['pk']}/set_password/",
|
||||
{"password": rock_password})
|
||||
if status not in (200, 204):
|
||||
die(f"set_password for rock -> {status} {pw_res}")
|
||||
kubectl_create_secret("iam", "rock-credentials", {
|
||||
"username": "rock",
|
||||
"password": rock_password,
|
||||
})
|
||||
print(" rock created, credentials stored in iam/rock-credentials")
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
print("[4/5] Ensuring OAuth2 providers + applications for grafana/minio/forgejo/argocd...")
|
||||
|
||||
SERVICES = {
|
||||
"grafana": {
|
||||
"client_secret_source": ("logging", "grafana-oidc", "GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET"),
|
||||
"redirect_uris": ["https://grafana.riotpiao.com/login/generic_oauth"],
|
||||
"launch_url": "https://grafana.riotpiao.com",
|
||||
"display_name": "Grafana",
|
||||
},
|
||||
"minio": {
|
||||
"client_secret_source": ("storage", "minio-oidc", "MINIO_IDENTITY_OPENID_CLIENT_SECRET"),
|
||||
"redirect_uris": ["https://minio.riotpiao.com/oauth_callback"],
|
||||
"launch_url": "https://minio.riotpiao.com",
|
||||
"display_name": "MinIO",
|
||||
},
|
||||
"forgejo": {
|
||||
# No secret exists yet for forgejo - generate + store on first run.
|
||||
"client_secret_source": ("cicd", "forgejo-oidc", "CLIENT_SECRET"),
|
||||
"generate_if_missing": True,
|
||||
"redirect_uris": [
|
||||
"https://forgejo.riotpiao.com/user/oauth2/authentik/callback",
|
||||
"https://forgejo.riotpiao.com/user/oauth2/openidconnect/callback",
|
||||
],
|
||||
"launch_url": "https://forgejo.riotpiao.com",
|
||||
"display_name": "Forgejo",
|
||||
},
|
||||
"argocd": {
|
||||
# oidc-secret uses hyphenated keys (client-id/client-secret) per
|
||||
# argocd-values.yaml's `$oidc-secret:client-id` / `:client-secret` refs.
|
||||
"client_secret_source": ("argocd", "oidc-secret", "client-secret"),
|
||||
"generate_if_missing": True,
|
||||
"extra_secret_literals": {"client-id": "argocd"},
|
||||
"redirect_uris": ["https://argocd.riotpiao.com/auth/callback"],
|
||||
"launch_url": "https://argocd.riotpiao.com",
|
||||
"display_name": "Argo CD",
|
||||
},
|
||||
"homarr": {
|
||||
"client_secret_source": ("dashboard", "homarr-oidc", "client-secret"),
|
||||
"generate_if_missing": True,
|
||||
"extra_secret_literals": {"client-id": "homarr"},
|
||||
"redirect_uris": ["https://homarr.riotpiao.com/api/auth/callback/oidc"],
|
||||
"launch_url": "https://homarr.riotpiao.com",
|
||||
"display_name": "Homarr",
|
||||
},
|
||||
}
|
||||
|
||||
app_pks_for_binding = []
|
||||
|
||||
for name, cfg in SERVICES.items():
|
||||
ns, secret_name, key = cfg["client_secret_source"]
|
||||
client_secret = kubectl_get_secret_key(ns, secret_name, key)
|
||||
if client_secret is None:
|
||||
if not cfg.get("generate_if_missing"):
|
||||
print(f" WARNING: {ns}/{secret_name} key {key} not found and "
|
||||
f"generate_if_missing not set for '{name}' - skipping provider/app")
|
||||
continue
|
||||
client_secret = gen_secret(40)
|
||||
literals = {key: client_secret}
|
||||
literals.update(cfg.get("extra_secret_literals", {}))
|
||||
kubectl_create_secret(ns, secret_name, literals)
|
||||
print(f" {name}: generated new client secret -> {ns}/{secret_name}")
|
||||
else:
|
||||
print(f" {name}: using existing client secret from {ns}/{secret_name}")
|
||||
|
||||
provider = get_or_create(
|
||||
"/api/v3/providers/oauth2/", "/api/v3/providers/oauth2/",
|
||||
f"name={name}",
|
||||
{
|
||||
"name": name,
|
||||
"client_id": name,
|
||||
"client_secret": client_secret,
|
||||
"client_type": "confidential",
|
||||
"authorization_flow": AUTHORIZATION_FLOW_PK,
|
||||
"invalidation_flow": INVALIDATION_FLOW_PK,
|
||||
"signing_key": SIGNING_KEY_PK,
|
||||
"property_mappings": SCOPE_PKS,
|
||||
"sub_mode": "hashed_user_id",
|
||||
"include_claims_in_id_token": True,
|
||||
# authentik 2026.x requires grant_types to be set explicitly; the
|
||||
# API defaults it to [] when omitted, which makes /authorize reject
|
||||
# every login with "Invalid grant_type for provider" ->
|
||||
# invalid_request. authorization_code = the web SSO flow all these
|
||||
# apps use; refresh_token = long-lived sessions (offline_access).
|
||||
"grant_types": ["authorization_code", "refresh_token"],
|
||||
"redirect_uris": [
|
||||
{"matching_mode": "strict", "url": u} for u in cfg["redirect_uris"]
|
||||
],
|
||||
},
|
||||
# Keep the redirect_uris/mappings/grant_types in sync on re-run, but
|
||||
# never touch client_secret again once created (that's the source of
|
||||
# truth in the k8s Secret, and re-sending it here is harmless anyway).
|
||||
patch_existing={
|
||||
"property_mappings": SCOPE_PKS,
|
||||
"grant_types": ["authorization_code", "refresh_token"],
|
||||
"redirect_uris": [
|
||||
{"matching_mode": "strict", "url": u} for u in cfg["redirect_uris"]
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
# superuser_full_list=true is REQUIRED on the LIST: the applications list
|
||||
# applies access-policy filtering to the results array (these apps are bound
|
||||
# to homelab-admins, and the bootstrap-token user akadmin is not a member),
|
||||
# so without it the GET returns an empty results list even though the app
|
||||
# exists -> fall through to POST -> 400 "already exists".
|
||||
#
|
||||
# We deliberately do NOT patch_existing here: the application DETAIL endpoint
|
||||
# (PATCH /applications/{pk}/) enforces the same access policy and does NOT
|
||||
# honor superuser_full_list, so PATCH-by-pk returns 404 for akadmin once the
|
||||
# homelab-admins binding exists. That 404 aborted the loop before later
|
||||
# providers got their grant_types. slug/provider/launch_url are set at
|
||||
# creation and are stable (provider is get_or_create'd by name, stable pk),
|
||||
# so find-or-create is sufficient.
|
||||
application = get_or_create(
|
||||
"/api/v3/core/applications/", "/api/v3/core/applications/",
|
||||
f"slug={name}&superuser_full_list=true",
|
||||
{
|
||||
"name": cfg["display_name"],
|
||||
"slug": name,
|
||||
"provider": provider["pk"],
|
||||
"meta_launch_url": cfg["launch_url"],
|
||||
},
|
||||
)
|
||||
app_pks_for_binding.append((name, application["pk"]))
|
||||
print(f" {name}: provider pk={provider['pk']} application pk={application['pk']}")
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
print("[5/5] Binding homelab-admins to every application (guaranteed access for rock)...")
|
||||
for name, app_pk in app_pks_for_binding:
|
||||
get_or_create(
|
||||
"/api/v3/policies/bindings/", "/api/v3/policies/bindings/",
|
||||
f"target={app_pk}&group={homelab_admins['pk']}",
|
||||
{
|
||||
"target": app_pk,
|
||||
"group": homelab_admins["pk"],
|
||||
"order": 0,
|
||||
"enabled": True,
|
||||
},
|
||||
)
|
||||
print(f" {name}: homelab-admins bound")
|
||||
|
||||
print("\nDone. Summary:")
|
||||
print(" groups: homelab-admins (superuser), grafana-admins")
|
||||
print(" user: rock -> homelab-admins + grafana-admins")
|
||||
print(f" apps: {', '.join(n for n, _ in app_pks_for_binding)}")
|
||||
if rock_password:
|
||||
print(" NOTE: rock's password was generated this run - see")
|
||||
print(" kubectl -n iam get secret rock-credentials -o jsonpath='{.data.password}' | base64 -d")
|
||||
@@ -0,0 +1,174 @@
|
||||
# k8s/talos-iam/vault-values.yaml
|
||||
# HashiCorp Vault — secrets backend for the homelab.
|
||||
# Stores OIDC client secrets, TLS certs, and any other sensitive values.
|
||||
# Accessed via the `talos` CLI (talos-cli/) which wraps `vault kv get/put`.
|
||||
#
|
||||
# Storage backend: MinIO S3 (minio.storage.svc.cluster.local) — no extra PVC.
|
||||
# Auto-unseal: postStart hook reads unseal keys from vault-unseal-keys Secret
|
||||
# (written by setup_vault.sh after operator init; operator must run that script
|
||||
# once after first install to initialize and store the keys).
|
||||
|
||||
# ── Global ────────────────────────────────────────────────────────────────────
|
||||
# tlsDisable: true — TLS terminated at the nginx ingress (vault.riotpiao.com)
|
||||
# or at port-forward. In-cluster traffic to Vault is plain HTTP; this is acceptable
|
||||
# because all clients are on the pod network (not crossing node boundaries).
|
||||
global:
|
||||
enabled: true
|
||||
tlsDisable: true
|
||||
|
||||
# ── Agent Injector ────────────────────────────────────────────────────────────
|
||||
# The injector mutates pods to sidecar Vault Agent for automatic secret injection.
|
||||
# Not used here — secrets are fetched explicitly via the talos CLI.
|
||||
# Enabling it would add a webhook that intercepts all pod creates cluster-wide,
|
||||
# which is unnecessary overhead for a homelab with manual secret management.
|
||||
injector:
|
||||
enabled: false
|
||||
|
||||
server:
|
||||
replicas: 1
|
||||
|
||||
annotations:
|
||||
secret.reloader.stakater.com/reload: "vault-unseal-keys"
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
|
||||
# ── Scheduling ─────────────────────────────────────────────────────────────
|
||||
# Tolerate cp-1 so Vault can run there if worker-1 is down.
|
||||
# Prefer worker-1 under normal conditions (keeps Vault off the same node as etcd).
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- weight: 100
|
||||
preference:
|
||||
matchExpressions:
|
||||
- key: node-role.kubernetes.io/worker
|
||||
operator: Exists
|
||||
|
||||
# ── Environment variables ───────────────────────────────────────────────────
|
||||
# extraEnvironmentVars: non-secret config passed directly.
|
||||
extraEnvironmentVars:
|
||||
VAULT_LOG_LEVEL: info
|
||||
|
||||
# extraSecretEnvironmentVars: pulls values from K8s Secrets into env vars.
|
||||
# vault-minio-creds is created by the helmfile presync hook from MINIO_ROOT_USER/PASSWORD.
|
||||
# vault-unseal-keys is a placeholder created at first deploy; setup_vault.sh
|
||||
# overwrites it with real unseal keys after `vault operator init`.
|
||||
# Vault reads the keys from env on every pod start and the postStart hook unseals.
|
||||
extraSecretEnvironmentVars:
|
||||
- envName: AWS_ACCESS_KEY_ID
|
||||
secretName: vault-minio-creds
|
||||
secretKey: access_key
|
||||
- envName: AWS_SECRET_ACCESS_KEY
|
||||
secretName: vault-minio-creds
|
||||
secretKey: secret_key
|
||||
- envName: VAULT_UNSEAL_KEY_1
|
||||
secretName: vault-unseal-keys
|
||||
secretKey: key1
|
||||
- envName: VAULT_UNSEAL_KEY_2
|
||||
secretName: vault-unseal-keys
|
||||
secretKey: key2
|
||||
- envName: VAULT_UNSEAL_KEY_3
|
||||
secretName: vault-unseal-keys
|
||||
secretKey: key3
|
||||
|
||||
# ── Auto-unseal ─────────────────────────────────────────────────────────────
|
||||
# Vault starts sealed after every pod restart and can't serve requests until
|
||||
# unsealed. postStart runs immediately after the container starts, sleeps 5s
|
||||
# to let the Vault process bind its port, then feeds the unseal keys one by one.
|
||||
# `|| true` prevents the hook from failing if a key was already used (idempotent).
|
||||
# 3-of-5 Shamir unseal is the default — we stored all 3 used keys in the Secret.
|
||||
postStart:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
sleep 5
|
||||
vault operator unseal "$VAULT_UNSEAL_KEY_1" || true
|
||||
vault operator unseal "$VAULT_UNSEAL_KEY_2" || true
|
||||
vault operator unseal "$VAULT_UNSEAL_KEY_3" || true
|
||||
|
||||
# ── Vault config (HCL) ──────────────────────────────────────────────────────
|
||||
standalone:
|
||||
enabled: true
|
||||
config: |
|
||||
ui = false # UI served via Vault's own HTTP; enabled below via ui: enabled: true
|
||||
|
||||
listener "tcp" {
|
||||
address = "0.0.0.0:8200"
|
||||
tls_disable = "true"
|
||||
|
||||
# No auth in front of Vault's metrics endpoint — acceptable since all
|
||||
# Prometheus scrape traffic stays on the pod network (not exposed via ingress).
|
||||
telemetry {
|
||||
unauthenticated_metrics_access = "true"
|
||||
}
|
||||
}
|
||||
|
||||
telemetry {
|
||||
prometheus_retention_time = "30s"
|
||||
disable_hostname = true
|
||||
}
|
||||
|
||||
# S3 storage backend pointing at the in-cluster MinIO service.
|
||||
# AWS_ env vars (from vault-minio-creds Secret) supply the credentials.
|
||||
# s3_force_path_style: MinIO uses path-style URLs (not virtual-hosted).
|
||||
# disable_ssl: MinIO in this cluster has no TLS.
|
||||
storage "s3" {
|
||||
endpoint = "http://minio-cluster-hl.storage.svc.cluster.local:9000"
|
||||
bucket = "vault"
|
||||
region = "us-east-1"
|
||||
s3_force_path_style = "true"
|
||||
disable_ssl = "true"
|
||||
max_parallel = 128
|
||||
}
|
||||
|
||||
# api_addr: the address other Vault nodes (or HA standbys) use to reach
|
||||
# this node. Single-node standalone, but Vault requires it to be set.
|
||||
api_addr = "http://vault.iam.svc.cluster.local:8200"
|
||||
cluster_addr = "https://vault-0.vault-internal.iam.svc.cluster.local:8201"
|
||||
|
||||
# ── Service ─────────────────────────────────────────────────────────────────
|
||||
# NodePort 32171 — fallback for direct node access during bootstrap before
|
||||
# the ingress is up. Normal access is via nginx ingress (vault.riotpiao.com).
|
||||
service:
|
||||
type: NodePort
|
||||
port: 8200
|
||||
nodePort: 32171
|
||||
|
||||
# ── Persistence ─────────────────────────────────────────────────────────────
|
||||
# No PVC — all Vault state (secrets, policies, tokens) is stored in MinIO S3.
|
||||
# This means Vault survives node loss as long as MinIO is healthy.
|
||||
dataStorage:
|
||||
enabled: false
|
||||
|
||||
auditStorage:
|
||||
enabled: false
|
||||
|
||||
# ── UI ────────────────────────────────────────────────────────────────────────
|
||||
# Vault's web UI is used for the OIDC browser login flow (Vault as an OIDC
|
||||
# provider, if configured) and for manual operator inspection.
|
||||
# Accessible at http://vault.riotpiao.com or via port-forward.
|
||||
ui:
|
||||
enabled: true
|
||||
|
||||
# ── Metrics ───────────────────────────────────────────────────────────────────
|
||||
# vault_core_unsealed is the availability signal (0 after a restart until the
|
||||
# postStart hook above finishes unsealing). Pairs with the telemetry{} stanzas
|
||||
# in standalone.config above, which actually turn the /v1/sys/metrics endpoint on.
|
||||
serverTelemetry:
|
||||
serviceMonitor:
|
||||
enabled: true
|
||||
selectors: {}
|
||||
interval: 30s
|
||||
scrapeTimeout: 10s
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
adminPassword: ENC[AES256_GCM,data:qykgCu5Kv5IwzM1JFLXL7wgiAyI=,iv:bAph3Jd4fxuyglS7zrWIwzfUgK/wBwRrcEpKusgFhVk=,tag:ymwdnU/Dxnp54eGur2caoQ==,type:str]
|
||||
env:
|
||||
GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: ENC[AES256_GCM,data:FwldsYfbOkRYGcycxn5BHZk/dOZsu23e3UCPo7dJKrcyr0n8dXQ9nH5XVSRHDpLOdRk7wbQwl+4Rgb8bGs21/w==,iv:JkNEfbsEIeE9TGouDA1xgjS4fX+v/sDrimaGpc4iaME=,tag:6Z3X29GKbgd2x6ghWA+CUw==,type:str]
|
||||
sops:
|
||||
age:
|
||||
- enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQdVNSMWZaVDZuN2RBKyt0
|
||||
eWJHRm5vRlErZ1UweXA0N2xSNGdESE5leWo4CjQ2TTlONnpJK2VZWmxoamdhdHZN
|
||||
ZGVMSU84SUZpVTUxREZlek9JcUxaTXcKLS0tIHlNaEpzZG1iaVpvU0E1UTQzbkVk
|
||||
WnQxSHZ6S3cyZ2hUTlpTNXQyaVZzRU0KwU/0aRJ0D/UsqtFcrJLmuHpNELhTsfYS
|
||||
4WL2TQ/yaUz261s7LEVQdpKUza8JjQXiKqqNcV2gaqEWurdx20ILzQ==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
recipient: age1smu533f803gmd0jq60s2zaj9zlznajy0ca6rtewd4r37mr2hs3uqsrldfh
|
||||
lastmodified: "2026-07-15T23:02:08Z"
|
||||
mac: ENC[AES256_GCM,data:LeN0mm7i7fsDpwtGQ6La7/UzhHbKfG1f/K30wW7fZJ1XbqPZGh7cdq07kMqrodlR6oWvrq7r4gbyuhcH9ptlxKgN4NuD3OoUjn6/CiPEF+wzgKMMgg2JdqFLsZNxVvGL/Ig3Y5hP5iag91QdtQxXaJmaV6Q9+FF6maYofsOU2rA=,iv:Eik9jEhdu+7qJv1oN7e1Vydnv7rh0pnRpbcBPDhh8e8=,tag:Mkry9bpTsmsf9Xs1jW7b1g==,type:str]
|
||||
unencrypted_suffix: _unencrypted
|
||||
version: 3.13.2
|
||||
@@ -0,0 +1,200 @@
|
||||
# logging/grafana-values.yaml
|
||||
# Grafana — dashboarding and log/metrics exploration UI.
|
||||
# Deployed in the logging namespace alongside Loki and Promtail.
|
||||
#
|
||||
# Secrets provided by ArgoCD SOPS CMP plugin:
|
||||
# adminPassword — mounted from the grafana-admin K8s Secret (admin.existingSecret)
|
||||
# OAuth client secret — mounted from the grafana-oidc K8s Secret (envFromSecret below)
|
||||
|
||||
replicas: 1
|
||||
|
||||
# RWO PVC (Longhorn) — old pod must fully terminate before the new one can
|
||||
# mount the volume. Recreate avoids the "two pods fighting over one PVC" failure.
|
||||
deploymentStrategy:
|
||||
type: Recreate
|
||||
|
||||
podAnnotations:
|
||||
secret.reloader.stakater.com/reload: "grafana-oidc,grafana-admin"
|
||||
|
||||
adminUser: admin
|
||||
admin:
|
||||
existingSecret: grafana-admin
|
||||
existingSecretKey: admin-password
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 256Mi
|
||||
|
||||
# ── Persistence ───────────────────────────────────────────────────────────────
|
||||
# Stores dashboards saved in the UI, datasource edits, and user sessions.
|
||||
# Longhorn provides the RWO block volume; 5Gi is ample for dashboard JSON.
|
||||
persistence:
|
||||
enabled: true
|
||||
storageClassName: longhorn
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
size: 5Gi
|
||||
|
||||
# ── Grafana config ────────────────────────────────────────────────────────────
|
||||
grafana.ini:
|
||||
server:
|
||||
root_url: https://grafana.riotpiao.com
|
||||
|
||||
# No anonymous read access — every user must log in via Authentik SSO.
|
||||
auth.anonymous:
|
||||
enabled: false
|
||||
|
||||
# Explore tab: required for ad-hoc LogQL/PromQL queries against Loki/Prometheus.
|
||||
explore:
|
||||
enabled: true
|
||||
|
||||
# WAL for the embedded SQLite DB — prevents corruption on ungraceful shutdown.
|
||||
database:
|
||||
wal: true
|
||||
|
||||
# ── Authentik OIDC (generic OAuth2) ───────────────────────────────────────
|
||||
# Grafana v10+ supports OIDC auto-discovery; we wire it manually here because
|
||||
# Authentik's discovery endpoint is internal-only (no external DNS for iam.svc).
|
||||
# All URLs use the external hostname so CoreDNS rewrites them to
|
||||
# authentik-server.iam.svc — this keeps the Host header correct so Authentik
|
||||
# doesn't return localhost redirects in its token responses.
|
||||
#
|
||||
# role_attribute_path: JMESPath expression evaluated against the userinfo
|
||||
# response. Members of the 'grafana-admins' Authentik group get Admin role;
|
||||
# everyone else gets Viewer. The group name must match exactly what Authentik
|
||||
# sends in the 'groups' claim.
|
||||
auth.generic_oauth:
|
||||
enabled: true
|
||||
name: Authentik
|
||||
allow_sign_up: true
|
||||
client_id: grafana
|
||||
scopes: openid email profile
|
||||
auth_url: https://authentik.riotpiao.com/application/o/authorize/
|
||||
token_url: https://authentik.riotpiao.com/application/o/token/
|
||||
api_url: https://authentik.riotpiao.com/application/o/userinfo/
|
||||
role_attribute_path: "contains(groups[*], 'grafana-admins') && 'Admin' || 'Viewer'"
|
||||
use_pkce: false
|
||||
use_refresh_token: false
|
||||
skip_org_role_sync: false
|
||||
tls_skip_verify_insecure: true # Authentik uses self-signed cert; verify in prod
|
||||
|
||||
# GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET is injected from the grafana-oidc K8s
|
||||
# Secret (created by k8s/talos-iam/setup_talos_iam.sh). envFromSecret mounts
|
||||
# every key in that secret as an environment variable — keeps secrets out of
|
||||
# values files and out of git.
|
||||
envFromSecret: grafana-oidc
|
||||
|
||||
# ── Datasources ───────────────────────────────────────────────────────────────
|
||||
# Provisioned at install — no manual "Add datasource" step in the UI.
|
||||
# Loki is the default (log exploration); Prometheus is secondary (metrics).
|
||||
# Both use in-cluster service DNS — Grafana never leaves the cluster for data.
|
||||
datasources:
|
||||
datasources.yaml:
|
||||
apiVersion: 1
|
||||
datasources:
|
||||
- name: Loki
|
||||
type: loki
|
||||
uid: loki
|
||||
access: proxy
|
||||
url: http://loki.logging.svc.cluster.local:3100
|
||||
isDefault: true
|
||||
version: 1
|
||||
editable: true
|
||||
jsonData:
|
||||
maxLines: 1000
|
||||
timeout: 60
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
uid: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus-kube-prometheus-prometheus.monitoring.svc.cluster.local:9090
|
||||
isDefault: false
|
||||
version: 1
|
||||
editable: true
|
||||
jsonData:
|
||||
timeInterval: 30s
|
||||
timeout: 60
|
||||
|
||||
# ── Dashboard providers ───────────────────────────────────────────────────────
|
||||
# Tells Grafana to watch a directory for dashboard JSON files.
|
||||
# The `dashboards` block below populates that directory via an init container
|
||||
# that downloads from grafana.com at install time.
|
||||
dashboardProviders:
|
||||
dashboardproviders.yaml:
|
||||
apiVersion: 1
|
||||
providers:
|
||||
- name: default
|
||||
orgId: 1
|
||||
folder: ""
|
||||
type: file
|
||||
disableDeletion: false
|
||||
editable: true
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards/default
|
||||
|
||||
# ── Pre-loaded dashboards ──────────────────────────────────────────────────────
|
||||
# Fetched from grafana.com by an init container at helm-install time and baked
|
||||
# into ConfigMaps. gnetId + revision pin the exact dashboard version so an
|
||||
# upstream change on grafana.com doesn't silently alter what's deployed.
|
||||
dashboards:
|
||||
default:
|
||||
node-exporter-full:
|
||||
gnetId: 1860
|
||||
revision: 37
|
||||
datasource: Prometheus
|
||||
kubernetes-cluster:
|
||||
gnetId: 7249
|
||||
revision: 1
|
||||
datasource: Prometheus
|
||||
|
||||
# Allow scheduling on the control-plane node (talos-cp-1 carries NoSchedule taint).
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
|
||||
# Pin to az-a (talos-cp-1) — the only node running Longhorn. Without this the
|
||||
# scheduler may place grafana on cp-2/cp-3, where its Longhorn PVC cannot attach
|
||||
# ("CSINode does not contain driver driver.longhorn.io").
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: topology.kubernetes.io/zone
|
||||
operator: In
|
||||
values: [az-a]
|
||||
|
||||
# ClusterIP only — access via ingress (grafana.riotpiao.com) or port-forward.
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 80
|
||||
|
||||
# Ingress managed by k8s/ingress/ingress.yaml — not the chart's built-in ingress.
|
||||
ingress:
|
||||
enabled: false
|
||||
|
||||
serviceMonitor:
|
||||
enabled: false
|
||||
|
||||
# ── Dashboard-as-code (sidecar discovery) ──────────────────────────────────────
|
||||
# Additive to the gnetId-download mechanism above. The sidecar watches for
|
||||
# ConfigMaps labeled grafana_dashboard=1 in ANY namespace and loads them live —
|
||||
# no Grafana restart needed when a new dashboard ConfigMap is applied.
|
||||
sidecar:
|
||||
dashboards:
|
||||
enabled: true
|
||||
label: grafana_dashboard
|
||||
labelValue: "1"
|
||||
folder: /var/lib/grafana/dashboards/custom
|
||||
folderAnnotation: grafana_folder
|
||||
provider:
|
||||
name: custom
|
||||
folder: "Homelab"
|
||||
disableDelete: false
|
||||
foldersFromFilesStructure: true
|
||||
searchNamespace: ALL
|
||||
@@ -0,0 +1,24 @@
|
||||
loki:
|
||||
storage:
|
||||
s3:
|
||||
accessKeyId: ENC[AES256_GCM,data:rUe9BzMuNEk7Lg==,iv:a6oz4kQciHc/sqT1lu05CqI8+iJmi82JAQJ9BM5yBjo=,tag:M1CJmBQlbz/H08S2Y7eUCA==,type:str]
|
||||
secretAccessKey: ENC[AES256_GCM,data:WEvUewgbriABBYllwMtyKsQw850Eb0J6CHc/MEY/NlsolWp9+eq6wKChknI=,iv:0rq4mv8cQ1za8ZLlR8voePiwZFdzK8sbRDDqgloO68g=,tag:r4GNYDa2fL1+d0gY5ndG1Q==,type:str]
|
||||
bucketNames:
|
||||
chunks: ENC[AES256_GCM,data:cAwa5qibAOvuhCk=,iv:Jqt3iHhLaHae6g4CsylytZxg+3fnTdw4PUinhaUaOCI=,tag:jVduzj+ggN9vQSD62qZ0jw==,type:str]
|
||||
ruler: ENC[AES256_GCM,data:G1r90YhXWGkUOw==,iv:K1ddZrCVFAo+CZet+BV7KJh6RVPzGbsnVIkoZpjWCU0=,tag:o1xK1XLVtOqehTBIn9cNHg==,type:str]
|
||||
admin: ENC[AES256_GCM,data:LhWklk60wExptQ==,iv:DpuT40Jl9H1nH04u+gm7pHFox7QrZ52WqNQuBqGiz8Y=,tag:ylGjb9J5CGXq/CxHtq732g==,type:str]
|
||||
sops:
|
||||
age:
|
||||
- enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBRVW4vV0xvaGw4VUlGL1Fa
|
||||
Tjh4V3RUV2d5L1padVZRVkE1V3BTRjg4bzNzCndjSHowU0s5a3I4NGsxNlBUVXFo
|
||||
M1kwQjh3Rkw0VnlDWnh4Q3NSMmdUcXMKLS0tIGl4RWNvVzVyMHo5TC93SXJyWU1G
|
||||
QzBJK0RrSnFiaTgxNnNiZWthM1J4TnMKPb3uoQLHPSTW0+6Kgsapa5/u6cN7aJa1
|
||||
NcQIhd/PPibvvGb47OgLrh8djg7uGpVDMMn3v0Kv3Pu5bshJjZ2gBQ==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
recipient: age1smu533f803gmd0jq60s2zaj9zlznajy0ca6rtewd4r37mr2hs3uqsrldfh
|
||||
lastmodified: "2026-07-15T23:02:08Z"
|
||||
mac: ENC[AES256_GCM,data:Fg9WsV1LGCgCc9TQtxJ83rthGZCpi9Is/0ab7Udry90TxUHAnfhktP/V74lh8XQq+mVOjCT5ZAw1mpM93lj92/o3ZTb2LnGoaxD4Mj/3eL1T3oH6GQH9VRb7iq+aD5BkYZQd4jmxiIVET3K8a6a7Tt1hrWeKUYz1fcsC9IFIOJQ=,iv:sF8T/Q7+RBrU3dXBBDGS2j4SSiVIpE5wBlGgK7YKGTY=,tag:wqKm9p2SctnsS0jrk4jF/w==,type:str]
|
||||
unencrypted_suffix: _unencrypted
|
||||
version: 3.13.2
|
||||
@@ -0,0 +1,178 @@
|
||||
# logging/loki-values.yaml
|
||||
# Grafana Loki — log aggregation backend for the homelab.
|
||||
# Deployed in SingleBinary mode: one pod handles ingest, query, and compaction.
|
||||
# Chunks are stored in MinIO (S3-compatible) — no local PVC needed for log data.
|
||||
#
|
||||
# MinIO credentials are injected at deploy time via helmfile --set:
|
||||
# loki.storage.s3.accessKeyId ← MINIO_ROOT_USER
|
||||
# loki.storage.s3.secretAccessKey ← MINIO_ROOT_PASSWORD
|
||||
# The placeholder values below are overridden and never used.
|
||||
|
||||
# ── Deployment mode ───────────────────────────────────────────────────────────
|
||||
# SingleBinary collapses all Loki components (ingester, querier, compactor, ruler)
|
||||
# into one Deployment. Simpler ops for a homelab — no inter-component networking
|
||||
# or separate scaling to worry about. The tradeoff is no horizontal scaling.
|
||||
deploymentMode: SingleBinary
|
||||
|
||||
loki:
|
||||
# auth_enabled: false skips tenant header (X-Scope-OrgID) enforcement.
|
||||
# All Promtail → Loki traffic is internal; multi-tenancy adds no value here.
|
||||
auth_enabled: false
|
||||
|
||||
# ── Replication ─────────────────────────────────────────────────────────────
|
||||
# replication_factor: 1 — single replica, no write quorum needed.
|
||||
# Higher values require multiple ingesters (only valid outside SingleBinary).
|
||||
commonConfig:
|
||||
replication_factor: 1
|
||||
|
||||
# ── Storage backend ──────────────────────────────────────────────────────────
|
||||
# s3 type with s3ForcePathStyle: MinIO exposes buckets as paths
|
||||
# (http://host:9000/bucket) not subdomains (http://bucket.host:9000).
|
||||
# insecure: true — MinIO in this cluster has no TLS; traffic stays in-cluster.
|
||||
# Three buckets: chunks (log data), ruler (recording/alerting rules), admin (index).
|
||||
storage:
|
||||
type: s3
|
||||
s3:
|
||||
endpoint: minio.storage.svc.cluster.local:80 # operator svc maps :80 -> pod :9000
|
||||
region: us-east-1 # MinIO ignores region but Loki's S3 client requires it
|
||||
s3ForcePathStyle: true
|
||||
insecure: true
|
||||
# Interpolated at runtime from the loki-s3-creds Secret (SOPS-CMP emitted),
|
||||
# injected as env vars via extraEnvFrom below. Requires -config.expand-env.
|
||||
access_key_id: "${access_key_id}"
|
||||
secret_access_key: "${secret_access_key}"
|
||||
bucketNames:
|
||||
chunks: loki-chunks
|
||||
ruler: loki-ruler
|
||||
admin: loki-admin
|
||||
|
||||
# ── Schema ───────────────────────────────────────────────────────────────────
|
||||
# v13 + TSDB is the current recommended schema (Loki 3.x).
|
||||
# from: sets the date after which this schema applies — logs before this date
|
||||
# would use a previous schema config (none exists here, so all logs use v13).
|
||||
# period: 24h means one index table per day in the object store.
|
||||
schemaConfig:
|
||||
configs:
|
||||
- from: "2024-01-01"
|
||||
store: boltdb-shipper
|
||||
object_store: s3
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
# ── Ingester ─────────────────────────────────────────────────────────────────
|
||||
# Controls how log chunks are buffered before being flushed to MinIO.
|
||||
# chunk_idle_period: flush a chunk if no new logs arrive for 3m (reduces
|
||||
# open chunk count). chunk_retain_period: keep flushed chunks in memory
|
||||
# briefly so late-arriving out-of-order logs can still be appended.
|
||||
# WAL persists in-memory chunks to disk — required for boltdb-shipper.
|
||||
ingester:
|
||||
chunk_idle_period: 3m
|
||||
chunk_block_size: 262144
|
||||
chunk_retain_period: 1m
|
||||
wal:
|
||||
dir: /var/loki/wal
|
||||
|
||||
# ── Compactor ────────────────────────────────────────────────────────────────
|
||||
# Merges small index files written by ingesters into larger ones, and
|
||||
# enforces retention by deleting chunks older than retention_period.
|
||||
# retention_delete_delay: waits 2h after marking chunks for deletion before
|
||||
# actually removing them — safety window if a query is still reading them.
|
||||
compactor:
|
||||
working_directory: /var/loki/compactor
|
||||
compaction_interval: 10m
|
||||
retention_enabled: true
|
||||
retention_delete_delay: 2h
|
||||
retention_delete_worker_count: 150
|
||||
delete_request_store: s3
|
||||
|
||||
# ── Limits ───────────────────────────────────────────────────────────────────
|
||||
# retention_period: 10 days. Homelab — no long-term log storage needed.
|
||||
# ingestion_rate_mb / burst: rate limits per tenant (single tenant here).
|
||||
# 4 MB/s steady, 6 MB/s burst — plenty for a 3-node cluster.
|
||||
# max_query_series: caps how many unique label combinations a single query
|
||||
# can return — prevents runaway cardinality queries from OOMing the pod.
|
||||
# max_query_lookback: hard cap matching retention_period (no point querying
|
||||
# further back than what's stored).
|
||||
# allow_structured_metadata: false — required for boltdb-shipper index store.
|
||||
limits_config:
|
||||
retention_period: 240h
|
||||
ingestion_rate_mb: 4
|
||||
ingestion_burst_size_mb: 6
|
||||
max_query_series: 5000
|
||||
max_query_lookback: 240h
|
||||
max_label_names_per_series: 30
|
||||
allow_structured_metadata: false
|
||||
# Query timeout: increased to 120s to tolerate 5+ second network latency spikes
|
||||
# Default: 30s — too aggressive when pod-to-pod latency hits 5-10s
|
||||
query_timeout: 120s
|
||||
|
||||
# ── Single binary pod ─────────────────────────────────────────────────────────
|
||||
singleBinary:
|
||||
replicas: 1
|
||||
|
||||
# Interpolate ${access_key_id}/${secret_access_key} in the S3 config from the
|
||||
# loki-s3-creds Secret (SOPS-CMP emitted).
|
||||
extraArgs:
|
||||
- "-config.expand-env=true"
|
||||
extraEnvFrom:
|
||||
- secretRef:
|
||||
name: loki-s3-creds
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 1000m
|
||||
memory: 512Mi
|
||||
|
||||
# Persistent storage for indices + WAL. Durable log chunks live in MinIO.
|
||||
persistence:
|
||||
enabled: true
|
||||
storageClassName: longhorn
|
||||
size: 5Gi
|
||||
|
||||
# Pin to az-a (talos-cp-1) — sole Longhorn node; otherwise the PVC fails to
|
||||
# attach on cp-2/cp-3.
|
||||
nodeSelector:
|
||||
topology.kubernetes.io/zone: az-a
|
||||
|
||||
# ── Disable micro-service replicas ───────────────────────────────────────────
|
||||
# In SingleBinary mode the chart still templates read/write/backend Deployments
|
||||
# unless explicitly set to 0. Setting replicas: 0 keeps them out of the cluster.
|
||||
read:
|
||||
replicas: 0
|
||||
write:
|
||||
replicas: 0
|
||||
backend:
|
||||
replicas: 0
|
||||
|
||||
# Nginx gateway is only needed for multi-replica deployments that split read
|
||||
# and write paths. Not used in SingleBinary.
|
||||
gateway:
|
||||
enabled: false
|
||||
|
||||
# Disable the bundled MinIO subchart — we run our own minio-az-a/az-b releases
|
||||
# in the storage namespace with site replication.
|
||||
minio:
|
||||
enabled: false
|
||||
|
||||
# ── Monitoring ────────────────────────────────────────────────────────────────
|
||||
# Self-monitoring ships a Grafana Agent operator to scrape Loki's own metrics.
|
||||
# We use kube-prometheus-stack for that instead — avoid running two agents.
|
||||
# lokiCanary sends synthetic log lines to verify the write→read pipeline;
|
||||
# useful in production, too noisy for a homelab.
|
||||
monitoring:
|
||||
selfMonitoring:
|
||||
enabled: false
|
||||
grafanaAgent:
|
||||
installOperator: false
|
||||
lokiCanary:
|
||||
enabled: false
|
||||
serviceMonitor:
|
||||
enabled: false
|
||||
|
||||
test:
|
||||
enabled: false
|
||||
@@ -0,0 +1,238 @@
|
||||
# logging/promtail-values.yaml
|
||||
# Promtail — log shipper DaemonSet. One pod per node; reads container logs
|
||||
# from /var/log/pods and Talos kernel/service logs from /var/log/journal,
|
||||
# then pushes them to Loki.
|
||||
|
||||
daemonset:
|
||||
enabled: true
|
||||
|
||||
config:
|
||||
logLevel: warn # info is noisy at homelab scale; warn keeps Promtail quiet
|
||||
serverPort: 3101
|
||||
|
||||
# Push endpoint — in-cluster DNS, no auth (Loki has auth_enabled: false).
|
||||
clients:
|
||||
- url: http://loki.logging.svc.cluster.local:3100/loki/api/v1/push
|
||||
|
||||
snippets:
|
||||
scrapeConfigs: |
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# LOG AGGREGATION STRATEGY
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# Three scrape jobs balance log volume, visibility, and cost:
|
||||
#
|
||||
# 1. kubernetes-pods-critical (no sampling)
|
||||
# Namespaces: iam, monitoring, temporal, cicd, cert-manager
|
||||
# Strategy: Keep ALL logs (full fidelity for security/observability)
|
||||
# Rationale: Critical services justify full log retention
|
||||
#
|
||||
# 2. kubernetes-pods (50% sampling + selective drops)
|
||||
# Namespaces: all others (sqs, storage, ingress-nginx, llm, etc.)
|
||||
# Drops: health/heartbeat/ping/keepalive + DEBUG from verbose services
|
||||
# Sampling: 50% of INFO/DEBUG (ERROR/WARN always kept)
|
||||
# Rationale: Reduce noise while preserving error visibility
|
||||
#
|
||||
# 3. systemd-journal (no filtering)
|
||||
# Source: Talos node kernel/kubelet logs
|
||||
# Strategy: Keep all for system-level debugging
|
||||
#
|
||||
# Result: ~60-70% total log reduction vs. 90% before
|
||||
# All errors/warnings captured; info logs sampled intelligently
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
# ── Pod logs ──────────────────────────────────────────────────────────
|
||||
# Discovers pods via the Kubernetes API (kubernetes_sd_configs role: pod).
|
||||
# cri pipeline stage parses the CRI-O/containerd log format so timestamps
|
||||
# and stream (stdout/stderr) are extracted properly before the line is
|
||||
# forwarded to Loki.
|
||||
# relabel_configs build useful labels: namespace, pod, container, node,
|
||||
# and a job label of the form "namespace/pod-name" for easy filtering.
|
||||
- job_name: kubernetes-pods
|
||||
kubernetes_sd_configs:
|
||||
- role: pod
|
||||
relabel_configs:
|
||||
# Exclude critical namespaces (handled separately by kubernetes-pods-critical)
|
||||
- source_labels: [__meta_kubernetes_namespace]
|
||||
regex: '(iam|monitoring|temporal|cicd|cert-manager)'
|
||||
action: drop
|
||||
# Apply standard pod labels
|
||||
- source_labels: [__meta_kubernetes_pod_node_name]
|
||||
target_label: __host__
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_pod_label_(.+)
|
||||
- action: replace
|
||||
replacement: $1
|
||||
separator: /
|
||||
source_labels:
|
||||
- __meta_kubernetes_namespace
|
||||
- __meta_kubernetes_pod_name
|
||||
target_label: job
|
||||
- action: replace
|
||||
source_labels: [__meta_kubernetes_namespace]
|
||||
target_label: namespace
|
||||
- action: replace
|
||||
source_labels: [__meta_kubernetes_pod_name]
|
||||
target_label: pod
|
||||
- action: replace
|
||||
source_labels: [__meta_kubernetes_pod_container_name]
|
||||
target_label: container
|
||||
- replacement: /var/log/pods/*$1/*.log
|
||||
separator: /
|
||||
source_labels:
|
||||
- __meta_kubernetes_pod_uid
|
||||
- __meta_kubernetes_pod_container_name
|
||||
target_label: __path__
|
||||
- action: replace
|
||||
source_labels: [__meta_kubernetes_pod_node_name]
|
||||
target_label: node
|
||||
pipeline_stages:
|
||||
# ── Drop Rule 1: Noise — all services ──────────────────────────
|
||||
# Discard health checks, keep-alives, pings, trace logs
|
||||
- drop:
|
||||
expression: '(health|heartbeat|ping|pong|keepalive|KEEPALIVE|timeout.*retrying|level="trace"|level="TRACE")'
|
||||
|
||||
# ── Drop Rule 2: DEBUG from verbose services ────────────────────
|
||||
# storage, ingress-nginx, kube-system have chatty DEBUG logs
|
||||
# Example: {namespace="storage"...level="debug"}
|
||||
- drop:
|
||||
expression: 'namespace="(storage|ingress-nginx|kube-system|sqs)".*level="(debug|DEBUG)"'
|
||||
|
||||
# ── Sampling: 50% keep rate on INFO + DEBUG after drops ────────
|
||||
# Keep 100% of ERROR/WARN; keep 50% of INFO (if not dropped above)
|
||||
# This balances log volume with visibility
|
||||
- sampling:
|
||||
rate: 0.5
|
||||
enabled: true
|
||||
|
||||
# ── Pod logs from critical services (no sampling) ────────────────────
|
||||
# iam, monitoring, temporal, cicd = always keep all logs (no sampling)
|
||||
# These services are critical for debugging and deserve full log retention
|
||||
- job_name: kubernetes-pods-critical
|
||||
kubernetes_sd_configs:
|
||||
- role: pod
|
||||
pipeline_stages:
|
||||
# Only drop obvious noise; keep everything else
|
||||
- drop:
|
||||
expression: '(health|heartbeat|ping|pong|keepalive|KEEPALIVE)'
|
||||
relabel_configs:
|
||||
# Only select pods from critical namespaces
|
||||
- source_labels: [__meta_kubernetes_namespace]
|
||||
regex: '(iam|monitoring|temporal|cicd|cert-manager)'
|
||||
action: keep
|
||||
- source_labels: [__meta_kubernetes_pod_node_name]
|
||||
target_label: __host__
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_pod_label_(.+)
|
||||
- action: replace
|
||||
replacement: $1
|
||||
separator: /
|
||||
source_labels:
|
||||
- __meta_kubernetes_namespace
|
||||
- __meta_kubernetes_pod_name
|
||||
target_label: job
|
||||
- action: replace
|
||||
source_labels: [__meta_kubernetes_namespace]
|
||||
target_label: namespace
|
||||
- action: replace
|
||||
source_labels: [__meta_kubernetes_pod_name]
|
||||
target_label: pod
|
||||
- action: replace
|
||||
source_labels: [__meta_kubernetes_pod_container_name]
|
||||
target_label: container
|
||||
- replacement: /var/log/pods/*$1/*.log
|
||||
separator: /
|
||||
source_labels:
|
||||
- __meta_kubernetes_pod_uid
|
||||
- __meta_kubernetes_pod_container_name
|
||||
target_label: __path__
|
||||
- action: replace
|
||||
source_labels: [__meta_kubernetes_pod_node_name]
|
||||
target_label: node
|
||||
|
||||
# ── Talos systemd journal ──────────────────────────────────────────────
|
||||
# Talos runs containerd, kubelet, and kernel messages through systemd-
|
||||
# journald — they never appear in /var/log/pods. This job reads the
|
||||
# binary journal directly and emits unit (systemd unit name) and node
|
||||
# labels so you can filter by service (e.g. unit="kubelet.service").
|
||||
# max_age: 12h — only tail recent journal entries on startup; prevents
|
||||
# Promtail from replaying hours of history after a pod restart.
|
||||
- job_name: systemd-journal
|
||||
journal:
|
||||
path: /var/log/journal
|
||||
max_age: 12h
|
||||
labels:
|
||||
job: systemd-journal
|
||||
relabel_configs:
|
||||
- source_labels: [__journal__systemd_unit]
|
||||
target_label: unit
|
||||
- source_labels: [__journal__hostname]
|
||||
target_label: node
|
||||
|
||||
# ── Volume mounts ─────────────────────────────────────────────────────────────
|
||||
# hostPath mounts give Promtail access to the node's log directories.
|
||||
# /var/log/pods — container stdout/stderr (written by containerd's CRI layer)
|
||||
# /var/log/journal — Talos systemd journal (binary format, read via journald API)
|
||||
# DirectoryOrCreate on journal ensures the mount doesn't fail on fresh nodes
|
||||
# before journald has written anything.
|
||||
defaultVolumes:
|
||||
- name: pods-logs
|
||||
hostPath:
|
||||
path: /var/log/pods
|
||||
- name: journal
|
||||
hostPath:
|
||||
path: /var/log/journal
|
||||
type: DirectoryOrCreate
|
||||
|
||||
defaultVolumeMounts:
|
||||
- name: pods-logs
|
||||
mountPath: /var/log/pods
|
||||
readOnly: true
|
||||
- name: journal
|
||||
mountPath: /var/log/journal
|
||||
readOnly: true
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
cpu: 200m
|
||||
memory: 128Mi
|
||||
|
||||
# ── Security context ──────────────────────────────────────────────────────────
|
||||
# privileged: true — required to open the binary journal on Talos (journald
|
||||
# uses file locking that non-privileged processes can't bypass).
|
||||
# DAC_READ_SEARCH — lets Promtail read files owned by other UIDs in /var/log/pods
|
||||
# even with a read-only root filesystem. Without this, pod logs from containers
|
||||
# running as non-root UIDs would be unreadable.
|
||||
# readOnlyRootFilesystem: true — defence in depth; Promtail doesn't need to
|
||||
# write to its own container filesystem.
|
||||
# allowPrivilegeEscalation must be true when privileged: true — Kubernetes 1.26+
|
||||
# rejects privileged containers that explicitly set allowPrivilegeEscalation: false.
|
||||
containerSecurityContext:
|
||||
privileged: true
|
||||
allowPrivilegeEscalation: true
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
add:
|
||||
- DAC_READ_SEARCH
|
||||
|
||||
# runAsUser/Group: 0 (root) — needed to access journal files and pod log
|
||||
# directories that are owned by root on Talos nodes.
|
||||
podSecurityContext:
|
||||
runAsUser: 0
|
||||
runAsGroup: 0
|
||||
|
||||
# Tolerate every taint so Promtail runs on ALL nodes including the CP.
|
||||
# Without this, the control-plane node's logs (etcd, kube-apiserver) would
|
||||
# be missing from Loki entirely.
|
||||
tolerations:
|
||||
- effect: NoSchedule
|
||||
operator: Exists
|
||||
- effect: NoExecute
|
||||
operator: Exists
|
||||
|
||||
serviceMonitor:
|
||||
enabled: false
|
||||
@@ -0,0 +1,96 @@
|
||||
# PostSync hook Job that expands all Longhorn volumes to 3 replicas.
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: longhorn-expand-replicas
|
||||
namespace: longhorn-system
|
||||
annotations:
|
||||
argocd.argoproj.io/hook: PostSync
|
||||
argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
|
||||
spec:
|
||||
backoffLimit: 3
|
||||
template:
|
||||
metadata:
|
||||
name: longhorn-expand-replicas
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
serviceAccountName: longhorn-expand-replicas
|
||||
containers:
|
||||
- name: expand
|
||||
image: bitnami/kubectl:latest
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
set -euo pipefail
|
||||
|
||||
echo "Waiting for all 3 Longhorn nodes to be Ready..."
|
||||
for i in {1..30}; do
|
||||
READY_COUNT=$(kubectl -n longhorn-system get nodes.longhorn.io -o json | \
|
||||
jq -r '[.items[] | select(.status.conditions[] | select(.type=="Ready" and .status=="True"))] | length')
|
||||
|
||||
if [ "$READY_COUNT" -ge 3 ]; then
|
||||
echo "✓ All 3 nodes Ready"
|
||||
break
|
||||
fi
|
||||
|
||||
echo " $READY_COUNT/3 nodes ready, waiting..."
|
||||
sleep 10
|
||||
|
||||
if [ $i -eq 30 ]; then
|
||||
echo "✗ Timeout waiting for 3 nodes"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo
|
||||
echo "Expanding volumes with < 3 replicas..."
|
||||
|
||||
VOLUMES=$(kubectl -n longhorn-system get volumes.longhorn.io -o json | \
|
||||
jq -r '.items[] | select(.spec.numberOfReplicas < 3) | .metadata.name')
|
||||
|
||||
if [ -z "$VOLUMES" ]; then
|
||||
echo " No volumes need expansion"
|
||||
else
|
||||
echo "$VOLUMES" | while read -r vol; do
|
||||
CURRENT=$(kubectl -n longhorn-system get volume "$vol" -o jsonpath='{.spec.numberOfReplicas}')
|
||||
echo " $vol: $CURRENT → 3 replicas"
|
||||
kubectl -n longhorn-system patch volume "$vol" --type merge \
|
||||
-p '{"spec":{"numberOfReplicas":3}}'
|
||||
done
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "Done. Final replica counts:"
|
||||
kubectl -n longhorn-system get volumes.longhorn.io \
|
||||
-o custom-columns='NAME:.metadata.name,REPLICAS:.spec.numberOfReplicas' | head -20
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: longhorn-expand-replicas
|
||||
namespace: longhorn-system
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: longhorn-expand-replicas
|
||||
namespace: longhorn-system
|
||||
rules:
|
||||
- apiGroups: ["longhorn.io"]
|
||||
resources: ["volumes", "nodes"]
|
||||
verbs: ["get", "list", "patch"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: longhorn-expand-replicas
|
||||
namespace: longhorn-system
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: longhorn-expand-replicas
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: longhorn-expand-replicas
|
||||
namespace: longhorn-system
|
||||
@@ -0,0 +1,16 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: longhorn-system
|
||||
resources:
|
||||
- longhorn-storageclass.yaml
|
||||
- longhorn-cnpg-storageclass.yaml # CNPG-specific with postgres UID/GID
|
||||
- longhorn-servicemonitor.yaml
|
||||
- longhorn-taint-toleration.yaml
|
||||
- longhorn-nodes.yaml
|
||||
- expand-replicas-job.yaml
|
||||
- patch-csi-tolerations-job.yaml
|
||||
# Longhorn deployed via bootstrap script or Helm.
|
||||
# These manifests configure it: unified StorageClass (default, 3 replicas),
|
||||
# Prometheus ServiceMonitor, taint toleration for control-plane nodes, explicit
|
||||
# Node CRDs for cp-2/cp-3, CSI plugin tolerations, and a PostSync hook Job
|
||||
# that ensures all existing volumes have 3 replicas.
|
||||
@@ -0,0 +1,26 @@
|
||||
# StorageClass specifically for CNPG (CloudNativePG) PostgreSQL clusters
|
||||
# Fixes the "read-only filesystem" issue by mounting with postgres UID/GID
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
name: longhorn-cnpg
|
||||
namespace: longhorn-system
|
||||
annotations:
|
||||
storageclass.kubernetes.io/is-default-class: "false"
|
||||
provisioner: driver.longhorn.io
|
||||
allowVolumeExpansion: true
|
||||
parameters:
|
||||
numberOfReplicas: "3"
|
||||
staleReplicaTimeout: "30"
|
||||
fromBackup: ""
|
||||
dataLocality: "best-effort"
|
||||
fsType: "ext4"
|
||||
# Mount options to ensure PostgreSQL can write
|
||||
mkfsParams: "-O ^64bit,^metadata_csum"
|
||||
mountOptions:
|
||||
- "noatime"
|
||||
# Critical: mount with postgres UID/GID (26:26) to avoid permission issues
|
||||
- "uid=26"
|
||||
- "gid=26"
|
||||
reclaimPolicy: Delete
|
||||
volumeBindingMode: Immediate
|
||||
@@ -0,0 +1,35 @@
|
||||
# Longhorn Node CRDs for cp-2 and cp-3.
|
||||
# These nodes have the control-plane taint, so Longhorn doesn't auto-discover them.
|
||||
# Explicit Node CRDs + the taint-toleration setting enable storage across all 3 nodes.
|
||||
---
|
||||
apiVersion: longhorn.io/v1beta2
|
||||
kind: Node
|
||||
metadata:
|
||||
name: talos-cp-2
|
||||
namespace: longhorn-system
|
||||
spec:
|
||||
name: talos-cp-2
|
||||
allowScheduling: true
|
||||
disks:
|
||||
default-disk:
|
||||
allowScheduling: true
|
||||
path: /var/lib/longhorn
|
||||
storageReserved: 0
|
||||
tags: []
|
||||
tags: []
|
||||
---
|
||||
apiVersion: longhorn.io/v1beta2
|
||||
kind: Node
|
||||
metadata:
|
||||
name: talos-cp-3
|
||||
namespace: longhorn-system
|
||||
spec:
|
||||
name: talos-cp-3
|
||||
allowScheduling: true
|
||||
disks:
|
||||
default-disk:
|
||||
allowScheduling: true
|
||||
path: /var/lib/longhorn
|
||||
storageReserved: 0
|
||||
tags: []
|
||||
tags: []
|
||||
@@ -0,0 +1,23 @@
|
||||
# k8s/longhorn/longhorn-servicemonitor.yaml
|
||||
# Longhorn is deployed via raw manifest (cluster-config/longhorn_bootstrap.sh), not
|
||||
# Helm, so there's no chart values toggle for metrics — the manager already exposes
|
||||
# them on the existing longhorn-backend service (port 9500); this just wires Prometheus
|
||||
# to it. longhorn_volume_robustness is the availability signal (degraded/faulted).
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: longhorn-manager
|
||||
namespace: longhorn-system
|
||||
labels:
|
||||
app: longhorn-manager
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: longhorn-manager
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- longhorn-system
|
||||
endpoints:
|
||||
- port: manager
|
||||
interval: 30s
|
||||
scrapeTimeout: 10s
|
||||
@@ -0,0 +1,21 @@
|
||||
# Longhorn StorageClass - single unified storage class for the entire cluster
|
||||
# Replaces: longhorn-wffc, longhorn-kafka, longhorn-static (all deprecated)
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
name: longhorn
|
||||
annotations:
|
||||
storageclass.kubernetes.io/is-default-class: "true"
|
||||
description: "Longhorn distributed storage - 3 replicas, WaitForFirstConsumer"
|
||||
provisioner: driver.longhorn.io
|
||||
allowVolumeExpansion: true
|
||||
reclaimPolicy: Delete
|
||||
volumeBindingMode: Immediate # Immediate binding for StatefulSets (matches deployed config)
|
||||
parameters:
|
||||
numberOfReplicas: "3" # HA across all 3 nodes
|
||||
staleReplicaTimeout: "30"
|
||||
fromBackup: ""
|
||||
dataLocality: "disabled" # Match deployed config (not best-effort)
|
||||
fsType: "ext4"
|
||||
disableRevisionCounter: "true" # Performance optimization
|
||||
unmapMarkSnapChainRemoved: "ignored"
|
||||
@@ -0,0 +1,9 @@
|
||||
# Longhorn taint-toleration setting - allows Longhorn to run on all control-plane nodes.
|
||||
# Without this, Longhorn DaemonSet only runs on nodes without the control-plane taint,
|
||||
# which in a 3-CP cluster with dedicated cp-2/cp-3 means only cp-1 gets storage.
|
||||
apiVersion: longhorn.io/v1beta2
|
||||
kind: Setting
|
||||
metadata:
|
||||
name: taint-toleration
|
||||
namespace: longhorn-system
|
||||
value: "node-role.kubernetes.io/control-plane:NoSchedule"
|
||||
@@ -0,0 +1,84 @@
|
||||
# PostSync hook to patch longhorn-csi-plugin DaemonSet with control-plane tolerations
|
||||
# This runs after longhorn-config Application syncs, ensuring CSI plugin can run on all nodes
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: longhorn-patch-csi-tolerations
|
||||
namespace: longhorn-system
|
||||
annotations:
|
||||
argocd.argoproj.io/hook: PostSync
|
||||
argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
|
||||
spec:
|
||||
backoffLimit: 3
|
||||
template:
|
||||
metadata:
|
||||
name: patch-csi-tolerations
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
serviceAccountName: longhorn-patch-csi-tolerations
|
||||
containers:
|
||||
- name: patch
|
||||
image: bitnami/kubectl:latest
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
set -euo pipefail
|
||||
|
||||
echo "Patching longhorn-csi-plugin DaemonSet with control-plane tolerations..."
|
||||
|
||||
kubectl patch daemonset longhorn-csi-plugin -n longhorn-system --type=json -p='[
|
||||
{
|
||||
"op": "add",
|
||||
"path": "/spec/template/spec/tolerations/-",
|
||||
"value": {
|
||||
"key": "node-role.kubernetes.io/control-plane",
|
||||
"operator": "Exists",
|
||||
"effect": "NoSchedule"
|
||||
}
|
||||
}
|
||||
]'
|
||||
|
||||
echo "✓ Patch applied successfully"
|
||||
|
||||
echo ""
|
||||
echo "Waiting for CSI plugin pods to roll out to all nodes..."
|
||||
kubectl rollout status daemonset/longhorn-csi-plugin -n longhorn-system --timeout=120s
|
||||
|
||||
echo ""
|
||||
echo "Final status:"
|
||||
kubectl get daemonset longhorn-csi-plugin -n longhorn-system
|
||||
kubectl get pods -n longhorn-system -l app=longhorn-csi-plugin -o wide
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: longhorn-patch-csi-tolerations
|
||||
namespace: longhorn-system
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: longhorn-patch-csi-tolerations
|
||||
namespace: longhorn-system
|
||||
rules:
|
||||
- apiGroups: ["apps"]
|
||||
resources: ["daemonsets"]
|
||||
verbs: ["get", "patch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["pods"]
|
||||
verbs: ["list", "get"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: longhorn-patch-csi-tolerations
|
||||
namespace: longhorn-system
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: longhorn-patch-csi-tolerations
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: longhorn-patch-csi-tolerations
|
||||
namespace: longhorn-system
|
||||
@@ -0,0 +1,8 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: storage
|
||||
resources:
|
||||
- minio-tenant.yaml
|
||||
# The operator creates the minio S3/console/headless Services and the
|
||||
# declarative bucket + user from the Tenant spec — no hand-rolled Service or
|
||||
# Bucket/User CRs (those kinds don't exist in the operator CRD set).
|
||||
@@ -0,0 +1,37 @@
|
||||
# minio-operator-values.yaml
|
||||
# MinIO Operator deployment with metrics enabled
|
||||
|
||||
operator:
|
||||
image:
|
||||
repository: minio/operator
|
||||
tag: "v5.0.0"
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
replicaCount: 1
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
|
||||
metrics:
|
||||
enabled: true
|
||||
port: 8080
|
||||
|
||||
rbac:
|
||||
create: true
|
||||
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- weight: 100
|
||||
preference:
|
||||
matchExpressions:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
|
||||
console:
|
||||
enabled: false
|
||||
@@ -0,0 +1,72 @@
|
||||
apiVersion: ENC[AES256_GCM,data:nd0=,iv:fORqNsg82i9+5WRQkHi698q/qHI4A1JP+YEkOvMxEGw=,tag:bLdfd21vJn6Mn2k/ghx1iA==,type:str]
|
||||
kind: ENC[AES256_GCM,data:aPTyakwz,iv:0Fviiz5ngsaBD5IRn30p6Z93cELcuLNCWILJJQSn19M=,tag:0n0f69t4ZC5WULd9VBI9sg==,type:str]
|
||||
metadata:
|
||||
name: ENC[AES256_GCM,data:UQ0lTL18hb7bZR4=,iv:NDGvM319FCypxFLWwONcF12osaU/EL0IwD0c2xtqYnU=,tag:JYu3EMyW7Pc4HTCmU8IiTw==,type:str]
|
||||
namespace: ENC[AES256_GCM,data:H49h6xTVAQ==,iv:EgTi3jFfn+EYYxMTC1GCQmt1fuGGtVSD3fhytp+CiaM=,tag:ynf4iSonvnnaO5/Ujno3+A==,type:str]
|
||||
type: ENC[AES256_GCM,data:xDlXeb2q,iv:0pe80oQG0ZIerOLvqbiorJ76JlCPu6wIjt4lQfbsXjU=,tag:rmaLgRUr/ERPzIXeUD9qMA==,type:str]
|
||||
stringData:
|
||||
config.env: ENC[AES256_GCM,data:rM2sfC9apy4Bnlw609d+BXAenZzUcBxceQmyzlbrvgasp37JfS6hIAWOf5Qq62ZF+/fN7nXSB5oEeajAIV3YJoP8wKlqm3765zOkUVhFkX2YmhJH4CkoJa6XzX9CowCQtTW9h+7F5PwaGnLptQ==,iv:0RQVMi2mMHBRTJ1OX6IDFvq9Qr7zpqgQzt+MPPgBIHA=,tag:zu45S2b29SM9JjqxsKQHow==,type:str]
|
||||
sops:
|
||||
age:
|
||||
- enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBNZCtYcC9kanVtdFNSeksr
|
||||
NDhXVHBpWjhKaWJSVnR1Rk9QV3BVU25mRldjCitEOVNCNnZVVjNLQWhFejdGelJV
|
||||
V3Bhb3JTMU9xdEhOck00TGt4UFVaWTAKLS0tIEZ1VnFrVmkyTWZWR2hCc2hpWGVK
|
||||
Z0NuYjVIdXJYRlVkY3JSUWUzZzd1bkkK0jm57duPu+u2qwjhKaO/fTtH1+Gt8fSm
|
||||
vnLQsQZlkYZBYxHHwtBPC4JztYvmuKnPNYAPqRPfVB+9IEOpQEetkA==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
recipient: age1smu533f803gmd0jq60s2zaj9zlznajy0ca6rtewd4r37mr2hs3uqsrldfh
|
||||
lastmodified: "2026-07-20T19:41:57Z"
|
||||
mac: ENC[AES256_GCM,data:fFoW//RJnlx0jQ3aMsEtx/qy70LrnocVc4UfPUIFNJG377NG7GnDxs+JIQGXSNNzRoKH06U3AFmBl2BvIi6D67A1VbOS0/+zmSGcNxdmL3apauOmn34jgJ/ipyakuawUi1+CGgSy8eMqgLM9QCpBkGx6qTtjjnGVWHYbN1WG3iE=,iv:8MD02ysiRL3nxSQvhQ6e0ek9gVLHwcfUCGzX+Tmtdp4=,tag:X/SdhJes1rJLHy6S278ztg==,type:str]
|
||||
unencrypted_suffix: _unencrypted
|
||||
version: 3.13.2
|
||||
---
|
||||
apiVersion: ENC[AES256_GCM,data:Utg=,iv:SqcUMt/REXm9qIppTVP8PQ6tnVrgt2gc5aghD7S8KHI=,tag:zeSno+VCPn9EXDE1Uxs8Hg==,type:str]
|
||||
kind: ENC[AES256_GCM,data:nDUp5Fyy,iv:RVtkV8l4JvqOSUR/MlZHZxnR75+fAAR8gKPNbpOLWVk=,tag:AoNBqT3MVudiA56sIIqpnQ==,type:str]
|
||||
metadata:
|
||||
name: ENC[AES256_GCM,data:74hhzAFMvD+DuQ==,iv:wTw2eoooDA9EfB2df3jzb0IWUdtLSwcuKzx8yDBaihQ=,tag:mnnkksSy69Gtg4LyYYnjmg==,type:str]
|
||||
namespace: ENC[AES256_GCM,data:z8dPkzIQkg==,iv:zoH4ZWOtD3tF6pkAuQDrL2rSfuRp+9Nu8IJq2q0eqsY=,tag:/39AaFYlXKxKkCuZpeHZlg==,type:str]
|
||||
type: ENC[AES256_GCM,data:J7ALXn/x,iv:GUfgjXF1/z0jGCFdz5/ARVRQUBwrEAdpzfATiFNPVLs=,tag:HZem0E53fAykijNrthbs8Q==,type:str]
|
||||
stringData:
|
||||
MINIO_IDENTITY_OPENID_CLIENT_SECRET: ENC[AES256_GCM,data:w3SZTsxn59UfoS4rej78RToiQP3jzMOwUmNwjEFoPjkZAhHpKYX7Y4tYv5znt9h1cd2jGrorBVLf9YgIOihJxw==,iv:I1aGIikbHTKvh//4Inhp7UoNfhjlIXSpcyP4RqljvqM=,tag:mL3ayxC3PuvbgQosazgTow==,type:str]
|
||||
sops:
|
||||
age:
|
||||
- enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBNZCtYcC9kanVtdFNSeksr
|
||||
NDhXVHBpWjhKaWJSVnR1Rk9QV3BVU25mRldjCitEOVNCNnZVVjNLQWhFejdGelJV
|
||||
V3Bhb3JTMU9xdEhOck00TGt4UFVaWTAKLS0tIEZ1VnFrVmkyTWZWR2hCc2hpWGVK
|
||||
Z0NuYjVIdXJYRlVkY3JSUWUzZzd1bkkK0jm57duPu+u2qwjhKaO/fTtH1+Gt8fSm
|
||||
vnLQsQZlkYZBYxHHwtBPC4JztYvmuKnPNYAPqRPfVB+9IEOpQEetkA==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
recipient: age1smu533f803gmd0jq60s2zaj9zlznajy0ca6rtewd4r37mr2hs3uqsrldfh
|
||||
lastmodified: "2026-07-20T19:41:57Z"
|
||||
mac: ENC[AES256_GCM,data:fFoW//RJnlx0jQ3aMsEtx/qy70LrnocVc4UfPUIFNJG377NG7GnDxs+JIQGXSNNzRoKH06U3AFmBl2BvIi6D67A1VbOS0/+zmSGcNxdmL3apauOmn34jgJ/ipyakuawUi1+CGgSy8eMqgLM9QCpBkGx6qTtjjnGVWHYbN1WG3iE=,iv:8MD02ysiRL3nxSQvhQ6e0ek9gVLHwcfUCGzX+Tmtdp4=,tag:X/SdhJes1rJLHy6S278ztg==,type:str]
|
||||
unencrypted_suffix: _unencrypted
|
||||
version: 3.13.2
|
||||
---
|
||||
apiVersion: ENC[AES256_GCM,data:Mv4=,iv:u/JjNJfdyz7ehk8lHZRh1u9zkNuufHosyXv10Gm+T6E=,tag:J4W7uyf3iYImPh2OSVWvNA==,type:str]
|
||||
kind: ENC[AES256_GCM,data:VPEKGVDy,iv:DymezGpBHmKT3BM/CcUKEGIWZw/JXNutReSKfhfmoD4=,tag:xqIUhSu4RSxameNPoFZVNQ==,type:str]
|
||||
metadata:
|
||||
name: ENC[AES256_GCM,data:bckq1pAY9UIMm3FUyGgafvQ=,iv:4uX7+Sl1UzgsImBH+qm9p8DlOfJu0i0W3bpoE7cFxXg=,tag:wvrBClQahE2OeVzeiubOEg==,type:str]
|
||||
namespace: ENC[AES256_GCM,data:5GotV/mD9g==,iv:PQrfm67la6RJCRq0H7qpiY8n2hPUjcaK6gmJ/zPQW30=,tag:fCupPaa2X/ztuoHbtr/HEQ==,type:str]
|
||||
type: ENC[AES256_GCM,data:q8D3YEZZ,iv:EXOe0day1ScxvZ7ozn1QyHgaZgnvlPNmvSQG74aw7lM=,tag:lxOx4u4LGcXCyK32BCD1XQ==,type:str]
|
||||
stringData:
|
||||
CONSOLE_ACCESS_KEY: ENC[AES256_GCM,data:XJCgReZf/FVli+lfA9ynF+s=,iv:gGTaSKxk721GwMQWWyymqUPZlGLidrBzWCWdRtnlOr0=,tag:Ts1P68sbBrYAS9gdI8XaHw==,type:str]
|
||||
CONSOLE_SECRET_KEY: ENC[AES256_GCM,data:WQKvfXroupl4R2oUuRXDSgLthsd/ZaEedsy8lAHAu40=,iv:GXBhHQ8YiaVjRRMw2yG4FxdulVOuOurkLXXC0u/SFRA=,tag:kyxRviddcYUc0DcPEphKIg==,type:str]
|
||||
sops:
|
||||
age:
|
||||
- enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBNZCtYcC9kanVtdFNSeksr
|
||||
NDhXVHBpWjhKaWJSVnR1Rk9QV3BVU25mRldjCitEOVNCNnZVVjNLQWhFejdGelJV
|
||||
V3Bhb3JTMU9xdEhOck00TGt4UFVaWTAKLS0tIEZ1VnFrVmkyTWZWR2hCc2hpWGVK
|
||||
Z0NuYjVIdXJYRlVkY3JSUWUzZzd1bkkK0jm57duPu+u2qwjhKaO/fTtH1+Gt8fSm
|
||||
vnLQsQZlkYZBYxHHwtBPC4JztYvmuKnPNYAPqRPfVB+9IEOpQEetkA==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
recipient: age1smu533f803gmd0jq60s2zaj9zlznajy0ca6rtewd4r37mr2hs3uqsrldfh
|
||||
lastmodified: "2026-07-20T19:41:57Z"
|
||||
mac: ENC[AES256_GCM,data:fFoW//RJnlx0jQ3aMsEtx/qy70LrnocVc4UfPUIFNJG377NG7GnDxs+JIQGXSNNzRoKH06U3AFmBl2BvIi6D67A1VbOS0/+zmSGcNxdmL3apauOmn34jgJ/ipyakuawUi1+CGgSy8eMqgLM9QCpBkGx6qTtjjnGVWHYbN1WG3iE=,iv:8MD02ysiRL3nxSQvhQ6e0ek9gVLHwcfUCGzX+Tmtdp4=,tag:X/SdhJes1rJLHy6S278ztg==,type:str]
|
||||
unencrypted_suffix: _unencrypted
|
||||
version: 3.13.2
|
||||
@@ -0,0 +1,110 @@
|
||||
apiVersion: minio.min.io/v2
|
||||
kind: Tenant
|
||||
metadata:
|
||||
name: minio-cluster
|
||||
namespace: storage
|
||||
labels:
|
||||
app: minio
|
||||
annotations:
|
||||
# Let the operator own bucket/user provisioning declaratively.
|
||||
prometheus.io/path: /minio/v2/metrics/cluster
|
||||
prometheus.io/port: "9000"
|
||||
prometheus.io/scrape: "true"
|
||||
spec:
|
||||
image: minio/minio:RELEASE.2025-07-23T15-54-02Z
|
||||
|
||||
# Disable operator auto-TLS: MinIO serves plain HTTP internally on 9000.
|
||||
# External TLS is terminated at nginx ingress (wildcard riotpiao-com-tls cert).
|
||||
# Without this, MinIO auto-generates self-signed certs and serves HTTPS-only
|
||||
# on 9000, which breaks plain-HTTP internal clients like Vault's S3 backend
|
||||
# (they hang waiting for a TLS handshake that never completes on an HTTP request).
|
||||
requestAutoCert: false
|
||||
|
||||
# Root credentials. v5 pods read `configuration` — a Secret whose `config.env`
|
||||
# key holds shell `export MINIO_ROOT_USER=...` lines. Created out-of-band
|
||||
# (SOPS), see minio-secrets.enc.yaml. NOTE: the operator health-monitor logs a
|
||||
# cosmetic "empty tenant credentials" warning (it greps for legacy
|
||||
# access_key/secret_key keys) — MinIO itself authenticates fine; ignore it.
|
||||
configuration:
|
||||
name: minio-creds
|
||||
|
||||
# ── Single pool on the sole storage/scheduling node (talos-cp-1, az-a) ──────
|
||||
# Per the 3-CP topology only talos-cp-1 is schedulable and holds Longhorn, so
|
||||
# MinIO is a single-server tenant. 4 volumes give erasure-coded durability
|
||||
# (MinIO's minimum for parity) on that one node.
|
||||
pools:
|
||||
- name: az-a
|
||||
servers: 1
|
||||
volumesPerServer: 4
|
||||
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: topology.kubernetes.io/zone
|
||||
operator: In
|
||||
values: [az-a]
|
||||
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
|
||||
volumeClaimTemplate:
|
||||
metadata:
|
||||
name: data
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: longhorn
|
||||
resources:
|
||||
requests:
|
||||
storage: 25Gi
|
||||
|
||||
# ── Declarative buckets (operator creates on first boot) ────────────────────
|
||||
buckets:
|
||||
- name: loki-chunks
|
||||
- name: loki-ruler
|
||||
- name: loki-admin
|
||||
- name: vault
|
||||
|
||||
# Metrics are exposed at /minio/v2/metrics; scrape via a hand-rolled
|
||||
# ServiceMonitor in the monitoring stack rather than operator auto-wiring
|
||||
# (prometheusOperator:true makes the operator hunt for Prometheus in ns
|
||||
# 'default' and fail the reconcile).
|
||||
|
||||
# Public hostnames the tenant serves (S3 + console via the cluster ingress).
|
||||
features:
|
||||
domains:
|
||||
minio:
|
||||
- https://minio.riotpiao.com
|
||||
console: https://minio-console.riotpiao.com
|
||||
|
||||
# ── OIDC via Authentik (server-side env, valid in v2 schema) ────────────────
|
||||
env:
|
||||
- name: MINIO_IDENTITY_OPENID_CONFIG_URL
|
||||
value: "https://authentik.riotpiao.com/application/o/minio/.well-known/openid-configuration"
|
||||
- name: MINIO_IDENTITY_OPENID_CLIENT_ID
|
||||
value: "minio"
|
||||
- name: MINIO_IDENTITY_OPENID_CLIENT_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: minio-oidc
|
||||
key: MINIO_IDENTITY_OPENID_CLIENT_SECRET
|
||||
- name: MINIO_IDENTITY_OPENID_CLAIM_NAME
|
||||
value: "policy"
|
||||
- name: MINIO_IDENTITY_OPENID_REDIRECT_URI
|
||||
value: "https://minio.riotpiao.com/oauth_callback"
|
||||
- name: MINIO_IDENTITY_OPENID_DISPLAY_NAME
|
||||
value: "Authentik"
|
||||
- name: MINIO_IDENTITY_OPENID_SCOPES
|
||||
value: "openid,profile,email,minio"
|
||||
@@ -0,0 +1,34 @@
|
||||
# k8s/monitoring/ingress-alerts.yaml
|
||||
# ingress-nginx's chart has no built-in PrometheusRule block, so these rules are
|
||||
# a standalone CRD instance. Applied via the prometheus release's postsync hook
|
||||
# (after the operator/CRDs are confirmed up) — see helmfile.yaml.gotmpl.
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: ingress-nginx-rules
|
||||
namespace: ingress-nginx
|
||||
spec:
|
||||
groups:
|
||||
- name: ingress-nginx.rules
|
||||
rules:
|
||||
- alert: IngressHighErrorRate
|
||||
expr: |
|
||||
sum(rate(nginx_ingress_controller_requests{status=~"5.."}[5m])) by (ingress)
|
||||
/ sum(rate(nginx_ingress_controller_requests[5m])) by (ingress) > 0.05
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High 5xx rate on {{ $labels.ingress }}"
|
||||
description: "More than 5% of requests to {{ $labels.ingress }} have returned 5xx for 10 minutes."
|
||||
- alert: IngressHighLatencyP95
|
||||
expr: |
|
||||
histogram_quantile(0.95,
|
||||
sum(rate(nginx_ingress_controller_request_duration_seconds_bucket[5m])) by (ingress, le)
|
||||
) > 1
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "p95 latency > 1s on {{ $labels.ingress }}"
|
||||
description: "95th percentile request latency for {{ $labels.ingress }} has exceeded 1s for 10 minutes."
|
||||
@@ -0,0 +1,33 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: argocd-rules
|
||||
namespace: argocd
|
||||
spec:
|
||||
groups:
|
||||
- name: argocd.rules
|
||||
interval: 15s
|
||||
rules:
|
||||
- alert: ArgoCDHighErrorRate
|
||||
expr: |
|
||||
(
|
||||
sum(rate(argocd_http_request_total{status=~"5.."}[5m]))
|
||||
/
|
||||
sum(rate(argocd_http_request_total[5m]))
|
||||
) > 0.05
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High error rate on Argo CD"
|
||||
description: "5xx error rate exceeded 5% for 10 minutes. Value: {{ $value | humanizePercentage }}"
|
||||
|
||||
- alert: ArgoCDApplicationSyncFailure
|
||||
expr: |
|
||||
argocd_app_health_degraded_total > 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Argo CD applications in degraded state"
|
||||
description: "{{ $value | humanize }} application(s) have been degraded for 10 minutes"
|
||||
@@ -0,0 +1,33 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: authentik-rules
|
||||
namespace: iam
|
||||
spec:
|
||||
groups:
|
||||
- name: authentik.rules
|
||||
interval: 15s
|
||||
rules:
|
||||
- alert: AuthentikHighErrorRate
|
||||
expr: |
|
||||
(
|
||||
sum(rate(authentik_http_requests_total{status=~"5.."}[5m]))
|
||||
/
|
||||
sum(rate(authentik_http_requests_total[5m]))
|
||||
) > 0.05
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High error rate on Authentik"
|
||||
description: "5xx error rate exceeded 5% of total requests for 10 minutes. Value: {{ $value | humanizePercentage }}"
|
||||
|
||||
- alert: AuthentikOutpostDown
|
||||
expr: |
|
||||
authentik_outpost_total_up == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Authentik outpost is down"
|
||||
description: "Outpost {{ $labels.outpost_name }} (type: {{ $labels.outpost_type }}) has been offline for 5 minutes"
|
||||
@@ -0,0 +1,23 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: forgejo-rules
|
||||
namespace: forgejo
|
||||
spec:
|
||||
groups:
|
||||
- name: forgejo.rules
|
||||
interval: 15s
|
||||
rules:
|
||||
- alert: ForgejoHighErrorRate
|
||||
expr: |
|
||||
(
|
||||
sum(rate(forgejo_http_request_total{status=~"5.."}[5m]))
|
||||
/
|
||||
sum(rate(forgejo_http_request_total[5m]))
|
||||
) > 0.05
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High error rate on Forgejo"
|
||||
description: "5xx error rate exceeded 5% for 10 minutes. Value: {{ $value | humanizePercentage }}"
|
||||
@@ -0,0 +1,23 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: grafana-rules
|
||||
namespace: logging
|
||||
spec:
|
||||
groups:
|
||||
- name: grafana.rules
|
||||
interval: 15s
|
||||
rules:
|
||||
- alert: GrafanaHighErrorRate
|
||||
expr: |
|
||||
(
|
||||
sum(rate(grafana_http_request_total{status=~"5.."}[5m]))
|
||||
/
|
||||
sum(rate(grafana_http_request_total[5m]))
|
||||
) > 0.05
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High error rate on Grafana"
|
||||
description: "5xx error rate exceeded 5% for 10 minutes. Value: {{ $value | humanizePercentage }}"
|
||||
@@ -0,0 +1,57 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: minio-rules
|
||||
namespace: storage
|
||||
spec:
|
||||
groups:
|
||||
- name: minio.rules
|
||||
interval: 15s
|
||||
rules:
|
||||
- alert: MinIOHighErrorRate
|
||||
expr: |
|
||||
(
|
||||
sum(rate(minio_s3_requests_total{error="true"}[5m]))
|
||||
/
|
||||
sum(rate(minio_s3_requests_total[5m]))
|
||||
) > 0.05
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High error rate on MinIO"
|
||||
description: "S3 request error rate exceeded 5% for 10 minutes. Value: {{ $value | humanizePercentage }}"
|
||||
|
||||
- alert: MinIODiskSpaceLow
|
||||
expr: |
|
||||
(
|
||||
minio_cluster_capacity_usable_bytes
|
||||
/
|
||||
minio_cluster_capacity_raw_total_bytes
|
||||
) < 0.1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "MinIO disk space critically low"
|
||||
description: "Usable capacity < 10% of raw capacity. Free space: {{ $value | humanizePercentage }}"
|
||||
|
||||
- alert: MinIOReplicationLag
|
||||
expr: |
|
||||
minio_replication_metrics_replicating_byte_count > 1073741824
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "MinIO replication lag detected"
|
||||
description: "Bytes pending replication > 1GB for 15 minutes. Value: {{ $value | humanize1024 }}B"
|
||||
|
||||
- alert: MinIODriveOffline
|
||||
expr: |
|
||||
minio_cluster_health_drives_offline > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "MinIO drive offline"
|
||||
description: "{{ $value | humanize }} drive(s) offline in MinIO cluster"
|
||||
@@ -0,0 +1,33 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: vault-rules
|
||||
namespace: storage
|
||||
spec:
|
||||
groups:
|
||||
- name: vault.rules
|
||||
interval: 15s
|
||||
rules:
|
||||
- alert: VaultHighErrorRate
|
||||
expr: |
|
||||
(
|
||||
sum(rate(vault_core_handle_request_total{error="true"}[5m]))
|
||||
/
|
||||
sum(rate(vault_core_handle_request_total[5m]))
|
||||
) > 0.05
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High error rate on Vault"
|
||||
description: "Error rate exceeded 5% for 10 minutes. Value: {{ $value | humanizePercentage }}"
|
||||
|
||||
- alert: VaultSealed
|
||||
expr: |
|
||||
vault_core_unsealed == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Vault is sealed"
|
||||
description: "Vault has been sealed for 1 minute. Immediate attention required."
|
||||
@@ -0,0 +1,75 @@
|
||||
# k8s/monitoring/blackbox-exporter-values.yaml
|
||||
# Active black-box HTTP probing of every ingress-exposed service — gives an
|
||||
# uptime/availability signal independent of real traffic. Homelab services
|
||||
# like Vault/MinIO/Longhorn UI get almost no organic requests, so ingress RED
|
||||
# metrics alone can't tell "idle" from "down"; this closes that gap.
|
||||
|
||||
config:
|
||||
modules:
|
||||
http_2xx:
|
||||
prober: http
|
||||
timeout: 5s
|
||||
http:
|
||||
valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
|
||||
valid_status_codes: [] # any 2xx
|
||||
follow_redirects: true
|
||||
preferred_ip_protocol: "ip4"
|
||||
tls_config:
|
||||
insecure_skip_verify: true # homelab-ca is a private CA; skip verify for simplicity
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 20m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
cpu: 100m
|
||||
memory: 64Mi
|
||||
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
|
||||
serviceMonitor:
|
||||
enabled: true
|
||||
selfMonitor:
|
||||
enabled: true
|
||||
defaults:
|
||||
interval: 30s
|
||||
scrapeTimeout: 10s
|
||||
module: http_2xx
|
||||
targets:
|
||||
- name: grafana
|
||||
url: https://grafana.riotpiao.com/
|
||||
- name: loki
|
||||
url: https://loki.riotpiao.com/ready
|
||||
- name: authentik
|
||||
url: https://authentik.riotpiao.com/-/health/ready/
|
||||
- name: vault
|
||||
url: https://vault.riotpiao.com/v1/sys/health
|
||||
- name: minio-console
|
||||
url: https://minio.riotpiao.com/
|
||||
- name: minio-api
|
||||
url: https://minio-api.riotpiao.com/minio/health/live
|
||||
- name: prometheus
|
||||
url: https://prometheus.riotpiao.com/-/healthy
|
||||
- name: portainer
|
||||
url: https://portainer.riotpiao.com/
|
||||
- name: forgejo
|
||||
url: https://forgejo.riotpiao.com/api/healthz
|
||||
- name: argocd
|
||||
url: https://argocd.riotpiao.com/healthz
|
||||
- name: longhorn
|
||||
url: https://longhorn.riotpiao.com/
|
||||
|
||||
prometheusRule:
|
||||
enabled: true
|
||||
rules:
|
||||
- alert: ServiceProbeDown
|
||||
expr: probe_success == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Probe failing for {{ $labels.instance }}"
|
||||
description: "Blackbox probe for {{ $labels.instance }} has failed for more than 5 minutes."
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,55 @@
|
||||
# k8s/monitoring/dashboards/control-plane-logs.yaml
|
||||
# Surfaces controller/control-plane logs that are already in Loki today
|
||||
# (Promtail scrapes every namespace with no filter) — this dashboard is the
|
||||
# "make it visible" piece, not new log collection.
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: control-plane-logs-dashboard
|
||||
namespace: logging
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
data:
|
||||
control-plane-logs.json: |
|
||||
{
|
||||
"title": "Cluster Control Plane & Controllers (Logs)",
|
||||
"uid": "control-plane-logs",
|
||||
"schemaVersion": 39,
|
||||
"timezone": "browser",
|
||||
"time": { "from": "now-1h", "to": "now" },
|
||||
"refresh": "30s",
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "Error rate by namespace",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 6, "w": 24, "x": 0, "y": 0 },
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (namespace) (count_over_time({namespace=~\"kube-system|cert-manager|ingress-nginx|longhorn-system\"} |= \"error\" [5m]))"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "Control plane (kube-apiserver, controller-manager, scheduler)",
|
||||
"type": "logs",
|
||||
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 6 },
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"targets": [
|
||||
{ "expr": "{namespace=\"kube-system\"}" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "Cluster add-ons (cert-manager, ingress-nginx, longhorn)",
|
||||
"type": "logs",
|
||||
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 16 },
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"targets": [
|
||||
{ "expr": "{namespace=~\"cert-manager|ingress-nginx|longhorn-system\"}" }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
# k8s/monitoring/dashboards/hardware-overview.yaml
|
||||
# Trimmed operator at-a-glance view across all nodes — node-exporter already
|
||||
# powers the deep-dive "Node Exporter Full" (#1860, see grafana-values.yaml),
|
||||
# this is the quick health-check version, not a replacement for it.
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: hardware-overview-dashboard
|
||||
namespace: logging
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
data:
|
||||
hardware-overview.json: |
|
||||
{
|
||||
"title": "Hardware Statistics (Operator Overview)",
|
||||
"uid": "hardware-overview",
|
||||
"schemaVersion": 39,
|
||||
"timezone": "browser",
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"refresh": "30s",
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "Nodes up / down",
|
||||
"type": "stat",
|
||||
"gridPos": { "h": 5, "w": 24, "x": 0, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"mappings": [
|
||||
{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" } } },
|
||||
{ "type": "value", "options": { "1": { "text": "UP", "color": "green" } } }
|
||||
]
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{ "expr": "up{job=~\".*node-exporter.*\"}", "legendFormat": "{{instance}}" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "CPU usage % by node",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 5 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) by (instance)) * 100",
|
||||
"legendFormat": "{{instance}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "Memory usage % by node",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 5 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100",
|
||||
"legendFormat": "{{instance}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "Root filesystem usage % by node",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(1 - node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"}) * 100",
|
||||
"legendFormat": "{{instance}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "Root filesystem space remaining",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 13 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "bytes" } },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "node_filesystem_avail_bytes{mountpoint=\"/\"}",
|
||||
"legendFormat": "{{instance}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "Network errors/drops by node",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 21 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "expr": "rate(node_network_receive_errs_total[5m])", "legendFormat": "{{instance}} rx errs" },
|
||||
{ "expr": "rate(node_network_transmit_errs_total[5m])", "legendFormat": "{{instance}} tx errs" },
|
||||
{ "expr": "rate(node_network_receive_drop_total[5m])", "legendFormat": "{{instance}} rx drops" },
|
||||
{ "expr": "rate(node_network_transmit_drop_total[5m])", "legendFormat": "{{instance}} tx drops" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "Load average (1m / 5m) by node",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 21 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "expr": "node_load1", "legendFormat": "{{instance}} load1" },
|
||||
{ "expr": "node_load5", "legendFormat": "{{instance}} load5" }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
# k8s/monitoring/dashboards/kube-controller-health.yaml
|
||||
# Talos binds controller-manager/scheduler/etcd to 127.0.0.1, so Prometheus
|
||||
# can't scrape them directly (see prometheus-values.yaml). kube-apiserver is
|
||||
# the one control-plane component that's still reachable (its ServiceMonitor
|
||||
# targets the in-cluster `kubernetes` service, not localhost) — paired with
|
||||
# kube-state-metrics signals as a proxy for controller/scheduler health.
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: kube-controller-health-dashboard
|
||||
namespace: logging
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
data:
|
||||
kube-controller-health.json: |
|
||||
{
|
||||
"title": "Kube-Controller Health",
|
||||
"uid": "kube-controller-health",
|
||||
"schemaVersion": 39,
|
||||
"timezone": "browser",
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"refresh": "30s",
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "API server — up",
|
||||
"type": "stat",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"mappings": [
|
||||
{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" } } },
|
||||
{ "type": "value", "options": { "1": { "text": "UP", "color": "green" } } }
|
||||
]
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{ "expr": "min(up{job=\"apiserver\"})", "legendFormat": "apiserver" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "API server — request rate by verb/code",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 18, "x": 6, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(apiserver_request_total[5m])) by (verb, code)",
|
||||
"legendFormat": "{{verb}} {{code}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "API server — error rate % (5xx)",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "percent" } },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(apiserver_request_total{code=~\"5..\"}[5m])) / sum(rate(apiserver_request_total[5m])) * 100",
|
||||
"legendFormat": "5xx %"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "API server — latency p95 / p99",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "s" } },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))",
|
||||
"legendFormat": "p99"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "Pods stuck Pending",
|
||||
"type": "stat",
|
||||
"gridPos": { "h": 5, "w": 8, "x": 0, "y": 16 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "value": 0, "color": "green" },
|
||||
{ "value": 1, "color": "yellow" },
|
||||
{ "value": 5, "color": "red" }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{ "expr": "sum(kube_pod_status_phase{phase=\"Pending\"}) OR on() vector(0)", "legendFormat": "pending" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "CrashLoopBackOff containers",
|
||||
"type": "stat",
|
||||
"gridPos": { "h": 5, "w": 8, "x": 8, "y": 16 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "value": 0, "color": "green" },
|
||||
{ "value": 1, "color": "red" }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{ "expr": "sum(kube_pod_container_status_waiting_reason{reason=\"CrashLoopBackOff\"}) OR on() vector(0)", "legendFormat": "crashlooping" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "Nodes NotReady",
|
||||
"type": "stat",
|
||||
"gridPos": { "h": 5, "w": 8, "x": 16, "y": 16 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "value": 0, "color": "green" },
|
||||
{ "value": 1, "color": "red" }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{ "expr": "count(kube_node_status_condition{condition=\"Ready\", status=\"true\"} == 0) OR on() vector(0)", "legendFormat": "not ready" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"title": "Failed Jobs",
|
||||
"type": "table",
|
||||
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "expr": "kube_job_status_failed > 0", "format": "table", "instant": true }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"title": "Deployments with unavailable replicas",
|
||||
"type": "table",
|
||||
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "expr": "kube_deployment_status_replicas_unavailable > 0", "format": "table", "instant": true }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"title": "Container restart rate by pod",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 28 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(kube_pod_container_status_restarts_total[15m])) by (namespace, pod)",
|
||||
"legendFormat": "{{namespace}}/{{pod}}"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
# k8s/monitoring/dashboards/service-availability.yaml
|
||||
# Active uptime/availability from blackbox-exporter probes — the signal that
|
||||
# covers low-traffic services (Vault, MinIO, Longhorn UI) where RED metrics
|
||||
# alone can't distinguish "idle" from "down".
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: service-availability-dashboard
|
||||
namespace: logging
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
data:
|
||||
service-availability.json: |
|
||||
{
|
||||
"title": "Service Availability & Certificate Expiration",
|
||||
"uid": "svc-availability",
|
||||
"schemaVersion": 39,
|
||||
"timezone": "browser",
|
||||
"time": { "from": "now-24h", "to": "now" },
|
||||
"refresh": "30s",
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "Up / Down — all probed services",
|
||||
"type": "stat",
|
||||
"gridPos": { "h": 6, "w": 24, "x": 0, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"mappings": [
|
||||
{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" } } },
|
||||
{ "type": "value", "options": { "1": { "text": "UP", "color": "green" } } }
|
||||
],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "value": 0, "color": "red" },
|
||||
{ "value": 1, "color": "green" }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{ "expr": "probe_success", "legendFormat": "{{instance}}" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "Uptime % trend",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "avg_over_time(probe_success[$__rate_interval]) * 100",
|
||||
"legendFormat": "{{instance}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "Probe latency",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "s" } },
|
||||
"targets": [
|
||||
{ "expr": "probe_duration_seconds", "legendFormat": "{{instance}}" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "7-day SLO (% successful probes)",
|
||||
"type": "table",
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 14 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "value": 0, "color": "red" },
|
||||
{ "value": 99, "color": "yellow" },
|
||||
{ "value": 99.9, "color": "green" }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "avg_over_time(probe_success[7d]) * 100",
|
||||
"format": "table",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "Services DOWN right now",
|
||||
"type": "stat",
|
||||
"gridPos": { "h": 4, "w": 12, "x": 0, "y": 22 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "value": 0, "color": "green" },
|
||||
{ "value": 1, "color": "red" }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{ "expr": "count(probe_success == 0) OR on() vector(0)", "legendFormat": "down" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "Certs expiring in < 14 days",
|
||||
"type": "stat",
|
||||
"gridPos": { "h": 4, "w": 12, "x": 12, "y": 22 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "value": 0, "color": "green" },
|
||||
{ "value": 1, "color": "red" }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "count((certmanager_certificate_expiration_timestamp_seconds - time()) / 86400 < 14) OR on() vector(0)",
|
||||
"legendFormat": "expiring"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "Certificate expiry — days remaining",
|
||||
"type": "table",
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 26 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "value": 0, "color": "red" },
|
||||
{ "value": 14, "color": "yellow" },
|
||||
{ "value": 30, "color": "green" }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(certmanager_certificate_expiration_timestamp_seconds - time()) / 86400",
|
||||
"legendFormat": "{{name}}",
|
||||
"format": "table",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
# k8s/monitoring/dashboards/service-golden-signals.yaml
|
||||
# RED metrics (rate/errors/duration) for every service fronted by ingress-nginx.
|
||||
# Picked up automatically by Grafana's sidecar (grafana_dashboard=1 label) — see
|
||||
# sidecar.dashboards in k8s/logging/grafana-values.yaml.
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: service-golden-signals-dashboard
|
||||
namespace: logging
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
data:
|
||||
service-golden-signals.json: |
|
||||
{
|
||||
"title": "Latency & Golden Signals (Ingress RED)",
|
||||
"uid": "svc-golden-signals",
|
||||
"schemaVersion": 39,
|
||||
"timezone": "browser",
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"refresh": "30s",
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "ingress",
|
||||
"type": "query",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"query": "label_values(nginx_ingress_controller_requests, ingress)",
|
||||
"refresh": 2,
|
||||
"includeAll": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "Request rate by status — $ingress",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\"}[5m])) by (status)",
|
||||
"legendFormat": "{{status}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "Error rate % (4xx / 5xx) — $ingress",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "percent" } },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\", status=~\"5..\"}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\"}[5m])) * 100",
|
||||
"legendFormat": "5xx"
|
||||
},
|
||||
{
|
||||
"expr": "sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\", status=~\"4..\"}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\"}[5m])) * 100",
|
||||
"legendFormat": "4xx"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "Latency p50 / p95 / p99 — $ingress",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "s" } },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"$ingress\"}[5m])) by (le))",
|
||||
"legendFormat": "p50"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"$ingress\"}[5m])) by (le))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"$ingress\"}[5m])) by (le))",
|
||||
"legendFormat": "p99"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "All services — traffic overview",
|
||||
"type": "table",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "topk(11, sum(rate(nginx_ingress_controller_requests[5m])) by (ingress))",
|
||||
"format": "table",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "Customer-facing failures (5xx count, window total)",
|
||||
"type": "stat",
|
||||
"gridPos": { "h": 5, "w": 12, "x": 0, "y": 16 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "value": 0, "color": "green" },
|
||||
{ "value": 1, "color": "yellow" },
|
||||
{ "value": 50, "color": "red" }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(increase(nginx_ingress_controller_requests{status=~\"5..\"}[$__range])) OR on() vector(0)",
|
||||
"legendFormat": "5xx total"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "Top 5 error-contributing services",
|
||||
"type": "table",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "topk(5, sum(rate(nginx_ingress_controller_requests{status=~\"5..\"}[5m])) by (ingress))",
|
||||
"format": "table",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
# k8s/monitoring/dashboards/service-internals.yaml
|
||||
# Native per-service metrics — the "why" layer behind the ingress RED/uptime
|
||||
# dashboards (e.g. ingress shows MinIO is slow; this shows disk offline).
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: service-internals-dashboard
|
||||
namespace: logging
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
data:
|
||||
service-internals.json: |
|
||||
{
|
||||
"title": "Service Internals (MinIO / Forgejo / Argo CD / cert-manager / Vault / Longhorn)",
|
||||
"uid": "svc-internals",
|
||||
"schemaVersion": 39,
|
||||
"timezone": "browser",
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"refresh": "30s",
|
||||
"panels": [
|
||||
{ "id": 1, "title": "MinIO — disk/node offline", "type": "timeseries",
|
||||
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "expr": "minio_cluster_disk_offline_total", "legendFormat": "disks offline" },
|
||||
{ "expr": "minio_cluster_nodes_offline_total", "legendFormat": "nodes offline" }
|
||||
]
|
||||
},
|
||||
{ "id": 2, "title": "MinIO — S3 request errors", "type": "timeseries",
|
||||
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "expr": "sum(rate(minio_s3_requests_errors_total[5m])) by (api)", "legendFormat": "{{api}}" }
|
||||
]
|
||||
},
|
||||
{ "id": 3, "title": "MinIO — S3 TTFB latency", "type": "timeseries",
|
||||
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 6 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "s" } },
|
||||
"targets": [
|
||||
{ "expr": "minio_s3_time_ttfb_seconds_distribution", "legendFormat": "{{api}}" }
|
||||
]
|
||||
},
|
||||
{ "id": 4, "title": "Forgejo — repos / orgs", "type": "stat",
|
||||
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 6 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "expr": "gitea_repositories", "legendFormat": "repos" },
|
||||
{ "expr": "gitea_organizations", "legendFormat": "orgs" }
|
||||
]
|
||||
},
|
||||
{ "id": 5, "title": "Forgejo — process health (CPU/mem)", "type": "timeseries",
|
||||
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 12 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "expr": "rate(process_cpu_seconds_total{job=~\".*forgejo.*|.*gitea.*\"}[5m])", "legendFormat": "cpu" },
|
||||
{ "expr": "process_resident_memory_bytes{job=~\".*forgejo.*|.*gitea.*\"}", "legendFormat": "mem" }
|
||||
]
|
||||
},
|
||||
{ "id": 6, "title": "Argo CD — app sync/health status", "type": "table",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "expr": "argocd_app_info", "format": "table", "instant": true }
|
||||
]
|
||||
},
|
||||
{ "id": 7, "title": "cert-manager — days to cert expiry", "type": "stat",
|
||||
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 18 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "value": 0, "color": "red" },
|
||||
{ "value": 14, "color": "yellow" },
|
||||
{ "value": 30, "color": "green" }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{ "expr": "(certmanager_certificate_expiration_timestamp_seconds - time()) / 86400", "legendFormat": "{{name}}" }
|
||||
]
|
||||
},
|
||||
{ "id": 8, "title": "Vault — sealed/unsealed", "type": "stat",
|
||||
"gridPos": { "h": 6, "w": 6, "x": 12, "y": 20 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"mappings": [
|
||||
{ "type": "value", "options": { "0": { "text": "SEALED", "color": "red" } } },
|
||||
{ "type": "value", "options": { "1": { "text": "UNSEALED", "color": "green" } } }
|
||||
]
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{ "expr": "vault_core_unsealed", "legendFormat": "vault" }
|
||||
]
|
||||
},
|
||||
{ "id": 9, "title": "Longhorn — volume robustness", "type": "table",
|
||||
"gridPos": { "h": 6, "w": 6, "x": 18, "y": 20 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "expr": "longhorn_volume_robustness", "format": "table", "instant": true }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: svc-argocd-dashboard
|
||||
namespace: logging
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
annotations:
|
||||
grafana_folder: "Argo CD"
|
||||
data:
|
||||
svc-argocd.json: |
|
||||
{"title":"Argo CD — Service Overview","uid":"svc-argocd","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"}}},"targets":[{"expr":"min(up{job=~\"argocd-.*\"})"}]},{"id":3,"title":"HTTP requests","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_http_request_total[5m])) by (status)","legendFormat":"{{status}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_http_request_total{status=~\"5..\"}[5m])) / sum(rate(argocd_http_request_total[5m])) * 100"}]},{"id":5,"title":"Request latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(argocd_http_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"argocd\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"argocd\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restarts","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"argocd\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Applications & Sync","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Applications","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"argocd_app_total"}]},{"id":22,"title":"Sync by status","type":"timeseries","gridPos":{"h":6,"w":9,"x":6,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_app_sync_total[5m])) by (sync_status)","legendFormat":"{{sync_status}}"}]},{"id":23,"title":"Degraded apps","type":"stat","gridPos":{"h":6,"w":6,"x":15,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"argocd_app_health_degraded_total"}]},{"id":24,"title":"Git sync ops","type":"timeseries","gridPos":{"h":6,"w":12,"x":0,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_git_sync_total[5m])) by (git_operation,git_status)","legendFormat":"{{git_operation}}/{{git_status}}"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"argocd\"}"}]}]}]}
|
||||
@@ -0,0 +1,143 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: svc-authentik-dashboard
|
||||
namespace: logging
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
annotations:
|
||||
grafana_folder: "Authentik"
|
||||
data:
|
||||
svc-authentik.json: |
|
||||
{
|
||||
"title": "Authentik — Service Overview",
|
||||
"uid": "svc-authentik",
|
||||
"schemaVersion": 39,
|
||||
"timezone": "browser",
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"refresh": "30s",
|
||||
"panels": [
|
||||
{
|
||||
"id": 1, "title": "Row: Availability & Golden Signals", "type": "row",
|
||||
"collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||
"panels": [
|
||||
{
|
||||
"id": 2, "title": "Up", "type": "stat",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "thresholds" },
|
||||
"mappings": [
|
||||
{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }
|
||||
],
|
||||
"thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" }, { "value": 1, "color": "green" } ] }
|
||||
}
|
||||
},
|
||||
"targets": [{ "expr": "min(up{job=\"authentik-server\"})" }]
|
||||
},
|
||||
{
|
||||
"id": 3, "title": "HTTP request rate by status", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 9, "x": 6, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "sum(rate(authentik_flows_execution_stage_time_count[5m])) by (flow_slug)", "legendFormat": "{{flow_slug}}" }]
|
||||
},
|
||||
{
|
||||
"id": 4, "title": "Error rate % (5xx)", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 9, "x": 15, "y": 1 },
|
||||
"fieldConfig": { "defaults": { "unit": "percent" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "(1 - (authentik_flows_cached / authentik_flows_execution_stage_time_count)) * 100" }]
|
||||
},
|
||||
{
|
||||
"id": 5, "title": "Request duration p50/p95/p99", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
|
||||
"fieldConfig": { "defaults": { "unit": "s" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "expr": "histogram_quantile(0.50, sum(rate(authentik_main_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p50" },
|
||||
{ "expr": "histogram_quantile(0.95, sum(rate(authentik_main_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p95" },
|
||||
{ "expr": "histogram_quantile(0.99, sum(rate(authentik_main_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p99" }
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10, "title": "Row: Resource Usage", "type": "row",
|
||||
"collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 },
|
||||
"panels": [
|
||||
{
|
||||
"id": 11, "title": "CPU by pod", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 2 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"iam\",pod=~\"authentik.*\"}[5m])) by (pod)", "legendFormat": "{{pod}}" }]
|
||||
},
|
||||
{
|
||||
"id": 12, "title": "Memory by pod", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 2 },
|
||||
"fieldConfig": { "defaults": { "unit": "bytes" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "sum(container_memory_working_set_bytes{namespace=\"iam\",pod=~\"authentik.*\"}) by (pod)", "legendFormat": "{{pod}}" }]
|
||||
},
|
||||
{
|
||||
"id": 13, "title": "Restart rate by pod", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 2 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "sum(rate(kube_pod_container_status_restarts_total{namespace=\"iam\",pod=~\"authentik.*\"}[15m])) by (pod)", "legendFormat": "{{pod}}" }]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 20, "title": "Row: Identity Provider (OIDC / OAuth2)", "type": "row",
|
||||
"collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 },
|
||||
"panels": [
|
||||
{
|
||||
"id": 21, "title": "Outpost connections", "type": "stat",
|
||||
"gridPos": { "h": 7, "w": 6, "x": 0, "y": 3 },
|
||||
"fieldConfig": { "defaults": { "unit": "short" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "authentik_outposts_connected" }]
|
||||
},
|
||||
{
|
||||
"id": 22, "title": "Flows cached", "type": "stat",
|
||||
"gridPos": { "h": 7, "w": 6, "x": 6, "y": 3 },
|
||||
"fieldConfig": { "defaults": { "unit": "short" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "authentik_flows_cached" }]
|
||||
},
|
||||
{
|
||||
"id": 23, "title": "Policies cached", "type": "stat",
|
||||
"gridPos": { "h": 7, "w": 6, "x": 12, "y": 3 },
|
||||
"fieldConfig": { "defaults": { "unit": "short" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "authentik_policies_cached" }]
|
||||
},
|
||||
{
|
||||
"id": 24, "title": "Queued tasks", "type": "stat",
|
||||
"gridPos": { "h": 7, "w": 6, "x": 18, "y": 3 },
|
||||
"fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "unit": "short" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "authentik_tasks_queued" }]
|
||||
},
|
||||
{
|
||||
"id": 25, "title": "Admin workers", "type": "timeseries",
|
||||
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 10 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "authentik_admin_workers" }]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 30, "title": "Row: Logs", "type": "row",
|
||||
"collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 },
|
||||
"panels": [
|
||||
{
|
||||
"id": 31, "title": "Recent logs", "type": "logs",
|
||||
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 4 },
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"targets": [{ "expr": "{namespace=\"iam\"}" }]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: svc-forgejo-dashboard
|
||||
namespace: logging
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
annotations:
|
||||
grafana_folder: "Forgejo"
|
||||
data:
|
||||
svc-forgejo.json: |
|
||||
{"title":"Forgejo — Service Overview","uid":"svc-forgejo","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"}}},"targets":[{"expr":"min(up{job=\"forgejo\"})"}]},{"id":3,"title":"HTTP requests by method","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_http_request_total[5m])) by (method)","legendFormat":"{{method}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_http_request_total{status=~\"5..\"}[5m])) / sum(rate(forgejo_http_request_total[5m])) * 100"}]},{"id":5,"title":"Request latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(forgejo_http_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"forgejo\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"forgejo\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restarts","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"forgejo\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Git Operations","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Repositories","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"forgejo_repositories_total"}]},{"id":22,"title":"Users","type":"stat","gridPos":{"h":6,"w":6,"x":6,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"forgejo_users_total"}]},{"id":23,"title":"Git ops rate","type":"timeseries","gridPos":{"h":6,"w":12,"x":12,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_git_operations_total[5m])) by (operation_type)","legendFormat":"{{operation_type}}"}]},{"id":24,"title":"Runner tasks","type":"timeseries","gridPos":{"h":6,"w":12,"x":0,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_runner_tasks_total[5m])) by (status)","legendFormat":"{{status}}"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"forgejo\"}"}]}]}]}
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: svc-grafana-dashboard
|
||||
namespace: logging
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
annotations:
|
||||
grafana_folder: "Grafana"
|
||||
data:
|
||||
svc-grafana.json: |
|
||||
{"title":"Grafana — Service Overview","uid":"svc-grafana","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}]}},"targets":[{"expr":"min(up{job=\"grafana\"})"}]},{"id":3,"title":"HTTP requests","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(grafana_http_request_total[5m])) by (status)","legendFormat":"{{status}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(grafana_http_request_total{status=~\"5..\"}[5m])) / sum(rate(grafana_http_request_total[5m])) * 100"}]},{"id":5,"title":"Request latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(grafana_http_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"logging\",pod=~\"grafana.*\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"logging\",pod=~\"grafana.*\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restarts","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"logging\",pod=~\"grafana.*\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Dashboards & Users","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Total dashboards","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"grafana_dashboard_total"}]},{"id":22,"title":"Total users","type":"stat","gridPos":{"h":6,"w":6,"x":6,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"grafana_user_total"}]},{"id":23,"title":"Total alerts","type":"stat","gridPos":{"h":6,"w":6,"x":12,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"grafana_alerts_total"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"logging\",container=\"grafana\"}"}]}]}]}
|
||||
@@ -0,0 +1,143 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: svc-minio-dashboard
|
||||
namespace: logging
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
annotations:
|
||||
grafana_folder: "MinIO"
|
||||
data:
|
||||
svc-minio.json: |
|
||||
{
|
||||
"title": "MinIO — Service Overview",
|
||||
"uid": "svc-minio",
|
||||
"schemaVersion": 39,
|
||||
"timezone": "browser",
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"refresh": "30s",
|
||||
"panels": [
|
||||
{
|
||||
"id": 1, "title": "Row: Availability & Golden Signals", "type": "row",
|
||||
"collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||
"panels": [
|
||||
{
|
||||
"id": 2, "title": "Up", "type": "stat",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "thresholds" },
|
||||
"mappings": [
|
||||
{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }
|
||||
],
|
||||
"thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" }, { "value": 1, "color": "green" } ] }
|
||||
}
|
||||
},
|
||||
"targets": [{ "expr": "min(up{job=\"minio\"})" }]
|
||||
},
|
||||
{
|
||||
"id": 3, "title": "S3 request rate by method", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 9, "x": 6, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "sum(rate(minio_s3_requests_total[5m])) by (method)", "legendFormat": "{{method}}" }]
|
||||
},
|
||||
{
|
||||
"id": 4, "title": "Error rate %", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 9, "x": 15, "y": 1 },
|
||||
"fieldConfig": { "defaults": { "unit": "percent" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "sum(rate(minio_s3_requests_total{error=\"true\"}[5m])) / sum(rate(minio_s3_requests_total[5m])) * 100" }]
|
||||
},
|
||||
{
|
||||
"id": 5, "title": "Request duration p50/p95/p99", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
|
||||
"fieldConfig": { "defaults": { "unit": "s" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "expr": "histogram_quantile(0.50, sum(rate(minio_s3_requests_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p50" },
|
||||
{ "expr": "histogram_quantile(0.95, sum(rate(minio_s3_requests_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p95" },
|
||||
{ "expr": "histogram_quantile(0.99, sum(rate(minio_s3_requests_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p99" }
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10, "title": "Row: Resource Usage", "type": "row",
|
||||
"collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 },
|
||||
"panels": [
|
||||
{
|
||||
"id": 11, "title": "CPU by pod", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 2 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"storage\",pod=~\"minio.*\"}[5m])) by (pod)", "legendFormat": "{{pod}}" }]
|
||||
},
|
||||
{
|
||||
"id": 12, "title": "Memory by pod", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 2 },
|
||||
"fieldConfig": { "defaults": { "unit": "bytes" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "sum(container_memory_working_set_bytes{namespace=\"storage\",pod=~\"minio.*\"}) by (pod)", "legendFormat": "{{pod}}" }]
|
||||
},
|
||||
{
|
||||
"id": 13, "title": "Restart rate by pod", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 2 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "sum(rate(kube_pod_container_status_restarts_total{namespace=\"storage\",pod=~\"minio.*\"}[15m])) by (pod)", "legendFormat": "{{pod}}" }]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 20, "title": "Row: Storage & Replication", "type": "row",
|
||||
"collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 },
|
||||
"panels": [
|
||||
{
|
||||
"id": 21, "title": "Usable vs Raw capacity", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 3 },
|
||||
"fieldConfig": { "defaults": { "unit": "bytes", "custom": { "lineWidth": 2 } } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "expr": "minio_cluster_capacity_usable_bytes", "legendFormat": "Usable" },
|
||||
{ "expr": "minio_cluster_capacity_raw_total_bytes", "legendFormat": "Raw Total" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 22, "title": "Drive health (online/offline)", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 3 },
|
||||
"fieldConfig": { "defaults": { "unit": "short" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "expr": "minio_cluster_health_drives_online", "legendFormat": "Online" },
|
||||
{ "expr": "minio_cluster_health_drives_offline", "legendFormat": "Offline" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 23, "title": "Replication lag (bytes pending)", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 11 },
|
||||
"fieldConfig": { "defaults": { "unit": "bytes" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "minio_replication_metrics_replicating_byte_count", "legendFormat": "Pending replication" }]
|
||||
},
|
||||
{
|
||||
"id": 24, "title": "Replication failures (bytes)", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 11 },
|
||||
"fieldConfig": { "defaults": { "unit": "bytes" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "minio_replication_metrics_failed_byte_count", "legendFormat": "Failed replication" }]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 30, "title": "Row: Logs", "type": "row",
|
||||
"collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 },
|
||||
"panels": [
|
||||
{
|
||||
"id": 31, "title": "Recent logs", "type": "logs",
|
||||
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 4 },
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"targets": [{ "expr": "{namespace=\"storage\"}" }]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: svc-vault-dashboard
|
||||
namespace: logging
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
annotations:
|
||||
grafana_folder: "Vault"
|
||||
data:
|
||||
svc-vault.json: |
|
||||
{"title":"Vault — Service Overview","uid":"svc-vault","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability & Golden Signals","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"red"},{"value":1,"color":"green"}]}}},"targets":[{"expr":"min(up{job=\"vault\"})"}]},{"id":3,"title":"Request rate by status","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(vault_core_handle_request_total[5m])) by (method)","legendFormat":"{{method}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(vault_core_handle_request_total{error=\"true\"}[5m])) / sum(rate(vault_core_handle_request_total[5m])) * 100"}]},{"id":5,"title":"Request duration p50/p95/p99","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.50, sum(rate(vault_core_handle_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95, sum(rate(vault_core_handle_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99, sum(rate(vault_core_handle_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p99"}]}]},{"id":10,"title":"Row: Resource Usage","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU by pod","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"iam\",pod=~\"vault.*\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory by pod","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"iam\",pod=~\"vault.*\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restart rate","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"iam\",pod=~\"vault.*\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Vault Seal State","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Sealed","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"UNSEALED","color":"green"},"1":{"text":"SEALED","color":"red"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]}}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_core_unsealed"}]},{"id":22,"title":"Active","type":"stat","gridPos":{"h":6,"w":6,"x":6,"y":3},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"INACTIVE","color":"red"},"1":{"text":"ACTIVE","color":"green"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"red"},{"value":1,"color":"green"}]}}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_core_active"}]},{"id":23,"title":"Replication (Primary)","type":"stat","gridPos":{"h":6,"w":6,"x":12,"y":3},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"SECONDARY","color":"orange"},"1":{"text":"PRIMARY","color":"green"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"orange"},{"value":1,"color":"green"}]}}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_core_replication_primary"}]},{"id":24,"title":"Active tokens","type":"stat","gridPos":{"h":6,"w":6,"x":18,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_token_total"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"iam\",container=\"vault\"}"}]}]}]}
|
||||
@@ -0,0 +1,30 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
# Single source for cluster monitoring config (alerts, servicemonitors, dashboards).
|
||||
# NO top-level `namespace:` transformer — PrometheusRules deliberately live in
|
||||
# per-app namespaces (forgejo, iam, argocd, storage, ...); a transformer would
|
||||
# wrongly rewrite them all to one ns.
|
||||
resources:
|
||||
- alerts/ingress-alerts.yaml
|
||||
- alerts/svc-argocd-rules.yaml
|
||||
- alerts/svc-authentik-rules.yaml
|
||||
- alerts/svc-forgejo-rules.yaml
|
||||
- alerts/svc-grafana-rules.yaml
|
||||
- alerts/svc-minio-rules.yaml
|
||||
- alerts/svc-vault-rules.yaml
|
||||
- servicemonitors/argocd.yaml
|
||||
- servicemonitors/authentik.yaml
|
||||
- servicemonitors/forgejo.yaml
|
||||
- servicemonitors/minio.yaml
|
||||
- dashboards/control-plane-logs.yaml
|
||||
- dashboards/hardware-overview.yaml
|
||||
- dashboards/kube-controller-health.yaml
|
||||
- dashboards/service-availability.yaml
|
||||
- dashboards/service-golden-signals.yaml
|
||||
- dashboards/service-internals.yaml
|
||||
- dashboards/svc-argocd.yaml
|
||||
- dashboards/svc-authentik.yaml
|
||||
- dashboards/svc-forgejo.yaml
|
||||
- dashboards/svc-grafana.yaml
|
||||
- dashboards/svc-minio.yaml
|
||||
- dashboards/svc-vault.yaml
|
||||
@@ -0,0 +1,140 @@
|
||||
# monitoring/prometheus-values.yaml
|
||||
# Chart: prometheus-community/kube-prometheus-stack
|
||||
# Release name: prometheus (affects all generated resource names)
|
||||
#
|
||||
# What this installs:
|
||||
# - Prometheus Operator (manages the Prometheus CRD)
|
||||
# - Prometheus StatefulSet (scrapes metrics, stores on Longhorn PVC)
|
||||
# - node-exporter DaemonSet (kernel CPU/RAM/disk/network per node)
|
||||
# - kube-state-metrics Deployment (K8s object state — pod resource requests, phases)
|
||||
#
|
||||
# What this deliberately omits:
|
||||
# - Grafana: already deployed in the logging namespace
|
||||
# - Alertmanager: enable later when you want Slack/PagerDuty routing
|
||||
# - kubeControllerManager / kubeScheduler / kubeEtcd: Talos only binds these
|
||||
# on 127.0.0.1 — the default ServiceMonitors can't reach them
|
||||
# - kubeProxy: removed cluster-wide; Cilium handles routing instead
|
||||
|
||||
# ── Grafana ───────────────────────────────────────────────────────────────────
|
||||
grafana:
|
||||
enabled: false
|
||||
|
||||
# ── Alertmanager ──────────────────────────────────────────────────────────────
|
||||
alertmanager:
|
||||
enabled: false
|
||||
|
||||
# ── Prometheus ────────────────────────────────────────────────────────────────
|
||||
prometheus:
|
||||
prometheusSpec:
|
||||
retention: 15d
|
||||
retentionSize: "18GB"
|
||||
|
||||
# scrapeTimeout MUST be <= scrapeInterval or the operator refuses to generate
|
||||
# the Prometheus config ("scrapeTimeout greater than scrapeInterval") and no
|
||||
# STS is created. Use a larger interval to keep the 60s timeout headroom.
|
||||
scrapeInterval: 60s
|
||||
scrapeTimeout: 60s
|
||||
evaluationInterval: 30s
|
||||
|
||||
# Pin to az-a (talos-cp-1) — sole Longhorn node; else the RWO PVC can't
|
||||
# attach on cp-2/cp-3 (CSINode lacks driver.longhorn.io).
|
||||
nodeSelector:
|
||||
topology.kubernetes.io/zone: az-a
|
||||
|
||||
# Persistent storage — metrics survive node reboots and pod restarts.
|
||||
# longhorn-wffc (WaitForFirstConsumer) binds after scheduling so the volume
|
||||
# lands on the node the pod runs on.
|
||||
storageSpec:
|
||||
volumeClaimTemplate:
|
||||
spec:
|
||||
storageClassName: longhorn-wffc
|
||||
accessModes: ["ReadWriteOnce"]
|
||||
resources:
|
||||
requests:
|
||||
storage: 20Gi
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 200m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: 1000m
|
||||
memory: 1Gi
|
||||
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
|
||||
# Match ServiceMonitors/PodMonitors from all namespaces, not just the ones
|
||||
# the chart itself creates. Required to scrape workloads in other namespaces.
|
||||
serviceMonitorSelectorNilUsesHelmValues: false
|
||||
podMonitorSelectorNilUsesHelmValues: false
|
||||
ruleSelectorNilUsesHelmValues: false
|
||||
|
||||
# ── Prometheus Operator ────────────────────────────────────────────────────────
|
||||
prometheusOperator:
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 200m
|
||||
memory: 256Mi
|
||||
|
||||
# ── node-exporter ─────────────────────────────────────────────────────────────
|
||||
# DaemonSet: one pod per node, reads /proc and /sys directly via hostPID.
|
||||
# These filesystem excludes prevent scrape errors on Talos's read-only overlayfs
|
||||
# mounts and containerd's ephemeral snapshot filesystems.
|
||||
nodeExporter:
|
||||
enabled: true
|
||||
|
||||
prometheus-node-exporter:
|
||||
extraArgs:
|
||||
- --collector.filesystem.mount-points-exclude=^/(dev|proc|run/credentials/.+|sys|var/lib/containerd/.+|var/lib/kubelet/.+|run/.+)($|/)
|
||||
- --collector.filesystem.fs-types-exclude=^(autofs|binfmt_misc|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$
|
||||
tolerations:
|
||||
- operator: Exists # schedule on every node regardless of taints
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
|
||||
# ── kube-state-metrics ────────────────────────────────────────────────────────
|
||||
# Watches the K8s API; surfaces pod CPU/memory requests, deployment replica
|
||||
# counts, pod phase, etc. — the "are my workloads healthy?" layer.
|
||||
kubeStateMetrics:
|
||||
enabled: true
|
||||
|
||||
kube-state-metrics:
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
|
||||
# ── Disable unreachable control-plane scrape targets ──────────────────────────
|
||||
kubeControllerManager:
|
||||
enabled: false
|
||||
|
||||
kubeScheduler:
|
||||
enabled: false
|
||||
|
||||
kubeEtcd:
|
||||
enabled: false
|
||||
|
||||
kubeProxy:
|
||||
enabled: false
|
||||
@@ -0,0 +1,37 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: argocd
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- cicd
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: argocd-metrics
|
||||
endpoints:
|
||||
- port: metrics
|
||||
interval: 30s
|
||||
scrapeTimeout: 10s
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: argocd-server
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- cicd
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: argocd-server-metrics
|
||||
endpoints:
|
||||
- port: metrics
|
||||
interval: 30s
|
||||
scrapeTimeout: 10s
|
||||
@@ -0,0 +1,18 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: authentik
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- iam
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: authentik
|
||||
endpoints:
|
||||
- port: metrics
|
||||
interval: 30s
|
||||
scrapeTimeout: 10s
|
||||
@@ -0,0 +1,19 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: forgejo
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- cicd
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: gitea
|
||||
endpoints:
|
||||
- port: http
|
||||
path: /metrics
|
||||
interval: 30s
|
||||
scrapeTimeout: 10s
|
||||
@@ -0,0 +1,19 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: minio
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- storage
|
||||
selector:
|
||||
matchLabels:
|
||||
app: minio
|
||||
endpoints:
|
||||
- port: minio-api
|
||||
path: /minio/v2/metrics/cluster
|
||||
interval: 30s
|
||||
scrapeTimeout: 10s
|
||||
Reference in New Issue
Block a user