rock
12c66fb612
Grafana gains one contact point, `gotify`, of type webhook. It posts to the bridge in the notifications namespace because Gotify rejects a Grafana payload with 400 "Field 'message' is required". maxAlerts 0 keeps every alert in one request, and the bridge names the extra count in the message body instead of dropping alerts.
234 lines
9.1 KiB
YAML
234 lines
9.1 KiB
YAML
# logging/grafana-values.yaml
|
|
# Grafana — dashboarding and log/metrics exploration UI.
|
|
# Deployed in the logging namespace alongside Loki and Promtail.
|
|
#
|
|
# Secrets provided by ArgoCD SOPS CMP plugin:
|
|
# adminPassword — mounted from the grafana-admin K8s Secret (admin.existingSecret)
|
|
# OAuth client secret — mounted from the grafana-oidc K8s Secret (envFromSecret below)
|
|
|
|
replicas: 1
|
|
|
|
# RWO PVC (Longhorn) — old pod must fully terminate before the new one can
|
|
# mount the volume. Recreate avoids the "two pods fighting over one PVC" failure.
|
|
deploymentStrategy:
|
|
type: Recreate
|
|
|
|
podAnnotations:
|
|
secret.reloader.stakater.com/reload: "grafana-oidc,grafana-admin"
|
|
|
|
adminUser: admin
|
|
admin:
|
|
existingSecret: grafana-admin
|
|
existingSecretKey: admin-password
|
|
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 128Mi
|
|
limits:
|
|
cpu: 500m
|
|
memory: 256Mi
|
|
|
|
# ── Persistence ───────────────────────────────────────────────────────────────
|
|
# Stores dashboards saved in the UI, datasource edits, and user sessions.
|
|
# Longhorn provides the RWO block volume; 5Gi is ample for dashboard JSON.
|
|
persistence:
|
|
enabled: true
|
|
storageClassName: longhorn
|
|
accessModes:
|
|
- ReadWriteOnce
|
|
size: 5Gi
|
|
|
|
# ── Grafana config ────────────────────────────────────────────────────────────
|
|
grafana.ini:
|
|
server:
|
|
root_url: https://grafana.riotpiao.com
|
|
|
|
# Allow embedding dashboards in iframes (NextJS integration)
|
|
security:
|
|
allow_embedding: true
|
|
|
|
# No anonymous read access — every user must log in via Authentik SSO.
|
|
auth.anonymous:
|
|
enabled: false
|
|
|
|
# Explore tab: required for ad-hoc LogQL/PromQL queries against Loki/Prometheus.
|
|
explore:
|
|
enabled: true
|
|
|
|
# WAL for the embedded SQLite DB — prevents corruption on ungraceful shutdown.
|
|
database:
|
|
wal: true
|
|
|
|
# ── ZITADEL OIDC (generic OAuth2) ────────────────────────────────────────
|
|
# Grafana v10+ supports OIDC auto-discovery; we wire it manually here because
|
|
# ZITADEL's discovery endpoint is internal-only (no external DNS for iam.svc).
|
|
# All URLs use the external hostname so CoreDNS rewrites them to
|
|
# zitadel.iam.svc — this keeps the Host header correct so ZITADEL
|
|
# doesn't return localhost redirects in its token responses.
|
|
#
|
|
# role_attribute_path: JMESPath expression evaluated against the userinfo
|
|
# response. akadmin gets GrafanaAdmin (server admin, can impersonate);
|
|
# homelab-admins role members get Admin (org admin); everyone else Viewer.
|
|
auth.generic_oauth:
|
|
enabled: true
|
|
name: Zitadel
|
|
allow_sign_up: true # Allow sync to create/update users from OAuth claims
|
|
client_id: "396757801965191169"
|
|
scopes: openid email profile groups
|
|
auth_url: https://zitadel.riotpiao.com/oauth/v2/authorize
|
|
token_url: https://zitadel.riotpiao.com/oauth/v2/token
|
|
api_url: https://zitadel.riotpiao.com/oidc/v1/userinfo
|
|
# Read identity straight from the userinfo/id_token claims. Without these,
|
|
# Grafana falls back to the GitHub-style "<api_url>/emails" call, which
|
|
# ZITADEL doesn't serve — request 404s with "Error getting email address"
|
|
# and the whole OAuth login fails.
|
|
email_attribute_path: email
|
|
login_attribute_path: preferred_username # Use preferred_username — OpenID standard claim, always present
|
|
name_attribute_path: name
|
|
role_attribute_path: "preferred_username == 'akadmin' && 'GrafanaAdmin' || contains(groups[*], 'homelab-admins') && 'Admin' || 'Viewer'"
|
|
allow_assign_grafana_admin: true
|
|
use_pkce: false
|
|
use_refresh_token: false
|
|
skip_org_role_sync: false # Sync org roles from OAuth groups
|
|
tls_skip_verify_insecure: true # In-cluster cert; verify in prod
|
|
|
|
# GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET is injected from the grafana-oidc K8s
|
|
# Secret (k8s/argocd/secrets/grafana-oidc-secrets.enc.yaml). envFromSecret mounts
|
|
# every key in that secret as an environment variable — keeps secrets out of
|
|
# values files and out of git.
|
|
envFromSecret: grafana-oidc
|
|
|
|
# ── Datasources ───────────────────────────────────────────────────────────────
|
|
# Provisioned at install — no manual "Add datasource" step in the UI.
|
|
# Loki is the default (log exploration); Prometheus is secondary (metrics).
|
|
# Both use in-cluster service DNS — Grafana never leaves the cluster for data.
|
|
datasources:
|
|
datasources.yaml:
|
|
apiVersion: 1
|
|
datasources:
|
|
- name: Loki
|
|
type: loki
|
|
uid: loki
|
|
access: proxy
|
|
url: http://loki.logging.svc.cluster.local:3100
|
|
isDefault: true
|
|
version: 1
|
|
editable: true
|
|
jsonData:
|
|
maxLines: 1000
|
|
timeout: 60
|
|
- name: Prometheus
|
|
type: prometheus
|
|
uid: prometheus
|
|
access: proxy
|
|
url: http://prometheus-kube-prometheus-prometheus.monitoring.svc.cluster.local:9090
|
|
isDefault: false
|
|
version: 1
|
|
editable: true
|
|
jsonData:
|
|
timeInterval: 30s
|
|
timeout: 60
|
|
|
|
# ── Dashboard providers ───────────────────────────────────────────────────────
|
|
# Tells Grafana to watch a directory for dashboard JSON files.
|
|
# The `dashboards` block below populates that directory via an init container
|
|
# that downloads from grafana.com at install time.
|
|
dashboardProviders:
|
|
dashboardproviders.yaml:
|
|
apiVersion: 1
|
|
providers:
|
|
- name: default
|
|
orgId: 1
|
|
folder: ""
|
|
type: file
|
|
disableDeletion: false
|
|
editable: true
|
|
options:
|
|
path: /var/lib/grafana/dashboards/default
|
|
|
|
# ── Pre-loaded dashboards ──────────────────────────────────────────────────────
|
|
# Fetched from grafana.com by an init container at helm-install time and baked
|
|
# into ConfigMaps. gnetId + revision pin the exact dashboard version so an
|
|
# upstream change on grafana.com doesn't silently alter what's deployed.
|
|
dashboards:
|
|
default:
|
|
node-exporter-full:
|
|
gnetId: 1860
|
|
revision: 37
|
|
datasource: Prometheus
|
|
kubernetes-cluster:
|
|
gnetId: 7249
|
|
revision: 1
|
|
datasource: Prometheus
|
|
|
|
# Allow scheduling on the control-plane node (talos-cp-1 carries NoSchedule taint).
|
|
tolerations:
|
|
- key: node-role.kubernetes.io/control-plane
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
|
|
# Pin to az-a (talos-cp-1) — the only node running Longhorn. Without this the
|
|
# scheduler may place grafana on cp-2/cp-3, where its Longhorn PVC cannot attach
|
|
# ("CSINode does not contain driver driver.longhorn.io").
|
|
affinity:
|
|
nodeAffinity:
|
|
requiredDuringSchedulingIgnoredDuringExecution:
|
|
nodeSelectorTerms:
|
|
- matchExpressions:
|
|
- key: topology.kubernetes.io/zone
|
|
operator: In
|
|
values: [az-a]
|
|
|
|
# ClusterIP only — access via ingress (grafana.riotpiao.com) or port-forward.
|
|
service:
|
|
type: ClusterIP
|
|
port: 80
|
|
|
|
# Ingress managed by k8s/ingress/ingress.yaml — not the chart's built-in ingress.
|
|
ingress:
|
|
enabled: false
|
|
|
|
serviceMonitor:
|
|
enabled: false
|
|
|
|
# ── Dashboard-as-code (sidecar discovery) ──────────────────────────────────────
|
|
# Additive to the gnetId-download mechanism above. The sidecar watches for
|
|
# ConfigMaps labeled grafana_dashboard=1 in ANY namespace and loads them live —
|
|
# no Grafana restart needed when a new dashboard ConfigMap is applied.
|
|
sidecar:
|
|
dashboards:
|
|
enabled: true
|
|
label: grafana_dashboard
|
|
labelValue: "1"
|
|
folder: /var/lib/grafana/dashboards/custom
|
|
folderAnnotation: grafana_folder
|
|
provider:
|
|
name: custom
|
|
folder: "Homelab"
|
|
disableDelete: false
|
|
foldersFromFilesStructure: true
|
|
searchNamespace: ALL
|
|
|
|
# ── Alerting ──────────────────────────────────────────────────────────────────
|
|
# One contact point: Gotify. Gotify refuses Grafana's payload shape with
|
|
# 400 "Field 'message' is required", so a bridge service translates the JSON
|
|
# first (k8s/apps/gotify/bridge). Grafana posts to the bridge, the bridge posts
|
|
# to /message with an app token.
|
|
alerting:
|
|
contactpoints.yaml:
|
|
apiVersion: 1
|
|
contactPoints:
|
|
- orgId: 1
|
|
name: gotify
|
|
receivers:
|
|
- uid: gotify-webhook
|
|
type: webhook
|
|
disableResolveMessage: false
|
|
settings:
|
|
url: http://grafana-gotify-bridge.notifications.svc.cluster.local:8080/alert
|
|
httpMethod: POST
|
|
# 0 sends every alert in one request. The bridge names the extra
|
|
# count in the message body.
|
|
maxAlerts: 0
|