feat(grafana): provision a Gotify contact point

Grafana gains one contact point, `gotify`, of type webhook. It posts to the
bridge in the notifications namespace because Gotify rejects a Grafana payload
with 400 "Field 'message' is required".

maxAlerts 0 keeps every alert in one request, and the bridge names the extra
count in the message body instead of dropping alerts.

k8s/apps/gotify/bridge/configmap.yaml

@@ -0,0 +1,110 @@
+# The bridge script is mounted as a ConfigMap so no image build is needed.
+# Regenerate this block from grafana_gotify_bridge.py when the script changes.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-gotify-bridge
+  namespace: notifications
+data:
+  grafana_gotify_bridge.py: |
+    #!/usr/bin/env python3
+    """Translate a Grafana alert webhook into a Gotify message.
+    
+    Grafana sends its own payload shape:
+      {"receiver": "...", "status": "firing", "alerts": [{"labels": {...}, "annotations": {...}}]}
+    
+    Gotify requires:
+      {"title": "...", "message": "...", "priority": 0..10}
+    
+    Gotify answers 400 "Field 'message' is required" to a Grafana payload, so a
+    conversion step is mandatory. This service is that step and nothing else.
+    Standard library only, so it needs no image build and no dependencies.
+    """
+    import json
+    import os
+    import urllib.request
+    from http.server import BaseHTTPRequestHandler, HTTPServer
+    
+    GOTIFY_BASE = os.environ.get("GOTIFY_URL", "http://gotify.notifications.svc.cluster.local")
+    GOTIFY_TOKEN = os.environ["GOTIFY_TOKEN"]
+    LISTEN_PORT = int(os.environ.get("PORT", "8080"))
+    
+    # Firing is louder than resolved. Gotify priority runs 0 to 10.
+    PRIORITY = {"firing": 8, "resolved": 3}
+    
+    
+    def to_gotify(payload):
+        alerts = payload.get("alerts") or []
+        status = (payload.get("status") or "firing").lower()
+        priority = PRIORITY.get(status, 5)
+    
+        if not alerts:
+            title = payload.get("title") or "Grafana notification"
+            return title, json.dumps(payload)[:2000], priority
+    
+        first = alerts[0]
+        labels = first.get("labels") or {}
+        annotations = first.get("annotations") or {}
+        name = labels.get("alertname", "Grafana alert")
+    
+        title = "[%s] %s" % (status.upper(), name)
+        lines = []
+        summary = annotations.get("summary") or annotations.get("description")
+        if summary:
+            lines.append(summary)
+        for key in ("severity", "namespace", "pod", "instance", "job"):
+            if labels.get(key):
+                lines.append("%s: %s" % (key, labels[key]))
+        if len(alerts) > 1:
+            lines.append("+%d more alert(s) in this notification" % (len(alerts) - 1))
+    
+        return title, "\n".join(lines) or name, priority
+    
+    
+    class Handler(BaseHTTPRequestHandler):
+        def _reply(self, code, body=b"ok"):
+            self.send_response(code)
+            self.send_header("Content-Type", "text/plain")
+            self.end_headers()
+            self.wfile.write(body)
+    
+        def do_GET(self):
+            self._reply(200)
+    
+        def do_POST(self):
+            length = int(self.headers.get("Content-Length") or 0)
+            raw = self.rfile.read(length) if length else b"{}"
+            try:
+                payload = json.loads(raw.decode("utf-8", "replace") or "{}")
+            except ValueError:
+                payload = {}
+    
+            title, message, priority = to_gotify(payload)
+            body = json.dumps({
+                "title": title[:200],
+                "message": message[:4000],
+                "priority": priority,
+            }).encode()
+    
+            request = urllib.request.Request(
+                GOTIFY_BASE.rstrip("/") + "/message",
+                data=body,
+                headers={"Content-Type": "application/json", "X-Gotify-Key": GOTIFY_TOKEN},
+            )
+            try:
+                urllib.request.urlopen(request, timeout=10).read()
+            except Exception as exc:  # noqa: BLE001 - log and report to Grafana
+                print("forward to gotify failed: %s" % exc, flush=True)
+                self._reply(502, b"gotify rejected the message")
+                return
+    
+            print("forwarded: %s" % title, flush=True)
+            self._reply(200)
+    
+        def log_message(self, fmt, *args):  # keep the log to one line per delivery
+            print("%s - %s" % (self.address_string(), fmt % args), flush=True)
+    
+    
+    if __name__ == "__main__":
+        print("grafana-gotify bridge listening on %d" % LISTEN_PORT, flush=True)
+        HTTPServer(("0.0.0.0", LISTEN_PORT), Handler).serve_forever()

k8s/apps/gotify/bridge/deployment.yaml

@@ -0,0 +1,80 @@
+# Grafana sends its own JSON and Gotify rejects it, so this Deployment is the
+# translation step between them. Grafana posts to the Service, the bridge posts
+# to /message with the app token from a SOPS secret.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: grafana-gotify-bridge
+  namespace: notifications
+  labels:
+    app: grafana-gotify-bridge
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: grafana-gotify-bridge
+  template:
+    metadata:
+      labels:
+        app: grafana-gotify-bridge
+    spec:
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: bridge
+          image: python:3.12-alpine
+          command: ["python3", "/app/grafana_gotify_bridge.py"]
+          env:
+            - name: GOTIFY_URL
+              value: "http://gotify.notifications.svc.cluster.local"
+            - name: PORT
+              value: "8080"
+            - name: GOTIFY_TOKEN
+              valueFrom:
+                secretKeyRef: {name: grafana-gotify-bridge, key: app-token}
+          ports:
+            - name: http
+              containerPort: 8080
+          readinessProbe:
+            httpGet: {path: /, port: http}
+            initialDelaySeconds: 3
+            periodSeconds: 10
+          livenessProbe:
+            httpGet: {path: /, port: http}
+            initialDelaySeconds: 10
+            periodSeconds: 20
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: script
+              mountPath: /app
+              readOnly: true
+          resources:
+            requests: {cpu: 10m, memory: 32Mi}
+            limits: {cpu: 200m, memory: 96Mi}
+      volumes:
+        - name: script
+          configMap:
+            name: grafana-gotify-bridge
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: grafana-gotify-bridge
+  namespace: notifications
+  labels:
+    app: grafana-gotify-bridge
+spec:
+  type: ClusterIP
+  selector:
+    app: grafana-gotify-bridge
+  ports:
+    - name: http
+      port: 8080
+      targetPort: http

k8s/apps/gotify/bridge/grafana_gotify_bridge.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+"""Translate a Grafana alert webhook into a Gotify message.
+
+Grafana sends its own payload shape:
+  {"receiver": "...", "status": "firing", "alerts": [{"labels": {...}, "annotations": {...}}]}
+
+Gotify requires:
+  {"title": "...", "message": "...", "priority": 0..10}
+
+Gotify answers 400 "Field 'message' is required" to a Grafana payload, so a
+conversion step is mandatory. This service is that step and nothing else.
+Standard library only, so it needs no image build and no dependencies.
+"""
+import json
+import os
+import urllib.request
+from http.server import BaseHTTPRequestHandler, HTTPServer
+
+GOTIFY_BASE = os.environ.get("GOTIFY_URL", "http://gotify.notifications.svc.cluster.local")
+GOTIFY_TOKEN = os.environ["GOTIFY_TOKEN"]
+LISTEN_PORT = int(os.environ.get("PORT", "8080"))
+
+# Firing is louder than resolved. Gotify priority runs 0 to 10.
+PRIORITY = {"firing": 8, "resolved": 3}
+
+
+def to_gotify(payload):
+    alerts = payload.get("alerts") or []
+    status = (payload.get("status") or "firing").lower()
+    priority = PRIORITY.get(status, 5)
+
+    if not alerts:
+        title = payload.get("title") or "Grafana notification"
+        return title, json.dumps(payload)[:2000], priority
+
+    first = alerts[0]
+    labels = first.get("labels") or {}
+    annotations = first.get("annotations") or {}
+    name = labels.get("alertname", "Grafana alert")
+
+    title = "[%s] %s" % (status.upper(), name)
+    lines = []
+    summary = annotations.get("summary") or annotations.get("description")
+    if summary:
+        lines.append(summary)
+    for key in ("severity", "namespace", "pod", "instance", "job"):
+        if labels.get(key):
+            lines.append("%s: %s" % (key, labels[key]))
+    if len(alerts) > 1:
+        lines.append("+%d more alert(s) in this notification" % (len(alerts) - 1))
+
+    return title, "\n".join(lines) or name, priority
+
+
+class Handler(BaseHTTPRequestHandler):
+    def _reply(self, code, body=b"ok"):
+        self.send_response(code)
+        self.send_header("Content-Type", "text/plain")
+        self.end_headers()
+        self.wfile.write(body)
+
+    def do_GET(self):
+        self._reply(200)
+
+    def do_POST(self):
+        length = int(self.headers.get("Content-Length") or 0)
+        raw = self.rfile.read(length) if length else b"{}"
+        try:
+            payload = json.loads(raw.decode("utf-8", "replace") or "{}")
+        except ValueError:
+            payload = {}
+
+        title, message, priority = to_gotify(payload)
+        body = json.dumps({
+            "title": title[:200],
+            "message": message[:4000],
+            "priority": priority,
+        }).encode()
+
+        request = urllib.request.Request(
+            GOTIFY_BASE.rstrip("/") + "/message",
+            data=body,
+            headers={"Content-Type": "application/json", "X-Gotify-Key": GOTIFY_TOKEN},
+        )
+        try:
+            urllib.request.urlopen(request, timeout=10).read()
+        except Exception as exc:  # noqa: BLE001 - log and report to Grafana
+            print("forward to gotify failed: %s" % exc, flush=True)
+            self._reply(502, b"gotify rejected the message")
+            return
+
+        print("forwarded: %s" % title, flush=True)
+        self._reply(200)
+
+    def log_message(self, fmt, *args):  # keep the log to one line per delivery
+        print("%s - %s" % (self.address_string(), fmt % args), flush=True)
+
+
+if __name__ == "__main__":
+    print("grafana-gotify bridge listening on %d" % LISTEN_PORT, flush=True)
+    HTTPServer(("0.0.0.0", LISTEN_PORT), Handler).serve_forever()

k8s/apps/gotify/kustomization.yaml

@@ -6,6 +6,8 @@ resources:
   - deployment.yaml
   - service.yaml
   - ingress.yaml
+  - bridge/configmap.yaml
+  - bridge/deployment.yaml
   - igotify-pvc.yaml
   - igotify-deployment.yaml
   - igotify-service.yaml

k8s/argocd/secrets/grafana-gotify-bridge.enc.yaml

@@ -0,0 +1,28 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: grafana-gotify-bridge
+    namespace: notifications
+type: Opaque
+data:
+    app-token: ENC[AES256_GCM,data:MnkKT0/ubJctmLTFXjdCZjNlz+KgQLsPwIKCmPF2weEtMw6ieRfb3+B7OUlIiDqpktdPaJA/znsMOYPXT2aXDqyKquA=,iv:vcAGkgfTheoViZ7OzJJhbNf0boIbRBZLr60ieW5DhLk=,tag:iBDJEjGScxHopkNExh7jDw==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBYYWZYNFBHbVhRSkZoa3dH
+            dWp3WllyVmRPcExPTndZUlFsTG1YMHN6WW1jCjRiS1M2U0tIM0tCdGhJOUQ0c2JW
+            bXNTTExDeVVmcExuNStldFdhTHMrVTgKLS0tIDN3d3JKSEtIUUFhVHROOVBUaUNO
+            UUVGcVVkOVJmN294dE5DS1R1UkxiZzgKx0MEkIrFsO2dQwjY2410Nj0Nyy3QgC3L
+            T844m4WT+mxFwB69/y0luGkn4aXMS3xrHHWUC+vjAkM4Unp+MKhhrg==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2026-10-27T19:35:38Z"
+    mac: ENC[AES256_GCM,data:Gf5jnvWP+jBRc8fvQHyrnHnTDaaUTcXQgoOqm7Drax5FpBYEHa36pdKaUq+XR6VBd7ch6Ia/f7YavhS47ybgR3jn+w+trMxkpwtZhm5clw2Lo3Hm7Cvi4fG4qhcjm1C8/NOnWGsi5Qm+TJqyG509p9HNTvM99SzElRovPZldLL4=,iv:10hA6nq0WIO0a2I4ZW6uPGXPjesFgxrwmXohNO3FZz4=,tag:bJ37e8jvgEactq6T/pM2UQ==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.9.3

k8s/argocd/secrets/secret-generator.yaml

@@ -37,6 +37,7 @@ files:
     - zitadel-masterkey.enc.yaml
     - smtp-gotify-secrets.enc.yaml
     - igotify-client-token.enc.yaml
+    - grafana-gotify-bridge.enc.yaml
     # Login UI RSA keypair, adopted from the chart-generated Secret so ArgoCD
     # owns it. Without this, the chart re-generates the keypair on every render,
     # which changes checksum/secret-login-service-key and leaves both Zitadel

k8s/infra/logging/grafana-values.yaml

@@ -209,3 +209,25 @@ sidecar:
       disableDelete: false
       foldersFromFilesStructure: true
     searchNamespace: ALL
+
+# ── Alerting ──────────────────────────────────────────────────────────────────
+# One contact point: Gotify. Gotify refuses Grafana's payload shape with
+# 400 "Field 'message' is required", so a bridge service translates the JSON
+# first (k8s/apps/gotify/bridge). Grafana posts to the bridge, the bridge posts
+# to /message with an app token.
+alerting:
+  contactpoints.yaml:
+    apiVersion: 1
+    contactPoints:
+      - orgId: 1
+        name: gotify
+        receivers:
+          - uid: gotify-webhook
+            type: webhook
+            disableResolveMessage: false
+            settings:
+              url: http://grafana-gotify-bridge.notifications.svc.cluster.local:8080/alert
+              httpMethod: POST
+              # 0 sends every alert in one request. The bridge names the extra
+              # count in the message body.
+              maxAlerts: 0