From 50a751de449cb020cc9e10eceb35bfba368c9f70 Mon Sep 17 00:00:00 2001 From: rock Date: Tue, 27 Oct 2026 12:35:44 -0700 Subject: [PATCH] feat(gotify): bridge Grafana alerts into Gotify Gotify rejects a Grafana payload with 400 "Field 'message' is required", so a translation step is mandatory. This adds a small standard-library Python service: it accepts Grafana's JSON, builds a Gotify message, and posts it with an app token. Mapping: firing -> priority 8, title "[FIRING] " resolved -> priority 3, title "[RESOLVED] " body -> the summary annotation plus severity, namespace, pod, instance, job A multi-alert notification names the alert count instead of dropping the rest. The script ships as a ConfigMap so no image build is needed. The app token comes from a SOPS secret and belongs to Gotify application `grafana`. --- k8s/apps/gotify/bridge/configmap.yaml | 110 ++++++++++++++++++ k8s/apps/gotify/bridge/deployment.yaml | 80 +++++++++++++ .../gotify/bridge/grafana_gotify_bridge.py | 101 ++++++++++++++++ k8s/apps/gotify/kustomization.yaml | 2 + .../secrets/grafana-gotify-bridge.enc.yaml | 28 +++++ k8s/argocd/secrets/secret-generator.yaml | 1 + 6 files changed, 322 insertions(+) create mode 100644 k8s/apps/gotify/bridge/configmap.yaml create mode 100644 k8s/apps/gotify/bridge/deployment.yaml create mode 100644 k8s/apps/gotify/bridge/grafana_gotify_bridge.py create mode 100644 k8s/argocd/secrets/grafana-gotify-bridge.enc.yaml diff --git a/k8s/apps/gotify/bridge/configmap.yaml b/k8s/apps/gotify/bridge/configmap.yaml new file mode 100644 index 0000000..1118086 --- /dev/null +++ b/k8s/apps/gotify/bridge/configmap.yaml @@ -0,0 +1,110 @@ +# The bridge script is mounted as a ConfigMap so no image build is needed. +# Regenerate this block from grafana_gotify_bridge.py when the script changes. +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-gotify-bridge + namespace: notifications +data: + grafana_gotify_bridge.py: | + #!/usr/bin/env python3 + """Translate a Grafana alert webhook into a Gotify message. + + Grafana sends its own payload shape: + {"receiver": "...", "status": "firing", "alerts": [{"labels": {...}, "annotations": {...}}]} + + Gotify requires: + {"title": "...", "message": "...", "priority": 0..10} + + Gotify answers 400 "Field 'message' is required" to a Grafana payload, so a + conversion step is mandatory. This service is that step and nothing else. + Standard library only, so it needs no image build and no dependencies. + """ + import json + import os + import urllib.request + from http.server import BaseHTTPRequestHandler, HTTPServer + + GOTIFY_BASE = os.environ.get("GOTIFY_URL", "http://gotify.notifications.svc.cluster.local") + GOTIFY_TOKEN = os.environ["GOTIFY_TOKEN"] + LISTEN_PORT = int(os.environ.get("PORT", "8080")) + + # Firing is louder than resolved. Gotify priority runs 0 to 10. + PRIORITY = {"firing": 8, "resolved": 3} + + + def to_gotify(payload): + alerts = payload.get("alerts") or [] + status = (payload.get("status") or "firing").lower() + priority = PRIORITY.get(status, 5) + + if not alerts: + title = payload.get("title") or "Grafana notification" + return title, json.dumps(payload)[:2000], priority + + first = alerts[0] + labels = first.get("labels") or {} + annotations = first.get("annotations") or {} + name = labels.get("alertname", "Grafana alert") + + title = "[%s] %s" % (status.upper(), name) + lines = [] + summary = annotations.get("summary") or annotations.get("description") + if summary: + lines.append(summary) + for key in ("severity", "namespace", "pod", "instance", "job"): + if labels.get(key): + lines.append("%s: %s" % (key, labels[key])) + if len(alerts) > 1: + lines.append("+%d more alert(s) in this notification" % (len(alerts) - 1)) + + return title, "\n".join(lines) or name, priority + + + class Handler(BaseHTTPRequestHandler): + def _reply(self, code, body=b"ok"): + self.send_response(code) + self.send_header("Content-Type", "text/plain") + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + self._reply(200) + + def do_POST(self): + length = int(self.headers.get("Content-Length") or 0) + raw = self.rfile.read(length) if length else b"{}" + try: + payload = json.loads(raw.decode("utf-8", "replace") or "{}") + except ValueError: + payload = {} + + title, message, priority = to_gotify(payload) + body = json.dumps({ + "title": title[:200], + "message": message[:4000], + "priority": priority, + }).encode() + + request = urllib.request.Request( + GOTIFY_BASE.rstrip("/") + "/message", + data=body, + headers={"Content-Type": "application/json", "X-Gotify-Key": GOTIFY_TOKEN}, + ) + try: + urllib.request.urlopen(request, timeout=10).read() + except Exception as exc: # noqa: BLE001 - log and report to Grafana + print("forward to gotify failed: %s" % exc, flush=True) + self._reply(502, b"gotify rejected the message") + return + + print("forwarded: %s" % title, flush=True) + self._reply(200) + + def log_message(self, fmt, *args): # keep the log to one line per delivery + print("%s - %s" % (self.address_string(), fmt % args), flush=True) + + + if __name__ == "__main__": + print("grafana-gotify bridge listening on %d" % LISTEN_PORT, flush=True) + HTTPServer(("0.0.0.0", LISTEN_PORT), Handler).serve_forever() diff --git a/k8s/apps/gotify/bridge/deployment.yaml b/k8s/apps/gotify/bridge/deployment.yaml new file mode 100644 index 0000000..fe9f082 --- /dev/null +++ b/k8s/apps/gotify/bridge/deployment.yaml @@ -0,0 +1,80 @@ +# Grafana sends its own JSON and Gotify rejects it, so this Deployment is the +# translation step between them. Grafana posts to the Service, the bridge posts +# to /message with the app token from a SOPS secret. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana-gotify-bridge + namespace: notifications + labels: + app: grafana-gotify-bridge +spec: + replicas: 1 + selector: + matchLabels: + app: grafana-gotify-bridge + template: + metadata: + labels: + app: grafana-gotify-bridge + spec: + securityContext: + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + containers: + - name: bridge + image: python:3.12-alpine + command: ["python3", "/app/grafana_gotify_bridge.py"] + env: + - name: GOTIFY_URL + value: "http://gotify.notifications.svc.cluster.local" + - name: PORT + value: "8080" + - name: GOTIFY_TOKEN + valueFrom: + secretKeyRef: {name: grafana-gotify-bridge, key: app-token} + ports: + - name: http + containerPort: 8080 + readinessProbe: + httpGet: {path: /, port: http} + initialDelaySeconds: 3 + periodSeconds: 10 + livenessProbe: + httpGet: {path: /, port: http} + initialDelaySeconds: 10 + periodSeconds: 20 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: script + mountPath: /app + readOnly: true + resources: + requests: {cpu: 10m, memory: 32Mi} + limits: {cpu: 200m, memory: 96Mi} + volumes: + - name: script + configMap: + name: grafana-gotify-bridge +--- +apiVersion: v1 +kind: Service +metadata: + name: grafana-gotify-bridge + namespace: notifications + labels: + app: grafana-gotify-bridge +spec: + type: ClusterIP + selector: + app: grafana-gotify-bridge + ports: + - name: http + port: 8080 + targetPort: http diff --git a/k8s/apps/gotify/bridge/grafana_gotify_bridge.py b/k8s/apps/gotify/bridge/grafana_gotify_bridge.py new file mode 100644 index 0000000..4bb9903 --- /dev/null +++ b/k8s/apps/gotify/bridge/grafana_gotify_bridge.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Translate a Grafana alert webhook into a Gotify message. + +Grafana sends its own payload shape: + {"receiver": "...", "status": "firing", "alerts": [{"labels": {...}, "annotations": {...}}]} + +Gotify requires: + {"title": "...", "message": "...", "priority": 0..10} + +Gotify answers 400 "Field 'message' is required" to a Grafana payload, so a +conversion step is mandatory. This service is that step and nothing else. +Standard library only, so it needs no image build and no dependencies. +""" +import json +import os +import urllib.request +from http.server import BaseHTTPRequestHandler, HTTPServer + +GOTIFY_BASE = os.environ.get("GOTIFY_URL", "http://gotify.notifications.svc.cluster.local") +GOTIFY_TOKEN = os.environ["GOTIFY_TOKEN"] +LISTEN_PORT = int(os.environ.get("PORT", "8080")) + +# Firing is louder than resolved. Gotify priority runs 0 to 10. +PRIORITY = {"firing": 8, "resolved": 3} + + +def to_gotify(payload): + alerts = payload.get("alerts") or [] + status = (payload.get("status") or "firing").lower() + priority = PRIORITY.get(status, 5) + + if not alerts: + title = payload.get("title") or "Grafana notification" + return title, json.dumps(payload)[:2000], priority + + first = alerts[0] + labels = first.get("labels") or {} + annotations = first.get("annotations") or {} + name = labels.get("alertname", "Grafana alert") + + title = "[%s] %s" % (status.upper(), name) + lines = [] + summary = annotations.get("summary") or annotations.get("description") + if summary: + lines.append(summary) + for key in ("severity", "namespace", "pod", "instance", "job"): + if labels.get(key): + lines.append("%s: %s" % (key, labels[key])) + if len(alerts) > 1: + lines.append("+%d more alert(s) in this notification" % (len(alerts) - 1)) + + return title, "\n".join(lines) or name, priority + + +class Handler(BaseHTTPRequestHandler): + def _reply(self, code, body=b"ok"): + self.send_response(code) + self.send_header("Content-Type", "text/plain") + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + self._reply(200) + + def do_POST(self): + length = int(self.headers.get("Content-Length") or 0) + raw = self.rfile.read(length) if length else b"{}" + try: + payload = json.loads(raw.decode("utf-8", "replace") or "{}") + except ValueError: + payload = {} + + title, message, priority = to_gotify(payload) + body = json.dumps({ + "title": title[:200], + "message": message[:4000], + "priority": priority, + }).encode() + + request = urllib.request.Request( + GOTIFY_BASE.rstrip("/") + "/message", + data=body, + headers={"Content-Type": "application/json", "X-Gotify-Key": GOTIFY_TOKEN}, + ) + try: + urllib.request.urlopen(request, timeout=10).read() + except Exception as exc: # noqa: BLE001 - log and report to Grafana + print("forward to gotify failed: %s" % exc, flush=True) + self._reply(502, b"gotify rejected the message") + return + + print("forwarded: %s" % title, flush=True) + self._reply(200) + + def log_message(self, fmt, *args): # keep the log to one line per delivery + print("%s - %s" % (self.address_string(), fmt % args), flush=True) + + +if __name__ == "__main__": + print("grafana-gotify bridge listening on %d" % LISTEN_PORT, flush=True) + HTTPServer(("0.0.0.0", LISTEN_PORT), Handler).serve_forever() diff --git a/k8s/apps/gotify/kustomization.yaml b/k8s/apps/gotify/kustomization.yaml index 4c7e0b6..51f734f 100644 --- a/k8s/apps/gotify/kustomization.yaml +++ b/k8s/apps/gotify/kustomization.yaml @@ -6,6 +6,8 @@ resources: - deployment.yaml - service.yaml - ingress.yaml + - bridge/configmap.yaml + - bridge/deployment.yaml - igotify-pvc.yaml - igotify-deployment.yaml - igotify-service.yaml diff --git a/k8s/argocd/secrets/grafana-gotify-bridge.enc.yaml b/k8s/argocd/secrets/grafana-gotify-bridge.enc.yaml new file mode 100644 index 0000000..e789c4c --- /dev/null +++ b/k8s/argocd/secrets/grafana-gotify-bridge.enc.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: Secret +metadata: + name: grafana-gotify-bridge + namespace: notifications +type: Opaque +data: + app-token: ENC[AES256_GCM,data:MnkKT0/ubJctmLTFXjdCZjNlz+KgQLsPwIKCmPF2weEtMw6ieRfb3+B7OUlIiDqpktdPaJA/znsMOYPXT2aXDqyKquA=,iv:vcAGkgfTheoViZ7OzJJhbNf0boIbRBZLr60ieW5DhLk=,tag:iBDJEjGScxHopkNExh7jDw==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBYYWZYNFBHbVhRSkZoa3dH + dWp3WllyVmRPcExPTndZUlFsTG1YMHN6WW1jCjRiS1M2U0tIM0tCdGhJOUQ0c2JW + bXNTTExDeVVmcExuNStldFdhTHMrVTgKLS0tIDN3d3JKSEtIUUFhVHROOVBUaUNO + UUVGcVVkOVJmN294dE5DS1R1UkxiZzgKx0MEkIrFsO2dQwjY2410Nj0Nyy3QgC3L + T844m4WT+mxFwB69/y0luGkn4aXMS3xrHHWUC+vjAkM4Unp+MKhhrg== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2026-10-27T19:35:38Z" + mac: ENC[AES256_GCM,data:Gf5jnvWP+jBRc8fvQHyrnHnTDaaUTcXQgoOqm7Drax5FpBYEHa36pdKaUq+XR6VBd7ch6Ia/f7YavhS47ybgR3jn+w+trMxkpwtZhm5clw2Lo3Hm7Cvi4fG4qhcjm1C8/NOnWGsi5Qm+TJqyG509p9HNTvM99SzElRovPZldLL4=,iv:10hA6nq0WIO0a2I4ZW6uPGXPjesFgxrwmXohNO3FZz4=,tag:bJ37e8jvgEactq6T/pM2UQ==,type:str] + pgp: [] + encrypted_regex: ^(data|stringData)$ + version: 3.9.3 diff --git a/k8s/argocd/secrets/secret-generator.yaml b/k8s/argocd/secrets/secret-generator.yaml index 56ead34..ad7e9c8 100644 --- a/k8s/argocd/secrets/secret-generator.yaml +++ b/k8s/argocd/secrets/secret-generator.yaml @@ -37,6 +37,7 @@ files: - zitadel-masterkey.enc.yaml - smtp-gotify-secrets.enc.yaml - igotify-client-token.enc.yaml + - grafana-gotify-bridge.enc.yaml # Login UI RSA keypair, adopted from the chart-generated Secret so ArgoCD # owns it. Without this, the chart re-generates the keypair on every render, # which changes checksum/secret-login-service-key and leaves both Zitadel