fix(iam): stop rotating client secrets on every apply, repair Grafana OIDC

Grafana sign-in failed with "Failed to get token from provider". The deployed
client secret no longer matched, because _ensure_oidc_apps called
_generate_client_secret for every app on every run, and ZITADEL stores exactly one
client secret per app: each call replaced the previous value and invalidated every
deployed copy.

Two changes:
1. Generate a secret only when the app is created. For an app that already exists,
   reuse the value from state.json, because the API cannot read a secret back.
2. Re-encrypt grafana-oidc-secrets.enc.yaml with the live secret, so Grafana's
   token exchange succeeds again.

Verified with the introspection endpoint:
  old deployed secret -> {"error":"unauthorized_client"}
  new secret          -> {"active":false}   (credentials accepted)

k8s/apps/gotify/bridge/configmap.yaml

@@ -0,0 +1,110 @@
+# The bridge script is mounted as a ConfigMap so no image build is needed.
+# Regenerate this block from grafana_gotify_bridge.py when the script changes.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-gotify-bridge
+  namespace: notifications
+data:
+  grafana_gotify_bridge.py: |
+    #!/usr/bin/env python3
+    """Translate a Grafana alert webhook into a Gotify message.
+    
+    Grafana sends its own payload shape:
+      {"receiver": "...", "status": "firing", "alerts": [{"labels": {...}, "annotations": {...}}]}
+    
+    Gotify requires:
+      {"title": "...", "message": "...", "priority": 0..10}
+    
+    Gotify answers 400 "Field 'message' is required" to a Grafana payload, so a
+    conversion step is mandatory. This service is that step and nothing else.
+    Standard library only, so it needs no image build and no dependencies.
+    """
+    import json
+    import os
+    import urllib.request
+    from http.server import BaseHTTPRequestHandler, HTTPServer
+    
+    GOTIFY_BASE = os.environ.get("GOTIFY_URL", "http://gotify.notifications.svc.cluster.local")
+    GOTIFY_TOKEN = os.environ["GOTIFY_TOKEN"]
+    LISTEN_PORT = int(os.environ.get("PORT", "8080"))
+    
+    # Firing is louder than resolved. Gotify priority runs 0 to 10.
+    PRIORITY = {"firing": 8, "resolved": 3}
+    
+    
+    def to_gotify(payload):
+        alerts = payload.get("alerts") or []
+        status = (payload.get("status") or "firing").lower()
+        priority = PRIORITY.get(status, 5)
+    
+        if not alerts:
+            title = payload.get("title") or "Grafana notification"
+            return title, json.dumps(payload)[:2000], priority
+    
+        first = alerts[0]
+        labels = first.get("labels") or {}
+        annotations = first.get("annotations") or {}
+        name = labels.get("alertname", "Grafana alert")
+    
+        title = "[%s] %s" % (status.upper(), name)
+        lines = []
+        summary = annotations.get("summary") or annotations.get("description")
+        if summary:
+            lines.append(summary)
+        for key in ("severity", "namespace", "pod", "instance", "job"):
+            if labels.get(key):
+                lines.append("%s: %s" % (key, labels[key]))
+        if len(alerts) > 1:
+            lines.append("+%d more alert(s) in this notification" % (len(alerts) - 1))
+    
+        return title, "\n".join(lines) or name, priority
+    
+    
+    class Handler(BaseHTTPRequestHandler):
+        def _reply(self, code, body=b"ok"):
+            self.send_response(code)
+            self.send_header("Content-Type", "text/plain")
+            self.end_headers()
+            self.wfile.write(body)
+    
+        def do_GET(self):
+            self._reply(200)
+    
+        def do_POST(self):
+            length = int(self.headers.get("Content-Length") or 0)
+            raw = self.rfile.read(length) if length else b"{}"
+            try:
+                payload = json.loads(raw.decode("utf-8", "replace") or "{}")
+            except ValueError:
+                payload = {}
+    
+            title, message, priority = to_gotify(payload)
+            body = json.dumps({
+                "title": title[:200],
+                "message": message[:4000],
+                "priority": priority,
+            }).encode()
+    
+            request = urllib.request.Request(
+                GOTIFY_BASE.rstrip("/") + "/message",
+                data=body,
+                headers={"Content-Type": "application/json", "X-Gotify-Key": GOTIFY_TOKEN},
+            )
+            try:
+                urllib.request.urlopen(request, timeout=10).read()
+            except Exception as exc:  # noqa: BLE001 - log and report to Grafana
+                print("forward to gotify failed: %s" % exc, flush=True)
+                self._reply(502, b"gotify rejected the message")
+                return
+    
+            print("forwarded: %s" % title, flush=True)
+            self._reply(200)
+    
+        def log_message(self, fmt, *args):  # keep the log to one line per delivery
+            print("%s - %s" % (self.address_string(), fmt % args), flush=True)
+    
+    
+    if __name__ == "__main__":
+        print("grafana-gotify bridge listening on %d" % LISTEN_PORT, flush=True)
+        HTTPServer(("0.0.0.0", LISTEN_PORT), Handler).serve_forever()

k8s/apps/gotify/bridge/deployment.yaml

@@ -0,0 +1,80 @@
+# Grafana sends its own JSON and Gotify rejects it, so this Deployment is the
+# translation step between them. Grafana posts to the Service, the bridge posts
+# to /message with the app token from a SOPS secret.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: grafana-gotify-bridge
+  namespace: notifications
+  labels:
+    app: grafana-gotify-bridge
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: grafana-gotify-bridge
+  template:
+    metadata:
+      labels:
+        app: grafana-gotify-bridge
+    spec:
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: bridge
+          image: python:3.12-alpine
+          command: ["python3", "/app/grafana_gotify_bridge.py"]
+          env:
+            - name: GOTIFY_URL
+              value: "http://gotify.notifications.svc.cluster.local"
+            - name: PORT
+              value: "8080"
+            - name: GOTIFY_TOKEN
+              valueFrom:
+                secretKeyRef: {name: grafana-gotify-bridge, key: app-token}
+          ports:
+            - name: http
+              containerPort: 8080
+          readinessProbe:
+            httpGet: {path: /, port: http}
+            initialDelaySeconds: 3
+            periodSeconds: 10
+          livenessProbe:
+            httpGet: {path: /, port: http}
+            initialDelaySeconds: 10
+            periodSeconds: 20
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: script
+              mountPath: /app
+              readOnly: true
+          resources:
+            requests: {cpu: 10m, memory: 32Mi}
+            limits: {cpu: 200m, memory: 96Mi}
+      volumes:
+        - name: script
+          configMap:
+            name: grafana-gotify-bridge
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: grafana-gotify-bridge
+  namespace: notifications
+  labels:
+    app: grafana-gotify-bridge
+spec:
+  type: ClusterIP
+  selector:
+    app: grafana-gotify-bridge
+  ports:
+    - name: http
+      port: 8080
+      targetPort: http

k8s/apps/gotify/bridge/grafana_gotify_bridge.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+"""Translate a Grafana alert webhook into a Gotify message.
+
+Grafana sends its own payload shape:
+  {"receiver": "...", "status": "firing", "alerts": [{"labels": {...}, "annotations": {...}}]}
+
+Gotify requires:
+  {"title": "...", "message": "...", "priority": 0..10}
+
+Gotify answers 400 "Field 'message' is required" to a Grafana payload, so a
+conversion step is mandatory. This service is that step and nothing else.
+Standard library only, so it needs no image build and no dependencies.
+"""
+import json
+import os
+import urllib.request
+from http.server import BaseHTTPRequestHandler, HTTPServer
+
+GOTIFY_BASE = os.environ.get("GOTIFY_URL", "http://gotify.notifications.svc.cluster.local")
+GOTIFY_TOKEN = os.environ["GOTIFY_TOKEN"]
+LISTEN_PORT = int(os.environ.get("PORT", "8080"))
+
+# Firing is louder than resolved. Gotify priority runs 0 to 10.
+PRIORITY = {"firing": 8, "resolved": 3}
+
+
+def to_gotify(payload):
+    alerts = payload.get("alerts") or []
+    status = (payload.get("status") or "firing").lower()
+    priority = PRIORITY.get(status, 5)
+
+    if not alerts:
+        title = payload.get("title") or "Grafana notification"
+        return title, json.dumps(payload)[:2000], priority
+
+    first = alerts[0]
+    labels = first.get("labels") or {}
+    annotations = first.get("annotations") or {}
+    name = labels.get("alertname", "Grafana alert")
+
+    title = "[%s] %s" % (status.upper(), name)
+    lines = []
+    summary = annotations.get("summary") or annotations.get("description")
+    if summary:
+        lines.append(summary)
+    for key in ("severity", "namespace", "pod", "instance", "job"):
+        if labels.get(key):
+            lines.append("%s: %s" % (key, labels[key]))
+    if len(alerts) > 1:
+        lines.append("+%d more alert(s) in this notification" % (len(alerts) - 1))
+
+    return title, "\n".join(lines) or name, priority
+
+
+class Handler(BaseHTTPRequestHandler):
+    def _reply(self, code, body=b"ok"):
+        self.send_response(code)
+        self.send_header("Content-Type", "text/plain")
+        self.end_headers()
+        self.wfile.write(body)
+
+    def do_GET(self):
+        self._reply(200)
+
+    def do_POST(self):
+        length = int(self.headers.get("Content-Length") or 0)
+        raw = self.rfile.read(length) if length else b"{}"
+        try:
+            payload = json.loads(raw.decode("utf-8", "replace") or "{}")
+        except ValueError:
+            payload = {}
+
+        title, message, priority = to_gotify(payload)
+        body = json.dumps({
+            "title": title[:200],
+            "message": message[:4000],
+            "priority": priority,
+        }).encode()
+
+        request = urllib.request.Request(
+            GOTIFY_BASE.rstrip("/") + "/message",
+            data=body,
+            headers={"Content-Type": "application/json", "X-Gotify-Key": GOTIFY_TOKEN},
+        )
+        try:
+            urllib.request.urlopen(request, timeout=10).read()
+        except Exception as exc:  # noqa: BLE001 - log and report to Grafana
+            print("forward to gotify failed: %s" % exc, flush=True)
+            self._reply(502, b"gotify rejected the message")
+            return
+
+        print("forwarded: %s" % title, flush=True)
+        self._reply(200)
+
+    def log_message(self, fmt, *args):  # keep the log to one line per delivery
+        print("%s - %s" % (self.address_string(), fmt % args), flush=True)
+
+
+if __name__ == "__main__":
+    print("grafana-gotify bridge listening on %d" % LISTEN_PORT, flush=True)
+    HTTPServer(("0.0.0.0", LISTEN_PORT), Handler).serve_forever()

k8s/apps/gotify/igotify-deployment.yaml

@@ -28,9 +28,17 @@ spec:
               value: "true"
             - name: ENABLE_SCALAR_UI
               value: "true"
-            # GOTIFY_URLS / GOTIFY_CLIENT_TOKENS / SECNTFY_TOKENS intentionally
-            # left unset — not required when Gotify is reachable via public domain
-            # (gotify.riotpiao.com). iGotify iOS app registers itself at runtime.
+            # The bridge authenticates with a Gotify client token, not with
+            # ZITADEL. Gotify has no way to accept a forwarded identity, so the
+            # token is the only credential that works server-side. The token
+            # belongs to user `rock`, never expires, and is revoked by deleting
+            # the client row in Gotify.
+            - name: GOTIFY_URLS
+              value: "https://gotify.riotpiao.com"
+            - name: GOTIFY_CLIENT_TOKENS
+              valueFrom:
+                secretKeyRef: {name: igotify-client-token, key: client-token}
+            # SECNTFY_TOKENS intentionally left unset: secntfy is not deployed.
           volumeMounts:
             - name: data
               mountPath: /app/data

k8s/apps/gotify/ingress.yaml

@@ -9,12 +9,6 @@ metadata:
     # WebSocket support for Gotify client connections (/stream endpoint)
     # nginx-ingress handles Upgrade/Connection headers natively with http/1.1
     nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
-    # Single sign-on. Gotify has no OIDC, so every request is handed to
-    # oauth2-proxy for an allow or deny decision. oauth2-proxy skips the token
-    # API paths itself, so /message keeps Gotify's own X-Gotify-Key check.
-    nginx.ingress.kubernetes.io/auth-url: "http://gotify-oauth2-proxy.notifications.svc.cluster.local:4180/oauth2/auth"
-    nginx.ingress.kubernetes.io/auth-signin: "https://gotify.riotpiao.com/oauth2/start?rd=$scheme://$host$request_uri"
-    nginx.ingress.kubernetes.io/auth-response-headers: "x-auth-request-user,x-auth-request-email"
 spec:
   ingressClassName: nginx
   rules:

k8s/apps/gotify/kustomization.yaml

@@ -5,10 +5,9 @@ resources:
   - db.yaml
   - deployment.yaml
   - service.yaml
-  - oauth2-proxy-deployment.yaml
-  - oauth2-proxy-service.yaml
-  - oauth2-proxy-ingress.yaml
   - ingress.yaml
+  - bridge/configmap.yaml
+  - bridge/deployment.yaml
   - igotify-pvc.yaml
   - igotify-deployment.yaml
   - igotify-service.yaml

k8s/apps/gotify/oauth2-proxy-deployment.yaml

@@ -1,82 +0,0 @@
-# oauth2-proxy in front of Gotify. Gotify has no OIDC support of its own, so it
-# cannot validate a ZITADEL token. This proxy is the front door: nginx asks it
-# for an auth decision on every request (auth_request), and it answers from a
-# ZITADEL session cookie.
-#
-# The Gotify REST API keeps its own client-token auth (X-Gotify-Key) and must
-# stay reachable without a browser session, or every push sender breaks:
-# Forgejo webhooks, the harness, poimen-sender and the ntfy-style relays all post
-# to /message. skip-auth-route leaves those paths to Gotify's own token check.
-#
-# Gotify's own login still exists behind the proxy. ZITADEL decides who may
-# reach the service; Gotify decides what that person may do inside it.
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: gotify-oauth2-proxy
-  namespace: notifications
-  labels:
-    app: gotify-oauth2-proxy
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: gotify-oauth2-proxy
-  template:
-    metadata:
-      labels:
-        app: gotify-oauth2-proxy
-    spec:
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 65534
-        seccompProfile:
-          type: RuntimeDefault
-      containers:
-        - name: oauth2-proxy
-          image: quay.io/oauth2-proxy/oauth2-proxy:v7.7.1
-          args:
-            - --provider=oidc
-            - --oidc-issuer-url=https://zitadel.riotpiao.com
-            - --email-domain=*
-            - --upstream=static://202
-            - --reverse-proxy=true
-            - --set-xauthrequest=true
-            - --cookie-secure=true
-            - --cookie-expire=12h
-            - --cookie-refresh=1h
-            - --skip-provider-button=true
-            - --redirect-url=https://gotify.riotpiao.com/oauth2/callback
-            - --http-address=0.0.0.0:4180
-            # Browser paths keep SSO. API paths fall through to Gotify's own
-            # client-token check, so existing senders and the Android client
-            # keep working without a browser session.
-            - --skip-auth-route=^/(message|application|client|user|version|stream|health)(/.*)?$
-          env:
-            - name: OAUTH2_PROXY_CLIENT_ID
-              valueFrom:
-                secretKeyRef: {name: gotify-oauth2-proxy, key: client-id}
-            - name: OAUTH2_PROXY_CLIENT_SECRET
-              valueFrom:
-                secretKeyRef: {name: gotify-oauth2-proxy, key: client-secret}
-            - name: OAUTH2_PROXY_COOKIE_SECRET
-              valueFrom:
-                secretKeyRef: {name: gotify-oauth2-proxy, key: cookie-secret}
-          ports:
-            - name: http
-              containerPort: 4180
-          readinessProbe:
-            httpGet: {path: /ping, port: http}
-            initialDelaySeconds: 5
-            periodSeconds: 10
-          livenessProbe:
-            httpGet: {path: /ping, port: http}
-            initialDelaySeconds: 15
-            periodSeconds: 20
-          securityContext:
-            allowPrivilegeEscalation: false
-            capabilities:
-              drop: ["ALL"]
-          resources:
-            requests: {cpu: 10m, memory: 32Mi}
-            limits: {cpu: 200m, memory: 128Mi}

k8s/apps/gotify/oauth2-proxy-ingress.yaml

@@ -1,22 +0,0 @@
-# The proxy needs its own ingress path. nginx auth_request also guards
-# /oauth2/start and /oauth2/callback, which would loop: the login entry point
-# would itself require a login. A longer path match sends /oauth2 to the proxy
-# directly, and the prefix rule on the gotify ingress handles everything else.
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: gotify-oauth2
-  namespace: notifications
-spec:
-  ingressClassName: nginx
-  rules:
-    - host: gotify.riotpiao.com
-      http:
-        paths:
-          - path: /oauth2
-            pathType: Prefix
-            backend:
-              service:
-                name: gotify-oauth2-proxy
-                port:
-                  number: 4180

k8s/apps/gotify/oauth2-proxy-service.yaml

@@ -1,15 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: gotify-oauth2-proxy
-  namespace: notifications
-  labels:
-    app: gotify-oauth2-proxy
-spec:
-  type: ClusterIP
-  selector:
-    app: gotify-oauth2-proxy
-  ports:
-    - name: http
-      port: 4180
-      targetPort: http

k8s/argocd/secrets/gotify-oauth2-proxy-secrets.enc.yaml

@@ -1,30 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-    name: gotify-oauth2-proxy
-    namespace: notifications
-type: Opaque
-data:
-    client-id: ENC[AES256_GCM,data:Q+XQAR6cfHKoiuxhPZs6khTvqT0A/JG6,iv:LqhfuhXOYPVB6FQpPpERzWRQ+nxgxHZo4qdJO/g52y8=,tag:V2eK2fyscQKmM3/MmDnNtg==,type:str]
-    client-secret: ENC[AES256_GCM,data:P7F+83CLDhHnTN+Qx3SeCFbCjVH4JHb2rSteCr3Dywcb1ysMSNvC+3HyBXzsfJJ/EErUrCPWuwlRmV5ySts1oIrWhKyWMJxNrUTzv4eoclat4RyV6SaNDg==,iv:ESO1OUIqZbZPEVUuHrqjskA4tuTX/sc5m+K6Eo319kI=,tag:2isdxjFbtosov88FHZdeaQ==,type:str]
-    cookie-secret: ENC[AES256_GCM,data:WoWiANSdju+Ep5MuZQNeT0BQdKf5cFvrDUZpe+4B0VeUmQqnvblFi3dciJOeBFom9xYYzUCAI2tUSmX+a6tdAObyoGvjdFmLh+WTfw==,iv:41gYYRkyDIqC3GPuZkoob81Wl2njFTY07j4Puzaq0s4=,tag:x6z33zZnFwvmWUQIVPcxdQ==,type:str]
-sops:
-    kms: []
-    gcp_kms: []
-    azure_kv: []
-    hc_vault: []
-    age:
-        - recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2UURWblY4aTBOQjFSNDRL
-            YkRXVjVHK1QvYW1iR2o0UWVmdE94c2lkWDJjCnlXK1k3cnZBNEExbDU3eDhQZ3FT
-            TVcyckFiWHdRSjZHQ201Nnl6UFdTN28KLS0tIExQeE90VnM1WHRCYkpycjJtR0ht
-            ZHFDNWhKeENSQzljZUNmdktDYWloUDQK/fOG8AjTb2TNqLbqNfArOs0mzInYNJVd
-            iaYxnjH6AjhbLKdnmCpecTQpgx8GeFGIkefV/x5pgrdSUwn0lcvpSQ==
-            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-10-27T19:05:35Z"
-    mac: ENC[AES256_GCM,data:KZV3JNebQ+DbZZZ+e3fvtOGckwC9fmkIchh8DBJ0DMq9lbrPEhP4Juordmnrrs9vJvGxuFJ/KwUv9EELqBsuSy8c47+z7flEAXJiJCJkvVPL/jMidABsFqMALvNNrCqBPWvMHLZUj762DwxB2gKhSx3GLfRUXhm+ZiNcD+39EVw=,iv:ifUlEq/GhBc/QomJApkCaWc3f4ggXoLsvqjzZElO0fU=,tag:fMo+HBDUZdRXgfck+8AJbA==,type:str]
-    pgp: []
-    encrypted_regex: ^(data|stringData)$
-    version: 3.9.3

k8s/argocd/secrets/grafana-gotify-bridge.enc.yaml

@@ -0,0 +1,28 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: grafana-gotify-bridge
+    namespace: notifications
+type: Opaque
+data:
+    app-token: ENC[AES256_GCM,data:MnkKT0/ubJctmLTFXjdCZjNlz+KgQLsPwIKCmPF2weEtMw6ieRfb3+B7OUlIiDqpktdPaJA/znsMOYPXT2aXDqyKquA=,iv:vcAGkgfTheoViZ7OzJJhbNf0boIbRBZLr60ieW5DhLk=,tag:iBDJEjGScxHopkNExh7jDw==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBYYWZYNFBHbVhRSkZoa3dH
+            dWp3WllyVmRPcExPTndZUlFsTG1YMHN6WW1jCjRiS1M2U0tIM0tCdGhJOUQ0c2JW
+            bXNTTExDeVVmcExuNStldFdhTHMrVTgKLS0tIDN3d3JKSEtIUUFhVHROOVBUaUNO
+            UUVGcVVkOVJmN294dE5DS1R1UkxiZzgKx0MEkIrFsO2dQwjY2410Nj0Nyy3QgC3L
+            T844m4WT+mxFwB69/y0luGkn4aXMS3xrHHWUC+vjAkM4Unp+MKhhrg==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2026-10-27T19:35:38Z"
+    mac: ENC[AES256_GCM,data:Gf5jnvWP+jBRc8fvQHyrnHnTDaaUTcXQgoOqm7Drax5FpBYEHa36pdKaUq+XR6VBd7ch6Ia/f7YavhS47ybgR3jn+w+trMxkpwtZhm5clw2Lo3Hm7Cvi4fG4qhcjm1C8/NOnWGsi5Qm+TJqyG509p9HNTvM99SzElRovPZldLL4=,iv:10hA6nq0WIO0a2I4ZW6uPGXPjesFgxrwmXohNO3FZz4=,tag:bJ37e8jvgEactq6T/pM2UQ==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.9.3

k8s/argocd/secrets/grafana-oidc-secrets.enc.yaml

@@ -1,23 +1,28 @@
-apiVersion: ENC[AES256_GCM,data:rGU=,iv:w1yJeQvN0MPZlPx8VQZmC6zkRaHW4rtjjROea0Kx9X4=,tag:qgYgVUGMQ2BtN0RZ0nZYjQ==,type:str]
-kind: ENC[AES256_GCM,data:wp2QP1he,iv:C8c/jKh5pnBOtHqXsTEchGf9DACU/5hOJ0Mp/+Jk1kg=,tag:QUZNeguRXdi3JO6Ov6GMDg==,type:str]
+apiVersion: v1
+kind: Secret
 metadata:
-    name: ENC[AES256_GCM,data:jVsnAcFTlRiKxDoU,iv:bWEqo28kH3bxzfI3ipiY6dsqhIcl9wIxBjFu0mxVGLg=,tag:PVWTveI8xhvLxvXPldhimQ==,type:str]
-    namespace: ENC[AES256_GCM,data:ZHnVZyd3wA==,iv:3evUnFTQfRz/ZilUQIl/9dsCba3nsX6o/iQsGXMBfgM=,tag:+9kp+VRu1ksroDTbTGmGPA==,type:str]
-type: ENC[AES256_GCM,data:bHX+Bu9o,iv:8qqNddsuw4YoHAuJfUcPIqjcqNySIh5Ln8OhsfZxuk8=,tag:P7QGUXo9Io2Y19YqlWv5sQ==,type:str]
+    name: grafana-oidc
+    namespace: logging
+type: Opaque
 stringData:
-    GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: ENC[AES256_GCM,data:nJz9pqa6w9ZGOjanZ32D38Bolg2YrdHs1crhb2uZkKyaaKDpklVb4FOKZ46K+NS1NXjoS6XDUAlV69CSvm1PeA==,iv:g9F/iLuzNC0WogtDJ/TAb3+97N6iGkbOTlb4SzYzwx8=,tag:4F3kUheGbtW2U6jgOvRO1A==,type:str]
+    GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: ENC[AES256_GCM,data:fGqA1ofVme1lVLxFjWIiU7sbvE9ip2Af7C1ua3+HdidlcVgkwW59vVlXZhgBRdoif86C8LKQSyoCGUqiO+740g==,iv:6Kv2ybbJg1GT8DLqz9P5y+E1zr7vcIBDMUErmvxNTaw=,tag:0wK28B3G/owCVPPqb2sLmQ==,type:str]
 sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
     age:
-        - enc: |
+        - recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+          enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrUkxLMEV3aEpILytic0Ux
-            V2VEV2EvcENJVnNNRUtBbGkwT0d6eUxYeEFvCjZPSkU2MG5qME5jZ01mRVJwTUFL
-            M3M0V3c3QXBoRml1Vm9NWW5JTlNRSG8KLS0tIEJnNlFmdFJXb1BPaVJzTmo1RDQv
-            VXZOVHRHUEZJRUZ4eGZGM1RxSURyWXcK3ZyYCPhRUvpvT/pjPQGJoLIwaktZY6Tg
-            LS0cIDUhjPFbeu7qUATOrat6vMUi0UREbXSFZ1KyAjNAwJZaUusPMA==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSArTXMxMWdVRHIvbWh1bVNk
+            bEFFNlgzNU9zTjlzY2o3VGdnbFpnU2lkYmxrCjF4aWVGSlJaNkcxODJ6RDkrVFo2
+            UkV2RUwwVHFCd0xMNG9RWXdTUHZGaXcKLS0tIHVaWjhaNHRtS2c0UkVzRk83ckhj
+            OCtBVFR0d1F5amVhV2E0emRZWThvYjAKFW46i9brIL9ZoO1CEn+IewIdoUhvYK/A
+            RqzZ6xV+mhlDj0zC+SpVcEm/Ki83nMYOj4+vQingZlr/Kr3Nwb0iLQ==
             -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    lastmodified: "2026-10-26T04:03:16Z"
-    mac: ENC[AES256_GCM,data:0cgVNtFMwhbEHBxu7T1XfstcQlgUTxijq9Mz36zC60E7kYXRNwGt2/0KoCaqaE0leypwAdrPn2OjeoDPeCN6x+n0vLRVX2tnli6HazBLbRsPHljDAV0ASF07iNhb0PvgtWay432Mm+sGATkel3ITdAaP0lISNLUYfEe1/TbYHkk=,iv:FZ9Bgtk7LOC3aOSOo1+50zwLGhlcUZYJaUQ9r05cAn0=,tag:tsLQ1frZ7li5l717CCtgoQ==,type:str]
-    unencrypted_suffix: _unencrypted
-    version: 3.13.2
+    lastmodified: "2026-10-27T19:44:32Z"
+    mac: ENC[AES256_GCM,data:EJew+p/kJrurcNpLlFxJGTl69hQYxn3IkzP7EDtdSriW/WI7X1xNLC2dp/dTiLomGmG1uh6+2JI9Sc8gXmrRAWLELAbiOaYbcoU6gi89BjTCWGDAMX4lNSIuH40rsGs4YtYj5U7SGfgx5qto4ZH7LxjBmxzPWM/Xz2Uq/I5jV4k=,iv:LdhjhwCfPUIa7D0gved8Twf7ujN+dg3lN8lCnFBFs0g=,tag:bjkwUCMtR8IBL9BkbuXcnw==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.9.3

k8s/argocd/secrets/igotify-client-token.enc.yaml

@@ -0,0 +1,28 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: igotify-client-token
+    namespace: notifications
+type: Opaque
+data:
+    client-token: ENC[AES256_GCM,data:W+InmDyZkU0i+OqQUsIaRNY/3p6zjMC19ASj/qYHTldhtGmHwj4koQX3tIxdCQV72xZKOxsc8eV3hGX3pqbW02c2/ng=,iv:wWI5AZdpxnzX61f8iTiWmxKcDFMaFM4PHFYo4DMBK4Q=,tag:CpE8Kg5V8fL7P3oAjWjSvQ==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwaUlFU0xKaEs0SWcxWEEr
+            T0tUSmErR08zVkhuLy9VMEc3VXdmU1o3cWdrCjhuOW1yTmZPdWxDZHhMeUZtQlcz
+            UmVzaEMyVHZOZS9KVHdMd2pndUJwUW8KLS0tIGxYeENwVEFDdUpDTGszdGtNRHBQ
+            Y041WW9HdTJwUlJCUGhMN2RGZWZpUmMKiQbi/O9X3cFJgqw3tvwWcyy8QKkUuq/7
+            Y+V51R/WFobEQuAZLmDKgWcPSGt51effufB/Vm9LRRqtbkbYQAIoXQ==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2026-10-27T19:14:20Z"
+    mac: ENC[AES256_GCM,data:Hxl7AGzMOMK0pwe6T9ZmrE6K45XhQkq4qk/7YkrVuneL1e7jc8Bmv4NlywqRkt9fPVOCMwKikeX9U3ZcsqK13xPPfm1CmAxV3CCluwK0/9589N70OQxL4qJSkO6CXyGaa0Y9wVRX4y/N3wSjW+d25+GOYUXrOxkRpVyU4GFSqAQ=,iv:0Af/MCZBEYJVI3yi7DpPX1eBKwSANXJPCpSAEE3zCq4=,tag:SS1U9TfRQR3z8KkoQ6ZuZA==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.9.3

k8s/argocd/secrets/secret-generator.yaml

@@ -36,7 +36,8 @@ files:
     - etcd-defrag-creds.enc.yaml
     - zitadel-masterkey.enc.yaml
     - smtp-gotify-secrets.enc.yaml
-    - gotify-oauth2-proxy-secrets.enc.yaml
+    - igotify-client-token.enc.yaml
+    - grafana-gotify-bridge.enc.yaml
     # Login UI RSA keypair, adopted from the chart-generated Secret so ArgoCD
     # owns it. Without this, the chart re-generates the keypair on every render,
     # which changes checksum/secret-login-service-key and leaves both Zitadel

k8s/infra/logging/grafana-values.yaml

@@ -209,3 +209,25 @@ sidecar:
       disableDelete: false
       foldersFromFilesStructure: true
     searchNamespace: ALL
+
+# ── Alerting ──────────────────────────────────────────────────────────────────
+# One contact point: Gotify. Gotify refuses Grafana's payload shape with
+# 400 "Field 'message' is required", so a bridge service translates the JSON
+# first (k8s/apps/gotify/bridge). Grafana posts to the bridge, the bridge posts
+# to /message with an app token.
+alerting:
+  contactpoints.yaml:
+    apiVersion: 1
+    contactPoints:
+      - orgId: 1
+        name: gotify
+        receivers:
+          - uid: gotify-webhook
+            type: webhook
+            disableResolveMessage: false
+            settings:
+              url: http://grafana-gotify-bridge.notifications.svc.cluster.local:8080/alert
+              httpMethod: POST
+              # 0 sends every alert in one request. The bridge names the extra
+              # count in the message body.
+              maxAlerts: 0

scripts/iam/zitadel-provision.py

@@ -122,11 +122,6 @@ OIDC_APPS = {
         "https://forgejo.riotpiao.com/user/oauth2/zitadel/callback",
         "http://localhost:3000/user/oauth2/authentik/callback",
         "https://forgejo.riotpiao.com/user/oauth2/authentik/callback"]},
-    # Gotify has no native OIDC, so oauth2-proxy sits in front of it and this app
-    # is the proxy's client. The callback path is the proxy's, not Gotify's.
-    "gotify": {**_WEB, "redirect_uris": [
-        "http://localhost:4180/oauth2/callback",
-        "https://gotify.riotpiao.com/oauth2/callback"]},
     "grafana": {**_WEB, "redirect_uris": [
         "http://localhost:3000/login/generic_oauth",
         "https://grafana.riotpiao.com/login/generic_oauth"]},
@@ -355,13 +350,16 @@ def _ensure_roles(token: str, project_id: str, roles: dict[str, str]) -> None:
             die(f"create role {key} failed: {code} {d}")
 
 
-def _ensure_oidc_apps(token: str, project_id: str, apps: dict[str, dict]) -> tuple[dict, dict]:
+def _ensure_oidc_apps(token: str, project_id: str, apps: dict[str, dict],
+                      known_secrets: dict[str, str] | None = None) -> tuple[dict, dict]:
     code, d = _api("POST", f"/management/v1/projects/{project_id}/apps/_search", token, {})
     existing = {a.get("name"): a for a in (d.get("result") or [])}
     ids: dict[str, str] = {}
     secrets: dict[str, str] = {}
+    known_secrets = known_secrets or {}
     for name, spec in apps.items():
         row = existing.get(name)
+        created = False
         if row:
             ids[name] = (row.get("oidcConfig") or {}).get("clientId", "")
             app_id = row.get("id")
@@ -384,8 +382,16 @@ def _ensure_oidc_apps(token: str, project_id: str, apps: dict[str, dict]) -> tup
                 die(f"create app {name} failed: {code} {d}")
             app_id = d["appId"]
             ids[name] = d.get("clientId", "")
+            created = True
         if spec.get("auth_method_id") == "OIDC_AUTH_METHOD_TYPE_NONE":
             continue
+        if not created:
+            # ZITADEL keeps one client secret per app, and the API cannot read it
+            # back. Generating a new one here would replace the deployed copy and
+            # break that app's login, so reuse the value already on disk.
+            if known_secrets.get(name):
+                secrets[name] = known_secrets[name]
+            continue
         code, d = _api("POST",
                        f"/management/v1/projects/{project_id}/apps/{app_id}"
                        "/oidc_config/_generate_client_secret", token, {})
@@ -487,7 +493,13 @@ def phase_apply(_args: argparse.Namespace) -> int:
     print(f"project   : {PROJECT_NAME} ({project_id})")
     _ensure_roles(token, project_id, ROLES)
     print(f"roles     : {len(ROLES)} ensured")
-    ids, secrets = _ensure_oidc_apps(token, project_id, OIDC_APPS)
+    known: dict[str, str] = {}
+    try:
+        known = (json.loads(STATE_FILE.read_bytes()).get("client_secrets", {})
+                 .get("value", {}) or {})
+    except (OSError, ValueError):
+        known = {}
+    ids, secrets = _ensure_oidc_apps(token, project_id, OIDC_APPS, known)
     print(f"oidc apps : {len(ids)} ensured, {len(secrets)} secrets issued")
 
     state = {