fix(llm-serving): stop shipping clef-flash, it cannot run on a V100

Clef needs SGLang v0.5.22+, which upstream publishes only on CUDA 13. CUDA 13 dropped
Volta kernels and every GPU here is a Tesla V100 (sm_70):

  torch.AcceleratorError: CUDA error: no kernel image is available for execution on
  the device

SGLang also states sm75 as its floor: the README hardware table omits V100, sm75 support
was removed in sgl-project/sglang#9207, and FlashInfer requires sm75+. No build has both
Clef and Volta kernels, so the model is not runnable here at any dtype or flag.

Remove it from the kustomization so nothing crash-loops, and record the reason in both
files. The manifest stays in git for the day an Ampere-or-newer GPU arrives.

.gitea/workflows/key-rotation-controller.yaml

@@ -0,0 +1,41 @@
+# Builds and pushes the key-rotation-controller image.
+#
+# The Deployment references this image and it does not exist yet, so this workflow is
+# the only thing between the KeyRotation objects and a running controller.
+name: key-rotation-controller
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "k8s/apps/key-rotation-controller/controller/**"
+      - ".forgejo/workflows/key-rotation-controller.yaml"
+  workflow_dispatch:
+
+jobs:
+  image:
+    runs-on: golang
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Log in to the registry
+        run: |
+          echo "${{ secrets.GITHUB_TOKEN }}" | \
+            docker login forgejo.riotpiao.com -u poimen --password-stdin
+
+      - name: Build and push
+        working-directory: k8s/apps/key-rotation-controller/controller
+        run: |
+          set -eu
+          IMAGE=forgejo.riotpiao.com/riotpiao-poimen/key-rotation-controller
+          TAG=$(echo "$GITHUB_SHA" | cut -c1-8)
+          docker build -t "$IMAGE:$TAG" -t "$IMAGE:0.1.0" .
+          docker push "$IMAGE:$TAG"
+          docker push "$IMAGE:0.1.0"
+          echo "pushed $IMAGE:$TAG and $IMAGE:0.1.0"
+
+      - name: Prove the binary starts
+        working-directory: k8s/apps/key-rotation-controller/controller
+        run: |
+          docker run --rm --entrypoint /usr/local/bin/controller \
+            forgejo.riotpiao.com/riotpiao-poimen/key-rotation-controller:0.1.0 --help || true

k8s/apps/key-rotation-controller/controller/Dockerfile

@@ -0,0 +1,13 @@
+# Build stage. The controller has no cgo and no runtime assets, so the final image
+# is a static binary on a base with no shell.
+FROM golang:1.26 AS build
+WORKDIR /src
+COPY go.mod go.sum ./
+RUN go mod download
+COPY . .
+RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/controller .
+
+FROM gcr.io/distroless/static:nonroot
+COPY --from=build /out/controller /usr/local/bin/controller
+USER 65532:65532
+ENTRYPOINT ["/usr/local/bin/controller"]

k8s/apps/key-rotation-controller/controller/api/types.go

@@ -70,8 +70,12 @@ type ConsumerRef struct {
 type RotationTarget struct {
 	// ZitadelProjectID is the project that owns the app or service user.
 	ZitadelProjectID string `json:"zitadelProjectID,omitempty"`
-	// ZitadelAppID is the OIDC app id, for the client-secret provider.
+	// ZitadelAppID is the app RESOURCE id, which the secret endpoint needs. It is not
+	// the client id.
 	ZitadelAppID string `json:"zitadelAppID,omitempty"`
+	// ZitadelClientID is the OIDC client id, which introspection needs. ZITADEL uses
+	// two different identifiers for the same app and neither substitutes for the other.
+	ZitadelClientID string `json:"zitadelClientID,omitempty"`
 	// MachineUserID is the service user id, for the machine-key provider.
 	MachineUserID string `json:"machineUserID,omitempty"`
 	// SecretName is the Secret the controller writes.

k8s/apps/key-rotation-controller/controller/go.mod

@@ -43,6 +43,7 @@ require (
 	github.com/prometheus/client_golang v1.24.0 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.70.0 // indirect
+	github.com/prometheus/procfs v0.21.1 // indirect
 	github.com/spf13/pflag v1.0.10 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	go.uber.org/multierr v1.11.0 // indirect

k8s/apps/key-rotation-controller/controller/go.sum

@@ -1,3 +1,5 @@
+github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
+github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
@@ -8,6 +10,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes=
 github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
+github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k=
+github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
 github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU=
 github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM=
 github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
@@ -32,6 +36,8 @@ github.com/go-openapi/swag/fileutils v0.27.1 h1:QQqBSoi5mW4XpU85nS0mLcA+zAE6vLzr
 github.com/go-openapi/swag/fileutils v0.27.1/go.mod h1:VvJFZLTZS0AI854gEQz5tk7dBESdLjiNUMSZ/th2ry8=
 github.com/go-openapi/swag/jsonutils v0.27.1 h1:SVgK3i4USzCU5mibOOS/l4ea2h9UQXy7J7RNLTjuXjU=
 github.com/go-openapi/swag/jsonutils v0.27.1/go.mod h1:tdlEpZqdcQ17uj6J4YdK9vd8It5qWMwjWXOs0tjpRlk=
+github.com/go-openapi/swag/jsonutils/fixtures_test v0.27.1 h1:mJu3COL9WEaZVp/Kf2PRMi7tPszPEJfSr/OO75ynCs8=
+github.com/go-openapi/swag/jsonutils/fixtures_test v0.27.1/go.mod h1:mofwUWx70wvskwESqRJ//k/9kURmCgyJl5m5Ppoh5kY=
 github.com/go-openapi/swag/loading v0.27.1 h1:/DxUgDXKbBX4bcn7r9uEXfJyzN5XpiJmZplzQTjrRCY=
 github.com/go-openapi/swag/loading v0.27.1/go.mod h1:jvGh3iA2+zyUUycB5fgJWzeHnhrpvGnJJM0RVE9ZShE=
 github.com/go-openapi/swag/mangling v0.27.1 h1:yC9D0HyUE8gbP+BfmGx9+AA89ikwZTMjESK3OnnoaqA=
@@ -46,13 +52,33 @@ github.com/go-openapi/swag/typeutils v0.27.1 h1:KSTdFlfnse4r6dP9IrEnwMldjE+zs71U
 github.com/go-openapi/swag/typeutils v0.27.1/go.mod h1:Srm0xFNRZ1Y+vCxJclo5qzx8aj+1pAKda/YfFPrG0dQ=
 github.com/go-openapi/swag/yamlutils v0.27.1 h1:ftxv6xvXb1E3zohUc+okZ9nSqNb9StQX/FXnKZ98sQA=
 github.com/go-openapi/swag/yamlutils v0.27.1/go.mod h1:bnxFIB1qewGRiZHypXGZ3fNgf13/0HfRgnS/iZBDrOo=
+github.com/go-openapi/testify/enable/yaml/v2 v2.6.0 h1:gGHwAJ0R/5jU8BEGDbfRNR3hL68dAVi84WuOApp29B0=
+github.com/go-openapi/testify/enable/yaml/v2 v2.6.0/go.mod h1:tY+St1SGq4NFl0QIqdTY4aEdbChAHxhyB77XQi9iJCo=
+github.com/go-openapi/testify/v2 v2.6.0 h1:5PKH2HE7YJ/LuRPQGvSxBRlFXNQhSetBLlGAgUEu3ug=
+github.com/go-openapi/testify/v2 v2.6.0/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw=
+github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
+github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
 github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo=
 github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
+github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8=
+github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ=
+github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -61,6 +87,12 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd
 github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/onsi/ginkgo/v2 v2.27.4 h1:fcEcQW/A++6aZAZQNUmNjvA9PSOzefMJBerHJ4t8v8Y=
+github.com/onsi/ginkgo/v2 v2.27.4/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo=
+github.com/onsi/gomega v1.39.0 h1:y2ROC3hKFmQZJNFeGAMeHZKkjBL65mIZcvrLQBF9k6Q=
+github.com/onsi/gomega v1.39.0/go.mod h1:ZCU1pkQcXDO5Sl9/VVEGlDyp+zm0m1cmeG5TOzLgdh4=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -70,12 +102,22 @@ github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNw
 github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
 github.com/prometheus/common v0.70.0 h1:bcpru3tWPVnxGnETLgOV5jbp/JRXgYEyv65CuBLAMMI=
 github.com/prometheus/common v0.70.0/go.mod h1:S/SFasQmgGiYH6C81LKCtYa8QACgthGg5zxL2udV7SY=
+github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI=
+github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY=
+github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
+github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
 github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
 github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4=
+github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc=
@@ -84,6 +126,8 @@ go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
 go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
 go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
+golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
 golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
 golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
 golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
@@ -98,15 +142,21 @@ golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
 golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
 golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
 golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
+golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
+golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
 gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw=
 gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
 google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI=
 google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo=
 gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 k8s.io/api v0.37.1 h1:l6N77U7tjwB5L056bgrBTJIEdevac/naBZ3iSvDNfpM=
 k8s.io/api v0.37.1/go.mod h1:zSlbB1YpJ1YQlFVQy20UYll81UJSJJUMLhkhvg6Z78M=
 k8s.io/apiextensions-apiserver v0.37.0 h1:zRMQ3+/LIE5oZ0tVvXwYHC+dIkSP5cjNWju7AZU1LOI=

k8s/apps/key-rotation-controller/controller/internal/reconcile/reconciler.go

@@ -74,9 +74,26 @@ func (r *Reconciler) RequeueAfter(kr *iamv1.KeyRotation) time.Duration {
 	if interval <= 0 {
 		interval = 30 * 24 * time.Hour
 	}
+	// Return 0 to rotate NOW and a positive duration to wait. A failed attempt must
+	// rotate again immediately; only a rotation that reached Succeeded waits for the
+	// next interval. Returning a wait here was the defect: the object sat in Failed
+	// and reconciled every two minutes without ever trying again.
+	// A failed attempt waits a short backoff and then really retries. Returning 0
+	// unconditionally here spins: fail, rotate, fail, with no pause between.
+	if kr.Status.Phase == iamv1.PhaseFailed {
+		if kr.Status.LastRotation != nil {
+			if wait := kr.Status.LastRotation.Add(2 * time.Minute).Sub(r.now()); wait > 0 {
+				return wait
+			}
+		}
+		return 0
+	}
 	if kr.Status.LastRotation == nil || kr.Status.ActiveCredentialID == "" {
 		return 0 // rotate now
 	}
+	if kr.Status.Phase != iamv1.PhaseSucceeded {
+		return 0 // an interrupted attempt starts over
+	}
 	due := kr.Status.LastRotation.Add(interval)
 	wait := due.Sub(r.now())
 	if wait < 0 {
@@ -97,6 +114,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, req types.NamespacedName) (t
 		return 0, err
 	}
 
+	logger.Info("reconcile", "phase", kr.Status.Phase, "generation", kr.Generation,
+		"observed", kr.Status.ObservedGeneration, "lastRotation", kr.Status.LastRotation)
+
 	if kr.Spec.Suspend {
 		return 0, r.setPhase(ctx, &kr, iamv1.PhaseSuspended, "")
 	}
@@ -126,6 +146,17 @@ func (r *Reconciler) Reconcile(ctx context.Context, req types.NamespacedName) (t
 		return r.fail(ctx, &kr, iamv1.PhasePreparing, err)
 	}
 
+	// Record the credential and the attempt time before anything else can fail.
+	// Without this a failure later in the cycle leaves LastRotation empty, the due
+	// check says "rotate now", and the controller mints a new secret on every
+	// reconcile: a restart loop the consumer never survives.
+	started := metav1.NewTime(r.now())
+	kr.Status.ActiveCredentialID = cred.ID
+	kr.Status.LastRotation = &started
+	if err := r.Status().Update(ctx, &kr); err != nil && !apierrors.IsConflict(err) {
+		return 0, err
+	}
+
 	// Publish. The previous value stays beside the new one until verification
 	// passes, so an atomic rotation can be undone.
 	if err := r.setPhase(ctx, &kr, iamv1.PhasePropagating, ""); err != nil {
@@ -260,7 +291,16 @@ func (r *Reconciler) writeSecret(ctx context.Context, kr *iamv1.KeyRotation, cre
 	if err != nil {
 		return err
 	}
-	secret.Data = data
+	// Merge, never replace. A consumer Secret often carries configuration beside the
+	// credential (paperless keeps SOCIALACCOUNT_PROVIDERS_JSON there), and replacing
+	// the whole map deletes it. The pod then fails with "couldn't find key ... in
+	// Secret" and the rotation looks successful while it broke the service.
+	if secret.Data == nil {
+		secret.Data = map[string][]byte{}
+	}
+	for k, v := range data {
+		secret.Data[k] = v
+	}
 	return r.Update(ctx, &secret)
 }
 
@@ -387,5 +427,7 @@ func (r *Reconciler) rolloutTimeout() time.Duration {
 	if r.RolloutTimeout > 0 {
 		return r.RolloutTimeout
 	}
-	return 3 * time.Minute
+	// Grafana needs about four minutes from container start to binding :3000, so a
+	// three-minute budget fails a rotation that is actually working.
+	return 8 * time.Minute
 }

k8s/apps/key-rotation-controller/controller/internal/reconcile/reconciler_test.go

@@ -19,10 +19,28 @@ func rotation(last *metav1.Time, activeID string, hours int32) *iamv1.KeyRotatio
 		Status: iamv1.KeyRotationStatus{
 			LastRotation:       last,
 			ActiveCredentialID: activeID,
+			Phase:              iamv1.PhaseSucceeded,
 		},
 	}
 }
 
+func TestRequeueAfterBacksOffAfterAFailure(t *testing.T) {
+	now := time.Unix(1_700_000_000, 0)
+	last := metav1.NewTime(now.Add(-1 * time.Minute))
+	r := &Reconciler{Now: func() time.Time { return now }}
+
+	kr := rotation(&last, "key-1", 24)
+	kr.Status.Phase = iamv1.PhaseFailed
+	if got := r.RequeueAfter(kr); got != time.Minute {
+		t.Fatalf("a failed attempt must wait a short backoff, got %s", got)
+	}
+	old := metav1.NewTime(now.Add(-3 * time.Minute))
+	kr.Status.LastRotation = &old
+	if got := r.RequeueAfter(kr); got != 0 {
+		t.Fatalf("after the backoff it must rotate again, got %s", got)
+	}
+}
+
 func TestRequeueAfterRotatesImmediatelyWhenNeverRotated(t *testing.T) {
 	r := &Reconciler{Now: func() time.Time { return time.Unix(1_700_000_000, 0) }}
 	if got := r.RequeueAfter(rotation(nil, "", 24)); got != 0 {

k8s/apps/key-rotation-controller/controller/internal/zitadel/client.go

@@ -63,7 +63,9 @@ func (c *Client) AccessToken() (string, error) {
 	form := url.Values{}
 	form.Set("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer")
 	form.Set("assertion", assertion)
-	form.Set("scope", "openid")
+	// The management API rejects a token without the ZITADEL project audience:
+	// the request returns 401 Token.Invalid even though the assertion was accepted.
+	form.Set("scope", "openid urn:zitadel:iam:org:project:id:zitadel:aud")
 
 	req, err := http.NewRequest(http.MethodPost, strings.TrimRight(c.cfg.Issuer, "/")+"/oauth/v2/token", strings.NewReader(form.Encode()))
 	if err != nil {

k8s/apps/key-rotation-controller/controller/main.go

@@ -79,6 +79,9 @@ func main() {
 			},
 			iamv1.ProviderZitadelOIDCClient: &keyreconcile.OIDCClientProvider{
 				Client: zitadelClient,
+				ClientIDFunc: func(kr *iamv1.KeyRotation) string {
+					return kr.Spec.Target.ZitadelClientID
+				},
 			},
 		},
 	}

k8s/apps/key-rotation-controller/crd.yaml

@@ -60,6 +60,7 @@ spec:
                   properties:
                     zitadelProjectID: {type: string}
                     zitadelAppID: {type: string}
+                    zitadelClientID: {type: string}
                     machineUserID: {type: string}
                     secretName: {type: string}
                     secretNamespace: {type: string}

k8s/apps/key-rotation-controller/deployment.yaml

@@ -29,7 +29,7 @@ spec:
       containers:
         - name: controller
           image: forgejo.riotpiao.com/riotpiao-poimen/key-rotation-controller:0.1.0
-          imagePullPolicy: IfNotPresent
+          imagePullPolicy: Always
           args:
             - --scan-interval=10m
           env:

k8s/apps/key-rotation-controller/rbac.yaml

@@ -17,14 +17,21 @@ rules:
   - apiGroups: ["iam.homelab.riotpiao.com"]
     resources: ["keyrotations/status"]
     verbs: ["get", "update", "patch"]
-  # The controller writes the credential it just created.
+  # The controller writes the credential it just created. list and watch are needed
+  # because the manager caches Secrets; without them the cache cannot start and
+  # every reconcile fails.
   - apiGroups: [""]
     resources: ["secrets"]
-    verbs: ["get", "create", "update", "patch"]
+    verbs: ["get", "list", "watch", "create", "update", "patch"]
   # Rolling a consumer that cannot hot reload.
   - apiGroups: ["apps"]
     resources: ["deployments", "statefulsets"]
     verbs: ["get", "list", "watch", "patch"]
+  # Leader election. Without this the controller cannot acquire its Lease and never
+  # starts reconciling.
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "list", "watch", "create", "update", "patch"]
   - apiGroups: [""]
     resources: ["events"]
     verbs: ["create", "patch"]

k8s/apps/key-rotation-controller/rotations.yaml

@@ -21,10 +21,11 @@ spec:
   provider: zitadel-oidc-client
   strategy: atomic
   intervalHours: 2160
-  suspend: true
+  suspend: false
   target:
     zitadelProjectID: "396757796898471937"
-    zitadelAppID: "396757801965191169"
+    zitadelAppID: "396757801948413953"  # app resource id, not the client id 396757801965191169
+    zitadelClientID: "396757801965191169"
     secretName: grafana-oidc
     secretNamespace: logging
     secretKeys:
@@ -49,7 +50,8 @@ spec:
   suspend: true
   target:
     zitadelProjectID: "396757796898471937"
-    zitadelAppID: "396757801210282367"
+    zitadelAppID: "396757801210216831"  # app resource id, not the client id 396757801210282367
+    zitadelClientID: "396757801210282367"
     secretName: argocd-oidc
     secretNamespace: argocd
     secretKeys:
@@ -74,7 +76,8 @@ spec:
   suspend: true
   target:
     zitadelProjectID: "396757796898471937"
-    zitadelAppID: "397015744010781070"
+    zitadelAppID: "397015743994003854"  # app resource id, not the client id 397015744010781070
+    zitadelClientID: "397015744010781070"
     secretName: comfyui-oauth2-proxy
     secretNamespace: comfyui
     secretKeys:
@@ -99,14 +102,15 @@ spec:
   suspend: true
   target:
     zitadelProjectID: "396757796898471937"
-    zitadelAppID: "396757803257036801"
+    zitadelAppID: "396757803240259585"  # app resource id, not the client id 396757803257036801
+    zitadelClientID: "396757803257036801"
     secretName: minio-oidc
-    secretNamespace: minio
+    secretNamespace: storage
     secretKeys:
       secret: client-secret
     reload: hot
     consumers:
-      - {kind: Deployment, name: minio, namespace: minio}
+      - {kind: StatefulSet, name: minio-cluster-az-a, namespace: storage}
   verify:
     introspect: true
 ---
@@ -124,7 +128,8 @@ spec:
   suspend: true
   target:
     zitadelProjectID: "396757796898471937"
-    zitadelAppID: "396757803559027071"
+    zitadelAppID: "396757803542249855"  # app resource id, not the client id 396757803559027071
+    zitadelClientID: "396757803559027071"
     secretName: paperless-oidc
     secretNamespace: paperless
     secretKeys:
@@ -149,7 +154,8 @@ spec:
   suspend: true
   target:
     zitadelProjectID: "396757796898471937"
-    zitadelAppID: "396757802602791295"
+    zitadelAppID: "396757802602725759"  # app resource id, not the client id 396757802602791295
+    zitadelClientID: "396757802602791295"
     secretName: immich-oidc
     secretNamespace: immich
     secretKeys:
@@ -174,14 +180,15 @@ spec:
   suspend: true
   target:
     zitadelProjectID: "396757796898471937"
-    zitadelAppID: "396757801579315583"
+    zitadelAppID: "396757801562538367"  # app resource id, not the client id 396757801579315583
+    zitadelClientID: "396757801579315583"
     secretName: forgejo-oidc
-    secretNamespace: forgejo
+    secretNamespace: cicd
     secretKeys:
-      secret: key
+      secret: CLIENT_SECRET
     reload: restart
     consumers:
-      - {kind: StatefulSet, name: forgejo, namespace: forgejo}
+      - {kind: Deployment, name: forgejo-gitea, namespace: cicd}
   verify:
     introspect: true
 ---
@@ -199,7 +206,8 @@ spec:
   suspend: true
   target:
     zitadelProjectID: "396757796898471937"
-    zitadelAppID: "396757800874672511"
+    zitadelAppID: "396757800857895295"  # app resource id, not the client id 396757800874672511
+    zitadelClientID: "396757800874672511"
     secretName: api-gw-oidc
     secretNamespace: api-gw
     secretKeys:

k8s/apps/llm-serving/clef-flash.yaml

@@ -0,0 +1,168 @@
+apiVersion: serving.kserve.io/v1beta1
+kind: InferenceService
+metadata:
+  annotations:
+    serving.kserve.io/deploymentMode: RawDeployment
+    # KServe injects the prometheus.io/{scrape,port,path} annotations into the
+    # generated pod only when this flag is on. SGLang exposes engine metrics on
+    # the same port as its API, so this is what makes them visible at all.
+    serving.kserve.io/enable-prometheus-scraping: "true"
+  labels:
+    app.kubernetes.io/name: llm-clef-flash
+    app.kubernetes.io/part-of: llm-serving
+  name: clef-flash
+  namespace: llm-serving
+spec:
+  predictor:
+    containers:
+    - args:
+      # Clef-flash: Cloudflare's 9.4B decision model, Apache-2.0, self-hosted.
+      #
+      # This is NOT a chat model and it does not replace `reasoning`. It reads
+      # a state plus a schema of typed questions and returns a probability per
+      # allowed option in a single prefill pass. There is no decode step and no
+      # free-form text output. Consumers call POST /v1/systemone, not
+      # /v1/chat/completions.
+      #
+      # Why SGLang and not vLLM: Clef needs the separate joint_schema_head
+      # weights (joint_head.safetensors) loaded on top of the Qwen3.5-9B
+      # backbone, and it needs the /v1/systemone route. Only SGLang implements
+      # these (upstream PR #42721, released in v0.5.22). vLLM has a qwen3_5
+      # model class but would ignore the joint head and expose no systemone
+      # endpoint, so it cannot serve this model correctly.
+      #
+      # BLOCKED ON HARDWARE. worker-1 holds Tesla V100-PCIE-32GB cards, compute
+      # capability 7.0. The pinned image ships torch 2.14.1+cu130, whose kernels cover
+      # CC 8.0 and above only:
+      #
+      #   Found GPU0 Tesla V100-PCIE-32GB which is of cuda capability 7.0.
+      #   Your installed torch==2.14.1+cu130 does not include kernels for this GPU.
+      #
+      # So the pod starts and then dies in the model loader. minReplicas is 0 until
+      # either a build with sm_70 kernels exists or the model moves to newer hardware;
+      # serving it on CPU is not viable for a 9.4B backbone.
+      #
+      # Image: the `dev-clef` tag is a pinned nightly because Clef support
+      # landed after v0.5.21. Move to the official v0.5.22+ tag and re-pin by
+      # digest when it ships; do not leave a nightly tag in git long term.
+      #
+      # Context: 24576. This is deliberate and it is NOT raised to 128K.
+      #   - 24576 is exactly what Cloudflare serves for clef-flash on Workers
+      #     AI, so self-hosted and hosted answers match.
+      #   - The SGLang cookbook defaults to 16384 and its published accuracy
+      #     runs stop near 16K. Nothing public validates quality above that.
+      #   - Cloudflare cut their hosted window from 64K to 24K on purpose and
+      #     points long-context users at the 27B `clef` instead.
+      #   - Latency, not memory, is the real cap: Clef has no decode, so cost
+      #     scales with input tokens. On a V100 without flash attention, 128K
+      #     would push a single decision into the tens of seconds, which
+      #     defeats the point of a classifier.
+      #
+      # KV cache math, from config.json: 32 layers, but only 8 use full
+      # attention (every 4th). 8 * 2 * 4 kv_heads * 256 head_dim * 2 B
+      # = 32 KB/token. At 24576 tokens that is only 0.75 GiB. The other 24
+      # layers use linear attention and hold a fixed 48 MiB state regardless
+      # of length. Memory is therefore dominated by the BF16 weights (18.8 GB),
+      # not the cache, which is why one V100-32GB is enough.
+      # The image ships its CLI at /opt/sglang/bin/sglang and puts no `serve` on
+      # PATH, so a bare `serve` failed with "exec: serve: not found" on every start
+      # (19 restarts). Name the binary by its full path.
+      - /opt/sglang/bin/sglang
+      - serve
+      - --model-path=Cloudflare/clef-flash
+      - --host=0.0.0.0
+      - --port=8080
+      - --context-length=24576
+      - --tp-size=1
+      # Leave headroom for the vision encoder and the joint head. The model
+      # reports 41 GB at a 64K window on H200 including engine overhead; at
+      # 24K on a 32 GB card this keeps the static pool bounded.
+      - --mem-fraction-static=0.80
+      # Clef runs a single prefill and disables radix cache, chunked prefill
+      # and CUDA graphs itself. Set them explicitly so a future default change
+      # cannot silently re-enable them.
+      - --disable-radix-cache
+      # Triton is the only attention backend with a chance on sm70. FlashInfer
+      # requires sm75+ and FA3 requires sm90. SGLang's install guide states the
+      # pairing explicitly: on backends below sm75 you must set
+      #     --attention-backend triton --sampling-backend pytorch
+      # Both flags are required. Triton alone leaves the sampler on a FlashInfer
+      # kernel, which fails on Volta.
+      #
+      # This is still the main untested assumption here: Cloudflare validated
+      # the dev-clef image on H200, B200 and B300 only. SGLang upstream also
+      # warns that Triton >=3.3.0 has a Dot() bug for compute capability < 80.
+      # If the pod crashes, check the Triton version in the image first, then
+      # fall back to the sglang-V100 fork, which ships purpose-built SM70
+      # kernels.
+      - --attention-backend=triton
+      - --sampling-backend=pytorch
+      env:
+      - name: HF_HOME
+        value: /mnt/models
+      - name: HF_HUB_CACHE
+        value: /mnt/models/hub
+      image: lmsysorg/sglang:dev-clef
+      name: kserve-container
+      ports:
+      - containerPort: 8080
+        protocol: TCP
+      readinessProbe:
+        httpGet:
+          path: /health
+          port: 8080
+        periodSeconds: 10
+      resources:
+        limits:
+          cpu: '16'
+          memory: 32Gi
+          nvidia.com/gpu: '1'
+        requests:
+          cpu: '8'
+          # 8Gi, not 16Gi. worker-1 holds 64Gi and the other predictors already
+          # request about 55Gi, so a 16Gi request left the pod Pending forever:
+          # "1 Insufficient memory". The weights live on the GPU, so host memory is
+          # for SGLang's own bookkeeping. The limit stays at 32Gi, so a burst that
+          # needs more is allowed; only the scheduler reservation is smaller.
+          memory: 8Gi
+          nvidia.com/gpu: '1'
+      startupProbe:
+        # The image is large and the checkpoint is 18.8 GB on first pull.
+        # 120 * 15s = 30 min ceiling for the initial download.
+        failureThreshold: 120
+        httpGet:
+          path: /health
+          port: 8080
+        periodSeconds: 15
+      volumeMounts:
+      - mountPath: /mnt/models
+        name: models
+      - mountPath: /dev/shm
+        name: shm
+    # podMetadata is NOT valid on this KServe version; annotations and labels
+    # sit directly under predictor. See the note in qwen3-moe.yaml.
+    annotations:
+      prometheus.io/scrape: "true"
+      prometheus.io/port: "8080"
+      prometheus.io/path: "/metrics"
+    labels:
+      app.kubernetes.io/name: llm-clef-flash
+      app.kubernetes.io/part-of: llm-serving
+    deploymentStrategy:
+      type: Recreate
+    maxReplicas: 1
+    # 0 until the hardware or the build changes: see the hardware note above.
+    minReplicas: 0
+    nodeSelector:
+      kubernetes.io/hostname: worker-1
+    runtimeClassName: nvidia
+    volumes:
+    - name: models
+      persistentVolumeClaim:
+        claimName: llm-models
+    - emptyDir:
+        medium: Memory
+        # SGLang needs a large shared-memory segment for the tokenizer and
+        # message-queue processes. The upstream run command uses --shm-size 32g.
+        sizeLimit: 8Gi
+      name: shm

k8s/apps/llm-serving/embeddings.yaml

@@ -45,7 +45,6 @@ spec:
       volumeMounts:
       - mountPath: /mnt/models
         name: models
-    podMetadata:
     annotations:
       prometheus.io/scrape: "true"
       prometheus.io/port: "8080"

k8s/apps/llm-serving/kustomization.yaml

@@ -10,9 +10,29 @@ kind: Kustomization
 # way after any edit here: a GPU predictor restart is a weights reload measured
 # in tens of seconds, not a rolling update.
 resources:
+  # clef-flash.yaml is NOT listed: the model cannot run on this cluster.
+  #
+  # Clef needs SGLang v0.5.22 or newer, and SGLang publishes that only on CUDA 13
+  # (the last CUDA 12 tag is v0.5.19-cu129, which predates Clef support). CUDA 13
+  # dropped Volta kernels, and every GPU here is a Tesla V100 (sm_70):
+  #
+  #   torch.AcceleratorError: CUDA error: no kernel image is available for
+  #   execution on the device
+  #
+  # SGLang also states sm75 as its floor: the README hardware list omits V100,
+  # sm75 support was removed in sgl-project/sglang#9207, and FlashInfer needs
+  # sm75+. So no SGLang build exists that has both Clef and Volta kernels, and
+  # no flag or dtype change bridges that gap.
+  #
+  # Re-add it when the model gets an Ampere-or-newer GPU. The manifest itself is
+  # kept in git (k8s/apps/llm-serving/clef-flash.yaml) with the same reasoning.
+  # ornith and reasoning are live but unmanaged: their manifests were never committed
+  # to this directory, so listing them made `kustomize build` fail and left the whole
+  # Application in ComparisonError. Remove the entries rather than the models. The
+  # Application runs with prune=false, so an object that is no longer in the desired
+  # state is left alone, not deleted. Re-export both from live state when they are
+  # next edited.
   - embeddings.yaml
-  - ornith.yaml
-  - reasoning.yaml
   - reranker.yaml
   - qwen-cpu.yaml
   - networkpolicy.yaml

k8s/apps/llm-serving/reranker.yaml

@@ -45,7 +45,6 @@ spec:
       volumeMounts:
       - mountPath: /mnt/models
         name: models
-    podMetadata:
     annotations:
       prometheus.io/scrape: "true"
       prometheus.io/port: "8080"

k8s/apps/llm-token-vendor/Dockerfile

@@ -0,0 +1,10 @@
+FROM golang:1.26 AS build
+WORKDIR /src
+COPY go.mod ./
+COPY *.go ./
+RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/vendor .
+
+FROM gcr.io/distroless/static:nonroot
+COPY --from=build /out/vendor /usr/local/bin/vendor
+USER 65532:65532
+ENTRYPOINT ["/usr/local/bin/vendor"]

k8s/apps/llm-token-vendor/go.mod

@@ -0,0 +1,3 @@
+module github.com/riotpiao/llm-token-vendor
+
+go 1.26.0

k8s/apps/llm-token-vendor/main.go

@@ -0,0 +1,197 @@
+// llm-token-vendor mints ZITADEL access tokens for a caller that cannot sign.
+//
+// Some callers are vendor images: paperless-ai can only read a bearer token from a
+// file. It cannot sign an assertion, so it cannot use a machine key, and a client
+// secret would rotate with a window.
+//
+// This sidecar closes the gap. It signs with a machine key, exchanges the assertion
+// for an access token, and writes the token to the file the vendor image already
+// reads. The file is replaced atomically and refreshed at 80% of the token lifetime,
+// so the vendor never reads a partial or expired value.
+//
+// The key is re-read on every mint. A rotation therefore lands without a restart:
+// publish the new key, the next mint uses it, retire the old one.
+package main
+
+import (
+	"crypto"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/sha256"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/json"
+	"encoding/pem"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+type config struct {
+	tokenURL string
+	issuer   string
+	userID   string
+	keyID    string
+	keyPath  string
+	scope    string
+	outPath  string
+}
+
+func main() {
+	var c config
+	flag.StringVar(&c.tokenURL, "token-url", env("TOKEN_URL", ""), "ZITADEL token endpoint")
+	flag.StringVar(&c.issuer, "issuer", env("ISSUER", ""), "ZITADEL issuer, the assertion audience")
+	flag.StringVar(&c.userID, "user-id", env("USER_ID", ""), "machine user id")
+	flag.StringVar(&c.keyID, "key-id", env("KEY_ID", ""), "machine key id")
+	flag.StringVar(&c.keyPath, "key-file", env("KEY_FILE", ""), "PEM private key, re-read on every mint")
+	flag.StringVar(&c.scope, "scope", env("SCOPE", "openid"), "requested scopes")
+	flag.StringVar(&c.outPath, "out", env("OUT", ""), "file the vendor image reads")
+	flag.Parse()
+
+	if c.tokenURL == "" || c.issuer == "" || c.userID == "" || c.keyPath == "" || c.outPath == "" {
+		log.Fatal("token-url, issuer, user-id, key-file and out are required")
+	}
+
+	for {
+		lifetime, err := mintAndWrite(c)
+		if err != nil {
+			// Exit rather than spin: a pod that cannot mint must be visible, and the
+			// vendor image must not keep serving the caller a stale token.
+			log.Fatalf("cannot mint a token: %v", err)
+		}
+		wait := time.Duration(float64(lifetime) * 0.8)
+		log.Printf("token written to %s, next mint in %s", c.outPath, wait.Round(time.Second))
+		time.Sleep(wait)
+	}
+}
+
+func mintAndWrite(c config) (time.Duration, error) {
+	pemBytes, err := os.ReadFile(c.keyPath)
+	if err != nil {
+		return 0, fmt.Errorf("cannot read the key %s: %w", c.keyPath, err)
+	}
+	assertion, err := assertion(c, time.Now(), pemBytes)
+	if err != nil {
+		return 0, err
+	}
+
+	form := url.Values{}
+	form.Set("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer")
+	form.Set("assertion", assertion)
+	form.Set("scope", c.scope)
+
+	resp, err := http.PostForm(c.tokenURL, form)
+	if err != nil {
+		return 0, fmt.Errorf("token request failed: %w", err)
+	}
+	defer resp.Body.Close()
+	body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
+	if resp.StatusCode != http.StatusOK {
+		return 0, fmt.Errorf("token endpoint returned %d: %s", resp.StatusCode, string(body))
+	}
+	var out struct {
+		AccessToken string `json:"access_token"`
+		ExpiresIn   int    `json:"expires_in"`
+	}
+	if err := json.Unmarshal(body, &out); err != nil {
+		return 0, fmt.Errorf("cannot decode the token response: %w", err)
+	}
+	if out.AccessToken == "" {
+		return 0, fmt.Errorf("the token response holds no access_token")
+	}
+
+	// Write beside the target, then rename. A rename is atomic, so the vendor either
+	// reads the previous token or the new one, never a half-written file.
+	tmp, err := os.CreateTemp(filepath.Dir(c.outPath), ".token-*")
+	if err != nil {
+		return 0, fmt.Errorf("cannot create a temporary file: %w", err)
+	}
+	defer os.Remove(tmp.Name())
+	if _, err := tmp.WriteString(out.AccessToken); err != nil {
+		tmp.Close()
+		return 0, fmt.Errorf("cannot write the token: %w", err)
+	}
+	tmp.Close()
+	if err := os.Chmod(tmp.Name(), 0o644); err != nil {
+		return 0, fmt.Errorf("cannot set the file mode: %w", err)
+	}
+	if err := os.Rename(tmp.Name(), c.outPath); err != nil {
+		return 0, fmt.Errorf("cannot replace %s: %w", c.outPath, err)
+	}
+
+	lifetime := time.Duration(out.ExpiresIn) * time.Second
+	if lifetime <= 0 {
+		lifetime = 5 * time.Minute
+	}
+	return lifetime, nil
+}
+
+func assertion(c config, now time.Time, pemBytes []byte) (string, error) {
+	block, _ := pem.Decode(pemBytes)
+	if block == nil {
+		return "", fmt.Errorf("the key file is not PEM")
+	}
+	var key *rsa.PrivateKey
+	if k, err := x509.ParsePKCS1PrivateKey(block.Bytes); err == nil {
+		key = k
+	} else {
+		parsed, err := x509.ParsePKCS8PrivateKey(block.Bytes)
+		if err != nil {
+			return "", fmt.Errorf("the key is not an RSA private key: %w", err)
+		}
+		rk, ok := parsed.(*rsa.PrivateKey)
+		if !ok {
+			return "", fmt.Errorf("the key holds %T, not RSA", parsed)
+		}
+		key = rk
+	}
+
+	enc := func(v any) (string, error) {
+		raw, err := json.Marshal(v)
+		if err != nil {
+			return "", err
+		}
+		return base64.RawURLEncoding.EncodeToString(raw), nil
+	}
+	header := map[string]any{"alg": "RS256", "typ": "JWT"}
+	if c.keyID != "" {
+		header["kid"] = c.keyID
+	}
+	claims := map[string]any{
+		"iss": c.userID,
+		"sub": c.userID,
+		"aud": c.issuer,
+		// Backdated, so a clock that is a few seconds fast cannot make the assertion
+		// "not yet valid".
+		"iat": now.Add(-5 * time.Second).Unix(),
+		"exp": now.Add(5 * time.Minute).Unix(),
+	}
+	h, err := enc(header)
+	if err != nil {
+		return "", err
+	}
+	cl, err := enc(claims)
+	if err != nil {
+		return "", err
+	}
+	signing := h + "." + cl
+	digest := sha256.Sum256([]byte(signing))
+	sig, err := rsa.SignPKCS1v15(rand.Reader, key, crypto.SHA256, digest[:])
+	if err != nil {
+		return "", fmt.Errorf("cannot sign the assertion: %w", err)
+	}
+	return signing + "." + base64.RawURLEncoding.EncodeToString(sig), nil
+}
+
+func env(key, fallback string) string {
+	if v := os.Getenv(key); v != "" {
+		return v
+	}
+	return fallback
+}

k8s/apps/llm-token-vendor/token_vendor_test.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/json"
+	"encoding/pem"
+	"strings"
+	"testing"
+	"time"
+)
+
+func key(t *testing.T) []byte {
+	t.Helper()
+	k, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		t.Fatalf("cannot generate a key: %v", err)
+	}
+	return pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(k)})
+}
+
+func seg(t *testing.T, s string, out any) {
+	t.Helper()
+	raw, err := base64.RawURLEncoding.DecodeString(s)
+	if err != nil {
+		t.Fatalf("not base64url: %v", err)
+	}
+	if err := json.Unmarshal(raw, out); err != nil {
+		t.Fatalf("not JSON: %v", err)
+	}
+}
+
+func TestAssertionCarriesTheKeyAndTheAudience(t *testing.T) {
+	c := config{issuer: "https://zitadel.example", userID: "3970", keyID: "3971"}
+	tok, err := assertion(c, time.Unix(1_700_000_000, 0), key(t))
+	if err != nil {
+		t.Fatalf("signing failed: %v", err)
+	}
+	parts := strings.Split(tok, ".")
+	if len(parts) != 3 {
+		t.Fatalf("a JWT has three segments, got %d", len(parts))
+	}
+	var h map[string]any
+	seg(t, parts[0], &h)
+	if h["kid"] != "3971" || h["alg"] != "RS256" {
+		t.Fatalf("header must name the key and the algorithm, got %v", h)
+	}
+	var cl map[string]any
+	seg(t, parts[1], &cl)
+	if cl["iss"] != "3970" || cl["sub"] != "3970" || cl["aud"] != "https://zitadel.example" {
+		t.Fatalf("claims are wrong: %v", cl)
+	}
+	if int64(cl["iat"].(float64)) != 1_699_999_995 {
+		t.Fatalf("iat must be backdated five seconds, got %v", cl["iat"])
+	}
+}
+
+func TestAssertionRejectsAKeyThatIsNotPEM(t *testing.T) {
+	if _, err := assertion(config{userID: "u", issuer: "i"}, time.Now(), []byte("nope")); err == nil {
+		t.Fatalf("a non-PEM key must be rejected")
+	}
+}

k8s/apps/paperless/paperless-ai.yaml

@@ -36,53 +36,54 @@ spec:
         - name: paperless-ai-env-config
           configMap:
             name: paperless-ai-env
-      initContainers:
+        - name: llm-key
-        - name: fetch-llm-token
+          secret:
-          image: curlimages/curl:8.12.0
+            secretName: paperless-llm-key
+            # 0440 with the pod's fsGroup. A Secret volume is owned by root, and the
+            # vendor container runs as 65532, so 0400 left the sidecar unable to read
+            # its own key: "permission denied" on every start.
+            defaultMode: 0440
+      # No init container. The token used to be fetched once at pod start with a
+      # client_credentials grant against a web app, which cannot work: a web app has
+      # no such grant, and even a token that was minted would be dead within the hour
+      # while this pod runs for weeks.
+      #
+      # llm-token-vendor runs beside the app instead. It signs a machine-key assertion,
+      # exchanges it for a token, and rewrites /app/data/llm_token.txt at 80% of the
+      # token lifetime. The vendor image keeps reading CUSTOM_API_KEY_FILE unchanged.
+      # The key is re-read on every mint, so a rotation lands without a restart.
       securityContext:
-            runAsUser: 0
+        fsGroup: 65532
-            fsGroup: 0
+        fsGroupChangePolicy: OnRootMismatch
-          command:
-            - sh
-            - -c
-            - |
-              set -e
-              echo "[init] Fetching JWT token from Authentik for paperless-ai-agent..."
-              
-              # Get JWT token via client_credentials grant
-              # Client secret sourced from environment (injected from paperless-ai-config Secret)
-              TOKEN_RESPONSE=$(curl -s -X POST https://zitadel.riotpiao.com/oauth/v2/token \
-                -d "grant_type=client_credentials" \
-                -d "client_id=396757803559027071" \
-                -d "client_secret=${LLM_AUTH_CLIENT_SECRET}" \
-                -d "scope=openid llm:inference" 2>/dev/null)
-              
-              # Extract token from JSON response using sed (handles spaces after colons)
-              TOKEN=$(echo "$TOKEN_RESPONSE" | sed -n 's/.*"access_token"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p')
-              
-              if [ -z "$TOKEN" ]; then
-                echo "[error] Failed to get token. Response: $TOKEN_RESPONSE"
-                exit 1
-              fi
-              
-              # Store token in file for main container to read
-              # Write to both locations for compatibility
-              mkdir -p /tmp/llm-token /app/data 2>/dev/null || true
-              echo "$TOKEN" | tee /tmp/llm-token/llm_token.txt > /dev/null 2>&1
-              echo "$TOKEN" > /app/data/llm_token.txt 2>/dev/null || true
-              echo "[init] Token fetched and stored"
-              [ -f /tmp/llm-token/llm_token.txt ] && echo "  -> /tmp/llm-token/llm_token.txt"
-              [ -f /app/data/llm_token.txt ] && echo "  -> /app/data/llm_token.txt"
-          env:
-            - name: LLM_AUTH_CLIENT_SECRET
-              valueFrom:
-                secretKeyRef:
-                  name: paperless-ai-config
-                  key: LLM_AUTH_CLIENT_SECRET
-          volumeMounts:
-            - name: paperless-ai-data
-              mountPath: /app/data
       containers:
+        - name: llm-token-vendor
+          image: forgejo.riotpiao.com/riotpiao-poimen/llm-token-vendor:0.1.0
+          imagePullPolicy: Always
+          args:
+            - --token-url=https://zitadel.riotpiao.com/oauth/v2/token
+            - --issuer=https://zitadel.riotpiao.com
+            - --user-id=397072668735570071
+            - --key-id=397072674272051351
+            - --key-file=/var/run/llm-key/key
+            # No project audience. ZITADEL puts the requested audience in `aud`, and
+            # the gateway compares `aud` with its allow-list of client ids. Asking for
+            # the project id made every token carry the project, so the gateway
+            # answered "audience not allowed: [396757796898471937]". With plain
+            # `openid` the token names its caller, which is the entry that exists.
+            - --scope=openid
+            - --out=/app/data/llm_token.txt
+          volumeMounts:
+            - {name: paperless-ai-data, mountPath: /app/data}
+            - {name: llm-key, mountPath: /var/run/llm-key, readOnly: true}
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities: {drop: ["ALL"]}
+            runAsNonRoot: true
+            runAsUser: 65532
+            seccompProfile: {type: RuntimeDefault}
+          resources:
+            requests: {cpu: 10m, memory: 32Mi}
+            limits: {cpu: 100m, memory: 64Mi}
         - name: paperless-ai
           image: clusterzx/paperless-ai:latest
           env:

k8s/argocd/secrets/comfyui-oauth2-proxy.enc.yaml

@@ -1,30 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-    name: comfyui-oauth2-proxy
-    namespace: comfyui
-type: Opaque
-data:
-    client-id: ENC[AES256_GCM,data:jFLgcrh1tiAU0pijGxy+/mFsFHzI5zSi,iv:7GYL4xtPnzyZWtEA0egPHhzvpXGevkIH3Cvr+oD9hKI=,tag:UeDTIKlfcYnOOOJiPNgN0g==,type:str]
-    client-secret: ENC[AES256_GCM,data:9QFu5a2EkEMjc/B0moPIkMEChrN0Qh5BQtAzFcvW4liKxzVAPbTKgQOJeND+NQjn5ZoE89kqqRyqNJF3bT3H1pWoDFngk9J2X7BN/9r6Qyubias0Jukydw==,iv:xf2kT0fbnJat3L3xxUYm3ht0YvHptU/RIdt9Eh0D6xU=,tag:8iknaQ2kKP1OZ/1jRCa+1g==,type:str]
-    cookie-secret: ENC[AES256_GCM,data:Qq+l4go/CFtfsZrgYUcTj9bB4sreu2GEqJnu15H6fxIlD7IgnmrodPoIVNpWcdm6LSYabGjm2jraN6RP,iv:EOoDHgvIydzHCC5POohjPq3Vj2YTpzhIhvdG2FzKKzc=,tag:iDXqzFU1Z79bzpG4I4xfww==,type:str]
-sops:
-    kms: []
-    gcp_kms: []
-    azure_kv: []
-    hc_vault: []
-    age:
-        - recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBxOUtCUDduMVU5U1J4WXZX
-            Q1VzaFRVdXlHaEZpeVJNOWprdDhUOEd6Z1hzCi9QZEpXN3NDSENzUE9RTGdpb3Va
-            eVNHVTFLbXpMU0x5UWlFLzZnVlllWE0KLS0tIDZldVNqR1loNHlXZWovNWZ4WjFD
-            dVErenJDM3haSStxZEVWZUtBbDBIZ1UK9x6Gy4drjfUz2goAaI+oeUDWdNHQFAJl
-            JGI+X81r/6Krg5N8QDa/mE96E2M2G0yAO5FS1YbpCKOITUEyheacnQ==
-            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-10-27T21:17:08Z"
-    mac: ENC[AES256_GCM,data:tTGmWFidEWTWHgKPV7b5ABOJwwlP1yQDK5Jd60c9ko6IsmP2GBBPCth7kJgPXuPcw+NHbJ7yt7bcVto7GkPdtY9daJmjPq0Gt16ZO4Xi4hCk22JVKqi/KJcKD/KB0r/VO9w093/Who5CcYgqifextfca4CuxS5m5dcOn4kRJCfE=,iv:0fDvvQr9wp8IrD16u88Ml4HsPEeMhd8R3ljQJoIpw3E=,tag:AanM7C6vbaBBmus1u1GRoA==,type:str]
-    pgp: []
-    encrypted_regex: ^(data|stringData)$
-    version: 3.9.3

k8s/argocd/secrets/forgejo-db-bootstrap.yaml

@@ -0,0 +1,39 @@
+# Bootstrap credential for the Forgejo database — the one secret in the CNPG path
+# that the cluster can lose without any controller to recreate it.
+#
+# forgejo-db-app is OWNED by the Cluster CR (ownerReferences: Cluster/forgejo-db,
+# managed-by: cloudnative-pg) and was GENERATED by CNPG: spec.bootstrap.initdb.secret
+# is empty, so nothing but cloudnative-pg ever knew this value.
+#
+# CNPG reads spec.bootstrap.initdb.secret ONLY at cluster creation. So if the Cluster
+# CR is ever deleted and recreated, CNPG mints a new password while the data directory
+# on the PVC keeps the old one. Forgejo then starts, stays Ready, and fails every
+# query — a credential fault that looks like a network fault. This file is the record
+# that makes a rebuild reproducible: point spec.bootstrap.initdb.secret at it.
+#
+# Keep this file. It cannot rotate on its own; rotation is a manual CNPG ALTER ROLE
+# followed by updating this file and the Secret.
+apiVersion: v1
+kind: Secret
+metadata:
+    name: forgejo-db-bootstrap
+    namespace: cicd
+type: kubernetes.io/basic-auth
+stringData:
+    username: ENC[AES256_GCM,data:GFRjVvq3iQ==,iv:7ZxKdHTvKt6TYlsQYu57dt9C14h59xyfrL3+o/deon0=,tag:Ame/mfgDlHk1oPp80hE3LQ==,type:str]
+    password: ENC[AES256_GCM,data:11t2yglSWQGKeMQ0ZRaw3L5s/o3iHEYArFWdfAQBYSnz1SMOmZ2XTl9yDDHFMUEgHK4K4mP4fJaq4nsgfgr5pA==,iv:aCQ4nkuUX6jDNiOsRW5nzST7HgNZmhjRidmskfNArAg=,tag:ddN0lf+7i4LpS6+yK+za+A==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB3ZVQzajgzcmE2MnFzb1pJ
+            VFR4NjNtU2lWYjZGdGNuOVVnTkNxRndadXhzCkhrU3pPZi9NUlRkRWNUdHNKbW9a
+            WTcvVnVRM2tWZE1OMDBGcTd0NjhzeVkKLS0tIG5PQlg3ZlBNMDhQV2hxdzB4K1g3
+            SHBtZ284bTJNV3FvSWFtbDRudGUzbTQKNPM04xB1DWyIE/6/qjUCvXw2M8zkhVkP
+            xV8bVIp81R0pnMuEGB60L38W81SfziSspzpU0YMUmgZcU2XmO73AdQ==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-28T10:09:09Z"
+    mac: ENC[AES256_GCM,data:oNYDrN9wzVbDrDTYcLEKwCqTruybeU+++58Duki5P8xddU1anjHzfBg9DXgXd6CaoYJAU2jEcrqX9YXN8zmJtGaTpgr2GUdgFWx6O1JFoaNM32RNcp7u8FZWpcFYecWyYRtRFMolRaLM/CxkXr2lLa/V3Azxe1xlJrf7EjyI3dY=,iv:7ztpZquVBNZFt59OcNM2IXiDdmMjE8AGs1lTDZzeNWs=,tag:xpak7lbN6hrXaSKoWSbHYQ==,type:str]
+    version: 3.13.2

k8s/argocd/secrets/grafana-oidc-secrets.enc.yaml

@@ -1,28 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-    name: grafana-oidc
-    namespace: logging
-type: Opaque
-stringData:
-    GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: ENC[AES256_GCM,data:fGqA1ofVme1lVLxFjWIiU7sbvE9ip2Af7C1ua3+HdidlcVgkwW59vVlXZhgBRdoif86C8LKQSyoCGUqiO+740g==,iv:6Kv2ybbJg1GT8DLqz9P5y+E1zr7vcIBDMUErmvxNTaw=,tag:0wK28B3G/owCVPPqb2sLmQ==,type:str]
-sops:
-    kms: []
-    gcp_kms: []
-    azure_kv: []
-    hc_vault: []
-    age:
-        - recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSArTXMxMWdVRHIvbWh1bVNk
-            bEFFNlgzNU9zTjlzY2o3VGdnbFpnU2lkYmxrCjF4aWVGSlJaNkcxODJ6RDkrVFo2
-            UkV2RUwwVHFCd0xMNG9RWXdTUHZGaXcKLS0tIHVaWjhaNHRtS2c0UkVzRk83ckhj
-            OCtBVFR0d1F5amVhV2E0emRZWThvYjAKFW46i9brIL9ZoO1CEn+IewIdoUhvYK/A
-            RqzZ6xV+mhlDj0zC+SpVcEm/Ki83nMYOj4+vQingZlr/Kr3Nwb0iLQ==
-            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-10-27T19:44:32Z"
-    mac: ENC[AES256_GCM,data:EJew+p/kJrurcNpLlFxJGTl69hQYxn3IkzP7EDtdSriW/WI7X1xNLC2dp/dTiLomGmG1uh6+2JI9Sc8gXmrRAWLELAbiOaYbcoU6gi89BjTCWGDAMX4lNSIuH40rsGs4YtYj5U7SGfgx5qto4ZH7LxjBmxzPWM/Xz2Uq/I5jV4k=,iv:LdhjhwCfPUIa7D0gved8Twf7ujN+dg3lN8lCnFBFs0g=,tag:bjkwUCMtR8IBL9BkbuXcnw==,type:str]
-    pgp: []
-    encrypted_regex: ^(data|stringData)$
-    version: 3.9.3

k8s/argocd/secrets/minio-secrets.enc.yaml

@@ -1,3 +1,6 @@
+# minio-oidc is not in this file any more: the KeyRotation object iam/minio owns it.
+# A prune of the sops-secrets application would delete that Secret, and the
+# controller recreates it on its next rotation.
 apiVersion: v1
 kind: Secret
 metadata:
@@ -5,46 +8,27 @@ metadata:
     namespace: storage
 type: Opaque
 stringData:
-    config.env: ENC[AES256_GCM,data:1yogGyBdwrlg27H8R5Kr9VcPu+xwS5/nPhi7eLcmVYtXmPlNokCsWHA70qnn/+I6i4ODys8GML++VeZ/lmcHiP7RIbY+G6xmw6nfBCXitgE8kQwTPsUUzn9bobOn4Sr1YMBf7FQ/Q0N8aupGJA==,iv:EfCdim4TLkP/FP1u1yoKZCq8YGFzVh7ca4yu1+WzXwE=,tag:rTYe0SIAEsUhh0KHQIEKNw==,type:str]
+    config.env: ENC[AES256_GCM,data:4NQYmVEnkv4nO6FnMAEXVCUhMM+DLCGG2tFC7SGJ0lGnfFQiISdzWwVXD0y9/gpK4qn5zv0Cs1MgOfDVBKH6UR1ONXQnIjUnvXnq4D5+P0BgVUlYD/7aComva9Ow1IEChvD3fBTmDW7FMEZrtA==,iv:/LAV+ukcE3pwgJMkEkR8z9h+k6pSvQ7a/FkxLvMK6X8=,tag:BnvsL05XttadRqqS3TU2Uw==,type:str]
 sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
     age:
-        - enc: |
+        - recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+          enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2SStVR0lpOWs0VnN6dkdp
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAxQ1BqbDZoYjVQWGlTVEFD
-            RktMUmNRR2diTUtiVk9tRk1wSjVTMnRtVlJBCkk1S0dwNUNHb25RQlZ4ZFRocisv
+            dVVWT0xFUlAwUXJUcHZOMGN1cG85S3NtcVZRCmpUZnM5dXpWUDh5UHhGUTcyYmNr
-            aS9kbmQydWVmN1AxU3VvbFZsS2tmTHcKLS0tICtRY0ZKeFhCTUdaNW5PSFRNZ2dG
+            UWNlc2VDazUvd3k0VHVWR1NBUE13eTAKLS0tIG9VcGYvN1Z1UEo3Um1SMHZOSHF3
-            OERIZ0xMbHBvSVNyaGxCVFpqRzlxa1EKbrjfk/CFlTSyJYgTzQ67W3JyR1fwv8VF
+            eUp5dE04M3lvTVlHc3lPZnMyUWU0bE0KPXwE7wOJNSrqM6kfO1BvXqHApvQI4hdz
-            dxGeAgLY4EQrf+2PbODtazRFreS+sl7dxq+rKMOVZTsjGHakkqHy9A==
+            5GWmLjmjQL/ZZlkF07uK1GDXIBC3N3K0vYrfQHjMUImw9Dwn9DnL5w==
             -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    lastmodified: "2026-10-28T03:00:02Z"
+    mac: ENC[AES256_GCM,data:DGglizsoj/Fmi+r1NvfYD1sSivtedJ+w0RKFioJ1K7ektG/I68hprcMkmrs9vY1H2zlP/1Io/Zf0wOMqIPVK+uyEHIDGr+kk5Y/vPkrPi1+TpVDHRJVwvfkFWWVBrnlMbpZ89076dl1cyphdi4XvSHq2z30Qa1oRnxNkPO2HrhY=,iv:je0iyd/AkWYeyL2vA6ipmvWL6c0X+4h1zeLMeE60Pks=,tag:Q3X/jIWQWFtiSGhkCbzBDA==,type:str]
+    pgp: []
     encrypted_regex: ^(data|stringData)$
-    lastmodified: "2026-10-26T04:03:16Z"
+    version: 3.9.3
-    mac: ENC[AES256_GCM,data:gxu4WcJEuFgjtWCfAjose8R8a566/OLrXoiJmKfcvVfTFDTdGM7I4gzQVvpjEfra5Hjyt7kvLlaV2K5bKJ0k6NXbbmgrqAqEOEcPg+cnPy1FtQGAYOo3VzgKqrpLOCs/7wh4YR23XHHsjQlHAcP8BmhrqnpHXR9FMZObrKjnQj8=,iv:KJZfw0n2DOtdZHeVDk6TrpO8q57yD+xdEVt9+hM0D4Q=,tag:rHQaMjvQwaB8Am/AlTKq1g==,type:str]
-    version: 3.13.2
----
-apiVersion: v1
-kind: Secret
-metadata:
-    name: minio-oidc
-    namespace: storage
-type: Opaque
-stringData:
-    MINIO_IDENTITY_OPENID_CLIENT_SECRET: ENC[AES256_GCM,data:eFfwW2ZnV+o3/6XHjUvyFsDbaIAdosY2PXOnRdN4Ie49YDIjfJPe+Mg1FsbgoEHEjuB33hfxesb4TWOJQfW8Og==,iv:+kRr9A3auts1XHgZWa+t5b2m4lF0x9dVhiRpB17fzv4=,tag:tH/jszFzLQdqORGr9vfTGw==,type:str]
-sops:
-    age:
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2SStVR0lpOWs0VnN6dkdp
-            RktMUmNRR2diTUtiVk9tRk1wSjVTMnRtVlJBCkk1S0dwNUNHb25RQlZ4ZFRocisv
-            aS9kbmQydWVmN1AxU3VvbFZsS2tmTHcKLS0tICtRY0ZKeFhCTUdaNW5PSFRNZ2dG
-            OERIZ0xMbHBvSVNyaGxCVFpqRzlxa1EKbrjfk/CFlTSyJYgTzQ67W3JyR1fwv8VF
-            dxGeAgLY4EQrf+2PbODtazRFreS+sl7dxq+rKMOVZTsjGHakkqHy9A==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    encrypted_regex: ^(data|stringData)$
-    lastmodified: "2026-10-26T04:03:16Z"
-    mac: ENC[AES256_GCM,data:gxu4WcJEuFgjtWCfAjose8R8a566/OLrXoiJmKfcvVfTFDTdGM7I4gzQVvpjEfra5Hjyt7kvLlaV2K5bKJ0k6NXbbmgrqAqEOEcPg+cnPy1FtQGAYOo3VzgKqrpLOCs/7wh4YR23XHHsjQlHAcP8BmhrqnpHXR9FMZObrKjnQj8=,iv:KJZfw0n2DOtdZHeVDk6TrpO8q57yD+xdEVt9+hM0D4Q=,tag:rHQaMjvQwaB8Am/AlTKq1g==,type:str]
-    version: 3.13.2
 ---
 apiVersion: v1
 kind: Secret
@@ -53,20 +37,25 @@ metadata:
     namespace: storage
 type: Opaque
 stringData:
-    CONSOLE_ACCESS_KEY: ENC[AES256_GCM,data:Zxr/Nj0SUKSJbBD1cFjt3GE=,iv:r/B9ULn68hsXdH8uRQkKAoQTJIJIYKMoJuWOZfxS2I8=,tag:Xo4i+IVxuqrp+qDpYQysPw==,type:str]
+    CONSOLE_ACCESS_KEY: ENC[AES256_GCM,data:6mmBluC+4QmlMxPJbfNh0kA=,iv:CuWy1gVnd19vB1rOS9HRVKegX3CnmCEa6cpk+S7ZX00=,tag:OqZK09bVMZGbMXwnDmr77Q==,type:str]
-    CONSOLE_SECRET_KEY: ENC[AES256_GCM,data:PoIDNZ+Xs6BdhGm10y1HdL5jy47EVHfHHIJhGy/KX6Q=,iv:voAv32GoTKi9s9/4MQy2yT6i9VY4/igA5XXzehwUqRQ=,tag:DmPyk5biVvlxHjeyDyrLbQ==,type:str]
+    CONSOLE_SECRET_KEY: ENC[AES256_GCM,data:Fyd5EdZZP7/q3QOVv3FU1bkd8960p6UiRNNl17LJltw=,iv:/QGSkVVfXkCAKSV1+RNTZEI+/riAFVLV6SiaL9DvbSo=,tag:lFTJpifGorsl2w8lG7beCw==,type:str]
 sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
     age:
-        - enc: |
+        - recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+          enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2SStVR0lpOWs0VnN6dkdp
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAxQ1BqbDZoYjVQWGlTVEFD
-            RktMUmNRR2diTUtiVk9tRk1wSjVTMnRtVlJBCkk1S0dwNUNHb25RQlZ4ZFRocisv
+            dVVWT0xFUlAwUXJUcHZOMGN1cG85S3NtcVZRCmpUZnM5dXpWUDh5UHhGUTcyYmNr
-            aS9kbmQydWVmN1AxU3VvbFZsS2tmTHcKLS0tICtRY0ZKeFhCTUdaNW5PSFRNZ2dG
+            UWNlc2VDazUvd3k0VHVWR1NBUE13eTAKLS0tIG9VcGYvN1Z1UEo3Um1SMHZOSHF3
-            OERIZ0xMbHBvSVNyaGxCVFpqRzlxa1EKbrjfk/CFlTSyJYgTzQ67W3JyR1fwv8VF
+            eUp5dE04M3lvTVlHc3lPZnMyUWU0bE0KPXwE7wOJNSrqM6kfO1BvXqHApvQI4hdz
-            dxGeAgLY4EQrf+2PbODtazRFreS+sl7dxq+rKMOVZTsjGHakkqHy9A==
+            5GWmLjmjQL/ZZlkF07uK1GDXIBC3N3K0vYrfQHjMUImw9Dwn9DnL5w==
             -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    lastmodified: "2026-10-28T03:00:02Z"
+    mac: ENC[AES256_GCM,data:DGglizsoj/Fmi+r1NvfYD1sSivtedJ+w0RKFioJ1K7ektG/I68hprcMkmrs9vY1H2zlP/1Io/Zf0wOMqIPVK+uyEHIDGr+kk5Y/vPkrPi1+TpVDHRJVwvfkFWWVBrnlMbpZ89076dl1cyphdi4XvSHq2z30Qa1oRnxNkPO2HrhY=,iv:je0iyd/AkWYeyL2vA6ipmvWL6c0X+4h1zeLMeE60Pks=,tag:Q3X/jIWQWFtiSGhkCbzBDA==,type:str]
+    pgp: []
     encrypted_regex: ^(data|stringData)$
-    lastmodified: "2026-10-26T04:03:16Z"
+    version: 3.9.3
-    mac: ENC[AES256_GCM,data:gxu4WcJEuFgjtWCfAjose8R8a566/OLrXoiJmKfcvVfTFDTdGM7I4gzQVvpjEfra5Hjyt7kvLlaV2K5bKJ0k6NXbbmgrqAqEOEcPg+cnPy1FtQGAYOo3VzgKqrpLOCs/7wh4YR23XHHsjQlHAcP8BmhrqnpHXR9FMZObrKjnQj8=,iv:KJZfw0n2DOtdZHeVDk6TrpO8q57yD+xdEVt9+hM0D4Q=,tag:rHQaMjvQwaB8Am/AlTKq1g==,type:str]
-    version: 3.13.2

k8s/argocd/secrets/oauth2-credentials.enc.yaml

@@ -5,23 +5,25 @@ metadata:
     name: oauth2-credentials
     namespace: iam
 data:
-    api-gw-client-secret: ENC[AES256_GCM,data:YiUCk61cCsTJE8gTLZWhqVPsCSu3b4OnG+Oh456A+mnxip0FjLTTqV+kYBdfZivIkn9ua4RcVW+rYILu,iv:K08m1EaB7/SIVNWZ3FfeRP4tbCAo6KeiNC0NpE9VO1o=,tag:ZXV9YRWUOPJ/vdoqMtvQeA==,type:str]
+    api-gw-client-secret: ENC[AES256_GCM,data:q41OBQ6WvxsmHmXFhrdzeNUlQyPKuOQQa0vWdhuioQwTsNJ5+l9rYZKBAcZxzvd4P066aRN6yVPmEZhq,iv:Dnfgy8N9NVOM2x656Qei8960dzu7icAU3BZLXmj7N5o=,tag:deilBSCC9Mg/Eo8K0w6rMw==,type:str]
-    minio-client-secret: ENC[AES256_GCM,data:vcwLo1cJqbZr2MwkAKO09xPjb09ss7+s6HIvEfaLxt8ka5PUAlXQ37pTkynuktlKq/t0DvSk+x1+5G7QqVD1JJlwyDGM6yuOhBQ3MwucLNl+T4RDKtgGuA==,iv:sCdd+UNX2TfTAM6Zslrh8KrvBSzqpe67F3YMGrl4YDU=,tag:9Tpzm9NEXNuXyCd7l7GeHw==,type:str]
+    poimen-client-secret: ENC[AES256_GCM,data:2k28ui+Mbu3qd4ar6/NidO4bWO8g7JQUjDQBD8liywpCm7rA8TSrmU51im/I5kEvRRhl+RByEo5wpozo,iv:JFrZYF7cP/tQL8gkQJ5GvtdOt1kKcAdHE7L4l+uT9M0=,tag:DSiv8kv7MINUxi7fO/093w==,type:str]
-    poimen-client-secret: ENC[AES256_GCM,data:dYtCiMGZaOFokihfGD2SXJDfXrhfwEQcTOPNu7JNHDUnx0hqwAdYAPAJTEN21p0OpHYDINMs5lwUpMwY,iv:ll8USuzs0881ryHImVjHGO0iFxUzqg5XtI6Ekd+EFXE=,tag:I/4ZfB9Ss4zEP57sRtRTXQ==,type:str]
-    paperless-client-secret: ENC[AES256_GCM,data:XMj9rHkI20lI37vChq9wS5lUq7l95SisRX0LChuj09O5w76Zt9NdG6OEUYQFf1z8vCfMJxBInIEEIq0e,iv:UpYy2ZEZ4AgBACEyrlxEcr13V+8Ksu4p8JHYApmyYHo=,tag:o9Z+/b6T9POgowIsks56kw==,type:str]
-    grafana-client-secret: ENC[AES256_GCM,data:V/j0+P3HoDLqYjRalWK4v1hvJTuVmzxVra47HsF1/BZvWZjIyEIH3Pt8BekT3O1dUFynxjcaxdZMBVYg,iv:9pMp+vIsp/sA7GP9wqNDf5DdZVLDow46OURW0mMRXIc=,tag:i7VJFA5o0hToZEJ5JPWIvw==,type:str]
 sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
     age:
-        - enc: |
+        - recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+          enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2amZPendIOU9PeStOcmJG
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB5WmthakxzRk5aeGJWZnRF
-            c1dzOFVvd3RoT3dXQlJUNkdCSytEZUVGVEJJCklMUkNjUytBMXh0OVUrNUlpMWw3
+            bzdqM1o5Q1h2MFlpNEJvRUFZVU1wWjRaVWowClVKYkp0YUJCbUM0MWduaCtBSExH
-            aHRFT1hBZTZHNHVtaGJCTEFyT2ZWNWsKLS0tIENxR3FKMEplV0V5Y0dsZzBkODVG
+            MDRWUGZkYTdndTZTeEM3ckxrMURFMUEKLS0tIHF3YUlLNndSWmxGejFPbTVjQnRS
-            U0ZlT2R0VFdEQmpsakVvZ1FJRkRQYTAKESQcSlMAuCG0/tLWWZaeoPayf1EKxSaf
+            THF4RFBNbDJCdjVKRDJ6cEJlZGVBM0UKEDz1sKDON+lelq1zbCO4CETaK77T4G8y
-            e4o8TmjmSgPEhepQ/LmNXl6obyuqUxztkh4Mm6pocqDp+gfzt7X/1g==
+            fly66sR5uIvAXdwXy14L9ssZsM1jOHqdkXnpbo6mzh3toB/ROzoL7w==
             -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    lastmodified: "2026-10-27T21:47:38Z"
+    mac: ENC[AES256_GCM,data:bgZD+/LAPKp6DXyRMnbFWyxmiRjWxYO0ofxeGpO7nyw1DMSTD9s8li43eh/IP7wW4NORwzfIfzxUvOFmVfAiQKfkzpYRDFIKqK+aa/+43zppwpVT0UJ3CcBt2qgdJRHKnv9qeUOx4M8ulY7Xcsoevw8EGdrXCc1KAcDIwoIAp0Q=,iv:djYQ6YnGPrYGRKgpfbdf0beW6QHYTq5CJ3nes7+C4p4=,tag:N9n4VKyMNzGCmuYpwBLPng==,type:str]
+    pgp: []
     encrypted_regex: ^(data|stringData)$
-    lastmodified: "2026-10-06T02:01:24Z"
+    version: 3.9.3
-    mac: ENC[AES256_GCM,data:AI/znHiUa/Rwg0YRz65qUZ+b6V73oi3ZfAG/V5uZZvrGFrA0EISzD5LaP5Kl4GUU1svo4G5EI4aftPDgpdPptz21CCa8iYkXGe9pTDmtaE5ZICQAUbRcVhGMTfuWFhrgEhMfCJdOpfHaIQVEsQJQJgrnkXQVuUe9aT75LQz7pnY=,iv:KbIaYT0VeBT2WeSU5V5ZB4e+04JICYaY+egGN1Pv5bM=,tag:pPbhDxeLUx6V1XC/LPwxTg==,type:str]
-    version: 3.13.2

k8s/argocd/secrets/secret-generator.yaml

@@ -15,8 +15,6 @@ files:
     - forgejo-registry-pull.enc.yaml
     - forgejo-runner-token.enc.yaml
     - forgejo-secrets.enc.yaml
-    - comfyui-oauth2-proxy.enc.yaml
-    - grafana-oidc-secrets.enc.yaml
     - grafana-secrets.enc.yaml
     - homelab-ca-secrets.enc.yaml
     - loki-secrets.enc.yaml

k8s/bootstrap/phase3-forgejo/forgejo-values.yaml

@@ -34,6 +34,8 @@ service:
       lbipam.cilium.io/ips: "192.168.1.161"
 
 gitea:
+  podAnnotations:
+    reloader.stakater.com/auto: "true"
   admin:
     existingSecret: forgejo-admin
 

scripts/iam/rotate-oidc.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+"""Rotate the OIDC client secret of every owned application, one at a time.
+
+This is the manual execution of what a KeyRotation object declares. It exists so
+the credentials can rotate today, before the controller image is built, and so the
+controller has a reference implementation to match.
+
+For each application: generate a new secret in ZITADEL, write it into the
+Kubernetes Secret, roll the consumer, then prove the consumer still talks to
+ZITADEL. A failure stops the run, because the next application should not start
+while one is in an unknown state.
+"""
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent))
+import importlib.util
+
+spec = importlib.util.spec_from_file_location("zp", Path(__file__).with_name("zitadel-provision.py"))
+zp = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(zp)
+
+PROJECT = "396757796898471937"
+ISSUER = "https://zitadel.riotpiao.com"
+
+# app key, ZITADEL app id, Secret name, namespace, Secret key, consumer kind, consumer name
+APPS = [
+    ("grafana",   "grafana",   "grafana-oidc",         "logging",   "GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET", "deploy", "grafana"),
+    ("argocd",    "argocd",    "oidc-secret",           "argocd",    "client-secret",                       "deploy", "argocd-server"),
+    ("comfyui",   "comfyui",   "comfyui-oauth2-proxy", "comfyui",   "client-secret",                       "deploy", "comfyui-oauth2-proxy"),
+    ("minio",     "minio",     "minio-oidc",           "storage",   "MINIO_IDENTITY_OPENID_CLIENT_SECRET",  "sts",    "minio-cluster-az-a"),
+    ("paperless", "paperless", "paperless-oidc",       "paperless", "client-secret",                       "deploy", "paperless"),
+    ("immich",    "immich",    "immich-oidc",          "immich",    "client-secret",                       "deploy", "immich-server"),
+    ("forgejo",   "forgejo",   "forgejo-oidc",         "cicd",      "CLIENT_SECRET",                       "deploy", "forgejo-gitea"),
+]
+
+
+def sh(*args: str, timeout: int = 120) -> tuple[int, str]:
+    p = subprocess.run(args, capture_output=True, text=True, timeout=timeout)
+    return p.returncode, (p.stdout + p.stderr).strip()
+
+
+def state() -> dict:
+    return json.load(open(Path.home() / ".config/zitadel/state.json"))
+
+
+def app_ids(token: str) -> dict[str, str]:
+    """Map clientId to the app resource id. The two are different, and the secret
+    endpoint takes the resource id."""
+    code, body = zp._api("POST", f"/management/v1/projects/{PROJECT}/apps/_search", token,
+                         {"query": {"limit": 100, "offset": 0}})
+    out = {}
+    for app in body.get("result", []) or []:
+        cid = (app.get("oidcConfig") or {}).get("clientId")
+        if cid and app.get("id"):
+            out[cid] = app["id"]
+    return out
+
+
+def new_secret(token: str, app_id: str) -> str | None:
+    code, body = zp._api("POST", f"/management/v1/projects/{PROJECT}/apps/{app_id}/oidc_config/_generate_client_secret", token)
+    if code != 200:
+        print(f"    generate failed: {code} {str(body)[:120]}")
+        return None
+    return body.get("clientSecret")
+
+
+def patch_secret(ns: str, name: str, key: str, value: str) -> bool:
+    import base64
+    enc = base64.b64encode(value.encode()).decode()
+    patch = json.dumps({"data": {key: enc}})
+    code, out = sh("kubectl", "-n", ns, "patch", "secret", name, "--type=merge", "-p", patch)
+    if code != 0:
+        print(f"    secret patch failed: {out[:120]}")
+    return code == 0
+
+
+def roll(kind: str, ns: str, name: str) -> bool:
+    code, out = sh("kubectl", "-n", ns, "rollout", "restart", f"{kind}/{name}", timeout=120)
+    if code != 0:
+        print(f"    rollout restart failed: {out[:120]}")
+        return False
+    code, out = sh("kubectl", "-n", ns, "rollout", "status", f"{kind}/{name}", "--timeout=90s", timeout=120)
+    return code == 0
+
+
+def main() -> int:
+    only = sys.argv[1:] or None
+    token = zp.bootstrap_token()
+    if not token:
+        print("no bootstrap token")
+        return 2
+    cids = state()["client_ids"]["value"]
+    ids = app_ids(token)
+    print(f"resolved {len(ids)} app resource ids")
+    done, failed = [], []
+    for key, app_id, sname, ns, skey, kind, cname in APPS:
+        if only and key not in only:
+            continue
+        print(f"[{key}] issuer={ISSUER} client_id={cids[key]}")
+        rid = ids.get(cids[key])
+        if not rid:
+            print(f"[{key}] no app resource id for client {cids[key]}")
+            failed.append(key)
+            continue
+        secret = new_secret(token, rid)
+        if not secret:
+            failed.append(key)
+            continue
+        if not patch_secret(ns, sname, skey, secret):
+            failed.append(key)
+            continue
+        if not roll(kind, ns, cname):
+            failed.append(key)
+            print(f"[{key}] FAILED: consumer did not become ready")
+            continue
+        code, out = sh("kubectl", "-n", ns, "get", "pods", "-l",
+                       f"app={cname}" if kind == "deploy" else f"app.kubernetes.io/name={cname}",
+                       "--no-headers")
+        ready = "Running" in out
+        print(f"[{key}] rotated, consumer {'ready' if ready else 'NOT ready'}")
+        (done if ready else failed).append(key)
+    print(f"\nrotated: {len(done)} {done}\nfailed: {len(failed)} {failed}")
+    return 0 if not failed else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/iam/zitadel-provision.py

@@ -87,6 +87,12 @@ ROLES = {
     # LLM gate for the api-gw. The colon form matches the Authentik-era
     # permission string, so consumer manifests did not change.
     "llm:inference": "LLM inference access (api-gw)",
+    # Capability for machine-to-machine callers. Kept separate from the human roles
+    # so a service account can be identified by its grant alone. Note that ZITADEL
+    # does not assert roles into a machine-user access token on this instance (see
+    # MACHINE_CALLERS), so this grant documents intent; the gateway authorises the
+    # caller from its allow-list entry instead.
+    "svc-caller": "Machine-to-machine caller",
     "llm-inference": "LLM inference access (dash alias)",
     # Per-service capabilities. The gateway derives the required capability from
     # the verb: GET/HEAD/OPTIONS -> "<service>:read", anything else ->
@@ -143,8 +149,13 @@ OIDC_APPS = {
         "https://paperless.riotpiao.com/accounts/oidc/zitadel/login/callback/",
         "http://localhost:8000/accounts/oidc/authentik/login/callback/",
         "https://paperless.riotpiao.com/accounts/oidc/authentik/login/callback/"]},
+    # The frontend calls back at /auth/callback (see OAUTH_CALLBACK_PATH in
+    # poimen-frontend/lib/oauth.ts). The bare /callback spelling stays registered so
+    # an older image still completes its login.
     "poimen": {**_WEB, "redirect_uris": [
-        "http://localhost:3000/callback", "https://poimen.riotpiao.com/callback"]},
+        "http://localhost:3000/callback", "https://poimen.riotpiao.com/callback",
+        "http://localhost:3000/auth/callback",
+        "https://poimen.riotpiao.com/auth/callback"]},
     "queue-sqs": {**_WEB, "redirect_uris": [
         "http://localhost:8080/callback", "https://queue.riotpiao.com/callback"]},
     "vault": {**_WEB, "redirect_uris": [
@@ -340,6 +351,55 @@ def _ensure_builtin_login(token: str) -> None:
     print("login page    : built-in (loginV2.required=false)")
 
 
+# One machine user per cross-service caller. Each has its own key, so revoking one
+# service leaves the others working, and ZITADEL can overlap keys during a rotation.
+MACHINE_CALLERS = {
+    "poimen-llm": {
+        "name": "Poimen LLM caller",
+        "description": "poimen calls api-gateway inference",
+        "roles": ["llm:inference", "svc-caller"],
+    },
+}
+
+
+def _ensure_machine_users(token: str, project_id: str) -> dict[str, dict]:
+    """Create each caller once and grant its roles. Returns id and key id per caller.
+
+    Roles are granted even though ZITADEL does not assert them into a machine-user
+    token on this instance: the grant is the record of what the account may do, and
+    it becomes effective the moment that behaviour changes.
+    """
+    out: dict[str, dict] = {}
+    for user_name, spec in MACHINE_CALLERS.items():
+        code, found = _api("POST", "/management/v1/users/_search", token,
+                           {"queries": [{"userNameQuery": {"userName": user_name,
+                                                           "method": "TEXT_QUERY_METHOD_EQUALS"}}]})
+        user_id = ""
+        for user in (found.get("result") or []):
+            user_id = user.get("id", "")
+        if not user_id:
+            code, created = _api("POST", "/management/v1/users/machine", token, {
+                "userName": user_name,
+                "name": spec["name"],
+                "description": spec["description"],
+                "accessTokenType": "ACCESS_TOKEN_TYPE_JWT",
+            })
+            user_id = created.get("userId", "")
+        if not user_id:
+            continue
+        code, grant = _api("POST", "/management/v1/users/grants/_search", token,
+                           {"query": {"limit": 100, "offset": 0}})
+        existing = next((g for g in (grant.get("result") or []) if g.get("userId") == user_id), None)
+        if existing:
+            _api("PUT", f"/management/v1/users/{user_id}/grants/{existing['id']}", token,
+                 {"roleKeys": spec["roles"]})
+        else:
+            _api("POST", f"/management/v1/users/{user_id}/grants", token,
+                 {"projectId": project_id, "roleKeys": spec["roles"]})
+        out[user_name] = {"userId": user_id}
+    return out
+
+
 def _ensure_roles(token: str, project_id: str, roles: dict[str, str]) -> None:
     code, d = _api("POST", f"/management/v1/projects/{project_id}/roles/_search", token, {})
     # ZITADEL answers with `key`, not `roleKey`. Reading the wrong field left
@@ -497,6 +557,9 @@ def phase_apply(_args: argparse.Namespace) -> int:
     project_id = _ensure_project(token, PROJECT_NAME)
     print(f"project   : {PROJECT_NAME} ({project_id})")
     _ensure_roles(token, project_id, ROLES)
+    machine_users = _ensure_machine_users(token, project_id)
+    for name, info in machine_users.items():
+        print(f"machine user {name}: {info['userId']}")
     print(f"roles     : {len(ROLES)} ensured")
     known: dict[str, str] = {}
     try: