From eaf98043747fb3ba4fb518483dc26ef5f3474202 Mon Sep 17 00:00:00 2001 From: rock Date: Mon, 26 Oct 2026 23:08:20 -0700 Subject: [PATCH] fix(iam): use the built-in ZITADEL login page Sign-in failed with "Could not create session for user". The ZITADEL API logged, on every session call: authz: verify access token -> invalid token token verifier repo: issuer does not match: Expected https://zitadel.riotpiao.com, got: login-client zitadel-login: [Password] Could not update session The hosted Login UI authenticates as the machine user `login-client`. That user did not exist in this instance, because the chart's setup job creates it and setupJob is disabled after the first sync. Without the user the UI cannot write a session, so no one can sign in, at ZITADEL or at Forgejo. Disable the hosted Login UI so ZITADEL serves its built-in login, which needs no machine user. Re-enable only after creating `login-client` and registering the public key that matches zitadel-login-service-key. Also register both callback spellings for the forgejo and paperless apps: the callback path follows the login-source name, so a rename between authentik and zitadel must not break sign-in. --- k8s/apps/llm-serving/ornith.yaml | 107 ------------------ k8s/apps/llm-serving/reasoning.yaml | 162 ---------------------------- k8s/infra/iam/zitadel-values.yaml | 15 ++- scripts/iam/zitadel-provision.py | 7 ++ 4 files changed, 21 insertions(+), 270 deletions(-) delete mode 100644 k8s/apps/llm-serving/ornith.yaml delete mode 100644 k8s/apps/llm-serving/reasoning.yaml diff --git a/k8s/apps/llm-serving/ornith.yaml b/k8s/apps/llm-serving/ornith.yaml deleted file mode 100644 index be9c841..0000000 --- a/k8s/apps/llm-serving/ornith.yaml +++ /dev/null @@ -1,107 +0,0 @@ -apiVersion: serving.kserve.io/v1beta1 -kind: InferenceService -metadata: - annotations: - serving.kserve.io/deploymentMode: RawDeployment - # The konghq.com/{connect,read,write}-timeout annotations that used to live - # here went with Kong (retired 2026-08-19). They existed because Kong read - # its upstream timeouts off the Kubernetes Service, and its 60s default cut - # off the first request after any pod restart — a restart flushes VRAM and - # reloading ornith:35b takes longer than that. OLLAMA_KEEP_ALIVE=-1 hid the - # problem in steady state. - # - # The equivalent budget now belongs to the Go gateway's per-route timeout - # config in rock/homelab-frontend, not to an annotation on this object. - labels: - app.kubernetes.io/name: llm-ornith - app.kubernetes.io/part-of: llm-serving - name: ornith - namespace: llm-serving -spec: - predictor: - containers: - - command: - - /bin/sh - - -c - - 'set -e - - ollama serve & - - SERVE_PID=$! - - until ollama list >/dev/null 2>&1; do sleep 2; done - - ollama pull ornith:35b - - ollama run ornith:35b "ok" >/dev/null 2>&1 || true - - wait $SERVE_PID - - ' - env: - - name: OLLAMA_HOST - value: 0.0.0.0:8080 - - name: OLLAMA_MODELS - value: /mnt/models/ollama - - name: OLLAMA_CONTEXT_LENGTH - value: '65536' - - name: OLLAMA_KEEP_ALIVE - value: '-1' - - name: OLLAMA_NUM_PARALLEL - value: '1' - - name: OLLAMA_MAX_LOADED_MODELS - value: '1' - image: ollama/ollama:0.32.9@sha256:1685741456770df6e3cceb2a945a5f75e020f658d1701509668d6f4688f1dd3f - name: kserve-container - ports: - - containerPort: 8080 - protocol: TCP - readinessProbe: - exec: - command: - - /bin/sh - - -c - - ollama ps 2>/dev/null | grep -q ornith - periodSeconds: 10 - resources: - limits: - cpu: '16' - memory: 16Gi - nvidia.com/gpu: '1' - requests: - cpu: '8' - memory: 8Gi - nvidia.com/gpu: '1' - startupProbe: - exec: - command: - - /bin/sh - - -c - - ollama ps 2>/dev/null | grep -q ornith - failureThreshold: 120 - periodSeconds: 15 - volumeMounts: - - mountPath: /mnt/models - name: models - podMetadata: - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "8080" - prometheus.io/path: "/metrics" - labels: - app.kubernetes.io/name: llm-ornith - app.kubernetes.io/part-of: llm-serving - deploymentStrategy: - type: Recreate - # 1 replica -- ornith:35b only. qwen2.5:3b moved to CPU on cp-2. - # Frees 1 GPU for ComfyUI. - maxReplicas: 1 - minReplicas: 1 - nodeSelector: - kubernetes.io/hostname: worker-1 - runtimeClassName: nvidia - volumes: - - name: models - persistentVolumeClaim: - claimName: llm-models - diff --git a/k8s/apps/llm-serving/reasoning.yaml b/k8s/apps/llm-serving/reasoning.yaml deleted file mode 100644 index 5585ad2..0000000 --- a/k8s/apps/llm-serving/reasoning.yaml +++ /dev/null @@ -1,162 +0,0 @@ -apiVersion: serving.kserve.io/v1beta1 -kind: InferenceService -metadata: - annotations: - serving.kserve.io/deploymentMode: RawDeployment - labels: - app.kubernetes.io/name: llm-reasoning - app.kubernetes.io/part-of: llm-serving - name: reasoning - namespace: llm-serving -spec: - predictor: - containers: - - args: - # bnb-4bit retired: no int4 tensor cores on sm70/V100, dequant-then- - # matmul is two slow kernel launches instead of one fused int4 GEMM, - # decode crawled at 2.5-10 tok/s regardless of TP/PP. Switched to - # JunHowie/Qwen3-32B-GPTQ-Int4 -- same dense Qwen3-32B weights, same - # hermes/qwen3 parser stack (no narration-bug risk, same as before), - # only the quant format changes. Plain (non-Marlin) GPTQ kernel is - # confirmed Volta-compatible; Marlin needs sm80+ and vLLM would try - # to auto-upgrade to it, so --quantization is pinned explicitly to - # `gptq` to force the plain kernel. Verified checkpoint size: 19.34GB - # (summed from the real safetensors index, not bits-per-param math). - # max-model-len=131072 is Qwen3-32B's real ceiling (config.json YaRN: - # factor=4.0, original_max_position_embeddings=32768) -- 200k was - # asked for but exceeds this architecturally regardless of VRAM. - # KV cache math: 256KB/token total (64 layers, 8 KV heads, 128 - # head_dim, fp16), PP=2 splits both weights and KV load ~evenly, so - # each GPU carries ~9.67GB weights + ~128KB/token KV. At - # gpu-memory-utilization=0.95 (30.4GB/GPU usable), that leaves - # ~20.7GB/GPU for KV cache. NOTE this is a projection, not a - # measurement: the live reading at 0.90 is num_gpu_blocks=9527 x - # block_size=16 = 152,432 tokens (vllm:cache_config_info). Re-read that - # metric after the rollout -- max_num_batched_tokens=8192 also raises - # activation memory out of the same budget, so the post-boot - # num_gpu_blocks is the authoritative number, and --max-num-seqs=5 is - # only valid if it still covers 2x65k + 3x8k. The old - # OffloadingConnector CPU-DRAM spillover (tuned for the previous - # model's much smaller 16384 context) is no longer needed and is - # dropped. Staying on PP=2 and vLLM 0.11.0 (no version bump needed, - # this checkpoint only requires vllm>=0.9.2) -- plain GPTQ has no - # TP>1 restriction unlike bnb, so tensor-parallel-size=2 is worth - # trying later, but not risking a parallelism-strategy change in the - # same rollout as the quant+context-length change. - # This GPTQ requant's own config.json ships max_position_embeddings= - # 40960 and rope_scaling=None -- confirmed directly (curl'd the raw - # config.json), the base Qwen3-32B repo's YaRN block did NOT carry - # over during quantization. Re-applying it explicitly here restores - # the same math the base model documents (32768 * 4.0 = 131072); - # without this, --max-model-len=131072 fails ModelConfig validation - # against the checkpoint's own (unscaled) 40960 ceiling. - - --model=JunHowie/Qwen3-32B-GPTQ-Int4 - - --served-model-name=reasoning - - --quantization=gptq - - --dtype=float16 - - --kv-cache-dtype=auto - - --rope-scaling={"rope_type":"yarn","factor":4.0,"original_max_position_embeddings":32768} - - --tensor-parallel-size=1 - - --pipeline-parallel-size=2 - - --max-model-len=131072 - - --gpu-memory-utilization=0.95 - - --max-num-seqs=5 - # Prefill scheduling -- see the measured starvation this fixes. - # V1's scheduler has no prefill/decode phase: it walks self.running and - # hands out tokens from one shared budget (vllm/v1/core/sched/scheduler.py: - # "There's no decoding phase nor prefill phase in the scheduler"). The cap - # per request is applied BEFORE the budget clamp: - # if 0 < long_prefill_token_threshold < num_new_tokens: take = threshold - # take = min(take, token_budget) - # The loop is `while ... and token_budget > 0`. With the threshold at its - # default of 0 the cap is disabled, so one long prompt takes the ENTIRE - # budget, token_budget hits 0, the loop exits and every other request is - # skipped for that step -- measured as 262 tok/s contended prefill and - # 0.0-0.1 tok/s generation for everything else, Running: 2-4 reqs. - # Hence threshold must be strictly BELOW the budget or it never binds: - # at 8192/8192 a single request is served per step (still starved), while - # 8192/2048 serves four. Verified by replaying the loop. - - --max-num-batched-tokens=8192 - - --long-prefill-token-threshold=2048 - - --enable-chunked-prefill - - --enable-prefix-caching - # - # NOT set, deliberately: --max-num-partial-prefills and - # --max-long-partial-prefills. Those are V0-only knobs. vLLM 0.11.0 ships - # V1 only (vllm/core/ absent), and EngineArgs._is_v1_supported_oracle - # rejects any value other than the default for either one, calling - # _raise_or_fallback("Concurrent Partial Prefill") -> "Falling back to V0" - # -> AssertionError, because there is no V0 engine to fall back to. - # The Deployment uses strategy Recreate, so setting them is an outage. - # V1 does concurrent chunked prefill natively; it needs no flag, only the - # budget/threshold ratio above. - # qwen3 is vLLM's dedicated reasoning parser for this family's - # blocks. - - --reasoning-parser=qwen3 - # hermes is the documented tool-call parser for general (non-Coder) - # Qwen3 models -- native chat template support, not narrated text. - - --enable-auto-tool-choice - - --tool-call-parser=hermes - - --host=0.0.0.0 - - --port=8080 - env: - - name: VLLM_USE_FLASHINFER_SAMPLER - value: '0' - - name: VLLM_ATTENTION_BACKEND - value: TRITON_ATTN - - name: HF_HOME - value: /mnt/models - image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c - name: kserve-container - ports: - - containerPort: 8080 - protocol: TCP - readinessProbe: - httpGet: - path: /health - port: 8080 - periodSeconds: 10 - resources: - limits: - cpu: '16' - memory: 36Gi - nvidia.com/gpu: '2' - requests: - cpu: '8' - memory: 12Gi - nvidia.com/gpu: '2' - startupProbe: - failureThreshold: 80 - httpGet: - path: /health - port: 8080 - periodSeconds: 15 - volumeMounts: - - mountPath: /mnt/models - name: models - - mountPath: /dev/shm - name: shm - podMetadata: - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "8080" - prometheus.io/path: "/metrics" - labels: - app.kubernetes.io/name: llm-reasoning - app.kubernetes.io/part-of: llm-serving - deploymentStrategy: - type: Recreate - maxReplicas: 1 - minReplicas: 1 - nodeSelector: - kubernetes.io/hostname: worker-1 - runtimeClassName: nvidia - volumes: - - name: models - persistentVolumeClaim: - claimName: llm-models - - emptyDir: - medium: Memory - sizeLimit: 2Gi - name: shm - diff --git a/k8s/infra/iam/zitadel-values.yaml b/k8s/infra/iam/zitadel-values.yaml index 0fd50f7..586a2e7 100644 --- a/k8s/infra/iam/zitadel-values.yaml +++ b/k8s/infra/iam/zitadel-values.yaml @@ -171,7 +171,20 @@ ingress: controller: nginx login: - enabled: true + # Built-in login page, not the hosted Login UI. + # + # The hosted UI authenticates to the ZITADEL API as the machine user + # `login-client`, using a JWT-profile assertion signed by the keypair in + # zitadel-login-service-key. That machine user is created by the chart's + # setup job, which this install runs only once (setupJob is deliberately + # disabled after the first sync). Without the user, every session write + # fails and the user sees "Could not create session for user", so no one + # can sign in here or at any relying party such as Forgejo. + # + # The built-in login serves the same flows from the ZITADEL server and + # needs no machine user. Turn this back on only after creating + # `login-client` and registering the matching public key. + enabled: false # ksops owns this Secret (k8s/argocd/secrets/zitadel-login-service-key.enc.yaml, # rendered by the sops-secrets Application). Pointing the chart at it stops the # chart from auto-generating an RSA keypair every render, which changed diff --git a/scripts/iam/zitadel-provision.py b/scripts/iam/zitadel-provision.py index e2d4fb1..8ecccc0 100755 --- a/scripts/iam/zitadel-provision.py +++ b/scripts/iam/zitadel-provision.py @@ -96,7 +96,12 @@ OIDC_APPS = { "argocd": {**_WEB, "redirect_uris": [ "http://localhost:8080/auth/callback", "https://argocd.riotpiao.com/auth/callback"]}, + # Forgejo derives the callback path from the login-source NAME + # (/user/oauth2//callback). Both spellings are registered so a + # rename between authentik and zitadel never breaks sign-in. "forgejo": {**_WEB, "redirect_uris": [ + "http://localhost:3000/user/oauth2/zitadel/callback", + "https://forgejo.riotpiao.com/user/oauth2/zitadel/callback", "http://localhost:3000/user/oauth2/authentik/callback", "https://forgejo.riotpiao.com/user/oauth2/authentik/callback"]}, "grafana": {**_WEB, "redirect_uris": [ @@ -111,6 +116,8 @@ OIDC_APPS = { "http://localhost:9000/auth/sso/oauth2/code", "https://minio.riotpiao.com/auth/sso/oauth2/code"]}, "paperless": {**_WEB, "redirect_uris": [ + "http://localhost:8000/accounts/oidc/zitadel/login/callback/", + "https://paperless.riotpiao.com/accounts/oidc/zitadel/login/callback/", "http://localhost:8000/accounts/oidc/authentik/login/callback/", "https://paperless.riotpiao.com/accounts/oidc/authentik/login/callback/"]}, "poimen": {**_WEB, "redirect_uris": [