refactor(iam): retire terraform/zitadel, provision through the API

The Terraform root module duplicated a writer that the provisioning script could be, and its state carried generated client secrets. Declare the objects in the script instead: project, 19 roles, 12 OIDC apps, applied through the management API with the iam-admin PAT. State is a 0600 JSON file under ~/.config/zitadel. Also add zitadel to the Cloudflare tunnel hostname list, with the import command and the note that the tunnel's Public Hostname entry is a separate dashboard step: DNS alone yields a 404 from the tunnel.
2026-10-25 19:35:01 -07:00
parent be44a576d4
commit d4e9dbf350
13 changed files with 213 additions and 653 deletions
@@ -6,7 +6,7 @@ One command replaces the manual sequence the Authentik script
 fixed order and every phase is safe to re-run:

  keys         extract the iam-admin machine key and PAT from namespace iam
-  apply        terraform apply terraform/zitadel (project, roles, apps, machines)
+  apply        create project, roles and OIDC apps through the ZITADEL API
  secrets      write generated client ids/secrets back into the SOPS files
  users        import one human user, converting the Django password hash
  verify       check discovery, client count and the groups-claim gate
@@ -43,13 +43,85 @@ from pathlib import Path
 import yaml

 REPO = Path(__file__).resolve().parents[2]
-TF_DIR = REPO / "terraform" / "zitadel"
 SOPS_DIR = REPO / "k8s" / "argocd" / "secrets"
 SOPS_CONFIG = REPO / ".sops.yaml"
 CONF_DIR = Path(os.environ.get("ZITADEL_CONF_DIR", Path.home() / ".config" / "zitadel"))
+STATE_FILE = CONF_DIR / "state.json"
 NAMESPACE = os.environ.get("ZITADEL_NAMESPACE", "iam")
 BASE_URL = os.environ.get("ZITADEL_BASE_URL", "https://zitadel.riotpiao.com")

+# ── desired ZITADEL state ──────────────────────────────────────────────────
+# Declared here, applied through the API by the `apply` phase. This replaces the
+# retired terraform/zitadel root module: same objects, no state file to drift and
+# no second writer.
+PROJECT_NAME = os.environ.get("ZITADEL_PROJECT_NAME", "homelab")
+PROJECT_GROUP = "homelab"
+
+# Role keys EQUAL the Authentik group names, so the `groups` claim keeps working
+# for every consumer that maps on it (kube-apiserver RBAC, MinIO policies,
+# Grafana roles).
+ROLES = {
+    "homelab-admins": "Cluster administrators with full access",
+    "minio-admins": "MinIO administrators",
+    "minio-photos": "Photos bucket (Immich) access",
+    "minio-documents": "Documents bucket (Paperless) access",
+    "minio-backups": "Backups bucket read-only access",
+    "poimen-admins": "Poimen memory administrators",
+    "poimen-devs": "Dev and staging projects access",
+    "poimen-prod-readonly": "Production projects read-only access",
+    "paperless-admins": "Paperless administrators",
+    "paperless-finance": "Finance documents",
+    "paperless-legal": "Legal documents",
+    "paperless-hr": "HR documents",
+    "grafana-admins": "Grafana administrators",
+    "grafana-editors": "Grafana dashboard editors",
+    "grafana-viewers": "Grafana dashboard viewers",
+    "sqs-users": "SQS/Temporal queue read access",
+    "sqs-writers": "SQS/Temporal queue read/write access",
+    "s3-users": "S3 read access",
+    "s3-writers": "S3 read/write access",
+}
+
+_WEB = {"app_type_id": "OIDC_APP_TYPE_WEB", "auth_method_id": "OIDC_AUTH_METHOD_TYPE_BASIC"}
+_NATIVE = {"app_type_id": "OIDC_APP_TYPE_NATIVE", "auth_method_id": "OIDC_AUTH_METHOD_TYPE_NONE"}
+_PKCE = {"app_type_id": "OIDC_APP_TYPE_USER_AGENT", "auth_method_id": "OIDC_AUTH_METHOD_TYPE_NONE"}
+
+OIDC_APPS = {
+    "api-gw": {**_WEB, "redirect_uris": [
+        "http://localhost:3000/callback", "https://api.riotpiao.com/callback"]},
+    "argocd": {**_WEB, "redirect_uris": [
+        "http://localhost:8080/auth/callback",
+        "https://argocd.riotpiao.com/auth/callback"]},
+    "forgejo": {**_WEB, "redirect_uris": [
+        "http://localhost:3000/user/oauth2/authentik/callback",
+        "https://forgejo.riotpiao.com/user/oauth2/authentik/callback"]},
+    "grafana": {**_WEB, "redirect_uris": [
+        "http://localhost:3000/login/generic_oauth",
+        "https://grafana.riotpiao.com/login/generic_oauth"]},
+    "homarr": {**_WEB, "redirect_uris": [
+        "http://localhost:7575/auth/callback",
+        "https://homarr.riotpiao.com/auth/callback"]},
+    "immich": {**_WEB, "app_type_id": "OIDC_APP_TYPE_NATIVE", "redirect_uris": [
+        "app.immich:///oauth-callback", "https://img.riotpiao.com/auth/login",
+        "https://img.riotpiao.com/user/oauth2/callback"]},
+    "kubernetes": {**_PKCE, "dev_mode": True, "redirect_uris": [
+        "http://localhost:8000", "http://localhost:18000"]},
+    "minio": {**_WEB, "redirect_uris": [
+        "http://localhost:9000/auth/sso/oauth2/code",
+        "https://minio.riotpiao.com/auth/sso/oauth2/code"]},
+    "paperless": {**_WEB, "redirect_uris": [
+        "http://localhost:8000/accounts/oidc/authentik/login/callback/",
+        "https://paperless.riotpiao.com/accounts/oidc/authentik/login/callback/"]},
+    "poimen": {**_WEB, "redirect_uris": [
+        "http://localhost:3000/callback", "https://poimen.riotpiao.com/callback"]},
+    "queue-sqs": {**_WEB, "redirect_uris": [
+        "http://localhost:8080/callback", "https://queue.riotpiao.com/callback"]},
+    "vault": {**_WEB, "redirect_uris": [
+        "http://localhost:8200/ui/vault/auth/oidc/oidc/callback",
+        "https://vault.riotpiao.com/ui/vault/auth/oidc/oidc/callback"]},
+}
+
+
 _TEMP_FILES: set[str] = set()


@@ -155,6 +227,13 @@ def curl_json(method: str, url: str, token: str | None = None, body: dict | None
    cfg = [f'url = "{esc(url)}"', f'request = "{method}"', "silent", "show-error",
           f"max-time = {timeout}", 'write-out = "\\n%{http_code}"',
           'header = "Content-Type: application/json"']
+    # Reaching the instance by name needs either public DNS through the tunnel
+    # or a pinned address. ZITADEL picks the instance from the Host header, so a
+    # plain port-forward fails with "Instance not found". Set this to bypass DNS:
+    #   ZITADEL_CURL_RESOLVE=zitadel.riotpiao.com:443:192.168.1.160
+    pinned = os.environ.get("ZITADEL_CURL_RESOLVE", "").strip()
+    if pinned:
+        cfg.append(f'resolve = "{esc(pinned)}"')
    if token:
        cfg.append(f'header = "Authorization: Bearer {esc(token)}"')
    if body is not None:
@@ -179,27 +258,99 @@ def phase_keys(_args: argparse.Namespace) -> int:

 # ── phase: apply ───────────────────────────────────────────────────────────

-def phase_apply(args: argparse.Namespace) -> int:
-    need("terraform")
-    if not args.org_id:
-        die("apply requires --org-id (ZITADEL console -> Organization settings -> Resource ID)")
-    key = CONF_DIR / "iam-admin.json"
-    if not key.exists():
-        die(f"{key} missing — run the keys phase first")
-    env = dict(os.environ, ZITADEL_JWT_PROFILE_FILE=str(key))
+def _api(method: str, path: str, token: str, body: dict | None = None,
+         timeout: int = 30) -> tuple[int, dict]:
+    code, payload = curl_json(method, f"{BASE_URL}{path}", token=token, body=body, timeout=timeout)
+    try:
+        return code, (json.loads(payload) if payload.strip() else {})
+    except json.JSONDecodeError:
+        return code, {"_raw": payload}

-    for cmd in (["init", "-input=false"], ["fmt", "-check", "-recursive"]):
-        cp = run(["terraform", f"-chdir={TF_DIR}", *cmd], env=env)
-        if cp.returncode != 0:
-            die(f"terraform {' '.join(cmd)} failed: {cp.stderr.strip() or cp.stdout.strip()}")

-    cp = run(["terraform", f"-chdir={TF_DIR}", "apply", "-input=false", "-auto-approve",
-              f"-var=org_id={args.org_id}"], env=env)
-    sys.stdout.write(cp.stdout)
-    sys.stderr.write(cp.stderr)
-    if cp.returncode != 0:
-        die("terraform apply failed — no SOPS value was written. Objects created before the "
-            "failure stay in ZITADEL; re-run apply once the cause is fixed")
+def _ensure_project(token: str, name: str) -> str:
+    code, d = _api("POST", "/management/v1/projects/_search", token, {})
+    for row in d.get("result") or []:
+        if row.get("name") == name:
+            return row["id"]
+    code, d = _api("POST", "/management/v1/projects", token, {"name": name})
+    if code not in (200, 201) or not d.get("id"):
+        die(f"create project {name} failed: {code} {d}")
+    return d["id"]
+
+
+def _ensure_roles(token: str, project_id: str, roles: dict[str, str]) -> None:
+    code, d = _api("POST", f"/management/v1/projects/{project_id}/roles/_search", token, {})
+    have = {r.get("roleKey") for r in (d.get("result") or [])}
+    for key, display in roles.items():
+        if key in have:
+            continue
+        code, d = _api("POST", f"/management/v1/projects/{project_id}/roles", token,
+                       {"roleKey": key, "displayName": display, "group": PROJECT_GROUP})
+        if code not in (200, 201):
+            die(f"create role {key} failed: {code} {d}")
+
+
+def _ensure_oidc_apps(token: str, project_id: str, apps: dict[str, dict]) -> tuple[dict, dict]:
+    code, d = _api("POST", f"/management/v1/projects/{project_id}/apps/_search", token, {})
+    existing = {a.get("name"): a for a in (d.get("result") or [])}
+    ids: dict[str, str] = {}
+    secrets: dict[str, str] = {}
+    for name, spec in apps.items():
+        row = existing.get(name)
+        if row:
+            ids[name] = (row.get("oidcConfig") or {}).get("clientId", "")
+            app_id = row.get("id")
+        else:
+            body = {
+                "name": name,
+                "redirectUris": spec["redirect_uris"],
+                "responseTypes": ["OIDC_RESPONSE_TYPE_CODE"],
+                "grantTypes": ["OIDC_GRANT_TYPE_AUTHORIZATION_CODE",
+                               "OIDC_GRANT_TYPE_REFRESH_TOKEN"],
+                "appType": spec.get("app_type_id", "OIDC_APP_TYPE_WEB"),
+                "authMethodType": spec.get("auth_method_id", "OIDC_AUTH_METHOD_TYPE_BASIC"),
+                "devMode": bool(spec.get("dev_mode")),
+                "accessTokenType": "OIDC_TOKEN_TYPE_BEARER",
+                "version": "OIDC_VERSION_1_0",
+                "idTokenRoleAssertion": True,
+            }
+            code, d = _api("POST", f"/management/v1/projects/{project_id}/apps/oidc", token, body)
+            if code not in (200, 201) or not d.get("appId"):
+                die(f"create app {name} failed: {code} {d}")
+            app_id = d["appId"]
+            ids[name] = d.get("clientId", "")
+        if spec.get("auth_method_id") == "OIDC_AUTH_METHOD_TYPE_NONE":
+            continue
+        code, d = _api("POST",
+                       f"/management/v1/projects/{project_id}/apps/{app_id}"
+                       "/oidc_config/_generate_client_secret", token, {})
+        if code in (200, 201) and d.get("clientSecret"):
+            secrets[name] = d["clientSecret"]
+    return ids, secrets
+
+
+def phase_apply(_args: argparse.Namespace) -> int:
+    """Declare the ZITADEL objects through the API. No Terraform, no state drift."""
+    token = bootstrap_token()
+    if not token:
+        die("no iam-admin PAT — run the keys phase first")
+
+    project_id = _ensure_project(token, PROJECT_NAME)
+    print(f"project   : {PROJECT_NAME} ({project_id})")
+    _ensure_roles(token, project_id, ROLES)
+    print(f"roles     : {len(ROLES)} ensured")
+    ids, secrets = _ensure_oidc_apps(token, project_id, OIDC_APPS)
+    print(f"oidc apps : {len(ids)} ensured, {len(secrets)} secrets issued")
+
+    state = {
+        "project_id": {"value": project_id},
+        "client_ids": {"value": ids},
+        "client_secrets": {"value": secrets},
+        "machine_keys": {"value": {}},
+        "groups_claim_enabled": {"value": False},
+    }
+    write_private(STATE_FILE, json.dumps(state, indent=2, sort_keys=True).encode())
+    print(f"state     -> {STATE_FILE} (0600)")
    return 0


@@ -238,16 +389,11 @@ MANUAL_ONLY = {
 }


-def terraform_outputs() -> dict:
-    need("terraform")
-    key = CONF_DIR / "iam-admin.json"
-    if not key.exists():
-        die(f"{key} missing — run the keys phase first")
-    env = dict(os.environ, ZITADEL_JWT_PROFILE_FILE=str(key))
-    cp = run(["terraform", f"-chdir={TF_DIR}", "output", "-json"], env=env)
-    if cp.returncode != 0:
-        die(f"terraform output failed: {cp.stderr.strip()}")
-    return json.loads(cp.stdout or "{}")
+def state_outputs() -> dict:
+    """Read the object map written by the apply phase."""
+    if not STATE_FILE.exists():
+        die(f"{STATE_FILE} missing — run the apply phase first")
+    return json.loads(STATE_FILE.read_text())


 def sops_env() -> dict:
@@ -345,13 +491,13 @@ def update_client_id(client: str, value: str, dry_run: bool, apply_config: bool)

 def phase_secrets(args: argparse.Namespace) -> int:
    need("sops")
-    out = terraform_outputs()
+    out = state_outputs()
    ids = out.get("client_ids", {}).get("value", {})
    secrets = out.get("client_secrets", {}).get("value", {})
    machine_keys = out.get("machine_keys", {}).get("value", {})

    if not ids and not machine_keys:
-        die("terraform has no ZITADEL outputs yet — run the apply phase first")
+        die("no ZITADEL objects recorded yet — run the apply phase first")

    print("[1/3] client secrets into SOPS")
    for client, entries in SECRET_MAP.items():
@@ -438,7 +584,7 @@ def phase_verify(_args: argparse.Namespace) -> int:
    if code != 200:
        die("discovery endpoint is not 200 — the instance or the ingress is not ready")

-    out = terraform_outputs()
+    out = state_outputs()
    ids = out.get("client_ids", {}).get("value", {})
    print(f"client apps    : {len(ids)}")
    enabled = bool(out.get("groups_claim_enabled", {}).get("value"))