iam: switch service accounts to roles-based auth

- Roles stored in user attributes, not groups
- Property mapping looks up roles by client_id for client_credentials
- Service account apps have no policy bindings (client_secret = access control)
- Cleanup stale bindings on re-provision
- JWT claims: azp (service identity) + roles (capabilities)

k8s/infra/iam/authentik-provision-job.yaml

@@ -1,252 +0,0 @@
-# Authentik OAuth provisioning — MANUAL operation, NOT auto-run.
-#
-# Security-sensitive IAM changes should be reviewed and run locally:
-#   export AUTHENTIK_BOOTSTRAP_TOKEN=$(kubectl -n iam get secret authentik-secrets \
-#     -o jsonpath='{.data.AUTHENTIK_BOOTSTRAP_TOKEN}' | base64 -d)
-#   sed 's|http://authentik-server.iam.svc.cluster.local|https://authentik.riotpiao.com|g' \
-#     k8s/infra/iam/scripts/authentik-provision.py | python3
-#
-# What it does (see scripts/authentik-provision.py docstring): creates scope
-# mappings (groups, permissions, memory, policy, immich_role), admin groups,
-# the "rock" admin user, OAuth2 providers + Applications, service accounts
-# (portfolio-agent, memory-agent), and binds groups to applications.
-#
-# This file provides the RBAC (ServiceAccount + RoleBindings) needed if you
-# ever want to run the Job in-cluster manually (kubectl create -f), but the
-# PostSync hook is deliberately removed — IAM is not GitOps-auto-deployed.
-#
-# RBAC: this Job only touches Secrets (get existing client secrets, create new
-# ones) across the namespaces those services live in.
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: authentik-provisioner
-  namespace: iam
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: authentik-provisioner
-rules:
-  - apiGroups: [""]
-    resources: ["secrets"]
-    verbs: ["get", "list", "create", "update", "patch"]
----
-# One RoleBinding per namespace the script touches (least-privilege: Secrets
-# only, and only in these 5 namespaces — not a cluster-wide ClusterRoleBinding).
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: iam
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: cicd
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: argocd
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: logging
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: storage
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: paperless
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: immich
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: llm-serving
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: portfolio
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: poimen
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: dashboard
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: authentik-provision
-  namespace: iam
-  # No ArgoCD hook - run manually when IAM changes are needed
-spec:
-  ttlSecondsAfterFinished: 600
-  backoffLimit: 3
-  template:
-    spec:
-      serviceAccountName: authentik-provisioner
-      restartPolicy: Never
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 1000
-        seccompProfile:
-          type: RuntimeDefault
-      containers:
-        - name: provision
-          image: python:3.12-alpine
-          securityContext:
-            allowPrivilegeEscalation: false
-            capabilities:
-              drop: ["ALL"]
-          env:
-            - name: AUTHENTIK_BOOTSTRAP_TOKEN
-              valueFrom:
-                secretKeyRef:
-                  name: authentik-secrets
-                  key: AUTHENTIK_BOOTSTRAP_TOKEN
-          volumeMounts:
-            - name: script
-              mountPath: /script
-          command:
-            - /bin/sh
-            - -c
-            - |
-              set -e
-              echo "waiting for authentik-server..."
-              until wget -q -O /dev/null http://authentik-server.iam.svc.cluster.local/-/health/ready/ 2>/dev/null; do
-                sleep 5
-              done
-              echo "installing kubectl (via python urllib - no apk/curl: this"
-              echo "container runs as non-root UID 1000 and can't write to"
-              echo "apk's directories or /usr/local/bin, both root-owned in"
-              echo "the python:3.12-alpine image; /tmp is world-writable)..."
-              python3 -c "
-              import urllib.request, os, stat
-              kver = urllib.request.urlopen('https://dl.k8s.io/release/stable.txt').read().decode().strip()
-              url = f'https://dl.k8s.io/release/{kver}/bin/linux/amd64/kubectl'
-              urllib.request.urlretrieve(url, '/tmp/kubectl')
-              st = os.stat('/tmp/kubectl')
-              os.chmod('/tmp/kubectl', st.st_mode | stat.S_IEXEC)
-              "
-              export PATH="/tmp:$PATH"
-              echo "running provisioning script..."
-              python3 /script/authentik-provision.py
-      volumes:
-        - name: script
-          configMap:
-            name: authentik-provision-script

k8s/infra/iam/kustomization.yaml

@@ -1,35 +1,12 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-# NOTE: no top-level `namespace:` transformer here (removed) - it used to
+
-# force-rewrite metadata.namespace to "iam" on every resource in this
-# kustomization, which was harmless while every manifest here only ever
-# targeted the iam namespace itself. authentik-provision-job.yaml's
-# RoleBindings deliberately target cicd/argocd/logging/storage (least-
-# privilege access for the authentik-provisioner ServiceAccount to touch
-# Secrets in those namespaces) - the namespace transformer would have
-# silently rewritten all of them back to iam, breaking the RBAC. Every
-# manifest in this directory already sets its own explicit
-# metadata.namespace, so dropping the transformer changes nothing for the
-# existing resources/.
 resources:
-  - authentik-provision-job.yaml
   - rbac-dashboard-rolebinding.yaml
 
-# Provisioning/verification python lives in scripts/*.py (real files, linted +
+# IAM provisioning is manual-only (security-sensitive).
-# diff-friendly) and is generated into ConfigMaps here rather than embedded in
+# Script: scripts/iam/authentik-provision.py
-# the job YAML. disableNameSuffixHash keeps the names stable so the Jobs'
+# Run:
-# configMap volume refs and PostSync hook-delete semantics keep working; each
+#   export AUTHENTIK_BOOTSTRAP_TOKEN=$(kubectl -n iam get secret authentik-secrets \
-# hook Job is recreated per sync so it always mounts the latest script.
+#     -o jsonpath='{.data.AUTHENTIK_BOOTSTRAP_TOKEN}' | base64 -d)
-configMapGenerator:
+#   python3 scripts/iam/authentik-provision.py
-  - name: authentik-provision-script
-    namespace: iam
-    files:
-      - authentik-provision.py=scripts/authentik-provision.py
-
-generatorOptions:
-  disableNameSuffixHash: true
-# authentik-migrations-job.yaml removed — redundant + broken. The authentik
-# `server` entrypoint runs migrations itself; this standalone job lacked the
-# authentik-secrets envFrom (Secret key missing) and always failed.
-# SOPS secrets (*.enc.yaml) handled by ArgoCD SOPS plugin at sync time
-# authentik/vault deployed via ArgoCD Helm source

scripts/iam/authentik-provision.py

@@ -37,7 +37,7 @@ import sys
 import urllib.error
 import urllib.request
 
-AUTHENTIK_URL = "http://authentik-server.iam.svc.cluster.local"
+AUTHENTIK_URL = "https://authentik.riotpiao.com"
 TOKEN = os.environ["AUTHENTIK_BOOTSTRAP_TOKEN"]
 
 
@@ -204,6 +204,38 @@ permissions_mapping = get_or_create(
 )
 PERMISSIONS_MAPPING_PK = permissions_mapping["pk"]
 
+# Roles claim for service accounts - for client_credentials flow, the auto-generated
+# user doesn't have roles, so we look up the actual service account by client_id.
+# Format: ["llm:inference", "memory:read", "memory:write"]
+_ROLES_EXPR = """
+from authentik.core.models import User
+
+# Try user's own roles first (password grant uses the actual user)
+roles = request.user.attributes.get("roles", [])
+
+# For client_credentials, user is auto-generated - look up by client_id
+if not roles and request.http_request:
+    client_id = request.http_request.POST.get("client_id", "")
+    if client_id:
+        sa_user = User.objects.filter(username=client_id, path="service-accounts").first()
+        if sa_user:
+            roles = sa_user.attributes.get("roles", [])
+
+return {"roles": roles}
+""".strip()
+roles_mapping = get_or_create(
+    "/api/v3/propertymappings/provider/scope/",
+    "/api/v3/propertymappings/provider/scope/",
+    "scope_name=roles",
+    {
+        "name": "homelab: roles claim",
+        "scope_name": "roles",
+        "expression": _ROLES_EXPR,
+    },
+    patch_existing={"expression": _ROLES_EXPR},
+)
+ROLES_MAPPING_PK = roles_mapping["pk"]
+
 # Immich reads a "immich_role" claim on every login (not just user-creation -
 # fixed upstream in immich-app/immich#29991) and syncs isAdmin from it, so
 # this is the actual mechanism that makes "rock" an Immich admin - not
@@ -720,14 +752,61 @@ app_pks_for_binding.append(("kubernetes", k8s_application["pk"]))
 print(f"  kubernetes: provider pk={k8s_provider['pk']} application pk={k8s_application['pk']}")
 
 # -----------------------------------------------------------------------------
-# Service accounts for programmatic API access (client_credentials flow).
+# Headless authentication flow for service accounts (password grant).
+# Default flow has MFA/interactive stages that break password grant.
+# This flow: identification -> password -> login (no MFA, no consent prompts).
+print("\n[SERVICE ACCOUNT FLOW] Ensuring headless authentication flow...")
+
+sa_flow = get_or_create(
+    "/api/v3/flows/instances/", "/api/v3/flows/instances/",
+    "slug=service-account-authentication",
+    {
+        "slug": "service-account-authentication",
+        "name": "Service Account Authentication (Headless)",
+        "title": "Service Account Login",
+        "designation": "authentication",
+        "policy_engine_mode": "any",
+    },
+)
+SA_AUTHENTICATION_FLOW_PK = sa_flow["pk"]
+print(f"  flow pk={SA_AUTHENTICATION_FLOW_PK}")
+
+# Bind stages to the flow (identification -> password -> login)
+# Get default stages (reuse existing ones)
+status, ident_stages = api("GET", "/api/v3/stages/identification/")
+status, pass_stages = api("GET", "/api/v3/stages/password/")
+status, login_stages = api("GET", "/api/v3/stages/user_login/")
+
+if ident_stages.get("results") and pass_stages.get("results") and login_stages.get("results"):
+    ident_pk = ident_stages["results"][0]["pk"]
+    pass_pk = pass_stages["results"][0]["pk"]
+    login_pk = login_stages["results"][0]["pk"]
+    
+    # Check if bindings already exist
+    status, existing_bindings = api("GET", f"/api/v3/flows/bindings/?target={SA_AUTHENTICATION_FLOW_PK}")
+    if not existing_bindings.get("results"):
+        # Create bindings
+        api("POST", "/api/v3/flows/bindings/", {"target": SA_AUTHENTICATION_FLOW_PK, "stage": ident_pk, "order": 10})
+        api("POST", "/api/v3/flows/bindings/", {"target": SA_AUTHENTICATION_FLOW_PK, "stage": pass_pk, "order": 20})
+        api("POST", "/api/v3/flows/bindings/", {"target": SA_AUTHENTICATION_FLOW_PK, "stage": login_pk, "order": 30})
+        print("  bound stages: identification -> password -> login")
+    else:
+        print(f"  stages already bound ({len(existing_bindings['results'])} bindings)")
+else:
+    print("  WARNING: Could not find default stages to bind")
+
+# -----------------------------------------------------------------------------
+# Service accounts for programmatic API access (password grant).
 # These are Authentik users with type=service_account, not OAuth applications.
-# They authenticate via client_credentials grant to get JWTs with claims.
+# They authenticate via password grant to get JWTs with user claims.
 print("\n[SERVICE ACCOUNTS] Creating service accounts for API access...")
 
+# Roles define what APIs a service can access - stored in user attributes,
+# output as "roles" claim in JWT. Gateway checks roles, not groups.
+# Format: "<api>:<action>" e.g. "llm:inference", "memory:write"
 SERVICE_ACCOUNTS = {
     "portfolio-agent": {
-        "groups": [],  # Permissions come from SA_CONFIGS in memory scope mapping
+        "roles": ["llm:inference", "memory:read"],
         "attributes": {
             "memory_projects": ["homelab", "portfolio"],
             "memory_visibility": "public",
@@ -736,7 +815,7 @@ SERVICE_ACCOUNTS = {
         "secret_name": "portfolio-agent-oidc",
     },
     "memory-agent": {
-        "groups": [],  # Permissions come from SA_CONFIGS in memory scope mapping
+        "roles": ["llm:inference", "memory:read", "memory:write"],
         "attributes": {
             "memory_projects": ["*"],
             "memory_visibility": "private",
@@ -755,36 +834,40 @@ for sa_name, sa_cfg in SERVICE_ACCOUNTS.items():
     if status != 200:
         die(f"GET users for {sa_name} -> {status} {res}")
     
+    # Merge roles into attributes
+    sa_attributes = {**sa_cfg["attributes"], "roles": sa_cfg["roles"]}
+    
     if res.get("results"):
         sa_user = res["results"][0]
-        # Update attributes and groups
+        # Update attributes (roles stored here, not in groups)
-        group_pks = [service_admin_groups[g]["pk"] for g in sa_cfg["groups"] if g in service_admin_groups]
         status, sa_user = api("PATCH", f"/api/v3/core/users/{sa_user['pk']}/", {
-            "attributes": sa_cfg["attributes"],
+            "attributes": sa_attributes,
-            "groups": group_pks,
             "is_active": True,
         })
         if status not in (200, 201):
             die(f"PATCH service account {sa_name} -> {status} {sa_user}")
-        print(f"  {sa_name}: updated existing service account")
+        print(f"  {sa_name}: updated (roles: {sa_cfg['roles']})")
     else:
         # Create new service account user
-        group_pks = [service_admin_groups[g]["pk"] for g in sa_cfg["groups"] if g in service_admin_groups]
         status, sa_user = api("POST", "/api/v3/core/users/", {
             "username": sa_name,
             "name": f"Service Account: {sa_name}",
             "type": "service_account",
             "path": "service-accounts",
             "is_active": True,
-            "attributes": sa_cfg["attributes"],
+            "attributes": sa_attributes,
-            "groups": group_pks,
         })
         if status not in (200, 201):
             die(f"POST service account {sa_name} -> {status} {sa_user}")
-        print(f"  {sa_name}: created new service account")
+        print(f"  {sa_name}: created (roles: {sa_cfg['roles']})")
     
-    # Create OAuth provider for this service account (client_credentials flow)
+    print(f"  {sa_name}: roles={sa_cfg['roles']}")
+    
+    # Create OAuth provider for this service account
+    # Supports both password grant (user claims) and client_credentials (fallback)
+    # Password grant requires authentication_flow and app_password token
     sa_client_secret = existing_secret or gen_secret(40)
+    sa_grant_types = ["password", "client_credentials", "refresh_token"]
     sa_provider = get_or_create(
         "/api/v3/providers/oauth2/", "/api/v3/providers/oauth2/",
         f"name={sa_name}",
@@ -794,17 +877,19 @@ for sa_name, sa_cfg in SERVICE_ACCOUNTS.items():
             "client_secret": sa_client_secret,
             "client_type": "confidential",
             "authorization_flow": AUTHORIZATION_FLOW_PK,
+            "authentication_flow": SA_AUTHENTICATION_FLOW_PK,  # Headless flow for password grant
             "invalidation_flow": INVALIDATION_FLOW_PK,
             "signing_key": SIGNING_KEY_PK,
-            "property_mappings": SCOPE_PKS + [MEMORY_MAPPING_PK],
+            "property_mappings": SCOPE_PKS + [ROLES_MAPPING_PK, MEMORY_MAPPING_PK],
             "sub_mode": "hashed_user_id",
             "include_claims_in_id_token": True,
-            "grant_types": ["client_credentials"],
+            "grant_types": sa_grant_types,
-            "redirect_uris": [],  # No redirects for client_credentials
+            "redirect_uris": [],
         },
         patch_existing={
-            "property_mappings": SCOPE_PKS + [MEMORY_MAPPING_PK],
+            "property_mappings": SCOPE_PKS + [ROLES_MAPPING_PK, MEMORY_MAPPING_PK],
-            "grant_types": ["client_credentials"],
+            "grant_types": sa_grant_types,
+            "authentication_flow": SA_AUTHENTICATION_FLOW_PK,
         },
     )
     
@@ -819,17 +904,70 @@ for sa_name, sa_cfg in SERVICE_ACCOUNTS.items():
             "meta_launch_url": "",
         },
     )
-    app_pks_for_binding.append((sa_name, sa_application["pk"]))
+    
+    # Service account apps have NO policy bindings - client_secret is the access control.
+    # Clean up any existing bindings (from old provisioning logic).
+    status, existing_bindings = api("GET", f"/api/v3/policies/bindings/?target={sa_application['pk']}")
+    if status == 200:
+        for binding in existing_bindings.get("results", []):
+            api("DELETE", f"/api/v3/policies/bindings/{binding['pk']}/")
+            print(f"  {sa_name}: removed stale binding {binding.get('group_obj', {}).get('name', binding['pk'])}")
+    
+    print(f"  {sa_name}: provider pk={sa_provider['pk']} application pk={sa_application['pk']}")
+    
+    # Create App Password token for password grant (optional, client_credentials also works)
+    # Authentik's password grant checks against Token with intent=app_password, not user password
+    # Note: API doesn't allow setting key at creation, must use set_key endpoint after
+    existing_app_password = kubectl_get_secret_key(sa_cfg["secret_ns"], sa_cfg["secret_name"], "APP_PASSWORD")
+    if not existing_app_password:
+        token_identifier = f"{sa_name}-app-password"
+        
+        # Check if token exists
+        status, existing_tokens = api("GET", f"/api/v3/core/tokens/?identifier={token_identifier}")
+        if status == 200 and existing_tokens.get("results"):
+            # Token exists, just set a new key
+            pass
+        else:
+            # Create the token first
+            status, token_resp = api("POST", "/api/v3/core/tokens/", {
+                "identifier": token_identifier,
+                "user": sa_user["pk"],
+                "intent": "app_password",
+                "expiring": False,
+                "description": f"App password for {sa_name} OAuth password grant",
+            })
+            if status not in (200, 201):
+                print(f"  WARNING: Failed to create app password token for {sa_name}: {token_resp}")
+        
+        # Set the key via set_key endpoint (works on existing or newly created token)
+        app_password_key = gen_secret(40)
+        status, _ = api("POST", f"/api/v3/core/tokens/{token_identifier}/set_key/", {
+            "key": app_password_key,
+        })
+        if status not in (200, 204):
+            print(f"  WARNING: Failed to set app password key for {sa_name}")
+            app_password_key = None
+        else:
+            print(f"  {sa_name}: created/updated app password token")
+    else:
+        app_password_key = existing_app_password
+        print(f"  {sa_name}: reusing existing app password")
     
     # Store credentials in k8s Secret
-    if not existing_secret:
+    # Supports both password grant (APP_PASSWORD) and client_credentials (CLIENT_SECRET)
-        kubectl_create_secret(sa_cfg["secret_ns"], sa_cfg["secret_name"], {
+    secret_data = {
         "CLIENT_ID": sa_name,
         "CLIENT_SECRET": sa_client_secret,
+        "USERNAME": sa_name,
         "TOKEN_URL": "https://authentik.riotpiao.com/application/o/token/",
         "ISSUER": f"https://authentik.riotpiao.com/application/o/{sa_name}/",
-        })
+    }
-        print(f"  {sa_name}: generated credentials -> {sa_cfg['secret_ns']}/{sa_cfg['secret_name']}")
+    if app_password_key:
+        secret_data["APP_PASSWORD"] = app_password_key
+    
+    if not existing_secret or (app_password_key and not existing_app_password):
+        kubectl_create_secret(sa_cfg["secret_ns"], sa_cfg["secret_name"], secret_data)
+        print(f"  {sa_name}: stored credentials -> {sa_cfg['secret_ns']}/{sa_cfg['secret_name']}")
     else:
         print(f"  {sa_name}: reusing existing credentials from {sa_cfg['secret_ns']}/{sa_cfg['secret_name']}")