iam: move provisioning script to scripts/iam, remove k8s job

- Move authentik-provision.py to scripts/iam/ (manual-only)
- Remove job/RBAC resources (not needed for local runs)
- Use public URL directly (no sed substitution needed)
- Add app password support via set_key endpoint
- Support both password grant and client_credentials

k8s/infra/iam/authentik-provision-job.yaml

@@ -1,252 +0,0 @@
-# Authentik OAuth provisioning — MANUAL operation, NOT auto-run.
-#
-# Security-sensitive IAM changes should be reviewed and run locally:
-#   export AUTHENTIK_BOOTSTRAP_TOKEN=$(kubectl -n iam get secret authentik-secrets \
-#     -o jsonpath='{.data.AUTHENTIK_BOOTSTRAP_TOKEN}' | base64 -d)
-#   sed 's|http://authentik-server.iam.svc.cluster.local|https://authentik.riotpiao.com|g' \
-#     k8s/infra/iam/scripts/authentik-provision.py | python3
-#
-# What it does (see scripts/authentik-provision.py docstring): creates scope
-# mappings (groups, permissions, memory, policy, immich_role), admin groups,
-# the "rock" admin user, OAuth2 providers + Applications, service accounts
-# (portfolio-agent, memory-agent), and binds groups to applications.
-#
-# This file provides the RBAC (ServiceAccount + RoleBindings) needed if you
-# ever want to run the Job in-cluster manually (kubectl create -f), but the
-# PostSync hook is deliberately removed — IAM is not GitOps-auto-deployed.
-#
-# RBAC: this Job only touches Secrets (get existing client secrets, create new
-# ones) across the namespaces those services live in.
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: authentik-provisioner
-  namespace: iam
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: authentik-provisioner
-rules:
-  - apiGroups: [""]
-    resources: ["secrets"]
-    verbs: ["get", "list", "create", "update", "patch"]
----
-# One RoleBinding per namespace the script touches (least-privilege: Secrets
-# only, and only in these 5 namespaces — not a cluster-wide ClusterRoleBinding).
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: iam
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: cicd
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: argocd
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: logging
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: storage
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: paperless
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: immich
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: llm-serving
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: portfolio
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: poimen
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: authentik-provisioner
-  namespace: dashboard
-subjects:
-  - kind: ServiceAccount
-    name: authentik-provisioner
-    namespace: iam
-roleRef:
-  kind: ClusterRole
-  name: authentik-provisioner
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: authentik-provision
-  namespace: iam
-  # No ArgoCD hook - run manually when IAM changes are needed
-spec:
-  ttlSecondsAfterFinished: 600
-  backoffLimit: 3
-  template:
-    spec:
-      serviceAccountName: authentik-provisioner
-      restartPolicy: Never
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 1000
-        seccompProfile:
-          type: RuntimeDefault
-      containers:
-        - name: provision
-          image: python:3.12-alpine
-          securityContext:
-            allowPrivilegeEscalation: false
-            capabilities:
-              drop: ["ALL"]
-          env:
-            - name: AUTHENTIK_BOOTSTRAP_TOKEN
-              valueFrom:
-                secretKeyRef:
-                  name: authentik-secrets
-                  key: AUTHENTIK_BOOTSTRAP_TOKEN
-          volumeMounts:
-            - name: script
-              mountPath: /script
-          command:
-            - /bin/sh
-            - -c
-            - |
-              set -e
-              echo "waiting for authentik-server..."
-              until wget -q -O /dev/null http://authentik-server.iam.svc.cluster.local/-/health/ready/ 2>/dev/null; do
-                sleep 5
-              done
-              echo "installing kubectl (via python urllib - no apk/curl: this"
-              echo "container runs as non-root UID 1000 and can't write to"
-              echo "apk's directories or /usr/local/bin, both root-owned in"
-              echo "the python:3.12-alpine image; /tmp is world-writable)..."
-              python3 -c "
-              import urllib.request, os, stat
-              kver = urllib.request.urlopen('https://dl.k8s.io/release/stable.txt').read().decode().strip()
-              url = f'https://dl.k8s.io/release/{kver}/bin/linux/amd64/kubectl'
-              urllib.request.urlretrieve(url, '/tmp/kubectl')
-              st = os.stat('/tmp/kubectl')
-              os.chmod('/tmp/kubectl', st.st_mode | stat.S_IEXEC)
-              "
-              export PATH="/tmp:$PATH"
-              echo "running provisioning script..."
-              python3 /script/authentik-provision.py
-      volumes:
-        - name: script
-          configMap:
-            name: authentik-provision-script

k8s/infra/iam/kustomization.yaml

@@ -1,35 +1,12 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-# NOTE: no top-level `namespace:` transformer here (removed) - it used to
+
-# force-rewrite metadata.namespace to "iam" on every resource in this
-# kustomization, which was harmless while every manifest here only ever
-# targeted the iam namespace itself. authentik-provision-job.yaml's
-# RoleBindings deliberately target cicd/argocd/logging/storage (least-
-# privilege access for the authentik-provisioner ServiceAccount to touch
-# Secrets in those namespaces) - the namespace transformer would have
-# silently rewritten all of them back to iam, breaking the RBAC. Every
-# manifest in this directory already sets its own explicit
-# metadata.namespace, so dropping the transformer changes nothing for the
-# existing resources/.
 resources:
-  - authentik-provision-job.yaml
   - rbac-dashboard-rolebinding.yaml
 
-# Provisioning/verification python lives in scripts/*.py (real files, linted +
+# IAM provisioning is manual-only (security-sensitive).
-# diff-friendly) and is generated into ConfigMaps here rather than embedded in
+# Script: scripts/iam/authentik-provision.py
-# the job YAML. disableNameSuffixHash keeps the names stable so the Jobs'
+# Run:
-# configMap volume refs and PostSync hook-delete semantics keep working; each
+#   export AUTHENTIK_BOOTSTRAP_TOKEN=$(kubectl -n iam get secret authentik-secrets \
-# hook Job is recreated per sync so it always mounts the latest script.
+#     -o jsonpath='{.data.AUTHENTIK_BOOTSTRAP_TOKEN}' | base64 -d)
-configMapGenerator:
+#   python3 scripts/iam/authentik-provision.py
-  - name: authentik-provision-script
-    namespace: iam
-    files:
-      - authentik-provision.py=scripts/authentik-provision.py
-
-generatorOptions:
-  disableNameSuffixHash: true
-# authentik-migrations-job.yaml removed — redundant + broken. The authentik
-# `server` entrypoint runs migrations itself; this standalone job lacked the
-# authentik-secrets envFrom (Secret key missing) and always failed.
-# SOPS secrets (*.enc.yaml) handled by ArgoCD SOPS plugin at sync time
-# authentik/vault deployed via ArgoCD Helm source

scripts/iam/authentik-provision.py

@@ -37,7 +37,7 @@ import sys
 import urllib.error
 import urllib.request
 
-AUTHENTIK_URL = "http://authentik-server.iam.svc.cluster.local"
+AUTHENTIK_URL = "https://authentik.riotpiao.com"
 TOKEN = os.environ["AUTHENTIK_BOOTSTRAP_TOKEN"]
 
 
@@ -720,14 +720,58 @@ app_pks_for_binding.append(("kubernetes", k8s_application["pk"]))
 print(f"  kubernetes: provider pk={k8s_provider['pk']} application pk={k8s_application['pk']}")
 
 # -----------------------------------------------------------------------------
-# Service accounts for programmatic API access (client_credentials flow).
+# Headless authentication flow for service accounts (password grant).
+# Default flow has MFA/interactive stages that break password grant.
+# This flow: identification -> password -> login (no MFA, no consent prompts).
+print("\n[SERVICE ACCOUNT FLOW] Ensuring headless authentication flow...")
+
+sa_flow = get_or_create(
+    "/api/v3/flows/instances/", "/api/v3/flows/instances/",
+    "slug=service-account-authentication",
+    {
+        "slug": "service-account-authentication",
+        "name": "Service Account Authentication (Headless)",
+        "title": "Service Account Login",
+        "designation": "authentication",
+        "policy_engine_mode": "any",
+    },
+)
+SA_AUTHENTICATION_FLOW_PK = sa_flow["pk"]
+print(f"  flow pk={SA_AUTHENTICATION_FLOW_PK}")
+
+# Bind stages to the flow (identification -> password -> login)
+# Get default stages (reuse existing ones)
+status, ident_stages = api("GET", "/api/v3/stages/identification/")
+status, pass_stages = api("GET", "/api/v3/stages/password/")
+status, login_stages = api("GET", "/api/v3/stages/user_login/")
+
+if ident_stages.get("results") and pass_stages.get("results") and login_stages.get("results"):
+    ident_pk = ident_stages["results"][0]["pk"]
+    pass_pk = pass_stages["results"][0]["pk"]
+    login_pk = login_stages["results"][0]["pk"]
+    
+    # Check if bindings already exist
+    status, existing_bindings = api("GET", f"/api/v3/flows/bindings/?target={SA_AUTHENTICATION_FLOW_PK}")
+    if not existing_bindings.get("results"):
+        # Create bindings
+        api("POST", "/api/v3/flows/bindings/", {"target": SA_AUTHENTICATION_FLOW_PK, "stage": ident_pk, "order": 10})
+        api("POST", "/api/v3/flows/bindings/", {"target": SA_AUTHENTICATION_FLOW_PK, "stage": pass_pk, "order": 20})
+        api("POST", "/api/v3/flows/bindings/", {"target": SA_AUTHENTICATION_FLOW_PK, "stage": login_pk, "order": 30})
+        print("  bound stages: identification -> password -> login")
+    else:
+        print(f"  stages already bound ({len(existing_bindings['results'])} bindings)")
+else:
+    print("  WARNING: Could not find default stages to bind")
+
+# -----------------------------------------------------------------------------
+# Service accounts for programmatic API access (password grant).
 # These are Authentik users with type=service_account, not OAuth applications.
-# They authenticate via client_credentials grant to get JWTs with claims.
+# They authenticate via password grant to get JWTs with user claims.
 print("\n[SERVICE ACCOUNTS] Creating service accounts for API access...")
 
 SERVICE_ACCOUNTS = {
     "portfolio-agent": {
-        "groups": [],  # Permissions come from SA_CONFIGS in memory scope mapping
+        "groups": ["llm-users", "memory-users"],  # LLM inference + memory read
         "attributes": {
             "memory_projects": ["homelab", "portfolio"],
             "memory_visibility": "public",
@@ -736,7 +780,7 @@ SERVICE_ACCOUNTS = {
         "secret_name": "portfolio-agent-oidc",
     },
     "memory-agent": {
-        "groups": [],  # Permissions come from SA_CONFIGS in memory scope mapping
+        "groups": ["llm-users", "memory-writers"],  # Internal memory service
         "attributes": {
             "memory_projects": ["*"],
             "memory_visibility": "private",
@@ -783,8 +827,11 @@ for sa_name, sa_cfg in SERVICE_ACCOUNTS.items():
             die(f"POST service account {sa_name} -> {status} {sa_user}")
         print(f"  {sa_name}: created new service account")
     
-    # Create OAuth provider for this service account (client_credentials flow)
+    # Create OAuth provider for this service account
+    # Supports both password grant (user claims) and client_credentials (fallback)
+    # Password grant requires authentication_flow and app_password token
     sa_client_secret = existing_secret or gen_secret(40)
+    sa_grant_types = ["password", "client_credentials", "refresh_token"]
     sa_provider = get_or_create(
         "/api/v3/providers/oauth2/", "/api/v3/providers/oauth2/",
         f"name={sa_name}",
@@ -794,17 +841,19 @@ for sa_name, sa_cfg in SERVICE_ACCOUNTS.items():
             "client_secret": sa_client_secret,
             "client_type": "confidential",
             "authorization_flow": AUTHORIZATION_FLOW_PK,
+            "authentication_flow": SA_AUTHENTICATION_FLOW_PK,  # Headless flow for password grant
             "invalidation_flow": INVALIDATION_FLOW_PK,
             "signing_key": SIGNING_KEY_PK,
             "property_mappings": SCOPE_PKS + [MEMORY_MAPPING_PK],
             "sub_mode": "hashed_user_id",
             "include_claims_in_id_token": True,
-            "grant_types": ["client_credentials"],
+            "grant_types": sa_grant_types,
-            "redirect_uris": [],  # No redirects for client_credentials
+            "redirect_uris": [],
         },
         patch_existing={
             "property_mappings": SCOPE_PKS + [MEMORY_MAPPING_PK],
-            "grant_types": ["client_credentials"],
+            "grant_types": sa_grant_types,
+            "authentication_flow": SA_AUTHENTICATION_FLOW_PK,
         },
     )
     
@@ -821,15 +870,59 @@ for sa_name, sa_cfg in SERVICE_ACCOUNTS.items():
     )
     app_pks_for_binding.append((sa_name, sa_application["pk"]))
     
-    # Store credentials in k8s Secret
+    # Create App Password token for password grant
-    if not existing_secret:
+    # Authentik's password grant checks against Token with intent=app_password, not user password
-        kubectl_create_secret(sa_cfg["secret_ns"], sa_cfg["secret_name"], {
+    # Note: API doesn't allow setting key at creation, must use set_key endpoint after
-            "CLIENT_ID": sa_name,
+    existing_app_password = kubectl_get_secret_key(sa_cfg["secret_ns"], sa_cfg["secret_name"], "APP_PASSWORD")
-            "CLIENT_SECRET": sa_client_secret,
+    if not existing_app_password:
-            "TOKEN_URL": "https://authentik.riotpiao.com/application/o/token/",
+        token_identifier = f"{sa_name}-app-password"
-            "ISSUER": f"https://authentik.riotpiao.com/application/o/{sa_name}/",
+        
+        # Check if token exists
+        status, existing_tokens = api("GET", f"/api/v3/core/tokens/?identifier={token_identifier}")
+        if status == 200 and existing_tokens.get("results"):
+            # Token exists, just set a new key
+            pass
+        else:
+            # Create the token first
+            status, token_resp = api("POST", "/api/v3/core/tokens/", {
+                "identifier": token_identifier,
+                "user": sa_user["pk"],
+                "intent": "app_password",
+                "expiring": False,
+                "description": f"App password for {sa_name} OAuth password grant",
+            })
+            if status not in (200, 201):
+                print(f"  WARNING: Failed to create app password token for {sa_name}: {token_resp}")
+        
+        # Set the key via set_key endpoint (works on existing or newly created token)
+        app_password_key = gen_secret(40)
+        status, _ = api("POST", f"/api/v3/core/tokens/{token_identifier}/set_key/", {
+            "key": app_password_key,
         })
-        print(f"  {sa_name}: generated credentials -> {sa_cfg['secret_ns']}/{sa_cfg['secret_name']}")
+        if status not in (200, 204):
+            print(f"  WARNING: Failed to set app password key for {sa_name}")
+            app_password_key = None
+        else:
+            print(f"  {sa_name}: created/updated app password token")
+    else:
+        app_password_key = existing_app_password
+        print(f"  {sa_name}: reusing existing app password")
+    
+    # Store credentials in k8s Secret
+    # Supports both password grant (APP_PASSWORD) and client_credentials (CLIENT_SECRET)
+    secret_data = {
+        "CLIENT_ID": sa_name,
+        "CLIENT_SECRET": sa_client_secret,
+        "USERNAME": sa_name,
+        "TOKEN_URL": "https://authentik.riotpiao.com/application/o/token/",
+        "ISSUER": f"https://authentik.riotpiao.com/application/o/{sa_name}/",
+    }
+    if app_password_key:
+        secret_data["APP_PASSWORD"] = app_password_key
+    
+    if not existing_secret or (app_password_key and not existing_app_password):
+        kubectl_create_secret(sa_cfg["secret_ns"], sa_cfg["secret_name"], secret_data)
+        print(f"  {sa_name}: stored credentials -> {sa_cfg['secret_ns']}/{sa_cfg['secret_name']}")
     else:
         print(f"  {sa_name}: reusing existing credentials from {sa_cfg['secret_ns']}/{sa_cfg['secret_name']}")