homelab/k8s/infra/minio/minio-tenant.yaml

apiVersion: minio.min.io/v2
kind: Tenant
metadata:
  name: minio-cluster
  namespace: storage
  labels:
    app: minio
  annotations:
    # Let the operator own bucket/user provisioning declaratively.
    prometheus.io/path: /minio/v2/metrics/cluster
    prometheus.io/port: "9000"
    prometheus.io/scrape: "true"
spec:
  # Was minio/minio:RELEASE.2025-07-23T15-54-02Z — that Docker Hub repository no longer
  # exists (404 for every tag), so the tenant sat in ImagePullBackOff and Vault's S3
  # backend could not resolve minio-cluster-hl. Official images now need registry auth
  # (quay.io/minio/minio returns 401), so this pins a public mirror of the same upstream
  # release; swap it for the official image once MinIO registry credentials exist.
  image: coollabsio/minio:RELEASE.2025-10-15T17-29-55Z

  # Disable operator auto-TLS: MinIO serves plain HTTP internally on 9000.
  # External TLS is terminated at nginx ingress (wildcard riotpiao-com-tls cert).
  # Without this, MinIO auto-generates self-signed certs and serves HTTPS-only
  # on 9000, which breaks plain-HTTP internal clients like Vault's S3 backend
  # (they hang waiting for a TLS handshake that never completes on an HTTP request).
  requestAutoCert: false

  # Root credentials. v5 pods read `configuration` — a Secret whose `config.env`
  # key holds shell `export MINIO_ROOT_USER=...` lines. Created out-of-band
  # (SOPS), see minio-secrets.enc.yaml. NOTE: the operator health-monitor logs a
  # cosmetic "empty tenant credentials" warning (it greps for legacy
  # access_key/secret_key keys) — MinIO itself authenticates fine; ignore it.
  configuration:
    name: minio-creds

  # ── Single pool on the sole storage/scheduling node (talos-cp-1, az-a) ──────
  # Per the 3-CP topology only talos-cp-1 is schedulable and holds Longhorn, so
  # MinIO is a single-server tenant. 4 volumes give erasure-coded durability
  # (MinIO's minimum for parity) on that one node.
  pools:
    - name: az-a
      servers: 1
      volumesPerServer: 4

      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: topology.kubernetes.io/zone
                    operator: In
                    values: [az-a]

      tolerations:
        - key: node-role.kubernetes.io/control-plane
          operator: Exists
          effect: NoSchedule

      resources:
        requests:
          cpu: 250m
          memory: 512Mi
        limits:
          cpu: "1"
          memory: 1Gi

      volumeClaimTemplate:
        metadata:
          name: data
        spec:
          accessModes:
            - ReadWriteOnce
          storageClassName: longhorn
          resources:
            requests:
              storage: 25Gi

  # ── Declarative buckets (operator creates on first boot) ────────────────────
  buckets:
    - name: loki-chunks
    - name: loki-ruler
    - name: loki-admin
    - name: vault
    - name: paperless

  # Metrics are exposed at /minio/v2/metrics; scrape via a hand-rolled
  # ServiceMonitor in the monitoring stack rather than operator auto-wiring
  # (prometheusOperator:true makes the operator hunt for Prometheus in ns
  # 'default' and fail the reconcile).

  # Public hostnames the tenant serves (S3 + console via the cluster ingress).
  # Must match actual ingress backends: the "minio" Ingress (host
  # minio.riotpiao.com) routes to minio-cluster-console:9090, and "minio-api"
  # (host minio-api.riotpiao.com) routes to minio:9000. Previously these were
  # swapped, so MinIO's own console-domain validation rejected the OIDC
  # callback arriving on minio.riotpiao.com's Host header.
  features:
    domains:
      minio:
        - https://minio-api.riotpiao.com
      console: https://minio.riotpiao.com

  # ── OIDC via Authentik (server-side env, valid in v2 schema) ────────────────
  # Use in-cluster URL for config fetch (pod→authentik); browser redirects use
  # public URLs embedded in the OIDC metadata response (issuer stays public).
  env:
    - name: MINIO_IDENTITY_OPENID_CONFIG_URL
      # Must use external URL — well-known response contains external issuer/jwks_uri.
      # MinIO validates issuer in JWT matches well-known issuer. Internal URL = mismatch.
      # Hairpins through ingress-nginx but stays in-cluster.
      value: "https://authentik.riotpiao.com/application/o/minio/.well-known/openid-configuration"
    - name: MINIO_IDENTITY_OPENID_CLIENT_ID
      value: "minio"
    - name: MINIO_IDENTITY_OPENID_CLIENT_SECRET
      valueFrom:
        secretKeyRef:
          name: minio-oidc
          key: MINIO_IDENTITY_OPENID_CLIENT_SECRET
    - name: MINIO_IDENTITY_OPENID_CLAIM_NAME
      value: "groups"
    - name: MINIO_IDENTITY_OPENID_CLAIM_PREFIX
      value: ""
    - name: MINIO_IDENTITY_OPENID_REDIRECT_URI
      value: "https://minio.riotpiao.com/oauth_callback"
    - name: MINIO_IDENTITY_OPENID_DISPLAY_NAME
      value: "Authentik"
    - name: MINIO_IDENTITY_OPENID_SCOPES
      value: "openid,profile,email,groups"
    - name: MINIO_BROWSER_LOGIN_ANIMATION
      value: "off"
    - name: MINIO_BROWSER_REDIRECT_URL
      value: "https://minio.riotpiao.com"