diff --git a/k8s/infrastructure/minio/buckets.yaml b/k8s/infrastructure/minio/buckets.yaml deleted file mode 100644 index 4b363fd..0000000 --- a/k8s/infrastructure/minio/buckets.yaml +++ /dev/null @@ -1,65 +0,0 @@ ---- -# Buckets -apiVersion: minio.min.io/v1alpha1 -kind: Bucket -metadata: - name: riotpiao-models - namespace: storage -spec: - name: riotpiao-models - versioning: - enabled: true - tags: - app: ollama - layer: applications - purpose: model-storage - ---- -# Policies: define access scopes -apiVersion: minio.min.io/v1alpha1 -kind: Policy -metadata: - name: policy-ollama - namespace: storage -spec: - name: policy-ollama - statements: - - Effect: Allow - Action: - - s3:GetObject - - s3:PutObject - Resource: - - arn:aws:s3:::riotpiao-models/* - - Effect: Allow - Action: - - s3:ListBucket - Resource: - - arn:aws:s3:::riotpiao-models - ---- -# Users: service accounts with scoped access -apiVersion: minio.min.io/v1alpha1 -kind: User -metadata: - name: user-ollama - namespace: storage -spec: - accessKey: ollama-access-key - secretKey: ollama-secret-key-changeme # Override via Secret - policies: - - policy-ollama - status: enabled - memberOf: [] - ---- -# Future buckets -# - terraform-state: for Terraform state -# - logs: for application logs -# - artifacts: for CI/CD artifacts -# - backups: for database backups - -# Future users -# - terraform: read/write terraform-state -# - logging: write logs -# - ci-cd: write artifacts -# - backups: write backups diff --git a/k8s/infrastructure/minio/kustomization.yaml b/k8s/infrastructure/minio/kustomization.yaml index 3cb798a..fa50943 100644 --- a/k8s/infrastructure/minio/kustomization.yaml +++ b/k8s/infrastructure/minio/kustomization.yaml @@ -3,7 +3,7 @@ kind: Kustomization namespace: storage resources: - minio-tenant.yaml - - minio-service.yaml - - minio-replication-job.yaml - - buckets.yaml -# Legacy: minio-legacy-alias.yaml skipped (migration artifact, conflicts with namespace transform) + - minio-policy-job.yaml +# The operator creates the minio S3/console/headless Services and the +# declarative bucket + user from the Tenant spec — no hand-rolled Service or +# Bucket/User CRs (those kinds don't exist in the operator CRD set). diff --git a/k8s/infrastructure/minio/minio-legacy-alias.yaml b/k8s/infrastructure/minio/minio-legacy-alias.yaml deleted file mode 100644 index 6baaba8..0000000 --- a/k8s/infrastructure/minio/minio-legacy-alias.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Backward-compatibility alias: anything still pointing at the old -# minio.logging.svc.cluster.local address resolves to the universal -# storage frontend. Apply AFTER `helm uninstall minio -n logging` -# (the old release owns the Service name `minio` until then). -apiVersion: v1 -kind: Service -metadata: - name: minio - namespace: logging - labels: - app: minio -spec: - type: ExternalName - externalName: minio.storage.svc.cluster.local diff --git a/k8s/infrastructure/minio/minio-policy-job.yaml b/k8s/infrastructure/minio/minio-policy-job.yaml new file mode 100644 index 0000000..8e04fd1 --- /dev/null +++ b/k8s/infrastructure/minio/minio-policy-job.yaml @@ -0,0 +1,85 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: minio-policy-setup + namespace: storage + annotations: + # Run after the Tenant (and its declarative bucket/user) exist. + argocd.argoproj.io/sync-wave: "2" + argocd.argoproj.io/hook: PostSync + argocd.argoproj.io/hook-delete-policy: HookSucceeded +spec: + ttlSecondsAfterFinished: 600 + backoffLimit: 10 + template: + spec: + restartPolicy: OnFailure + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + securityContext: + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: mc + image: minio/mc:latest + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + command: + - /bin/sh + - -c + - | + set -e + + # Wait for the tenant S3 endpoint to answer. + until mc alias set local "http://minio.storage.svc.cluster.local:9000" \ + "$ROOT_USER" "$ROOT_PASSWORD" 2>/dev/null; do + echo "waiting for minio..."; sleep 5 + done + + # Scoped policy for the ollama service user: read/write its bucket only. + cat >/tmp/policy-ollama.json <<'JSON' + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject"], + "Resource": ["arn:aws:s3:::riotpiao-models/*"] + }, + { + "Effect": "Allow", + "Action": ["s3:ListBucket"], + "Resource": ["arn:aws:s3:::riotpiao-models"] + } + ] + } + JSON + + mc admin policy create local policy-ollama /tmp/policy-ollama.json || true + + # Attach the policy to the declaratively-created user. + mc admin policy attach local policy-ollama --user "$OLLAMA_ACCESS_KEY" || true + + echo "ollama policy configured" + env: + - name: ROOT_USER + valueFrom: + secretKeyRef: + name: minio-creds + key: accesskey + - name: ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: minio-creds + key: secretkey + - name: OLLAMA_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio-user-ollama + key: CONSOLE_ACCESS_KEY diff --git a/k8s/infrastructure/minio/minio-replication-job.yaml b/k8s/infrastructure/minio/minio-replication-job.yaml deleted file mode 100644 index db3271e..0000000 --- a/k8s/infrastructure/minio/minio-replication-job.yaml +++ /dev/null @@ -1,75 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: minio-site-replication-setup - namespace: storage -spec: - # Auto-delete the Job pod 10 minutes after completion - ttlSecondsAfterFinished: 600 - template: - spec: - restartPolicy: OnFailure - containers: - - name: mc - image: minio/mc:latest - command: - - /bin/sh - - -c - - | - set -e - - # Alias every site. Service names follow the release naming - # convention minio-.storage.svc.cluster.local. - for site in $SITES; do - mc alias set "$site" "http://minio-${site}.storage.svc.cluster.local:9000" \ - "$ROOT_USER" "$ROOT_PASSWORD" - done - - first=$(echo $SITES | cut -d' ' -f1) - - # Idempotent: skip only if every requested site is already part - # of the replication group. A partially-configured group (e.g. - # az-c newly added to SITES) falls through to `replicate add`, - # which expands an existing group in place. - # NOTE: pure-shell matching — the minio/mc image has no grep. - info=$(mc admin replicate info "$first" 2>/dev/null || true) - missing=0 - for site in $SITES; do - case "$info" in - *"$site"*) ;; - *) missing=1 ;; - esac - done - if [ "$missing" -eq 0 ]; then - echo "Site replication already spans all sites ($SITES) — nothing to do" - mc admin replicate info "$first" - exit 0 - fi - - echo "Configuring site replication across: $SITES" - mc admin replicate add $SITES - - echo "Replication status:" - mc admin replicate info "$first" - env: - # Space-separated list of replication sites, named after the AZ - # node labels. Each site must have a Helm release minio- - # (e.g. minio-az-a) so the Service DNS resolves. To add an AZ - # later: deploy minio-az-, append "az-" here, then delete - # and re-apply this Job. - - name: SITES - value: "az-a az-b az-c" - # MinIO site replication requires identical root credentials on - # every site, so reading one release's Secret covers all of them. - # The MinIO chart creates a Secret named after the release with - # keys rootUser and rootPassword. - - name: ROOT_USER - valueFrom: - secretKeyRef: - name: minio-az-a - key: rootUser - - name: ROOT_PASSWORD - valueFrom: - secretKeyRef: - name: minio-az-a - key: rootPassword diff --git a/k8s/infrastructure/minio/minio-secrets.enc.yaml b/k8s/infrastructure/minio/minio-secrets.enc.yaml new file mode 100644 index 0000000..db3c190 --- /dev/null +++ b/k8s/infrastructure/minio/minio-secrets.enc.yaml @@ -0,0 +1,73 @@ +apiVersion: ENC[AES256_GCM,data:w/A=,iv:Qum0Jux4oTEBSsbhDhFJHjoI9SsIAYpQazHaGCpNi4Y=,tag:9QGbAc8+zx1kpHdsmvY2YQ==,type:str] +kind: ENC[AES256_GCM,data:gNunoeDN,iv:baVsCtIdMzvIxiDYUWJZofazYmy9IxJ7Qy7iq0423LU=,tag:yFhxqH6pjZcRg8wYu+Euig==,type:str] +metadata: + name: ENC[AES256_GCM,data:PEmF8pnwETVjnys=,iv:qhljxXFqi2YInSuh8umcMzexekEombvc+P7vjdis4us=,tag:Z0P3hvrrCqACd1X2kyWuJg==,type:str] + namespace: ENC[AES256_GCM,data:+xu5BsH0rQ==,iv:QXGrT2y8YRlPloD3vtIEWi6KKEulapx0qDReKBcx94s=,tag:lJeuU8ze4S0nCvD6HDO8vQ==,type:str] +type: ENC[AES256_GCM,data:SGp0jv5g,iv:0cGTcnKMpo9szxciiq3lVmjKu9PGEBei4DIRg5ykCSk=,tag:zVNwJrGzjwFiEJ8YsJl4OA==,type:str] +stringData: + accesskey: ENC[AES256_GCM,data:RDGA1yzgfLp4Qg==,iv:6ahifHVN0+oLcPK+wU9zQ6mzCfYletVa6OrLMgGjvZU=,tag:aTjrX9GTOWhQ1JAdD7mkdw==,type:str] + secretkey: ENC[AES256_GCM,data:1TBLP5i7RcHBqpSRrQwYRS+R5eZhEx2NmM8Pe5/ffg4PmEWW/O4hHirpcyg=,iv:uUCaDPzBRsbDJjJKiaT1Cl+LAjaRTtih1QQNbMB2UkU=,tag:UIp5CsdPec1NoJ5Mr1p07Q==,type:str] +sops: + age: + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBLN1Y4QmxCMlgxeHlWQ3M1 + U2ZJd20vQVRISzc2eFVwamplSi8vOEVxWGw4CjFVc2twMllGUURKaFl4Y2ZqTjRq + aFVMbHpuNWR0VHpQMlNyUkM5dFFnS1kKLS0tIFhRU09TM0xZTG0rR1dOM0IzQ3NL + V0loSURhSHh2alVPdWNCMFFqNUpPK1UKLZF4LxAfFjj55v4GXki3NVa4d+uGjCoM + rx4bv8VBsrgCXUewv97R2CK2UYTn5BwzIP3XEFQ6GYh7aW58ilU5Ww== + -----END AGE ENCRYPTED FILE----- + recipient: age1smu533f803gmd0jq60s2zaj9zlznajy0ca6rtewd4r37mr2hs3uqsrldfh + lastmodified: "2026-07-20T19:27:04Z" + mac: ENC[AES256_GCM,data:LLAp9hGlb/OvPRsTHoCZvYh7VMOUPY6Bv3FUJrM0CE0+E2TRR2Y4QwfDrHhWIW/z7ODgeSGScFBXvkMjVnuFgjXW3HHvfLDrW58BxaBOlPkOxliur8DTJ+P+ow9LGUwPAM4rAoeD05/fAQeZiRLFtEzMcoqBQBGEMXyAtUKE6kU=,iv:TWtlzUoqAZCORgRFwcPCn6XfV/m7WcB4AcOMMn0iA/w=,tag:EoABMArHUUTiYoTja6CUwQ==,type:str] + unencrypted_suffix: _unencrypted + version: 3.13.2 +--- +apiVersion: ENC[AES256_GCM,data:nT4=,iv:oDoOIsSdLse/e5NWZfB77VDCKZg7H1ryo2G1yO/kMSQ=,tag:mh0742mY6JkziEDjVvJraQ==,type:str] +kind: ENC[AES256_GCM,data:/b9jpUAE,iv:t9ePbTrB312I/DbwwU2+TbPdadu017NRCf86tB3xgo8=,tag:bQ4kkgIdK+ok+Q8Ib5RQWQ==,type:str] +metadata: + name: ENC[AES256_GCM,data:njhz0CfjRVGOsA==,iv:SaHtjjPxAW+RWZ3++5ki9coI042/ZfDVbzvBhRjf9Qs=,tag:6/P18CD8nsMSLfdSKS3gwA==,type:str] + namespace: ENC[AES256_GCM,data:2uvto8XrRQ==,iv:kFe9jRZQPxh2e4xYeGG1qbJ1wv1oIGiwHqDb26EF7ZQ=,tag:YtfAVMk03Nutfqcz7yLj/Q==,type:str] +type: ENC[AES256_GCM,data:UcCLVcHy,iv:QbcI8GveNxBPirq6WvbCfm+ToTsjFCYsLKgd7q4/6Ns=,tag:HyhZMa8++GR56Kr66SGukw==,type:str] +stringData: + MINIO_IDENTITY_OPENID_CLIENT_SECRET: ENC[AES256_GCM,data:AfQjsyC5sVF7CbUZd04lRsMGWoUASagUjcQr+KbhMfvfwjKXMxSu9Zpt2kDxTaIwm+CaUnvWf01G1n4oj5wPPA==,iv:QmZDkftfSs2h+ktlHcDNQJJzuVtUi5MnKaYAD+7ZzeA=,tag:VZgouvb5sLighY73k1ssFg==,type:str] +sops: + age: + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBLN1Y4QmxCMlgxeHlWQ3M1 + U2ZJd20vQVRISzc2eFVwamplSi8vOEVxWGw4CjFVc2twMllGUURKaFl4Y2ZqTjRq + aFVMbHpuNWR0VHpQMlNyUkM5dFFnS1kKLS0tIFhRU09TM0xZTG0rR1dOM0IzQ3NL + V0loSURhSHh2alVPdWNCMFFqNUpPK1UKLZF4LxAfFjj55v4GXki3NVa4d+uGjCoM + rx4bv8VBsrgCXUewv97R2CK2UYTn5BwzIP3XEFQ6GYh7aW58ilU5Ww== + -----END AGE ENCRYPTED FILE----- + recipient: age1smu533f803gmd0jq60s2zaj9zlznajy0ca6rtewd4r37mr2hs3uqsrldfh + lastmodified: "2026-07-20T19:27:04Z" + mac: ENC[AES256_GCM,data:LLAp9hGlb/OvPRsTHoCZvYh7VMOUPY6Bv3FUJrM0CE0+E2TRR2Y4QwfDrHhWIW/z7ODgeSGScFBXvkMjVnuFgjXW3HHvfLDrW58BxaBOlPkOxliur8DTJ+P+ow9LGUwPAM4rAoeD05/fAQeZiRLFtEzMcoqBQBGEMXyAtUKE6kU=,iv:TWtlzUoqAZCORgRFwcPCn6XfV/m7WcB4AcOMMn0iA/w=,tag:EoABMArHUUTiYoTja6CUwQ==,type:str] + unencrypted_suffix: _unencrypted + version: 3.13.2 +--- +apiVersion: ENC[AES256_GCM,data:Z/g=,iv:EbbztYw9O2eWwDxlFFEaBgmAnIhN9mmwA8GPzZLZF6I=,tag:2Xe1qSCcaCLtC/CYgDT21Q==,type:str] +kind: ENC[AES256_GCM,data:DkLVktg5,iv:e5g6/fBulvT1/1Jjd+8UWOsVCk1mWntGvSXLJxzf1Lo=,tag:x8XB6vxsezq/8hkqGUjMBA==,type:str] +metadata: + name: ENC[AES256_GCM,data:6CsuJOz36cL1qqTdm/Ry3gU=,iv:WzCfjwMQMonViXgOvr8s0ekp/49vuhOcv6xH/diV8i0=,tag:EHMnvtfC2zB5hWkgkVOHaw==,type:str] + namespace: ENC[AES256_GCM,data:rYh2ReE2Hw==,iv:j1kYlDhZYGcjn/CXMSq0ZBtht/jPLb/Bzle2NCWbe1Y=,tag:u7MiEnxVFKeycEfS/MLeSw==,type:str] +type: ENC[AES256_GCM,data:8oyW6AP2,iv:qEjwjABAu7AJz/4h17JnwH6VYPU07onKfUparanC0Zs=,tag:HrH2IJHxnktY8wvWNWNMHQ==,type:str] +stringData: + CONSOLE_ACCESS_KEY: ENC[AES256_GCM,data:VtXwL7ngWkb3mDucpRtigqE=,iv:+r7DQLbjVMU82Ig8nlOwIgFEA2wE3KpMVtXIzpjE/o4=,tag:p3/7UOOCvH9qnOPN+sTCmw==,type:str] + CONSOLE_SECRET_KEY: ENC[AES256_GCM,data:CG09bCWQP8zP++t8iYn3jIietAikzLOJhsYSfb1kq+s=,iv:svztBK1rZky5YQjWc4q2YzHQNJO5u8rhSsZHbGLnhnw=,tag:m0lGFXshydA596KAqMHbzg==,type:str] +sops: + age: + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBLN1Y4QmxCMlgxeHlWQ3M1 + U2ZJd20vQVRISzc2eFVwamplSi8vOEVxWGw4CjFVc2twMllGUURKaFl4Y2ZqTjRq + aFVMbHpuNWR0VHpQMlNyUkM5dFFnS1kKLS0tIFhRU09TM0xZTG0rR1dOM0IzQ3NL + V0loSURhSHh2alVPdWNCMFFqNUpPK1UKLZF4LxAfFjj55v4GXki3NVa4d+uGjCoM + rx4bv8VBsrgCXUewv97R2CK2UYTn5BwzIP3XEFQ6GYh7aW58ilU5Ww== + -----END AGE ENCRYPTED FILE----- + recipient: age1smu533f803gmd0jq60s2zaj9zlznajy0ca6rtewd4r37mr2hs3uqsrldfh + lastmodified: "2026-07-20T19:27:04Z" + mac: ENC[AES256_GCM,data:LLAp9hGlb/OvPRsTHoCZvYh7VMOUPY6Bv3FUJrM0CE0+E2TRR2Y4QwfDrHhWIW/z7ODgeSGScFBXvkMjVnuFgjXW3HHvfLDrW58BxaBOlPkOxliur8DTJ+P+ow9LGUwPAM4rAoeD05/fAQeZiRLFtEzMcoqBQBGEMXyAtUKE6kU=,iv:TWtlzUoqAZCORgRFwcPCn6XfV/m7WcB4AcOMMn0iA/w=,tag:EoABMArHUUTiYoTja6CUwQ==,type:str] + unencrypted_suffix: _unencrypted + version: 3.13.2 diff --git a/k8s/infrastructure/minio/minio-service.yaml b/k8s/infrastructure/minio/minio-service.yaml deleted file mode 100644 index cceef77..0000000 --- a/k8s/infrastructure/minio/minio-service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: minio - namespace: storage -spec: - type: ClusterIP - # Helm chart selector — routes to minio-az-a/b/c pods - selector: - app: minio - ports: - - name: s3 - port: 9000 - targetPort: 9000 - - name: console - port: 9001 - targetPort: 9001 diff --git a/k8s/infrastructure/minio/minio-tenant.yaml b/k8s/infrastructure/minio/minio-tenant.yaml index 2b28784..0261156 100644 --- a/k8s/infrastructure/minio/minio-tenant.yaml +++ b/k8s/infrastructure/minio/minio-tenant.yaml @@ -5,19 +5,27 @@ metadata: namespace: storage labels: app: minio + annotations: + # Let the operator own bucket/user provisioning declaratively. + prometheus.io/path: /minio/v2/metrics/cluster + prometheus.io/port: "9000" + prometheus.io/scrape: "true" spec: image: minio/minio:RELEASE.2024-06-13T20-48-48Z + # Root credentials (rootUser/rootPassword). Created out-of-band — see + # BOOTSTRAP.md / SOPS. Operator reads CONSOLE_ACCESS_KEY style from here. credsSecret: name: minio-creds - # 3-zone distributed cluster (one server per zone) + # ── Single pool on the sole storage/scheduling node (talos-cp-1, az-a) ────── + # Per the 3-CP topology only talos-cp-1 is schedulable and holds Longhorn, so + # MinIO is a single-server tenant. 4 volumes give erasure-coded durability + # (MinIO's minimum for parity) on that one node. pools: - name: az-a servers: 1 - volumesPerServer: 1 - size: 100Gi - storageClass: longhorn + volumesPerServer: 4 affinity: nodeAffinity: @@ -33,31 +41,13 @@ spec: operator: Exists effect: NoSchedule - volumeClaimTemplate: - metadata: - name: data - spec: - accessModes: - - ReadWriteOnce - storageClassName: longhorn - resources: - requests: - storage: 100Gi - - - name: az-b - servers: 1 - volumesPerServer: 1 - size: 100Gi - storageClass: longhorn - - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: topology.kubernetes.io/zone - operator: In - values: [az-b] + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "1" + memory: 1Gi volumeClaimTemplate: metadata: @@ -68,47 +58,28 @@ spec: storageClassName: longhorn resources: requests: - storage: 100Gi + storage: 25Gi - - name: az-c - servers: 1 - volumesPerServer: 1 - size: 100Gi - storageClass: longhorn + # ── Declarative buckets (operator creates on first boot) ──────────────────── + buckets: + - name: riotpiao-models - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: topology.kubernetes.io/zone - operator: In - values: [az-c] + # ── Declarative users (each references a Secret of the same name holding + # CONSOLE_ACCESS_KEY / CONSOLE_SECRET_KEY) ───────────────────────────── + users: + - name: minio-user-ollama - volumeClaimTemplate: - metadata: - name: data - spec: - accessModes: - - ReadWriteOnce - storageClassName: longhorn - resources: - requests: - storage: 100Gi + # Expose ServiceMonitor for the kube-prometheus-stack. + prometheusOperator: true - # Console (web UI) - console: - image: minio/console:v0.30.0 - replicas: 1 - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - cpu: 500m - memory: 512Mi + # Public hostnames the tenant serves (S3 + console via the cluster ingress). + features: + domains: + minio: + - https://minio.riotpiao.com + console: https://minio-console.riotpiao.com - # Environment variables — OIDC config + # ── OIDC via Authentik (server-side env, valid in v2 schema) ──────────────── env: - name: MINIO_IDENTITY_OPENID_CONFIG_URL value: "https://authentik.riotpiao.com/application/o/minio/.well-known/openid-configuration" @@ -128,14 +99,5 @@ spec: - name: MINIO_IDENTITY_OPENID_DISPLAY_NAME value: "Authentik" - # Metrics - metrics: - enabled: true - port: 9000 - - # No auto-TLS (using cert-manager) + # cert-manager handles TLS; no operator auto-cert. requestAutoCert: false - - # No built-in ingress - ingress: - enabled: false