fix(minio): correct operator chart source + rewrite Tenant to v5 schema + config.env creds — tenant now boots

This commit is contained in:
Story Crater Bot
2026-08-18 15:08:02 -07:00
parent 2623eecfca
commit a6465f7158
10 changed files with 208 additions and 254 deletions
@@ -11,9 +11,9 @@ metadata:
spec: spec:
project: homelab project: homelab
sources: sources:
- repoURL: https://charts.min.io/ - repoURL: https://operator.min.io/
chart: operator chart: operator
targetRevision: "*" targetRevision: "5.0.18"
helm: helm:
valueFiles: valueFiles:
- $values/k8s/infrastructure/minio/minio-operator-values.yaml - $values/k8s/infrastructure/minio/minio-operator-values.yaml
+1
View File
@@ -16,6 +16,7 @@ spec:
- https://cloudnative-pg.github.io/charts - https://cloudnative-pg.github.io/charts
- https://dl.gitea.com/charts/ - https://dl.gitea.com/charts/
- https://charts.min.io/ - https://charts.min.io/
- https://operator.min.io/
- https://prometheus-community.github.io/helm-charts - https://prometheus-community.github.io/helm-charts
- https://grafana.github.io/helm-charts - https://grafana.github.io/helm-charts
- https://helm.releases.hashicorp.com - https://helm.releases.hashicorp.com
-65
View File
@@ -1,65 +0,0 @@
---
# Buckets
apiVersion: minio.min.io/v1alpha1
kind: Bucket
metadata:
name: riotpiao-models
namespace: storage
spec:
name: riotpiao-models
versioning:
enabled: true
tags:
app: ollama
layer: applications
purpose: model-storage
---
# Policies: define access scopes
apiVersion: minio.min.io/v1alpha1
kind: Policy
metadata:
name: policy-ollama
namespace: storage
spec:
name: policy-ollama
statements:
- Effect: Allow
Action:
- s3:GetObject
- s3:PutObject
Resource:
- arn:aws:s3:::riotpiao-models/*
- Effect: Allow
Action:
- s3:ListBucket
Resource:
- arn:aws:s3:::riotpiao-models
---
# Users: service accounts with scoped access
apiVersion: minio.min.io/v1alpha1
kind: User
metadata:
name: user-ollama
namespace: storage
spec:
accessKey: ollama-access-key
secretKey: ollama-secret-key-changeme # Override via Secret
policies:
- policy-ollama
status: enabled
memberOf: []
---
# Future buckets
# - terraform-state: for Terraform state
# - logs: for application logs
# - artifacts: for CI/CD artifacts
# - backups: for database backups
# Future users
# - terraform: read/write terraform-state
# - logging: write logs
# - ci-cd: write artifacts
# - backups: write backups
+4 -4
View File
@@ -3,7 +3,7 @@ kind: Kustomization
namespace: storage namespace: storage
resources: resources:
- minio-tenant.yaml - minio-tenant.yaml
- minio-service.yaml - minio-policy-job.yaml
- minio-replication-job.yaml # The operator creates the minio S3/console/headless Services and the
- buckets.yaml # declarative bucket + user from the Tenant spec — no hand-rolled Service or
# Legacy: minio-legacy-alias.yaml skipped (migration artifact, conflicts with namespace transform) # Bucket/User CRs (those kinds don't exist in the operator CRD set).
@@ -1,14 +0,0 @@
# Backward-compatibility alias: anything still pointing at the old
# minio.logging.svc.cluster.local address resolves to the universal
# storage frontend. Apply AFTER `helm uninstall minio -n logging`
# (the old release owns the Service name `minio` until then).
apiVersion: v1
kind: Service
metadata:
name: minio
namespace: logging
labels:
app: minio
spec:
type: ExternalName
externalName: minio.storage.svc.cluster.local
@@ -0,0 +1,85 @@
apiVersion: batch/v1
kind: Job
metadata:
name: minio-policy-setup
namespace: storage
annotations:
# Run after the Tenant (and its declarative bucket/user) exist.
argocd.argoproj.io/sync-wave: "2"
argocd.argoproj.io/hook: PostSync
argocd.argoproj.io/hook-delete-policy: HookSucceeded
spec:
ttlSecondsAfterFinished: 600
backoffLimit: 10
template:
spec:
restartPolicy: OnFailure
tolerations:
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
securityContext:
runAsNonRoot: true
runAsUser: 1000
seccompProfile:
type: RuntimeDefault
containers:
- name: mc
image: minio/mc:latest
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
command:
- /bin/sh
- -c
- |
set -e
# Wait for the tenant S3 endpoint to answer.
until mc alias set local "http://minio.storage.svc.cluster.local:9000" \
"$ROOT_USER" "$ROOT_PASSWORD" 2>/dev/null; do
echo "waiting for minio..."; sleep 5
done
# Scoped policy for the ollama service user: read/write its bucket only.
cat >/tmp/policy-ollama.json <<'JSON'
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:PutObject"],
"Resource": ["arn:aws:s3:::riotpiao-models/*"]
},
{
"Effect": "Allow",
"Action": ["s3:ListBucket"],
"Resource": ["arn:aws:s3:::riotpiao-models"]
}
]
}
JSON
mc admin policy create local policy-ollama /tmp/policy-ollama.json || true
# Attach the policy to the declaratively-created user.
mc admin policy attach local policy-ollama --user "$OLLAMA_ACCESS_KEY" || true
echo "ollama policy configured"
env:
- name: ROOT_USER
valueFrom:
secretKeyRef:
name: minio-creds
key: accesskey
- name: ROOT_PASSWORD
valueFrom:
secretKeyRef:
name: minio-creds
key: secretkey
- name: OLLAMA_ACCESS_KEY
valueFrom:
secretKeyRef:
name: minio-user-ollama
key: CONSOLE_ACCESS_KEY
@@ -1,75 +0,0 @@
apiVersion: batch/v1
kind: Job
metadata:
name: minio-site-replication-setup
namespace: storage
spec:
# Auto-delete the Job pod 10 minutes after completion
ttlSecondsAfterFinished: 600
template:
spec:
restartPolicy: OnFailure
containers:
- name: mc
image: minio/mc:latest
command:
- /bin/sh
- -c
- |
set -e
# Alias every site. Service names follow the release naming
# convention minio-<site>.storage.svc.cluster.local.
for site in $SITES; do
mc alias set "$site" "http://minio-${site}.storage.svc.cluster.local:9000" \
"$ROOT_USER" "$ROOT_PASSWORD"
done
first=$(echo $SITES | cut -d' ' -f1)
# Idempotent: skip only if every requested site is already part
# of the replication group. A partially-configured group (e.g.
# az-c newly added to SITES) falls through to `replicate add`,
# which expands an existing group in place.
# NOTE: pure-shell matching — the minio/mc image has no grep.
info=$(mc admin replicate info "$first" 2>/dev/null || true)
missing=0
for site in $SITES; do
case "$info" in
*"$site"*) ;;
*) missing=1 ;;
esac
done
if [ "$missing" -eq 0 ]; then
echo "Site replication already spans all sites ($SITES) — nothing to do"
mc admin replicate info "$first"
exit 0
fi
echo "Configuring site replication across: $SITES"
mc admin replicate add $SITES
echo "Replication status:"
mc admin replicate info "$first"
env:
# Space-separated list of replication sites, named after the AZ
# node labels. Each site must have a Helm release minio-<site>
# (e.g. minio-az-a) so the Service DNS resolves. To add an AZ
# later: deploy minio-az-<X>, append "az-<X>" here, then delete
# and re-apply this Job.
- name: SITES
value: "az-a az-b az-c"
# MinIO site replication requires identical root credentials on
# every site, so reading one release's Secret covers all of them.
# The MinIO chart creates a Secret named after the release with
# keys rootUser and rootPassword.
- name: ROOT_USER
valueFrom:
secretKeyRef:
name: minio-az-a
key: rootUser
- name: ROOT_PASSWORD
valueFrom:
secretKeyRef:
name: minio-az-a
key: rootPassword
@@ -0,0 +1,72 @@
apiVersion: ENC[AES256_GCM,data:nd0=,iv:fORqNsg82i9+5WRQkHi698q/qHI4A1JP+YEkOvMxEGw=,tag:bLdfd21vJn6Mn2k/ghx1iA==,type:str]
kind: ENC[AES256_GCM,data:aPTyakwz,iv:0Fviiz5ngsaBD5IRn30p6Z93cELcuLNCWILJJQSn19M=,tag:0n0f69t4ZC5WULd9VBI9sg==,type:str]
metadata:
name: ENC[AES256_GCM,data:UQ0lTL18hb7bZR4=,iv:NDGvM319FCypxFLWwONcF12osaU/EL0IwD0c2xtqYnU=,tag:JYu3EMyW7Pc4HTCmU8IiTw==,type:str]
namespace: ENC[AES256_GCM,data:H49h6xTVAQ==,iv:EgTi3jFfn+EYYxMTC1GCQmt1fuGGtVSD3fhytp+CiaM=,tag:ynf4iSonvnnaO5/Ujno3+A==,type:str]
type: ENC[AES256_GCM,data:xDlXeb2q,iv:0pe80oQG0ZIerOLvqbiorJ76JlCPu6wIjt4lQfbsXjU=,tag:rmaLgRUr/ERPzIXeUD9qMA==,type:str]
stringData:
config.env: ENC[AES256_GCM,data:rM2sfC9apy4Bnlw609d+BXAenZzUcBxceQmyzlbrvgasp37JfS6hIAWOf5Qq62ZF+/fN7nXSB5oEeajAIV3YJoP8wKlqm3765zOkUVhFkX2YmhJH4CkoJa6XzX9CowCQtTW9h+7F5PwaGnLptQ==,iv:0RQVMi2mMHBRTJ1OX6IDFvq9Qr7zpqgQzt+MPPgBIHA=,tag:zu45S2b29SM9JjqxsKQHow==,type:str]
sops:
age:
- enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBNZCtYcC9kanVtdFNSeksr
NDhXVHBpWjhKaWJSVnR1Rk9QV3BVU25mRldjCitEOVNCNnZVVjNLQWhFejdGelJV
V3Bhb3JTMU9xdEhOck00TGt4UFVaWTAKLS0tIEZ1VnFrVmkyTWZWR2hCc2hpWGVK
Z0NuYjVIdXJYRlVkY3JSUWUzZzd1bkkK0jm57duPu+u2qwjhKaO/fTtH1+Gt8fSm
vnLQsQZlkYZBYxHHwtBPC4JztYvmuKnPNYAPqRPfVB+9IEOpQEetkA==
-----END AGE ENCRYPTED FILE-----
recipient: age1smu533f803gmd0jq60s2zaj9zlznajy0ca6rtewd4r37mr2hs3uqsrldfh
lastmodified: "2026-07-20T19:41:57Z"
mac: ENC[AES256_GCM,data:fFoW//RJnlx0jQ3aMsEtx/qy70LrnocVc4UfPUIFNJG377NG7GnDxs+JIQGXSNNzRoKH06U3AFmBl2BvIi6D67A1VbOS0/+zmSGcNxdmL3apauOmn34jgJ/ipyakuawUi1+CGgSy8eMqgLM9QCpBkGx6qTtjjnGVWHYbN1WG3iE=,iv:8MD02ysiRL3nxSQvhQ6e0ek9gVLHwcfUCGzX+Tmtdp4=,tag:X/SdhJes1rJLHy6S278ztg==,type:str]
unencrypted_suffix: _unencrypted
version: 3.13.2
---
apiVersion: ENC[AES256_GCM,data:Utg=,iv:SqcUMt/REXm9qIppTVP8PQ6tnVrgt2gc5aghD7S8KHI=,tag:zeSno+VCPn9EXDE1Uxs8Hg==,type:str]
kind: ENC[AES256_GCM,data:nDUp5Fyy,iv:RVtkV8l4JvqOSUR/MlZHZxnR75+fAAR8gKPNbpOLWVk=,tag:AoNBqT3MVudiA56sIIqpnQ==,type:str]
metadata:
name: ENC[AES256_GCM,data:74hhzAFMvD+DuQ==,iv:wTw2eoooDA9EfB2df3jzb0IWUdtLSwcuKzx8yDBaihQ=,tag:mnnkksSy69Gtg4LyYYnjmg==,type:str]
namespace: ENC[AES256_GCM,data:z8dPkzIQkg==,iv:zoH4ZWOtD3tF6pkAuQDrL2rSfuRp+9Nu8IJq2q0eqsY=,tag:/39AaFYlXKxKkCuZpeHZlg==,type:str]
type: ENC[AES256_GCM,data:J7ALXn/x,iv:GUfgjXF1/z0jGCFdz5/ARVRQUBwrEAdpzfATiFNPVLs=,tag:HZem0E53fAykijNrthbs8Q==,type:str]
stringData:
MINIO_IDENTITY_OPENID_CLIENT_SECRET: ENC[AES256_GCM,data:w3SZTsxn59UfoS4rej78RToiQP3jzMOwUmNwjEFoPjkZAhHpKYX7Y4tYv5znt9h1cd2jGrorBVLf9YgIOihJxw==,iv:I1aGIikbHTKvh//4Inhp7UoNfhjlIXSpcyP4RqljvqM=,tag:mL3ayxC3PuvbgQosazgTow==,type:str]
sops:
age:
- enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBNZCtYcC9kanVtdFNSeksr
NDhXVHBpWjhKaWJSVnR1Rk9QV3BVU25mRldjCitEOVNCNnZVVjNLQWhFejdGelJV
V3Bhb3JTMU9xdEhOck00TGt4UFVaWTAKLS0tIEZ1VnFrVmkyTWZWR2hCc2hpWGVK
Z0NuYjVIdXJYRlVkY3JSUWUzZzd1bkkK0jm57duPu+u2qwjhKaO/fTtH1+Gt8fSm
vnLQsQZlkYZBYxHHwtBPC4JztYvmuKnPNYAPqRPfVB+9IEOpQEetkA==
-----END AGE ENCRYPTED FILE-----
recipient: age1smu533f803gmd0jq60s2zaj9zlznajy0ca6rtewd4r37mr2hs3uqsrldfh
lastmodified: "2026-07-20T19:41:57Z"
mac: ENC[AES256_GCM,data:fFoW//RJnlx0jQ3aMsEtx/qy70LrnocVc4UfPUIFNJG377NG7GnDxs+JIQGXSNNzRoKH06U3AFmBl2BvIi6D67A1VbOS0/+zmSGcNxdmL3apauOmn34jgJ/ipyakuawUi1+CGgSy8eMqgLM9QCpBkGx6qTtjjnGVWHYbN1WG3iE=,iv:8MD02ysiRL3nxSQvhQ6e0ek9gVLHwcfUCGzX+Tmtdp4=,tag:X/SdhJes1rJLHy6S278ztg==,type:str]
unencrypted_suffix: _unencrypted
version: 3.13.2
---
apiVersion: ENC[AES256_GCM,data:Mv4=,iv:u/JjNJfdyz7ehk8lHZRh1u9zkNuufHosyXv10Gm+T6E=,tag:J4W7uyf3iYImPh2OSVWvNA==,type:str]
kind: ENC[AES256_GCM,data:VPEKGVDy,iv:DymezGpBHmKT3BM/CcUKEGIWZw/JXNutReSKfhfmoD4=,tag:xqIUhSu4RSxameNPoFZVNQ==,type:str]
metadata:
name: ENC[AES256_GCM,data:bckq1pAY9UIMm3FUyGgafvQ=,iv:4uX7+Sl1UzgsImBH+qm9p8DlOfJu0i0W3bpoE7cFxXg=,tag:wvrBClQahE2OeVzeiubOEg==,type:str]
namespace: ENC[AES256_GCM,data:5GotV/mD9g==,iv:PQrfm67la6RJCRq0H7qpiY8n2hPUjcaK6gmJ/zPQW30=,tag:fCupPaa2X/ztuoHbtr/HEQ==,type:str]
type: ENC[AES256_GCM,data:q8D3YEZZ,iv:EXOe0day1ScxvZ7ozn1QyHgaZgnvlPNmvSQG74aw7lM=,tag:lxOx4u4LGcXCyK32BCD1XQ==,type:str]
stringData:
CONSOLE_ACCESS_KEY: ENC[AES256_GCM,data:XJCgReZf/FVli+lfA9ynF+s=,iv:gGTaSKxk721GwMQWWyymqUPZlGLidrBzWCWdRtnlOr0=,tag:Ts1P68sbBrYAS9gdI8XaHw==,type:str]
CONSOLE_SECRET_KEY: ENC[AES256_GCM,data:WQKvfXroupl4R2oUuRXDSgLthsd/ZaEedsy8lAHAu40=,iv:GXBhHQ8YiaVjRRMw2yG4FxdulVOuOurkLXXC0u/SFRA=,tag:kyxRviddcYUc0DcPEphKIg==,type:str]
sops:
age:
- enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBNZCtYcC9kanVtdFNSeksr
NDhXVHBpWjhKaWJSVnR1Rk9QV3BVU25mRldjCitEOVNCNnZVVjNLQWhFejdGelJV
V3Bhb3JTMU9xdEhOck00TGt4UFVaWTAKLS0tIEZ1VnFrVmkyTWZWR2hCc2hpWGVK
Z0NuYjVIdXJYRlVkY3JSUWUzZzd1bkkK0jm57duPu+u2qwjhKaO/fTtH1+Gt8fSm
vnLQsQZlkYZBYxHHwtBPC4JztYvmuKnPNYAPqRPfVB+9IEOpQEetkA==
-----END AGE ENCRYPTED FILE-----
recipient: age1smu533f803gmd0jq60s2zaj9zlznajy0ca6rtewd4r37mr2hs3uqsrldfh
lastmodified: "2026-07-20T19:41:57Z"
mac: ENC[AES256_GCM,data:fFoW//RJnlx0jQ3aMsEtx/qy70LrnocVc4UfPUIFNJG377NG7GnDxs+JIQGXSNNzRoKH06U3AFmBl2BvIi6D67A1VbOS0/+zmSGcNxdmL3apauOmn34jgJ/ipyakuawUi1+CGgSy8eMqgLM9QCpBkGx6qTtjjnGVWHYbN1WG3iE=,iv:8MD02ysiRL3nxSQvhQ6e0ek9gVLHwcfUCGzX+Tmtdp4=,tag:X/SdhJes1rJLHy6S278ztg==,type:str]
unencrypted_suffix: _unencrypted
version: 3.13.2
@@ -1,17 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: minio
namespace: storage
spec:
type: ClusterIP
# Helm chart selector — routes to minio-az-a/b/c pods
selector:
app: minio
ports:
- name: s3
port: 9000
targetPort: 9000
- name: console
port: 9001
targetPort: 9001
+44 -77
View File
@@ -5,19 +5,30 @@ metadata:
namespace: storage namespace: storage
labels: labels:
app: minio app: minio
annotations:
# Let the operator own bucket/user provisioning declaratively.
prometheus.io/path: /minio/v2/metrics/cluster
prometheus.io/port: "9000"
prometheus.io/scrape: "true"
spec: spec:
image: minio/minio:RELEASE.2024-06-13T20-48-48Z image: minio/minio:RELEASE.2025-07-23T15-54-02Z
credsSecret: # Root credentials. v5 pods read `configuration` — a Secret whose `config.env`
# key holds shell `export MINIO_ROOT_USER=...` lines. Created out-of-band
# (SOPS), see minio-secrets.enc.yaml. NOTE: the operator health-monitor logs a
# cosmetic "empty tenant credentials" warning (it greps for legacy
# access_key/secret_key keys) — MinIO itself authenticates fine; ignore it.
configuration:
name: minio-creds name: minio-creds
# 3-zone distributed cluster (one server per zone) # ── Single pool on the sole storage/scheduling node (talos-cp-1, az-a) ──────
# Per the 3-CP topology only talos-cp-1 is schedulable and holds Longhorn, so
# MinIO is a single-server tenant. 4 volumes give erasure-coded durability
# (MinIO's minimum for parity) on that one node.
pools: pools:
- name: az-a - name: az-a
servers: 1 servers: 1
volumesPerServer: 1 volumesPerServer: 4
size: 100Gi
storageClass: longhorn
affinity: affinity:
nodeAffinity: nodeAffinity:
@@ -33,31 +44,13 @@ spec:
operator: Exists operator: Exists
effect: NoSchedule effect: NoSchedule
volumeClaimTemplate: resources:
metadata: requests:
name: data cpu: 250m
spec: memory: 512Mi
accessModes: limits:
- ReadWriteOnce cpu: "1"
storageClassName: longhorn memory: 1Gi
resources:
requests:
storage: 100Gi
- name: az-b
servers: 1
volumesPerServer: 1
size: 100Gi
storageClass: longhorn
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: topology.kubernetes.io/zone
operator: In
values: [az-b]
volumeClaimTemplate: volumeClaimTemplate:
metadata: metadata:
@@ -68,47 +61,30 @@ spec:
storageClassName: longhorn storageClassName: longhorn
resources: resources:
requests: requests:
storage: 100Gi storage: 25Gi
- name: az-c # ── Declarative buckets (operator creates on first boot) ────────────────────
servers: 1 buckets:
volumesPerServer: 1 - name: riotpiao-models
size: 100Gi
storageClass: longhorn
affinity: # ── Declarative users (each references a Secret of the same name holding
nodeAffinity: # CONSOLE_ACCESS_KEY / CONSOLE_SECRET_KEY) ─────────────────────────────
requiredDuringSchedulingIgnoredDuringExecution: users:
nodeSelectorTerms: - name: minio-user-ollama
- matchExpressions:
- key: topology.kubernetes.io/zone
operator: In
values: [az-c]
volumeClaimTemplate: # Metrics are exposed at /minio/v2/metrics; scrape via a hand-rolled
metadata: # ServiceMonitor in the monitoring stack rather than operator auto-wiring
name: data # (prometheusOperator:true makes the operator hunt for Prometheus in ns
spec: # 'default' and fail the reconcile).
accessModes:
- ReadWriteOnce
storageClassName: longhorn
resources:
requests:
storage: 100Gi
# Console (web UI) # Public hostnames the tenant serves (S3 + console via the cluster ingress).
console: features:
image: minio/console:v0.30.0 domains:
replicas: 1 minio:
resources: - https://minio.riotpiao.com
requests: console: https://minio-console.riotpiao.com
cpu: 100m
memory: 256Mi
limits:
cpu: 500m
memory: 512Mi
# Environment variables — OIDC config # ── OIDC via Authentik (server-side env, valid in v2 schema) ────────────────
env: env:
- name: MINIO_IDENTITY_OPENID_CONFIG_URL - name: MINIO_IDENTITY_OPENID_CONFIG_URL
value: "https://authentik.riotpiao.com/application/o/minio/.well-known/openid-configuration" value: "https://authentik.riotpiao.com/application/o/minio/.well-known/openid-configuration"
@@ -128,14 +104,5 @@ spec:
- name: MINIO_IDENTITY_OPENID_DISPLAY_NAME - name: MINIO_IDENTITY_OPENID_DISPLAY_NAME
value: "Authentik" value: "Authentik"
# Metrics # cert-manager handles TLS; no operator auto-cert.
metrics:
enabled: true
port: 9000
# No auto-TLS (using cert-manager)
requestAutoCert: false requestAutoCert: false
# No built-in ingress
ingress:
enabled: false