fix(minio): rewrite Tenant to operator-v5 schema, single-node pool, declarative buckets/users — removes invalid Bucket/Policy/User CRs and dead multi-site replication

Old Tenant used unknown v2 fields (pools[].size/storageClass, spec.console/metrics/ingress)
and referenced nonexistent minio.min.io/v1alpha1 Bucket/Policy/User kinds, so the app
never synced. Rewrites to valid v2: single erasure-coded pool (4 vols) pinned to
talos-cp-1/az-a (only schedulable+Longhorn node per 3-CP topology), spec.buckets +
spec.users declarative provisioning, prometheusOperator ServiceMonitor, features.domains.
Drops hand-rolled minio-service (operator owns it), dead multi-site replication job,
and legacy alias. Adds mc-based PostSync job for the ollama scoped policy, and
SOPS-encrypted minio-creds/oidc/user secrets for IaC record.
This commit is contained in:
Story Crater Bot
2026-07-20 12:27:44 -07:00
parent 7728f20d2b
commit 4cfac71a73
8 changed files with 199 additions and 250 deletions
-65
View File
@@ -1,65 +0,0 @@
---
# Buckets
apiVersion: minio.min.io/v1alpha1
kind: Bucket
metadata:
name: riotpiao-models
namespace: storage
spec:
name: riotpiao-models
versioning:
enabled: true
tags:
app: ollama
layer: applications
purpose: model-storage
---
# Policies: define access scopes
apiVersion: minio.min.io/v1alpha1
kind: Policy
metadata:
name: policy-ollama
namespace: storage
spec:
name: policy-ollama
statements:
- Effect: Allow
Action:
- s3:GetObject
- s3:PutObject
Resource:
- arn:aws:s3:::riotpiao-models/*
- Effect: Allow
Action:
- s3:ListBucket
Resource:
- arn:aws:s3:::riotpiao-models
---
# Users: service accounts with scoped access
apiVersion: minio.min.io/v1alpha1
kind: User
metadata:
name: user-ollama
namespace: storage
spec:
accessKey: ollama-access-key
secretKey: ollama-secret-key-changeme # Override via Secret
policies:
- policy-ollama
status: enabled
memberOf: []
---
# Future buckets
# - terraform-state: for Terraform state
# - logs: for application logs
# - artifacts: for CI/CD artifacts
# - backups: for database backups
# Future users
# - terraform: read/write terraform-state
# - logging: write logs
# - ci-cd: write artifacts
# - backups: write backups
+4 -4
View File
@@ -3,7 +3,7 @@ kind: Kustomization
namespace: storage
resources:
- minio-tenant.yaml
- minio-service.yaml
- minio-replication-job.yaml
- buckets.yaml
# Legacy: minio-legacy-alias.yaml skipped (migration artifact, conflicts with namespace transform)
- minio-policy-job.yaml
# The operator creates the minio S3/console/headless Services and the
# declarative bucket + user from the Tenant spec — no hand-rolled Service or
# Bucket/User CRs (those kinds don't exist in the operator CRD set).
@@ -1,14 +0,0 @@
# Backward-compatibility alias: anything still pointing at the old
# minio.logging.svc.cluster.local address resolves to the universal
# storage frontend. Apply AFTER `helm uninstall minio -n logging`
# (the old release owns the Service name `minio` until then).
apiVersion: v1
kind: Service
metadata:
name: minio
namespace: logging
labels:
app: minio
spec:
type: ExternalName
externalName: minio.storage.svc.cluster.local
@@ -0,0 +1,85 @@
apiVersion: batch/v1
kind: Job
metadata:
name: minio-policy-setup
namespace: storage
annotations:
# Run after the Tenant (and its declarative bucket/user) exist.
argocd.argoproj.io/sync-wave: "2"
argocd.argoproj.io/hook: PostSync
argocd.argoproj.io/hook-delete-policy: HookSucceeded
spec:
ttlSecondsAfterFinished: 600
backoffLimit: 10
template:
spec:
restartPolicy: OnFailure
tolerations:
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
securityContext:
runAsNonRoot: true
runAsUser: 1000
seccompProfile:
type: RuntimeDefault
containers:
- name: mc
image: minio/mc:latest
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
command:
- /bin/sh
- -c
- |
set -e
# Wait for the tenant S3 endpoint to answer.
until mc alias set local "http://minio.storage.svc.cluster.local:9000" \
"$ROOT_USER" "$ROOT_PASSWORD" 2>/dev/null; do
echo "waiting for minio..."; sleep 5
done
# Scoped policy for the ollama service user: read/write its bucket only.
cat >/tmp/policy-ollama.json <<'JSON'
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:PutObject"],
"Resource": ["arn:aws:s3:::riotpiao-models/*"]
},
{
"Effect": "Allow",
"Action": ["s3:ListBucket"],
"Resource": ["arn:aws:s3:::riotpiao-models"]
}
]
}
JSON
mc admin policy create local policy-ollama /tmp/policy-ollama.json || true
# Attach the policy to the declaratively-created user.
mc admin policy attach local policy-ollama --user "$OLLAMA_ACCESS_KEY" || true
echo "ollama policy configured"
env:
- name: ROOT_USER
valueFrom:
secretKeyRef:
name: minio-creds
key: accesskey
- name: ROOT_PASSWORD
valueFrom:
secretKeyRef:
name: minio-creds
key: secretkey
- name: OLLAMA_ACCESS_KEY
valueFrom:
secretKeyRef:
name: minio-user-ollama
key: CONSOLE_ACCESS_KEY
@@ -1,75 +0,0 @@
apiVersion: batch/v1
kind: Job
metadata:
name: minio-site-replication-setup
namespace: storage
spec:
# Auto-delete the Job pod 10 minutes after completion
ttlSecondsAfterFinished: 600
template:
spec:
restartPolicy: OnFailure
containers:
- name: mc
image: minio/mc:latest
command:
- /bin/sh
- -c
- |
set -e
# Alias every site. Service names follow the release naming
# convention minio-<site>.storage.svc.cluster.local.
for site in $SITES; do
mc alias set "$site" "http://minio-${site}.storage.svc.cluster.local:9000" \
"$ROOT_USER" "$ROOT_PASSWORD"
done
first=$(echo $SITES | cut -d' ' -f1)
# Idempotent: skip only if every requested site is already part
# of the replication group. A partially-configured group (e.g.
# az-c newly added to SITES) falls through to `replicate add`,
# which expands an existing group in place.
# NOTE: pure-shell matching — the minio/mc image has no grep.
info=$(mc admin replicate info "$first" 2>/dev/null || true)
missing=0
for site in $SITES; do
case "$info" in
*"$site"*) ;;
*) missing=1 ;;
esac
done
if [ "$missing" -eq 0 ]; then
echo "Site replication already spans all sites ($SITES) — nothing to do"
mc admin replicate info "$first"
exit 0
fi
echo "Configuring site replication across: $SITES"
mc admin replicate add $SITES
echo "Replication status:"
mc admin replicate info "$first"
env:
# Space-separated list of replication sites, named after the AZ
# node labels. Each site must have a Helm release minio-<site>
# (e.g. minio-az-a) so the Service DNS resolves. To add an AZ
# later: deploy minio-az-<X>, append "az-<X>" here, then delete
# and re-apply this Job.
- name: SITES
value: "az-a az-b az-c"
# MinIO site replication requires identical root credentials on
# every site, so reading one release's Secret covers all of them.
# The MinIO chart creates a Secret named after the release with
# keys rootUser and rootPassword.
- name: ROOT_USER
valueFrom:
secretKeyRef:
name: minio-az-a
key: rootUser
- name: ROOT_PASSWORD
valueFrom:
secretKeyRef:
name: minio-az-a
key: rootPassword
@@ -0,0 +1,73 @@
apiVersion: ENC[AES256_GCM,data:w/A=,iv:Qum0Jux4oTEBSsbhDhFJHjoI9SsIAYpQazHaGCpNi4Y=,tag:9QGbAc8+zx1kpHdsmvY2YQ==,type:str]
kind: ENC[AES256_GCM,data:gNunoeDN,iv:baVsCtIdMzvIxiDYUWJZofazYmy9IxJ7Qy7iq0423LU=,tag:yFhxqH6pjZcRg8wYu+Euig==,type:str]
metadata:
name: ENC[AES256_GCM,data:PEmF8pnwETVjnys=,iv:qhljxXFqi2YInSuh8umcMzexekEombvc+P7vjdis4us=,tag:Z0P3hvrrCqACd1X2kyWuJg==,type:str]
namespace: ENC[AES256_GCM,data:+xu5BsH0rQ==,iv:QXGrT2y8YRlPloD3vtIEWi6KKEulapx0qDReKBcx94s=,tag:lJeuU8ze4S0nCvD6HDO8vQ==,type:str]
type: ENC[AES256_GCM,data:SGp0jv5g,iv:0cGTcnKMpo9szxciiq3lVmjKu9PGEBei4DIRg5ykCSk=,tag:zVNwJrGzjwFiEJ8YsJl4OA==,type:str]
stringData:
accesskey: ENC[AES256_GCM,data:RDGA1yzgfLp4Qg==,iv:6ahifHVN0+oLcPK+wU9zQ6mzCfYletVa6OrLMgGjvZU=,tag:aTjrX9GTOWhQ1JAdD7mkdw==,type:str]
secretkey: ENC[AES256_GCM,data:1TBLP5i7RcHBqpSRrQwYRS+R5eZhEx2NmM8Pe5/ffg4PmEWW/O4hHirpcyg=,iv:uUCaDPzBRsbDJjJKiaT1Cl+LAjaRTtih1QQNbMB2UkU=,tag:UIp5CsdPec1NoJ5Mr1p07Q==,type:str]
sops:
age:
- enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBLN1Y4QmxCMlgxeHlWQ3M1
U2ZJd20vQVRISzc2eFVwamplSi8vOEVxWGw4CjFVc2twMllGUURKaFl4Y2ZqTjRq
aFVMbHpuNWR0VHpQMlNyUkM5dFFnS1kKLS0tIFhRU09TM0xZTG0rR1dOM0IzQ3NL
V0loSURhSHh2alVPdWNCMFFqNUpPK1UKLZF4LxAfFjj55v4GXki3NVa4d+uGjCoM
rx4bv8VBsrgCXUewv97R2CK2UYTn5BwzIP3XEFQ6GYh7aW58ilU5Ww==
-----END AGE ENCRYPTED FILE-----
recipient: age1smu533f803gmd0jq60s2zaj9zlznajy0ca6rtewd4r37mr2hs3uqsrldfh
lastmodified: "2026-07-20T19:27:04Z"
mac: ENC[AES256_GCM,data:LLAp9hGlb/OvPRsTHoCZvYh7VMOUPY6Bv3FUJrM0CE0+E2TRR2Y4QwfDrHhWIW/z7ODgeSGScFBXvkMjVnuFgjXW3HHvfLDrW58BxaBOlPkOxliur8DTJ+P+ow9LGUwPAM4rAoeD05/fAQeZiRLFtEzMcoqBQBGEMXyAtUKE6kU=,iv:TWtlzUoqAZCORgRFwcPCn6XfV/m7WcB4AcOMMn0iA/w=,tag:EoABMArHUUTiYoTja6CUwQ==,type:str]
unencrypted_suffix: _unencrypted
version: 3.13.2
---
apiVersion: ENC[AES256_GCM,data:nT4=,iv:oDoOIsSdLse/e5NWZfB77VDCKZg7H1ryo2G1yO/kMSQ=,tag:mh0742mY6JkziEDjVvJraQ==,type:str]
kind: ENC[AES256_GCM,data:/b9jpUAE,iv:t9ePbTrB312I/DbwwU2+TbPdadu017NRCf86tB3xgo8=,tag:bQ4kkgIdK+ok+Q8Ib5RQWQ==,type:str]
metadata:
name: ENC[AES256_GCM,data:njhz0CfjRVGOsA==,iv:SaHtjjPxAW+RWZ3++5ki9coI042/ZfDVbzvBhRjf9Qs=,tag:6/P18CD8nsMSLfdSKS3gwA==,type:str]
namespace: ENC[AES256_GCM,data:2uvto8XrRQ==,iv:kFe9jRZQPxh2e4xYeGG1qbJ1wv1oIGiwHqDb26EF7ZQ=,tag:YtfAVMk03Nutfqcz7yLj/Q==,type:str]
type: ENC[AES256_GCM,data:UcCLVcHy,iv:QbcI8GveNxBPirq6WvbCfm+ToTsjFCYsLKgd7q4/6Ns=,tag:HyhZMa8++GR56Kr66SGukw==,type:str]
stringData:
MINIO_IDENTITY_OPENID_CLIENT_SECRET: ENC[AES256_GCM,data:AfQjsyC5sVF7CbUZd04lRsMGWoUASagUjcQr+KbhMfvfwjKXMxSu9Zpt2kDxTaIwm+CaUnvWf01G1n4oj5wPPA==,iv:QmZDkftfSs2h+ktlHcDNQJJzuVtUi5MnKaYAD+7ZzeA=,tag:VZgouvb5sLighY73k1ssFg==,type:str]
sops:
age:
- enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBLN1Y4QmxCMlgxeHlWQ3M1
U2ZJd20vQVRISzc2eFVwamplSi8vOEVxWGw4CjFVc2twMllGUURKaFl4Y2ZqTjRq
aFVMbHpuNWR0VHpQMlNyUkM5dFFnS1kKLS0tIFhRU09TM0xZTG0rR1dOM0IzQ3NL
V0loSURhSHh2alVPdWNCMFFqNUpPK1UKLZF4LxAfFjj55v4GXki3NVa4d+uGjCoM
rx4bv8VBsrgCXUewv97R2CK2UYTn5BwzIP3XEFQ6GYh7aW58ilU5Ww==
-----END AGE ENCRYPTED FILE-----
recipient: age1smu533f803gmd0jq60s2zaj9zlznajy0ca6rtewd4r37mr2hs3uqsrldfh
lastmodified: "2026-07-20T19:27:04Z"
mac: ENC[AES256_GCM,data:LLAp9hGlb/OvPRsTHoCZvYh7VMOUPY6Bv3FUJrM0CE0+E2TRR2Y4QwfDrHhWIW/z7ODgeSGScFBXvkMjVnuFgjXW3HHvfLDrW58BxaBOlPkOxliur8DTJ+P+ow9LGUwPAM4rAoeD05/fAQeZiRLFtEzMcoqBQBGEMXyAtUKE6kU=,iv:TWtlzUoqAZCORgRFwcPCn6XfV/m7WcB4AcOMMn0iA/w=,tag:EoABMArHUUTiYoTja6CUwQ==,type:str]
unencrypted_suffix: _unencrypted
version: 3.13.2
---
apiVersion: ENC[AES256_GCM,data:Z/g=,iv:EbbztYw9O2eWwDxlFFEaBgmAnIhN9mmwA8GPzZLZF6I=,tag:2Xe1qSCcaCLtC/CYgDT21Q==,type:str]
kind: ENC[AES256_GCM,data:DkLVktg5,iv:e5g6/fBulvT1/1Jjd+8UWOsVCk1mWntGvSXLJxzf1Lo=,tag:x8XB6vxsezq/8hkqGUjMBA==,type:str]
metadata:
name: ENC[AES256_GCM,data:6CsuJOz36cL1qqTdm/Ry3gU=,iv:WzCfjwMQMonViXgOvr8s0ekp/49vuhOcv6xH/diV8i0=,tag:EHMnvtfC2zB5hWkgkVOHaw==,type:str]
namespace: ENC[AES256_GCM,data:rYh2ReE2Hw==,iv:j1kYlDhZYGcjn/CXMSq0ZBtht/jPLb/Bzle2NCWbe1Y=,tag:u7MiEnxVFKeycEfS/MLeSw==,type:str]
type: ENC[AES256_GCM,data:8oyW6AP2,iv:qEjwjABAu7AJz/4h17JnwH6VYPU07onKfUparanC0Zs=,tag:HrH2IJHxnktY8wvWNWNMHQ==,type:str]
stringData:
CONSOLE_ACCESS_KEY: ENC[AES256_GCM,data:VtXwL7ngWkb3mDucpRtigqE=,iv:+r7DQLbjVMU82Ig8nlOwIgFEA2wE3KpMVtXIzpjE/o4=,tag:p3/7UOOCvH9qnOPN+sTCmw==,type:str]
CONSOLE_SECRET_KEY: ENC[AES256_GCM,data:CG09bCWQP8zP++t8iYn3jIietAikzLOJhsYSfb1kq+s=,iv:svztBK1rZky5YQjWc4q2YzHQNJO5u8rhSsZHbGLnhnw=,tag:m0lGFXshydA596KAqMHbzg==,type:str]
sops:
age:
- enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBLN1Y4QmxCMlgxeHlWQ3M1
U2ZJd20vQVRISzc2eFVwamplSi8vOEVxWGw4CjFVc2twMllGUURKaFl4Y2ZqTjRq
aFVMbHpuNWR0VHpQMlNyUkM5dFFnS1kKLS0tIFhRU09TM0xZTG0rR1dOM0IzQ3NL
V0loSURhSHh2alVPdWNCMFFqNUpPK1UKLZF4LxAfFjj55v4GXki3NVa4d+uGjCoM
rx4bv8VBsrgCXUewv97R2CK2UYTn5BwzIP3XEFQ6GYh7aW58ilU5Ww==
-----END AGE ENCRYPTED FILE-----
recipient: age1smu533f803gmd0jq60s2zaj9zlznajy0ca6rtewd4r37mr2hs3uqsrldfh
lastmodified: "2026-07-20T19:27:04Z"
mac: ENC[AES256_GCM,data:LLAp9hGlb/OvPRsTHoCZvYh7VMOUPY6Bv3FUJrM0CE0+E2TRR2Y4QwfDrHhWIW/z7ODgeSGScFBXvkMjVnuFgjXW3HHvfLDrW58BxaBOlPkOxliur8DTJ+P+ow9LGUwPAM4rAoeD05/fAQeZiRLFtEzMcoqBQBGEMXyAtUKE6kU=,iv:TWtlzUoqAZCORgRFwcPCn6XfV/m7WcB4AcOMMn0iA/w=,tag:EoABMArHUUTiYoTja6CUwQ==,type:str]
unencrypted_suffix: _unencrypted
version: 3.13.2
@@ -1,17 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: minio
namespace: storage
spec:
type: ClusterIP
# Helm chart selector — routes to minio-az-a/b/c pods
selector:
app: minio
ports:
- name: s3
port: 9000
targetPort: 9000
- name: console
port: 9001
targetPort: 9001
+37 -75
View File
@@ -5,19 +5,27 @@ metadata:
namespace: storage
labels:
app: minio
annotations:
# Let the operator own bucket/user provisioning declaratively.
prometheus.io/path: /minio/v2/metrics/cluster
prometheus.io/port: "9000"
prometheus.io/scrape: "true"
spec:
image: minio/minio:RELEASE.2024-06-13T20-48-48Z
# Root credentials (rootUser/rootPassword). Created out-of-band — see
# BOOTSTRAP.md / SOPS. Operator reads CONSOLE_ACCESS_KEY style from here.
credsSecret:
name: minio-creds
# 3-zone distributed cluster (one server per zone)
# ── Single pool on the sole storage/scheduling node (talos-cp-1, az-a) ──────
# Per the 3-CP topology only talos-cp-1 is schedulable and holds Longhorn, so
# MinIO is a single-server tenant. 4 volumes give erasure-coded durability
# (MinIO's minimum for parity) on that one node.
pools:
- name: az-a
servers: 1
volumesPerServer: 1
size: 100Gi
storageClass: longhorn
volumesPerServer: 4
affinity:
nodeAffinity:
@@ -33,31 +41,13 @@ spec:
operator: Exists
effect: NoSchedule
volumeClaimTemplate:
metadata:
name: data
spec:
accessModes:
- ReadWriteOnce
storageClassName: longhorn
resources:
requests:
storage: 100Gi
- name: az-b
servers: 1
volumesPerServer: 1
size: 100Gi
storageClass: longhorn
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: topology.kubernetes.io/zone
operator: In
values: [az-b]
resources:
requests:
cpu: 250m
memory: 512Mi
limits:
cpu: "1"
memory: 1Gi
volumeClaimTemplate:
metadata:
@@ -68,47 +58,28 @@ spec:
storageClassName: longhorn
resources:
requests:
storage: 100Gi
storage: 25Gi
- name: az-c
servers: 1
volumesPerServer: 1
size: 100Gi
storageClass: longhorn
# ── Declarative buckets (operator creates on first boot) ────────────────────
buckets:
- name: riotpiao-models
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: topology.kubernetes.io/zone
operator: In
values: [az-c]
# ── Declarative users (each references a Secret of the same name holding
# CONSOLE_ACCESS_KEY / CONSOLE_SECRET_KEY) ─────────────────────────────
users:
- name: minio-user-ollama
volumeClaimTemplate:
metadata:
name: data
spec:
accessModes:
- ReadWriteOnce
storageClassName: longhorn
resources:
requests:
storage: 100Gi
# Expose ServiceMonitor for the kube-prometheus-stack.
prometheusOperator: true
# Console (web UI)
console:
image: minio/console:v0.30.0
replicas: 1
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: 500m
memory: 512Mi
# Public hostnames the tenant serves (S3 + console via the cluster ingress).
features:
domains:
minio:
- https://minio.riotpiao.com
console: https://minio-console.riotpiao.com
# Environment variables — OIDC config
# ── OIDC via Authentik (server-side env, valid in v2 schema) ────────────────
env:
- name: MINIO_IDENTITY_OPENID_CONFIG_URL
value: "https://authentik.riotpiao.com/application/o/minio/.well-known/openid-configuration"
@@ -128,14 +99,5 @@ spec:
- name: MINIO_IDENTITY_OPENID_DISPLAY_NAME
value: "Authentik"
# Metrics
metrics:
enabled: true
port: 9000
# No auto-TLS (using cert-manager)
# cert-manager handles TLS; no operator auto-cert.
requestAutoCert: false
# No built-in ingress
ingress:
enabled: false