fix(minio): rewrite Tenant to operator-v5 schema, single-node pool, declarative buckets/users — removes invalid Bucket/Policy/User CRs and dead multi-site replication

Old Tenant used unknown v2 fields (pools[].size/storageClass, spec.console/metrics/ingress)
and referenced nonexistent minio.min.io/v1alpha1 Bucket/Policy/User kinds, so the app
never synced. Rewrites to valid v2: single erasure-coded pool (4 vols) pinned to
talos-cp-1/az-a (only schedulable+Longhorn node per 3-CP topology), spec.buckets +
spec.users declarative provisioning, prometheusOperator ServiceMonitor, features.domains.
Drops hand-rolled minio-service (operator owns it), dead multi-site replication job,
and legacy alias. Adds mc-based PostSync job for the ollama scoped policy, and
SOPS-encrypted minio-creds/oidc/user secrets for IaC record.
This commit is contained in:
Story Crater Bot
2026-07-20 12:27:44 -07:00
parent 7728f20d2b
commit 4cfac71a73
8 changed files with 199 additions and 250 deletions
@@ -0,0 +1,85 @@
apiVersion: batch/v1
kind: Job
metadata:
name: minio-policy-setup
namespace: storage
annotations:
# Run after the Tenant (and its declarative bucket/user) exist.
argocd.argoproj.io/sync-wave: "2"
argocd.argoproj.io/hook: PostSync
argocd.argoproj.io/hook-delete-policy: HookSucceeded
spec:
ttlSecondsAfterFinished: 600
backoffLimit: 10
template:
spec:
restartPolicy: OnFailure
tolerations:
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
securityContext:
runAsNonRoot: true
runAsUser: 1000
seccompProfile:
type: RuntimeDefault
containers:
- name: mc
image: minio/mc:latest
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
command:
- /bin/sh
- -c
- |
set -e
# Wait for the tenant S3 endpoint to answer.
until mc alias set local "http://minio.storage.svc.cluster.local:9000" \
"$ROOT_USER" "$ROOT_PASSWORD" 2>/dev/null; do
echo "waiting for minio..."; sleep 5
done
# Scoped policy for the ollama service user: read/write its bucket only.
cat >/tmp/policy-ollama.json <<'JSON'
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:PutObject"],
"Resource": ["arn:aws:s3:::riotpiao-models/*"]
},
{
"Effect": "Allow",
"Action": ["s3:ListBucket"],
"Resource": ["arn:aws:s3:::riotpiao-models"]
}
]
}
JSON
mc admin policy create local policy-ollama /tmp/policy-ollama.json || true
# Attach the policy to the declaratively-created user.
mc admin policy attach local policy-ollama --user "$OLLAMA_ACCESS_KEY" || true
echo "ollama policy configured"
env:
- name: ROOT_USER
valueFrom:
secretKeyRef:
name: minio-creds
key: accesskey
- name: ROOT_PASSWORD
valueFrom:
secretKeyRef:
name: minio-creds
key: secretkey
- name: OLLAMA_ACCESS_KEY
valueFrom:
secretKeyRef:
name: minio-user-ollama
key: CONSOLE_ACCESS_KEY