k8s/storage: add minio s3 with 3-way replication and oidc

- MinIO 3-node site replication (az-a/b/c)
- S3 backend for Loki chunks (10-day retention)
- OIDC integration with Authentik
- envFrom for secret injection

k8s/storage/.env.example

@@ -0,0 +1,9 @@
+# storage/.env.example
+# Copy to storage/.env and fill in real values.
+# The real .env is gitignored — never commit it.
+# Used by: k8s/storage/bootstrap.sh
+
+# MinIO root credentials (S3-compatible object store)
+#   openssl rand -base64 24
+MINIO_ROOT_USER=
+MINIO_ROOT_PASSWORD=

k8s/storage/bootstrap.sh

@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+# k8s/storage/bootstrap.sh
+# Deploys the MinIO multi-AZ object store into the storage namespace.
+#
+# Prerequisites:
+#   - kubectl configured (KUBECONFIG pointing to cluster-config/kubeconfig)
+#   - helm >= 3.x installed
+#   - k8s/storage/.env file containing:
+#       MINIO_ROOT_USER=...
+#       MINIO_ROOT_PASSWORD=...
+#     OR those variables already exported in the calling shell.
+#
+# Idempotent: safe to re-run — helm upgrade --install and kubectl apply are both idempotent.
+# Disaster recovery order: run this before k8s/logging/bootstrap.sh (Loki depends on MinIO).
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+KUBECONFIG="${KUBECONFIG:-${SCRIPT_DIR}/../../cluster-config/kubeconfig}"
+export KUBECONFIG
+
+# ── Load credentials ──────────────────────────────────────────────────────────
+if [[ -f "${SCRIPT_DIR}/.env" ]]; then
+  # shellcheck source=/dev/null
+  source "${SCRIPT_DIR}/.env"
+fi
+
+for var in MINIO_ROOT_USER MINIO_ROOT_PASSWORD; do
+  if [[ -z "${!var:-}" ]]; then
+    echo "ERROR: ${var} is not set. Export it or place it in k8s/storage/.env"
+    exit 1
+  fi
+done
+
+# ── Namespace ─────────────────────────────────────────────────────────────────
+echo "==> Creating storage namespace with privileged pod security..."
+kubectl create namespace storage --dry-run=client -o yaml | kubectl apply -f -
+kubectl label namespace storage \
+  pod-security.kubernetes.io/enforce=privileged \
+  pod-security.kubernetes.io/enforce-version=latest \
+  --overwrite
+
+# ── Helm repos ────────────────────────────────────────────────────────────────
+echo "==> Adding Helm repositories..."
+helm repo add minio https://charts.min.io
+helm repo update minio
+
+# ── PVC for az-a (adopts existing volume) ────────────────────────────────────
+echo "==> Applying az-a PVC (adopts pre-existing Longhorn volume)..."
+kubectl apply -f "${SCRIPT_DIR}/minio-az-a-pvc.yaml"
+
+# ── MinIO az-a (control-plane node, zone az-a) ───────────────────────────────
+echo "==> Installing MinIO az-a..."
+helm upgrade --install minio-az-a minio/minio \
+  --namespace storage \
+  --values "${SCRIPT_DIR}/minio-az-a-values.yaml" \
+  --set rootUser="${MINIO_ROOT_USER}" \
+  --set rootPassword="${MINIO_ROOT_PASSWORD}" \
+  --wait \
+  --timeout 5m
+
+echo "==> Waiting for minio-az-a Deployment to be ready..."
+kubectl rollout status deployment/minio-az-a -n storage --timeout=120s
+
+# ── MinIO az-b (worker node, zone az-b) ──────────────────────────────────────
+echo "==> Installing MinIO az-b..."
+helm upgrade --install minio-az-b minio/minio \
+  --namespace storage \
+  --values "${SCRIPT_DIR}/minio-az-b-values.yaml" \
+  --set rootUser="${MINIO_ROOT_USER}" \
+  --set rootPassword="${MINIO_ROOT_PASSWORD}" \
+  --wait \
+  --timeout 5m
+
+echo "==> Waiting for minio-az-b Deployment to be ready..."
+kubectl rollout status deployment/minio-az-b -n storage --timeout=120s
+
+# ── Universal frontend service + legacy alias ─────────────────────────────────
+echo "==> Applying universal frontend service and legacy ExternalName alias..."
+kubectl apply -f "${SCRIPT_DIR}/minio-service.yaml"
+kubectl apply -f "${SCRIPT_DIR}/minio-legacy-alias.yaml"
+
+# ── Site replication ──────────────────────────────────────────────────────────
+echo "==> Running site replication setup job..."
+# Delete any leftover completed job first (kubectl apply on Jobs is not idempotent)
+kubectl delete job minio-site-replication-setup -n storage --ignore-not-found
+kubectl apply -f "${SCRIPT_DIR}/minio-replication-job.yaml"
+
+echo "==> Waiting for replication job to complete..."
+kubectl wait --for=condition=complete job/minio-site-replication-setup \
+  -n storage --timeout=120s
+
+echo "==> Replication job output:"
+kubectl logs -n storage \
+  -l job-name=minio-site-replication-setup \
+  --tail=20
+
+# ── Done ──────────────────────────────────────────────────────────────────────
+echo ""
+echo "==> MinIO multi-AZ stack is up."
+echo ""
+echo "MinIO console (S3 browser):"
+echo "  kubectl port-forward -n storage svc/minio 9001:9001"
+echo "  http://localhost:9001  (${MINIO_ROOT_USER} / <your MINIO_ROOT_PASSWORD>)"
+echo ""
+echo "MinIO S3 endpoint for apps (active-active frontend):"
+echo "  http://minio.storage.svc.cluster.local:9000"
+echo ""
+echo "Verify site replication:"
+echo "  kubectl logs -n storage -l job-name=minio-site-replication-setup"

k8s/storage/longhorn-ingress-oauth2.yaml

@@ -0,0 +1,25 @@
+# Ingress for Longhorn storage UI — routes to OAuth2-Proxy
+# TLS terminated here; oauth2-proxy handles OIDC auth
+
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: longhorn
+  namespace: longhorn-system
+spec:
+  ingressClassName: nginx
+  tls:
+  - secretName: longhorn-tls
+    hosts:
+    - longhorn.riotpiao.homelab.com
+  rules:
+  - host: longhorn.riotpiao.homelab.com
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: oauth2-proxy
+            port:
+              number: 4180

k8s/storage/longhorn-oauth2-proxy.yaml

@@ -0,0 +1,106 @@
+# OAuth2-Proxy for Longhorn storage UI
+# Protects persistent volume management interface with Authentik OIDC
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: oauth2-proxy
+  namespace: longhorn-system
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: oauth2-proxy
+  namespace: longhorn-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: oauth2-proxy
+  template:
+    metadata:
+      labels:
+        app: oauth2-proxy
+      annotations:
+        secret.reloader.stakater.com/reload: "longhorn-oidc"
+    spec:
+      serviceAccountName: oauth2-proxy
+      containers:
+      - name: oauth2-proxy
+        image: quay.io/oauth2-proxy/oauth2-proxy:v7.5.1
+        imagePullPolicy: IfNotPresent
+        ports:
+        - name: http
+          containerPort: 4180
+          protocol: TCP
+        env:
+        - name: OAUTH2_PROXY_PROVIDER
+          value: "oidc"
+        - name: OAUTH2_PROXY_OIDC_ISSUER_URL
+          value: "https://authentik.riotpiao.homelab.com/application/o/longhorn/"
+        - name: OAUTH2_PROXY_CLIENT_ID
+          value: "longhorn"
+        - name: OAUTH2_PROXY_CLIENT_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: longhorn-oidc
+              key: clientSecret
+        - name: OAUTH2_PROXY_COOKIE_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: longhorn-oidc
+              key: cookieSecret
+        - name: OAUTH2_PROXY_REDIRECT_URL
+          value: "https://longhorn.riotpiao.homelab.com/oauth2/callback"
+        - name: OAUTH2_PROXY_UPSTREAM
+          value: "http://longhorn-frontend:80"
+        - name: OAUTH2_PROXY_COOKIE_SECURE
+          value: "true"
+        - name: OAUTH2_PROXY_COOKIE_HTTPONLY
+          value: "true"
+        - name: OAUTH2_PROXY_COOKIE_SAMESITE
+          value: "Lax"
+        - name: OAUTH2_PROXY_EMAIL_DOMAIN
+          value: "*"
+        - name: OAUTH2_PROXY_SKIP_AUTH_REGEX
+          value: "^/health"
+        - name: OAUTH2_PROXY_PASS_AUTHORIZATION_HEADER
+          value: "true"
+        - name: OAUTH2_PROXY_REVERSE_PROXY
+          value: "true"
+        resources:
+          requests:
+            cpu: 50m
+            memory: 64Mi
+          limits:
+            cpu: 100m
+            memory: 128Mi
+        livenessProbe:
+          httpGet:
+            path: /ping
+            port: http
+          initialDelaySeconds: 10
+          periodSeconds: 10
+        readinessProbe:
+          httpGet:
+            path: /ping
+            port: http
+          initialDelaySeconds: 5
+          periodSeconds: 5
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: oauth2-proxy
+  namespace: longhorn-system
+spec:
+  type: ClusterIP
+  ports:
+  - port: 4180
+    targetPort: http
+    protocol: TCP
+    name: http
+  selector:
+    app: oauth2-proxy

k8s/storage/minio-bucket-init.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+# Safe MinIO bucket initialization via Job (credentials in Secret, not env)
+# Usage: ./minio-bucket-init.sh <namespace> <bucket1> [bucket2] ...
+set -euo pipefail
+
+NAMESPACE="${1:?Missing namespace}"
+shift
+BUCKETS=("$@")
+
+if [ ${#BUCKETS[@]} -eq 0 ]; then
+  echo "Usage: $0 <namespace> <bucket1> [bucket2] ..." >&2
+  exit 1
+fi
+
+# Create Secret with Minio credentials (safe: sourced from env, not exposed in pod)
+kubectl create secret generic minio-creds -n "$NAMESPACE" \
+  --from-literal=MINIO_ROOT_USER="${MINIO_ROOT_USER:?Missing MINIO_ROOT_USER}" \
+  --from-literal=MINIO_ROOT_PASSWORD="${MINIO_ROOT_PASSWORD:?Missing MINIO_ROOT_PASSWORD}" \
+  --dry-run=client -o yaml | kubectl apply -f -
+
+# Create init Job that mounts Secret as volume, preventing env exposure
+BUCKET_ARGS=$(printf '"%s", ' "${BUCKETS[@]}" | sed 's/, $//')
+kubectl apply -f - <<EOF
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: minio-bucket-init-$RANDOM
+  namespace: $NAMESPACE
+spec:
+  ttlSecondsAfterFinished: 300
+  backoffLimit: 2
+  template:
+    spec:
+      restartPolicy: Never
+      containers:
+      - name: minio-init
+        image: minio/mc:latest
+        volumeMounts:
+        - name: minio-secret
+          mountPath: /var/run/secrets/minio
+          readOnly: true
+        command:
+        - sh
+        - -c
+        - |
+          MINIO_ROOT_USER=\$(cat /var/run/secrets/minio/MINIO_ROOT_USER)
+          MINIO_ROOT_PASSWORD=\$(cat /var/run/secrets/minio/MINIO_ROOT_PASSWORD)
+          mc alias set local http://minio.storage.svc.cluster.local:9000 "\$MINIO_ROOT_USER" "\$MINIO_ROOT_PASSWORD"
+          mc mb --ignore-existing $BUCKET_ARGS
+      volumes:
+      - name: minio-secret
+        secret:
+          secretName: minio-creds
+EOF
+
+echo "✓ MinIO bucket init job submitted for: ${BUCKETS[*]}"
+

k8s/storage/minio-legacy-alias.yaml

@@ -0,0 +1,14 @@
+# Backward-compatibility alias: anything still pointing at the old
+# minio.logging.svc.cluster.local address resolves to the universal
+# storage frontend. Apply AFTER `helm uninstall minio -n logging`
+# (the old release owns the Service name `minio` until then).
+apiVersion: v1
+kind: Service
+metadata:
+  name: minio
+  namespace: logging
+  labels:
+    app: minio
+spec:
+  type: ExternalName
+  externalName: minio.storage.svc.cluster.local

k8s/storage/minio-operator-values.yaml

@@ -0,0 +1,48 @@
+# minio-operator-values.yaml
+# MinIO Operator deployment with metrics enabled
+
+operator:
+  image:
+    repository: minio/operator
+    tag: "v5.0.0"
+    pullPolicy: IfNotPresent
+
+  replicaCount: 1
+
+  resources:
+    requests:
+      cpu: 100m
+      memory: 256Mi
+    limits:
+      cpu: 500m
+      memory: 512Mi
+
+  metrics:
+    enabled: true
+    port: 8080
+
+  rbac:
+    create: true
+
+  affinity:
+    nodeAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+        - weight: 100
+          preference:
+            matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+
+console:
+  enabled: true
+  replicaCount: 1
+  image:
+    repository: minio/console
+    tag: "v0.30.0"
+  resources:
+    requests:
+      cpu: 50m
+      memory: 128Mi
+    limits:
+      cpu: 200m
+      memory: 256Mi

k8s/storage/minio-replication-job.yaml

@@ -0,0 +1,75 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: minio-site-replication-setup
+  namespace: storage
+spec:
+  # Auto-delete the Job pod 10 minutes after completion
+  ttlSecondsAfterFinished: 600
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+        - name: mc
+          image: minio/mc:latest
+          command:
+            - /bin/sh
+            - -c
+            - |
+              set -e
+
+              # Alias every site. Service names follow the release naming
+              # convention minio-<site>.storage.svc.cluster.local.
+              for site in $SITES; do
+                mc alias set "$site" "http://minio-${site}.storage.svc.cluster.local:9000" \
+                  "$ROOT_USER" "$ROOT_PASSWORD"
+              done
+
+              first=$(echo $SITES | cut -d' ' -f1)
+
+              # Idempotent: skip only if every requested site is already part
+              # of the replication group. A partially-configured group (e.g.
+              # az-c newly added to SITES) falls through to `replicate add`,
+              # which expands an existing group in place.
+              # NOTE: pure-shell matching — the minio/mc image has no grep.
+              info=$(mc admin replicate info "$first" 2>/dev/null || true)
+              missing=0
+              for site in $SITES; do
+                case "$info" in
+                  *"$site"*) ;;
+                  *) missing=1 ;;
+                esac
+              done
+              if [ "$missing" -eq 0 ]; then
+                echo "Site replication already spans all sites ($SITES) — nothing to do"
+                mc admin replicate info "$first"
+                exit 0
+              fi
+
+              echo "Configuring site replication across: $SITES"
+              mc admin replicate add $SITES
+
+              echo "Replication status:"
+              mc admin replicate info "$first"
+          env:
+            # Space-separated list of replication sites, named after the AZ
+            # node labels. Each site must have a Helm release minio-<site>
+            # (e.g. minio-az-a) so the Service DNS resolves. To add an AZ
+            # later: deploy minio-az-<X>, append "az-<X>" here, then delete
+            # and re-apply this Job.
+            - name: SITES
+              value: "az-a az-b az-c"
+            # MinIO site replication requires identical root credentials on
+            # every site, so reading one release's Secret covers all of them.
+            # The MinIO chart creates a Secret named after the release with
+            # keys rootUser and rootPassword.
+            - name: ROOT_USER
+              valueFrom:
+                secretKeyRef:
+                  name: minio-az-a
+                  key: rootUser
+            - name: ROOT_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: minio-az-a
+                  key: rootPassword

k8s/storage/minio-service.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: minio
+  namespace: storage
+spec:
+  type: ClusterIP
+  # Helm chart selector — routes to minio-az-a/b/c pods
+  selector:
+    app: minio
+  ports:
+    - name: s3
+      port: 9000
+      targetPort: 9000
+    - name: console
+      port: 9001
+      targetPort: 9001

k8s/storage/minio-tenant.yaml

@@ -0,0 +1,141 @@
+apiVersion: minio.min.io/v2
+kind: Tenant
+metadata:
+  name: minio-cluster
+  namespace: storage
+  labels:
+    app: minio
+spec:
+  image: minio/minio:RELEASE.2024-06-13T20-48-48Z
+
+  credsSecret:
+    name: minio-creds
+
+  # 3-zone distributed cluster (one server per zone)
+  pools:
+    - name: az-a
+      servers: 1
+      volumesPerServer: 1
+      size: 100Gi
+      storageClass: longhorn
+
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: topology.kubernetes.io/zone
+                    operator: In
+                    values: [az-a]
+
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+
+      volumeClaimTemplate:
+        metadata:
+          name: data
+        spec:
+          accessModes:
+            - ReadWriteOnce
+          storageClassName: longhorn
+          resources:
+            requests:
+              storage: 100Gi
+
+    - name: az-b
+      servers: 1
+      volumesPerServer: 1
+      size: 100Gi
+      storageClass: longhorn
+
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: topology.kubernetes.io/zone
+                    operator: In
+                    values: [az-b]
+
+      volumeClaimTemplate:
+        metadata:
+          name: data
+        spec:
+          accessModes:
+            - ReadWriteOnce
+          storageClassName: longhorn
+          resources:
+            requests:
+              storage: 100Gi
+
+    - name: az-c
+      servers: 1
+      volumesPerServer: 1
+      size: 100Gi
+      storageClass: longhorn
+
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: topology.kubernetes.io/zone
+                    operator: In
+                    values: [az-c]
+
+      volumeClaimTemplate:
+        metadata:
+          name: data
+        spec:
+          accessModes:
+            - ReadWriteOnce
+          storageClassName: longhorn
+          resources:
+            requests:
+              storage: 100Gi
+
+  # Console (web UI)
+  console:
+    image: minio/console:v0.30.0
+    replicas: 1
+    resources:
+      requests:
+        cpu: 100m
+        memory: 256Mi
+      limits:
+        cpu: 500m
+        memory: 512Mi
+
+  # Environment variables — OIDC config
+  env:
+    - name: MINIO_IDENTITY_OPENID_CONFIG_URL
+      value: "https://authentik.riotpiao.homelab.com/application/o/minio/.well-known/openid-configuration"
+    - name: MINIO_IDENTITY_OPENID_CLIENT_ID
+      value: "minio"
+    - name: MINIO_IDENTITY_OPENID_CLIENT_SECRET
+      valueFrom:
+        secretKeyRef:
+          name: minio-oidc
+          key: MINIO_IDENTITY_OPENID_CLIENT_SECRET
+    - name: MINIO_IDENTITY_OPENID_CLAIM_NAME
+      value: "policy"
+    - name: MINIO_IDENTITY_OPENID_SCOPES
+      value: "openid,profile,email,minio"
+    - name: MINIO_IDENTITY_OPENID_REDIRECT_URI
+      value: "https://minio.riotpiao.homelab.com/oauth_callback"
+    - name: MINIO_IDENTITY_OPENID_DISPLAY_NAME
+      value: "Authentik"
+
+  # Metrics
+  metrics:
+    enabled: true
+    port: 9000
+
+  # No auto-TLS (using cert-manager)
+  requestAutoCert: false
+
+  # No built-in ingress
+  ingress:
+    enabled: false

k8s/storage/minio-values.yaml

@@ -0,0 +1,64 @@
+mode: standalone
+
+rootUser: ""
+rootPassword: ""
+
+persistence:
+  enabled: true
+  storageClass: longhorn
+  size: 100Gi
+
+deploymentUpdate:
+  type: Recreate
+
+resources:
+  requests:
+    cpu: 100m
+    memory: 256Mi
+  limits:
+    cpu: 500m
+    memory: 512Mi
+
+service:
+  type: ClusterIP
+  port: 9000
+
+consoleService:
+  type: ClusterIP
+  port: 9001
+
+ingress:
+  enabled: false
+
+consoleIngress:
+  enabled: false
+
+metrics:
+  serviceMonitor:
+    enabled: true
+
+envFrom:
+  - secretRef:
+      name: minio-oidc
+
+environment:
+  MINIO_IDENTITY_OPENID_CONFIG_URL: "https://authentik.riotpiao.homelab.com/application/o/minio/.well-known/openid-configuration"
+  MINIO_IDENTITY_OPENID_CLIENT_ID: "minio"
+  MINIO_IDENTITY_OPENID_CLAIM_NAME: "policy"
+  MINIO_IDENTITY_OPENID_SCOPES: "openid,profile,email,minio"
+  MINIO_IDENTITY_OPENID_REDIRECT_URI: "https://minio.riotpiao.homelab.com/oauth_callback"
+  MINIO_IDENTITY_OPENID_DISPLAY_NAME: "Authentik"
+
+podAnnotations:
+  secret.reloader.stakater.com/reload: "minio-oidc"
+  configmap.reloader.stakater.com/reload: "homelab-ca"
+
+extraVolumes:
+  - name: homelab-ca
+    configMap:
+      name: homelab-ca
+
+extraVolumeMounts:
+  - name: homelab-ca
+    mountPath: /etc/minio/certs/CAs
+    readOnly: true

k8s/storage/test/README.md

@@ -0,0 +1,137 @@
+# MinIO CRUD Example (Go)
+
+A minimal Go program that exercises the cluster's object storage through the
+**universal storage frontend** — `minio.storage.svc.cluster.local:9000` — the
+single DNS name that load-balances across both node-pinned MinIO instances
+(`minio-az-a` on talos-cp-1, `minio-az-b` on talos-worker-1).
+
+It runs a full CRUD cycle with a random text file:
+
+| Step | S3 call | What it proves |
+|------|---------|----------------|
+| Ensure bucket | `BucketExists` / `MakeBucket` | bucket `crud-test` exists (idempotent) |
+| **C**reate | `PutObject` | write path through the frontend |
+| **R**ead | `GetObject` + byte compare | content round-trips intact |
+| **U**pdate | `PutObject` (overwrite) | S3 update semantics (objects are replaced, not edited) |
+| List | `ListObjects` prefix `demo/` | enumeration |
+| **D**elete | `RemoveObject` + `StatObject` | object gone (`NoSuchKey` confirmed) |
+
+Every operation emits `[SERVICE_METRIC] s3.<op>.latency_ms=<n> ms`; any failure
+emits `[APP_METRIC] ERROR s3.<op> failed ... | trace=...` and exits non-zero.
+
+## Run it
+
+```bash
+# 1. expose the frontend locally (leave running in another terminal)
+kubectl port-forward svc/minio -n storage 9000:9000
+
+# 2. credentials — same root creds used by both MinIO sites
+source logging/.env        # exports MINIO_ROOT_USER / MINIO_ROOT_PASSWORD
+
+# 3. run
+cd storage/test
+go mod tidy && go run .
+```
+
+In-cluster (e.g. from a Job), skip the port-forward and set
+`MINIO_ENDPOINT=minio.storage.svc.cluster.local:9000`.
+
+Expected output:
+
+```
+[SERVICE_METRIC] s3.ensure_bucket.latency_ms=145 ms
+[SERVICE_METRIC] s3.put.latency_ms=19 ms
+created crud-test/demo/<unix-ts>.txt (256 bytes of random text)
+[SERVICE_METRIC] s3.get.latency_ms=9 ms
+read back and verified content
+[SERVICE_METRIC] s3.update.latency_ms=70 ms
+updated (overwrote) object
+  demo/<unix-ts>.txt  140 bytes  <timestamp>
+[SERVICE_METRIC] s3.list.latency_ms=11 ms
+[SERVICE_METRIC] s3.delete.latency_ms=68 ms
+deleted and verified gone — CRUD cycle complete
+```
+
+## Validating each state with kubectl
+
+The program verifies itself in-process (read-back compare, post-delete stat),
+but every state is also independently observable from outside with `kubectl`.
+The helper below drops you into a throwaway `mc` shell wired to both sites —
+all subsequent checks use it:
+
+```bash
+source logging/.env
+kubectl run mc-shell --rm -it --restart=Never --image=minio/mc -n storage \
+  --env="U=$MINIO_ROOT_USER" --env="P=$MINIO_ROOT_PASSWORD" \
+  --command -- /bin/sh -c '
+mc alias set front http://minio.storage.svc.cluster.local:9000 "$U" "$P"
+mc alias set az-a  http://minio-az-a.storage.svc.cluster.local:9000 "$U" "$P"
+mc alias set az-b  http://minio-az-b.storage.svc.cluster.local:9000 "$U" "$P"
+exec /bin/sh'
+```
+
+> The demo deletes its object at the end, so to inspect the CREATE/UPDATE
+> states at your own pace, comment out the `// DELETE` block in `main.go`
+> and re-run (the delete is idempotent to re-apply later).
+
+**0. Frontend is healthy (before running anything)**
+
+```bash
+kubectl get endpoints minio -n storage          # expect TWO pod IPs on :9000
+kubectl get pods -n storage -o wide             # az-a on talos-cp-1, az-b on talos-worker-1
+```
+
+**1. Bucket created** — and replicated to BOTH sites
+
+```bash
+# inside mc-shell — the bucket must appear on each site individually
+mc ls az-a | grep crud-test
+mc ls az-b | grep crud-test     # proves site replication propagated it
+```
+
+**2. Object created (CREATE)** — 256 bytes, present on both nodes
+
+```bash
+mc ls az-a/crud-test/demo/      # <ts>.txt, 256 B
+mc ls az-b/crud-test/demo/      # same object, replicated (allow ~seconds of lag)
+mc cat front/crud-test/demo/<ts>.txt   # the random text itself
+```
+
+**3. Object updated (UPDATE)** — size changed 256 → 140 bytes, content starts with `UPDATED ---`
+
+```bash
+mc stat az-a/crud-test/demo/<ts>.txt   # Size: 140 B, fresh LastModified
+mc cat  az-b/crud-test/demo/<ts>.txt | head -1    # "UPDATED ---" (replicated overwrite)
+```
+
+**4. Object deleted (DELETE)** — gone from both sites
+
+```bash
+mc ls az-a/crud-test/demo/      # empty
+mc ls az-b/crud-test/demo/      # empty — deletes replicate too
+mc stat front/crud-test/demo/<ts>.txt   # error: Object does not exist
+```
+
+**5. Replication layer itself**
+
+```bash
+# inside mc-shell
+mc admin replicate status az-a   # buckets/policies/users "in sync"
+```
+
+**6. Storage layer under it**
+
+```bash
+kubectl get volumes.longhorn.io -n longhorn-system   # both volumes attached / healthy
+```
+
+## Troubleshooting
+
+- `connection refused` on localhost:9000 → the port-forward isn't running.
+- `[APP_METRIC] ERROR config missing` → `source logging/.env` first.
+- Object visible on az-a but not az-b → check `mc admin replicate status az-a`;
+  replication is near-synchronous, not instant. Persistent divergence:
+  `mc admin replicate resync start az-a az-b`.
+- Frontend has one endpoint instead of two → a MinIO pod is unready;
+  `kubectl describe pod -n storage <pod>`. Traffic still flows via the
+  surviving pod (that's the failover design — see `minio_migration.html`).

k8s/storage/test/go.mod

@@ -0,0 +1,23 @@
+module homelab/storage/test
+
+go 1.22
+
+require github.com/minio/minio-go/v7 v7.0.80
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/go-ini/ini v1.67.0 // indirect
+	github.com/goccy/go-json v0.10.3 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/klauspost/compress v1.17.11 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
+	github.com/minio/md5-simd v1.1.2 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/rs/xid v1.6.0 // indirect
+	golang.org/x/crypto v0.28.0 // indirect
+	golang.org/x/net v0.30.0 // indirect
+	golang.org/x/sys v0.26.0 // indirect
+	golang.org/x/text v0.19.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

k8s/storage/test/go.sum

@@ -0,0 +1,37 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
+github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
+github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
+github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
+github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
+github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
+github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
+github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
+github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
+github.com/minio/minio-go/v7 v7.0.80 h1:2mdUHXEykRdY/BigLt3Iuu1otL0JTogT0Nmltg0wujk=
+github.com/minio/minio-go/v7 v7.0.80/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
+github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
+golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
+golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
+golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
+golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
+golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

k8s/storage/test/main.go

@@ -0,0 +1,152 @@
+// CRUD demo against the universal MinIO storage frontend
+// (minio.storage.svc.cluster.local:9000).
+//
+// Run from outside the cluster via a port-forward:
+//
+//	kubectl port-forward svc/minio -n storage 9000:9000 &
+//	source logging/.env
+//	cd storage/test && go mod tidy && go run .
+//
+// In-cluster, set MINIO_ENDPOINT=minio.storage.svc.cluster.local:9000.
+//
+// Every S3 call emits [SERVICE_METRIC] op latency; every failure emits
+// [APP_METRIC] ERROR with context and aborts (no silent catches).
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"math/rand"
+	"os"
+	"time"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+const bucket = "crud-test"
+
+func getenv(key, fallback string) string {
+	if v := os.Getenv(key); v != "" {
+		return v
+	}
+	return fallback
+}
+
+// timed wraps an S3 operation: emits a serviceMetric on success,
+// an applicationMetric and exit(1) on failure.
+func timed(op string, fn func() error) {
+	start := time.Now()
+	if err := fn(); err != nil {
+		fmt.Printf("[APP_METRIC] ERROR s3.%s failed bucket=%s | trace=%v\n", op, bucket, err)
+		os.Exit(1)
+	}
+	fmt.Printf("[SERVICE_METRIC] s3.%s.latency_ms=%d ms\n", op, time.Since(start).Milliseconds())
+}
+
+func randomText(n int) []byte {
+	const letters = "abcdefghijklmnopqrstuvwxyz \n"
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = letters[rand.Intn(len(letters))]
+	}
+	return b
+}
+
+func main() {
+	endpoint := getenv("MINIO_ENDPOINT", "localhost:9000")
+	user := os.Getenv("MINIO_ROOT_USER")
+	pass := os.Getenv("MINIO_ROOT_PASSWORD")
+	if user == "" || pass == "" {
+		fmt.Println("[APP_METRIC] ERROR config missing | trace=MINIO_ROOT_USER / MINIO_ROOT_PASSWORD not set (source storage/.env)")
+		os.Exit(1)
+	}
+
+	ctx := context.Background()
+	client, err := minio.New(endpoint, &minio.Options{
+		Creds:  credentials.NewStaticV4(user, pass, ""),
+		Secure: false, // in-cluster traffic, no TLS
+	})
+	if err != nil {
+		fmt.Printf("[APP_METRIC] ERROR s3.connect failed endpoint=%s | trace=%v\n", endpoint, err)
+		os.Exit(1)
+	}
+
+	key := fmt.Sprintf("demo/%d.txt", time.Now().Unix())
+	original := randomText(256)
+	updated := append([]byte("UPDATED ---\n"), randomText(128)...)
+
+	// Ensure bucket (idempotent). Site replication propagates it to az-b.
+	timed("ensure_bucket", func() error {
+		exists, err := client.BucketExists(ctx, bucket)
+		if err != nil || exists {
+			return err
+		}
+		return client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{})
+	})
+
+	// CREATE
+	timed("put", func() error {
+		_, err := client.PutObject(ctx, bucket, key,
+			bytes.NewReader(original), int64(len(original)),
+			minio.PutObjectOptions{ContentType: "text/plain"})
+		return err
+	})
+	fmt.Printf("created %s/%s (%d bytes of random text)\n", bucket, key, len(original))
+
+	// READ — and verify content round-trips
+	timed("get", func() error {
+		obj, err := client.GetObject(ctx, bucket, key, minio.GetObjectOptions{})
+		if err != nil {
+			return err
+		}
+		defer obj.Close()
+		got, err := io.ReadAll(obj)
+		if err != nil {
+			return err
+		}
+		if !bytes.Equal(got, original) {
+			return fmt.Errorf("read-back mismatch: want %d bytes, got %d", len(original), len(got))
+		}
+		return nil
+	})
+	fmt.Println("read back and verified content")
+
+	// UPDATE — S3 semantics: overwrite the object in place
+	timed("update", func() error {
+		_, err := client.PutObject(ctx, bucket, key,
+			bytes.NewReader(updated), int64(len(updated)),
+			minio.PutObjectOptions{ContentType: "text/plain"})
+		return err
+	})
+	fmt.Println("updated (overwrote) object")
+
+	// LIST the demo/ prefix
+	timed("list", func() error {
+		for obj := range client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: "demo/", Recursive: true}) {
+			if obj.Err != nil {
+				return obj.Err
+			}
+			fmt.Printf("  %s  %d bytes  %s\n", obj.Key, obj.Size, obj.LastModified.Format(time.RFC3339))
+		}
+		return nil
+	})
+
+	// DELETE — and verify it is gone
+	timed("delete", func() error {
+		if err := client.RemoveObject(ctx, bucket, key, minio.RemoveObjectOptions{}); err != nil {
+			return err
+		}
+		_, err := client.StatObject(ctx, bucket, key, minio.StatObjectOptions{})
+		if err == nil {
+			return fmt.Errorf("object %s still exists after delete", key)
+		}
+		if minio.ToErrorResponse(err).Code != "NoSuchKey" {
+			return err
+		}
+		return nil
+	})
+	fmt.Println("deleted and verified gone — CRUD cycle complete")
+}