From 36aea89e47af9374e075b33a5c5d35ff2119d0cb Mon Sep 17 00:00:00 2001 From: Story Crater Bot <19826264+Riotpiaole@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:16:56 -0700 Subject: [PATCH] k8s/storage: add minio s3 with 3-way replication and oidc - MinIO 3-node site replication (az-a/b/c) - S3 backend for Loki chunks (10-day retention) - OIDC integration with Authentik - envFrom for secret injection --- k8s/storage/.env.example | 9 ++ k8s/storage/bootstrap.sh | 109 ++++++++++++++++ k8s/storage/longhorn-ingress-oauth2.yaml | 25 ++++ k8s/storage/longhorn-oauth2-proxy.yaml | 106 ++++++++++++++++ k8s/storage/minio-bucket-init.sh | 57 +++++++++ k8s/storage/minio-legacy-alias.yaml | 14 +++ k8s/storage/minio-operator-values.yaml | 48 +++++++ k8s/storage/minio-replication-job.yaml | 75 +++++++++++ k8s/storage/minio-service.yaml | 17 +++ k8s/storage/minio-tenant.yaml | 141 +++++++++++++++++++++ k8s/storage/minio-values.yaml | 64 ++++++++++ k8s/storage/test/README.md | 137 ++++++++++++++++++++ k8s/storage/test/go.mod | 23 ++++ k8s/storage/test/go.sum | 37 ++++++ k8s/storage/test/main.go | 152 +++++++++++++++++++++++ 15 files changed, 1014 insertions(+) create mode 100644 k8s/storage/.env.example create mode 100755 k8s/storage/bootstrap.sh create mode 100644 k8s/storage/longhorn-ingress-oauth2.yaml create mode 100644 k8s/storage/longhorn-oauth2-proxy.yaml create mode 100755 k8s/storage/minio-bucket-init.sh create mode 100644 k8s/storage/minio-legacy-alias.yaml create mode 100644 k8s/storage/minio-operator-values.yaml create mode 100644 k8s/storage/minio-replication-job.yaml create mode 100644 k8s/storage/minio-service.yaml create mode 100644 k8s/storage/minio-tenant.yaml create mode 100644 k8s/storage/minio-values.yaml create mode 100644 k8s/storage/test/README.md create mode 100644 k8s/storage/test/go.mod create mode 100644 k8s/storage/test/go.sum create mode 100644 k8s/storage/test/main.go diff --git a/k8s/storage/.env.example b/k8s/storage/.env.example new file mode 100644 index 0000000..211f6f0 --- /dev/null +++ b/k8s/storage/.env.example @@ -0,0 +1,9 @@ +# storage/.env.example +# Copy to storage/.env and fill in real values. +# The real .env is gitignored — never commit it. +# Used by: k8s/storage/bootstrap.sh + +# MinIO root credentials (S3-compatible object store) +# openssl rand -base64 24 +MINIO_ROOT_USER= +MINIO_ROOT_PASSWORD= diff --git a/k8s/storage/bootstrap.sh b/k8s/storage/bootstrap.sh new file mode 100755 index 0000000..63e5887 --- /dev/null +++ b/k8s/storage/bootstrap.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# k8s/storage/bootstrap.sh +# Deploys the MinIO multi-AZ object store into the storage namespace. +# +# Prerequisites: +# - kubectl configured (KUBECONFIG pointing to cluster-config/kubeconfig) +# - helm >= 3.x installed +# - k8s/storage/.env file containing: +# MINIO_ROOT_USER=... +# MINIO_ROOT_PASSWORD=... +# OR those variables already exported in the calling shell. +# +# Idempotent: safe to re-run — helm upgrade --install and kubectl apply are both idempotent. +# Disaster recovery order: run this before k8s/logging/bootstrap.sh (Loki depends on MinIO). +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +KUBECONFIG="${KUBECONFIG:-${SCRIPT_DIR}/../../cluster-config/kubeconfig}" +export KUBECONFIG + +# ── Load credentials ────────────────────────────────────────────────────────── +if [[ -f "${SCRIPT_DIR}/.env" ]]; then + # shellcheck source=/dev/null + source "${SCRIPT_DIR}/.env" +fi + +for var in MINIO_ROOT_USER MINIO_ROOT_PASSWORD; do + if [[ -z "${!var:-}" ]]; then + echo "ERROR: ${var} is not set. Export it or place it in k8s/storage/.env" + exit 1 + fi +done + +# ── Namespace ───────────────────────────────────────────────────────────────── +echo "==> Creating storage namespace with privileged pod security..." +kubectl create namespace storage --dry-run=client -o yaml | kubectl apply -f - +kubectl label namespace storage \ + pod-security.kubernetes.io/enforce=privileged \ + pod-security.kubernetes.io/enforce-version=latest \ + --overwrite + +# ── Helm repos ──────────────────────────────────────────────────────────────── +echo "==> Adding Helm repositories..." +helm repo add minio https://charts.min.io +helm repo update minio + +# ── PVC for az-a (adopts existing volume) ──────────────────────────────────── +echo "==> Applying az-a PVC (adopts pre-existing Longhorn volume)..." +kubectl apply -f "${SCRIPT_DIR}/minio-az-a-pvc.yaml" + +# ── MinIO az-a (control-plane node, zone az-a) ─────────────────────────────── +echo "==> Installing MinIO az-a..." +helm upgrade --install minio-az-a minio/minio \ + --namespace storage \ + --values "${SCRIPT_DIR}/minio-az-a-values.yaml" \ + --set rootUser="${MINIO_ROOT_USER}" \ + --set rootPassword="${MINIO_ROOT_PASSWORD}" \ + --wait \ + --timeout 5m + +echo "==> Waiting for minio-az-a Deployment to be ready..." +kubectl rollout status deployment/minio-az-a -n storage --timeout=120s + +# ── MinIO az-b (worker node, zone az-b) ────────────────────────────────────── +echo "==> Installing MinIO az-b..." +helm upgrade --install minio-az-b minio/minio \ + --namespace storage \ + --values "${SCRIPT_DIR}/minio-az-b-values.yaml" \ + --set rootUser="${MINIO_ROOT_USER}" \ + --set rootPassword="${MINIO_ROOT_PASSWORD}" \ + --wait \ + --timeout 5m + +echo "==> Waiting for minio-az-b Deployment to be ready..." +kubectl rollout status deployment/minio-az-b -n storage --timeout=120s + +# ── Universal frontend service + legacy alias ───────────────────────────────── +echo "==> Applying universal frontend service and legacy ExternalName alias..." +kubectl apply -f "${SCRIPT_DIR}/minio-service.yaml" +kubectl apply -f "${SCRIPT_DIR}/minio-legacy-alias.yaml" + +# ── Site replication ────────────────────────────────────────────────────────── +echo "==> Running site replication setup job..." +# Delete any leftover completed job first (kubectl apply on Jobs is not idempotent) +kubectl delete job minio-site-replication-setup -n storage --ignore-not-found +kubectl apply -f "${SCRIPT_DIR}/minio-replication-job.yaml" + +echo "==> Waiting for replication job to complete..." +kubectl wait --for=condition=complete job/minio-site-replication-setup \ + -n storage --timeout=120s + +echo "==> Replication job output:" +kubectl logs -n storage \ + -l job-name=minio-site-replication-setup \ + --tail=20 + +# ── Done ────────────────────────────────────────────────────────────────────── +echo "" +echo "==> MinIO multi-AZ stack is up." +echo "" +echo "MinIO console (S3 browser):" +echo " kubectl port-forward -n storage svc/minio 9001:9001" +echo " http://localhost:9001 (${MINIO_ROOT_USER} / )" +echo "" +echo "MinIO S3 endpoint for apps (active-active frontend):" +echo " http://minio.storage.svc.cluster.local:9000" +echo "" +echo "Verify site replication:" +echo " kubectl logs -n storage -l job-name=minio-site-replication-setup" diff --git a/k8s/storage/longhorn-ingress-oauth2.yaml b/k8s/storage/longhorn-ingress-oauth2.yaml new file mode 100644 index 0000000..54db685 --- /dev/null +++ b/k8s/storage/longhorn-ingress-oauth2.yaml @@ -0,0 +1,25 @@ +# Ingress for Longhorn storage UI — routes to OAuth2-Proxy +# TLS terminated here; oauth2-proxy handles OIDC auth + +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: longhorn + namespace: longhorn-system +spec: + ingressClassName: nginx + tls: + - secretName: longhorn-tls + hosts: + - longhorn.riotpiao.homelab.com + rules: + - host: longhorn.riotpiao.homelab.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: oauth2-proxy + port: + number: 4180 diff --git a/k8s/storage/longhorn-oauth2-proxy.yaml b/k8s/storage/longhorn-oauth2-proxy.yaml new file mode 100644 index 0000000..afb8bea --- /dev/null +++ b/k8s/storage/longhorn-oauth2-proxy.yaml @@ -0,0 +1,106 @@ +# OAuth2-Proxy for Longhorn storage UI +# Protects persistent volume management interface with Authentik OIDC + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: oauth2-proxy + namespace: longhorn-system + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: oauth2-proxy + namespace: longhorn-system +spec: + replicas: 1 + selector: + matchLabels: + app: oauth2-proxy + template: + metadata: + labels: + app: oauth2-proxy + annotations: + secret.reloader.stakater.com/reload: "longhorn-oidc" + spec: + serviceAccountName: oauth2-proxy + containers: + - name: oauth2-proxy + image: quay.io/oauth2-proxy/oauth2-proxy:v7.5.1 + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 4180 + protocol: TCP + env: + - name: OAUTH2_PROXY_PROVIDER + value: "oidc" + - name: OAUTH2_PROXY_OIDC_ISSUER_URL + value: "https://authentik.riotpiao.homelab.com/application/o/longhorn/" + - name: OAUTH2_PROXY_CLIENT_ID + value: "longhorn" + - name: OAUTH2_PROXY_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: longhorn-oidc + key: clientSecret + - name: OAUTH2_PROXY_COOKIE_SECRET + valueFrom: + secretKeyRef: + name: longhorn-oidc + key: cookieSecret + - name: OAUTH2_PROXY_REDIRECT_URL + value: "https://longhorn.riotpiao.homelab.com/oauth2/callback" + - name: OAUTH2_PROXY_UPSTREAM + value: "http://longhorn-frontend:80" + - name: OAUTH2_PROXY_COOKIE_SECURE + value: "true" + - name: OAUTH2_PROXY_COOKIE_HTTPONLY + value: "true" + - name: OAUTH2_PROXY_COOKIE_SAMESITE + value: "Lax" + - name: OAUTH2_PROXY_EMAIL_DOMAIN + value: "*" + - name: OAUTH2_PROXY_SKIP_AUTH_REGEX + value: "^/health" + - name: OAUTH2_PROXY_PASS_AUTHORIZATION_HEADER + value: "true" + - name: OAUTH2_PROXY_REVERSE_PROXY + value: "true" + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 100m + memory: 128Mi + livenessProbe: + httpGet: + path: /ping + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /ping + port: http + initialDelaySeconds: 5 + periodSeconds: 5 + +--- +apiVersion: v1 +kind: Service +metadata: + name: oauth2-proxy + namespace: longhorn-system +spec: + type: ClusterIP + ports: + - port: 4180 + targetPort: http + protocol: TCP + name: http + selector: + app: oauth2-proxy diff --git a/k8s/storage/minio-bucket-init.sh b/k8s/storage/minio-bucket-init.sh new file mode 100755 index 0000000..a38cfec --- /dev/null +++ b/k8s/storage/minio-bucket-init.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# Safe MinIO bucket initialization via Job (credentials in Secret, not env) +# Usage: ./minio-bucket-init.sh [bucket2] ... +set -euo pipefail + +NAMESPACE="${1:?Missing namespace}" +shift +BUCKETS=("$@") + +if [ ${#BUCKETS[@]} -eq 0 ]; then + echo "Usage: $0 [bucket2] ..." >&2 + exit 1 +fi + +# Create Secret with Minio credentials (safe: sourced from env, not exposed in pod) +kubectl create secret generic minio-creds -n "$NAMESPACE" \ + --from-literal=MINIO_ROOT_USER="${MINIO_ROOT_USER:?Missing MINIO_ROOT_USER}" \ + --from-literal=MINIO_ROOT_PASSWORD="${MINIO_ROOT_PASSWORD:?Missing MINIO_ROOT_PASSWORD}" \ + --dry-run=client -o yaml | kubectl apply -f - + +# Create init Job that mounts Secret as volume, preventing env exposure +BUCKET_ARGS=$(printf '"%s", ' "${BUCKETS[@]}" | sed 's/, $//') +kubectl apply -f - <.storage.svc.cluster.local. + for site in $SITES; do + mc alias set "$site" "http://minio-${site}.storage.svc.cluster.local:9000" \ + "$ROOT_USER" "$ROOT_PASSWORD" + done + + first=$(echo $SITES | cut -d' ' -f1) + + # Idempotent: skip only if every requested site is already part + # of the replication group. A partially-configured group (e.g. + # az-c newly added to SITES) falls through to `replicate add`, + # which expands an existing group in place. + # NOTE: pure-shell matching — the minio/mc image has no grep. + info=$(mc admin replicate info "$first" 2>/dev/null || true) + missing=0 + for site in $SITES; do + case "$info" in + *"$site"*) ;; + *) missing=1 ;; + esac + done + if [ "$missing" -eq 0 ]; then + echo "Site replication already spans all sites ($SITES) — nothing to do" + mc admin replicate info "$first" + exit 0 + fi + + echo "Configuring site replication across: $SITES" + mc admin replicate add $SITES + + echo "Replication status:" + mc admin replicate info "$first" + env: + # Space-separated list of replication sites, named after the AZ + # node labels. Each site must have a Helm release minio- + # (e.g. minio-az-a) so the Service DNS resolves. To add an AZ + # later: deploy minio-az-, append "az-" here, then delete + # and re-apply this Job. + - name: SITES + value: "az-a az-b az-c" + # MinIO site replication requires identical root credentials on + # every site, so reading one release's Secret covers all of them. + # The MinIO chart creates a Secret named after the release with + # keys rootUser and rootPassword. + - name: ROOT_USER + valueFrom: + secretKeyRef: + name: minio-az-a + key: rootUser + - name: ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: minio-az-a + key: rootPassword diff --git a/k8s/storage/minio-service.yaml b/k8s/storage/minio-service.yaml new file mode 100644 index 0000000..cceef77 --- /dev/null +++ b/k8s/storage/minio-service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: minio + namespace: storage +spec: + type: ClusterIP + # Helm chart selector — routes to minio-az-a/b/c pods + selector: + app: minio + ports: + - name: s3 + port: 9000 + targetPort: 9000 + - name: console + port: 9001 + targetPort: 9001 diff --git a/k8s/storage/minio-tenant.yaml b/k8s/storage/minio-tenant.yaml new file mode 100644 index 0000000..bf74ef8 --- /dev/null +++ b/k8s/storage/minio-tenant.yaml @@ -0,0 +1,141 @@ +apiVersion: minio.min.io/v2 +kind: Tenant +metadata: + name: minio-cluster + namespace: storage + labels: + app: minio +spec: + image: minio/minio:RELEASE.2024-06-13T20-48-48Z + + credsSecret: + name: minio-creds + + # 3-zone distributed cluster (one server per zone) + pools: + - name: az-a + servers: 1 + volumesPerServer: 1 + size: 100Gi + storageClass: longhorn + + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: topology.kubernetes.io/zone + operator: In + values: [az-a] + + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + + volumeClaimTemplate: + metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + storageClassName: longhorn + resources: + requests: + storage: 100Gi + + - name: az-b + servers: 1 + volumesPerServer: 1 + size: 100Gi + storageClass: longhorn + + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: topology.kubernetes.io/zone + operator: In + values: [az-b] + + volumeClaimTemplate: + metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + storageClassName: longhorn + resources: + requests: + storage: 100Gi + + - name: az-c + servers: 1 + volumesPerServer: 1 + size: 100Gi + storageClass: longhorn + + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: topology.kubernetes.io/zone + operator: In + values: [az-c] + + volumeClaimTemplate: + metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + storageClassName: longhorn + resources: + requests: + storage: 100Gi + + # Console (web UI) + console: + image: minio/console:v0.30.0 + replicas: 1 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + + # Environment variables — OIDC config + env: + - name: MINIO_IDENTITY_OPENID_CONFIG_URL + value: "https://authentik.riotpiao.homelab.com/application/o/minio/.well-known/openid-configuration" + - name: MINIO_IDENTITY_OPENID_CLIENT_ID + value: "minio" + - name: MINIO_IDENTITY_OPENID_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: minio-oidc + key: MINIO_IDENTITY_OPENID_CLIENT_SECRET + - name: MINIO_IDENTITY_OPENID_CLAIM_NAME + value: "policy" + - name: MINIO_IDENTITY_OPENID_SCOPES + value: "openid,profile,email,minio" + - name: MINIO_IDENTITY_OPENID_REDIRECT_URI + value: "https://minio.riotpiao.homelab.com/oauth_callback" + - name: MINIO_IDENTITY_OPENID_DISPLAY_NAME + value: "Authentik" + + # Metrics + metrics: + enabled: true + port: 9000 + + # No auto-TLS (using cert-manager) + requestAutoCert: false + + # No built-in ingress + ingress: + enabled: false diff --git a/k8s/storage/minio-values.yaml b/k8s/storage/minio-values.yaml new file mode 100644 index 0000000..3730fed --- /dev/null +++ b/k8s/storage/minio-values.yaml @@ -0,0 +1,64 @@ +mode: standalone + +rootUser: "" +rootPassword: "" + +persistence: + enabled: true + storageClass: longhorn + size: 100Gi + +deploymentUpdate: + type: Recreate + +resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + +service: + type: ClusterIP + port: 9000 + +consoleService: + type: ClusterIP + port: 9001 + +ingress: + enabled: false + +consoleIngress: + enabled: false + +metrics: + serviceMonitor: + enabled: true + +envFrom: + - secretRef: + name: minio-oidc + +environment: + MINIO_IDENTITY_OPENID_CONFIG_URL: "https://authentik.riotpiao.homelab.com/application/o/minio/.well-known/openid-configuration" + MINIO_IDENTITY_OPENID_CLIENT_ID: "minio" + MINIO_IDENTITY_OPENID_CLAIM_NAME: "policy" + MINIO_IDENTITY_OPENID_SCOPES: "openid,profile,email,minio" + MINIO_IDENTITY_OPENID_REDIRECT_URI: "https://minio.riotpiao.homelab.com/oauth_callback" + MINIO_IDENTITY_OPENID_DISPLAY_NAME: "Authentik" + +podAnnotations: + secret.reloader.stakater.com/reload: "minio-oidc" + configmap.reloader.stakater.com/reload: "homelab-ca" + +extraVolumes: + - name: homelab-ca + configMap: + name: homelab-ca + +extraVolumeMounts: + - name: homelab-ca + mountPath: /etc/minio/certs/CAs + readOnly: true diff --git a/k8s/storage/test/README.md b/k8s/storage/test/README.md new file mode 100644 index 0000000..f8d2a85 --- /dev/null +++ b/k8s/storage/test/README.md @@ -0,0 +1,137 @@ +# MinIO CRUD Example (Go) + +A minimal Go program that exercises the cluster's object storage through the +**universal storage frontend** — `minio.storage.svc.cluster.local:9000` — the +single DNS name that load-balances across both node-pinned MinIO instances +(`minio-az-a` on talos-cp-1, `minio-az-b` on talos-worker-1). + +It runs a full CRUD cycle with a random text file: + +| Step | S3 call | What it proves | +|------|---------|----------------| +| Ensure bucket | `BucketExists` / `MakeBucket` | bucket `crud-test` exists (idempotent) | +| **C**reate | `PutObject` | write path through the frontend | +| **R**ead | `GetObject` + byte compare | content round-trips intact | +| **U**pdate | `PutObject` (overwrite) | S3 update semantics (objects are replaced, not edited) | +| List | `ListObjects` prefix `demo/` | enumeration | +| **D**elete | `RemoveObject` + `StatObject` | object gone (`NoSuchKey` confirmed) | + +Every operation emits `[SERVICE_METRIC] s3..latency_ms= ms`; any failure +emits `[APP_METRIC] ERROR s3. failed ... | trace=...` and exits non-zero. + +## Run it + +```bash +# 1. expose the frontend locally (leave running in another terminal) +kubectl port-forward svc/minio -n storage 9000:9000 + +# 2. credentials — same root creds used by both MinIO sites +source logging/.env # exports MINIO_ROOT_USER / MINIO_ROOT_PASSWORD + +# 3. run +cd storage/test +go mod tidy && go run . +``` + +In-cluster (e.g. from a Job), skip the port-forward and set +`MINIO_ENDPOINT=minio.storage.svc.cluster.local:9000`. + +Expected output: + +``` +[SERVICE_METRIC] s3.ensure_bucket.latency_ms=145 ms +[SERVICE_METRIC] s3.put.latency_ms=19 ms +created crud-test/demo/.txt (256 bytes of random text) +[SERVICE_METRIC] s3.get.latency_ms=9 ms +read back and verified content +[SERVICE_METRIC] s3.update.latency_ms=70 ms +updated (overwrote) object + demo/.txt 140 bytes +[SERVICE_METRIC] s3.list.latency_ms=11 ms +[SERVICE_METRIC] s3.delete.latency_ms=68 ms +deleted and verified gone — CRUD cycle complete +``` + +## Validating each state with kubectl + +The program verifies itself in-process (read-back compare, post-delete stat), +but every state is also independently observable from outside with `kubectl`. +The helper below drops you into a throwaway `mc` shell wired to both sites — +all subsequent checks use it: + +```bash +source logging/.env +kubectl run mc-shell --rm -it --restart=Never --image=minio/mc -n storage \ + --env="U=$MINIO_ROOT_USER" --env="P=$MINIO_ROOT_PASSWORD" \ + --command -- /bin/sh -c ' +mc alias set front http://minio.storage.svc.cluster.local:9000 "$U" "$P" +mc alias set az-a http://minio-az-a.storage.svc.cluster.local:9000 "$U" "$P" +mc alias set az-b http://minio-az-b.storage.svc.cluster.local:9000 "$U" "$P" +exec /bin/sh' +``` + +> The demo deletes its object at the end, so to inspect the CREATE/UPDATE +> states at your own pace, comment out the `// DELETE` block in `main.go` +> and re-run (the delete is idempotent to re-apply later). + +**0. Frontend is healthy (before running anything)** + +```bash +kubectl get endpoints minio -n storage # expect TWO pod IPs on :9000 +kubectl get pods -n storage -o wide # az-a on talos-cp-1, az-b on talos-worker-1 +``` + +**1. Bucket created** — and replicated to BOTH sites + +```bash +# inside mc-shell — the bucket must appear on each site individually +mc ls az-a | grep crud-test +mc ls az-b | grep crud-test # proves site replication propagated it +``` + +**2. Object created (CREATE)** — 256 bytes, present on both nodes + +```bash +mc ls az-a/crud-test/demo/ # .txt, 256 B +mc ls az-b/crud-test/demo/ # same object, replicated (allow ~seconds of lag) +mc cat front/crud-test/demo/.txt # the random text itself +``` + +**3. Object updated (UPDATE)** — size changed 256 → 140 bytes, content starts with `UPDATED ---` + +```bash +mc stat az-a/crud-test/demo/.txt # Size: 140 B, fresh LastModified +mc cat az-b/crud-test/demo/.txt | head -1 # "UPDATED ---" (replicated overwrite) +``` + +**4. Object deleted (DELETE)** — gone from both sites + +```bash +mc ls az-a/crud-test/demo/ # empty +mc ls az-b/crud-test/demo/ # empty — deletes replicate too +mc stat front/crud-test/demo/.txt # error: Object does not exist +``` + +**5. Replication layer itself** + +```bash +# inside mc-shell +mc admin replicate status az-a # buckets/policies/users "in sync" +``` + +**6. Storage layer under it** + +```bash +kubectl get volumes.longhorn.io -n longhorn-system # both volumes attached / healthy +``` + +## Troubleshooting + +- `connection refused` on localhost:9000 → the port-forward isn't running. +- `[APP_METRIC] ERROR config missing` → `source logging/.env` first. +- Object visible on az-a but not az-b → check `mc admin replicate status az-a`; + replication is near-synchronous, not instant. Persistent divergence: + `mc admin replicate resync start az-a az-b`. +- Frontend has one endpoint instead of two → a MinIO pod is unready; + `kubectl describe pod -n storage `. Traffic still flows via the + surviving pod (that's the failover design — see `minio_migration.html`). diff --git a/k8s/storage/test/go.mod b/k8s/storage/test/go.mod new file mode 100644 index 0000000..291a942 --- /dev/null +++ b/k8s/storage/test/go.mod @@ -0,0 +1,23 @@ +module homelab/storage/test + +go 1.22 + +require github.com/minio/minio-go/v7 v7.0.80 + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/goccy/go-json v0.10.3 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rs/xid v1.6.0 // indirect + golang.org/x/crypto v0.28.0 // indirect + golang.org/x/net v0.30.0 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/text v0.19.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/k8s/storage/test/go.sum b/k8s/storage/test/go.sum new file mode 100644 index 0000000..3cdb8bd --- /dev/null +++ b/k8s/storage/test/go.sum @@ -0,0 +1,37 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= +github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.80 h1:2mdUHXEykRdY/BigLt3Iuu1otL0JTogT0Nmltg0wujk= +github.com/minio/minio-go/v7 v7.0.80/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/k8s/storage/test/main.go b/k8s/storage/test/main.go new file mode 100644 index 0000000..33475fb --- /dev/null +++ b/k8s/storage/test/main.go @@ -0,0 +1,152 @@ +// CRUD demo against the universal MinIO storage frontend +// (minio.storage.svc.cluster.local:9000). +// +// Run from outside the cluster via a port-forward: +// +// kubectl port-forward svc/minio -n storage 9000:9000 & +// source logging/.env +// cd storage/test && go mod tidy && go run . +// +// In-cluster, set MINIO_ENDPOINT=minio.storage.svc.cluster.local:9000. +// +// Every S3 call emits [SERVICE_METRIC] op latency; every failure emits +// [APP_METRIC] ERROR with context and aborts (no silent catches). +package main + +import ( + "bytes" + "context" + "fmt" + "io" + "math/rand" + "os" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +const bucket = "crud-test" + +func getenv(key, fallback string) string { + if v := os.Getenv(key); v != "" { + return v + } + return fallback +} + +// timed wraps an S3 operation: emits a serviceMetric on success, +// an applicationMetric and exit(1) on failure. +func timed(op string, fn func() error) { + start := time.Now() + if err := fn(); err != nil { + fmt.Printf("[APP_METRIC] ERROR s3.%s failed bucket=%s | trace=%v\n", op, bucket, err) + os.Exit(1) + } + fmt.Printf("[SERVICE_METRIC] s3.%s.latency_ms=%d ms\n", op, time.Since(start).Milliseconds()) +} + +func randomText(n int) []byte { + const letters = "abcdefghijklmnopqrstuvwxyz \n" + b := make([]byte, n) + for i := range b { + b[i] = letters[rand.Intn(len(letters))] + } + return b +} + +func main() { + endpoint := getenv("MINIO_ENDPOINT", "localhost:9000") + user := os.Getenv("MINIO_ROOT_USER") + pass := os.Getenv("MINIO_ROOT_PASSWORD") + if user == "" || pass == "" { + fmt.Println("[APP_METRIC] ERROR config missing | trace=MINIO_ROOT_USER / MINIO_ROOT_PASSWORD not set (source storage/.env)") + os.Exit(1) + } + + ctx := context.Background() + client, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(user, pass, ""), + Secure: false, // in-cluster traffic, no TLS + }) + if err != nil { + fmt.Printf("[APP_METRIC] ERROR s3.connect failed endpoint=%s | trace=%v\n", endpoint, err) + os.Exit(1) + } + + key := fmt.Sprintf("demo/%d.txt", time.Now().Unix()) + original := randomText(256) + updated := append([]byte("UPDATED ---\n"), randomText(128)...) + + // Ensure bucket (idempotent). Site replication propagates it to az-b. + timed("ensure_bucket", func() error { + exists, err := client.BucketExists(ctx, bucket) + if err != nil || exists { + return err + } + return client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + }) + + // CREATE + timed("put", func() error { + _, err := client.PutObject(ctx, bucket, key, + bytes.NewReader(original), int64(len(original)), + minio.PutObjectOptions{ContentType: "text/plain"}) + return err + }) + fmt.Printf("created %s/%s (%d bytes of random text)\n", bucket, key, len(original)) + + // READ — and verify content round-trips + timed("get", func() error { + obj, err := client.GetObject(ctx, bucket, key, minio.GetObjectOptions{}) + if err != nil { + return err + } + defer obj.Close() + got, err := io.ReadAll(obj) + if err != nil { + return err + } + if !bytes.Equal(got, original) { + return fmt.Errorf("read-back mismatch: want %d bytes, got %d", len(original), len(got)) + } + return nil + }) + fmt.Println("read back and verified content") + + // UPDATE — S3 semantics: overwrite the object in place + timed("update", func() error { + _, err := client.PutObject(ctx, bucket, key, + bytes.NewReader(updated), int64(len(updated)), + minio.PutObjectOptions{ContentType: "text/plain"}) + return err + }) + fmt.Println("updated (overwrote) object") + + // LIST the demo/ prefix + timed("list", func() error { + for obj := range client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: "demo/", Recursive: true}) { + if obj.Err != nil { + return obj.Err + } + fmt.Printf(" %s %d bytes %s\n", obj.Key, obj.Size, obj.LastModified.Format(time.RFC3339)) + } + return nil + }) + + // DELETE — and verify it is gone + timed("delete", func() error { + if err := client.RemoveObject(ctx, bucket, key, minio.RemoveObjectOptions{}); err != nil { + return err + } + _, err := client.StatObject(ctx, bucket, key, minio.StatObjectOptions{}) + if err == nil { + return fmt.Errorf("object %s still exists after delete", key) + } + if minio.ToErrorResponse(err).Code != "NoSuchKey" { + return err + } + return nil + }) + fmt.Println("deleted and verified gone — CRUD cycle complete") +}