k8s/storage: add minio s3 with 3-way replication and oidc
- MinIO 3-node site replication (az-a/b/c) - S3 backend for Loki chunks (10-day retention) - OIDC integration with Authentik - envFrom for secret injection
This commit is contained in:
Story Crater Bot
@@ -0,0 +1,9 @@
|
||||
# storage/.env.example
|
||||
# Copy to storage/.env and fill in real values.
|
||||
# The real .env is gitignored — never commit it.
|
||||
# Used by: k8s/storage/bootstrap.sh
|
||||
|
||||
# MinIO root credentials (S3-compatible object store)
|
||||
# openssl rand -base64 24
|
||||
MINIO_ROOT_USER=
|
||||
MINIO_ROOT_PASSWORD=
|
||||
Executable
+109
@@ -0,0 +1,109 @@
|
||||
#!/usr/bin/env bash
|
||||
# k8s/storage/bootstrap.sh
|
||||
# Deploys the MinIO multi-AZ object store into the storage namespace.
|
||||
#
|
||||
# Prerequisites:
|
||||
# - kubectl configured (KUBECONFIG pointing to cluster-config/kubeconfig)
|
||||
# - helm >= 3.x installed
|
||||
# - k8s/storage/.env file containing:
|
||||
# MINIO_ROOT_USER=...
|
||||
# MINIO_ROOT_PASSWORD=...
|
||||
# OR those variables already exported in the calling shell.
|
||||
#
|
||||
# Idempotent: safe to re-run — helm upgrade --install and kubectl apply are both idempotent.
|
||||
# Disaster recovery order: run this before k8s/logging/bootstrap.sh (Loki depends on MinIO).
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
KUBECONFIG="${KUBECONFIG:-${SCRIPT_DIR}/../../cluster-config/kubeconfig}"
|
||||
export KUBECONFIG
|
||||
|
||||
# ── Load credentials ──────────────────────────────────────────────────────────
|
||||
if [[ -f "${SCRIPT_DIR}/.env" ]]; then
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/.env"
|
||||
fi
|
||||
|
||||
for var in MINIO_ROOT_USER MINIO_ROOT_PASSWORD; do
|
||||
if [[ -z "${!var:-}" ]]; then
|
||||
echo "ERROR: ${var} is not set. Export it or place it in k8s/storage/.env"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# ── Namespace ─────────────────────────────────────────────────────────────────
|
||||
echo "==> Creating storage namespace with privileged pod security..."
|
||||
kubectl create namespace storage --dry-run=client -o yaml | kubectl apply -f -
|
||||
kubectl label namespace storage \
|
||||
pod-security.kubernetes.io/enforce=privileged \
|
||||
pod-security.kubernetes.io/enforce-version=latest \
|
||||
--overwrite
|
||||
|
||||
# ── Helm repos ────────────────────────────────────────────────────────────────
|
||||
echo "==> Adding Helm repositories..."
|
||||
helm repo add minio https://charts.min.io
|
||||
helm repo update minio
|
||||
|
||||
# ── PVC for az-a (adopts existing volume) ────────────────────────────────────
|
||||
echo "==> Applying az-a PVC (adopts pre-existing Longhorn volume)..."
|
||||
kubectl apply -f "${SCRIPT_DIR}/minio-az-a-pvc.yaml"
|
||||
|
||||
# ── MinIO az-a (control-plane node, zone az-a) ───────────────────────────────
|
||||
echo "==> Installing MinIO az-a..."
|
||||
helm upgrade --install minio-az-a minio/minio \
|
||||
--namespace storage \
|
||||
--values "${SCRIPT_DIR}/minio-az-a-values.yaml" \
|
||||
--set rootUser="${MINIO_ROOT_USER}" \
|
||||
--set rootPassword="${MINIO_ROOT_PASSWORD}" \
|
||||
--wait \
|
||||
--timeout 5m
|
||||
|
||||
echo "==> Waiting for minio-az-a Deployment to be ready..."
|
||||
kubectl rollout status deployment/minio-az-a -n storage --timeout=120s
|
||||
|
||||
# ── MinIO az-b (worker node, zone az-b) ──────────────────────────────────────
|
||||
echo "==> Installing MinIO az-b..."
|
||||
helm upgrade --install minio-az-b minio/minio \
|
||||
--namespace storage \
|
||||
--values "${SCRIPT_DIR}/minio-az-b-values.yaml" \
|
||||
--set rootUser="${MINIO_ROOT_USER}" \
|
||||
--set rootPassword="${MINIO_ROOT_PASSWORD}" \
|
||||
--wait \
|
||||
--timeout 5m
|
||||
|
||||
echo "==> Waiting for minio-az-b Deployment to be ready..."
|
||||
kubectl rollout status deployment/minio-az-b -n storage --timeout=120s
|
||||
|
||||
# ── Universal frontend service + legacy alias ─────────────────────────────────
|
||||
echo "==> Applying universal frontend service and legacy ExternalName alias..."
|
||||
kubectl apply -f "${SCRIPT_DIR}/minio-service.yaml"
|
||||
kubectl apply -f "${SCRIPT_DIR}/minio-legacy-alias.yaml"
|
||||
|
||||
# ── Site replication ──────────────────────────────────────────────────────────
|
||||
echo "==> Running site replication setup job..."
|
||||
# Delete any leftover completed job first (kubectl apply on Jobs is not idempotent)
|
||||
kubectl delete job minio-site-replication-setup -n storage --ignore-not-found
|
||||
kubectl apply -f "${SCRIPT_DIR}/minio-replication-job.yaml"
|
||||
|
||||
echo "==> Waiting for replication job to complete..."
|
||||
kubectl wait --for=condition=complete job/minio-site-replication-setup \
|
||||
-n storage --timeout=120s
|
||||
|
||||
echo "==> Replication job output:"
|
||||
kubectl logs -n storage \
|
||||
-l job-name=minio-site-replication-setup \
|
||||
--tail=20
|
||||
|
||||
# ── Done ──────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "==> MinIO multi-AZ stack is up."
|
||||
echo ""
|
||||
echo "MinIO console (S3 browser):"
|
||||
echo " kubectl port-forward -n storage svc/minio 9001:9001"
|
||||
echo " http://localhost:9001 (${MINIO_ROOT_USER} / <your MINIO_ROOT_PASSWORD>)"
|
||||
echo ""
|
||||
echo "MinIO S3 endpoint for apps (active-active frontend):"
|
||||
echo " http://minio.storage.svc.cluster.local:9000"
|
||||
echo ""
|
||||
echo "Verify site replication:"
|
||||
echo " kubectl logs -n storage -l job-name=minio-site-replication-setup"
|
||||
@@ -0,0 +1,25 @@
|
||||
# Ingress for Longhorn storage UI — routes to OAuth2-Proxy
|
||||
# TLS terminated here; oauth2-proxy handles OIDC auth
|
||||
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: longhorn
|
||||
namespace: longhorn-system
|
||||
spec:
|
||||
ingressClassName: nginx
|
||||
tls:
|
||||
- secretName: longhorn-tls
|
||||
hosts:
|
||||
- longhorn.riotpiao.homelab.com
|
||||
rules:
|
||||
- host: longhorn.riotpiao.homelab.com
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: oauth2-proxy
|
||||
port:
|
||||
number: 4180
|
||||
@@ -0,0 +1,106 @@
|
||||
# OAuth2-Proxy for Longhorn storage UI
|
||||
# Protects persistent volume management interface with Authentik OIDC
|
||||
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: oauth2-proxy
|
||||
namespace: longhorn-system
|
||||
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: oauth2-proxy
|
||||
namespace: longhorn-system
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: oauth2-proxy
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: oauth2-proxy
|
||||
annotations:
|
||||
secret.reloader.stakater.com/reload: "longhorn-oidc"
|
||||
spec:
|
||||
serviceAccountName: oauth2-proxy
|
||||
containers:
|
||||
- name: oauth2-proxy
|
||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.5.1
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 4180
|
||||
protocol: TCP
|
||||
env:
|
||||
- name: OAUTH2_PROXY_PROVIDER
|
||||
value: "oidc"
|
||||
- name: OAUTH2_PROXY_OIDC_ISSUER_URL
|
||||
value: "https://authentik.riotpiao.homelab.com/application/o/longhorn/"
|
||||
- name: OAUTH2_PROXY_CLIENT_ID
|
||||
value: "longhorn"
|
||||
- name: OAUTH2_PROXY_CLIENT_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: longhorn-oidc
|
||||
key: clientSecret
|
||||
- name: OAUTH2_PROXY_COOKIE_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: longhorn-oidc
|
||||
key: cookieSecret
|
||||
- name: OAUTH2_PROXY_REDIRECT_URL
|
||||
value: "https://longhorn.riotpiao.homelab.com/oauth2/callback"
|
||||
- name: OAUTH2_PROXY_UPSTREAM
|
||||
value: "http://longhorn-frontend:80"
|
||||
- name: OAUTH2_PROXY_COOKIE_SECURE
|
||||
value: "true"
|
||||
- name: OAUTH2_PROXY_COOKIE_HTTPONLY
|
||||
value: "true"
|
||||
- name: OAUTH2_PROXY_COOKIE_SAMESITE
|
||||
value: "Lax"
|
||||
- name: OAUTH2_PROXY_EMAIL_DOMAIN
|
||||
value: "*"
|
||||
- name: OAUTH2_PROXY_SKIP_AUTH_REGEX
|
||||
value: "^/health"
|
||||
- name: OAUTH2_PROXY_PASS_AUTHORIZATION_HEADER
|
||||
value: "true"
|
||||
- name: OAUTH2_PROXY_REVERSE_PROXY
|
||||
value: "true"
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /ping
|
||||
port: http
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /ping
|
||||
port: http
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: oauth2-proxy
|
||||
namespace: longhorn-system
|
||||
spec:
|
||||
type: ClusterIP
|
||||
ports:
|
||||
- port: 4180
|
||||
targetPort: http
|
||||
protocol: TCP
|
||||
name: http
|
||||
selector:
|
||||
app: oauth2-proxy
|
||||
Executable
+57
@@ -0,0 +1,57 @@
|
||||
#!/bin/bash
|
||||
# Safe MinIO bucket initialization via Job (credentials in Secret, not env)
|
||||
# Usage: ./minio-bucket-init.sh <namespace> <bucket1> [bucket2] ...
|
||||
set -euo pipefail
|
||||
|
||||
NAMESPACE="${1:?Missing namespace}"
|
||||
shift
|
||||
BUCKETS=("$@")
|
||||
|
||||
if [ ${#BUCKETS[@]} -eq 0 ]; then
|
||||
echo "Usage: $0 <namespace> <bucket1> [bucket2] ..." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create Secret with Minio credentials (safe: sourced from env, not exposed in pod)
|
||||
kubectl create secret generic minio-creds -n "$NAMESPACE" \
|
||||
--from-literal=MINIO_ROOT_USER="${MINIO_ROOT_USER:?Missing MINIO_ROOT_USER}" \
|
||||
--from-literal=MINIO_ROOT_PASSWORD="${MINIO_ROOT_PASSWORD:?Missing MINIO_ROOT_PASSWORD}" \
|
||||
--dry-run=client -o yaml | kubectl apply -f -
|
||||
|
||||
# Create init Job that mounts Secret as volume, preventing env exposure
|
||||
BUCKET_ARGS=$(printf '"%s", ' "${BUCKETS[@]}" | sed 's/, $//')
|
||||
kubectl apply -f - <<EOF
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: minio-bucket-init-$RANDOM
|
||||
namespace: $NAMESPACE
|
||||
spec:
|
||||
ttlSecondsAfterFinished: 300
|
||||
backoffLimit: 2
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: minio-init
|
||||
image: minio/mc:latest
|
||||
volumeMounts:
|
||||
- name: minio-secret
|
||||
mountPath: /var/run/secrets/minio
|
||||
readOnly: true
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
MINIO_ROOT_USER=\$(cat /var/run/secrets/minio/MINIO_ROOT_USER)
|
||||
MINIO_ROOT_PASSWORD=\$(cat /var/run/secrets/minio/MINIO_ROOT_PASSWORD)
|
||||
mc alias set local http://minio.storage.svc.cluster.local:9000 "\$MINIO_ROOT_USER" "\$MINIO_ROOT_PASSWORD"
|
||||
mc mb --ignore-existing $BUCKET_ARGS
|
||||
volumes:
|
||||
- name: minio-secret
|
||||
secret:
|
||||
secretName: minio-creds
|
||||
EOF
|
||||
|
||||
echo "✓ MinIO bucket init job submitted for: ${BUCKETS[*]}"
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
# Backward-compatibility alias: anything still pointing at the old
|
||||
# minio.logging.svc.cluster.local address resolves to the universal
|
||||
# storage frontend. Apply AFTER `helm uninstall minio -n logging`
|
||||
# (the old release owns the Service name `minio` until then).
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: minio
|
||||
namespace: logging
|
||||
labels:
|
||||
app: minio
|
||||
spec:
|
||||
type: ExternalName
|
||||
externalName: minio.storage.svc.cluster.local
|
||||
@@ -0,0 +1,48 @@
|
||||
# minio-operator-values.yaml
|
||||
# MinIO Operator deployment with metrics enabled
|
||||
|
||||
operator:
|
||||
image:
|
||||
repository: minio/operator
|
||||
tag: "v5.0.0"
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
replicaCount: 1
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
|
||||
metrics:
|
||||
enabled: true
|
||||
port: 8080
|
||||
|
||||
rbac:
|
||||
create: true
|
||||
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- weight: 100
|
||||
preference:
|
||||
matchExpressions:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
|
||||
console:
|
||||
enabled: true
|
||||
replicaCount: 1
|
||||
image:
|
||||
repository: minio/console
|
||||
tag: "v0.30.0"
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 200m
|
||||
memory: 256Mi
|
||||
@@ -0,0 +1,75 @@
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: minio-site-replication-setup
|
||||
namespace: storage
|
||||
spec:
|
||||
# Auto-delete the Job pod 10 minutes after completion
|
||||
ttlSecondsAfterFinished: 600
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: mc
|
||||
image: minio/mc:latest
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
set -e
|
||||
|
||||
# Alias every site. Service names follow the release naming
|
||||
# convention minio-<site>.storage.svc.cluster.local.
|
||||
for site in $SITES; do
|
||||
mc alias set "$site" "http://minio-${site}.storage.svc.cluster.local:9000" \
|
||||
"$ROOT_USER" "$ROOT_PASSWORD"
|
||||
done
|
||||
|
||||
first=$(echo $SITES | cut -d' ' -f1)
|
||||
|
||||
# Idempotent: skip only if every requested site is already part
|
||||
# of the replication group. A partially-configured group (e.g.
|
||||
# az-c newly added to SITES) falls through to `replicate add`,
|
||||
# which expands an existing group in place.
|
||||
# NOTE: pure-shell matching — the minio/mc image has no grep.
|
||||
info=$(mc admin replicate info "$first" 2>/dev/null || true)
|
||||
missing=0
|
||||
for site in $SITES; do
|
||||
case "$info" in
|
||||
*"$site"*) ;;
|
||||
*) missing=1 ;;
|
||||
esac
|
||||
done
|
||||
if [ "$missing" -eq 0 ]; then
|
||||
echo "Site replication already spans all sites ($SITES) — nothing to do"
|
||||
mc admin replicate info "$first"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Configuring site replication across: $SITES"
|
||||
mc admin replicate add $SITES
|
||||
|
||||
echo "Replication status:"
|
||||
mc admin replicate info "$first"
|
||||
env:
|
||||
# Space-separated list of replication sites, named after the AZ
|
||||
# node labels. Each site must have a Helm release minio-<site>
|
||||
# (e.g. minio-az-a) so the Service DNS resolves. To add an AZ
|
||||
# later: deploy minio-az-<X>, append "az-<X>" here, then delete
|
||||
# and re-apply this Job.
|
||||
- name: SITES
|
||||
value: "az-a az-b az-c"
|
||||
# MinIO site replication requires identical root credentials on
|
||||
# every site, so reading one release's Secret covers all of them.
|
||||
# The MinIO chart creates a Secret named after the release with
|
||||
# keys rootUser and rootPassword.
|
||||
- name: ROOT_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: minio-az-a
|
||||
key: rootUser
|
||||
- name: ROOT_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: minio-az-a
|
||||
key: rootPassword
|
||||
@@ -0,0 +1,17 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: minio
|
||||
namespace: storage
|
||||
spec:
|
||||
type: ClusterIP
|
||||
# Helm chart selector — routes to minio-az-a/b/c pods
|
||||
selector:
|
||||
app: minio
|
||||
ports:
|
||||
- name: s3
|
||||
port: 9000
|
||||
targetPort: 9000
|
||||
- name: console
|
||||
port: 9001
|
||||
targetPort: 9001
|
||||
@@ -0,0 +1,141 @@
|
||||
apiVersion: minio.min.io/v2
|
||||
kind: Tenant
|
||||
metadata:
|
||||
name: minio-cluster
|
||||
namespace: storage
|
||||
labels:
|
||||
app: minio
|
||||
spec:
|
||||
image: minio/minio:RELEASE.2024-06-13T20-48-48Z
|
||||
|
||||
credsSecret:
|
||||
name: minio-creds
|
||||
|
||||
# 3-zone distributed cluster (one server per zone)
|
||||
pools:
|
||||
- name: az-a
|
||||
servers: 1
|
||||
volumesPerServer: 1
|
||||
size: 100Gi
|
||||
storageClass: longhorn
|
||||
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: topology.kubernetes.io/zone
|
||||
operator: In
|
||||
values: [az-a]
|
||||
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
|
||||
volumeClaimTemplate:
|
||||
metadata:
|
||||
name: data
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: longhorn
|
||||
resources:
|
||||
requests:
|
||||
storage: 100Gi
|
||||
|
||||
- name: az-b
|
||||
servers: 1
|
||||
volumesPerServer: 1
|
||||
size: 100Gi
|
||||
storageClass: longhorn
|
||||
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: topology.kubernetes.io/zone
|
||||
operator: In
|
||||
values: [az-b]
|
||||
|
||||
volumeClaimTemplate:
|
||||
metadata:
|
||||
name: data
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: longhorn
|
||||
resources:
|
||||
requests:
|
||||
storage: 100Gi
|
||||
|
||||
- name: az-c
|
||||
servers: 1
|
||||
volumesPerServer: 1
|
||||
size: 100Gi
|
||||
storageClass: longhorn
|
||||
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: topology.kubernetes.io/zone
|
||||
operator: In
|
||||
values: [az-c]
|
||||
|
||||
volumeClaimTemplate:
|
||||
metadata:
|
||||
name: data
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: longhorn
|
||||
resources:
|
||||
requests:
|
||||
storage: 100Gi
|
||||
|
||||
# Console (web UI)
|
||||
console:
|
||||
image: minio/console:v0.30.0
|
||||
replicas: 1
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
|
||||
# Environment variables — OIDC config
|
||||
env:
|
||||
- name: MINIO_IDENTITY_OPENID_CONFIG_URL
|
||||
value: "https://authentik.riotpiao.homelab.com/application/o/minio/.well-known/openid-configuration"
|
||||
- name: MINIO_IDENTITY_OPENID_CLIENT_ID
|
||||
value: "minio"
|
||||
- name: MINIO_IDENTITY_OPENID_CLIENT_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: minio-oidc
|
||||
key: MINIO_IDENTITY_OPENID_CLIENT_SECRET
|
||||
- name: MINIO_IDENTITY_OPENID_CLAIM_NAME
|
||||
value: "policy"
|
||||
- name: MINIO_IDENTITY_OPENID_SCOPES
|
||||
value: "openid,profile,email,minio"
|
||||
- name: MINIO_IDENTITY_OPENID_REDIRECT_URI
|
||||
value: "https://minio.riotpiao.homelab.com/oauth_callback"
|
||||
- name: MINIO_IDENTITY_OPENID_DISPLAY_NAME
|
||||
value: "Authentik"
|
||||
|
||||
# Metrics
|
||||
metrics:
|
||||
enabled: true
|
||||
port: 9000
|
||||
|
||||
# No auto-TLS (using cert-manager)
|
||||
requestAutoCert: false
|
||||
|
||||
# No built-in ingress
|
||||
ingress:
|
||||
enabled: false
|
||||
@@ -0,0 +1,64 @@
|
||||
mode: standalone
|
||||
|
||||
rootUser: ""
|
||||
rootPassword: ""
|
||||
|
||||
persistence:
|
||||
enabled: true
|
||||
storageClass: longhorn
|
||||
size: 100Gi
|
||||
|
||||
deploymentUpdate:
|
||||
type: Recreate
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 9000
|
||||
|
||||
consoleService:
|
||||
type: ClusterIP
|
||||
port: 9001
|
||||
|
||||
ingress:
|
||||
enabled: false
|
||||
|
||||
consoleIngress:
|
||||
enabled: false
|
||||
|
||||
metrics:
|
||||
serviceMonitor:
|
||||
enabled: true
|
||||
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: minio-oidc
|
||||
|
||||
environment:
|
||||
MINIO_IDENTITY_OPENID_CONFIG_URL: "https://authentik.riotpiao.homelab.com/application/o/minio/.well-known/openid-configuration"
|
||||
MINIO_IDENTITY_OPENID_CLIENT_ID: "minio"
|
||||
MINIO_IDENTITY_OPENID_CLAIM_NAME: "policy"
|
||||
MINIO_IDENTITY_OPENID_SCOPES: "openid,profile,email,minio"
|
||||
MINIO_IDENTITY_OPENID_REDIRECT_URI: "https://minio.riotpiao.homelab.com/oauth_callback"
|
||||
MINIO_IDENTITY_OPENID_DISPLAY_NAME: "Authentik"
|
||||
|
||||
podAnnotations:
|
||||
secret.reloader.stakater.com/reload: "minio-oidc"
|
||||
configmap.reloader.stakater.com/reload: "homelab-ca"
|
||||
|
||||
extraVolumes:
|
||||
- name: homelab-ca
|
||||
configMap:
|
||||
name: homelab-ca
|
||||
|
||||
extraVolumeMounts:
|
||||
- name: homelab-ca
|
||||
mountPath: /etc/minio/certs/CAs
|
||||
readOnly: true
|
||||
@@ -0,0 +1,137 @@
|
||||
# MinIO CRUD Example (Go)
|
||||
|
||||
A minimal Go program that exercises the cluster's object storage through the
|
||||
**universal storage frontend** — `minio.storage.svc.cluster.local:9000` — the
|
||||
single DNS name that load-balances across both node-pinned MinIO instances
|
||||
(`minio-az-a` on talos-cp-1, `minio-az-b` on talos-worker-1).
|
||||
|
||||
It runs a full CRUD cycle with a random text file:
|
||||
|
||||
| Step | S3 call | What it proves |
|
||||
|------|---------|----------------|
|
||||
| Ensure bucket | `BucketExists` / `MakeBucket` | bucket `crud-test` exists (idempotent) |
|
||||
| **C**reate | `PutObject` | write path through the frontend |
|
||||
| **R**ead | `GetObject` + byte compare | content round-trips intact |
|
||||
| **U**pdate | `PutObject` (overwrite) | S3 update semantics (objects are replaced, not edited) |
|
||||
| List | `ListObjects` prefix `demo/` | enumeration |
|
||||
| **D**elete | `RemoveObject` + `StatObject` | object gone (`NoSuchKey` confirmed) |
|
||||
|
||||
Every operation emits `[SERVICE_METRIC] s3.<op>.latency_ms=<n> ms`; any failure
|
||||
emits `[APP_METRIC] ERROR s3.<op> failed ... | trace=...` and exits non-zero.
|
||||
|
||||
## Run it
|
||||
|
||||
```bash
|
||||
# 1. expose the frontend locally (leave running in another terminal)
|
||||
kubectl port-forward svc/minio -n storage 9000:9000
|
||||
|
||||
# 2. credentials — same root creds used by both MinIO sites
|
||||
source logging/.env # exports MINIO_ROOT_USER / MINIO_ROOT_PASSWORD
|
||||
|
||||
# 3. run
|
||||
cd storage/test
|
||||
go mod tidy && go run .
|
||||
```
|
||||
|
||||
In-cluster (e.g. from a Job), skip the port-forward and set
|
||||
`MINIO_ENDPOINT=minio.storage.svc.cluster.local:9000`.
|
||||
|
||||
Expected output:
|
||||
|
||||
```
|
||||
[SERVICE_METRIC] s3.ensure_bucket.latency_ms=145 ms
|
||||
[SERVICE_METRIC] s3.put.latency_ms=19 ms
|
||||
created crud-test/demo/<unix-ts>.txt (256 bytes of random text)
|
||||
[SERVICE_METRIC] s3.get.latency_ms=9 ms
|
||||
read back and verified content
|
||||
[SERVICE_METRIC] s3.update.latency_ms=70 ms
|
||||
updated (overwrote) object
|
||||
demo/<unix-ts>.txt 140 bytes <timestamp>
|
||||
[SERVICE_METRIC] s3.list.latency_ms=11 ms
|
||||
[SERVICE_METRIC] s3.delete.latency_ms=68 ms
|
||||
deleted and verified gone — CRUD cycle complete
|
||||
```
|
||||
|
||||
## Validating each state with kubectl
|
||||
|
||||
The program verifies itself in-process (read-back compare, post-delete stat),
|
||||
but every state is also independently observable from outside with `kubectl`.
|
||||
The helper below drops you into a throwaway `mc` shell wired to both sites —
|
||||
all subsequent checks use it:
|
||||
|
||||
```bash
|
||||
source logging/.env
|
||||
kubectl run mc-shell --rm -it --restart=Never --image=minio/mc -n storage \
|
||||
--env="U=$MINIO_ROOT_USER" --env="P=$MINIO_ROOT_PASSWORD" \
|
||||
--command -- /bin/sh -c '
|
||||
mc alias set front http://minio.storage.svc.cluster.local:9000 "$U" "$P"
|
||||
mc alias set az-a http://minio-az-a.storage.svc.cluster.local:9000 "$U" "$P"
|
||||
mc alias set az-b http://minio-az-b.storage.svc.cluster.local:9000 "$U" "$P"
|
||||
exec /bin/sh'
|
||||
```
|
||||
|
||||
> The demo deletes its object at the end, so to inspect the CREATE/UPDATE
|
||||
> states at your own pace, comment out the `// DELETE` block in `main.go`
|
||||
> and re-run (the delete is idempotent to re-apply later).
|
||||
|
||||
**0. Frontend is healthy (before running anything)**
|
||||
|
||||
```bash
|
||||
kubectl get endpoints minio -n storage # expect TWO pod IPs on :9000
|
||||
kubectl get pods -n storage -o wide # az-a on talos-cp-1, az-b on talos-worker-1
|
||||
```
|
||||
|
||||
**1. Bucket created** — and replicated to BOTH sites
|
||||
|
||||
```bash
|
||||
# inside mc-shell — the bucket must appear on each site individually
|
||||
mc ls az-a | grep crud-test
|
||||
mc ls az-b | grep crud-test # proves site replication propagated it
|
||||
```
|
||||
|
||||
**2. Object created (CREATE)** — 256 bytes, present on both nodes
|
||||
|
||||
```bash
|
||||
mc ls az-a/crud-test/demo/ # <ts>.txt, 256 B
|
||||
mc ls az-b/crud-test/demo/ # same object, replicated (allow ~seconds of lag)
|
||||
mc cat front/crud-test/demo/<ts>.txt # the random text itself
|
||||
```
|
||||
|
||||
**3. Object updated (UPDATE)** — size changed 256 → 140 bytes, content starts with `UPDATED ---`
|
||||
|
||||
```bash
|
||||
mc stat az-a/crud-test/demo/<ts>.txt # Size: 140 B, fresh LastModified
|
||||
mc cat az-b/crud-test/demo/<ts>.txt | head -1 # "UPDATED ---" (replicated overwrite)
|
||||
```
|
||||
|
||||
**4. Object deleted (DELETE)** — gone from both sites
|
||||
|
||||
```bash
|
||||
mc ls az-a/crud-test/demo/ # empty
|
||||
mc ls az-b/crud-test/demo/ # empty — deletes replicate too
|
||||
mc stat front/crud-test/demo/<ts>.txt # error: Object does not exist
|
||||
```
|
||||
|
||||
**5. Replication layer itself**
|
||||
|
||||
```bash
|
||||
# inside mc-shell
|
||||
mc admin replicate status az-a # buckets/policies/users "in sync"
|
||||
```
|
||||
|
||||
**6. Storage layer under it**
|
||||
|
||||
```bash
|
||||
kubectl get volumes.longhorn.io -n longhorn-system # both volumes attached / healthy
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
- `connection refused` on localhost:9000 → the port-forward isn't running.
|
||||
- `[APP_METRIC] ERROR config missing` → `source logging/.env` first.
|
||||
- Object visible on az-a but not az-b → check `mc admin replicate status az-a`;
|
||||
replication is near-synchronous, not instant. Persistent divergence:
|
||||
`mc admin replicate resync start az-a az-b`.
|
||||
- Frontend has one endpoint instead of two → a MinIO pod is unready;
|
||||
`kubectl describe pod -n storage <pod>`. Traffic still flows via the
|
||||
surviving pod (that's the failover design — see `minio_migration.html`).
|
||||
@@ -0,0 +1,23 @@
|
||||
module homelab/storage/test
|
||||
|
||||
go 1.22
|
||||
|
||||
require github.com/minio/minio-go/v7 v7.0.80
|
||||
|
||||
require (
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
github.com/goccy/go-json v0.10.3 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/klauspost/compress v1.17.11 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
golang.org/x/crypto v0.28.0 // indirect
|
||||
golang.org/x/net v0.30.0 // indirect
|
||||
golang.org/x/sys v0.26.0 // indirect
|
||||
golang.org/x/text v0.19.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
@@ -0,0 +1,37 @@
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
|
||||
github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
|
||||
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
|
||||
github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.0.80 h1:2mdUHXEykRdY/BigLt3Iuu1otL0JTogT0Nmltg0wujk=
|
||||
github.com/minio/minio-go/v7 v7.0.80/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
|
||||
golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
|
||||
golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
|
||||
golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
|
||||
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
|
||||
golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -0,0 +1,152 @@
|
||||
// CRUD demo against the universal MinIO storage frontend
|
||||
// (minio.storage.svc.cluster.local:9000).
|
||||
//
|
||||
// Run from outside the cluster via a port-forward:
|
||||
//
|
||||
// kubectl port-forward svc/minio -n storage 9000:9000 &
|
||||
// source logging/.env
|
||||
// cd storage/test && go mod tidy && go run .
|
||||
//
|
||||
// In-cluster, set MINIO_ENDPOINT=minio.storage.svc.cluster.local:9000.
|
||||
//
|
||||
// Every S3 call emits [SERVICE_METRIC] op latency; every failure emits
|
||||
// [APP_METRIC] ERROR with context and aborts (no silent catches).
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
const bucket = "crud-test"
|
||||
|
||||
func getenv(key, fallback string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
// timed wraps an S3 operation: emits a serviceMetric on success,
|
||||
// an applicationMetric and exit(1) on failure.
|
||||
func timed(op string, fn func() error) {
|
||||
start := time.Now()
|
||||
if err := fn(); err != nil {
|
||||
fmt.Printf("[APP_METRIC] ERROR s3.%s failed bucket=%s | trace=%v\n", op, bucket, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Printf("[SERVICE_METRIC] s3.%s.latency_ms=%d ms\n", op, time.Since(start).Milliseconds())
|
||||
}
|
||||
|
||||
func randomText(n int) []byte {
|
||||
const letters = "abcdefghijklmnopqrstuvwxyz \n"
|
||||
b := make([]byte, n)
|
||||
for i := range b {
|
||||
b[i] = letters[rand.Intn(len(letters))]
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func main() {
|
||||
endpoint := getenv("MINIO_ENDPOINT", "localhost:9000")
|
||||
user := os.Getenv("MINIO_ROOT_USER")
|
||||
pass := os.Getenv("MINIO_ROOT_PASSWORD")
|
||||
if user == "" || pass == "" {
|
||||
fmt.Println("[APP_METRIC] ERROR config missing | trace=MINIO_ROOT_USER / MINIO_ROOT_PASSWORD not set (source storage/.env)")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
client, err := minio.New(endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(user, pass, ""),
|
||||
Secure: false, // in-cluster traffic, no TLS
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Printf("[APP_METRIC] ERROR s3.connect failed endpoint=%s | trace=%v\n", endpoint, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
key := fmt.Sprintf("demo/%d.txt", time.Now().Unix())
|
||||
original := randomText(256)
|
||||
updated := append([]byte("UPDATED ---\n"), randomText(128)...)
|
||||
|
||||
// Ensure bucket (idempotent). Site replication propagates it to az-b.
|
||||
timed("ensure_bucket", func() error {
|
||||
exists, err := client.BucketExists(ctx, bucket)
|
||||
if err != nil || exists {
|
||||
return err
|
||||
}
|
||||
return client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{})
|
||||
})
|
||||
|
||||
// CREATE
|
||||
timed("put", func() error {
|
||||
_, err := client.PutObject(ctx, bucket, key,
|
||||
bytes.NewReader(original), int64(len(original)),
|
||||
minio.PutObjectOptions{ContentType: "text/plain"})
|
||||
return err
|
||||
})
|
||||
fmt.Printf("created %s/%s (%d bytes of random text)\n", bucket, key, len(original))
|
||||
|
||||
// READ — and verify content round-trips
|
||||
timed("get", func() error {
|
||||
obj, err := client.GetObject(ctx, bucket, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer obj.Close()
|
||||
got, err := io.ReadAll(obj)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !bytes.Equal(got, original) {
|
||||
return fmt.Errorf("read-back mismatch: want %d bytes, got %d", len(original), len(got))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
fmt.Println("read back and verified content")
|
||||
|
||||
// UPDATE — S3 semantics: overwrite the object in place
|
||||
timed("update", func() error {
|
||||
_, err := client.PutObject(ctx, bucket, key,
|
||||
bytes.NewReader(updated), int64(len(updated)),
|
||||
minio.PutObjectOptions{ContentType: "text/plain"})
|
||||
return err
|
||||
})
|
||||
fmt.Println("updated (overwrote) object")
|
||||
|
||||
// LIST the demo/ prefix
|
||||
timed("list", func() error {
|
||||
for obj := range client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: "demo/", Recursive: true}) {
|
||||
if obj.Err != nil {
|
||||
return obj.Err
|
||||
}
|
||||
fmt.Printf(" %s %d bytes %s\n", obj.Key, obj.Size, obj.LastModified.Format(time.RFC3339))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
// DELETE — and verify it is gone
|
||||
timed("delete", func() error {
|
||||
if err := client.RemoveObject(ctx, bucket, key, minio.RemoveObjectOptions{}); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := client.StatObject(ctx, bucket, key, minio.StatObjectOptions{})
|
||||
if err == nil {
|
||||
return fmt.Errorf("object %s still exists after delete", key)
|
||||
}
|
||||
if minio.ToErrorResponse(err).Code != "NoSuchKey" {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
fmt.Println("deleted and verified gone — CRUD cycle complete")
|
||||
}
|
||||
Reference in New Issue
Block a user