rock/homelab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(k8s): Reorganize into 5-layer structure with production kustomizations

Browse Source
This commit is contained in:
Story Crater Bot
2026-07-16 14:28:19 -07:00
parent a81b9b6169
commit 2d7330798b
214 changed files with 405 additions and 885 deletions
Download Patch File Download Diff File Expand all files Collapse all files
k8s/infrastructure/kustomization.yaml
+14
View File
@@ -0,0 +1,14 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
metadata:
name: infrastructure
namespace: infrastructure
# Layer 1: Infrastructure — storage, persistent volumes
# Dependencies: None (base layer)
# Order: Applied first
resources:
- minio/
- longhorn/
k8s/infrastructure/longhorn/kustomization.yaml
+5
View File
@@ -0,0 +1,5 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: longhorn-system
resources: []
# Longhorn deployed via Helm chart (values in parent layer)
k8s/infrastructure/longhorn/longhorn-servicemonitor.yaml
+23
View File
@@ -0,0 +1,23 @@
# k8s/longhorn/longhorn-servicemonitor.yaml
# Longhorn is deployed via raw manifest (cluster-config/longhorn_bootstrap.sh), not
# Helm, so there's no chart values toggle for metrics — the manager already exposes
# them on the existing longhorn-backend service (port 9500); this just wires Prometheus
# to it. longhorn_volume_robustness is the availability signal (degraded/faulted).
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: longhorn-manager
namespace: longhorn-system
labels:
app: longhorn-manager
spec:
selector:
matchLabels:
app: longhorn-manager
namespaceSelector:
matchNames:
- longhorn-system
endpoints:
- port: manager
interval: 30s
scrapeTimeout: 10s
k8s/infrastructure/minio/.env.example
+9
View File
@@ -0,0 +1,9 @@
# storage/.env.example
# Copy to storage/.env and fill in real values.
# The real .env is gitignored — never commit it.
# Used by: k8s/storage/bootstrap.sh
# MinIO root credentials (S3-compatible object store)
# openssl rand -base64 24
MINIO_ROOT_USER=
MINIO_ROOT_PASSWORD=
k8s/infrastructure/minio/bootstrap.sh
Executable
+109
View File
@@ -0,0 +1,109 @@
#!/usr/bin/env bash
# k8s/storage/bootstrap.sh
# Deploys the MinIO multi-AZ object store into the storage namespace.
#
# Prerequisites:
# - kubectl configured (KUBECONFIG pointing to cluster-config/kubeconfig)
# - helm >= 3.x installed
# - k8s/storage/.env file containing:
# MINIO_ROOT_USER=...
# MINIO_ROOT_PASSWORD=...
# OR those variables already exported in the calling shell.
#
# Idempotent: safe to re-run — helm upgrade --install and kubectl apply are both idempotent.
# Disaster recovery order: run this before k8s/logging/bootstrap.sh (Loki depends on MinIO).
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
KUBECONFIG="${KUBECONFIG:-${SCRIPT_DIR}/../../cluster-config/kubeconfig}"
export KUBECONFIG
# ── Load credentials ──────────────────────────────────────────────────────────
if [[ -f "${SCRIPT_DIR}/.env" ]]; then
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/.env"
fi
for var in MINIO_ROOT_USER MINIO_ROOT_PASSWORD; do
if [[ -z "${!var:-}" ]]; then
echo "ERROR: ${var} is not set. Export it or place it in k8s/storage/.env"
exit 1
fi
done
# ── Namespace ─────────────────────────────────────────────────────────────────
echo "==> Creating storage namespace with privileged pod security..."
kubectl create namespace storage --dry-run=client -o yaml | kubectl apply -f -
kubectl label namespace storage \
pod-security.kubernetes.io/enforce=privileged \
pod-security.kubernetes.io/enforce-version=latest \
--overwrite
# ── Helm repos ────────────────────────────────────────────────────────────────
echo "==> Adding Helm repositories..."
helm repo add minio https://charts.min.io
helm repo update minio
# ── PVC for az-a (adopts existing volume) ────────────────────────────────────
echo "==> Applying az-a PVC (adopts pre-existing Longhorn volume)..."
kubectl apply -f "${SCRIPT_DIR}/minio-az-a-pvc.yaml"
# ── MinIO az-a (control-plane node, zone az-a) ───────────────────────────────
echo "==> Installing MinIO az-a..."
helm upgrade --install minio-az-a minio/minio \
--namespace storage \
--values "${SCRIPT_DIR}/minio-az-a-values.yaml" \
--set rootUser="${MINIO_ROOT_USER}" \
--set rootPassword="${MINIO_ROOT_PASSWORD}" \
--wait \
--timeout 5m
echo "==> Waiting for minio-az-a Deployment to be ready..."
kubectl rollout status deployment/minio-az-a -n storage --timeout=120s
# ── MinIO az-b (worker node, zone az-b) ──────────────────────────────────────
echo "==> Installing MinIO az-b..."
helm upgrade --install minio-az-b minio/minio \
--namespace storage \
--values "${SCRIPT_DIR}/minio-az-b-values.yaml" \
--set rootUser="${MINIO_ROOT_USER}" \
--set rootPassword="${MINIO_ROOT_PASSWORD}" \
--wait \
--timeout 5m
echo "==> Waiting for minio-az-b Deployment to be ready..."
kubectl rollout status deployment/minio-az-b -n storage --timeout=120s
# ── Universal frontend service + legacy alias ─────────────────────────────────
echo "==> Applying universal frontend service and legacy ExternalName alias..."
kubectl apply -f "${SCRIPT_DIR}/minio-service.yaml"
kubectl apply -f "${SCRIPT_DIR}/minio-legacy-alias.yaml"
# ── Site replication ──────────────────────────────────────────────────────────
echo "==> Running site replication setup job..."
# Delete any leftover completed job first (kubectl apply on Jobs is not idempotent)
kubectl delete job minio-site-replication-setup -n storage --ignore-not-found
kubectl apply -f "${SCRIPT_DIR}/minio-replication-job.yaml"
echo "==> Waiting for replication job to complete..."
kubectl wait --for=condition=complete job/minio-site-replication-setup \
-n storage --timeout=120s
echo "==> Replication job output:"
kubectl logs -n storage \
-l job-name=minio-site-replication-setup \
--tail=20
# ── Done ──────────────────────────────────────────────────────────────────────
echo ""
echo "==> MinIO multi-AZ stack is up."
echo ""
echo "MinIO console (S3 browser):"
echo " kubectl port-forward -n storage svc/minio 9001:9001"
echo " http://localhost:9001 (${MINIO_ROOT_USER} / <your MINIO_ROOT_PASSWORD>)"
echo ""
echo "MinIO S3 endpoint for apps (active-active frontend):"
echo " http://minio.storage.svc.cluster.local:9000"
echo ""
echo "Verify site replication:"
echo " kubectl logs -n storage -l job-name=minio-site-replication-setup"
k8s/infrastructure/minio/kustomization.yaml
+8
View File
@@ -0,0 +1,8 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: minio
resources:
- minio-tenant.yaml
- minio-service.yaml
- minio-replication-job.yaml
# Legacy: minio-legacy-alias.yaml skipped (migration artifact, conflicts with namespace transform)
k8s/infrastructure/minio/minio-bucket-init.sh
Executable
+57
View File
@@ -0,0 +1,57 @@
#!/bin/bash
# Safe MinIO bucket initialization via Job (credentials in Secret, not env)
# Usage: ./minio-bucket-init.sh <namespace> <bucket1> [bucket2] ...
set -euo pipefail
NAMESPACE="${1:?Missing namespace}"
shift
BUCKETS=("$@")
if [ ${#BUCKETS[@]} -eq 0 ]; then
echo "Usage: $0 <namespace> <bucket1> [bucket2] ..." >&2
exit 1
fi
# Create Secret with Minio credentials (safe: sourced from env, not exposed in pod)
kubectl create secret generic minio-creds -n "$NAMESPACE" \
--from-literal=MINIO_ROOT_USER="${MINIO_ROOT_USER:?Missing MINIO_ROOT_USER}" \
--from-literal=MINIO_ROOT_PASSWORD="${MINIO_ROOT_PASSWORD:?Missing MINIO_ROOT_PASSWORD}" \
--dry-run=client -o yaml | kubectl apply -f -
# Create init Job that mounts Secret as volume, preventing env exposure
BUCKET_ARGS=$(printf '"%s", ' "${BUCKETS[@]}" | sed 's/, $//')
kubectl apply -f - <<EOF
apiVersion: batch/v1
kind: Job
metadata:
name: minio-bucket-init-$RANDOM
namespace: $NAMESPACE
spec:
ttlSecondsAfterFinished: 300
backoffLimit: 2
template:
spec:
restartPolicy: Never
containers:
- name: minio-init
image: minio/mc:latest
volumeMounts:
- name: minio-secret
mountPath: /var/run/secrets/minio
readOnly: true
command:
- sh
- -c
- |
MINIO_ROOT_USER=\$(cat /var/run/secrets/minio/MINIO_ROOT_USER)
MINIO_ROOT_PASSWORD=\$(cat /var/run/secrets/minio/MINIO_ROOT_PASSWORD)
mc alias set local http://minio.storage.svc.cluster.local:9000 "\$MINIO_ROOT_USER" "\$MINIO_ROOT_PASSWORD"
mc mb --ignore-existing $BUCKET_ARGS
volumes:
- name: minio-secret
secret:
secretName: minio-creds
EOF
echo "✓ MinIO bucket init job submitted for: ${BUCKETS[*]}"
k8s/infrastructure/minio/minio-legacy-alias.yaml
+14
View File
@@ -0,0 +1,14 @@
# Backward-compatibility alias: anything still pointing at the old
# minio.logging.svc.cluster.local address resolves to the universal
# storage frontend. Apply AFTER `helm uninstall minio -n logging`
# (the old release owns the Service name `minio` until then).
apiVersion: v1
kind: Service
metadata:
name: minio
namespace: logging
labels:
app: minio
spec:
type: ExternalName
externalName: minio.storage.svc.cluster.local
k8s/infrastructure/minio/minio-operator-values.yaml
+48
View File
@@ -0,0 +1,48 @@
# minio-operator-values.yaml
# MinIO Operator deployment with metrics enabled
operator:
image:
repository: minio/operator
tag: "v5.0.0"
pullPolicy: IfNotPresent
replicaCount: 1
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: 500m
memory: 512Mi
metrics:
enabled: true
port: 8080
rbac:
create: true
affinity:
nodeAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
matchExpressions:
- key: node-role.kubernetes.io/control-plane
operator: Exists
console:
enabled: true
replicaCount: 1
image:
repository: minio/console
tag: "v0.30.0"
resources:
requests:
cpu: 50m
memory: 128Mi
limits:
cpu: 200m
memory: 256Mi
k8s/infrastructure/minio/minio-replication-job.yaml
+75
View File
@@ -0,0 +1,75 @@
apiVersion: batch/v1
kind: Job
metadata:
name: minio-site-replication-setup
namespace: storage
spec:
# Auto-delete the Job pod 10 minutes after completion
ttlSecondsAfterFinished: 600
template:
spec:
restartPolicy: OnFailure
containers:
- name: mc
image: minio/mc:latest
command:
- /bin/sh
- -c
- |
set -e
# Alias every site. Service names follow the release naming
# convention minio-<site>.storage.svc.cluster.local.
for site in $SITES; do
mc alias set "$site" "http://minio-${site}.storage.svc.cluster.local:9000" \
"$ROOT_USER" "$ROOT_PASSWORD"
done
first=$(echo $SITES | cut -d' ' -f1)
# Idempotent: skip only if every requested site is already part
# of the replication group. A partially-configured group (e.g.
# az-c newly added to SITES) falls through to `replicate add`,
# which expands an existing group in place.
# NOTE: pure-shell matching — the minio/mc image has no grep.
info=$(mc admin replicate info "$first" 2>/dev/null || true)
missing=0
for site in $SITES; do
case "$info" in
*"$site"*) ;;
*) missing=1 ;;
esac
done
if [ "$missing" -eq 0 ]; then
echo "Site replication already spans all sites ($SITES) — nothing to do"
mc admin replicate info "$first"
exit 0
fi
echo "Configuring site replication across: $SITES"
mc admin replicate add $SITES
echo "Replication status:"
mc admin replicate info "$first"
env:
# Space-separated list of replication sites, named after the AZ
# node labels. Each site must have a Helm release minio-<site>
# (e.g. minio-az-a) so the Service DNS resolves. To add an AZ
# later: deploy minio-az-<X>, append "az-<X>" here, then delete
# and re-apply this Job.
- name: SITES
value: "az-a az-b az-c"
# MinIO site replication requires identical root credentials on
# every site, so reading one release's Secret covers all of them.
# The MinIO chart creates a Secret named after the release with
# keys rootUser and rootPassword.
- name: ROOT_USER
valueFrom:
secretKeyRef:
name: minio-az-a
key: rootUser
- name: ROOT_PASSWORD
valueFrom:
secretKeyRef:
name: minio-az-a
key: rootPassword
k8s/infrastructure/minio/minio-service.yaml
+17
View File
@@ -0,0 +1,17 @@
apiVersion: v1
kind: Service
metadata:
name: minio
namespace: storage
spec:
type: ClusterIP
# Helm chart selector — routes to minio-az-a/b/c pods
selector:
app: minio
ports:
- name: s3
port: 9000
targetPort: 9000
- name: console
port: 9001
targetPort: 9001
k8s/infrastructure/minio/minio-tenant.yaml
+141
View File
@@ -0,0 +1,141 @@
apiVersion: minio.min.io/v2
kind: Tenant
metadata:
name: minio-cluster
namespace: storage
labels:
app: minio
spec:
image: minio/minio:RELEASE.2024-06-13T20-48-48Z
credsSecret:
name: minio-creds
# 3-zone distributed cluster (one server per zone)
pools:
- name: az-a
servers: 1
volumesPerServer: 1
size: 100Gi
storageClass: longhorn
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: topology.kubernetes.io/zone
operator: In
values: [az-a]
tolerations:
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
volumeClaimTemplate:
metadata:
name: data
spec:
accessModes:
- ReadWriteOnce
storageClassName: longhorn
resources:
requests:
storage: 100Gi
- name: az-b
servers: 1
volumesPerServer: 1
size: 100Gi
storageClass: longhorn
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: topology.kubernetes.io/zone
operator: In
values: [az-b]
volumeClaimTemplate:
metadata:
name: data
spec:
accessModes:
- ReadWriteOnce
storageClassName: longhorn
resources:
requests:
storage: 100Gi
- name: az-c
servers: 1
volumesPerServer: 1
size: 100Gi
storageClass: longhorn
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: topology.kubernetes.io/zone
operator: In
values: [az-c]
volumeClaimTemplate:
metadata:
name: data
spec:
accessModes:
- ReadWriteOnce
storageClassName: longhorn
resources:
requests:
storage: 100Gi
# Console (web UI)
console:
image: minio/console:v0.30.0
replicas: 1
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: 500m
memory: 512Mi
# Environment variables — OIDC config
env:
- name: MINIO_IDENTITY_OPENID_CONFIG_URL
value: "https://authentik.riotpiao.homelab.com/application/o/minio/.well-known/openid-configuration"
- name: MINIO_IDENTITY_OPENID_CLIENT_ID
value: "minio"
- name: MINIO_IDENTITY_OPENID_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: minio-oidc
key: MINIO_IDENTITY_OPENID_CLIENT_SECRET
- name: MINIO_IDENTITY_OPENID_CLAIM_NAME
value: "policy"
- name: MINIO_IDENTITY_OPENID_SCOPES
value: "openid,profile,email,minio"
- name: MINIO_IDENTITY_OPENID_REDIRECT_URI
value: "https://minio.riotpiao.homelab.com/oauth_callback"
- name: MINIO_IDENTITY_OPENID_DISPLAY_NAME
value: "Authentik"
# Metrics
metrics:
enabled: true
port: 9000
# No auto-TLS (using cert-manager)
requestAutoCert: false
# No built-in ingress
ingress:
enabled: false
k8s/infrastructure/minio/minio-values.yaml
+64
View File
@@ -0,0 +1,64 @@
mode: standalone
rootUser: ""
rootPassword: ""
persistence:
enabled: true
storageClass: longhorn
size: 100Gi
deploymentUpdate:
type: Recreate
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: 500m
memory: 512Mi
service:
type: ClusterIP
port: 9000
consoleService:
type: ClusterIP
port: 9001
ingress:
enabled: false
consoleIngress:
enabled: false
metrics:
serviceMonitor:
enabled: true
envFrom:
- secretRef:
name: minio-oidc
environment:
MINIO_IDENTITY_OPENID_CONFIG_URL: "https://authentik.riotpiao.homelab.com/application/o/minio/.well-known/openid-configuration"
MINIO_IDENTITY_OPENID_CLIENT_ID: "minio"
MINIO_IDENTITY_OPENID_CLAIM_NAME: "policy"
MINIO_IDENTITY_OPENID_SCOPES: "openid,profile,email,minio"
MINIO_IDENTITY_OPENID_REDIRECT_URI: "https://minio.riotpiao.homelab.com/oauth_callback"
MINIO_IDENTITY_OPENID_DISPLAY_NAME: "Authentik"
podAnnotations:
secret.reloader.stakater.com/reload: "minio-oidc"
configmap.reloader.stakater.com/reload: "homelab-ca"
extraVolumes:
- name: homelab-ca
configMap:
name: homelab-ca
extraVolumeMounts:
- name: homelab-ca
mountPath: /etc/minio/certs/CAs
readOnly: true
k8s/infrastructure/minio/test/README.md
+137
View File
@@ -0,0 +1,137 @@
# MinIO CRUD Example (Go)
A minimal Go program that exercises the cluster's object storage through the
**universal storage frontend** — `minio.storage.svc.cluster.local:9000` — the
single DNS name that load-balances across both node-pinned MinIO instances
(`minio-az-a` on talos-cp-1, `minio-az-b` on talos-worker-1).
It runs a full CRUD cycle with a random text file:
| Step | S3 call | What it proves |
|------|---------|----------------|
| Ensure bucket | `BucketExists` / `MakeBucket` | bucket `crud-test` exists (idempotent) |
| **C**reate | `PutObject` | write path through the frontend |
| **R**ead | `GetObject` + byte compare | content round-trips intact |
| **U**pdate | `PutObject` (overwrite) | S3 update semantics (objects are replaced, not edited) |
| List | `ListObjects` prefix `demo/` | enumeration |
| **D**elete | `RemoveObject` + `StatObject` | object gone (`NoSuchKey` confirmed) |
Every operation emits `[SERVICE_METRIC] s3.<op>.latency_ms=<n> ms`; any failure
emits `[APP_METRIC] ERROR s3.<op> failed ... | trace=...` and exits non-zero.
## Run it
```bash
# 1. expose the frontend locally (leave running in another terminal)
kubectl port-forward svc/minio -n storage 9000:9000
# 2. credentials — same root creds used by both MinIO sites
source logging/.env # exports MINIO_ROOT_USER / MINIO_ROOT_PASSWORD
# 3. run
cd storage/test
go mod tidy && go run .
```
In-cluster (e.g. from a Job), skip the port-forward and set
`MINIO_ENDPOINT=minio.storage.svc.cluster.local:9000`.
Expected output:
```
[SERVICE_METRIC] s3.ensure_bucket.latency_ms=145 ms
[SERVICE_METRIC] s3.put.latency_ms=19 ms
created crud-test/demo/<unix-ts>.txt (256 bytes of random text)
[SERVICE_METRIC] s3.get.latency_ms=9 ms
read back and verified content
[SERVICE_METRIC] s3.update.latency_ms=70 ms
updated (overwrote) object
demo/<unix-ts>.txt 140 bytes <timestamp>
[SERVICE_METRIC] s3.list.latency_ms=11 ms
[SERVICE_METRIC] s3.delete.latency_ms=68 ms
deleted and verified gone — CRUD cycle complete
```
## Validating each state with kubectl
The program verifies itself in-process (read-back compare, post-delete stat),
but every state is also independently observable from outside with `kubectl`.
The helper below drops you into a throwaway `mc` shell wired to both sites —
all subsequent checks use it:
```bash
source logging/.env
kubectl run mc-shell --rm -it --restart=Never --image=minio/mc -n storage \
--env="U=$MINIO_ROOT_USER" --env="P=$MINIO_ROOT_PASSWORD" \
--command -- /bin/sh -c '
mc alias set front http://minio.storage.svc.cluster.local:9000 "$U" "$P"
mc alias set az-a http://minio-az-a.storage.svc.cluster.local:9000 "$U" "$P"
mc alias set az-b http://minio-az-b.storage.svc.cluster.local:9000 "$U" "$P"
exec /bin/sh'
```
> The demo deletes its object at the end, so to inspect the CREATE/UPDATE
> states at your own pace, comment out the `// DELETE` block in `main.go`
> and re-run (the delete is idempotent to re-apply later).
**0. Frontend is healthy (before running anything)**
```bash
kubectl get endpoints minio -n storage # expect TWO pod IPs on :9000
kubectl get pods -n storage -o wide # az-a on talos-cp-1, az-b on talos-worker-1
```
**1. Bucket created** — and replicated to BOTH sites
```bash
# inside mc-shell — the bucket must appear on each site individually
mc ls az-a | grep crud-test
mc ls az-b | grep crud-test # proves site replication propagated it
```
**2. Object created (CREATE)** — 256 bytes, present on both nodes
```bash
mc ls az-a/crud-test/demo/ # <ts>.txt, 256 B
mc ls az-b/crud-test/demo/ # same object, replicated (allow ~seconds of lag)
mc cat front/crud-test/demo/<ts>.txt # the random text itself
```
**3. Object updated (UPDATE)** — size changed 256 → 140 bytes, content starts with `UPDATED ---`
```bash
mc stat az-a/crud-test/demo/<ts>.txt # Size: 140 B, fresh LastModified
mc cat az-b/crud-test/demo/<ts>.txt | head -1 # "UPDATED ---" (replicated overwrite)
```
**4. Object deleted (DELETE)** — gone from both sites
```bash
mc ls az-a/crud-test/demo/ # empty
mc ls az-b/crud-test/demo/ # empty — deletes replicate too
mc stat front/crud-test/demo/<ts>.txt # error: Object does not exist
```
**5. Replication layer itself**
```bash
# inside mc-shell
mc admin replicate status az-a # buckets/policies/users "in sync"
```
**6. Storage layer under it**
```bash
kubectl get volumes.longhorn.io -n longhorn-system # both volumes attached / healthy
```
## Troubleshooting
- `connection refused` on localhost:9000 → the port-forward isn't running.
- `[APP_METRIC] ERROR config missing` → `source logging/.env` first.
- Object visible on az-a but not az-b → check `mc admin replicate status az-a`;
replication is near-synchronous, not instant. Persistent divergence:
`mc admin replicate resync start az-a az-b`.
- Frontend has one endpoint instead of two → a MinIO pod is unready;
`kubectl describe pod -n storage <pod>`. Traffic still flows via the
surviving pod (that's the failover design — see `minio_migration.html`).
k8s/infrastructure/minio/test/go.mod
+23
View File
@@ -0,0 +1,23 @@
module homelab/storage/test
go 1.22
require github.com/minio/minio-go/v7 v7.0.80
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/goccy/go-json v0.10.3 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/klauspost/compress v1.17.11 // indirect
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rs/xid v1.6.0 // indirect
golang.org/x/crypto v0.28.0 // indirect
golang.org/x/net v0.30.0 // indirect
golang.org/x/sys v0.26.0 // indirect
golang.org/x/text v0.19.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
k8s/infrastructure/minio/test/go.sum
+37
View File
@@ -0,0 +1,37 @@
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.80 h1:2mdUHXEykRdY/BigLt3Iuu1otL0JTogT0Nmltg0wujk=
github.com/minio/minio-go/v7 v7.0.80/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
k8s/infrastructure/minio/test/main.go
+152
View File
@@ -0,0 +1,152 @@
// CRUD demo against the universal MinIO storage frontend
// (minio.storage.svc.cluster.local:9000).
//
// Run from outside the cluster via a port-forward:
//
// kubectl port-forward svc/minio -n storage 9000:9000 &
// source logging/.env
// cd storage/test && go mod tidy && go run .
//
// In-cluster, set MINIO_ENDPOINT=minio.storage.svc.cluster.local:9000.
//
// Every S3 call emits [SERVICE_METRIC] op latency; every failure emits
// [APP_METRIC] ERROR with context and aborts (no silent catches).
package main
import (
"bytes"
"context"
"fmt"
"io"
"math/rand"
"os"
"time"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
const bucket = "crud-test"
func getenv(key, fallback string) string {
if v := os.Getenv(key); v != "" {
return v
}
return fallback
}
// timed wraps an S3 operation: emits a serviceMetric on success,
// an applicationMetric and exit(1) on failure.
func timed(op string, fn func() error) {
start := time.Now()
if err := fn(); err != nil {
fmt.Printf("[APP_METRIC] ERROR s3.%s failed bucket=%s | trace=%v\n", op, bucket, err)
os.Exit(1)
}
fmt.Printf("[SERVICE_METRIC] s3.%s.latency_ms=%d ms\n", op, time.Since(start).Milliseconds())
}
func randomText(n int) []byte {
const letters = "abcdefghijklmnopqrstuvwxyz \n"
b := make([]byte, n)
for i := range b {
b[i] = letters[rand.Intn(len(letters))]
}
return b
}
func main() {
endpoint := getenv("MINIO_ENDPOINT", "localhost:9000")
user := os.Getenv("MINIO_ROOT_USER")
pass := os.Getenv("MINIO_ROOT_PASSWORD")
if user == "" || pass == "" {
fmt.Println("[APP_METRIC] ERROR config missing | trace=MINIO_ROOT_USER / MINIO_ROOT_PASSWORD not set (source storage/.env)")
os.Exit(1)
}
ctx := context.Background()
client, err := minio.New(endpoint, &minio.Options{
Creds: credentials.NewStaticV4(user, pass, ""),
Secure: false, // in-cluster traffic, no TLS
})
if err != nil {
fmt.Printf("[APP_METRIC] ERROR s3.connect failed endpoint=%s | trace=%v\n", endpoint, err)
os.Exit(1)
}
key := fmt.Sprintf("demo/%d.txt", time.Now().Unix())
original := randomText(256)
updated := append([]byte("UPDATED ---\n"), randomText(128)...)
// Ensure bucket (idempotent). Site replication propagates it to az-b.
timed("ensure_bucket", func() error {
exists, err := client.BucketExists(ctx, bucket)
if err != nil || exists {
return err
}
return client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{})
})
// CREATE
timed("put", func() error {
_, err := client.PutObject(ctx, bucket, key,
bytes.NewReader(original), int64(len(original)),
minio.PutObjectOptions{ContentType: "text/plain"})
return err
})
fmt.Printf("created %s/%s (%d bytes of random text)\n", bucket, key, len(original))
// READ — and verify content round-trips
timed("get", func() error {
obj, err := client.GetObject(ctx, bucket, key, minio.GetObjectOptions{})
if err != nil {
return err
}
defer obj.Close()
got, err := io.ReadAll(obj)
if err != nil {
return err
}
if !bytes.Equal(got, original) {
return fmt.Errorf("read-back mismatch: want %d bytes, got %d", len(original), len(got))
}
return nil
})
fmt.Println("read back and verified content")
// UPDATE — S3 semantics: overwrite the object in place
timed("update", func() error {
_, err := client.PutObject(ctx, bucket, key,
bytes.NewReader(updated), int64(len(updated)),
minio.PutObjectOptions{ContentType: "text/plain"})
return err
})
fmt.Println("updated (overwrote) object")
// LIST the demo/ prefix
timed("list", func() error {
for obj := range client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: "demo/", Recursive: true}) {
if obj.Err != nil {
return obj.Err
}
fmt.Printf(" %s %d bytes %s\n", obj.Key, obj.Size, obj.LastModified.Format(time.RFC3339))
}
return nil
})
// DELETE — and verify it is gone
timed("delete", func() error {
if err := client.RemoveObject(ctx, bucket, key, minio.RemoveObjectOptions{}); err != nil {
return err
}
_, err := client.StatObject(ctx, bucket, key, minio.StatObjectOptions{})
if err == nil {
return fmt.Errorf("object %s still exists after delete", key)
}
if minio.ToErrorResponse(err).Code != "NoSuchKey" {
return err
}
return nil
})
fmt.Println("deleted and verified gone — CRUD cycle complete")
}
k8s/infrastructure/minio/test/test
Executable
BIN
View File
Binary file not shown.