fix(minio): freeze the community image by digest and alert on pull failure

MinIO deleted the community edition images upstream. Docker Hub reports
{"message":"object not found"} for minio/minio, and an anonymous pull of
quay.io/minio/mc returns HTTP 401. talos-cp-1 dropped the cached image during
the DiskPressure event on 2026-09-22. The tenant pod was recreated on
2026-09-23 and has been in ImagePullBackOff since. MinIO served no traffic for
13 days, which put vault-0 and loki-0 into CrashLoopBackOff and left the
paperless backups stuck.

Pin the tenant image to the digest of the last community release, so a retag or
a deletion on the mirror cannot break the pull again. Repoint the two
quay.io/minio/mc references at the archived Bitnami client image, also by
digest. Alert on ImagePullBackOff and on a pod that stays not ready, because
neither case raised any alert.

Evidence: https://hub.docker.com/v2/repositories/minio/minio/tags -> {"message":"object not found"}
This commit is contained in:
2026-10-06 11:42:51 +08:00
parent b6e317b34c
commit d7b96fb29b
4 changed files with 25 additions and 4 deletions
+1 -1
View File
@@ -48,7 +48,7 @@ spec:
mountPath: /backup mountPath: /backup
containers: containers:
- name: mc-mirror - name: mc-mirror
image: quay.io/minio/mc:latest image: docker.io/bitnamilegacy/minio-client:2025.7.21-debian-12-r3@sha256:73bd39f7899a0cef12b8dd5df13aa93a3ed1aaa44236542442e9ac76819ac158
env: env:
- name: ACCESS_KEY - name: ACCESS_KEY
valueFrom: valueFrom:
@@ -52,14 +52,14 @@ spec:
restartPolicy: Never restartPolicy: Never
initContainers: initContainers:
- name: kubectl-copy - name: kubectl-copy
image: bitnami/kubectl:latest image: bitnami/kubectl:latest@sha256:f7f9e4f64d9e114650c115a4ac6fd383394b3d494061a8f17b1a0f6c8d55bc25
command: ["sh", "-c", "cp $(which kubectl) /shared/kubectl"] command: ["sh", "-c", "cp $(which kubectl) /shared/kubectl"]
volumeMounts: volumeMounts:
- name: shared - name: shared
mountPath: /shared mountPath: /shared
containers: containers:
- name: provision - name: provision
image: quay.io/minio/mc:latest image: docker.io/bitnamilegacy/minio-client:2025.7.21-debian-12-r3@sha256:73bd39f7899a0cef12b8dd5df13aa93a3ed1aaa44236542442e9ac76819ac158
volumeMounts: volumeMounts:
- name: shared - name: shared
mountPath: /shared mountPath: /shared
+4 -1
View File
@@ -16,7 +16,10 @@ spec:
# backend could not resolve minio-cluster-hl. Official images now need registry auth # backend could not resolve minio-cluster-hl. Official images now need registry auth
# (quay.io/minio/minio returns 401), so this pins a public mirror of the same upstream # (quay.io/minio/minio returns 401), so this pins a public mirror of the same upstream
# release; swap it for the official image once MinIO registry credentials exist. # release; swap it for the official image once MinIO registry credentials exist.
image: coollabsio/minio:RELEASE.2025-10-15T17-29-55Z # The digest freezes those bytes, so a retag or a deletion on the mirror cannot break
# the pull again. The same digest already sits in the containerd cache on talos-cp-1,
# so the first start needs no registry access.
image: docker.io/coollabsio/minio:RELEASE.2025-10-15T17-29-55Z@sha256:69b55a1c1c5dc285ce04db96689f5b2102317fc77a50680a1874ca6efd1c87f9
# Disable operator auto-TLS: MinIO serves plain HTTP internally on 9000. # Disable operator auto-TLS: MinIO serves plain HTTP internally on 9000.
# External TLS is terminated at nginx ingress (wildcard riotpiao-com-tls cert). # External TLS is terminated at nginx ingress (wildcard riotpiao-com-tls cert).
@@ -27,6 +27,24 @@ spec:
annotations: annotations:
summary: "{{ $value }} pod(s) stuck in Pending state for >10m" summary: "{{ $value }} pod(s) stuck in Pending state for >10m"
# ImagePullBackOff on a vendor-deleted image produced no alert for 13 days
- alert: PodImagePullFailure
expr: sum(kube_pod_container_status_waiting_reason{reason=~"ImagePullBackOff|ErrImagePull"}) by (namespace, pod) > 0
for: 10m
labels:
severity: critical
annotations:
summary: "{{ $labels.namespace }}/{{ $labels.pod }} cannot pull its image"
# Pod not ready
- alert: PodNotReady
expr: sum(kube_pod_status_ready{condition="false"}) by (namespace, pod) > 0
for: 15m
labels:
severity: warning
annotations:
summary: "{{ $labels.namespace }}/{{ $labels.pod }} is not ready"
# CrashLoopBackOff # CrashLoopBackOff
- alert: PodCrashLooping - alert: PodCrashLooping
expr: sum(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}) by (namespace, pod) > 0 expr: sum(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}) by (namespace, pod) > 0