From d7b96fb29b521b42d7a33f95943b22d498a6e9e6 Mon Sep 17 00:00:00 2001 From: rock Date: Tue, 6 Oct 2026 11:38:40 +0800 Subject: [PATCH] fix(minio): freeze the community image by digest and alert on pull failure MinIO deleted the community edition images upstream. Docker Hub reports {"message":"object not found"} for minio/minio, and an anonymous pull of quay.io/minio/mc returns HTTP 401. talos-cp-1 dropped the cached image during the DiskPressure event on 2026-09-22. The tenant pod was recreated on 2026-09-23 and has been in ImagePullBackOff since. MinIO served no traffic for 13 days, which put vault-0 and loki-0 into CrashLoopBackOff and left the paperless backups stuck. Pin the tenant image to the digest of the last community release, so a retag or a deletion on the mirror cannot break the pull again. Repoint the two quay.io/minio/mc references at the archived Bitnami client image, also by digest. Alert on ImagePullBackOff and on a pod that stays not ready, because neither case raised any alert. Evidence: https://hub.docker.com/v2/repositories/minio/minio/tags -> {"message":"object not found"} --- k8s/apps/paperless/backup-cronjob.yaml | 2 +- .../minio/minio-provision-paperless-job.yaml | 4 ++-- k8s/infra/minio/minio-tenant.yaml | 5 ++++- .../monitoring/alerts/cluster-alerts.yaml | 18 ++++++++++++++++++ 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/k8s/apps/paperless/backup-cronjob.yaml b/k8s/apps/paperless/backup-cronjob.yaml index 46dcbd6..be56123 100644 --- a/k8s/apps/paperless/backup-cronjob.yaml +++ b/k8s/apps/paperless/backup-cronjob.yaml @@ -48,7 +48,7 @@ spec: mountPath: /backup containers: - name: mc-mirror - image: quay.io/minio/mc:latest + image: docker.io/bitnamilegacy/minio-client:2025.7.21-debian-12-r3@sha256:73bd39f7899a0cef12b8dd5df13aa93a3ed1aaa44236542442e9ac76819ac158 env: - name: ACCESS_KEY valueFrom: diff --git a/k8s/infra/minio/minio-provision-paperless-job.yaml b/k8s/infra/minio/minio-provision-paperless-job.yaml index d1e849e..7329686 100644 --- a/k8s/infra/minio/minio-provision-paperless-job.yaml +++ b/k8s/infra/minio/minio-provision-paperless-job.yaml @@ -52,14 +52,14 @@ spec: restartPolicy: Never initContainers: - name: kubectl-copy - image: bitnami/kubectl:latest + image: bitnami/kubectl:latest@sha256:f7f9e4f64d9e114650c115a4ac6fd383394b3d494061a8f17b1a0f6c8d55bc25 command: ["sh", "-c", "cp $(which kubectl) /shared/kubectl"] volumeMounts: - name: shared mountPath: /shared containers: - name: provision - image: quay.io/minio/mc:latest + image: docker.io/bitnamilegacy/minio-client:2025.7.21-debian-12-r3@sha256:73bd39f7899a0cef12b8dd5df13aa93a3ed1aaa44236542442e9ac76819ac158 volumeMounts: - name: shared mountPath: /shared diff --git a/k8s/infra/minio/minio-tenant.yaml b/k8s/infra/minio/minio-tenant.yaml index 810821e..ab61e93 100644 --- a/k8s/infra/minio/minio-tenant.yaml +++ b/k8s/infra/minio/minio-tenant.yaml @@ -16,7 +16,10 @@ spec: # backend could not resolve minio-cluster-hl. Official images now need registry auth # (quay.io/minio/minio returns 401), so this pins a public mirror of the same upstream # release; swap it for the official image once MinIO registry credentials exist. - image: coollabsio/minio:RELEASE.2025-10-15T17-29-55Z + # The digest freezes those bytes, so a retag or a deletion on the mirror cannot break + # the pull again. The same digest already sits in the containerd cache on talos-cp-1, + # so the first start needs no registry access. + image: docker.io/coollabsio/minio:RELEASE.2025-10-15T17-29-55Z@sha256:69b55a1c1c5dc285ce04db96689f5b2102317fc77a50680a1874ca6efd1c87f9 # Disable operator auto-TLS: MinIO serves plain HTTP internally on 9000. # External TLS is terminated at nginx ingress (wildcard riotpiao-com-tls cert). diff --git a/k8s/infra/monitoring/alerts/cluster-alerts.yaml b/k8s/infra/monitoring/alerts/cluster-alerts.yaml index 040055b..2899fb2 100644 --- a/k8s/infra/monitoring/alerts/cluster-alerts.yaml +++ b/k8s/infra/monitoring/alerts/cluster-alerts.yaml @@ -27,6 +27,24 @@ spec: annotations: summary: "{{ $value }} pod(s) stuck in Pending state for >10m" + # ImagePullBackOff on a vendor-deleted image produced no alert for 13 days + - alert: PodImagePullFailure + expr: sum(kube_pod_container_status_waiting_reason{reason=~"ImagePullBackOff|ErrImagePull"}) by (namespace, pod) > 0 + for: 10m + labels: + severity: critical + annotations: + summary: "{{ $labels.namespace }}/{{ $labels.pod }} cannot pull its image" + + # Pod not ready + - alert: PodNotReady + expr: sum(kube_pod_status_ready{condition="false"}) by (namespace, pod) > 0 + for: 15m + labels: + severity: warning + annotations: + summary: "{{ $labels.namespace }}/{{ $labels.pod }} is not ready" + # CrashLoopBackOff - alert: PodCrashLooping expr: sum(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}) by (namespace, pod) > 0 -- 2.54.0