rock
d1a7d3be6e
The oidc-bootstrap ArgoCD Application runs with selfHeal and prune. Three Jobs in restart-apiserver.yaml kill the kube-apiserver process on each control plane, and bootstrap.yaml carried a fourth Job that deletes the mirror pods. ArgoCD recreated a deleted Job within two minutes, so the kill repeated and every API server reached CrashLoopBackOff. Remove all four. The OIDC authenticator already loaded, so the one-shot restart is not needed again.
58 lines
2.4 KiB
YAML
58 lines
2.4 KiB
YAML
# One-shot repair: the kubelets on cp-1, cp-2 and worker-1 stopped posting node status at
|
|
# 04:11-04:14 UTC and never reconnected, so those nodes are NotReady/unreachable and every
|
|
# pod on them (Cilium included) shows NotReady. This job runs on the one healthy node (cp-3),
|
|
# talks to the Talos API on the LAN, and uses Talos' endpoint->node proxying to reach the
|
|
# other three: it reports the kubelet service state, then restarts it.
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata:
|
|
name: talos-restart-stalled-kubelets
|
|
namespace: cicd
|
|
spec:
|
|
backoffLimit: 1
|
|
ttlSecondsAfterFinished: 86400
|
|
template:
|
|
spec:
|
|
restartPolicy: Never
|
|
nodeSelector:
|
|
kubernetes.io/hostname: talos-cp-3
|
|
tolerations:
|
|
- key: node-role.kubernetes.io/control-plane
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
containers:
|
|
- name: talosctl
|
|
image: docker.io/curlimages/curl:8.22.0
|
|
command: ["/bin/sh", "-c"]
|
|
args:
|
|
- |
|
|
set -u
|
|
echo "fetching talosctl v1.13.3 (matches the node version)"
|
|
curl -sSL -o /tmp/talosctl \
|
|
https://github.com/siderolabs/talos/releases/download/v1.13.3/talosctl-linux-amd64 \
|
|
|| { echo "DOWNLOAD FAILED"; exit 1; }
|
|
chmod +x /tmp/talosctl
|
|
T="/tmp/talosctl --talosconfig /talos/config -e 192.168.1.162"
|
|
NODES="192.168.1.166 192.168.1.214 192.168.1.223"
|
|
echo "=== BEFORE: kubelet service state ==="
|
|
for n in $NODES; do echo "--- $n"; $T -n $n service kubelet 2>&1 || true; done
|
|
echo "=== kernel log tail (last 15 lines per node) ==="
|
|
for n in $NODES; do echo "--- $n"; $T -n $n dmesg --tail 15 2>&1 | tail -15 || true; done
|
|
echo "=== ACTION: restart kubelet ==="
|
|
for n in $NODES; do echo "--- restarting $n"; $T -n $n restart kubelet 2>&1 || true; sleep 20; done
|
|
sleep 60
|
|
echo "=== AFTER ==="
|
|
for n in $NODES; do echo "--- $n"; $T -n $n service kubelet 2>&1 || true; done
|
|
volumeMounts:
|
|
- name: talos
|
|
mountPath: /talos
|
|
readOnly: true
|
|
volumes:
|
|
- name: talos
|
|
secret:
|
|
secretName: talos-repair-config
|
|
defaultMode: 0444
|
|
items:
|
|
- key: config
|
|
path: config
|