58 lines
2.4 KiB
YAML
58 lines
2.4 KiB
YAML
|
# One-shot repair: the kubelets on cp-1, cp-2 and worker-1 stopped posting node status at
|
||
|
|
# 04:11-04:14 UTC and never reconnected, so those nodes are NotReady/unreachable and every
|
||
|
|
# pod on them (Cilium included) shows NotReady. This job runs on the one healthy node (cp-3),
|
||
|
|
# talks to the Talos API on the LAN, and uses Talos' endpoint->node proxying to reach the
|
||
|
|
# other three: it reports the kubelet service state, then restarts it.
|
||
|
|
apiVersion: batch/v1
|
||
|
|
kind: Job
|
||
|
|
metadata:
|
||
|
|
name: talos-restart-stalled-kubelets
|
||
|
|
namespace: cicd
|
||
|
|
spec:
|
||
|
|
backoffLimit: 1
|
||
|
|
ttlSecondsAfterFinished: 86400
|
||
|
|
template:
|
||
|
|
spec:
|
||
|
|
restartPolicy: Never
|
||
|
|
nodeSelector:
|
||
|
|
kubernetes.io/hostname: talos-cp-3
|
||
|
|
tolerations:
|
||
|
|
- key: node-role.kubernetes.io/control-plane
|
||
|
|
operator: Exists
|
||
|
|
effect: NoSchedule
|
||
|
|
containers:
|
||
|
|
- name: talosctl
|
||
|
|
image: docker.io/curlimages/curl:8.22.0
|
||
|
|
command: ["/bin/sh", "-c"]
|
||
|
|
args:
|
||
|
|
- |
|
||
|
|
set -u
|
||
|
|
echo "fetching talosctl v1.13.3 (matches the node version)"
|
||
|
|
curl -sSL -o /tmp/talosctl \
|
||
|
|
https://github.com/siderolabs/talos/releases/download/v1.13.3/talosctl-linux-amd64 \
|
||
|
|
|| { echo "DOWNLOAD FAILED"; exit 1; }
|
||
|
|
chmod +x /tmp/talosctl
|
||
|
|
T="/tmp/talosctl --talosconfig /talos/config -e 192.168.1.162"
|
||
|
|
NODES="192.168.1.166 192.168.1.214 192.168.1.223"
|
||
|
|
echo "=== BEFORE: kubelet service state ==="
|
||
|
|
for n in $NODES; do echo "--- $n"; $T -n $n service kubelet 2>&1 || true; done
|
||
|
|
echo "=== kernel log tail (last 15 lines per node) ==="
|
||
|
|
for n in $NODES; do echo "--- $n"; $T -n $n dmesg --tail 15 2>&1 | tail -15 || true; done
|
||
|
|
echo "=== ACTION: restart kubelet ==="
|
||
|
|
for n in $NODES; do echo "--- restarting $n"; $T -n $n restart kubelet 2>&1 || true; sleep 20; done
|
||
|
|
sleep 60
|
||
|
|
echo "=== AFTER ==="
|
||
|
|
for n in $NODES; do echo "--- $n"; $T -n $n service kubelet 2>&1 || true; done
|
||
|
|
volumeMounts:
|
||
|
|
- name: talos
|
||
|
|
mountPath: /talos
|
||
|
|
readOnly: true
|
||
|
|
volumes:
|
||
|
|
- name: talos
|
||
|
|
secret:
|
||
|
|
secretName: talos-repair-config
|
||
|
|
defaultMode: 0444
|
||
|
|
items:
|
||
|
|
- key: config
|
||
|
|
path: config
|