fix(oidc): restart the apiservers by signalling the process, not by deleting the mirror pod
Static pods survive mirror-pod deletion, so the first bootstrap job had no effect. These jobs share the host PID namespace and SIGTERM the kube-apiserver process itself, staggered by 100s so only one control plane restarts at a time.
This commit is contained in:
@@ -0,0 +1,144 @@
|
|||||||
|
# One-shot: restart the control-plane apiservers so their OIDC authenticator initialises.
|
||||||
|
#
|
||||||
|
# Deleting the mirror pod does NOT restart a static pod - that is why the earlier job had no
|
||||||
|
# effect. These pods share the host PID namespace (privileged, allowed in the 'cicd' namespace)
|
||||||
|
# and send SIGTERM to the kube-apiserver PROCESS; the kubelet then recreates the static pod.
|
||||||
|
# Delays stagger the three control planes so only one API server is down at a time.
|
||||||
|
---
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: Job
|
||||||
|
metadata:
|
||||||
|
name: oidc-restart-apiserver-cp1
|
||||||
|
namespace: cicd
|
||||||
|
spec:
|
||||||
|
backoffLimit: 1
|
||||||
|
ttlSecondsAfterFinished: 3600
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
hostPID: true
|
||||||
|
restartPolicy: Never
|
||||||
|
nodeSelector:
|
||||||
|
kubernetes.io/hostname: talos-cp-1
|
||||||
|
tolerations:
|
||||||
|
- key: node-role.kubernetes.io/control-plane
|
||||||
|
operator: Exists
|
||||||
|
effect: NoSchedule
|
||||||
|
- key: node-role.kubernetes.io/master
|
||||||
|
operator: Exists
|
||||||
|
effect: NoSchedule
|
||||||
|
containers:
|
||||||
|
- name: kill-apiserver
|
||||||
|
image: docker.io/library/busybox:1.36
|
||||||
|
securityContext:
|
||||||
|
privileged: true
|
||||||
|
command: ["/bin/sh", "-c"]
|
||||||
|
args:
|
||||||
|
- |
|
||||||
|
echo "sleeping 10s, then restarting the apiserver process on this node"
|
||||||
|
sleep 10
|
||||||
|
killed=0
|
||||||
|
for p in /proc/[0-9]*; do
|
||||||
|
[ -r "$p/cmdline" ] || continue
|
||||||
|
cmd=$(tr '\0' ' ' < "$p/cmdline" 2>/dev/null)
|
||||||
|
case "$cmd" in
|
||||||
|
*kube-apiserver*)
|
||||||
|
pid=${p#/proc/}
|
||||||
|
echo "SIGTERM -> $pid ${cmd% }"
|
||||||
|
kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
echo "signalled $killed apiserver process(es); the kubelet will recreate the static pod"
|
||||||
|
|
||||||
|
---
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: Job
|
||||||
|
metadata:
|
||||||
|
name: oidc-restart-apiserver-cp2
|
||||||
|
namespace: cicd
|
||||||
|
spec:
|
||||||
|
backoffLimit: 1
|
||||||
|
ttlSecondsAfterFinished: 3600
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
hostPID: true
|
||||||
|
restartPolicy: Never
|
||||||
|
nodeSelector:
|
||||||
|
kubernetes.io/hostname: talos-cp-2
|
||||||
|
tolerations:
|
||||||
|
- key: node-role.kubernetes.io/control-plane
|
||||||
|
operator: Exists
|
||||||
|
effect: NoSchedule
|
||||||
|
- key: node-role.kubernetes.io/master
|
||||||
|
operator: Exists
|
||||||
|
effect: NoSchedule
|
||||||
|
containers:
|
||||||
|
- name: kill-apiserver
|
||||||
|
image: docker.io/library/busybox:1.36
|
||||||
|
securityContext:
|
||||||
|
privileged: true
|
||||||
|
command: ["/bin/sh", "-c"]
|
||||||
|
args:
|
||||||
|
- |
|
||||||
|
echo "sleeping 110s, then restarting the apiserver process on this node"
|
||||||
|
sleep 110
|
||||||
|
killed=0
|
||||||
|
for p in /proc/[0-9]*; do
|
||||||
|
[ -r "$p/cmdline" ] || continue
|
||||||
|
cmd=$(tr '\0' ' ' < "$p/cmdline" 2>/dev/null)
|
||||||
|
case "$cmd" in
|
||||||
|
*kube-apiserver*)
|
||||||
|
pid=${p#/proc/}
|
||||||
|
echo "SIGTERM -> $pid ${cmd% }"
|
||||||
|
kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
echo "signalled $killed apiserver process(es); the kubelet will recreate the static pod"
|
||||||
|
|
||||||
|
---
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: Job
|
||||||
|
metadata:
|
||||||
|
name: oidc-restart-apiserver-cp3
|
||||||
|
namespace: cicd
|
||||||
|
spec:
|
||||||
|
backoffLimit: 1
|
||||||
|
ttlSecondsAfterFinished: 3600
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
hostPID: true
|
||||||
|
restartPolicy: Never
|
||||||
|
nodeSelector:
|
||||||
|
kubernetes.io/hostname: talos-cp-3
|
||||||
|
tolerations:
|
||||||
|
- key: node-role.kubernetes.io/control-plane
|
||||||
|
operator: Exists
|
||||||
|
effect: NoSchedule
|
||||||
|
- key: node-role.kubernetes.io/master
|
||||||
|
operator: Exists
|
||||||
|
effect: NoSchedule
|
||||||
|
containers:
|
||||||
|
- name: kill-apiserver
|
||||||
|
image: docker.io/library/busybox:1.36
|
||||||
|
securityContext:
|
||||||
|
privileged: true
|
||||||
|
command: ["/bin/sh", "-c"]
|
||||||
|
args:
|
||||||
|
- |
|
||||||
|
echo "sleeping 210s, then restarting the apiserver process on this node"
|
||||||
|
sleep 210
|
||||||
|
killed=0
|
||||||
|
for p in /proc/[0-9]*; do
|
||||||
|
[ -r "$p/cmdline" ] || continue
|
||||||
|
cmd=$(tr '\0' ' ' < "$p/cmdline" 2>/dev/null)
|
||||||
|
case "$cmd" in
|
||||||
|
*kube-apiserver*)
|
||||||
|
pid=${p#/proc/}
|
||||||
|
echo "SIGTERM -> $pid ${cmd% }"
|
||||||
|
kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
echo "signalled $killed apiserver process(es); the kubelet will recreate the static pod"
|
||||||
|
|
||||||
Reference in New Issue
Block a user