fix(oidc): match /proc/<pid>/comm instead of cmdline when signalling the API server
Exit code 143 proved the previous job killed itself: the script's own cmdline contains the string 'kube-apiserver', so it matched its own process. Matching comm (== 'kube-apiserver') and skipping $$ removes that failure mode. Staggered 2 minutes apart.
This commit is contained in:
@@ -1,17 +1,20 @@
|
|||||||
# One-shot: restart the control-plane apiservers so their OIDC authenticator initialises.
|
# One-shot: restart the control-plane API servers so their OIDC authenticator initialises.
|
||||||
#
|
#
|
||||||
# Deleting the mirror pod does NOT restart a static pod - that is why the earlier job had no
|
# Two earlier mechanisms failed and are documented so nobody repeats them:
|
||||||
# effect. These pods share the host PID namespace (privileged, allowed in the 'cicd' namespace)
|
# 1. deleting the mirror pod does NOT restart a static pod (deletes returned 200, no restart);
|
||||||
# and send SIGTERM to the kube-apiserver PROCESS; the kubelet then recreates the static pod.
|
# 2. matching /proc/*/cmdline for 'kube-apiserver' also matched this script's own cmdline,
|
||||||
# Delays stagger the three control planes so only one API server is down at a time.
|
# so every pod SIGTERMed itself (exit 143) before reaching the real process.
|
||||||
|
# This version matches /proc/<pid>/comm (the process name) and skips \$\$, so self-kill is
|
||||||
|
# impossible. hostPID + privileged are permitted in the 'cicd' namespace. Delays stagger the
|
||||||
|
# three control planes by 2 minutes each, so only one API server is down at a time.
|
||||||
---
|
---
|
||||||
apiVersion: batch/v1
|
apiVersion: batch/v1
|
||||||
kind: Job
|
kind: Job
|
||||||
metadata:
|
metadata:
|
||||||
name: oidc-restart-apiserver-cp1
|
name: oidc-restart-apiserver-cp1-v2
|
||||||
namespace: cicd
|
namespace: cicd
|
||||||
spec:
|
spec:
|
||||||
backoffLimit: 1
|
backoffLimit: 0
|
||||||
ttlSecondsAfterFinished: 3600
|
ttlSecondsAfterFinished: 3600
|
||||||
template:
|
template:
|
||||||
spec:
|
spec:
|
||||||
@@ -27,37 +30,35 @@ spec:
|
|||||||
operator: Exists
|
operator: Exists
|
||||||
effect: NoSchedule
|
effect: NoSchedule
|
||||||
containers:
|
containers:
|
||||||
- name: kill-apiserver
|
- name: kill
|
||||||
image: docker.io/library/busybox:1.36
|
image: docker.io/library/busybox:1.36
|
||||||
securityContext:
|
securityContext:
|
||||||
privileged: true
|
privileged: true
|
||||||
command: ["/bin/sh", "-c"]
|
command: ["/bin/sh", "-c"]
|
||||||
args:
|
args:
|
||||||
- |
|
- |
|
||||||
echo "sleeping 10s, then restarting the apiserver process on this node"
|
echo "sleeping 10s before restarting the API server on talos-cp-1"
|
||||||
sleep 10
|
sleep 10
|
||||||
|
self=$$
|
||||||
killed=0
|
killed=0
|
||||||
for p in /proc/[0-9]*; do
|
for d in /proc/[0-9]*; do
|
||||||
[ -r "$p/cmdline" ] || continue
|
pid=${d#/proc/}
|
||||||
cmd=$(tr '\0' ' ' < "$p/cmdline" 2>/dev/null)
|
[ "$pid" = "$self" ] && continue
|
||||||
case "$cmd" in
|
comm=$(cat "$d/comm" 2>/dev/null)
|
||||||
*kube-apiserver*)
|
[ "$comm" = "kube-apiserver" ] || continue
|
||||||
pid=${p#/proc/}
|
echo "SIGTERM -> pid $pid"
|
||||||
echo "SIGTERM -> $pid ${cmd% }"
|
kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
|
||||||
kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
done
|
done
|
||||||
echo "signalled $killed apiserver process(es); the kubelet will recreate the static pod"
|
echo "signalled $killed API server process(es); kubelet recreates the static pod"
|
||||||
|
|
||||||
---
|
---
|
||||||
apiVersion: batch/v1
|
apiVersion: batch/v1
|
||||||
kind: Job
|
kind: Job
|
||||||
metadata:
|
metadata:
|
||||||
name: oidc-restart-apiserver-cp2
|
name: oidc-restart-apiserver-cp2-v2
|
||||||
namespace: cicd
|
namespace: cicd
|
||||||
spec:
|
spec:
|
||||||
backoffLimit: 1
|
backoffLimit: 0
|
||||||
ttlSecondsAfterFinished: 3600
|
ttlSecondsAfterFinished: 3600
|
||||||
template:
|
template:
|
||||||
spec:
|
spec:
|
||||||
@@ -73,37 +74,35 @@ spec:
|
|||||||
operator: Exists
|
operator: Exists
|
||||||
effect: NoSchedule
|
effect: NoSchedule
|
||||||
containers:
|
containers:
|
||||||
- name: kill-apiserver
|
- name: kill
|
||||||
image: docker.io/library/busybox:1.36
|
image: docker.io/library/busybox:1.36
|
||||||
securityContext:
|
securityContext:
|
||||||
privileged: true
|
privileged: true
|
||||||
command: ["/bin/sh", "-c"]
|
command: ["/bin/sh", "-c"]
|
||||||
args:
|
args:
|
||||||
- |
|
- |
|
||||||
echo "sleeping 110s, then restarting the apiserver process on this node"
|
echo "sleeping 120s before restarting the API server on talos-cp-2"
|
||||||
sleep 110
|
sleep 120
|
||||||
|
self=$$
|
||||||
killed=0
|
killed=0
|
||||||
for p in /proc/[0-9]*; do
|
for d in /proc/[0-9]*; do
|
||||||
[ -r "$p/cmdline" ] || continue
|
pid=${d#/proc/}
|
||||||
cmd=$(tr '\0' ' ' < "$p/cmdline" 2>/dev/null)
|
[ "$pid" = "$self" ] && continue
|
||||||
case "$cmd" in
|
comm=$(cat "$d/comm" 2>/dev/null)
|
||||||
*kube-apiserver*)
|
[ "$comm" = "kube-apiserver" ] || continue
|
||||||
pid=${p#/proc/}
|
echo "SIGTERM -> pid $pid"
|
||||||
echo "SIGTERM -> $pid ${cmd% }"
|
kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
|
||||||
kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
done
|
done
|
||||||
echo "signalled $killed apiserver process(es); the kubelet will recreate the static pod"
|
echo "signalled $killed API server process(es); kubelet recreates the static pod"
|
||||||
|
|
||||||
---
|
---
|
||||||
apiVersion: batch/v1
|
apiVersion: batch/v1
|
||||||
kind: Job
|
kind: Job
|
||||||
metadata:
|
metadata:
|
||||||
name: oidc-restart-apiserver-cp3
|
name: oidc-restart-apiserver-cp3-v2
|
||||||
namespace: cicd
|
namespace: cicd
|
||||||
spec:
|
spec:
|
||||||
backoffLimit: 1
|
backoffLimit: 0
|
||||||
ttlSecondsAfterFinished: 3600
|
ttlSecondsAfterFinished: 3600
|
||||||
template:
|
template:
|
||||||
spec:
|
spec:
|
||||||
@@ -119,26 +118,24 @@ spec:
|
|||||||
operator: Exists
|
operator: Exists
|
||||||
effect: NoSchedule
|
effect: NoSchedule
|
||||||
containers:
|
containers:
|
||||||
- name: kill-apiserver
|
- name: kill
|
||||||
image: docker.io/library/busybox:1.36
|
image: docker.io/library/busybox:1.36
|
||||||
securityContext:
|
securityContext:
|
||||||
privileged: true
|
privileged: true
|
||||||
command: ["/bin/sh", "-c"]
|
command: ["/bin/sh", "-c"]
|
||||||
args:
|
args:
|
||||||
- |
|
- |
|
||||||
echo "sleeping 210s, then restarting the apiserver process on this node"
|
echo "sleeping 240s before restarting the API server on talos-cp-3"
|
||||||
sleep 210
|
sleep 240
|
||||||
|
self=$$
|
||||||
killed=0
|
killed=0
|
||||||
for p in /proc/[0-9]*; do
|
for d in /proc/[0-9]*; do
|
||||||
[ -r "$p/cmdline" ] || continue
|
pid=${d#/proc/}
|
||||||
cmd=$(tr '\0' ' ' < "$p/cmdline" 2>/dev/null)
|
[ "$pid" = "$self" ] && continue
|
||||||
case "$cmd" in
|
comm=$(cat "$d/comm" 2>/dev/null)
|
||||||
*kube-apiserver*)
|
[ "$comm" = "kube-apiserver" ] || continue
|
||||||
pid=${p#/proc/}
|
echo "SIGTERM -> pid $pid"
|
||||||
echo "SIGTERM -> $pid ${cmd% }"
|
kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
|
||||||
kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
done
|
done
|
||||||
echo "signalled $killed apiserver process(es); the kubelet will recreate the static pod"
|
echo "signalled $killed API server process(es); kubelet recreates the static pod"
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user