riotpiao-poimen/homelab
2
0
Fork 0
Code Issues 23 Pull Requests 2 Actions Packages Projects 1 Releases Wiki Activity

fix(oidc): match /proc/<pid>/comm instead of cmdline when signalling the API server

Browse Source 
Exit code 143 proved the previous job killed itself: the script's own cmdline contains the
string 'kube-apiserver', so it matched its own process. Matching comm (== 'kube-apiserver')
and skipping $$ removes that failure mode. Staggered 2 minutes apart.
This commit is contained in:
rock
2026-10-07 04:07:37 +00:00
parent 9fa8d9cf60
commit 2d9e3abee8
1 changed files with 49 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
k8s/infra/oidc-bootstrap/restart-apiserver.yaml
+46 -49
View File
@@ -1,17 +1,20 @@
# One-shot: restart the control-plane apiservers so their OIDC authenticator initialises.
# One-shot: restart the control-plane API servers so their OIDC authenticator initialises.
#
# Deleting the mirror pod does NOT restart a static pod - that is why the earlier job had no
# effect. These pods share the host PID namespace (privileged, allowed in the 'cicd' namespace)
# and send SIGTERM to the kube-apiserver PROCESS; the kubelet then recreates the static pod.
# Delays stagger the three control planes so only one API server is down at a time.
# Two earlier mechanisms failed and are documented so nobody repeats them:
# 1. deleting the mirror pod does NOT restart a static pod (deletes returned 200, no restart);
# 2. matching /proc/*/cmdline for 'kube-apiserver' also matched this script's own cmdline,
# so every pod SIGTERMed itself (exit 143) before reaching the real process.
# This version matches /proc/<pid>/comm (the process name) and skips \$\$, so self-kill is
# impossible. hostPID + privileged are permitted in the 'cicd' namespace. Delays stagger the
# three control planes by 2 minutes each, so only one API server is down at a time.
---
apiVersion: batch/v1
kind: Job
metadata:
name: oidc-restart-apiserver-cp1
name: oidc-restart-apiserver-cp1-v2
namespace: cicd
spec:
backoffLimit: 1
backoffLimit: 0
ttlSecondsAfterFinished: 3600
template:
spec:
@@ -27,37 +30,35 @@ spec:
operator: Exists
effect: NoSchedule
containers:
- name: kill-apiserver
- name: kill
image: docker.io/library/busybox:1.36
securityContext:
privileged: true
command: ["/bin/sh", "-c"]
args:
- |
echo "sleeping 10s, then restarting the apiserver process on this node"
echo "sleeping 10s before restarting the API server on talos-cp-1"
sleep 10
self=$$
killed=0
for p in /proc/[0-9]*; do
[ -r "$p/cmdline" ] || continue
cmd=$(tr '\0' ' ' < "$p/cmdline" 2>/dev/null)
case "$cmd" in
*kube-apiserver*)
pid=${p#/proc/}
echo "SIGTERM -> $pid ${cmd% }"
for d in /proc/[0-9]*; do
pid=${d#/proc/}
[ "$pid" = "$self" ] && continue
comm=$(cat "$d/comm" 2>/dev/null)
[ "$comm" = "kube-apiserver" ] || continue
echo "SIGTERM -> pid $pid"
kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
;;
esac
done
echo "signalled $killed apiserver process(es); the kubelet will recreate the static pod"
echo "signalled $killed API server process(es); kubelet recreates the static pod"
---
apiVersion: batch/v1
kind: Job
metadata:
name: oidc-restart-apiserver-cp2
name: oidc-restart-apiserver-cp2-v2
namespace: cicd
spec:
backoffLimit: 1
backoffLimit: 0
ttlSecondsAfterFinished: 3600
template:
spec:
@@ -73,37 +74,35 @@ spec:
operator: Exists
effect: NoSchedule
containers:
- name: kill-apiserver
- name: kill
image: docker.io/library/busybox:1.36
securityContext:
privileged: true
command: ["/bin/sh", "-c"]
args:
- |
echo "sleeping 110s, then restarting the apiserver process on this node"
sleep 110
echo "sleeping 120s before restarting the API server on talos-cp-2"
sleep 120
self=$$
killed=0
for p in /proc/[0-9]*; do
[ -r "$p/cmdline" ] || continue
cmd=$(tr '\0' ' ' < "$p/cmdline" 2>/dev/null)
case "$cmd" in
*kube-apiserver*)
pid=${p#/proc/}
echo "SIGTERM -> $pid ${cmd% }"
for d in /proc/[0-9]*; do
pid=${d#/proc/}
[ "$pid" = "$self" ] && continue
comm=$(cat "$d/comm" 2>/dev/null)
[ "$comm" = "kube-apiserver" ] || continue
echo "SIGTERM -> pid $pid"
kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
;;
esac
done
echo "signalled $killed apiserver process(es); the kubelet will recreate the static pod"
echo "signalled $killed API server process(es); kubelet recreates the static pod"
---
apiVersion: batch/v1
kind: Job
metadata:
name: oidc-restart-apiserver-cp3
name: oidc-restart-apiserver-cp3-v2
namespace: cicd
spec:
backoffLimit: 1
backoffLimit: 0
ttlSecondsAfterFinished: 3600
template:
spec:
@@ -119,26 +118,24 @@ spec:
operator: Exists
effect: NoSchedule
containers:
- name: kill-apiserver
- name: kill
image: docker.io/library/busybox:1.36
securityContext:
privileged: true
command: ["/bin/sh", "-c"]
args:
- |
echo "sleeping 210s, then restarting the apiserver process on this node"
sleep 210
echo "sleeping 240s before restarting the API server on talos-cp-3"
sleep 240
self=$$
killed=0
for p in /proc/[0-9]*; do
[ -r "$p/cmdline" ] || continue
cmd=$(tr '\0' ' ' < "$p/cmdline" 2>/dev/null)
case "$cmd" in
*kube-apiserver*)
pid=${p#/proc/}
echo "SIGTERM -> $pid ${cmd% }"
for d in /proc/[0-9]*; do
pid=${d#/proc/}
[ "$pid" = "$self" ] && continue
comm=$(cat "$d/comm" 2>/dev/null)
[ "$comm" = "kube-apiserver" ] || continue
echo "SIGTERM -> pid $pid"
kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
;;
esac
done
echo "signalled $killed apiserver process(es); the kubelet will recreate the static pod"
echo "signalled $killed API server process(es); kubelet recreates the static pod"