diff --git a/k8s/infra/oidc-bootstrap/restart-apiserver.yaml b/k8s/infra/oidc-bootstrap/restart-apiserver.yaml index 4663996..021ab0b 100644 --- a/k8s/infra/oidc-bootstrap/restart-apiserver.yaml +++ b/k8s/infra/oidc-bootstrap/restart-apiserver.yaml @@ -1,17 +1,20 @@ -# One-shot: restart the control-plane apiservers so their OIDC authenticator initialises. +# One-shot: restart the control-plane API servers so their OIDC authenticator initialises. # -# Deleting the mirror pod does NOT restart a static pod - that is why the earlier job had no -# effect. These pods share the host PID namespace (privileged, allowed in the 'cicd' namespace) -# and send SIGTERM to the kube-apiserver PROCESS; the kubelet then recreates the static pod. -# Delays stagger the three control planes so only one API server is down at a time. +# Two earlier mechanisms failed and are documented so nobody repeats them: +# 1. deleting the mirror pod does NOT restart a static pod (deletes returned 200, no restart); +# 2. matching /proc/*/cmdline for 'kube-apiserver' also matched this script's own cmdline, +# so every pod SIGTERMed itself (exit 143) before reaching the real process. +# This version matches /proc//comm (the process name) and skips \$\$, so self-kill is +# impossible. hostPID + privileged are permitted in the 'cicd' namespace. Delays stagger the +# three control planes by 2 minutes each, so only one API server is down at a time. --- apiVersion: batch/v1 kind: Job metadata: - name: oidc-restart-apiserver-cp1 + name: oidc-restart-apiserver-cp1-v2 namespace: cicd spec: - backoffLimit: 1 + backoffLimit: 0 ttlSecondsAfterFinished: 3600 template: spec: @@ -27,37 +30,35 @@ spec: operator: Exists effect: NoSchedule containers: - - name: kill-apiserver + - name: kill image: docker.io/library/busybox:1.36 securityContext: privileged: true command: ["/bin/sh", "-c"] args: - | - echo "sleeping 10s, then restarting the apiserver process on this node" + echo "sleeping 10s before restarting the API server on talos-cp-1" sleep 10 + self=$$ killed=0 - for p in /proc/[0-9]*; do - [ -r "$p/cmdline" ] || continue - cmd=$(tr '\0' ' ' < "$p/cmdline" 2>/dev/null) - case "$cmd" in - *kube-apiserver*) - pid=${p#/proc/} - echo "SIGTERM -> $pid ${cmd% }" - kill -TERM "$pid" 2>/dev/null && killed=$((killed+1)) - ;; - esac + for d in /proc/[0-9]*; do + pid=${d#/proc/} + [ "$pid" = "$self" ] && continue + comm=$(cat "$d/comm" 2>/dev/null) + [ "$comm" = "kube-apiserver" ] || continue + echo "SIGTERM -> pid $pid" + kill -TERM "$pid" 2>/dev/null && killed=$((killed+1)) done - echo "signalled $killed apiserver process(es); the kubelet will recreate the static pod" + echo "signalled $killed API server process(es); kubelet recreates the static pod" --- apiVersion: batch/v1 kind: Job metadata: - name: oidc-restart-apiserver-cp2 + name: oidc-restart-apiserver-cp2-v2 namespace: cicd spec: - backoffLimit: 1 + backoffLimit: 0 ttlSecondsAfterFinished: 3600 template: spec: @@ -73,37 +74,35 @@ spec: operator: Exists effect: NoSchedule containers: - - name: kill-apiserver + - name: kill image: docker.io/library/busybox:1.36 securityContext: privileged: true command: ["/bin/sh", "-c"] args: - | - echo "sleeping 110s, then restarting the apiserver process on this node" - sleep 110 + echo "sleeping 120s before restarting the API server on talos-cp-2" + sleep 120 + self=$$ killed=0 - for p in /proc/[0-9]*; do - [ -r "$p/cmdline" ] || continue - cmd=$(tr '\0' ' ' < "$p/cmdline" 2>/dev/null) - case "$cmd" in - *kube-apiserver*) - pid=${p#/proc/} - echo "SIGTERM -> $pid ${cmd% }" - kill -TERM "$pid" 2>/dev/null && killed=$((killed+1)) - ;; - esac + for d in /proc/[0-9]*; do + pid=${d#/proc/} + [ "$pid" = "$self" ] && continue + comm=$(cat "$d/comm" 2>/dev/null) + [ "$comm" = "kube-apiserver" ] || continue + echo "SIGTERM -> pid $pid" + kill -TERM "$pid" 2>/dev/null && killed=$((killed+1)) done - echo "signalled $killed apiserver process(es); the kubelet will recreate the static pod" + echo "signalled $killed API server process(es); kubelet recreates the static pod" --- apiVersion: batch/v1 kind: Job metadata: - name: oidc-restart-apiserver-cp3 + name: oidc-restart-apiserver-cp3-v2 namespace: cicd spec: - backoffLimit: 1 + backoffLimit: 0 ttlSecondsAfterFinished: 3600 template: spec: @@ -119,26 +118,24 @@ spec: operator: Exists effect: NoSchedule containers: - - name: kill-apiserver + - name: kill image: docker.io/library/busybox:1.36 securityContext: privileged: true command: ["/bin/sh", "-c"] args: - | - echo "sleeping 210s, then restarting the apiserver process on this node" - sleep 210 + echo "sleeping 240s before restarting the API server on talos-cp-3" + sleep 240 + self=$$ killed=0 - for p in /proc/[0-9]*; do - [ -r "$p/cmdline" ] || continue - cmd=$(tr '\0' ' ' < "$p/cmdline" 2>/dev/null) - case "$cmd" in - *kube-apiserver*) - pid=${p#/proc/} - echo "SIGTERM -> $pid ${cmd% }" - kill -TERM "$pid" 2>/dev/null && killed=$((killed+1)) - ;; - esac + for d in /proc/[0-9]*; do + pid=${d#/proc/} + [ "$pid" = "$self" ] && continue + comm=$(cat "$d/comm" 2>/dev/null) + [ "$comm" = "kube-apiserver" ] || continue + echo "SIGTERM -> pid $pid" + kill -TERM "$pid" 2>/dev/null && killed=$((killed+1)) done - echo "signalled $killed apiserver process(es); the kubelet will recreate the static pod" + echo "signalled $killed API server process(es); kubelet recreates the static pod"