fix(oidc): restart the apiservers by signalling the process, not by deleting the mirror pod

Static pods survive mirror-pod deletion, so the first bootstrap job had no effect. These jobs share the host PID namespace and SIGTERM the kube-apiserver process itself, staggered by 100s so only one control plane restarts at a time.
2026-10-07 03:59:22 +00:00
parent 2e4fad3bf7
commit 9fa8d9cf60
1 changed files with 144 additions and 0 deletions
@@ -0,0 +1,144 @@
+# One-shot: restart the control-plane apiservers so their OIDC authenticator initialises.
+#
+# Deleting the mirror pod does NOT restart a static pod - that is why the earlier job had no
+# effect. These pods share the host PID namespace (privileged, allowed in the 'cicd' namespace)
+# and send SIGTERM to the kube-apiserver PROCESS; the kubelet then recreates the static pod.
+# Delays stagger the three control planes so only one API server is down at a time.
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: oidc-restart-apiserver-cp1
+  namespace: cicd
+spec:
+  backoffLimit: 1
+  ttlSecondsAfterFinished: 3600
+  template:
+    spec:
+      hostPID: true
+      restartPolicy: Never
+      nodeSelector:
+        kubernetes.io/hostname: talos-cp-1
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+        - key: node-role.kubernetes.io/master
+          operator: Exists
+          effect: NoSchedule
+      containers:
+        - name: kill-apiserver
+          image: docker.io/library/busybox:1.36
+          securityContext:
+            privileged: true
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              echo "sleeping 10s, then restarting the apiserver process on this node"
+              sleep 10
+              killed=0
+              for p in /proc/[0-9]*; do
+                [ -r "$p/cmdline" ] || continue
+                cmd=$(tr '\0' ' ' < "$p/cmdline" 2>/dev/null)
+                case "$cmd" in
+                  *kube-apiserver*)
+                    pid=${p#/proc/}
+                    echo "SIGTERM -> $pid  ${cmd% }"
+                    kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
+                    ;;
+                esac
+              done
+              echo "signalled $killed apiserver process(es); the kubelet will recreate the static pod"
+
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: oidc-restart-apiserver-cp2
+  namespace: cicd
+spec:
+  backoffLimit: 1
+  ttlSecondsAfterFinished: 3600
+  template:
+    spec:
+      hostPID: true
+      restartPolicy: Never
+      nodeSelector:
+        kubernetes.io/hostname: talos-cp-2
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+        - key: node-role.kubernetes.io/master
+          operator: Exists
+          effect: NoSchedule
+      containers:
+        - name: kill-apiserver
+          image: docker.io/library/busybox:1.36
+          securityContext:
+            privileged: true
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              echo "sleeping 110s, then restarting the apiserver process on this node"
+              sleep 110
+              killed=0
+              for p in /proc/[0-9]*; do
+                [ -r "$p/cmdline" ] || continue
+                cmd=$(tr '\0' ' ' < "$p/cmdline" 2>/dev/null)
+                case "$cmd" in
+                  *kube-apiserver*)
+                    pid=${p#/proc/}
+                    echo "SIGTERM -> $pid  ${cmd% }"
+                    kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
+                    ;;
+                esac
+              done
+              echo "signalled $killed apiserver process(es); the kubelet will recreate the static pod"
+
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: oidc-restart-apiserver-cp3
+  namespace: cicd
+spec:
+  backoffLimit: 1
+  ttlSecondsAfterFinished: 3600
+  template:
+    spec:
+      hostPID: true
+      restartPolicy: Never
+      nodeSelector:
+        kubernetes.io/hostname: talos-cp-3
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+        - key: node-role.kubernetes.io/master
+          operator: Exists
+          effect: NoSchedule
+      containers:
+        - name: kill-apiserver
+          image: docker.io/library/busybox:1.36
+          securityContext:
+            privileged: true
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              echo "sleeping 210s, then restarting the apiserver process on this node"
+              sleep 210
+              killed=0
+              for p in /proc/[0-9]*; do
+                [ -r "$p/cmdline" ] || continue
+                cmd=$(tr '\0' ' ' < "$p/cmdline" 2>/dev/null)
+                case "$cmd" in
+                  *kube-apiserver*)
+                    pid=${p#/proc/}
+                    echo "SIGTERM -> $pid  ${cmd% }"
+                    kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
+                    ;;
+                esac
+              done
+              echo "signalled $killed apiserver process(es); the kubelet will recreate the static pod"
+