homelab/k8s/infra/oidc-bootstrap/restart-apiserver.yaml

# One-shot: restart the control-plane API servers so their OIDC authenticator initialises.
#
# Two earlier mechanisms failed and are documented so nobody repeats them:
#   1. deleting the mirror pod does NOT restart a static pod (deletes returned 200, no restart);
#   2. matching /proc/*/cmdline for 'kube-apiserver' also matched this script's own cmdline,
#      so every pod SIGTERMed itself (exit 143) before reaching the real process.
# This version matches /proc/<pid>/comm (the process name) and skips \$\$, so self-kill is
# impossible. hostPID + privileged are permitted in the 'cicd' namespace. Delays stagger the
# three control planes by 2 minutes each, so only one API server is down at a time.
---
apiVersion: batch/v1
kind: Job
metadata:
  name: oidc-restart-apiserver-cp1-v2
  namespace: cicd
spec:
  backoffLimit: 0
  ttlSecondsAfterFinished: 3600
  template:
    spec:
      hostPID: true
      restartPolicy: Never
      nodeSelector:
        kubernetes.io/hostname: talos-cp-1
      tolerations:
        - key: node-role.kubernetes.io/control-plane
          operator: Exists
          effect: NoSchedule
        - key: node-role.kubernetes.io/master
          operator: Exists
          effect: NoSchedule
      containers:
        - name: kill
          image: docker.io/library/busybox:1.36
          securityContext:
            privileged: true
          command: ["/bin/sh", "-c"]
          args:
            - |
              echo "sleeping 10s before restarting the API server on talos-cp-1"
              sleep 10
              self=$$
              killed=0
              for d in /proc/[0-9]*; do
                pid=${d#/proc/}
                [ "$pid" = "$self" ] && continue
                comm=$(cat "$d/comm" 2>/dev/null)
                [ "$comm" = "kube-apiserver" ] || continue
                echo "SIGTERM -> pid $pid"
                kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
              done
              echo "signalled $killed API server process(es); kubelet recreates the static pod"

---
apiVersion: batch/v1
kind: Job
metadata:
  name: oidc-restart-apiserver-cp2-v2
  namespace: cicd
spec:
  backoffLimit: 0
  ttlSecondsAfterFinished: 3600
  template:
    spec:
      hostPID: true
      restartPolicy: Never
      nodeSelector:
        kubernetes.io/hostname: talos-cp-2
      tolerations:
        - key: node-role.kubernetes.io/control-plane
          operator: Exists
          effect: NoSchedule
        - key: node-role.kubernetes.io/master
          operator: Exists
          effect: NoSchedule
      containers:
        - name: kill
          image: docker.io/library/busybox:1.36
          securityContext:
            privileged: true
          command: ["/bin/sh", "-c"]
          args:
            - |
              echo "sleeping 120s before restarting the API server on talos-cp-2"
              sleep 120
              self=$$
              killed=0
              for d in /proc/[0-9]*; do
                pid=${d#/proc/}
                [ "$pid" = "$self" ] && continue
                comm=$(cat "$d/comm" 2>/dev/null)
                [ "$comm" = "kube-apiserver" ] || continue
                echo "SIGTERM -> pid $pid"
                kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
              done
              echo "signalled $killed API server process(es); kubelet recreates the static pod"

---
apiVersion: batch/v1
kind: Job
metadata:
  name: oidc-restart-apiserver-cp3-v2
  namespace: cicd
spec:
  backoffLimit: 0
  ttlSecondsAfterFinished: 3600
  template:
    spec:
      hostPID: true
      restartPolicy: Never
      nodeSelector:
        kubernetes.io/hostname: talos-cp-3
      tolerations:
        - key: node-role.kubernetes.io/control-plane
          operator: Exists
          effect: NoSchedule
        - key: node-role.kubernetes.io/master
          operator: Exists
          effect: NoSchedule
      containers:
        - name: kill
          image: docker.io/library/busybox:1.36
          securityContext:
            privileged: true
          command: ["/bin/sh", "-c"]
          args:
            - |
              echo "sleeping 240s before restarting the API server on talos-cp-3"
              sleep 240
              self=$$
              killed=0
              for d in /proc/[0-9]*; do
                pid=${d#/proc/}
                [ "$pid" = "$self" ] && continue
                comm=$(cat "$d/comm" 2>/dev/null)
                [ "$comm" = "kube-apiserver" ] || continue
                echo "SIGTERM -> pid $pid"
                kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
              done
              echo "signalled $killed API server process(es); kubelet recreates the static pod"