fix(oidc): remove the API-server restart Jobs from GitOps
The oidc-bootstrap ArgoCD Application runs with selfHeal and prune. Three Jobs in restart-apiserver.yaml kill the kube-apiserver process on each control plane, and bootstrap.yaml carried a fourth Job that deletes the mirror pods. ArgoCD recreated a deleted Job within two minutes, so the kill repeated and every API server reached CrashLoopBackOff. Remove all four. The OIDC authenticator already loaded, so the one-shot restart is not needed again.
This commit is contained in:
@@ -31,29 +31,3 @@ roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: oidc-bootstrap
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: oidc-bootstrap-restart-apiserver-v2
|
||||
namespace: kube-system
|
||||
spec:
|
||||
backoffLimit: 4
|
||||
ttlSecondsAfterFinished: 7200
|
||||
template:
|
||||
spec:
|
||||
serviceAccountName: oidc-bootstrap
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: restart
|
||||
image: docker.io/curlimages/curl:8.22.0
|
||||
command: ["/bin/sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
set -u
|
||||
TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
|
||||
CA=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
||||
for pod in kube-apiserver-talos-cp-1 kube-apiserver-talos-cp-2 kube-apiserver-talos-cp-3; do
|
||||
curl -sS --max-time 15 --cacert "$CA" -H "Authorization: Bearer $TOKEN" -X DELETE "https://kubernetes.default.svc/api/v1/namespaces/kube-system/pods/$pod?gracePeriodSeconds=0" -o /dev/null -w "$pod -> HTTP %{http_code}\n" || echo "$pod -> request failed"
|
||||
done
|
||||
echo "deletes requested; kubelet recreates each static pod in ~20-40s"
|
||||
@@ -1,141 +0,0 @@
|
||||
# One-shot: restart the control-plane API servers so their OIDC authenticator initialises.
|
||||
#
|
||||
# Two earlier mechanisms failed and are documented so nobody repeats them:
|
||||
# 1. deleting the mirror pod does NOT restart a static pod (deletes returned 200, no restart);
|
||||
# 2. matching /proc/*/cmdline for 'kube-apiserver' also matched this script's own cmdline,
|
||||
# so every pod SIGTERMed itself (exit 143) before reaching the real process.
|
||||
# This version matches /proc/<pid>/comm (the process name) and skips \$\$, so self-kill is
|
||||
# impossible. hostPID + privileged are permitted in the 'cicd' namespace. Delays stagger the
|
||||
# three control planes by 2 minutes each, so only one API server is down at a time.
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: oidc-restart-apiserver-cp1-v2
|
||||
namespace: cicd
|
||||
spec:
|
||||
backoffLimit: 0
|
||||
ttlSecondsAfterFinished: 3600
|
||||
template:
|
||||
spec:
|
||||
hostPID: true
|
||||
restartPolicy: Never
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: talos-cp-1
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
- key: node-role.kubernetes.io/master
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
containers:
|
||||
- name: kill
|
||||
image: docker.io/library/busybox:1.36
|
||||
securityContext:
|
||||
privileged: true
|
||||
command: ["/bin/sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
echo "sleeping 10s before restarting the API server on talos-cp-1"
|
||||
sleep 10
|
||||
self=$$
|
||||
killed=0
|
||||
for d in /proc/[0-9]*; do
|
||||
pid=${d#/proc/}
|
||||
[ "$pid" = "$self" ] && continue
|
||||
comm=$(cat "$d/comm" 2>/dev/null)
|
||||
[ "$comm" = "kube-apiserver" ] || continue
|
||||
echo "SIGTERM -> pid $pid"
|
||||
kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
|
||||
done
|
||||
echo "signalled $killed API server process(es); kubelet recreates the static pod"
|
||||
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: oidc-restart-apiserver-cp2-v2
|
||||
namespace: cicd
|
||||
spec:
|
||||
backoffLimit: 0
|
||||
ttlSecondsAfterFinished: 3600
|
||||
template:
|
||||
spec:
|
||||
hostPID: true
|
||||
restartPolicy: Never
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: talos-cp-2
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
- key: node-role.kubernetes.io/master
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
containers:
|
||||
- name: kill
|
||||
image: docker.io/library/busybox:1.36
|
||||
securityContext:
|
||||
privileged: true
|
||||
command: ["/bin/sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
echo "sleeping 120s before restarting the API server on talos-cp-2"
|
||||
sleep 120
|
||||
self=$$
|
||||
killed=0
|
||||
for d in /proc/[0-9]*; do
|
||||
pid=${d#/proc/}
|
||||
[ "$pid" = "$self" ] && continue
|
||||
comm=$(cat "$d/comm" 2>/dev/null)
|
||||
[ "$comm" = "kube-apiserver" ] || continue
|
||||
echo "SIGTERM -> pid $pid"
|
||||
kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
|
||||
done
|
||||
echo "signalled $killed API server process(es); kubelet recreates the static pod"
|
||||
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: oidc-restart-apiserver-cp3-v2
|
||||
namespace: cicd
|
||||
spec:
|
||||
backoffLimit: 0
|
||||
ttlSecondsAfterFinished: 3600
|
||||
template:
|
||||
spec:
|
||||
hostPID: true
|
||||
restartPolicy: Never
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: talos-cp-3
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
- key: node-role.kubernetes.io/master
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
containers:
|
||||
- name: kill
|
||||
image: docker.io/library/busybox:1.36
|
||||
securityContext:
|
||||
privileged: true
|
||||
command: ["/bin/sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
echo "sleeping 240s before restarting the API server on talos-cp-3"
|
||||
sleep 240
|
||||
self=$$
|
||||
killed=0
|
||||
for d in /proc/[0-9]*; do
|
||||
pid=${d#/proc/}
|
||||
[ "$pid" = "$self" ] && continue
|
||||
comm=$(cat "$d/comm" 2>/dev/null)
|
||||
[ "$comm" = "kube-apiserver" ] || continue
|
||||
echo "SIGTERM -> pid $pid"
|
||||
kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
|
||||
done
|
||||
echo "signalled $killed API server process(es); kubelet recreates the static pod"
|
||||
|
||||
Reference in New Issue
Block a user