Compare commits

..
20 changed files with 30 additions and 682 deletions
+1 -1
View File
@@ -48,7 +48,7 @@ spec:
mountPath: /backup mountPath: /backup
containers: containers:
- name: mc-mirror - name: mc-mirror
image: docker.io/bitnamilegacy/minio-client:2025.7.21-debian-12-r3@sha256:73bd39f7899a0cef12b8dd5df13aa93a3ed1aaa44236542442e9ac76819ac158 image: quay.io/minio/mc:latest
env: env:
- name: ACCESS_KEY - name: ACCESS_KEY
valueFrom: valueFrom:
-36
View File
@@ -1,36 +0,0 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: azhi-riotpiao
namespace: argocd
annotations:
argocd.argoproj.io/sync-wave: '8'
spec:
project: homelab
source:
# Private GitHub. The read-only credential is Secret azhi-infra-github in
# namespace argocd, created out-of-band.
repoURL: https://github.com/ToyoGlobal/azhi-infra.git
# The repository's default branch. It is not `main`: this repo's work
# happens on the staging slice branch, and `main` is behind it.
targetRevision: harness/staging-slice
# One directory, because ArgoCD reads a single path per source. That
# directory is a root kustomization over foundation, postgres and
# azhi-server.
path: k8s/overlays/azi-riotpiao
destination:
server: https://kubernetes.default.svc
namespace: azi-riotpiao
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true
# The two Secrets this namespace needs are credentials, so they are created
# by hand and NOT by this Application:
# acr-pull — the private Alibaba ACR pull credential
# azhi-riotpiao-app-env — DB credentials, JWT secrets, AES key
# A missing imagePullSecret fails the pod outright and a missing app-env
# Secret leaves it in CreateContainerConfigError, which is the intended
# loud failure rather than a pod that runs against unknown values.
-20
View File
@@ -1,20 +0,0 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: oidc-bootstrap
namespace: argocd
annotations:
argocd.argoproj.io/sync-wave: "5"
spec:
project: homelab
source:
repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
targetRevision: main
path: k8s/infra/oidc-bootstrap
destination:
server: https://kubernetes.default.svc
namespace: kube-system
syncPolicy:
automated:
prune: true
selfHeal: true
-6
View File
@@ -16,12 +16,6 @@ spec:
- https://github.com/Riotpiaole/Poimen-memory.git - https://github.com/Riotpiaole/Poimen-memory.git
- https://github.com/Riotpiaole/Poimen-workflows.git - https://github.com/Riotpiaole/Poimen-workflows.git
- https://github.com/Riotpiaole/poimen*.git - https://github.com/Riotpiaole/poimen*.git
# ToyoGlobal (GitHub, private). The azhi-infra Application reads its
# manifests from here, so the URL must be listed or ArgoCD rejects the
# Application with "repository not permitted". A read-only credential for
# this repo lives in Secret azhi-infra-github in namespace argocd; it is
# created out-of-band and is not committed, because it is a credential.
- https://github.com/ToyoGlobal/azhi-infra.git
# In-cluster Forgejo repos — explicit allowlist (no wildcard) # In-cluster Forgejo repos — explicit allowlist (no wildcard)
- https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git - https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
- https://forgejo.riotpiao.com/riotpiao-poimen/homelab-frontend.git - https://forgejo.riotpiao.com/riotpiao-poimen/homelab-frontend.git
@@ -1,27 +0,0 @@
apiVersion: v1
kind: Secret
metadata:
name: etcd-defrag-creds
namespace: kube-system
labels:
app.kubernetes.io/name: etcd-defrag
type: Opaque
stringData:
ca.crt: ENC[AES256_GCM,data:vso/Ouz5QRoRF2CFj9TsGYhoGkX8oGJIeiFCoS92AAQbyyGPTiZ/fPNFM03p+TpeKDb+Z+Fyo/HXuDeT5Fee5kjPnUHXf2hiUBmJLyHnPMp5i57dRzTC8X4VaNeV/1MmP2WpkWwieWOGBxJVIIrgeQhQexxmIzSlQ5Io9dVOH8F6iyakwdf7qfLFGdoer3rfrXadSOJtKRpLneFJDL2OmR1042LZ8KtNuRRXI/qrdpgr2bnyxFuCRc+tk5QY73mpwwWfKGn6FW/rVdjiEftkC+AvTUItx7RkYMrz0PlTYPUOLustEF2gIgWheYZhE5ebZDurVSG//sudjASliYJItwgSk3G5mZihOSDA7Jcus+U8p1iHQ9CQsey3CSAMmuSGRFmq42kI8135kjtBYrzVswHjX1/51qPMyA2TIeWlRJ7AJlHihG317gT1vPeM1ULl3PkuCxaBLCtjotZBzKLJ/6zXr1ig77Baz71YOVB0D0MgFvzIdao+mdxS8WLOZdiXOCwbI5Y2eo7wJHG4BnpKEdx1rjPkv7DI2NNEoXA7e5FvJWinyAxOy6YGH4Fv6dTlEVhI3Nl/1X+CeafL6Hk8YoqA04Hm+/dJCZi7SsF0r6zfPNNsZLivM8DN18h8KYM3eVY3CwtBldE7atz+oprQ6K/yGUbblHRQwuG3sH3AHrCKLPiUrMIi0jWc0IpdPZRZFL7SnMULfV05QgcvGySOJCuQU+/OYzrRs7umOjwpqvi9PyJYVXBliiYfJutvmfpFAhQj,iv:KnW9LYNzxZ6r3Q6LTeOrgSLLPgiJlibVp6YmTEKBkjE=,tag:KTL1nOWdNWDA4dLVaA4//g==,type:str]
client.crt: ENC[AES256_GCM,data:IEmZe505SwkW3U5vmRtFxR6PpuSFGWEHflUZwyhSODzYG1zLM8olVWDtH+yUVI54kLXxTtcpMkKrTBjpAxHG8zUTLPdTCVzPMq6U20jH4PYFMzoldnSgpRl/eiADUmE4I32CBbPdM5nkPvjVEYMmRafEGyJctjG6Shhh/sUO1yE9A+h1R7o3UCRPttWTozSKN4dPi75v1TSAOLyXqvQYl5Ea9iKW47ea7Q+XTcoKfMPXsTJp44kiZnVW0VT/7xNydR7AVWJJzYltY5380j0Ho+7PfNd+QqmdodkaZPS4b0PaoinMiZ40JnChGVhdS91j3IkUlU1g/+G2iVnPR9KB07xqRubqiASxIvpSaSCeYl3rR7+dXYX18VpWcWnJ6XoREoqxJiqmMQWYSROYDTd4G/zs4Usuh5r0wIodmCjud+g89mmXFtH0PbsMCVvZDUswjVfeT2PUBm036TXlxF/1aAy2GDqycbPgJgDLyw9tdiQUZe2iA+Z/Wx80qgQjmmqCaw77cDergLFj9KnByl3Tinwfm+IWFeQiDW+fwSzz+/vrpl55BkSVkVhw7r2Jd6zyg4VtLVXbxYhzdNz695N+fjkZHuPE5tCwr4WCzAg3vSS9hA7IHo8k93WO8s6+KlyjuwTdjPIYZghebO4ktwdqvhvuqxv8Nzwpc5J7Z2H2IChjXg==,iv:l8BwxxEfHfYrm+mM5UE6Yj42DeeU9y23jjIE1KjIiME=,tag:HwRT6/myIsxc84BYDj3K6w==,type:str]
client.key: ENC[AES256_GCM,data:QD/Glf3lFslBEdEoJ3CNrYzlw2HLCvGSeVuJLtxnFne2aJOuw0sk/e0h/kSAxvG4dZ+LIWJLLbqTjKHOAhkvS/ZOA7GSea4ubNkvTkw8+JBRuYwb2D2iB8kZhlER/CpiGkYJukjkgGP1CzHdvBY1M88pW9w6rH74MD3jY47qExIaNduQj5vkUQZ8ajqkd79l0nk98ZTawfPG4AFcEtAIKAPIuubT2GZ3qqNLjOI7agbtRv8xI44hhb2wj5J9xdl1GrbOWwG5EOo+hLNrcAAOCcWBpNTUCKBy7mrk2ID8jX3HsFWN76DDlQzkhcphMhlM7g==,iv:49HGJAEgllEkB3YSiZLmbWs/vlbFbY52kiKesdGVC/Y=,tag:g8YZ9at3V2l6RFRfvO2lAQ==,type:str]
sops:
age:
- enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4Z0h0R2dhbDdRcFJydE9o
R0hWeCtwVzluZWZlR0VtOXcxdW9FbklOQXlzCmo3eXNyNlFjejNETkJiVVRVWFhr
a2pORnhIQTI0UUk2N3dWMDZJcEhYU1UKLS0tIEduaHlmMHQza1A0TGVZb0JrbTYz
MWJwQjcrdkkzQXhOTmhycERsM0FtNVEKG+gBRcvUsyfRzuW999EQNzyDh2/5fY6p
8ab2xvTOJPiMUBTBuQZsA+iDImytUO/02ROQsbEZkALijp1nqrsV/g==
-----END AGE ENCRYPTED FILE-----
recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
encrypted_regex: ^(data|stringData)$
lastmodified: "2026-10-07T05:37:26Z"
mac: ENC[AES256_GCM,data:vu6ltV0jYQNGV4eqIcqAp9p2TJ8ULBX5JBELUgm/nA9GX+I6Pv3fyjpcHRSWY0HbLvCwgXd22CG4IYcj4HJFtQGZR7OcQA5WM5plZH3RCPdvPNix/798vkETfpJxorQjlHoq4nlr3dEmUCbzdtE07qV2W1M6GJr4C9xKvB947bQ=,iv:aA9mjZCFAoLEhGqkl6NfyBZ3x3vWGfLYYAEe8jXuA4U=,tag:Drdiji33ckDd8ildpvSEXw==,type:str]
version: 3.13.2
@@ -1,24 +0,0 @@
apiVersion: v1
kind: Secret
type: Opaque
metadata:
name: forgejo-registry-token
namespace: cicd
data:
username: ENC[AES256_GCM,data:oRNecs0a7rs=,iv:/5EFdNqpSjoxe7XDmAEDMd6lXd0SMsRiMmBM9SIwDHY=,tag:Wi+ggoP+9/vgz9rHrxB9nA==,type:str]
password: ENC[AES256_GCM,data:HbmwnWk3ST5SDWXwIhu+M5f/FOfFbJHaG0Q/IWrO63HRXmJTmLMpbjtWIXC9IMFoHeDZ1MNAV8A=,iv:ztJZgpaS//yYCiyr6/ubs/UiXkbGMnokvy4Pe45jQMk=,tag:WMpawc/SVAV4mxsCd6XqAA==,type:str]
sops:
age:
- enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBjQmhxN2E3ZnNTdkR6U2c1
ZStsQktkeWUwMEdmdFIyMkRYRVk1ZGppWlZ3CmUzR2JubWw2VjdWUGt6S2lpNmQ3
cDdSWjJiYzRIWS9UVERpVDFJenFnaDAKLS0tIGhDSmNMSjZwcDVPR3U0dUcwNk9Q
WmdGS1lyS3dkSXIvOFhyRklGRll2UlUK8NGts6a6Fv10c6gxIIPqIIqfgzQ3EEyk
SFrMThwbdaOqLY4AMeDrg+k0Y2t1Xj/XOvdNbRCRictDwZUsptvO7A==
-----END AGE ENCRYPTED FILE-----
recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
encrypted_regex: ^(data|stringData)$
lastmodified: "2026-10-06T02:01:24Z"
mac: ENC[AES256_GCM,data:imebczVzG9YWd42ER4a+w+Bi3dxbN0HFm/1igxzJt6Z0HG/q6i0s59swpGVruk9dQJWrlO8KKqX739ATp2KGC5szrEuRFvPemF594XFCg6ykW+ZeP5VtTvpJg2nHv7/Pu+71ljKKi4gqOJ4SJDn8owAbSN+su6PiT323wax930A=,iv:iG/G2+qQkvbBU95w/GG2PrbAHDOaamuvjqMDBxSW6zI=,tag:wy9Vn72HBDHqCyzTcV15/Q==,type:str]
version: 3.13.2
@@ -1,26 +0,0 @@
apiVersion: v1
kind: Secret
type: Opaque
metadata:
name: memory-agent-oidc
namespace: poimen
data:
CLIENT_ID: ENC[AES256_GCM,data:c/RvtV6anofX8h1BmlKu9w==,iv:3lQ1BR8kVvF1RHVfKstmb0Y7u/uX5OrvmGiCi/xTpjw=,tag:9JC2WkfQFWaiZroAEmPenw==,type:str]
CLIENT_SECRET: ENC[AES256_GCM,data:I258/6O43PccyllYN/wfoKvTgNgZONTivM/KFIr8TGqSl+2XQkwBXvK2VQc5Xgc5cpwlrgnjk8pjQbqr,iv:MZyA+V5z9Po2VoxqvOSlWh1fZp1lrkcbYMx7VcSIaJs=,tag:gXH0BpA8KOZiVd+4RnhRzg==,type:str]
ISSUER: ENC[AES256_GCM,data:GX/uQPDoeJWXXQZlzD6oOIvxo7H4LRAv4MrPAiCgmv6WxW59+KKkymuDT/T9MiDm4bNhxVPgB7nHbDcMHfPvU2zCNM2TWYUsNOgGwxNAqM8=,iv:5OEsdrUZvMLO2C8Ea2i5alsyFGtmEk3othTemk8fOk4=,tag:8CMlRY2JCe4z4fv4WuWjRQ==,type:str]
TOKEN_URL: ENC[AES256_GCM,data:KeU1pQzvQ2/1lQhOqNtcR7t5TGVx1J7NY8ogxCXS6IuHhVe0zWYIbC4BYFc2H9PAjsMJwCCg75N1FrIiCYBvpfK8ln0=,iv:NbPiJC/B7W1Zm+JwnEGDikE2cb3L4l6TBeQTKEb2boM=,tag:FRmllFLZzQW03lsCu9mAUA==,type:str]
sops:
age:
- enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBmK0dPSmhKT01FSitDdUNW
Um1XQU9FVnBUeDdoWXpCcXRzVXh0VXkvNGxNClZaQ3R5U2NkYTJjaFdHRWhUVmFS
K0tHZVRFRDZGREVRZzZqcWJ6dnRpUUEKLS0tIE15UUNtMStKbWwzb3dEVGEwN1VQ
cktlVTM2SUhpOGJZMXJOOG84RnhCQzAKOdM0pThEaN37SJtRrKIivncMu8Z91Btq
es6NOlJdBUhzIhnEJWfDzZ9aA6I/fYzugKbC+AfUwdGgThAgHx9crg==
-----END AGE ENCRYPTED FILE-----
recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
encrypted_regex: ^(data|stringData)$
lastmodified: "2026-10-06T02:01:24Z"
mac: ENC[AES256_GCM,data:/BTQ984D+hsk0LmnjD3hxopB65NAR54LhoAPNcATWA7MCTJXfdnYRu3gNxUzyiyCBd+rDaPp77TBnak4jVjMnOiYJsYLj1R7hrrhTZopz8pT+YNMupperwsirKZwK8it4Mu+fnpCyox2mtGED0hk+Nh/b6IteTRt5lS7noumo2M=,iv:RWIrVkNYj+8mqMSPiLueToexabzAtVbIR525m6bUrlg=,tag:5+wcNoNYsjGKpFlML+N70Q==,type:str]
version: 3.13.2
@@ -1,27 +0,0 @@
apiVersion: v1
kind: Secret
type: Opaque
metadata:
name: oauth2-credentials
namespace: iam
data:
api-gw-client-secret: ENC[AES256_GCM,data:YiUCk61cCsTJE8gTLZWhqVPsCSu3b4OnG+Oh456A+mnxip0FjLTTqV+kYBdfZivIkn9ua4RcVW+rYILu,iv:K08m1EaB7/SIVNWZ3FfeRP4tbCAo6KeiNC0NpE9VO1o=,tag:ZXV9YRWUOPJ/vdoqMtvQeA==,type:str]
minio-client-secret: ENC[AES256_GCM,data:vcwLo1cJqbZr2MwkAKO09xPjb09ss7+s6HIvEfaLxt8ka5PUAlXQ37pTkynuktlKq/t0DvSk+x1+5G7QqVD1JJlwyDGM6yuOhBQ3MwucLNl+T4RDKtgGuA==,iv:sCdd+UNX2TfTAM6Zslrh8KrvBSzqpe67F3YMGrl4YDU=,tag:9Tpzm9NEXNuXyCd7l7GeHw==,type:str]
poimen-client-secret: ENC[AES256_GCM,data:dYtCiMGZaOFokihfGD2SXJDfXrhfwEQcTOPNu7JNHDUnx0hqwAdYAPAJTEN21p0OpHYDINMs5lwUpMwY,iv:ll8USuzs0881ryHImVjHGO0iFxUzqg5XtI6Ekd+EFXE=,tag:I/4ZfB9Ss4zEP57sRtRTXQ==,type:str]
paperless-client-secret: ENC[AES256_GCM,data:XMj9rHkI20lI37vChq9wS5lUq7l95SisRX0LChuj09O5w76Zt9NdG6OEUYQFf1z8vCfMJxBInIEEIq0e,iv:UpYy2ZEZ4AgBACEyrlxEcr13V+8Ksu4p8JHYApmyYHo=,tag:o9Z+/b6T9POgowIsks56kw==,type:str]
grafana-client-secret: ENC[AES256_GCM,data:V/j0+P3HoDLqYjRalWK4v1hvJTuVmzxVra47HsF1/BZvWZjIyEIH3Pt8BekT3O1dUFynxjcaxdZMBVYg,iv:9pMp+vIsp/sA7GP9wqNDf5DdZVLDow46OURW0mMRXIc=,tag:i7VJFA5o0hToZEJ5JPWIvw==,type:str]
sops:
age:
- enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2amZPendIOU9PeStOcmJG
c1dzOFVvd3RoT3dXQlJUNkdCSytEZUVGVEJJCklMUkNjUytBMXh0OVUrNUlpMWw3
aHRFT1hBZTZHNHVtaGJCTEFyT2ZWNWsKLS0tIENxR3FKMEplV0V5Y0dsZzBkODVG
U0ZlT2R0VFdEQmpsakVvZ1FJRkRQYTAKESQcSlMAuCG0/tLWWZaeoPayf1EKxSaf
e4o8TmjmSgPEhepQ/LmNXl6obyuqUxztkh4Mm6pocqDp+gfzt7X/1g==
-----END AGE ENCRYPTED FILE-----
recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
encrypted_regex: ^(data|stringData)$
lastmodified: "2026-10-06T02:01:24Z"
mac: ENC[AES256_GCM,data:AI/znHiUa/Rwg0YRz65qUZ+b6V73oi3ZfAG/V5uZZvrGFrA0EISzD5LaP5Kl4GUU1svo4G5EI4aftPDgpdPptz21CCa8iYkXGe9pTDmtaE5ZICQAUbRcVhGMTfuWFhrgEhMfCJdOpfHaIQVEsQJQJgrnkXQVuUe9aT75LQz7pnY=,iv:KbIaYT0VeBT2WeSU5V5ZB4e+04JICYaY+egGN1Pv5bM=,tag:pPbhDxeLUx6V1XC/LPwxTg==,type:str]
version: 3.13.2
@@ -1,24 +0,0 @@
apiVersion: v1
kind: Secret
type: Opaque
metadata:
name: pi-code-agent-api-keys
namespace: poimen-sandbox
stringData:
ANTHROPIC_API_KEY: ENC[AES256_GCM,data:TBcno/dzj3qykHjeX96UKZekuA/WMQT2E7a1ziPyYo5J6lWLPydgli84KeJZhH62/8CnnFtFOhVIe4FWCGPGjvqKH6M8WbiQHaP9y7FN0qQUpFHKNVAQ2R+s0wInhw0eUmANvd7bRx8aIZ2X,iv:oex19yVtWUevoJ0A3mkMLCfWu85o1gPSNp43x1iV0LE=,tag:BW3zQnwvagPceeNird9CZg==,type:str]
DEEPSEEK_API_KEY: ENC[AES256_GCM,data:vq/5gOsMVjlVMd0oXiBtHFHFEr+IF2YR2iP2OBFWx3ZFNoQ=,iv:QhLU/BRIJb/7QKDCRyw+HWPphZifVEhIOLywYDgOb20=,tag:BZdtXw+WsuK+P4E0caTa0Q==,type:str]
sops:
age:
- enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqNm1LNE5nWktnd3c3STg3
UlVWbWJxWElxUHMwa0JVMk81Z0c4d0JaQm1vCnpLQWJWV3hxOU00NzJIaG51Mmlr
ZWpNTFJWcmdtV0w4YndQeHpvSUx2MTAKLS0tIDVnNkNuVWhPUFZ5V2hFOEt1eitG
blVqRy8yNnYzSU1FNzZGNkxrRElvWVEKL3J77INje75+CvKXaJHC3CvONgHv7d0a
a10MITvfreFr6M4AN3Z/mno49SDMp15Q4oQRagHHxSTS1K54mWHo8g==
-----END AGE ENCRYPTED FILE-----
recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
encrypted_regex: ^(data|stringData)$
lastmodified: "2026-10-06T01:19:49Z"
mac: ENC[AES256_GCM,data:GpYl8IRefms36GvO3RW41c0US7SOgoOhd4rumxsCdEf5ME9kJ0eOxPx8klMgnlYrkBnwHmad8jj7C+HcA5Eim7mdUaYOZZWPFohvC6N4GLHX85aH78P09CelqG5tdm/Gm+1QfxYu0vANYGCAVjLK7YbBbZjbZmbvVfQvAbs3mOY=,iv:y3lglGu8FhTHQpO1poSIKgGYJFu5J9yjIaOjulda6fY=,tag:5Sgh4TbtNFMw7xOJjYRn0g==,type:str]
version: 3.13.2
-5
View File
@@ -33,8 +33,3 @@ files:
- gotify-smtp-secrets.enc.yaml - gotify-smtp-secrets.enc.yaml
- forgejo-smtp-secrets.enc.yaml - forgejo-smtp-secrets.enc.yaml
- paperless-ai-secrets.enc.yaml - paperless-ai-secrets.enc.yaml
- memory-agent-oidc.enc.yaml
- oauth2-credentials.enc.yaml
- forgejo-registry-token.enc.yaml
- pi-sandbox-secrets.enc.yaml
- etcd-defrag-creds.enc.yaml
@@ -1,177 +0,0 @@
# etcd backend maintenance: compact, defrag, disarm NOSPACE.
#
# Why this exists
# ---------------
# etcd's backend file only grows. The apiserver compacts every 5m, but
# compaction marks superseded revisions free *inside* the file; only `defrag`
# returns that space to the filesystem. Nothing ran defrag, so the file grew
# until it crossed the 2 GiB quota:
#
# 2026-10-07 backend 2,148,843,520 B (2.1 GB) on talos-cp-1 and talos-cp-3
# -> etcd raised NOSPACE on those two members
# -> etcd rejected ALL writes cluster-wide
# -> kubelets could not post node status: 3 nodes NotReady
# -> apiserver writes returned 500, the tunnel API path 502
#
# The live data was tiny: 15,943 keys, 69 MB after compaction. The rest was
# dead pages. Recovery was `compact <current-revision>` then `defrag` per
# member then `alarm disarm`. Compaction had also stalled while the alarm was
# up, so `dbSizeInUse` read 100% of the file - defrag alone would NOT have
# worked. Order matters: compact first, then defrag.
#
# The earlier revision of this file never ran, for two independent reasons:
# 1. it existed only on branch fix/etcd-defrag-cron; the ArgoCD application
# `cluster-maintenance` tracks `main` (targetRevision: main), so the
# cluster never had the CronJob or the Secret.
# 2. the image was quay.io/coreos/etcd, which has NO /bin/sh (verified:
# `exec: "/bin/sh": no such file or directory`). Any container that runs
# a shell script needs an image that ships one.
# This revision fixes both: image bitnamilegacy/etcd (ships bash + etcdctl
# 3.6.4, matching the 3.6.11 servers) pinned by digest.
#
# Cadence: every 15 days. Measured growth is ~29 MB/day, so a 15-day cycle
# peaks near 500 MB - comfortably under the 2 GiB quota. The schedule is daily
# and the script exits unless the UTC epoch day is a multiple of 15. That gives
# an exact 15-day interval; `*/15` in the day-of-month field does not (it fires
# on the 1st, 16th and 31st, which puts two runs one day apart).
#
# Safety
# ------
# 1. Compaction uses the MINIMUM revision across members. Compacting to a
# revision a member has not reached wedges that member - measured on
# 2026-10-07, when talos-cp-2 froze ~1.2M entries behind the leader and
# needed a reboot to rejoin.
# 2. A member whose raft applied index lags the leader by more than 100000 is
# skipped and reported. Defragging a frozen member reclaims nothing.
# 3. Members are defragged one at a time. A defrag pauses the member's apply
# loop for the duration of the file copy.
#
# Credentials: Secret etcd-defrag-creds in kube-system, SOPS-managed in
# k8s/argocd/secrets/etcd-defrag-creds.enc.yaml (ksops generator 00-secrets).
# Its client certificate is signed by the cluster's etcd CA and expires
# 2036-10; rotate it before then by re-running the mint and re-encrypting.
#
# Companion alerts (monitoring): etcd_mvcc_db_total_size_in_bytes /
# etcd_server_quota_backend_bytes > 0.8, and an etcd NOSPACE alarm alert.
apiVersion: batch/v1
kind: CronJob
metadata:
name: etcd-defrag
namespace: kube-system
labels:
app.kubernetes.io/name: etcd-defrag
app.kubernetes.io/part-of: cluster-maintenance
spec:
schedule: "0 3 * * *" # daily; the script acts only every 15th day
concurrencyPolicy: Forbid
startingDeadlineSeconds: 600
successfulJobsHistoryLimit: 3
failedJobsHistoryLimit: 3
jobTemplate:
spec:
backoffLimit: 2
template:
spec:
restartPolicy: OnFailure
tolerations:
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
containers:
- name: defrag
image: bitnamilegacy/etcd:3.6.4@sha256:c86c06f1ce6a08b6df21eb7a9655b1818778b64c85b5cbb8413ebf7534789730
env:
- name: ETCDCTL_API
value: "3"
- name: ETCDCTL_CACERT
value: /etc/etcd/ca.crt
- name: ETCDCTL_CERT
value: /etc/etcd/client.crt
- name: ETCDCTL_KEY
value: /etc/etcd/client.key
- name: EPS
value: "https://192.168.1.166:2379 https://192.168.1.214:2379 https://192.168.1.162:2379"
command: ["/bin/bash", "-c"]
args:
- |
set -u
DAY=$(( $(date -u +%s) / 86400 ))
if [ $(( DAY % 15 )) -ne 0 ]; then
echo "skip: UTC epoch day $DAY is not a 15-day boundary"
exit 0
fi
echo "== before =="
for ep in $EPS; do
etcdctl --endpoints="$ep" endpoint status -w table || true
done
MINREV=""
MAXAPPLIED=0
for ep in $EPS; do
json=$(etcdctl --endpoints="$ep" endpoint status -w json 2>/dev/null) || continue
rev=$(printf '%s' "$json" | sed -n 's/.*"revision":\([0-9]*\).*/\1/p' | head -1)
app=$(printf '%s' "$json" | sed -n 's/.*"raftAppliedIndex":\([0-9]*\).*/\1/p' | head -1)
echo "member $ep revision=${rev:-?} raftAppliedIndex=${app:-?}"
if [ -n "$rev" ] && { [ -z "$MINREV" ] || [ "$rev" -lt "$MINREV" ]; }; then
MINREV="$rev"
fi
if [ -n "$app" ] && [ "$app" -gt "$MAXAPPLIED" ]; then
MAXAPPLIED="$app"
fi
done
if [ -z "$MINREV" ]; then
echo "FATAL: no member returned a revision; nothing attempted"
exit 1
fi
echo "== compact to revision $MINREV =="
etcdctl --endpoints="$(echo "$EPS" | awk '{print $1}')" compact "$MINREV" || echo "COMPACT-FAILED (continuing: defrag still reclaims free pages)"
# Physical deletion of superseded keys runs in the background
# after the compaction commit; give it a moment to drain.
sleep 20
echo "== defrag each member (sequential, one at a time) =="
rc=0
for ep in $EPS; do
json=$(etcdctl --endpoints="$ep" endpoint status -w json 2>/dev/null) || { echo "SKIP $ep (unreachable)"; rc=1; continue; }
app=$(printf '%s' "$json" | sed -n 's/.*"raftAppliedIndex":\([0-9]*\).*/\1/p' | head -1)
if [ -n "$app" ] && [ "$app" -lt $(( MAXAPPLIED - 100000 )) ]; then
echo "SKIP $ep (applied index $app lags leader $MAXAPPLIED; defrag would reclaim nothing)"
rc=1
continue
fi
echo "-- $ep"
if ! etcdctl --endpoints="$ep" defrag; then
echo "DEFRAG-FAILED $ep"
rc=1
fi
done
echo "== disarm NOSPACE =="
for ep in $EPS; do
etcdctl --endpoints="$ep" alarm list 2>/dev/null || true
etcdctl --endpoints="$ep" alarm disarm 2>/dev/null || true
done
echo "== after =="
for ep in $EPS; do
etcdctl --endpoints="$ep" endpoint status -w table || true
done
etcdctl --endpoints="$(echo "$EPS" | awk '{print $1}')" alarm list || true
exit $rc
resources:
requests: { cpu: 10m, memory: 32Mi }
limits: { cpu: 200m, memory: 128Mi }
volumeMounts:
- name: creds
mountPath: /etc/etcd
readOnly: true
volumes:
- name: creds
secret:
secretName: etcd-defrag-creds
defaultMode: 0444
@@ -52,14 +52,14 @@ spec:
restartPolicy: Never restartPolicy: Never
initContainers: initContainers:
- name: kubectl-copy - name: kubectl-copy
image: bitnami/kubectl:latest@sha256:f7f9e4f64d9e114650c115a4ac6fd383394b3d494061a8f17b1a0f6c8d55bc25 image: bitnami/kubectl:latest
command: ["sh", "-c", "cp $(which kubectl) /shared/kubectl"] command: ["sh", "-c", "cp $(which kubectl) /shared/kubectl"]
volumeMounts: volumeMounts:
- name: shared - name: shared
mountPath: /shared mountPath: /shared
containers: containers:
- name: provision - name: provision
image: docker.io/bitnamilegacy/minio-client:2025.7.21-debian-12-r3@sha256:73bd39f7899a0cef12b8dd5df13aa93a3ed1aaa44236542442e9ac76819ac158 image: quay.io/minio/mc:latest
volumeMounts: volumeMounts:
- name: shared - name: shared
mountPath: /shared mountPath: /shared
+1 -4
View File
@@ -16,10 +16,7 @@ spec:
# backend could not resolve minio-cluster-hl. Official images now need registry auth # backend could not resolve minio-cluster-hl. Official images now need registry auth
# (quay.io/minio/minio returns 401), so this pins a public mirror of the same upstream # (quay.io/minio/minio returns 401), so this pins a public mirror of the same upstream
# release; swap it for the official image once MinIO registry credentials exist. # release; swap it for the official image once MinIO registry credentials exist.
# The digest freezes those bytes, so a retag or a deletion on the mirror cannot break image: coollabsio/minio:RELEASE.2025-10-15T17-29-55Z
# the pull again. The same digest already sits in the containerd cache on talos-cp-1,
# so the first start needs no registry access.
image: docker.io/coollabsio/minio:RELEASE.2025-10-15T17-29-55Z@sha256:69b55a1c1c5dc285ce04db96689f5b2102317fc77a50680a1874ca6efd1c87f9
# Disable operator auto-TLS: MinIO serves plain HTTP internally on 9000. # Disable operator auto-TLS: MinIO serves plain HTTP internally on 9000.
# External TLS is terminated at nginx ingress (wildcard riotpiao-com-tls cert). # External TLS is terminated at nginx ingress (wildcard riotpiao-com-tls cert).
@@ -27,24 +27,6 @@ spec:
annotations: annotations:
summary: "{{ $value }} pod(s) stuck in Pending state for >10m" summary: "{{ $value }} pod(s) stuck in Pending state for >10m"
# ImagePullBackOff on a vendor-deleted image produced no alert for 13 days
- alert: PodImagePullFailure
expr: sum(kube_pod_container_status_waiting_reason{reason=~"ImagePullBackOff|ErrImagePull"}) by (namespace, pod) > 0
for: 10m
labels:
severity: critical
annotations:
summary: "{{ $labels.namespace }}/{{ $labels.pod }} cannot pull its image"
# Pod not ready
- alert: PodNotReady
expr: sum(kube_pod_status_ready{condition="false"}) by (namespace, pod) > 0
for: 15m
labels:
severity: warning
annotations:
summary: "{{ $labels.namespace }}/{{ $labels.pod }} is not ready"
# CrashLoopBackOff # CrashLoopBackOff
- alert: PodCrashLooping - alert: PodCrashLooping
expr: sum(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}) by (namespace, pod) > 0 expr: sum(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}) by (namespace, pod) > 0
-59
View File
@@ -1,59 +0,0 @@
# One-shot bootstrap: restart the control-plane apiservers ONCE so they pick up the
# OIDC authenticator (they last started before the Authentik provider existed, so OIDC
# tokens were never accepted -> every login got 401). Delete this whole directory after.
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: oidc-bootstrap
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: oidc-bootstrap
namespace: kube-system
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list", "delete"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: oidc-bootstrap
namespace: kube-system
subjects:
- kind: ServiceAccount
name: oidc-bootstrap
namespace: kube-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: oidc-bootstrap
---
apiVersion: batch/v1
kind: Job
metadata:
name: oidc-bootstrap-restart-apiserver-v2
namespace: kube-system
spec:
backoffLimit: 4
ttlSecondsAfterFinished: 7200
template:
spec:
serviceAccountName: oidc-bootstrap
restartPolicy: Never
containers:
- name: restart
image: docker.io/curlimages/curl:8.22.0
command: ["/bin/sh", "-c"]
args:
- |
set -u
TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
CA=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
for pod in kube-apiserver-talos-cp-1 kube-apiserver-talos-cp-2 kube-apiserver-talos-cp-3; do
curl -sS --max-time 15 --cacert "$CA" -H "Authorization: Bearer $TOKEN" -X DELETE "https://kubernetes.default.svc/api/v1/namespaces/kube-system/pods/$pod?gracePeriodSeconds=0" -o /dev/null -w "$pod -> HTTP %{http_code}\n" || echo "$pod -> request failed"
done
echo "deletes requested; kubelet recreates each static pod in ~20-40s"
@@ -1,141 +0,0 @@
# One-shot: restart the control-plane API servers so their OIDC authenticator initialises.
#
# Two earlier mechanisms failed and are documented so nobody repeats them:
# 1. deleting the mirror pod does NOT restart a static pod (deletes returned 200, no restart);
# 2. matching /proc/*/cmdline for 'kube-apiserver' also matched this script's own cmdline,
# so every pod SIGTERMed itself (exit 143) before reaching the real process.
# This version matches /proc/<pid>/comm (the process name) and skips \$\$, so self-kill is
# impossible. hostPID + privileged are permitted in the 'cicd' namespace. Delays stagger the
# three control planes by 2 minutes each, so only one API server is down at a time.
---
apiVersion: batch/v1
kind: Job
metadata:
name: oidc-restart-apiserver-cp1-v2
namespace: cicd
spec:
backoffLimit: 0
ttlSecondsAfterFinished: 3600
template:
spec:
hostPID: true
restartPolicy: Never
nodeSelector:
kubernetes.io/hostname: talos-cp-1
tolerations:
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule
containers:
- name: kill
image: docker.io/library/busybox:1.36
securityContext:
privileged: true
command: ["/bin/sh", "-c"]
args:
- |
echo "sleeping 10s before restarting the API server on talos-cp-1"
sleep 10
self=$$
killed=0
for d in /proc/[0-9]*; do
pid=${d#/proc/}
[ "$pid" = "$self" ] && continue
comm=$(cat "$d/comm" 2>/dev/null)
[ "$comm" = "kube-apiserver" ] || continue
echo "SIGTERM -> pid $pid"
kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
done
echo "signalled $killed API server process(es); kubelet recreates the static pod"
---
apiVersion: batch/v1
kind: Job
metadata:
name: oidc-restart-apiserver-cp2-v2
namespace: cicd
spec:
backoffLimit: 0
ttlSecondsAfterFinished: 3600
template:
spec:
hostPID: true
restartPolicy: Never
nodeSelector:
kubernetes.io/hostname: talos-cp-2
tolerations:
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule
containers:
- name: kill
image: docker.io/library/busybox:1.36
securityContext:
privileged: true
command: ["/bin/sh", "-c"]
args:
- |
echo "sleeping 120s before restarting the API server on talos-cp-2"
sleep 120
self=$$
killed=0
for d in /proc/[0-9]*; do
pid=${d#/proc/}
[ "$pid" = "$self" ] && continue
comm=$(cat "$d/comm" 2>/dev/null)
[ "$comm" = "kube-apiserver" ] || continue
echo "SIGTERM -> pid $pid"
kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
done
echo "signalled $killed API server process(es); kubelet recreates the static pod"
---
apiVersion: batch/v1
kind: Job
metadata:
name: oidc-restart-apiserver-cp3-v2
namespace: cicd
spec:
backoffLimit: 0
ttlSecondsAfterFinished: 3600
template:
spec:
hostPID: true
restartPolicy: Never
nodeSelector:
kubernetes.io/hostname: talos-cp-3
tolerations:
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule
containers:
- name: kill
image: docker.io/library/busybox:1.36
securityContext:
privileged: true
command: ["/bin/sh", "-c"]
args:
- |
echo "sleeping 240s before restarting the API server on talos-cp-3"
sleep 240
self=$$
killed=0
for d in /proc/[0-9]*; do
pid=${d#/proc/}
[ "$pid" = "$self" ] && continue
comm=$(cat "$d/comm" 2>/dev/null)
[ "$comm" = "kube-apiserver" ] || continue
echo "SIGTERM -> pid $pid"
kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
done
echo "signalled $killed API server process(es); kubelet recreates the static pod"
-1
View File
@@ -9,4 +9,3 @@ resources:
- poimen-sandbox-namespace.yaml - poimen-sandbox-namespace.yaml
- poimen-sandbox-quota.yaml - poimen-sandbox-quota.yaml
- pi-session-netpol.yaml - pi-session-netpol.yaml
- pi-agent-workspace-pv.yaml
@@ -1,57 +0,0 @@
apiVersion: v1
kind: PersistentVolume
metadata:
finalizers:
- kubernetes.io/pv-protection
labels:
app: pi-coding-agent-sandbox
part-of: poimen-toolkit
name: pi-agent-workspace-pv
annotations:
argocd.argoproj.io/sync-wave: '-1'
spec:
accessModes:
- ReadWriteOnce
capacity:
storage: 10Gi
claimRef:
apiVersion: v1
kind: PersistentVolumeClaim
name: pi-agent-workspace
namespace: poimen-sandbox
hostPath:
path: /var/lib/pi-agent-sandbox
type: DirectoryOrCreate
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
values:
- talos-cp-2
persistentVolumeReclaimPolicy: Retain
volumeMode: Filesystem
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
annotations:
pv.kubernetes.io/bind-completed: 'yes'
argocd.argoproj.io/sync-wave: '-1'
finalizers:
- kubernetes.io/pvc-protection
labels:
app: pi-coding-agent-sandbox
part-of: poimen-toolkit
name: pi-agent-workspace
namespace: poimen-sandbox
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10Gi
storageClassName: ''
volumeMode: Filesystem
volumeName: pi-agent-workspace-pv
-1
View File
@@ -39,7 +39,6 @@
rewrite name comfy.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local rewrite name comfy.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
rewrite name comfyui.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local rewrite name comfyui.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
rewrite name gotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local rewrite name gotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
rewrite name pi.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
rewrite name igotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local rewrite name igotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
rewrite name riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local rewrite name riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local