rock
f81db119f4
The Actions queue was stalling because Gitea had no CPU, not because the database was slow. Measured on the running pod before this change: nr_throttled 189528 of 1169014 CFS periods (16.2%) throttled_usec 4424898602 us (~74 minutes lost) actual usage 951m against a 1000m limit The user-visible symptom was slow Actions traffic in Gitea's own log: slow POST /api/actions/runner.v1.RunnerService/FetchTask ... 3469.9ms slow GET .../compare/... elapsed 3091.5ms / 3520.8ms / 3811.5ms So runners waited, timed out, and retried, which loaded the saturated process further. The database is exonerated. On the primary, a real action_runner UPDATE measured 2.133ms, pg_stat_activity showed zero lock waits, and no query ran longer than 3 seconds. Raising any database resource would not have helped. Why these numbers: - requests.cpu 200m -> 750m. A request below real usage earns the pod little scheduling weight during contention, which is exactly when it matters. - limits.cpu 1000m -> 3000m. Diff computation is bursty; a 3x ceiling lets a burst finish instead of being chopped into 100ms slices. - requests.memory 512Mi -> 768Mi. Observed usage was 621Mi, above the old request. - limits.memory stays at 2Gi. Observed usage was 621Mi, so memory was never the problem and does not need room. Caution recorded in the file: this pod runs on talos-cp-3, a control-plane node, and 3000m is 25% of its 11950m allocatable CPU. If Gitea sustains near this limit, move the workload off the control plane instead of raising the limit again. ArgoCD owns this file (app `forgejo`, chart gitea 12.7.0, selfHeal true), so this must reach main to take effect. A live patch is reverted within seconds.
202 lines
7.0 KiB
YAML
202 lines
7.0 KiB
YAML
# Forgejo Helm Values — Single Source of Truth
|
|
# Chart: https://codeberg.org/forgejo-contrib/forgejo-helm
|
|
|
|
# Recreate (not RollingUpdate): the gitea data volume is a single RWO PVC. With
|
|
# RollingUpdate the new pod tries to attach the PVC while the old pod still holds
|
|
# it -> "Multi-Attach error", new pod stuck Init forever, rollout wedged. Recreate
|
|
# terminates the old pod first so the PVC detaches before the new one starts.
|
|
strategy:
|
|
type: Recreate
|
|
|
|
# Disable bundled dependencies (use external CNPG + Redis instead)
|
|
postgresql-ha:
|
|
enabled: false
|
|
|
|
valkey:
|
|
enabled: false
|
|
|
|
valkey-cluster:
|
|
enabled: false
|
|
|
|
redis:
|
|
enabled: false
|
|
|
|
# External SSH access for git over the LAN. The chart's ssh Service becomes a
|
|
# LoadBalancer with a stable IP from the Cilium homelab-pool (192.168.1.160/28,
|
|
# L2-announced) so `git clone ssh://[email protected]:2222/...` works from the
|
|
# LAN. gitea's sshd listens on 2222 in-pod; port 2222 is exposed directly to
|
|
# avoid needing privileged :22.
|
|
service:
|
|
ssh:
|
|
type: LoadBalancer
|
|
port: 2222
|
|
annotations:
|
|
lbipam.cilium.io/ips: "192.168.1.161"
|
|
|
|
gitea:
|
|
podAnnotations:
|
|
reloader.stakater.com/auto: "true"
|
|
admin:
|
|
existingSecret: forgejo-admin
|
|
|
|
config:
|
|
server:
|
|
DOMAIN: forgejo.riotpiao.com
|
|
ROOT_URL: https://forgejo.riotpiao.com
|
|
# SSH clone URLs advertise git.riotpiao.com:2222 (the LoadBalancer above).
|
|
SSH_DOMAIN: git.riotpiao.com
|
|
SSH_PORT: 2222
|
|
SSH_LISTEN_PORT: 2222
|
|
|
|
# Self-signup was open (GET /user/sign_up returned 200), so anyone reaching
|
|
# forgejo.riotpiao.com could create an account. Accounts are provisioned by
|
|
# the admin only; admin create-user via UI/API is unaffected by this flag.
|
|
service:
|
|
DISABLE_REGISTRATION: true
|
|
|
|
database:
|
|
DB_TYPE: postgres
|
|
HOST: forgejo-db-rw.cicd.svc.cluster.local:5432
|
|
NAME: forgejo
|
|
# User/password injected via extraEnv (secretKeyRef doesn't work in config)
|
|
|
|
cache:
|
|
ADAPTER: redis
|
|
HOST: redis://forgejo-redis.cicd.svc.cluster.local:6379/0
|
|
|
|
session:
|
|
PROVIDER: redis
|
|
PROVIDER_CONFIG: redis://forgejo-redis.cicd.svc.cluster.local:6379/1
|
|
|
|
queue:
|
|
TYPE: redis
|
|
CONN_STR: redis://forgejo-redis.cicd.svc.cluster.local:6379/2
|
|
|
|
# Actions must be enabled globally, not just per-repo. Without this section
|
|
# app.ini carries no [actions] block at all and Forgejo never *creates* a
|
|
# workflow run — the API returns total_count: 0 for every repo even though
|
|
# each repo reports has_actions: true, the workflow file is on the default
|
|
# branch, and forgejo-runner has registered successfully with label
|
|
# [docker]. Registration does not require the dispatcher, so a healthy-
|
|
# looking runner sitting at "[poller 0] launched" with zero task pickups is
|
|
# the symptom of this being off, not of a bad workflow or a label mismatch.
|
|
#
|
|
# DEFAULT_ACTIONS_URL is left at its default (https://code.forgejo.org),
|
|
# which is where `uses: actions/checkout@v4` and friends resolve from. That
|
|
# requires egress from the runner; if it is ever blocked, pin the actions to
|
|
# local copies rather than turning this off.
|
|
actions:
|
|
ENABLED: true
|
|
|
|
mailer:
|
|
ENABLED: true
|
|
PROTOCOL: smtp+starttls
|
|
SMTP_ADDR: smtp.gmail.com
|
|
SMTP_PORT: 587
|
|
FROM: "Forgejo <[email protected]>"
|
|
# USER and PASSWD injected via env vars below (GITEA__MAILER__USER, GITEA__MAILER__PASSWD)
|
|
|
|
# Persistence (shared storage for repos)
|
|
persistence:
|
|
enabled: true
|
|
storageClass: longhorn
|
|
size: 20Gi
|
|
accessModes:
|
|
- ReadWriteOnce
|
|
|
|
# Ingress
|
|
ingress:
|
|
enabled: true
|
|
className: nginx
|
|
annotations:
|
|
# No cert-manager annotation. forgejo.riotpiao.com is covered by the
|
|
# *.riotpiao.com wildcard, served as nginx's default-ssl-certificate.
|
|
#
|
|
# This chart Ingress and the hand-written one in
|
|
# k8s/bootstrap/ingress/ingress.yaml both claim forgejo.riotpiao.com.
|
|
# ingress-nginx breaks the tie by oldest creationTimestamp, and the chart's
|
|
# is older, so it is the one actually serving — the annotations on the other
|
|
# have never applied. Duplicate should be removed; until then these must
|
|
# live here or they do nothing.
|
|
#
|
|
# proxy-body-size 0 is required for the OCI registry: nginx defaults to 1m,
|
|
# so any image layer above that fails the push with 413.
|
|
nginx.ingress.kubernetes.io/proxy-body-size: "0"
|
|
nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
|
|
nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
|
|
hosts:
|
|
- host: forgejo.riotpiao.com
|
|
paths:
|
|
- path: /
|
|
pathType: Prefix
|
|
# tls: intentionally omitted. The chart would create a second, single-host
|
|
# certificate (forgejo-tls) that expired on 2026-10-23 and competed with the
|
|
# wildcard. The wildcard covers this host.
|
|
|
|
# Resources
|
|
#
|
|
# The 1000m ceiling throttled Gitea badly. Measured on a running pod:
|
|
# nr_throttled 189528 of 1169014 CFS periods (16.2%)
|
|
# throttled_usec 4424898602 us (~74 minutes lost)
|
|
# actual usage 951m against the 1000m limit
|
|
# The visible symptom was slow Actions traffic: FetchTask took 3469.9ms and
|
|
# repository compare endpoints took 3091-3811ms, so runners timed out and
|
|
# retried, which added more load to an already saturated process.
|
|
#
|
|
# The database was never the bottleneck. A real action_runner UPDATE measured
|
|
# 2.133ms with zero lock waits, so this is CPU starvation in Gitea itself.
|
|
#
|
|
# requests.cpu rises to 750m because a request below real usage earns the pod
|
|
# little scheduling weight during contention. limits.cpu rises to 3000m
|
|
# because diff computation is bursty and needs room to run without throttling.
|
|
#
|
|
# Caution: this pod runs on talos-cp-3, a control-plane node. 3000m is 25% of
|
|
# that node's 11950m allocatable CPU. If Gitea sustains near this limit, move
|
|
# the workload off the control plane rather than raising the limit again.
|
|
resources:
|
|
requests:
|
|
cpu: 750m
|
|
memory: 768Mi
|
|
limits:
|
|
cpu: 3000m
|
|
memory: 2Gi
|
|
|
|
# Tolerations for control-plane
|
|
tolerations:
|
|
- key: node-role.kubernetes.io/control-plane
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
|
|
# ArgoCD adoption labels
|
|
labels:
|
|
argocd.argoproj.io/instance: forgejo
|
|
|
|
# Inject database credentials via environment variables (overrides app.ini)
|
|
deployment:
|
|
env:
|
|
- name: GITEA__DATABASE__USER
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: forgejo-db-app
|
|
key: username
|
|
- name: GITEA__DATABASE__PASSWD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: forgejo-db-app
|
|
key: password
|
|
- name: GITEA__MAILER__USER
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: forgejo-smtp
|
|
key: user
|
|
- name: GITEA__MAILER__PASSWD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: forgejo-smtp
|
|
key: password
|
|
# Allow webhooks to reach api-gw through the cluster ingress (resolves to
|
|
# internal ClusterIP 10.101.x.x which is blocked by default ALLOWED_HOST_LIST).
|
|
# api.riotpiao.com is our own gateway — safe to allowlist.
|
|
- name: GITEA__WEBHOOK__ALLOWED_HOST_LIST
|
|
value: "external,api.riotpiao.com"
|