Admin Bot
c5a91cf647
CI / CI (pull_request) Successful in 34m5s
The per-pod semaphore was not a cap: 3 replicas x 4 slots advertised 12 concurrent requests while the engine runs 4, and nothing coordinated the replicas. Admission is now gated on the engine's own published gauges, which is global by construction because every replica contends for the same engine. - new internal/proxy/engine_state.go: 2s scrape of the backend /metrics, hand-rolled exposition parse (no new dependency), unhealthy after 3 consecutive failed polls, and a startup self-check that raises observer_degraded rather than falling back to fail-open silently - gate: park while kv_cache_usage_perc >= 0.90 or running + waiting >= ADMISSION_MAX_INFLIGHT, bounded by ADMISSION_MAX_PARK_SECONDS, then 503 + Retry-After: 5. Unknown state fails open on purpose - streaming requests receive SSE keep-alive heartbeats while parked - router: extract only the top-level model field instead of unmarshalling the whole body into a map; body cap from MAX_BODY_BYTES (1 MiB, the model path previously inherited the 100 MB default) - GOMAXPROCS from the cgroup CPU quota: the container is limited to 500m while Go defaulted to the host core count and was CFS-throttled - VLLM_MAX_QUEUE_DEPTH default 20 -> 8 - delete k8s/configmap.yaml: absent from kustomization.yaml, and the live ConfigMap of that name carries an unrelated key - correct the llm-client NetworkPolicy comment; llm-serving-ingress now selects by namespace, not by that label Observers run only for models named in ADMISSION_OBSERVE_MODELS. reasoning is the only vLLM engine here; ornith and qwen-cpu are Ollama and embeddings/rerank are TEI, none of which publish vllm:* series, so observing them would hold observer_degraded at 1 permanently. Companion change (engine flags + dashboard panels) is riotpiao-poimen/homelab#51.
173 lines
5.3 KiB
YAML
173 lines
5.3 KiB
YAML
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: api-gateway
|
|
namespace: api
|
|
labels:
|
|
app: api-gateway
|
|
component: gateway
|
|
spec:
|
|
replicas: 3
|
|
strategy:
|
|
type: RollingUpdate
|
|
rollingUpdate:
|
|
maxSurge: 1
|
|
maxUnavailable: 0
|
|
selector:
|
|
matchLabels:
|
|
app: api-gateway
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: api-gateway
|
|
component: gateway
|
|
# The llm-serving-ingress NetworkPolicy in the llm-serving namespace now
|
|
# selects by namespace (kubernetes.io/metadata.name: api), not by this
|
|
# label. Kept for older NetworkPolicy revisions that still admit on
|
|
# llm-client=true.
|
|
llm-client: "true"
|
|
annotations:
|
|
reloader.stakater.com/auto: "true"
|
|
prometheus.io/scrape: "true"
|
|
prometheus.io/port: "8080"
|
|
prometheus.io/path: "/metrics"
|
|
spec:
|
|
serviceAccountName: api-gateway
|
|
# The registry at forgejo.riotpiao.com requires auth — /v2/ answers 401.
|
|
# This Secret must exist in the api namespace before the first rollout.
|
|
imagePullSecrets:
|
|
- name: forgejo-registry
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
# 65532 is distroless's nonroot user, matching USER in the Dockerfile.
|
|
runAsUser: 65532
|
|
fsGroup: 65532
|
|
containers:
|
|
- name: gateway
|
|
# Tag is pinned in kustomization.yaml so there is exactly one place to
|
|
# bump it. Never :latest — Argo cannot make a deterministic rollout
|
|
# decision from a mutable tag, and 6.1 requires SHA tags.
|
|
image: forgejo.riotpiao.com/rock/api-gateway:latest
|
|
imagePullPolicy: Always
|
|
ports:
|
|
- name: http
|
|
containerPort: 8080
|
|
protocol: TCP
|
|
env:
|
|
- name: LISTEN_ADDR
|
|
value: "0.0.0.0:8080"
|
|
- name: CONFIG_PATH
|
|
value: "/etc/gateway/config.yaml"
|
|
- name: AUTH_CLIENT_SECRET
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: api-gw-client-secret
|
|
key: client-secret
|
|
optional: true
|
|
- name: SHUTDOWN_TIMEOUT
|
|
value: "5m"
|
|
- name: LOG_LEVEL
|
|
value: "info"
|
|
- name: ADMISSION_MAX_PARK_SECONDS
|
|
value: "120"
|
|
# Only backends that publish the vLLM scheduler gauges may be observed.
|
|
# `reasoning` is the only vLLM engine; ornith/qwen-cpu are Ollama and
|
|
# embeddings/rerank are TEI, so observing them would hold
|
|
# observer_degraded at 1 forever.
|
|
- name: ADMISSION_OBSERVE_MODELS
|
|
value: "reasoning"
|
|
- name: VLLM_MAX_QUEUE_DEPTH
|
|
value: "8"
|
|
- name: MAX_BODY_BYTES
|
|
value: "1048576"
|
|
# OpenTelemetry tracing configuration
|
|
- name: OTEL_EXPORTER_OTLP_ENDPOINT
|
|
value: "otel-collector.tracing.svc.cluster.local:4317"
|
|
- name: OTEL_SERVICE_NAME
|
|
value: "api-gateway"
|
|
- name: OTEL_SERVICE_VERSION
|
|
value: "1.0.0"
|
|
- name: OTEL_ENVIRONMENT
|
|
value: "production"
|
|
# Gotify integration — Forgejo webhook → push notifications
|
|
- name: GOTIFY_URL
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: gotify-webhook-secret
|
|
key: gotify-url
|
|
optional: true
|
|
- name: GOTIFY_APP_TOKEN
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: gotify-webhook-secret
|
|
key: gotify-app-token
|
|
optional: true
|
|
- name: FORGEJO_WEBHOOK_SECRET
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: gotify-webhook-secret
|
|
key: forgejo-webhook-secret
|
|
optional: true
|
|
- name: REGISTRY_PAT
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: forgejo-registry-pat
|
|
key: pat
|
|
optional: true
|
|
- name: REGISTRY_BASE
|
|
value: "http://forgejo-gitea-http.cicd.svc.cluster.local:3000"
|
|
volumeMounts:
|
|
- name: config
|
|
mountPath: /etc/gateway
|
|
readOnly: true
|
|
livenessProbe:
|
|
httpGet:
|
|
path: /healthz
|
|
port: http
|
|
scheme: HTTP
|
|
initialDelaySeconds: 5
|
|
periodSeconds: 10
|
|
timeoutSeconds: 2
|
|
failureThreshold: 3
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /readyz
|
|
port: http
|
|
scheme: HTTP
|
|
initialDelaySeconds: 5
|
|
periodSeconds: 5
|
|
timeoutSeconds: 2
|
|
failureThreshold: 2
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 128Mi
|
|
limits:
|
|
cpu: 500m
|
|
memory: 512Mi
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: true
|
|
runAsNonRoot: true
|
|
runAsUser: 65532
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
volumes:
|
|
- name: config
|
|
secret:
|
|
secretName: api-gateway-config
|
|
affinity:
|
|
podAntiAffinity:
|
|
preferredDuringSchedulingIgnoredDuringExecution:
|
|
- weight: 100
|
|
podAffinityTerm:
|
|
labelSelector:
|
|
matchExpressions:
|
|
- key: app
|
|
operator: In
|
|
values:
|
|
- api-gateway
|
|
topologyKey: kubernetes.io/hostname
|
|
terminationGracePeriodSeconds: 300
|