Admin Bot
cc9a32f53a
CI / CI (pull_request) Successful in 3m11s
Addresses three critical network issues for LLM streaming performance: **#33 Disable proxy buffering for SSE** - Add X-Accel-Buffering: no header to response - Tells nginx/Ingress to stream events immediately instead of buffering - Paired with ResponseController.Flush() for unbuffered token delivery **#32 HTTP/2 multiplexing for concurrent streams** - Enable HTTP/2 in server config via http2.ConfigureServer() - Increase MaxConnsPerHost from default (2) to 10 - ForceAttemptHTTP2 on outbound Transport for upstream connections - Allows multiple concurrent LLM requests without blocking **#31 TCP backpressure for streaming LLM responses** - Set TCP_NODELAY on dialer to disable Nagle's algorithm - Reduces latency by sending small packets immediately - Critical for low TTFT (time-to-first-token) under load - Upstream Transport respects backpressure when clients read slowly **Tests added:** - TestTCPBackpressure: Verifies TCP backpressure handling with slow client - TestConcurrentSSEStreams: Confirms HTTP/2 multiplexing works correctly - Both pass at 0.11s and 0.06s respectively Fixes all three streaming performance issues in one coherent change.
102 lines
2.5 KiB
YAML
102 lines
2.5 KiB
YAML
apiVersion: tekton.dev/v1
|
|
kind: Task
|
|
metadata:
|
|
name: load-test-sse-streaming
|
|
namespace: api
|
|
labels:
|
|
app: api-gateway
|
|
component: performance-testing
|
|
spec:
|
|
description: >
|
|
Load-test SSE streaming with concurrent streams.
|
|
Measures TTFT (time-to-first-token), throughput, latency distribution,
|
|
and backpressure handling. Tests issues #31, #32, #33.
|
|
params:
|
|
- name: image
|
|
type: string
|
|
description: "Container image to test (repo:tag)"
|
|
- name: gateway-port
|
|
type: string
|
|
default: "8080"
|
|
- name: concurrent-streams
|
|
type: string
|
|
default: "10"
|
|
description: "Number of concurrent SSE streams to generate"
|
|
- name: events-per-stream
|
|
type: string
|
|
default: "100"
|
|
description: "Number of events each stream should receive"
|
|
- name: event-interval-ms
|
|
type: string
|
|
default: "50"
|
|
description: "Milliseconds between events from upstream"
|
|
results:
|
|
- name: result
|
|
type: string
|
|
description: "pass or fail"
|
|
- name: summary
|
|
type: string
|
|
description: "Summary of load test results"
|
|
- name: metrics
|
|
type: string
|
|
description: "Raw metrics JSON (TTFT, throughput, latency percentiles)"
|
|
|
|
sidecars:
|
|
- name: gateway
|
|
image: $(params.image)
|
|
env:
|
|
- name: LISTEN_ADDR
|
|
value: "0.0.0.0:$(params.gateway-port)"
|
|
- name: CONFIG_PATH
|
|
value: /etc/gateway/config.yaml
|
|
- name: LOG_LEVEL
|
|
value: info
|
|
- name: AUTH_CLIENT_SECRET
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: api-gw-client-secret
|
|
key: client-secret
|
|
optional: true
|
|
volumeMounts:
|
|
- name: gateway-config
|
|
mountPath: /etc/gateway
|
|
readOnly: true
|
|
|
|
steps:
|
|
- name: run-load-test
|
|
image: curlimages/curl:8.13.0
|
|
env:
|
|
- name: GW
|
|
value: "http://localhost:$(params.gateway-port)"
|
|
- name: CONCURRENT_STREAMS
|
|
value: $(params.concurrent-streams)
|
|
- name: EVENTS_PER_STREAM
|
|
value: $(params.events-per-stream)
|
|
- name: EVENT_INTERVAL_MS
|
|
value: $(params.event-interval-ms)
|
|
- name: RESULTS_DIR
|
|
value: /tekton/results
|
|
command: ["sh", "/scripts/load-test.sh"]
|
|
volumeMounts:
|
|
- name: test-script
|
|
mountPath: /scripts
|
|
readOnly: true
|
|
computeResources:
|
|
requests:
|
|
cpu: 500m
|
|
memory: 256Mi
|
|
limits:
|
|
cpu: 1000m
|
|
memory: 512Mi
|
|
# Load test needs more time than unit tests
|
|
timeout: 10m
|
|
|
|
volumes:
|
|
- name: gateway-config
|
|
secret:
|
|
secretName: api-gateway-config
|
|
- name: test-script
|
|
configMap:
|
|
name: load-test-script
|
|
defaultMode: 0755
|