feat(network): SSE optimization for local LLM streaming (#31 #32 #33) (#26)

Addresses three critical network issues for LLM streaming performance: **#33 Disable proxy buffering for SSE** - Add X-Accel-Buffering: no header to response - Tells nginx/Ingress to stream events immediately instead of buffering **#32 HTTP/2 multiplexing for concurrent streams** - Enable HTTP/2 in server config via http2.ConfigureServer() - Increase MaxConnsPerHost to 10 for better concurrency - Allows multiple concurrent LLM requests without blocking **#31 TCP backpressure for streaming LLM responses** - Set TCP_NODELAY on dialer to disable Nagle's algorithm - Reduces latency by sending small packets immediately - Critical for low TTFT (time-to-first-token) under load **Tests added:** - TestTCPBackpressure: Verifies TCP backpressure handling with slow client - TestConcurrentSSEStreams: Confirms HTTP/2 multiplexing works correctly --------- Co-authored-by: poison <[email protected]> Reviewed-on: #26 Co-authored-by: poimen <[email protected]>
2026-09-13 23:37:33 +00:00
parent 7de71180b3
commit 30e0a83a50
23 changed files with 1606 additions and 797 deletions
@@ -0,0 +1,124 @@
+apiVersion: tekton.dev/v1
+kind: Pipeline
+metadata:
+  name: sse-optimization-tests
+  namespace: api
+  labels:
+    app: api-gateway
+    component: ci-cd
+spec:
+  description: >
+    Test pipeline for SSE optimization (issues #31, #32, #33).
+    Runs both functional integration tests and performance load tests.
+  
+  params:
+  - name: image
+    type: string
+    description: "Container image to test (repo:tag)"
+  - name: gateway-port
+    type: string
+    default: "8080"
+  
+  tasks:
+  # Functional integration tests first (quick smoke test)
+  - name: integration-tests
+    taskRef:
+      name: integration-test
+    params:
+    - name: image
+      value: $(params.image)
+    - name: gateway-port
+      value: $(params.gateway-port)
+  
+  # Workflow visibility tests (runs after integration tests pass)
+  - name: workflow-visibility-tests
+    runAfter:
+    - integration-tests
+    taskRef:
+      name: workflow-visibility-test
+    params:
+    - name: image
+      value: $(params.image)
+    - name: gateway-port
+      value: $(params.gateway-port)
+  
+  # Performance load tests (runs after integration tests pass)
+  - name: load-tests
+    runAfter:
+    - workflow-visibility-tests
+    taskRef:
+      name: load-test-sse-streaming
+    params:
+    - name: image
+      value: $(params.image)
+    - name: gateway-port
+      value: $(params.gateway-port)
+    - name: concurrent-streams
+      value: "10"
+    - name: events-per-stream
+      value: "100"
+    - name: event-interval-ms
+      value: "50"
+  
+  # Summary reporter
+  - name: report-results
+    runAfter:
+    - load-tests
+    - workflow-visibility-tests
+    taskSpec:
+      description: "Report combined test results"
+      params:
+      - name: integration-result
+        type: string
+      - name: integration-summary
+        type: string
+      - name: workflow-result
+        type: string
+      - name: workflow-summary
+        type: string
+      - name: load-result
+        type: string
+      - name: load-summary
+        type: string
+      - name: load-metrics
+        type: string
+      steps:
+      - name: print-summary
+        image: busybox
+        script: |
+          #!/bin/sh
+          echo "╔═══════════════════════════════════════════════════════════╗"
+          echo "║      SSE Optimization + Workflow Tests (PR #26)          ║"
+          echo "╠═══════════════════════════════════════════════════════════╣"
+          echo "║                                                           ║"
+          echo "║  Integration Tests:                                       ║"
+          echo "║    Status: $(params.integration-result)"
+          echo "║    Summary: $(params.integration-summary)"
+          echo "║                                                           ║"
+          echo "║  Workflow Visibility (namespace pass-down):               ║"
+          echo "║    Status: $(params.workflow-result)"
+          echo "║    Summary: $(params.workflow-summary)"
+          echo "║                                                           ║"
+          echo "║  Load Tests (Issues #31, #32, #33):                       ║"
+          echo "║    Status: $(params.load-result)"
+          echo "║    Summary: $(params.load-summary)"
+          echo "║                                                           ║"
+          echo "║  Performance Metrics:                                     ║"
+          echo "║    $(params.load-metrics)"
+          echo "║                                                           ║"
+          echo "╚═══════════════════════════════════════════════════════════╝"
+    params:
+    - name: integration-result
+      value: $(tasks.integration-tests.results.result)
+    - name: integration-summary
+      value: $(tasks.integration-tests.results.summary)
+    - name: workflow-result
+      value: $(tasks.workflow-visibility-tests.results.result)
+    - name: workflow-summary
+      value: $(tasks.workflow-visibility-tests.results.summary)
+    - name: load-result
+      value: $(tasks.load-tests.results.result)
+    - name: load-summary
+      value: $(tasks.load-tests.results.summary)
+    - name: load-metrics
+      value: $(tasks.load-tests.results.metrics)