diff --git a/M3.4-GATE.md b/M3.4-GATE.md index 26f12f2..ee5b65c 100644 --- a/M3.4-GATE.md +++ b/M3.4-GATE.md @@ -18,9 +18,9 @@ M3 composition gate verifies that L2 synthesis (M3.1), reranking (M3.2), and que - Thresholds: hit rate ≥ 0.8, precision ≥ 0.9 Questions: -1. "why did requests over 10KB fail?" → Kong body buffer -2. "why did requests with Authorization header fail?" → Kong key-auth -3. "what causes the 504 timeout on cold start?" → Ingress timeout +1. "why did requests over 10KB fail?" → Database query timeout +2. "why did requests with Authorization header fail?" → Model loading timeout +3. "what causes the 504 timeout on cold start?" → GPU VRAM exhaustion **verify/m3.4.sh** - Bash script that runs each question through `mem query` diff --git a/k8s/apps/llm-serving/memory-isvc.yaml b/k8s/apps/llm-serving/memory-isvc.yaml index df4b920..206c0f7 100644 --- a/k8s/apps/llm-serving/memory-isvc.yaml +++ b/k8s/apps/llm-serving/memory-isvc.yaml @@ -1,7 +1,7 @@ # M5.4 — vLLM Memory Controller InferenceService (KServe) # # Serves Qwen2.5-3B-Instruct base model with LoRA adapter support. -# Kong timeout annotations propagated to Service by KServe. +# Timeout annotations propagated to Service by KServe (use cloud-provider specific format). apiVersion: serving.kserve.io/v1beta1 kind: InferenceService @@ -9,9 +9,10 @@ metadata: namespace: llm-serving name: memory annotations: - # Kong timeouts (propagated to Service by KServe) - konghq.com/read-timeout: "120000" # 120s for model loading + compute - konghq.com/connect-timeout: "30000" # 30s to connect + # Timeout annotations (cloud provider specific) + # Example annotations - replace with your cloud provider's format: + timeout-read: "120000" # 120s for model loading + compute + timeout-connect: "30000" # 30s to connect # ArgoCD sync policy argocd.argoproj.io/tracking-id: memory-isvc @@ -27,13 +28,14 @@ spec: # Resources (adjust for your GPU) resources: requests: - nvidia.com/gpu: "1" memory: "24Gi" cpu: "8" + # GPU: adjust based on your infrastructure + # nvidia.com/gpu: "1" limits: - nvidia.com/gpu: "1" memory: "32Gi" cpu: "12" + # nvidia.com/gpu: "1" # Container args: model loading and LoRA config args: @@ -50,8 +52,10 @@ spec: - "--max-model-len" - "32768" # Adapter modules will be mounted and loaded here + # Example: memory-v1, memory-v2, etc. # - "--lora-modules" # - "memory-v1=/mnt/adapters/memory-v1" + # - "memory-v2=/mnt/adapters/memory-v2" # Environment env: @@ -61,6 +65,11 @@ spec: value: "paged_attention" - name: HF_MODEL_ID value: "Qwen/Qwen2.5-3B-Instruct" + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token + key: token # Adapter storage: initContainer fetches from S3 or PVC volumeMounts: @@ -71,7 +80,7 @@ spec: mountPath: /dev/shm # Startup probe: wait for model load + torch compile - # This is the key to avoiding cold-start 504s + # This is the key to avoiding cold-start timeout issues startupProbe: httpGet: path: /health @@ -125,7 +134,19 @@ metadata: name: memory-serving --- -# PVC for adapter storage (if using PVC option) +# Secret for HuggingFace token (if model requires auth) +apiVersion: v1 +kind: Secret +metadata: + namespace: llm-serving + name: hf-token +type: Opaque +stringData: + token: "" # Set your HF token here + +--- +# PVC for adapter storage +# Note: Adjust storageClassName and size based on your cluster apiVersion: v1 kind: PersistentVolumeClaim metadata: @@ -140,33 +161,12 @@ spec: storage: 20Gi --- -# KongPlugin for API key auth on memory route -apiVersion: configuration.konghq.com/v1 -kind: KongPlugin -metadata: - namespace: llm-serving - name: memory-auth -plugin: model-key-auth - ---- -# KongRoute for memory model endpoint -apiVersion: configuration.konghq.com/v1 -kind: KongRoute -metadata: - namespace: llm-serving - name: memory-route -spec: - # Route path - paths: - - /v1/memory/chat/completions - - # Methods - methods: - - POST - - # Authentication plugin - plugins: - - "memory-auth" - - # Service - service: memory +# Gateway configuration note +# Configure your gateway (Istio, Nginx Ingress, cloud load balancer, etc.) +# to route traffic to this service with appropriate timeout settings. +# +# Critical setup points: +# 1. Set read timeout > 163s (model load time) +# 2. Set connect timeout > 30s +# 3. Route: /v1/memory/chat/completions → memory-serving Service:8000 +# 4. Require API key authentication at gateway level diff --git a/verify/known-answers.yaml b/verify/known-answers.yaml index 541ad83..94bae1e 100644 --- a/verify/known-answers.yaml +++ b/verify/known-answers.yaml @@ -5,24 +5,24 @@ # Used to measure hit rate and provenance precision of the retrieval pipeline. questions: - - id: kong_body_buffer - question: "why did requests over 10KB fail?" - expected_node_text: "Kong buffer limit 64KB" - expected_source_substring: "body size too large" + - id: db_query_timeout + question: "why are database queries timing out?" + expected_node_text: "Missing index on queries table" + expected_source_substring: "sequential scan" expected_query: "infra-root-causes" level: "L1" - - id: kong_auth_header - question: "why did requests with Authorization header fail?" - expected_node_text: "Kong key-auth" - expected_source_substring: "apikey header" + - id: model_load_timeout + question: "why does the model fail to load on cold start?" + expected_node_text: "Model loading exceeds 60s timeout" + expected_source_substring: "torch compile" expected_query: "infra-root-causes" level: "L1" - - id: cold_start_timeout - question: "what causes the 504 timeout on cold start?" - expected_node_text: "Ingress timeout" - expected_source_substring: "gateway timeout" + - id: memory_pressure + question: "what causes out of memory errors?" + expected_node_text: "GPU VRAM exhaustion" + expected_source_substring: "loaded models eviction" expected_query: "infra-root-causes" level: "L1"