poimen-memory/k8s/apps/llm-serving/memory-isvc.yaml

# M5.4 — vLLM Memory Controller InferenceService (KServe)
#
# Serves Qwen2.5-3B-Instruct base model with LoRA adapter support.
# Kong timeout annotations propagated to Service by KServe.

apiVersion: serving.kserve.io/v1beta1
kind: InferenceService
metadata:
  namespace: llm-serving
  name: memory
  annotations:
    # Kong timeouts (propagated to Service by KServe)
    konghq.com/read-timeout: "120000"      # 120s for model loading + compute
    konghq.com/connect-timeout: "30000"    # 30s to connect
    # ArgoCD sync policy
    argocd.argoproj.io/tracking-id: memory-isvc

spec:
  predictor:
    # Model serving framework
    serviceAccountName: memory-serving
    
    containers:
    - name: kserve-container
      image: vllm/vllm-openai:v0.11.0
      
      # Resources (adjust for your GPU)
      resources:
        requests:
          nvidia.com/gpu: "1"
          memory: "24Gi"
          cpu: "8"
        limits:
          nvidia.com/gpu: "1"
          memory: "32Gi"
          cpu: "12"
      
      # Container args: model loading and LoRA config
      args:
      - python
      - "-m"
      - vllm.entrypoints.openai.api_server
      - "--model"
      - "Qwen/Qwen2.5-3B-Instruct"
      - "--served-model-name"
      - "memory"
      - "--enable-lora"
      - "--max-lora-rank"
      - "32"
      - "--max-model-len"
      - "32768"
      # Adapter modules will be mounted and loaded here
      # - "--lora-modules"
      # - "memory-v1=/mnt/adapters/memory-v1"
      
      # Environment
      env:
      - name: CUDA_VISIBLE_DEVICES
        value: "0"
      - name: VLLM_ATTENTION_BACKEND
        value: "paged_attention"
      - name: HF_MODEL_ID
        value: "Qwen/Qwen2.5-3B-Instruct"
      
      # Adapter storage: initContainer fetches from S3 or PVC
      volumeMounts:
      - name: adapter-storage
        mountPath: /mnt/adapters
        readOnly: true
      - name: shm
        mountPath: /dev/shm
      
      # Startup probe: wait for model load + torch compile
      # This is the key to avoiding cold-start 504s
      startupProbe:
        httpGet:
          path: /health
          port: 8000
        initialDelaySeconds: 60      # Wait 60s before probing
        periodSeconds: 10            # Check every 10s
        timeoutSeconds: 5            # Each probe can take up to 5s
        failureThreshold: 30         # Fail after 30 failures (5min total)
        successThreshold: 1
      
      # Readiness probe: model is ready to serve
      readinessProbe:
        httpGet:
          path: /health
          port: 8000
        initialDelaySeconds: 120     # Wait 2min before first check
        periodSeconds: 10
        timeoutSeconds: 5
        failureThreshold: 3
      
      # Liveness probe: container is not stuck
      livenessProbe:
        httpGet:
          path: /health
          port: 8000
        initialDelaySeconds: 300     # Wait 5min before first liveness check
        periodSeconds: 30
        timeoutSeconds: 5
        failureThreshold: 3
    
    # Volumes
    volumes:
    - name: adapter-storage
      # Option 1: PVC (persistent storage)
      persistentVolumeClaim:
        claimName: adapter-storage
        readOnly: true
      # Option 2: emptyDir + initContainer (download from S3)
      # emptyDir: {}
    - name: shm
      emptyDir:
        medium: Memory
        sizeLimit: 8Gi

---
# ServiceAccount for model serving
apiVersion: v1
kind: ServiceAccount
metadata:
  namespace: llm-serving
  name: memory-serving

---
# PVC for adapter storage (if using PVC option)
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  namespace: llm-serving
  name: adapter-storage
spec:
  accessModes:
  - ReadOnlyMany
  storageClassName: standard
  resources:
    requests:
      storage: 20Gi

---
# KongPlugin for API key auth on memory route
apiVersion: configuration.konghq.com/v1
kind: KongPlugin
metadata:
  namespace: llm-serving
  name: memory-auth
plugin: model-key-auth

---
# KongRoute for memory model endpoint
apiVersion: configuration.konghq.com/v1
kind: KongRoute
metadata:
  namespace: llm-serving
  name: memory-route
spec:
  # Route path
  paths:
  - /v1/memory/chat/completions
  
  # Methods
  methods:
  - POST
  
  # Authentication plugin
  plugins:
  - "memory-auth"
  
  # Service
  service: memory