poimen-memory/k8s/apps/llm-serving/memory-isvc.yaml

# M5.4 — vLLM Memory Controller InferenceService (KServe)
#
# Serves Qwen2.5-3B-Instruct base model with LoRA adapter support.
# Timeout annotations propagated to Service by KServe (use cloud-provider specific format).

apiVersion: serving.kserve.io/v1beta1
kind: InferenceService
metadata:
  namespace: llm-serving
  name: memory
  annotations:
    # Timeout annotations (cloud provider specific)
    # Example annotations - replace with your cloud provider's format:
    timeout-read: "120000"        # 120s for model loading + compute
    timeout-connect: "30000"      # 30s to connect
    # ArgoCD sync policy
    argocd.argoproj.io/tracking-id: memory-isvc

spec:
  predictor:
    # Model serving framework
    serviceAccountName: memory-serving
    
    containers:
    - name: kserve-container
      image: vllm/vllm-openai:v0.11.0
      
      # Resources (adjust for your GPU)
      resources:
        requests:
          memory: "24Gi"
          cpu: "8"
          # GPU: adjust based on your infrastructure
          # nvidia.com/gpu: "1"
        limits:
          memory: "32Gi"
          cpu: "12"
          # nvidia.com/gpu: "1"
      
      # Container args: model loading and LoRA config
      args:
      - python
      - "-m"
      - vllm.entrypoints.openai.api_server
      - "--model"
      - "Qwen/Qwen2.5-3B-Instruct"
      - "--served-model-name"
      - "memory"
      - "--enable-lora"
      - "--max-lora-rank"
      - "32"
      - "--max-model-len"
      - "32768"
      # Adapter modules will be mounted and loaded here
      # Example: memory-v1, memory-v2, etc.
      # - "--lora-modules"
      # - "memory-v1=/mnt/adapters/memory-v1"
      # - "memory-v2=/mnt/adapters/memory-v2"
      
      # Environment
      env:
      - name: CUDA_VISIBLE_DEVICES
        value: "0"
      - name: VLLM_ATTENTION_BACKEND
        value: "paged_attention"
      - name: HF_MODEL_ID
        value: "Qwen/Qwen2.5-3B-Instruct"
      - name: HF_TOKEN
        valueFrom:
          secretKeyRef:
            name: hf-token
            key: token
      
      # Adapter storage: initContainer fetches from S3 or PVC
      volumeMounts:
      - name: adapter-storage
        mountPath: /mnt/adapters
        readOnly: true
      - name: shm
        mountPath: /dev/shm
      
      # Startup probe: wait for model load + torch compile
      # This is the key to avoiding cold-start timeout issues
      startupProbe:
        httpGet:
          path: /health
          port: 8000
        initialDelaySeconds: 60      # Wait 60s before probing
        periodSeconds: 10            # Check every 10s
        timeoutSeconds: 5            # Each probe can take up to 5s
        failureThreshold: 30         # Fail after 30 failures (5min total)
        successThreshold: 1
      
      # Readiness probe: model is ready to serve
      readinessProbe:
        httpGet:
          path: /health
          port: 8000
        initialDelaySeconds: 120     # Wait 2min before first check
        periodSeconds: 10
        timeoutSeconds: 5
        failureThreshold: 3
      
      # Liveness probe: container is not stuck
      livenessProbe:
        httpGet:
          path: /health
          port: 8000
        initialDelaySeconds: 300     # Wait 5min before first liveness check
        periodSeconds: 30
        timeoutSeconds: 5
        failureThreshold: 3
    
    # Volumes
    volumes:
    - name: adapter-storage
      # Option 1: PVC (persistent storage)
      persistentVolumeClaim:
        claimName: adapter-storage
        readOnly: true
      # Option 2: emptyDir + initContainer (download from S3)
      # emptyDir: {}
    - name: shm
      emptyDir:
        medium: Memory
        sizeLimit: 8Gi

---
# ServiceAccount for model serving
apiVersion: v1
kind: ServiceAccount
metadata:
  namespace: llm-serving
  name: memory-serving

---
# Secret for HuggingFace token (if model requires auth)
apiVersion: v1
kind: Secret
metadata:
  namespace: llm-serving
  name: hf-token
type: Opaque
stringData:
  token: ""  # Set your HF token here

---
# PVC for adapter storage
# Note: Adjust storageClassName and size based on your cluster
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  namespace: llm-serving
  name: adapter-storage
spec:
  accessModes:
  - ReadOnlyMany
  storageClassName: standard
  resources:
    requests:
      storage: 20Gi

---
# Gateway configuration note
# Configure your gateway (Istio, Nginx Ingress, cloud load balancer, etc.)
# to route traffic to this service with appropriate timeout settings.
# 
# Critical setup points:
# 1. Set read timeout > 163s (model load time)
# 2. Set connect timeout > 30s
# 3. Route: /v1/memory/chat/completions → memory-serving Service:8000
# 4. Require API key authentication at gateway level