homelab/k8s/apps/llm-serving/ornith.yaml

apiVersion: serving.kserve.io/v1beta1
kind: InferenceService
metadata:
  annotations:
    serving.kserve.io/deploymentMode: RawDeployment
    # The konghq.com/{connect,read,write}-timeout annotations that used to live
    # here went with Kong (retired 2026-08-19). They existed because Kong read
    # its upstream timeouts off the Kubernetes Service, and its 60s default cut
    # off the first request after any pod restart — a restart flushes VRAM and
    # reloading ornith:35b takes longer than that. OLLAMA_KEEP_ALIVE=-1 hid the
    # problem in steady state.
    #
    # The equivalent budget now belongs to the Go gateway's per-route timeout
    # config in rock/homelab-frontend, not to an annotation on this object.
  labels:
    app.kubernetes.io/name: llm-ornith
    app.kubernetes.io/part-of: llm-serving
  name: ornith
  namespace: llm-serving
spec:
  predictor:
    containers:
    - command:
      - /bin/sh
      - -c
      - 'set -e

        ollama serve &

        SERVE_PID=$!

        until ollama list >/dev/null 2>&1; do sleep 2; done

        ollama pull ornith:35b

        ollama pull qwen2.5:3b-instruct

        ollama run ornith:35b "ok" >/dev/null 2>&1 || true

        ollama run qwen2.5:3b-instruct "ok" >/dev/null 2>&1 || true

        wait $SERVE_PID

        '
      env:
      - name: OLLAMA_HOST
        value: 0.0.0.0:8080
      - name: OLLAMA_MODELS
        value: /mnt/models/ollama
      - name: OLLAMA_CONTEXT_LENGTH
        value: '32768'
      - name: OLLAMA_KEEP_ALIVE
        value: '-1'
      - name: OLLAMA_NUM_PARALLEL
        value: '1'
      - name: OLLAMA_MAX_LOADED_MODELS
        value: '2'
      image: ollama/ollama:0.32.9@sha256:1685741456770df6e3cceb2a945a5f75e020f658d1701509668d6f4688f1dd3f
      name: kserve-container
      ports:
      - containerPort: 8080
        protocol: TCP
      readinessProbe:
        exec:
          command:
          - /bin/sh
          - -c
          - ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
            grep -q qwen2.5
        periodSeconds: 10
      resources:
        limits:
          cpu: '16'
          memory: 16Gi
          nvidia.com/gpu: '1'
        requests:
          cpu: '8'
          memory: 8Gi
          nvidia.com/gpu: '1'
      startupProbe:
        exec:
          command:
          - /bin/sh
          - -c
          - ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
            grep -q qwen2.5
        failureThreshold: 120
        periodSeconds: 15
      volumeMounts:
      - mountPath: /mnt/models
        name: models
    deploymentStrategy:
      type: Recreate
    # 2 replicas -- each its own GPU, each loading both ornith:35b and
    # qwen2.5:3b-instruct -- so 2 concurrent implementer-style calls each
    # get an independent instance instead of contending on one, at the
    # cost of judge/qwen traffic still sharing whichever replica an
    # implementer call also lands on.
    maxReplicas: 2
    minReplicas: 2
    nodeSelector:
      kubernetes.io/hostname: worker-1
    runtimeClassName: nvidia
    volumes:
    - name: models
      persistentVolumeClaim:
        claimName: llm-models