apiVersion: serving.kserve.io/v1beta1
kind: InferenceService
metadata:
  annotations:
    serving.kserve.io/deploymentMode: RawDeployment
    # Kong reads its timeouts from the Kubernetes Service, not the Ingress —
    # Ingress annotations configure Route entities (strip-path, methods,
    # plugins), these configure the Service entity. They were on
    # llm-chat-ornith's Ingress and therefore ignored, leaving Kong's 60s
    # default in force. KServe propagates InferenceService annotations to the
    # Service it generates, which is how they reach Kong from here.
    #
    # This was invisible while OLLAMA_KEEP_ALIVE=-1 kept the model resident: no
    # request ever waited on a cold load. A pod restart flushes VRAM, and
    # loading ornith:35b takes longer than 60s, so the first request after any
    # restart returned 504.
    konghq.com/connect-timeout: "10000"
    konghq.com/read-timeout: "3600000"
    konghq.com/write-timeout: "3600000"
  labels:
    app.kubernetes.io/name: llm-ornith
    app.kubernetes.io/part-of: llm-serving
  name: ornith
  namespace: llm-serving
spec:
  predictor:
    containers:
    - command:
      - /bin/sh
      - -c
      - 'set -e

        ollama serve &

        SERVE_PID=$!

        until ollama list >/dev/null 2>&1; do sleep 2; done

        ollama pull ornith:35b

        ollama run ornith:35b "ok" >/dev/null 2>&1 || true

        wait $SERVE_PID

        '
      env:
      - name: OLLAMA_HOST
        value: 0.0.0.0:8080
      - name: OLLAMA_MODELS
        value: /mnt/models/ollama
      - name: OLLAMA_CONTEXT_LENGTH
        value: '32768'
      - name: OLLAMA_KEEP_ALIVE
        value: '-1'
      - name: OLLAMA_NUM_PARALLEL
        value: '1'
      # qwen2.5:3b-instruct moved to its own dedicated GPU (llm-serving/grm.yaml)
      # so verification/judge traffic no longer contends with ornith:35b's
      # planner/implementer traffic on this one -- single model here now.
      - name: OLLAMA_MAX_LOADED_MODELS
        value: '1'
      image: ollama/ollama:0.32.9@sha256:1685741456770df6e3cceb2a945a5f75e020f658d1701509668d6f4688f1dd3f
      name: kserve-container
      ports:
      - containerPort: 8080
        protocol: TCP
      readinessProbe:
        exec:
          command:
          - /bin/sh
          - -c
          - ollama ps 2>/dev/null | grep -q ornith
        periodSeconds: 10
      resources:
        limits:
          cpu: '16'
          memory: 16Gi
          nvidia.com/gpu: '1'
        requests:
          cpu: '8'
          memory: 8Gi
          nvidia.com/gpu: '1'
      startupProbe:
        exec:
          command:
          - /bin/sh
          - -c
          - ollama ps 2>/dev/null | grep -q ornith
        failureThreshold: 120
        periodSeconds: 15
      volumeMounts:
      - mountPath: /mnt/models
        name: models
    deploymentStrategy:
      type: Recreate
    maxReplicas: 1
    minReplicas: 1
    nodeSelector:
      kubernetes.io/hostname: worker-1
    runtimeClassName: nvidia
    volumes:
    - name: models
      persistentVolumeClaim:
        claimName: llm-models