apiVersion: serving.kserve.io/v1beta1
kind: InferenceService
metadata:
  annotations:
    serving.kserve.io/deploymentMode: RawDeployment
  labels:
    app.kubernetes.io/name: llm-reranker
    app.kubernetes.io/part-of: llm-serving
  name: reranker
  namespace: llm-serving
spec:
  predictor:
    containers:
    - args:
      - --model-id=BAAI/bge-reranker-base
      - --port=8080
      - --hostname=0.0.0.0
      - --auto-truncate
      env:
      - name: HUGGINGFACE_HUB_CACHE
        value: /mnt/models
      image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.8.2@sha256:4d632b76bd14cb57044a1ffb0ad48ab0ba4939e705a9a615ccc740658575c26e
      name: kserve-container
      ports:
      - containerPort: 8080
        protocol: TCP
      readinessProbe:
        httpGet:
          path: /health
          port: 8080
        periodSeconds: 10
      resources:
        limits:
          cpu: '16'
          memory: 8Gi
        requests:
          cpu: '8'
          memory: 4Gi
      startupProbe:
        failureThreshold: 60
        httpGet:
          path: /health
          port: 8080
        periodSeconds: 10
      volumeMounts:
      - mountPath: /mnt/models
        name: models
    maxReplicas: 1
    minReplicas: 1
    nodeSelector:
      kubernetes.io/hostname: worker-1
    volumes:
    - name: models
      persistentVolumeClaim:
        claimName: llm-models