rock/homelab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(agent-pod): remote tui session for multi-agent

Browse Source
This commit is contained in:
Story Crater Bot
2026-08-18 15:08:04 -07:00
parent 0fe3d25936
commit afb9b35292
34 changed files with 1874 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
k8s/apps/llm-serving/embeddings.yaml
+56
View File
@@ -0,0 +1,56 @@
apiVersion: serving.kserve.io/v1beta1
kind: InferenceService
metadata:
annotations:
serving.kserve.io/deploymentMode: RawDeployment
labels:
app.kubernetes.io/name: llm-embeddings
app.kubernetes.io/part-of: llm-serving
name: embeddings
namespace: llm-serving
spec:
predictor:
containers:
- args:
- --model-id=nomic-ai/nomic-embed-text-v2-moe
- --port=8080
- --hostname=0.0.0.0
- --auto-truncate
env:
- name: HUGGINGFACE_HUB_CACHE
value: /mnt/models
image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.8.2@sha256:4d632b76bd14cb57044a1ffb0ad48ab0ba4939e705a9a615ccc740658575c26e
name: kserve-container
ports:
- containerPort: 8080
protocol: TCP
readinessProbe:
httpGet:
path: /health
port: 8080
periodSeconds: 10
resources:
limits:
cpu: '16'
memory: 8Gi
requests:
cpu: '8'
memory: 4Gi
startupProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8080
periodSeconds: 10
volumeMounts:
- mountPath: /mnt/models
name: models
maxReplicas: 1
minReplicas: 1
nodeSelector:
kubernetes.io/hostname: worker-1
volumes:
- name: models
persistentVolumeClaim:
claimName: llm-models
k8s/apps/llm-serving/kustomization.yaml
+19
View File
@@ -0,0 +1,19 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
# Explicit allowlist, matching k8s/apps/api. Anything added to this directory
# and not listed here is silently dropped — no error, no drift shown.
#
# These five were adopted from live state on 2026-08-15; they had been applied
# by hand and carried no ArgoCD ownership. Each was exported and verified with
# `kubectl diff -f <file>` returning empty before the Application below was
# created, so the first sync was a no-op rather than a redeploy. Re-verify that
# way after any edit here: a GPU predictor restart is a weights reload measured
# in tens of seconds, not a rolling update.
resources:
- embeddings.yaml
- ornith.yaml
- reasoning.yaml
- reranker.yaml
- verifier.yaml
# No namespace transformer: every file sets its own, and the transformer would
# rewrite metadata.namespace on anything cross-namespace added later.
k8s/apps/llm-serving/ornith.yaml
+108
View File
@@ -0,0 +1,108 @@
apiVersion: serving.kserve.io/v1beta1
kind: InferenceService
metadata:
annotations:
serving.kserve.io/deploymentMode: RawDeployment
# Kong reads its timeouts from the Kubernetes Service, not the Ingress —
# Ingress annotations configure Route entities (strip-path, methods,
# plugins), these configure the Service entity. They were on
# llm-chat-ornith's Ingress and therefore ignored, leaving Kong's 60s
# default in force. KServe propagates InferenceService annotations to the
# Service it generates, which is how they reach Kong from here.
#
# This was invisible while OLLAMA_KEEP_ALIVE=-1 kept the model resident: no
# request ever waited on a cold load. A pod restart flushes VRAM, and
# loading ornith:35b takes longer than 60s, so the first request after any
# restart returned 504.
konghq.com/connect-timeout: "10000"
konghq.com/read-timeout: "3600000"
konghq.com/write-timeout: "3600000"
labels:
app.kubernetes.io/name: llm-ornith
app.kubernetes.io/part-of: llm-serving
name: ornith
namespace: llm-serving
spec:
predictor:
containers:
- command:
- /bin/sh
- -c
- 'set -e
ollama serve &
SERVE_PID=$!
until ollama list >/dev/null 2>&1; do sleep 2; done
ollama pull ornith:35b
ollama pull qwen2.5:3b-instruct
ollama run ornith:35b "ok" >/dev/null 2>&1 || true
ollama run qwen2.5:3b-instruct "ok" >/dev/null 2>&1 || true
wait $SERVE_PID
'
env:
- name: OLLAMA_HOST
value: 0.0.0.0:8080
- name: OLLAMA_MODELS
value: /mnt/models/ollama
- name: OLLAMA_CONTEXT_LENGTH
value: '32768'
- name: OLLAMA_KEEP_ALIVE
value: '-1'
- name: OLLAMA_NUM_PARALLEL
value: '1'
- name: OLLAMA_MAX_LOADED_MODELS
value: '2'
image: ollama/ollama:0.32.9@sha256:1685741456770df6e3cceb2a945a5f75e020f658d1701509668d6f4688f1dd3f
name: kserve-container
ports:
- containerPort: 8080
protocol: TCP
readinessProbe:
exec:
command:
- /bin/sh
- -c
- ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
grep -q qwen2.5
periodSeconds: 10
resources:
limits:
cpu: '16'
memory: 16Gi
nvidia.com/gpu: '1'
requests:
cpu: '8'
memory: 8Gi
nvidia.com/gpu: '1'
startupProbe:
exec:
command:
- /bin/sh
- -c
- ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
grep -q qwen2.5
failureThreshold: 120
periodSeconds: 15
volumeMounts:
- mountPath: /mnt/models
name: models
deploymentStrategy:
type: Recreate
maxReplicas: 1
minReplicas: 1
nodeSelector:
kubernetes.io/hostname: worker-1
runtimeClassName: nvidia
volumes:
- name: models
persistentVolumeClaim:
claimName: llm-models
k8s/apps/llm-serving/reasoning.yaml
+81
View File
@@ -0,0 +1,81 @@
apiVersion: serving.kserve.io/v1beta1
kind: InferenceService
metadata:
annotations:
serving.kserve.io/deploymentMode: RawDeployment
labels:
app.kubernetes.io/name: llm-reasoning
app.kubernetes.io/part-of: llm-serving
name: reasoning
namespace: llm-serving
spec:
predictor:
containers:
- args:
- --model=unsloth/DeepSeek-R1-Distill-Qwen-32B-bnb-4bit
- --served-model-name=reasoning
- --quantization=bitsandbytes
- --dtype=float16
- --kv-cache-dtype=auto
- --tensor-parallel-size=1
- --max-model-len=16384
- --gpu-memory-utilization=0.90
- --max-num-seqs=4
- --enable-chunked-prefill
- --enable-prefix-caching
- --reasoning-parser=deepseek_r1
- --host=0.0.0.0
- --port=8080
env:
- name: VLLM_USE_FLASHINFER_SAMPLER
value: '0'
- name: VLLM_ATTENTION_BACKEND
value: TRITON_ATTN
- name: HF_HOME
value: /mnt/models
image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
name: kserve-container
ports:
- containerPort: 8080
protocol: TCP
readinessProbe:
httpGet:
path: /health
port: 8080
periodSeconds: 10
resources:
limits:
cpu: '16'
memory: 16Gi
nvidia.com/gpu: '1'
requests:
cpu: '8'
memory: 8Gi
nvidia.com/gpu: '1'
startupProbe:
failureThreshold: 80
httpGet:
path: /health
port: 8080
periodSeconds: 15
volumeMounts:
- mountPath: /mnt/models
name: models
- mountPath: /dev/shm
name: shm
deploymentStrategy:
type: Recreate
maxReplicas: 2
minReplicas: 2
nodeSelector:
kubernetes.io/hostname: worker-1
runtimeClassName: nvidia
volumes:
- name: models
persistentVolumeClaim:
claimName: llm-models
- emptyDir:
medium: Memory
sizeLimit: 2Gi
name: shm
k8s/apps/llm-serving/reranker.yaml
+56
View File
@@ -0,0 +1,56 @@
apiVersion: serving.kserve.io/v1beta1
kind: InferenceService
metadata:
annotations:
serving.kserve.io/deploymentMode: RawDeployment
labels:
app.kubernetes.io/name: llm-reranker
app.kubernetes.io/part-of: llm-serving
name: reranker
namespace: llm-serving
spec:
predictor:
containers:
- args:
- --model-id=BAAI/bge-reranker-base
- --port=8080
- --hostname=0.0.0.0
- --auto-truncate
env:
- name: HUGGINGFACE_HUB_CACHE
value: /mnt/models
image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.8.2@sha256:4d632b76bd14cb57044a1ffb0ad48ab0ba4939e705a9a615ccc740658575c26e
name: kserve-container
ports:
- containerPort: 8080
protocol: TCP
readinessProbe:
httpGet:
path: /health
port: 8080
periodSeconds: 10
resources:
limits:
cpu: '16'
memory: 8Gi
requests:
cpu: '8'
memory: 4Gi
startupProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8080
periodSeconds: 10
volumeMounts:
- mountPath: /mnt/models
name: models
maxReplicas: 1
minReplicas: 1
nodeSelector:
kubernetes.io/hostname: worker-1
volumes:
- name: models
persistentVolumeClaim:
claimName: llm-models
k8s/apps/llm-serving/verifier.yaml
+76
View File
@@ -0,0 +1,76 @@
apiVersion: serving.kserve.io/v1beta1
kind: InferenceService
metadata:
annotations:
serving.kserve.io/deploymentMode: RawDeployment
labels:
app.kubernetes.io/name: llm-verifier
app.kubernetes.io/part-of: llm-serving
name: verifier
namespace: llm-serving
spec:
predictor:
containers:
- args:
- --model=Qwen/Qwen2.5-Math-PRM-7B
- --served-model-name=verifier
- --runner=pooling
- --dtype=float16
- --tensor-parallel-size=1
- --max-model-len=4096
- --max-num-seqs=8
- --host=0.0.0.0
- --port=8080
env:
- name: VLLM_USE_FLASHINFER_SAMPLER
value: '0'
- name: VLLM_ATTENTION_BACKEND
value: XFORMERS
- name: HF_HOME
value: /mnt/models
image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
name: kserve-container
ports:
- containerPort: 8080
protocol: TCP
readinessProbe:
httpGet:
path: /health
port: 8080
periodSeconds: 10
resources:
limits:
cpu: '16'
memory: 16Gi
nvidia.com/gpu: '1'
requests:
cpu: '4'
memory: 8Gi
nvidia.com/gpu: '1'
startupProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8080
periodSeconds: 15
volumeMounts:
- mountPath: /mnt/models
name: models
- mountPath: /dev/shm
name: shm
deploymentStrategy:
type: Recreate
maxReplicas: 1
minReplicas: 1
nodeSelector:
kubernetes.io/hostname: worker-1
runtimeClassName: nvidia
volumes:
- name: models
persistentVolumeClaim:
claimName: llm-models
- emptyDir:
medium: Memory
sizeLimit: 1Gi
name: shm