feat(poimen): restrict the GraphQL server to the memory proxy

poimen-graphql is already ClusterIP on 8090 with no Ingress, so it is off the internet. It had no NetworkPolicy, so every pod in the cluster could reach it. The only caller is poimen-memory, which forwards its /graphql route to MEM_GQL_PORT=8090. No other workload references the service. Allows 8090 from poimen-memory pods only.
2026-11-04 05:38:26 +08:00
9 changed files with 4 additions and 484 deletions
@@ -1,92 +0,0 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: clef-candle
  namespace: llm-serving
  labels:
    app.kubernetes.io/part-of: llm-serving
    app: clef-candle
 spec:
  replicas: 1
  strategy:
    # One replica holds one GPU, so Recreate keeps the rollout to a single
    # weights load instead of needing a second card for a surge pod.
    type: Recreate
  selector:
    matchLabels:
      app: clef-candle
  template:
    metadata:
      labels:
        app.kubernetes.io/part-of: llm-serving
        # The clef-serve Service now selects this label (see clef-serve.yaml),
        # so clients keep addressing clef-serve:80 unchanged. The reference
        # pytorch Deployment must stay at replicas 0 while this runs, or the
        # Service splits traffic across two different implementations.
        app: clef-candle
    spec:
      # Required. Without the nvidia runtime class libcuda.so is never injected
      # and candle-vllm panics in cudarc at startup.
      runtimeClassName: nvidia
      containers:
        - name: candle
          # :sm70 is built by .gitea/workflows/candle-vllm-sm70.yaml on every
          # merge to main. It MUST be a build that contains the GDN conv fix
          # (candle-vllm PR: feat/clef-systemone). Older :sm70 images zero the
          # key channel band in causal_conv1d_fwd, which starves the delta rule
          # and makes every linear-attention layer return zero. The symptom is
          # a well-formed /v1/systemone response with wrong answers, not an error.
          image: forgejo.riotpiao.com/riotpiao-poimen/candle-vllm:sm70
          # :sm70 is a moving tag. Default IfNotPresent keeps a cached copy and a
          # rebuilt image never reaches the pod, so a merged fix looks like it did
          # not land. Always forces the pull on every pod start.
          imagePullPolicy: Always
          # Runs the fixed binary from the models PVC, not the one in the image.
          # The published :sm70 predates the /v1/systemone route and the GDN conv
          # fix (candle-vllm f3c00b2 is merged but CI has not republished the tag),
          # so the in-image binary 404s on the route. The PVC copy at
          # /mnt/models/bin/candle-vllm contains both fixes.
          #
          # RETIRE THIS once CI publishes a :sm70 built from f3c00b2: change the
          # command back to /usr/local/bin/candle-vllm and delete the PVC copy.
          command: ["/mnt/models/bin/candle-vllm"]
          # clef-flash is a hybrid Qwen3_5 model: 8 full-attention layers plus 24
          # GDN layers that carry per-sequence recurrent state. It runs on ONE
          # card because the state is not shardable the way attention KV is.
          args:
            - --w
            - /mnt/models/clef-flash
            - --d
            - "0"
            - --dtype
            - f16
            - --p
            - "8080"
            - --max-num-seqs
            - "4"
            # About 66.8K KV tokens. Clef is a single-shot classifier-style
            # endpoint, so it does not need the coder's pool.
            - --kv-fraction
            - "0.30"
            # Prefix caching stays on: clef requests repeat the schema block.
            - --prefill-chunk-size
            - "1024"
          ports:
            - containerPort: 8080
          resources:
            limits:
              cpu: "8"
              memory: 32Gi
              nvidia.com/gpu: "1"
            requests:
              cpu: "4"
              memory: 16Gi
              nvidia.com/gpu: "1"
          volumeMounts:
            - name: models
              mountPath: /mnt/models
              readOnly: true
      volumes:
        - name: models
          persistentVolumeClaim:
            claimName: llm-models
@@ -33,17 +33,7 @@ metadata:
    app: clef-serve
    app.kubernetes.io/part-of: llm-serving
 spec:
-  # One card. worker-1 holds four V100s: candle-vllm takes one for the coder, and
+  replicas: 1
  # this pod takes one, so two stay free for a TP=2 candle run.
  #
  # Enabled 2026-11-04 to serve /v1/systemone, which the gateway already routes to
  # this Service. It was parked at 0 while candle-vllm was brought up on the same
  # cards, and it served no inference request between 2026-10-30T06:23:40Z and then.
  #
  # Cold start is about four minutes: the pod pip-installs on every boot because the
  # torch base image cannot be pushed to the registry (Cloudflare answers 413 above
  # 100 MB per layer). startupProbe allows for that.
  replicas: 0
  # Recreate, not RollingUpdate. Every GPU card is allocated, so a rolling update leaves
  # the new pod Pending forever waiting for a card the old pod still holds.
  strategy:
@@ -127,11 +117,8 @@ metadata:
  labels:
    app.kubernetes.io/part-of: llm-serving
 spec:
  # Points at the candle-vllm implementation (clef-candle.yaml). Switching back
  # to the reference pytorch server is this line plus that Deployment's replicas;
  # no client changes, because the Service name and ClusterIP do not move.
  selector:
-    app: clef-candle
+    app: clef-serve
  ports:
  - name: http
    port: 80
@@ -1,106 +0,0 @@
 # Qwen2.5-Coder-14B-Instruct, fp16, tensor-parallel across two V100s.
 #
 # Why this model and this precision. The previous workload was Qwen3-Coder-30B-A3B
 # in GPTQ W4A16 on one V100. Profiling that deployment showed SM utilisation at
 # 98% while the memory controller sat at 19%: the V100 has FP16 tensor cores and
 # no INT4 ones, so every decode step ran a dequantisation pass on the SM. That is
 # a hardware-generation mismatch, not a tunable. Sequence throughput matches the
 # arithmetic: at batch 1 the step cost equalled a full-model read (17.6 ms against
 # 15 GB / 900 GB/s = 16.7 ms) while the routed experts alone are ~1.5 GB.
 #
 # Measured on this hardware, same harness, same 16-stream load:
 #
 #   GPTQ W4A16 30B-A3B, TP=1  : ITL 71.5 ms, 215 tok/s aggregate
 #   fp16 14B, TP=2            : ITL 39.7 ms, 287 tok/s aggregate
 #
 # The dense model degrades only 1.4x from 1 to 16 streams (27.9 -> 39.7 ms) while
 # the MoE degrades 4.1x (17.6 -> 71.5 ms), because a dense model reads its weights
 # once per step and amortises across the batch. The MoE pays expert weight traffic
 # per sequence. Single-stream latency is the one case where the MoE still wins
 # (17.6 ms against 27.9 ms), since it activates ~3B parameters per token.
 #
 # TP=2 notes that cost time to establish, so they are recorded here:
 #   - Sharding needs no code change. Every dimension divides by 2: attention heads
 #     40 -> 20, KV heads 8 -> 4, intermediate_size 13824 -> 6912, vocab
 #     152064 -> 76032. Verify this first for any replacement model.
 #   - TP=2 on a GPTQ checkpoint is what crashed (CUDA_ERROR_ILLEGAL_ADDRESS in
 #     forward()) on the previous model. The cause was quantised weight sharding
 #     and the g_idx guard in src/openai/models/linear.rs, not TP itself. A dense
 #     fp16 checkpoint has no g_idx, no packed weights and no scales, so it takes
 #     the plain column/row-parallel path and the crash does not reproduce.
 #   - These are V100-PCIE cards with no NVLink, so every layer's all-reduce
 #     crosses PCIe. It shows up in TTFT (275 ms at 1 stream, 1090 ms at 16)
 #     more than in ITL.
 #   - KV capacity is 11635 MB per rank, about 59K context tokens. The fp16
 #     weights consume the room the KV pool would otherwise use. This is well
 #     short of the 256K the previous model targeted, and it is the price of
 #     leaving the dequantisation path.
 #
 # Weights must be in the PVC as /mnt/models/coder-14b-fp16 (28 GB, 6 shards).
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: candle-vllm
  namespace: llm-serving
  labels:
    app.kubernetes.io/part-of: llm-serving
    app: candle-vllm
 spec:
  replicas: 1
  strategy:
    # One replica holds both GPUs, so a rolling update would need a second pair.
    # Recreate keeps the rollout to a single weights load.
    type: Recreate
  selector:
    matchLabels:
      app: candle-vllm
  template:
    metadata:
      labels:
        app.kubernetes.io/part-of: llm-serving
        app: candle-vllm
    spec:
      # Required. Without this the container runs under runc, libcuda.so is never
      # injected, and candle-vllm panics in cudarc at startup with
      # "Unable to dynamically load the cuda shared library".
      runtimeClassName: nvidia
      containers:
        - name: candle
          image: forgejo.riotpiao.com/riotpiao-poimen/candle-vllm:sm70-gptq-slim
          command: ["/usr/local/bin/candle-vllm"]
          # fp16, not GPTQ, and two ranks, not one. See the header for the
          # measurements and the TP=2 findings.
          args:
            - --w
            - /mnt/models/coder-14b-fp16
            - --d
            - 0,1
            - --dtype
            - f16
            - --p
            - "8080"
            - --max-num-seqs
            - "32"
            # 11.6 GB of KV per rank, about 59K context tokens. The fp16 weights
            # leave no room for more.
            - --kv-fraction
            - "0.85"
          ports:
            - containerPort: 8080
          resources:
            limits:
              cpu: "8"
              memory: 36Gi
              nvidia.com/gpu: "2"
            requests:
              cpu: "4"
              memory: 12Gi
              nvidia.com/gpu: "2"
          volumeMounts:
            - name: models
              mountPath: /mnt/models
              readOnly: true
      volumes:
        - name: models
          persistentVolumeClaim:
            claimName: llm-models
@@ -1,40 +0,0 @@
 # Stable client-facing name for the coder chat model, deliberately decoupled
 # from the process that serves it.
 #
 # poimen-memory addresses a Service, not a pod and not an InferenceService. The
 # vLLM -> candle-vllm handoff therefore changes this file's selector and nothing
 # else, so the client config never changes again and there is no window in which
 # the host name is wrong.
 #
 # The previous KTML predictor requested two V100s and never reached Ready, so it
 # held half the GPU pool while serving nothing. candle-vllm replaces it.
 #
 # Two ports on purpose:
 #   - 80    matches every other predictor Service in this namespace, and matches
 #           a client base URL that carries no port number.
 #   - 8080  is the container port. Publishing it directly avoids a port
 #           translation that is easy to get wrong in a curl.
 #
 # Do not delete and recreate this Service: a delete drops the ClusterIP, and a
 # client that cached the name during the gap fails until it re-resolves.
 apiVersion: v1
 kind: Service
 metadata:
  name: coder-predictor
  namespace: llm-serving
  labels:
    app.kubernetes.io/part-of: llm-serving
    app: coder-predictor
 spec:
  type: ClusterIP
  selector:
    app: candle-vllm
  ports:
    - name: http
      protocol: TCP
      port: 80
      targetPort: 8080
    - name: http-8080
      protocol: TCP
      port: 8080
      targetPort: 8080
@@ -1,134 +0,0 @@
 apiVersion: serving.kserve.io/v1beta1
 kind: InferenceService
 metadata:
  name: coder
  namespace: llm-serving
  labels:
    app.kubernetes.io/name: llm-coder
    app.kubernetes.io/part-of: llm-serving
  annotations:
    serving.kserve.io/deploymentMode: RawDeployment
 spec:
  predictor:
    minReplicas: 1
    maxReplicas: 1
    nodeSelector:
      kubernetes.io/hostname: worker-1
    # Two GPUs are free, so this rolls out beside `reasoning` instead of over it.
    # Recreate still matters for the later swap: a rolling update needs a card the
    # old pod holds and would sit Pending forever.
    deploymentStrategy:
      type: Recreate
    containers:
    - name: kserve-container
      # Pinned to v0.11.0 on purpose. This is the last image measured to work on this
      # node: torch 2.8.0+cu128, arch list sm_70..sm_120, driver 580.159.04.
      # v0.15.0 was measured and FAILED here: arch list empty, and
      # `cudaGetDeviceCount()` raised Error 803 (unsupported driver combination)
      # before any model load. Do not raise this tag without re-running that probe.
      image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
      args:
      # Qwen3-Coder-30B-A3B-Instruct: 30.5B total, 3.3B active. Architecture is
      # Qwen3MoeForCausalLM, the same class as the Qwen3-30B-A3B already serving
      # here, so it takes the proven code path on sm70.
      #
      # Qwen ships this model only as BF16 (61.1 GB) and FP8 (no FP8 on Volta).
      # 4-bit GPTQ is the only form that fits one 32 GB card. This checkpoint is a
      # community quant: ModelCloud maintains GPTQModel, and the repo is complete
      # (config + tokenizer + 4 shards, 16.9 GB). Verified by loading, not by trust.
      #
      # Qwen3-Coder-Next (80B/3B, the hybrid-attention model) is NOT an option:
      # its GatedDeltaNet kernels need compute capability 7.5+ and the `fla` module,
      # and sm70 has neither. Measured, not assumed.
      - --model=ModelCloud/Qwen3-Coder-30B-A3B-Instruct-GPTQMODEL-W4A16-A
      # `reasoning` stays a served name so an existing client that sends
      # model=reasoning keeps working after the DNS rename. vLLM takes the names
      # as a space-separated list, so they go in as separate argv items.
      - --served-model-name
      - coder
      - reasoning
      - --quantization=gptq
      - --dtype=float16
      - --kv-cache-dtype=auto
      # Native window is 262144 and rope_scaling is null, so 256K needs no YaRN.
      # One card cannot hold it: measured KV pool on a single V100 was 132,672
      # tokens. TP=2 halves the weights per card (7.8 GiB each), which frees room
      # for the larger pool. Measured after boot: see the KV line in the log.
      - --max-model-len=262144
      - --tensor-parallel-size=2
      - --pipeline-parallel-size=1
      - --gpu-memory-utilization=0.90
      # A single 262,144-token request consumes the whole pool, so keep few slots.
      - --max-num-seqs=4
      - --max-num-batched-tokens=8192
      - --long-prefill-token-threshold=2048
      - --enable-chunked-prefill
      - --enable-prefix-caching
      # Measured: with the default (custom all-reduce ON) the engine finished CUDA graph
      # capture in 6s at 17:01:56 and then went silent for 14+ minutes, emitting only
      # "No available shared memory broadcast block found in 60 seconds". The front-end
      # never received the ready message. vLLM's custom all-reduce registers its own
      # CUDA graph addresses and deadlocks on this Volta/PCIe pair. NCCL is the fallback
      # and is correct here, because TP=2 across two PCIe V100s gains nothing from the
      # custom kernel anyway.
      - --disable-custom-all-reduce
      - --reasoning-parser=qwen3
      - --enable-auto-tool-choice
      - --tool-call-parser=hermes
      - --host=0.0.0.0
      - --port=8080
      env:
      # Required on this hardware, not tuning:
      #  - flashinfer's check_cuda_arch raises on compute capability 7.0,
      #    so the sampler must stay off
      #  - FlashInfer and FlashAttention-2 need sm75+, so Triton serves attention
      - name: VLLM_USE_FLASHINFER_SAMPLER
        value: "0"
      - name: VLLM_ATTENTION_BACKEND
        value: TRITON_ATTN
      - name: HF_HOME
        value: /mnt/models
      ports:
      - containerPort: 8080
        protocol: TCP
      readinessProbe:
        httpGet:
          path: /health
          port: 8080
        periodSeconds: 10
      startupProbe:
        httpGet:
          path: /health
          port: 8080
        # TP=2 at 256K needs ~10 min before the API binds: 22s weights, 80s torch.compile,
        # 85s KV profiling, then graph capture. 120 x 15s = 30 min of headroom.
        failureThreshold: 120
        periodSeconds: 15
      resources:
        # worker-1 exposes only 63,495 MiB of allocatable memory, so a 24Gi request
        # never schedules. Two vLLM workers keep the weights on the GPU and need
        # little host RAM, so the request stays small and the limit allows burst.
        requests:
          cpu: "8"
          memory: 12Gi
          nvidia.com/gpu: "2"
        limits:
          cpu: "16"
          memory: 40Gi
          nvidia.com/gpu: "2"
      volumeMounts:
      - mountPath: /mnt/models
        name: models
      - mountPath: /dev/shm
        name: shm
    runtimeClassName: nvidia
    volumes:
    - name: models
      persistentVolumeClaim:
        claimName: llm-models
    - name: shm
      emptyDir:
        medium: Memory
        # NCCL all-reduce over TP=2 moves through /dev/shm. The 64Mi default is
        # too small and shows up as an init hang or a bus error.
        sizeLimit: 8Gi
@@ -13,24 +13,8 @@ resources:
  # clef-flash runs through clef-serve.yaml, not through an SGLang runtime. SGLang needs
  # v0.5.22+, published only on CUDA 13, and CUDA 13 dropped Volta kernels. The model
  # instead runs the implementation it ships (joint_schema_model.py) in fp16 on one V100.
  # coder-14b-fp16.yaml owns the `candle-vllm` Deployment name and the
  # `app: candle-vllm` label that coder-service.yaml selects. It must be the only
  # workload with that label, or the Service splits traffic across two models.
  #
  # fp16 on two V100s, not GPTQ on one: measured 39.7 ms ITL and 287 tok/s at 16
  # streams against 71.5 ms and 215 tok/s for the 4-bit MoE. The V100 has FP16
  # tensor cores and no INT4 ones, so 4-bit pays dequantisation on the SM while
  # the memory controller idles at 19%.
  # clef-candle.yaml serves clef-flash on candle-vllm (hybrid Qwen3_5, one GPU)
  # and owns the clef-candle label that clef-serve.yaml's Service now selects.
  # It must not sync against an image built before the GDN conv fix.
  - clef-candle.yaml
  - coder-14b-fp16.yaml
  - clef-serve.yaml
-  # Stable client-facing name for the chat model. poimen-memory points at this Service,
+  - reasoning.yaml
  # so the vLLM -> candle-vllm handoff is a selector edit here and no client change.
  - coder-service.yaml
  - reasoning-long-service.yaml
  # ornith is retired: its InferenceService still reports Ready=True while running zero
  # replicas, because desired equals available at 0. Its manifest was never committed,
  # so it is left unmanaged and is not listed here. Delete the CR when convenient.
@@ -66,21 +66,6 @@ spec:
          port: 80
        - protocol: TCP
          port: 8000
    # Allow from poimen namespace (poimen-memory ingest + query).
    # The client-side egress rule is the other half and must stay in step:
    # NetworkPolicy poimen/poimen-memory-llm-egress permits port 8080 to pods
    # labelled part-of=llm-serving, which is the same selector used here. This
    # policy had no matching ingress rule at all, so the packet was allowed out
    # and dropped here. The client then saw a bare timeout and the predictor log
    # stayed empty, which is why this failure looked like a hung inference engine
    # for 34 days.
    - from:
        - namespaceSelector:
            matchLabels:
              kubernetes.io/metadata.name: poimen
      ports:
        - protocol: TCP
          port: 8080
    # Allow intra-namespace (pod-to-pod within llm-serving)
    - from:
        - podSelector:
@@ -1,43 +0,0 @@
 # Stable client-facing name for the chat model, deliberately decoupled from the
 # process that currently serves it.
 #
 # poimen-memory addresses this Service, not a pod and not an InferenceService. That
 # means the vLLM -> candle-vllm handoff changes only this file's selector. The client
 # config never changes again, so the ingest cutover has no window in which the host
 # name is wrong.
 #
 # Two ports on purpose:
 #   - 80    matches every other predictor Service in this namespace, and matches
 #           poimen-memory's LLM_CHAT_BASE, which carries no port number.
 #   - 8080  is the container port. It is also the only port llm-serving-ingress and
 #           poimen-memory-llm-egress permit, and the port the handoff readiness check
 #           uses. Publishing it directly avoids a port translation that is easy to get
 #           wrong in a curl.
 #
 # The selector names the vLLM reasoning pod today. To hand off, replace the three
 # selector labels with candle-vllm's pod labels and nothing else. Do not delete and
 # recreate this Service: a delete drops the ClusterIP, and a client that cached the
 # name during the gap will fail until it re-resolves.
 apiVersion: v1
 kind: Service
 metadata:
  name: reasoning-long-predictor
  namespace: llm-serving
  labels:
    app.kubernetes.io/part-of: llm-serving
    app: reasoning-long-predictor
 spec:
  type: ClusterIP
  selector:
    app.kubernetes.io/name: llm-reasoning
    component: predictor
    serving.kserve.io/inferenceservice: reasoning
  ports:
    - name: http
      protocol: TCP
      port: 80
      targetPort: 8080
    - name: http-8080
      protocol: TCP
      port: 8080
      targetPort: 8080
@@ -30,28 +30,7 @@ spec:
      # Measured on this cluster, one V100-32GB, 64k context: 15.55 GiB of weights,
      # 137k tokens of KV cache, 2.09x concurrency at 65536 tokens per request.
      - --model=Qwen/Qwen3-30B-A3B-GPTQ-Int4
-      # Two served names, one pod, on purpose. `reasoning-long` is the name the goal map
+      - --served-model-name=reasoning
      # uses for this model, and the name qwen-finetune wiring targets. It is NOT a
      # separate InferenceService: this model already occupies the only free card, and a
      # second pod on the same weights cannot schedule (4 of 4 GPUs allocated: coder 2,
      # this 1, clef-serve 1). Serving both names from one process costs nothing and
      # keeps the two callers working without a second weights copy.
      #
      # `reasoning` is deliberately still served here. poimen-memory pins
      # LLM_CHAT_BASE=reasoning-predictor, so dropping the name here breaks ingest
      # immediately. coder-predictor also serves `reasoning`; that ambiguity is known
      # and documented in the PR. Resolve it by moving the client host, not by
      # removing the name here.
      #
      # The two names are separate list entries, NOT one comma-joined value. This
      # image's parser returns ['reasoning,reasoning-long'] for the comma form, i.e. a
      # single model whose name contains a comma, and api_server.py then iterates
      # args.served_model_name to build the model list. Verified against vLLM 0.11.0:
      #   FlexibleArgumentParser().parse_args([... "--served-model-name", "a", "b"])
      #     -> ['a', 'b']
      - --served-model-name
      - reasoning
      - reasoning-long
      - --quantization=gptq
      - --dtype=float16
      - --kv-cache-dtype=auto