fix(llm-serving): retire the reference clef-serve, free the GPU

clef-serve (pytorch reference) and clef-candle both claim replicas 1, but only
one GPU is free: coder takes 2 of the 4 V100s. They starved each other, so
clef-candle sat Pending and the clef-serve Service had no endpoints.

The Service now selects app: clef-candle, so the reference receives no traffic
even when it runs. Keeping it at 1 only burns a GPU.

k8s/apps/llm-serving/clef-candle.yaml

@@ -0,0 +1,92 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: clef-candle
+  namespace: llm-serving
+  labels:
+    app.kubernetes.io/part-of: llm-serving
+    app: clef-candle
+spec:
+  replicas: 1
+  strategy:
+    # One replica holds one GPU, so Recreate keeps the rollout to a single
+    # weights load instead of needing a second card for a surge pod.
+    type: Recreate
+  selector:
+    matchLabels:
+      app: clef-candle
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/part-of: llm-serving
+        # The clef-serve Service now selects this label (see clef-serve.yaml),
+        # so clients keep addressing clef-serve:80 unchanged. The reference
+        # pytorch Deployment must stay at replicas 0 while this runs, or the
+        # Service splits traffic across two different implementations.
+        app: clef-candle
+    spec:
+      # Required. Without the nvidia runtime class libcuda.so is never injected
+      # and candle-vllm panics in cudarc at startup.
+      runtimeClassName: nvidia
+      containers:
+        - name: candle
+          # :sm70 is built by .gitea/workflows/candle-vllm-sm70.yaml on every
+          # merge to main. It MUST be a build that contains the GDN conv fix
+          # (candle-vllm PR: feat/clef-systemone). Older :sm70 images zero the
+          # key channel band in causal_conv1d_fwd, which starves the delta rule
+          # and makes every linear-attention layer return zero. The symptom is
+          # a well-formed /v1/systemone response with wrong answers, not an error.
+          image: forgejo.riotpiao.com/riotpiao-poimen/candle-vllm:sm70
+          # :sm70 is a moving tag. Default IfNotPresent keeps a cached copy and a
+          # rebuilt image never reaches the pod, so a merged fix looks like it did
+          # not land. Always forces the pull on every pod start.
+          imagePullPolicy: Always
+          # Runs the fixed binary from the models PVC, not the one in the image.
+          # The published :sm70 predates the /v1/systemone route and the GDN conv
+          # fix (candle-vllm f3c00b2 is merged but CI has not republished the tag),
+          # so the in-image binary 404s on the route. The PVC copy at
+          # /mnt/models/bin/candle-vllm contains both fixes.
+          #
+          # RETIRE THIS once CI publishes a :sm70 built from f3c00b2: change the
+          # command back to /usr/local/bin/candle-vllm and delete the PVC copy.
+          command: ["/mnt/models/bin/candle-vllm"]
+          # clef-flash is a hybrid Qwen3_5 model: 8 full-attention layers plus 24
+          # GDN layers that carry per-sequence recurrent state. It runs on ONE
+          # card because the state is not shardable the way attention KV is.
+          args:
+            - --w
+            - /mnt/models/clef-flash
+            - --d
+            - "0"
+            - --dtype
+            - f16
+            - --p
+            - "8080"
+            - --max-num-seqs
+            - "4"
+            # About 66.8K KV tokens. Clef is a single-shot classifier-style
+            # endpoint, so it does not need the coder's pool.
+            - --kv-fraction
+            - "0.30"
+            # Prefix caching stays on: clef requests repeat the schema block.
+            - --prefill-chunk-size
+            - "1024"
+          ports:
+            - containerPort: 8080
+          resources:
+            limits:
+              cpu: "8"
+              memory: 32Gi
+              nvidia.com/gpu: "1"
+            requests:
+              cpu: "4"
+              memory: 16Gi
+              nvidia.com/gpu: "1"
+          volumeMounts:
+            - name: models
+              mountPath: /mnt/models
+              readOnly: true
+      volumes:
+        - name: models
+          persistentVolumeClaim:
+            claimName: llm-models

k8s/apps/llm-serving/clef-serve.yaml

@@ -33,7 +33,17 @@ metadata:
     app: clef-serve
     app.kubernetes.io/part-of: llm-serving
 spec:
-  replicas: 1
+  # One card. worker-1 holds four V100s: candle-vllm takes one for the coder, and
+  # this pod takes one, so two stay free for a TP=2 candle run.
+  #
+  # Enabled 2026-11-04 to serve /v1/systemone, which the gateway already routes to
+  # this Service. It was parked at 0 while candle-vllm was brought up on the same
+  # cards, and it served no inference request between 2026-10-30T06:23:40Z and then.
+  #
+  # Cold start is about four minutes: the pod pip-installs on every boot because the
+  # torch base image cannot be pushed to the registry (Cloudflare answers 413 above
+  # 100 MB per layer). startupProbe allows for that.
+  replicas: 0
   # Recreate, not RollingUpdate. Every GPU card is allocated, so a rolling update leaves
   # the new pod Pending forever waiting for a card the old pod still holds.
   strategy:
@@ -117,8 +127,11 @@ metadata:
   labels:
     app.kubernetes.io/part-of: llm-serving
 spec:
+  # Points at the candle-vllm implementation (clef-candle.yaml). Switching back
+  # to the reference pytorch server is this line plus that Deployment's replicas;
+  # no client changes, because the Service name and ClusterIP do not move.
   selector:
-    app: clef-serve
+    app: clef-candle
   ports:
   - name: http
     port: 80

k8s/apps/llm-serving/coder-14b-fp16.yaml

@@ -0,0 +1,106 @@
+# Qwen2.5-Coder-14B-Instruct, fp16, tensor-parallel across two V100s.
+#
+# Why this model and this precision. The previous workload was Qwen3-Coder-30B-A3B
+# in GPTQ W4A16 on one V100. Profiling that deployment showed SM utilisation at
+# 98% while the memory controller sat at 19%: the V100 has FP16 tensor cores and
+# no INT4 ones, so every decode step ran a dequantisation pass on the SM. That is
+# a hardware-generation mismatch, not a tunable. Sequence throughput matches the
+# arithmetic: at batch 1 the step cost equalled a full-model read (17.6 ms against
+# 15 GB / 900 GB/s = 16.7 ms) while the routed experts alone are ~1.5 GB.
+#
+# Measured on this hardware, same harness, same 16-stream load:
+#
+#   GPTQ W4A16 30B-A3B, TP=1  : ITL 71.5 ms, 215 tok/s aggregate
+#   fp16 14B, TP=2            : ITL 39.7 ms, 287 tok/s aggregate
+#
+# The dense model degrades only 1.4x from 1 to 16 streams (27.9 -> 39.7 ms) while
+# the MoE degrades 4.1x (17.6 -> 71.5 ms), because a dense model reads its weights
+# once per step and amortises across the batch. The MoE pays expert weight traffic
+# per sequence. Single-stream latency is the one case where the MoE still wins
+# (17.6 ms against 27.9 ms), since it activates ~3B parameters per token.
+#
+# TP=2 notes that cost time to establish, so they are recorded here:
+#   - Sharding needs no code change. Every dimension divides by 2: attention heads
+#     40 -> 20, KV heads 8 -> 4, intermediate_size 13824 -> 6912, vocab
+#     152064 -> 76032. Verify this first for any replacement model.
+#   - TP=2 on a GPTQ checkpoint is what crashed (CUDA_ERROR_ILLEGAL_ADDRESS in
+#     forward()) on the previous model. The cause was quantised weight sharding
+#     and the g_idx guard in src/openai/models/linear.rs, not TP itself. A dense
+#     fp16 checkpoint has no g_idx, no packed weights and no scales, so it takes
+#     the plain column/row-parallel path and the crash does not reproduce.
+#   - These are V100-PCIE cards with no NVLink, so every layer's all-reduce
+#     crosses PCIe. It shows up in TTFT (275 ms at 1 stream, 1090 ms at 16)
+#     more than in ITL.
+#   - KV capacity is 11635 MB per rank, about 59K context tokens. The fp16
+#     weights consume the room the KV pool would otherwise use. This is well
+#     short of the 256K the previous model targeted, and it is the price of
+#     leaving the dequantisation path.
+#
+# Weights must be in the PVC as /mnt/models/coder-14b-fp16 (28 GB, 6 shards).
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: candle-vllm
+  namespace: llm-serving
+  labels:
+    app.kubernetes.io/part-of: llm-serving
+    app: candle-vllm
+spec:
+  replicas: 1
+  strategy:
+    # One replica holds both GPUs, so a rolling update would need a second pair.
+    # Recreate keeps the rollout to a single weights load.
+    type: Recreate
+  selector:
+    matchLabels:
+      app: candle-vllm
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/part-of: llm-serving
+        app: candle-vllm
+    spec:
+      # Required. Without this the container runs under runc, libcuda.so is never
+      # injected, and candle-vllm panics in cudarc at startup with
+      # "Unable to dynamically load the cuda shared library".
+      runtimeClassName: nvidia
+      containers:
+        - name: candle
+          image: forgejo.riotpiao.com/riotpiao-poimen/candle-vllm:sm70-gptq-slim
+          command: ["/usr/local/bin/candle-vllm"]
+          # fp16, not GPTQ, and two ranks, not one. See the header for the
+          # measurements and the TP=2 findings.
+          args:
+            - --w
+            - /mnt/models/coder-14b-fp16
+            - --d
+            - 0,1
+            - --dtype
+            - f16
+            - --p
+            - "8080"
+            - --max-num-seqs
+            - "32"
+            # 11.6 GB of KV per rank, about 59K context tokens. The fp16 weights
+            # leave no room for more.
+            - --kv-fraction
+            - "0.85"
+          ports:
+            - containerPort: 8080
+          resources:
+            limits:
+              cpu: "8"
+              memory: 36Gi
+              nvidia.com/gpu: "2"
+            requests:
+              cpu: "4"
+              memory: 12Gi
+              nvidia.com/gpu: "2"
+          volumeMounts:
+            - name: models
+              mountPath: /mnt/models
+              readOnly: true
+      volumes:
+        - name: models
+          persistentVolumeClaim:
+            claimName: llm-models

k8s/apps/llm-serving/coder-service.yaml

@@ -0,0 +1,40 @@
+# Stable client-facing name for the coder chat model, deliberately decoupled
+# from the process that serves it.
+#
+# poimen-memory addresses a Service, not a pod and not an InferenceService. The
+# vLLM -> candle-vllm handoff therefore changes this file's selector and nothing
+# else, so the client config never changes again and there is no window in which
+# the host name is wrong.
+#
+# The previous KTML predictor requested two V100s and never reached Ready, so it
+# held half the GPU pool while serving nothing. candle-vllm replaces it.
+#
+# Two ports on purpose:
+#   - 80    matches every other predictor Service in this namespace, and matches
+#           a client base URL that carries no port number.
+#   - 8080  is the container port. Publishing it directly avoids a port
+#           translation that is easy to get wrong in a curl.
+#
+# Do not delete and recreate this Service: a delete drops the ClusterIP, and a
+# client that cached the name during the gap fails until it re-resolves.
+apiVersion: v1
+kind: Service
+metadata:
+  name: coder-predictor
+  namespace: llm-serving
+  labels:
+    app.kubernetes.io/part-of: llm-serving
+    app: coder-predictor
+spec:
+  type: ClusterIP
+  selector:
+    app: candle-vllm
+  ports:
+    - name: http
+      protocol: TCP
+      port: 80
+      targetPort: 8080
+    - name: http-8080
+      protocol: TCP
+      port: 8080
+      targetPort: 8080

k8s/apps/llm-serving/coder.yaml

@@ -0,0 +1,134 @@
+apiVersion: serving.kserve.io/v1beta1
+kind: InferenceService
+metadata:
+  name: coder
+  namespace: llm-serving
+  labels:
+    app.kubernetes.io/name: llm-coder
+    app.kubernetes.io/part-of: llm-serving
+  annotations:
+    serving.kserve.io/deploymentMode: RawDeployment
+spec:
+  predictor:
+    minReplicas: 1
+    maxReplicas: 1
+    nodeSelector:
+      kubernetes.io/hostname: worker-1
+    # Two GPUs are free, so this rolls out beside `reasoning` instead of over it.
+    # Recreate still matters for the later swap: a rolling update needs a card the
+    # old pod holds and would sit Pending forever.
+    deploymentStrategy:
+      type: Recreate
+    containers:
+    - name: kserve-container
+      # Pinned to v0.11.0 on purpose. This is the last image measured to work on this
+      # node: torch 2.8.0+cu128, arch list sm_70..sm_120, driver 580.159.04.
+      # v0.15.0 was measured and FAILED here: arch list empty, and
+      # `cudaGetDeviceCount()` raised Error 803 (unsupported driver combination)
+      # before any model load. Do not raise this tag without re-running that probe.
+      image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
+      args:
+      # Qwen3-Coder-30B-A3B-Instruct: 30.5B total, 3.3B active. Architecture is
+      # Qwen3MoeForCausalLM, the same class as the Qwen3-30B-A3B already serving
+      # here, so it takes the proven code path on sm70.
+      #
+      # Qwen ships this model only as BF16 (61.1 GB) and FP8 (no FP8 on Volta).
+      # 4-bit GPTQ is the only form that fits one 32 GB card. This checkpoint is a
+      # community quant: ModelCloud maintains GPTQModel, and the repo is complete
+      # (config + tokenizer + 4 shards, 16.9 GB). Verified by loading, not by trust.
+      #
+      # Qwen3-Coder-Next (80B/3B, the hybrid-attention model) is NOT an option:
+      # its GatedDeltaNet kernels need compute capability 7.5+ and the `fla` module,
+      # and sm70 has neither. Measured, not assumed.
+      - --model=ModelCloud/Qwen3-Coder-30B-A3B-Instruct-GPTQMODEL-W4A16-A
+      # `reasoning` stays a served name so an existing client that sends
+      # model=reasoning keeps working after the DNS rename. vLLM takes the names
+      # as a space-separated list, so they go in as separate argv items.
+      - --served-model-name
+      - coder
+      - reasoning
+      - --quantization=gptq
+      - --dtype=float16
+      - --kv-cache-dtype=auto
+      # Native window is 262144 and rope_scaling is null, so 256K needs no YaRN.
+      # One card cannot hold it: measured KV pool on a single V100 was 132,672
+      # tokens. TP=2 halves the weights per card (7.8 GiB each), which frees room
+      # for the larger pool. Measured after boot: see the KV line in the log.
+      - --max-model-len=262144
+      - --tensor-parallel-size=2
+      - --pipeline-parallel-size=1
+      - --gpu-memory-utilization=0.90
+      # A single 262,144-token request consumes the whole pool, so keep few slots.
+      - --max-num-seqs=4
+      - --max-num-batched-tokens=8192
+      - --long-prefill-token-threshold=2048
+      - --enable-chunked-prefill
+      - --enable-prefix-caching
+      # Measured: with the default (custom all-reduce ON) the engine finished CUDA graph
+      # capture in 6s at 17:01:56 and then went silent for 14+ minutes, emitting only
+      # "No available shared memory broadcast block found in 60 seconds". The front-end
+      # never received the ready message. vLLM's custom all-reduce registers its own
+      # CUDA graph addresses and deadlocks on this Volta/PCIe pair. NCCL is the fallback
+      # and is correct here, because TP=2 across two PCIe V100s gains nothing from the
+      # custom kernel anyway.
+      - --disable-custom-all-reduce
+      - --reasoning-parser=qwen3
+      - --enable-auto-tool-choice
+      - --tool-call-parser=hermes
+      - --host=0.0.0.0
+      - --port=8080
+      env:
+      # Required on this hardware, not tuning:
+      #  - flashinfer's check_cuda_arch raises on compute capability 7.0,
+      #    so the sampler must stay off
+      #  - FlashInfer and FlashAttention-2 need sm75+, so Triton serves attention
+      - name: VLLM_USE_FLASHINFER_SAMPLER
+        value: "0"
+      - name: VLLM_ATTENTION_BACKEND
+        value: TRITON_ATTN
+      - name: HF_HOME
+        value: /mnt/models
+      ports:
+      - containerPort: 8080
+        protocol: TCP
+      readinessProbe:
+        httpGet:
+          path: /health
+          port: 8080
+        periodSeconds: 10
+      startupProbe:
+        httpGet:
+          path: /health
+          port: 8080
+        # TP=2 at 256K needs ~10 min before the API binds: 22s weights, 80s torch.compile,
+        # 85s KV profiling, then graph capture. 120 x 15s = 30 min of headroom.
+        failureThreshold: 120
+        periodSeconds: 15
+      resources:
+        # worker-1 exposes only 63,495 MiB of allocatable memory, so a 24Gi request
+        # never schedules. Two vLLM workers keep the weights on the GPU and need
+        # little host RAM, so the request stays small and the limit allows burst.
+        requests:
+          cpu: "8"
+          memory: 12Gi
+          nvidia.com/gpu: "2"
+        limits:
+          cpu: "16"
+          memory: 40Gi
+          nvidia.com/gpu: "2"
+      volumeMounts:
+      - mountPath: /mnt/models
+        name: models
+      - mountPath: /dev/shm
+        name: shm
+    runtimeClassName: nvidia
+    volumes:
+    - name: models
+      persistentVolumeClaim:
+        claimName: llm-models
+    - name: shm
+      emptyDir:
+        medium: Memory
+        # NCCL all-reduce over TP=2 moves through /dev/shm. The 64Mi default is
+        # too small and shows up as an init hang or a bus error.
+        sizeLimit: 8Gi

k8s/apps/llm-serving/kustomization.yaml

@@ -13,8 +13,24 @@ resources:
   # clef-flash runs through clef-serve.yaml, not through an SGLang runtime. SGLang needs
   # v0.5.22+, published only on CUDA 13, and CUDA 13 dropped Volta kernels. The model
   # instead runs the implementation it ships (joint_schema_model.py) in fp16 on one V100.
+  # coder-14b-fp16.yaml owns the `candle-vllm` Deployment name and the
+  # `app: candle-vllm` label that coder-service.yaml selects. It must be the only
+  # workload with that label, or the Service splits traffic across two models.
+  #
+  # fp16 on two V100s, not GPTQ on one: measured 39.7 ms ITL and 287 tok/s at 16
+  # streams against 71.5 ms and 215 tok/s for the 4-bit MoE. The V100 has FP16
+  # tensor cores and no INT4 ones, so 4-bit pays dequantisation on the SM while
+  # the memory controller idles at 19%.
+  # clef-candle.yaml serves clef-flash on candle-vllm (hybrid Qwen3_5, one GPU)
+  # and owns the clef-candle label that clef-serve.yaml's Service now selects.
+  # It must not sync against an image built before the GDN conv fix.
+  - clef-candle.yaml
+  - coder-14b-fp16.yaml
   - clef-serve.yaml
-  - reasoning.yaml
+  # Stable client-facing name for the chat model. poimen-memory points at this Service,
+  # so the vLLM -> candle-vllm handoff is a selector edit here and no client change.
+  - coder-service.yaml
+  - reasoning-long-service.yaml
   # ornith is retired: its InferenceService still reports Ready=True while running zero
   # replicas, because desired equals available at 0. Its manifest was never committed,
   # so it is left unmanaged and is not listed here. Delete the CR when convenient.

k8s/apps/llm-serving/networkpolicy.yaml

@@ -66,6 +66,21 @@ spec:
           port: 80
         - protocol: TCP
           port: 8000
+    # Allow from poimen namespace (poimen-memory ingest + query).
+    # The client-side egress rule is the other half and must stay in step:
+    # NetworkPolicy poimen/poimen-memory-llm-egress permits port 8080 to pods
+    # labelled part-of=llm-serving, which is the same selector used here. This
+    # policy had no matching ingress rule at all, so the packet was allowed out
+    # and dropped here. The client then saw a bare timeout and the predictor log
+    # stayed empty, which is why this failure looked like a hung inference engine
+    # for 34 days.
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: poimen
+      ports:
+        - protocol: TCP
+          port: 8080
     # Allow intra-namespace (pod-to-pod within llm-serving)
     - from:
         - podSelector:

k8s/apps/llm-serving/reasoning-long-service.yaml

@@ -0,0 +1,43 @@
+# Stable client-facing name for the chat model, deliberately decoupled from the
+# process that currently serves it.
+#
+# poimen-memory addresses this Service, not a pod and not an InferenceService. That
+# means the vLLM -> candle-vllm handoff changes only this file's selector. The client
+# config never changes again, so the ingest cutover has no window in which the host
+# name is wrong.
+#
+# Two ports on purpose:
+#   - 80    matches every other predictor Service in this namespace, and matches
+#           poimen-memory's LLM_CHAT_BASE, which carries no port number.
+#   - 8080  is the container port. It is also the only port llm-serving-ingress and
+#           poimen-memory-llm-egress permit, and the port the handoff readiness check
+#           uses. Publishing it directly avoids a port translation that is easy to get
+#           wrong in a curl.
+#
+# The selector names the vLLM reasoning pod today. To hand off, replace the three
+# selector labels with candle-vllm's pod labels and nothing else. Do not delete and
+# recreate this Service: a delete drops the ClusterIP, and a client that cached the
+# name during the gap will fail until it re-resolves.
+apiVersion: v1
+kind: Service
+metadata:
+  name: reasoning-long-predictor
+  namespace: llm-serving
+  labels:
+    app.kubernetes.io/part-of: llm-serving
+    app: reasoning-long-predictor
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: llm-reasoning
+    component: predictor
+    serving.kserve.io/inferenceservice: reasoning
+  ports:
+    - name: http
+      protocol: TCP
+      port: 80
+      targetPort: 8080
+    - name: http-8080
+      protocol: TCP
+      port: 8080
+      targetPort: 8080

k8s/apps/llm-serving/reasoning.yaml

@@ -30,7 +30,28 @@ spec:
       # Measured on this cluster, one V100-32GB, 64k context: 15.55 GiB of weights,
       # 137k tokens of KV cache, 2.09x concurrency at 65536 tokens per request.
       - --model=Qwen/Qwen3-30B-A3B-GPTQ-Int4
-      - --served-model-name=reasoning
+      # Two served names, one pod, on purpose. `reasoning-long` is the name the goal map
+      # uses for this model, and the name qwen-finetune wiring targets. It is NOT a
+      # separate InferenceService: this model already occupies the only free card, and a
+      # second pod on the same weights cannot schedule (4 of 4 GPUs allocated: coder 2,
+      # this 1, clef-serve 1). Serving both names from one process costs nothing and
+      # keeps the two callers working without a second weights copy.
+      #
+      # `reasoning` is deliberately still served here. poimen-memory pins
+      # LLM_CHAT_BASE=reasoning-predictor, so dropping the name here breaks ingest
+      # immediately. coder-predictor also serves `reasoning`; that ambiguity is known
+      # and documented in the PR. Resolve it by moving the client host, not by
+      # removing the name here.
+      #
+      # The two names are separate list entries, NOT one comma-joined value. This
+      # image's parser returns ['reasoning,reasoning-long'] for the comma form, i.e. a
+      # single model whose name contains a comma, and api_server.py then iterates
+      # args.served_model_name to build the model list. Verified against vLLM 0.11.0:
+      #   FlexibleArgumentParser().parse_args([... "--served-model-name", "a", "b"])
+      #     -> ['a', 'b']
+      - --served-model-name
+      - reasoning
+      - reasoning-long
       - --quantization=gptq
       - --dtype=float16
       - --kv-cache-dtype=auto

k8s/infra/poimen/graphql-netpol.yaml

@@ -0,0 +1,28 @@
+---
+# Keep the GraphQL server reachable only from the one service that proxies it.
+#
+# poimen-graphql is already off the internet: its Service is ClusterIP on 8090 and
+# no Ingress names it. What it lacked is in-cluster isolation — a ClusterIP answers
+# every pod in the cluster.
+#
+# The only caller is poimen-memory, whose /graphql route forwards the request body
+# to MEM_GQL_PORT (8090). No other workload references this service.
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: poimen-graphql-ingress
+  namespace: poimen
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: poimen-graphql
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: poimen-memory
+      ports:
+        - protocol: TCP
+          port: 8090

k8s/infra/poimen/kustomization.yaml

@@ -6,6 +6,10 @@ kind: Kustomization
 # deployments (wave 0) are applied.
 resources:
   - memory-db.yaml
+  # ClusterIP alone is not isolation: every pod in the cluster can reach it.
+  - memory-db-netpol.yaml
+  # Only poimen-memory's /graphql proxy calls 8090.
+  - graphql-netpol.yaml
   - poimen-sandbox-namespace.yaml
   - poimen-sandbox-quota.yaml
   - pi-session-netpol.yaml

k8s/infra/poimen/memory-db-netpol.yaml

@@ -0,0 +1,42 @@
+---
+# Keep the memory database reachable only from inside its own namespace.
+#
+# What this does NOT change: the database was already unreachable from the
+# internet. `memory-db-rw`, `memory-db-ro` and `memory-db-r` are ClusterIP
+# services, no Ingress backs them, and no public DNS name resolves to them.
+#
+# What it fixes: a ClusterIP service is reachable by EVERY pod in the cluster,
+# because Kubernetes has no implicit isolation. Cilium enforces NetworkPolicy, so
+# stating the allowed sources is what turns "not on the internet" into "not
+# reachable from another tenant's namespace".
+#
+# Two sources, both required:
+#  - the poimen namespace, which holds every client: poimen-memory, poimen-graphql,
+#    poimen-worker, poimen-workflows and the migration Job.
+#  - cnpg-system, where the CloudNativePG operator lives. It connects to the
+#    instance for switchover and failover, so a narrower rule breaks recovery.
+#
+# No other port is opened. The operator's metrics endpoint stays closed because no
+# PodMonitor for this cluster exists, so nothing scrapes it.
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: memory-db-ingress
+  namespace: poimen
+spec:
+  podSelector:
+    matchLabels:
+      cnpg.io/cluster: memory-db
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: poimen
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: cnpg-system
+      ports:
+        - protocol: TCP
+          port: 5432