k8s/messaging: add kafka kmsvc and temporal workflows

- Kafka 3-broker cluster (RF=3, min-ISR=2) - kmsvc SQS-like API on Kafka - Redis dedup (standalone, can extend to HA) - Temporal workflow orchestration (Cassandra backend)
2026-08-18 15:08:00 -07:00
parent a3f261f548
commit 69ad5c371c
38 changed files with 2072 additions and 0 deletions
@@ -0,0 +1,5 @@
+apiVersion: v2
+name: kafka-cluster
+description: Strimzi Kafka/KafkaNodePool CRs for the kmsvc Kafka cluster (design.md §7)
+type: application
+version: 0.1.0
@@ -0,0 +1,30 @@
+apiVersion: kafka.strimzi.io/v1beta2
+kind: Kafka
+metadata:
+  name: {{ .Values.clusterName }}
+  namespace: {{ .Values.namespace }}
+  annotations:
+    strimzi.io/node-pools: enabled
+    strimzi.io/kraft: enabled
+spec:
+  kafka:
+    version: 4.0.0
+    metadataVersion: 4.0-IV3
+    listeners:
+      - name: plain
+        port: 9092
+        type: internal
+        tls: false
+      - name: tls
+        port: 9093
+        type: internal
+        tls: true
+    config:
+      default.replication.factor: {{ .Values.kafka.replicationFactor }}
+      min.insync.replicas: {{ .Values.kafka.minInsyncReplicas }}
+      offsets.topic.replication.factor: {{ .Values.kafka.replicationFactor }}
+      transaction.state.log.replication.factor: {{ .Values.kafka.replicationFactor }}
+      transaction.state.log.min.isr: {{ .Values.kafka.minInsyncReplicas }}
+  entityOperator:
+    topicOperator: {}
+    userOperator: {}
@@ -0,0 +1,39 @@
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaNodePool
+metadata:
+  name: {{ .Values.clusterName }}-pool
+  namespace: {{ .Values.namespace }}
+  labels:
+    strimzi.io/cluster: {{ .Values.clusterName }}
+spec:
+  replicas: {{ .Values.nodePool.replicas }}
+  roles:
+    - controller
+    - broker
+  storage:
+    type: persistent-claim
+    size: {{ .Values.nodePool.storage.sizeGi }}Gi
+    class: {{ .Values.nodePool.storage.class }}
+    deleteClaim: false
+  resources:
+    limits:
+      memory: {{ .Values.nodePool.resources.memory }}
+      cpu: {{ .Values.nodePool.resources.cpu | quote }}
+    requests:
+      memory: {{ .Values.nodePool.resources.memory }}
+      cpu: {{ .Values.nodePool.resources.cpu | quote }}
+  template:
+    pod:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                topologyKey: {{ .Values.nodePool.antiAffinityTopologyKey }}
+                labelSelector:
+                  matchLabels:
+                    strimzi.io/cluster: {{ .Values.clusterName }}
+    kafkaContainer:
+      env:
+        - name: KAFKA_HEAP_OPTS
+          value: {{ .Values.nodePool.heapOpts | quote }}
@@ -0,0 +1,19 @@
+{{- if eq .Values.nodePool.storage.class "longhorn-kafka" }}
+# The default "longhorn" StorageClass requests 3 replicas across 3 zone-labeled
+# nodes (az-a/az-b/az-c). With Longhorn's zone-aware anti-affinity, replicas
+# spread 1-per-zone for durability.
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: longhorn-kafka
+provisioner: driver.longhorn.io
+allowVolumeExpansion: true
+reclaimPolicy: Delete
+volumeBindingMode: Immediate
+parameters:
+  numberOfReplicas: "3"
+  staleReplicaTimeout: "30"
+  fromBackup: ""
+  fsType: "ext4"
+  dataLocality: "disabled"
+{{- end }}
@@ -0,0 +1,26 @@
+clusterName: kmsvc
+namespace: sqs
+
+nodePool:
+  replicas: 3
+  storage:
+    class: longhorn-kafka
+    # Longhorn's per-node scheduling budget on the current 2-node cluster has
+    # only ~36Gi of headroom left (other PVCs already reserve the rest), and
+    # each node hosts one replica of all 3 broker volumes -- so 3 * sizeGi
+    # must fit in that headroom. Revisit once the 3rd node joins.
+    sizeGi: 10
+  resources:
+    memory: 5Gi
+    cpu: "2"
+  heapOpts: "-Xms2g -Xmx2g"
+  # design.md §7: 3 real zones now exist (talos-cp-1=az-a, talos-worker-1=az-b,
+  # talos-worker-2=az-c), so anti-affinity keys off zone instead of hostname —
+  # spreads the 3 broker pods one-per-zone/one-per-node (equivalent today,
+  # but zone is the correct long-term key if a node ever gets replaced within
+  # the same zone).
+  antiAffinityTopologyKey: topology.kubernetes.io/zone
+
+kafka:
+  replicationFactor: 3
+  minInsyncReplicas: 2