k8s/messaging: add kafka kmsvc and temporal workflows

- Kafka 3-broker cluster (RF=3, min-ISR=2)
- kmsvc SQS-like API on Kafka
- Redis dedup (standalone, can extend to HA)
- Temporal workflow orchestration (Cassandra backend)
This commit is contained in:
Story Crater Bot
2026-08-18 15:08:00 -07:00
parent a3f261f548
commit 69ad5c371c
38 changed files with 2072 additions and 0 deletions
@@ -0,0 +1,30 @@
apiVersion: kafka.strimzi.io/v1beta2
kind: Kafka
metadata:
name: {{ .Values.clusterName }}
namespace: {{ .Values.namespace }}
annotations:
strimzi.io/node-pools: enabled
strimzi.io/kraft: enabled
spec:
kafka:
version: 4.0.0
metadataVersion: 4.0-IV3
listeners:
- name: plain
port: 9092
type: internal
tls: false
- name: tls
port: 9093
type: internal
tls: true
config:
default.replication.factor: {{ .Values.kafka.replicationFactor }}
min.insync.replicas: {{ .Values.kafka.minInsyncReplicas }}
offsets.topic.replication.factor: {{ .Values.kafka.replicationFactor }}
transaction.state.log.replication.factor: {{ .Values.kafka.replicationFactor }}
transaction.state.log.min.isr: {{ .Values.kafka.minInsyncReplicas }}
entityOperator:
topicOperator: {}
userOperator: {}
@@ -0,0 +1,39 @@
apiVersion: kafka.strimzi.io/v1beta2
kind: KafkaNodePool
metadata:
name: {{ .Values.clusterName }}-pool
namespace: {{ .Values.namespace }}
labels:
strimzi.io/cluster: {{ .Values.clusterName }}
spec:
replicas: {{ .Values.nodePool.replicas }}
roles:
- controller
- broker
storage:
type: persistent-claim
size: {{ .Values.nodePool.storage.sizeGi }}Gi
class: {{ .Values.nodePool.storage.class }}
deleteClaim: false
resources:
limits:
memory: {{ .Values.nodePool.resources.memory }}
cpu: {{ .Values.nodePool.resources.cpu | quote }}
requests:
memory: {{ .Values.nodePool.resources.memory }}
cpu: {{ .Values.nodePool.resources.cpu | quote }}
template:
pod:
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
topologyKey: {{ .Values.nodePool.antiAffinityTopologyKey }}
labelSelector:
matchLabels:
strimzi.io/cluster: {{ .Values.clusterName }}
kafkaContainer:
env:
- name: KAFKA_HEAP_OPTS
value: {{ .Values.nodePool.heapOpts | quote }}
@@ -0,0 +1,19 @@
{{- if eq .Values.nodePool.storage.class "longhorn-kafka" }}
# The default "longhorn" StorageClass requests 3 replicas across 3 zone-labeled
# nodes (az-a/az-b/az-c). With Longhorn's zone-aware anti-affinity, replicas
# spread 1-per-zone for durability.
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: longhorn-kafka
provisioner: driver.longhorn.io
allowVolumeExpansion: true
reclaimPolicy: Delete
volumeBindingMode: Immediate
parameters:
numberOfReplicas: "3"
staleReplicaTimeout: "30"
fromBackup: ""
fsType: "ext4"
dataLocality: "disabled"
{{- end }}