k8s/messaging: add kafka kmsvc and temporal workflows
- Kafka 3-broker cluster (RF=3, min-ISR=2) - kmsvc SQS-like API on Kafka - Redis dedup (standalone, can extend to HA) - Temporal workflow orchestration (Cassandra backend)
This commit is contained in:
Story Crater Bot
@@ -0,0 +1,25 @@
|
|||||||
|
apiVersion: argoproj.io/v1alpha1
|
||||||
|
kind: Application
|
||||||
|
metadata:
|
||||||
|
name: strimzi-operator
|
||||||
|
namespace: cicd
|
||||||
|
annotations:
|
||||||
|
argocd.argoproj.io/sync-wave: "0"
|
||||||
|
spec:
|
||||||
|
project: kmsvc
|
||||||
|
source:
|
||||||
|
repoURL: https://strimzi.io/charts/
|
||||||
|
chart: strimzi-kafka-operator
|
||||||
|
targetRevision: 0.46.0
|
||||||
|
helm:
|
||||||
|
values: |
|
||||||
|
watchNamespaces: ["sqs"]
|
||||||
|
destination:
|
||||||
|
server: https://kubernetes.default.svc
|
||||||
|
namespace: sqs
|
||||||
|
syncPolicy:
|
||||||
|
automated:
|
||||||
|
prune: true
|
||||||
|
selfHeal: true
|
||||||
|
syncOptions:
|
||||||
|
- CreateNamespace=true
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
apiVersion: argoproj.io/v1alpha1
|
||||||
|
kind: Application
|
||||||
|
metadata:
|
||||||
|
name: kafka-cluster
|
||||||
|
namespace: cicd
|
||||||
|
annotations:
|
||||||
|
argocd.argoproj.io/sync-wave: "1"
|
||||||
|
spec:
|
||||||
|
project: kmsvc
|
||||||
|
source:
|
||||||
|
repoURL: https://forgejo.riotpiao.homelab.com/rock/kafaka-management-service.git
|
||||||
|
targetRevision: main
|
||||||
|
path: k8s/charts/kafka-cluster
|
||||||
|
helm:
|
||||||
|
values: |
|
||||||
|
namespace: sqs
|
||||||
|
nodePool:
|
||||||
|
replicas: 3
|
||||||
|
storage:
|
||||||
|
class: longhorn
|
||||||
|
sizeGi: 50
|
||||||
|
resources:
|
||||||
|
memory: 5Gi
|
||||||
|
cpu: "2"
|
||||||
|
destination:
|
||||||
|
server: https://kubernetes.default.svc
|
||||||
|
namespace: sqs
|
||||||
|
syncPolicy:
|
||||||
|
automated:
|
||||||
|
prune: true
|
||||||
|
selfHeal: true
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
apiVersion: argoproj.io/v1alpha1
|
||||||
|
kind: Application
|
||||||
|
metadata:
|
||||||
|
name: kmsvc-redis
|
||||||
|
namespace: cicd
|
||||||
|
annotations:
|
||||||
|
argocd.argoproj.io/sync-wave: "1"
|
||||||
|
spec:
|
||||||
|
project: kmsvc
|
||||||
|
source:
|
||||||
|
repoURL: https://charts.bitnami.com/bitnami
|
||||||
|
chart: redis
|
||||||
|
targetRevision: 20.6.0
|
||||||
|
helm:
|
||||||
|
values: |
|
||||||
|
architecture: standalone
|
||||||
|
# docker.io/bitnami stopped publishing version-pinned tags; bitnamilegacy
|
||||||
|
# mirrors them for free. allowInsecureImages silences the chart's
|
||||||
|
# container-image allowlist check, which doesn't know about that mirror.
|
||||||
|
global:
|
||||||
|
security:
|
||||||
|
allowInsecureImages: true
|
||||||
|
image:
|
||||||
|
repository: bitnamilegacy/redis
|
||||||
|
auth:
|
||||||
|
enabled: false
|
||||||
|
master:
|
||||||
|
persistence:
|
||||||
|
enabled: true
|
||||||
|
storageClass: longhorn
|
||||||
|
size: 2Gi
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
memory: 1Gi
|
||||||
|
requests:
|
||||||
|
memory: 1Gi
|
||||||
|
destination:
|
||||||
|
server: https://kubernetes.default.svc
|
||||||
|
namespace: sqs
|
||||||
|
syncPolicy:
|
||||||
|
automated:
|
||||||
|
prune: true
|
||||||
|
selfHeal: true
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
apiVersion: argoproj.io/v1alpha1
|
||||||
|
kind: Application
|
||||||
|
metadata:
|
||||||
|
name: queue-crd
|
||||||
|
namespace: cicd
|
||||||
|
annotations:
|
||||||
|
argocd.argoproj.io/sync-wave: "2"
|
||||||
|
spec:
|
||||||
|
project: kmsvc
|
||||||
|
source:
|
||||||
|
repoURL: https://forgejo.riotpiao.homelab.com/rock/kafaka-management-service.git
|
||||||
|
targetRevision: main
|
||||||
|
path: k8s/charts/queue-crd
|
||||||
|
helm:
|
||||||
|
values: |
|
||||||
|
namespace: sqs
|
||||||
|
kafkaBrokers: "kmsvc-kafka-bootstrap.sqs.svc.cluster.local:9092"
|
||||||
|
redisAddr: "kmsvc-redis-master.sqs.svc.cluster.local:6379"
|
||||||
|
image:
|
||||||
|
repository: forgejo.riotpiao.homelab.com/rock/kafka-management-service-queue-operator
|
||||||
|
# CI (.forgejo/workflows/release.yaml) writes the released git tag
|
||||||
|
# here and pushes the commit -- ArgoCD picks it up on its next sync.
|
||||||
|
tag: latest
|
||||||
|
destination:
|
||||||
|
server: https://kubernetes.default.svc
|
||||||
|
namespace: sqs
|
||||||
|
syncPolicy:
|
||||||
|
automated:
|
||||||
|
prune: true
|
||||||
|
selfHeal: true
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
apiVersion: argoproj.io/v1alpha1
|
||||||
|
kind: Application
|
||||||
|
metadata:
|
||||||
|
name: management-service
|
||||||
|
namespace: cicd
|
||||||
|
annotations:
|
||||||
|
argocd.argoproj.io/sync-wave: "2"
|
||||||
|
spec:
|
||||||
|
project: kmsvc
|
||||||
|
source:
|
||||||
|
repoURL: https://forgejo.riotpiao.homelab.com/rock/kafaka-management-service.git
|
||||||
|
targetRevision: main
|
||||||
|
path: k8s/charts/management-service
|
||||||
|
helm:
|
||||||
|
values: |
|
||||||
|
namespace: sqs
|
||||||
|
image:
|
||||||
|
repository: forgejo.riotpiao.homelab.com/rock/kafka-management-service
|
||||||
|
# CI (.forgejo/workflows/release.yaml) writes the released git tag
|
||||||
|
# here and pushes the commit -- ArgoCD picks it up on its next sync.
|
||||||
|
tag: latest
|
||||||
|
env:
|
||||||
|
kafkaBrokers: "kmsvc-kafka-bootstrap.sqs.svc.cluster.local:9092"
|
||||||
|
redisAddr: "kmsvc-redis-master.sqs.svc.cluster.local:6379"
|
||||||
|
authentikIssuerURL: "https://authentik.riotpiao.homelab.com/application/o/kafaka/"
|
||||||
|
authentikAudience: "QI0gPtR99ar8VvhK8Tqox4SDkTKzbNU7lbgwBNSc"
|
||||||
|
ingress:
|
||||||
|
enabled: true
|
||||||
|
host: kmsvc.riotpiao.homelab.com
|
||||||
|
clusterIssuer: homelab-ca
|
||||||
|
destination:
|
||||||
|
server: https://kubernetes.default.svc
|
||||||
|
namespace: sqs
|
||||||
|
syncPolicy:
|
||||||
|
automated:
|
||||||
|
prune: true
|
||||||
|
selfHeal: true
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
apiVersion: argoproj.io/v1alpha1
|
||||||
|
kind: AppProject
|
||||||
|
metadata:
|
||||||
|
name: kmsvc
|
||||||
|
namespace: cicd
|
||||||
|
spec:
|
||||||
|
description: Kafka Management Service (design.md) -- Strimzi/Kafka, Redis, queue-operator, message-plane server
|
||||||
|
sourceRepos:
|
||||||
|
- https://forgejo.riotpiao.homelab.com/rock/kafaka-management-service.git
|
||||||
|
- https://strimzi.io/charts/
|
||||||
|
- https://charts.bitnami.com/bitnami
|
||||||
|
destinations:
|
||||||
|
- namespace: sqs
|
||||||
|
server: https://kubernetes.default.svc
|
||||||
|
- namespace: cicd
|
||||||
|
server: https://kubernetes.default.svc
|
||||||
|
clusterResourceWhitelist:
|
||||||
|
- group: "apiextensions.k8s.io"
|
||||||
|
kind: CustomResourceDefinition
|
||||||
|
- group: "rbac.authorization.k8s.io"
|
||||||
|
kind: ClusterRole
|
||||||
|
- group: "rbac.authorization.k8s.io"
|
||||||
|
kind: ClusterRoleBinding
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
apiVersion: argoproj.io/v1alpha1
|
||||||
|
kind: Application
|
||||||
|
metadata:
|
||||||
|
name: kmsvc-root
|
||||||
|
namespace: cicd
|
||||||
|
spec:
|
||||||
|
project: kmsvc
|
||||||
|
source:
|
||||||
|
repoURL: https://forgejo.riotpiao.homelab.com/rock/kafaka-management-service.git
|
||||||
|
targetRevision: main
|
||||||
|
path: k8s/argocd/apps
|
||||||
|
directory:
|
||||||
|
recurse: false
|
||||||
|
destination:
|
||||||
|
server: https://kubernetes.default.svc
|
||||||
|
namespace: cicd
|
||||||
|
syncPolicy:
|
||||||
|
automated:
|
||||||
|
prune: true
|
||||||
|
selfHeal: true
|
||||||
|
syncOptions:
|
||||||
|
- CreateNamespace=true
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
apiVersion: v2
|
||||||
|
name: kafka-cluster
|
||||||
|
description: Strimzi Kafka/KafkaNodePool CRs for the kmsvc Kafka cluster (design.md §7)
|
||||||
|
type: application
|
||||||
|
version: 0.1.0
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
apiVersion: kafka.strimzi.io/v1beta2
|
||||||
|
kind: Kafka
|
||||||
|
metadata:
|
||||||
|
name: {{ .Values.clusterName }}
|
||||||
|
namespace: {{ .Values.namespace }}
|
||||||
|
annotations:
|
||||||
|
strimzi.io/node-pools: enabled
|
||||||
|
strimzi.io/kraft: enabled
|
||||||
|
spec:
|
||||||
|
kafka:
|
||||||
|
version: 4.0.0
|
||||||
|
metadataVersion: 4.0-IV3
|
||||||
|
listeners:
|
||||||
|
- name: plain
|
||||||
|
port: 9092
|
||||||
|
type: internal
|
||||||
|
tls: false
|
||||||
|
- name: tls
|
||||||
|
port: 9093
|
||||||
|
type: internal
|
||||||
|
tls: true
|
||||||
|
config:
|
||||||
|
default.replication.factor: {{ .Values.kafka.replicationFactor }}
|
||||||
|
min.insync.replicas: {{ .Values.kafka.minInsyncReplicas }}
|
||||||
|
offsets.topic.replication.factor: {{ .Values.kafka.replicationFactor }}
|
||||||
|
transaction.state.log.replication.factor: {{ .Values.kafka.replicationFactor }}
|
||||||
|
transaction.state.log.min.isr: {{ .Values.kafka.minInsyncReplicas }}
|
||||||
|
entityOperator:
|
||||||
|
topicOperator: {}
|
||||||
|
userOperator: {}
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
apiVersion: kafka.strimzi.io/v1beta2
|
||||||
|
kind: KafkaNodePool
|
||||||
|
metadata:
|
||||||
|
name: {{ .Values.clusterName }}-pool
|
||||||
|
namespace: {{ .Values.namespace }}
|
||||||
|
labels:
|
||||||
|
strimzi.io/cluster: {{ .Values.clusterName }}
|
||||||
|
spec:
|
||||||
|
replicas: {{ .Values.nodePool.replicas }}
|
||||||
|
roles:
|
||||||
|
- controller
|
||||||
|
- broker
|
||||||
|
storage:
|
||||||
|
type: persistent-claim
|
||||||
|
size: {{ .Values.nodePool.storage.sizeGi }}Gi
|
||||||
|
class: {{ .Values.nodePool.storage.class }}
|
||||||
|
deleteClaim: false
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
memory: {{ .Values.nodePool.resources.memory }}
|
||||||
|
cpu: {{ .Values.nodePool.resources.cpu | quote }}
|
||||||
|
requests:
|
||||||
|
memory: {{ .Values.nodePool.resources.memory }}
|
||||||
|
cpu: {{ .Values.nodePool.resources.cpu | quote }}
|
||||||
|
template:
|
||||||
|
pod:
|
||||||
|
affinity:
|
||||||
|
podAntiAffinity:
|
||||||
|
preferredDuringSchedulingIgnoredDuringExecution:
|
||||||
|
- weight: 100
|
||||||
|
podAffinityTerm:
|
||||||
|
topologyKey: {{ .Values.nodePool.antiAffinityTopologyKey }}
|
||||||
|
labelSelector:
|
||||||
|
matchLabels:
|
||||||
|
strimzi.io/cluster: {{ .Values.clusterName }}
|
||||||
|
kafkaContainer:
|
||||||
|
env:
|
||||||
|
- name: KAFKA_HEAP_OPTS
|
||||||
|
value: {{ .Values.nodePool.heapOpts | quote }}
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
{{- if eq .Values.nodePool.storage.class "longhorn-kafka" }}
|
||||||
|
# The default "longhorn" StorageClass requests 3 replicas across 3 zone-labeled
|
||||||
|
# nodes (az-a/az-b/az-c). With Longhorn's zone-aware anti-affinity, replicas
|
||||||
|
# spread 1-per-zone for durability.
|
||||||
|
apiVersion: storage.k8s.io/v1
|
||||||
|
kind: StorageClass
|
||||||
|
metadata:
|
||||||
|
name: longhorn-kafka
|
||||||
|
provisioner: driver.longhorn.io
|
||||||
|
allowVolumeExpansion: true
|
||||||
|
reclaimPolicy: Delete
|
||||||
|
volumeBindingMode: Immediate
|
||||||
|
parameters:
|
||||||
|
numberOfReplicas: "3"
|
||||||
|
staleReplicaTimeout: "30"
|
||||||
|
fromBackup: ""
|
||||||
|
fsType: "ext4"
|
||||||
|
dataLocality: "disabled"
|
||||||
|
{{- end }}
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
clusterName: kmsvc
|
||||||
|
namespace: sqs
|
||||||
|
|
||||||
|
nodePool:
|
||||||
|
replicas: 3
|
||||||
|
storage:
|
||||||
|
class: longhorn-kafka
|
||||||
|
# Longhorn's per-node scheduling budget on the current 2-node cluster has
|
||||||
|
# only ~36Gi of headroom left (other PVCs already reserve the rest), and
|
||||||
|
# each node hosts one replica of all 3 broker volumes -- so 3 * sizeGi
|
||||||
|
# must fit in that headroom. Revisit once the 3rd node joins.
|
||||||
|
sizeGi: 10
|
||||||
|
resources:
|
||||||
|
memory: 5Gi
|
||||||
|
cpu: "2"
|
||||||
|
heapOpts: "-Xms2g -Xmx2g"
|
||||||
|
# design.md §7: 3 real zones now exist (talos-cp-1=az-a, talos-worker-1=az-b,
|
||||||
|
# talos-worker-2=az-c), so anti-affinity keys off zone instead of hostname —
|
||||||
|
# spreads the 3 broker pods one-per-zone/one-per-node (equivalent today,
|
||||||
|
# but zone is the correct long-term key if a node ever gets replaced within
|
||||||
|
# the same zone).
|
||||||
|
antiAffinityTopologyKey: topology.kubernetes.io/zone
|
||||||
|
|
||||||
|
kafka:
|
||||||
|
replicationFactor: 3
|
||||||
|
minInsyncReplicas: 2
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
apiVersion: v2
|
||||||
|
name: management-service
|
||||||
|
description: kmsvc message-plane gRPC+REST server (design.md §1, §7a, §9)
|
||||||
|
type: application
|
||||||
|
version: 0.1.0
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: management-service-config
|
||||||
|
namespace: {{ .Values.namespace }}
|
||||||
|
data:
|
||||||
|
KMSVC_KAFKA_BROKERS: {{ .Values.env.kafkaBrokers | quote }}
|
||||||
|
KMSVC_REDIS_ADDR: {{ .Values.env.redisAddr | quote }}
|
||||||
|
KMSVC_AUTHENTIK_ISSUER_URL: {{ .Values.env.authentikIssuerURL | quote }}
|
||||||
|
KMSVC_AUTHENTIK_AUDIENCE: {{ .Values.env.authentikAudience | quote }}
|
||||||
|
KMSVC_GRPC_LISTEN_ADDR: ":{{ .Values.grpcPort }}"
|
||||||
|
KMSVC_HTTP_LISTEN_ADDR: ":{{ .Values.httpPort }}"
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: management-service
|
||||||
|
namespace: {{ .Values.namespace }}
|
||||||
|
spec:
|
||||||
|
replicas: {{ .Values.replicaCount }}
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: management-service
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: management-service
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: management-service
|
||||||
|
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
|
||||||
|
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||||
|
ports:
|
||||||
|
- name: grpc
|
||||||
|
containerPort: {{ .Values.grpcPort }}
|
||||||
|
- name: http
|
||||||
|
containerPort: {{ .Values.httpPort }}
|
||||||
|
env:
|
||||||
|
- name: GOMEMLIMIT
|
||||||
|
value: {{ .Values.goMemLimit | quote }}
|
||||||
|
envFrom:
|
||||||
|
- configMapRef:
|
||||||
|
name: management-service-config
|
||||||
|
resources:
|
||||||
|
{{- toYaml .Values.resources | nindent 12 }}
|
||||||
|
readinessProbe:
|
||||||
|
tcpSocket:
|
||||||
|
port: {{ .Values.httpPort }}
|
||||||
|
initialDelaySeconds: 5
|
||||||
|
periodSeconds: 10
|
||||||
|
livenessProbe:
|
||||||
|
tcpSocket:
|
||||||
|
port: {{ .Values.httpPort }}
|
||||||
|
initialDelaySeconds: 10
|
||||||
|
periodSeconds: 20
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
{{- if .Values.hpa.enabled }}
|
||||||
|
apiVersion: autoscaling/v2
|
||||||
|
kind: HorizontalPodAutoscaler
|
||||||
|
metadata:
|
||||||
|
name: management-service
|
||||||
|
namespace: {{ .Values.namespace }}
|
||||||
|
spec:
|
||||||
|
scaleTargetRef:
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
name: management-service
|
||||||
|
minReplicas: {{ .Values.hpa.minReplicas }}
|
||||||
|
maxReplicas: {{ .Values.hpa.maxReplicas }}
|
||||||
|
metrics:
|
||||||
|
- type: Resource
|
||||||
|
resource:
|
||||||
|
name: cpu
|
||||||
|
target:
|
||||||
|
type: Utilization
|
||||||
|
averageUtilization: {{ .Values.hpa.targetCPUUtilizationPercentage }}
|
||||||
|
- type: Resource
|
||||||
|
resource:
|
||||||
|
name: memory
|
||||||
|
target:
|
||||||
|
type: Utilization
|
||||||
|
averageUtilization: {{ .Values.hpa.targetMemoryUtilizationPercentage }}
|
||||||
|
{{- end }}
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
{{- if and .Values.ingress.enabled .Values.ingress.grpcEnabled }}
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: Ingress
|
||||||
|
metadata:
|
||||||
|
name: management-service-grpc
|
||||||
|
namespace: {{ .Values.namespace }}
|
||||||
|
annotations:
|
||||||
|
cert-manager.io/cluster-issuer: {{ .Values.ingress.clusterIssuer }}
|
||||||
|
nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
|
||||||
|
spec:
|
||||||
|
ingressClassName: {{ .Values.ingress.className }}
|
||||||
|
tls:
|
||||||
|
- hosts:
|
||||||
|
- {{ .Values.ingress.host }}
|
||||||
|
secretName: {{ .Values.ingress.tlsSecretName }}
|
||||||
|
rules:
|
||||||
|
- host: {{ .Values.ingress.host }}
|
||||||
|
http:
|
||||||
|
paths:
|
||||||
|
- path: {{ .Values.ingress.grpcPathPrefix }}
|
||||||
|
pathType: Prefix
|
||||||
|
backend:
|
||||||
|
service:
|
||||||
|
name: management-service
|
||||||
|
port:
|
||||||
|
number: {{ .Values.grpcPort }}
|
||||||
|
{{- end }}
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
{{- if .Values.ingress.enabled }}
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: Ingress
|
||||||
|
metadata:
|
||||||
|
name: management-service
|
||||||
|
namespace: {{ .Values.namespace }}
|
||||||
|
annotations:
|
||||||
|
cert-manager.io/cluster-issuer: {{ .Values.ingress.clusterIssuer }}
|
||||||
|
spec:
|
||||||
|
ingressClassName: {{ .Values.ingress.className }}
|
||||||
|
tls:
|
||||||
|
- hosts:
|
||||||
|
- {{ .Values.ingress.host }}
|
||||||
|
secretName: {{ .Values.ingress.tlsSecretName }}
|
||||||
|
rules:
|
||||||
|
- host: {{ .Values.ingress.host }}
|
||||||
|
http:
|
||||||
|
paths:
|
||||||
|
- path: /
|
||||||
|
pathType: Prefix
|
||||||
|
backend:
|
||||||
|
service:
|
||||||
|
name: management-service
|
||||||
|
port:
|
||||||
|
number: {{ .Values.httpPort }}
|
||||||
|
{{- end }}
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: management-service
|
||||||
|
namespace: {{ .Values.namespace }}
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
app: management-service
|
||||||
|
ports:
|
||||||
|
- name: grpc
|
||||||
|
port: {{ .Values.grpcPort }}
|
||||||
|
targetPort: {{ .Values.grpcPort }}
|
||||||
|
- name: http
|
||||||
|
port: {{ .Values.httpPort }}
|
||||||
|
targetPort: {{ .Values.httpPort }}
|
||||||
|
type: ClusterIP
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
namespace: sqs
|
||||||
|
replicaCount: 3
|
||||||
|
|
||||||
|
image:
|
||||||
|
repository: forgejo.riotpiao.homelab.com/rock/kafka-management-service
|
||||||
|
tag: latest
|
||||||
|
pullPolicy: Always
|
||||||
|
|
||||||
|
grpcPort: 9090
|
||||||
|
httpPort: 8080
|
||||||
|
|
||||||
|
env:
|
||||||
|
kafkaBrokers: "kmsvc-kafka-bootstrap.sqs.svc.cluster.local:9092"
|
||||||
|
redisAddr: "kmsvc-redis-master.sqs.svc.cluster.local:6379"
|
||||||
|
authentikIssuerURL: ""
|
||||||
|
authentikAudience: ""
|
||||||
|
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 200m
|
||||||
|
memory: 256Mi
|
||||||
|
limits:
|
||||||
|
cpu: "1"
|
||||||
|
memory: 512Mi
|
||||||
|
|
||||||
|
# Go's GC only reacts to GOGC by default and has no idea about the cgroup
|
||||||
|
# memory limit above -- it'll happily grow heap until the kernel OOMKills it.
|
||||||
|
# Setting GOMEMLIMIT to ~90% of the container limit makes the GC self-throttle
|
||||||
|
# before that happens. Keep this in sync with resources.limits.memory.
|
||||||
|
goMemLimit: "460MiB"
|
||||||
|
|
||||||
|
hpa:
|
||||||
|
enabled: true
|
||||||
|
minReplicas: 3
|
||||||
|
maxReplicas: 9
|
||||||
|
targetCPUUtilizationPercentage: 70
|
||||||
|
targetMemoryUtilizationPercentage: 80
|
||||||
|
|
||||||
|
ingress:
|
||||||
|
enabled: true
|
||||||
|
className: nginx
|
||||||
|
clusterIssuer: homelab-ca
|
||||||
|
host: kmsvc.riotpiao.homelab.com
|
||||||
|
tlsSecretName: kmsvc-tls
|
||||||
|
# kmsvc-cli connects via gRPC directly to --server/KMSVC_SERVER (default
|
||||||
|
# kmsvc.riotpiao.homelab.com:443, see kmsvc-cli's README), so raw gRPC needs an
|
||||||
|
# external path too — scoped to the gRPC service's own path prefix on the
|
||||||
|
# same host/port, rather than opening the whole host to gRPC passthrough.
|
||||||
|
grpcEnabled: true
|
||||||
|
grpcPathPrefix: /kafkamgmt.v1.QueueService/
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
apiVersion: v2
|
||||||
|
name: queue-crd
|
||||||
|
description: Queue CRD definition + queue-operator Deployment/RBAC (design.md §2a)
|
||||||
|
type: application
|
||||||
|
version: 0.1.0
|
||||||
@@ -0,0 +1,274 @@
|
|||||||
|
---
|
||||||
|
apiVersion: apiextensions.k8s.io/v1
|
||||||
|
kind: CustomResourceDefinition
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
controller-gen.kubebuilder.io/version: v0.21.0
|
||||||
|
name: queues.kmsvc.io
|
||||||
|
spec:
|
||||||
|
group: kmsvc.io
|
||||||
|
names:
|
||||||
|
kind: Queue
|
||||||
|
listKind: QueueList
|
||||||
|
plural: queues
|
||||||
|
shortNames:
|
||||||
|
- queue
|
||||||
|
- queues
|
||||||
|
singular: queue
|
||||||
|
scope: Namespaced
|
||||||
|
versions:
|
||||||
|
- additionalPrinterColumns:
|
||||||
|
- jsonPath: .spec.fifoQueue
|
||||||
|
name: FIFO
|
||||||
|
type: boolean
|
||||||
|
- jsonPath: .status.phase
|
||||||
|
name: Phase
|
||||||
|
type: string
|
||||||
|
name: v1
|
||||||
|
schema:
|
||||||
|
openAPIV3Schema:
|
||||||
|
description: Queue is the Schema for the queues API — see design.md §2a.
|
||||||
|
properties:
|
||||||
|
apiVersion:
|
||||||
|
description: |-
|
||||||
|
APIVersion defines the versioned schema of this representation of an object.
|
||||||
|
Servers should convert recognized schemas to the latest internal value, and
|
||||||
|
may reject unrecognized values.
|
||||||
|
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
|
||||||
|
type: string
|
||||||
|
kind:
|
||||||
|
description: |-
|
||||||
|
Kind is a string value representing the REST resource this object represents.
|
||||||
|
Servers may infer this from the endpoint the client submits requests to.
|
||||||
|
Cannot be updated.
|
||||||
|
In CamelCase.
|
||||||
|
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
|
||||||
|
type: string
|
||||||
|
metadata:
|
||||||
|
type: object
|
||||||
|
spec:
|
||||||
|
description: QueueSpec defines the desired state of a Queue (design.md
|
||||||
|
§2a).
|
||||||
|
properties:
|
||||||
|
deadLetterTargetQueue:
|
||||||
|
description: |-
|
||||||
|
DeadLetterTargetQueue is the name of another Queue to route exhausted
|
||||||
|
messages to. Must not point at itself or at another DLQ (design.md §5).
|
||||||
|
type: string
|
||||||
|
delaySeconds:
|
||||||
|
description: DelaySeconds is the default delivery delay applied to
|
||||||
|
sent messages.
|
||||||
|
format: int32
|
||||||
|
maximum: 900
|
||||||
|
minimum: 0
|
||||||
|
type: integer
|
||||||
|
fifoQueue:
|
||||||
|
default: false
|
||||||
|
description: FIFOQueue enables per-MessageGroupId ordering and deduplication
|
||||||
|
semantics.
|
||||||
|
type: boolean
|
||||||
|
isDLQ:
|
||||||
|
description: |-
|
||||||
|
IsDLQ marks this queue as itself a dead-letter queue, used to enforce
|
||||||
|
the no-DLQ-chaining validation rule in design.md §5.
|
||||||
|
type: boolean
|
||||||
|
maxReceiveCount:
|
||||||
|
default: 5
|
||||||
|
description: |-
|
||||||
|
MaxReceiveCount is how many times a message may be redelivered before
|
||||||
|
being routed to DeadLetterTargetQueue.
|
||||||
|
format: int32
|
||||||
|
minimum: 1
|
||||||
|
type: integer
|
||||||
|
maxShards:
|
||||||
|
default: 8
|
||||||
|
description: MaxShards is the ceiling on shard count the operator
|
||||||
|
may split up to (design.md §2c).
|
||||||
|
format: int32
|
||||||
|
minimum: 1
|
||||||
|
type: integer
|
||||||
|
messageRetentionPeriodSeconds:
|
||||||
|
default: 345600
|
||||||
|
description: MessageRetentionPeriodSeconds maps to the underlying
|
||||||
|
Kafka topic's retention.ms.
|
||||||
|
format: int32
|
||||||
|
maximum: 1209600
|
||||||
|
minimum: 60
|
||||||
|
type: integer
|
||||||
|
minShards:
|
||||||
|
default: 1
|
||||||
|
description: MinShards is the floor on shard count; the operator never
|
||||||
|
merges below this.
|
||||||
|
format: int32
|
||||||
|
minimum: 1
|
||||||
|
type: integer
|
||||||
|
partitionsPerShard:
|
||||||
|
default: 6
|
||||||
|
description: PartitionsPerShard is the Kafka partition count on each
|
||||||
|
shard's topic.
|
||||||
|
format: int32
|
||||||
|
minimum: 1
|
||||||
|
type: integer
|
||||||
|
shardSplitCooldownSeconds:
|
||||||
|
default: 300
|
||||||
|
description: |-
|
||||||
|
ShardSplitCooldownSeconds is the minimum age a shard must reach before it
|
||||||
|
is eligible to be split again, preventing rapid re-splitting of a child
|
||||||
|
that hasn't yet absorbed its share of traffic.
|
||||||
|
format: int32
|
||||||
|
minimum: 0
|
||||||
|
type: integer
|
||||||
|
shardSplitThresholdBytesPerSec:
|
||||||
|
default: 5242880
|
||||||
|
description: |-
|
||||||
|
ShardSplitThresholdBytesPerSec is the sustained per-shard throughput that
|
||||||
|
triggers a split into two child shards (design.md §2c).
|
||||||
|
format: int64
|
||||||
|
minimum: 1
|
||||||
|
type: integer
|
||||||
|
visibilityTimeoutSeconds:
|
||||||
|
default: 30
|
||||||
|
description: |-
|
||||||
|
VisibilityTimeoutSeconds is how long a received-but-unacked message stays
|
||||||
|
invisible to other consumers before being redelivered.
|
||||||
|
format: int32
|
||||||
|
maximum: 43200
|
||||||
|
minimum: 0
|
||||||
|
type: integer
|
||||||
|
type: object
|
||||||
|
status:
|
||||||
|
description: QueueStatus defines the observed state of a Queue.
|
||||||
|
properties:
|
||||||
|
conditions:
|
||||||
|
description: Conditions hold detailed status information.
|
||||||
|
items:
|
||||||
|
description: Condition contains details for one aspect of the current
|
||||||
|
state of this API Resource.
|
||||||
|
properties:
|
||||||
|
lastTransitionTime:
|
||||||
|
description: |-
|
||||||
|
lastTransitionTime is the last time the condition transitioned from one status to another.
|
||||||
|
This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
|
||||||
|
format: date-time
|
||||||
|
type: string
|
||||||
|
message:
|
||||||
|
description: |-
|
||||||
|
message is a human readable message indicating details about the transition.
|
||||||
|
This may be an empty string.
|
||||||
|
maxLength: 32768
|
||||||
|
type: string
|
||||||
|
observedGeneration:
|
||||||
|
description: |-
|
||||||
|
observedGeneration represents the .metadata.generation that the condition was set based upon.
|
||||||
|
For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
|
||||||
|
with respect to the current state of the instance.
|
||||||
|
format: int64
|
||||||
|
minimum: 0
|
||||||
|
type: integer
|
||||||
|
reason:
|
||||||
|
description: |-
|
||||||
|
reason contains a programmatic identifier indicating the reason for the condition's last transition.
|
||||||
|
Producers of specific condition types may define expected values and meanings for this field,
|
||||||
|
and whether the values are considered a guaranteed API.
|
||||||
|
The value should be a CamelCase string.
|
||||||
|
This field may not be empty.
|
||||||
|
maxLength: 1024
|
||||||
|
minLength: 1
|
||||||
|
pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
|
||||||
|
type: string
|
||||||
|
status:
|
||||||
|
description: status of the condition, one of True, False, Unknown.
|
||||||
|
enum:
|
||||||
|
- "True"
|
||||||
|
- "False"
|
||||||
|
- Unknown
|
||||||
|
type: string
|
||||||
|
type:
|
||||||
|
description: type of condition in CamelCase or in foo.example.com/CamelCase.
|
||||||
|
maxLength: 316
|
||||||
|
pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
|
||||||
|
type: string
|
||||||
|
required:
|
||||||
|
- lastTransitionTime
|
||||||
|
- message
|
||||||
|
- reason
|
||||||
|
- status
|
||||||
|
- type
|
||||||
|
type: object
|
||||||
|
type: array
|
||||||
|
phase:
|
||||||
|
description: Phase is the current reconciliation phase.
|
||||||
|
enum:
|
||||||
|
- Pending
|
||||||
|
- Ready
|
||||||
|
- Failed
|
||||||
|
type: string
|
||||||
|
shards:
|
||||||
|
description: |-
|
||||||
|
Shards lists every shard backing this queue, active or draining
|
||||||
|
(design.md §2a/§2c).
|
||||||
|
items:
|
||||||
|
description: ShardStatus describes one shard backing a Queue (design.md
|
||||||
|
§2a/§2c).
|
||||||
|
properties:
|
||||||
|
availabilityZones:
|
||||||
|
description: |-
|
||||||
|
AvailabilityZones lists the topology.kubernetes.io/zone values of every
|
||||||
|
node currently hosting a Kafka replica of this shard's topic, resolved
|
||||||
|
from the broker pods' node placement each reconcile. Empty until the
|
||||||
|
first successful resolution (e.g. node lookup failed transiently).
|
||||||
|
items:
|
||||||
|
type: string
|
||||||
|
type: array
|
||||||
|
createdAt:
|
||||||
|
description: |-
|
||||||
|
CreatedAt timestamps when this shard was created, used to enforce
|
||||||
|
ShardSplitCooldownSeconds.
|
||||||
|
format: date-time
|
||||||
|
type: string
|
||||||
|
hashRangeEnd:
|
||||||
|
format: int64
|
||||||
|
type: integer
|
||||||
|
hashRangeStart:
|
||||||
|
description: |-
|
||||||
|
HashRangeStart/HashRangeEnd define the [start, end) murmur2 hash range
|
||||||
|
this shard owns over the 32-bit key space. Stored as int64 (not uint32)
|
||||||
|
because controller-gen maps Go uint32 to OpenAPI format:int32, whose max
|
||||||
|
(2147483647) is smaller than FullHashRangeEnd (0xFFFFFFFF) and the
|
||||||
|
apiserver rejects the status update.
|
||||||
|
format: int64
|
||||||
|
type: integer
|
||||||
|
id:
|
||||||
|
description: ID is the shard's identifier, used in its topic
|
||||||
|
name (kmsvc.{queue}.shard-{id}).
|
||||||
|
type: string
|
||||||
|
parentId:
|
||||||
|
description: |-
|
||||||
|
ParentID is the shard ID this shard was split from, empty for the
|
||||||
|
original shard-0.
|
||||||
|
type: string
|
||||||
|
phase:
|
||||||
|
description: Phase is this shard's lifecycle state.
|
||||||
|
enum:
|
||||||
|
- Active
|
||||||
|
- Closing
|
||||||
|
- Closed
|
||||||
|
type: string
|
||||||
|
topic:
|
||||||
|
description: Topic is the underlying Kafka topic name for this
|
||||||
|
shard.
|
||||||
|
type: string
|
||||||
|
required:
|
||||||
|
- hashRangeEnd
|
||||||
|
- hashRangeStart
|
||||||
|
- id
|
||||||
|
- phase
|
||||||
|
- topic
|
||||||
|
type: object
|
||||||
|
type: array
|
||||||
|
type: object
|
||||||
|
type: object
|
||||||
|
served: true
|
||||||
|
storage: true
|
||||||
|
subresources:
|
||||||
|
status: {}
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: queue-operator
|
||||||
|
namespace: {{ .Values.namespace }}
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: queue-operator
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: queue-operator
|
||||||
|
spec:
|
||||||
|
serviceAccountName: queue-operator
|
||||||
|
containers:
|
||||||
|
- name: queue-operator
|
||||||
|
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
|
||||||
|
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||||
|
env:
|
||||||
|
- name: KMSVC_KAFKA_BROKERS
|
||||||
|
value: {{ .Values.kafkaBrokers | quote }}
|
||||||
|
- name: KMSVC_REDIS_ADDR
|
||||||
|
value: {{ .Values.redisAddr | quote }}
|
||||||
|
- name: GOMEMLIMIT
|
||||||
|
value: {{ .Values.goMemLimit | quote }}
|
||||||
|
- name: KMSVC_NAMESPACE
|
||||||
|
valueFrom:
|
||||||
|
fieldRef:
|
||||||
|
fieldPath: metadata.namespace
|
||||||
|
- name: KMSVC_KAFKA_CLUSTER_NAME
|
||||||
|
value: {{ .Values.kafkaClusterName | quote }}
|
||||||
|
- name: KMSVC_KAFKA_POOL_NAME
|
||||||
|
value: {{ .Values.kafkaPoolName | quote }}
|
||||||
|
resources:
|
||||||
|
{{- toYaml .Values.resources | nindent 12 }}
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: queue-operator
|
||||||
|
namespace: {{ .Values.namespace }}
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRole
|
||||||
|
metadata:
|
||||||
|
name: queue-operator
|
||||||
|
rules:
|
||||||
|
- apiGroups: ["kmsvc.io"]
|
||||||
|
resources: ["queues"]
|
||||||
|
verbs: ["get", "list", "watch", "update", "patch"]
|
||||||
|
- apiGroups: ["kmsvc.io"]
|
||||||
|
resources: ["queues/status"]
|
||||||
|
verbs: ["get", "update", "patch"]
|
||||||
|
- apiGroups: ["kmsvc.io"]
|
||||||
|
resources: ["queues/finalizers"]
|
||||||
|
verbs: ["update"]
|
||||||
|
- apiGroups: ["coordination.k8s.io"]
|
||||||
|
resources: ["leases"]
|
||||||
|
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: ["events"]
|
||||||
|
verbs: ["create", "patch"]
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: ["pods", "nodes"]
|
||||||
|
verbs: ["get"]
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRoleBinding
|
||||||
|
metadata:
|
||||||
|
name: queue-operator
|
||||||
|
roleRef:
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
kind: ClusterRole
|
||||||
|
name: queue-operator
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: queue-operator
|
||||||
|
namespace: {{ .Values.namespace }}
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
namespace: sqs
|
||||||
|
|
||||||
|
image:
|
||||||
|
repository: forgejo.riotpiao.homelab.com/rock/kafka-management-service-queue-operator
|
||||||
|
tag: latest
|
||||||
|
pullPolicy: Always
|
||||||
|
|
||||||
|
kafkaBrokers: "kmsvc-kafka-bootstrap.sqs.svc.cluster.local:9092"
|
||||||
|
redisAddr: "kmsvc-redis-master.sqs.svc.cluster.local:6379"
|
||||||
|
|
||||||
|
# Must match kafka-cluster chart's clusterName/derived pool name -- used to
|
||||||
|
# resolve "<kafkaClusterName>-<kafkaPoolName>-<brokerID>" broker pod names
|
||||||
|
# for AZ-aware Queue status (design.md §2a).
|
||||||
|
kafkaClusterName: kmsvc
|
||||||
|
kafkaPoolName: kmsvc-pool
|
||||||
|
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 128Mi
|
||||||
|
limits:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 256Mi
|
||||||
|
|
||||||
|
# See management-service/values.yaml's goMemLimit comment -- same reasoning.
|
||||||
|
goMemLimit: "230MiB"
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
# design.md §7b: cluster-specific values for the homelab environment.
|
||||||
|
# No secrets here — Authentik client secret etc. flow through the existing
|
||||||
|
# Vault/talos-cli pattern, referenced at deploy time, not inlined.
|
||||||
|
namespace: sqs
|
||||||
|
|
||||||
|
kafkaCluster:
|
||||||
|
nodePool:
|
||||||
|
replicas: 3
|
||||||
|
storage:
|
||||||
|
# longhorn-kafka now uses numberOfReplicas: 3 across 3 zone-labeled nodes.
|
||||||
|
class: longhorn-kafka
|
||||||
|
# Per-node headroom: with 3 nodes and existing PVCs, estimate ~100+ Gi total
|
||||||
|
# available. Each node hosts one replica of all 3 broker volumes, so 3 *
|
||||||
|
# sizeGi must fit. Monitor usage during Kafka deployment.
|
||||||
|
sizeGi: 10
|
||||||
|
resources:
|
||||||
|
memory: 5Gi
|
||||||
|
cpu: "2"
|
||||||
|
|
||||||
|
redis:
|
||||||
|
storageClass: longhorn
|
||||||
|
memoryLimit: 1Gi
|
||||||
|
|
||||||
|
managementService:
|
||||||
|
ingress:
|
||||||
|
host: kmsvc.riotpiao.homelab.com
|
||||||
|
clusterIssuer: homelab-ca
|
||||||
|
authentikIssuerURL: "https://authentik.riotpiao.homelab.com/application/o/kafaka/"
|
||||||
|
authentikAudience: "QI0gPtR99ar8VvhK8Tqox4SDkTKzbNU7lbgwBNSc"
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
environments:
|
||||||
|
default:
|
||||||
|
values:
|
||||||
|
- environments/homelab.yaml
|
||||||
|
homelab:
|
||||||
|
values:
|
||||||
|
- environments/homelab.yaml
|
||||||
|
---
|
||||||
|
helmDefaults:
|
||||||
|
wait: true
|
||||||
|
timeout: 600
|
||||||
|
|
||||||
|
repositories:
|
||||||
|
- name: strimzi
|
||||||
|
url: https://strimzi.io/charts/
|
||||||
|
- name: bitnami
|
||||||
|
url: https://charts.bitnami.com/bitnami
|
||||||
|
|
||||||
|
releases:
|
||||||
|
- name: strimzi-operator
|
||||||
|
namespace: {{ .Values.namespace }}
|
||||||
|
chart: strimzi/strimzi-kafka-operator
|
||||||
|
version: 0.46.0
|
||||||
|
values:
|
||||||
|
- watchNamespaces: ["{{ .Values.namespace }}"]
|
||||||
|
|
||||||
|
- name: kafka-cluster
|
||||||
|
namespace: {{ .Values.namespace }}
|
||||||
|
chart: charts/kafka-cluster
|
||||||
|
needs:
|
||||||
|
- {{ .Values.namespace }}/strimzi-operator
|
||||||
|
values:
|
||||||
|
- namespace: {{ .Values.namespace }}
|
||||||
|
nodePool:
|
||||||
|
replicas: {{ .Values.kafkaCluster.nodePool.replicas }}
|
||||||
|
storage:
|
||||||
|
class: {{ .Values.kafkaCluster.nodePool.storage.class }}
|
||||||
|
sizeGi: {{ .Values.kafkaCluster.nodePool.storage.sizeGi }}
|
||||||
|
resources:
|
||||||
|
memory: {{ .Values.kafkaCluster.nodePool.resources.memory }}
|
||||||
|
cpu: {{ .Values.kafkaCluster.nodePool.resources.cpu | quote }}
|
||||||
|
|
||||||
|
- name: kmsvc-redis
|
||||||
|
namespace: {{ .Values.namespace }}
|
||||||
|
chart: bitnami/redis
|
||||||
|
version: 20.6.0
|
||||||
|
values:
|
||||||
|
- architecture: standalone
|
||||||
|
# Bitnami stopped publishing version-pinned tags under docker.io/bitnami
|
||||||
|
# (only `latest` remains there); bitnamilegacy/* mirrors the old
|
||||||
|
# versioned tags for free, so pin there instead of floating on `latest`.
|
||||||
|
# The chart's container-image allowlist check doesn't know about the
|
||||||
|
# legacy mirror, hence allowInsecureImages.
|
||||||
|
global:
|
||||||
|
security:
|
||||||
|
allowInsecureImages: true
|
||||||
|
image:
|
||||||
|
repository: bitnamilegacy/redis
|
||||||
|
auth:
|
||||||
|
enabled: false
|
||||||
|
master:
|
||||||
|
persistence:
|
||||||
|
enabled: true
|
||||||
|
storageClass: {{ .Values.redis.storageClass }}
|
||||||
|
size: 2Gi
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
memory: {{ .Values.redis.memoryLimit }}
|
||||||
|
requests:
|
||||||
|
memory: {{ .Values.redis.memoryLimit }}
|
||||||
|
|
||||||
|
- name: queue-crd
|
||||||
|
namespace: {{ .Values.namespace }}
|
||||||
|
chart: charts/queue-crd
|
||||||
|
needs:
|
||||||
|
- {{ .Values.namespace }}/kafka-cluster
|
||||||
|
- {{ .Values.namespace }}/kmsvc-redis
|
||||||
|
values:
|
||||||
|
- namespace: {{ .Values.namespace }}
|
||||||
|
kafkaBrokers: "kmsvc-kafka-bootstrap.{{ .Values.namespace }}.svc.cluster.local:9092"
|
||||||
|
redisAddr: "kmsvc-redis-master.{{ .Values.namespace }}.svc.cluster.local:6379"
|
||||||
|
|
||||||
|
- name: management-service
|
||||||
|
namespace: {{ .Values.namespace }}
|
||||||
|
chart: charts/management-service
|
||||||
|
needs:
|
||||||
|
- {{ .Values.namespace }}/kafka-cluster
|
||||||
|
- {{ .Values.namespace }}/kmsvc-redis
|
||||||
|
values:
|
||||||
|
- namespace: {{ .Values.namespace }}
|
||||||
|
env:
|
||||||
|
kafkaBrokers: "kmsvc-kafka-bootstrap.{{ .Values.namespace }}.svc.cluster.local:9092"
|
||||||
|
redisAddr: "kmsvc-redis-master.{{ .Values.namespace }}.svc.cluster.local:6379"
|
||||||
|
authentikIssuerURL: {{ .Values.managementService.authentikIssuerURL | quote }}
|
||||||
|
authentikAudience: {{ .Values.managementService.authentikAudience | quote }}
|
||||||
|
ingress:
|
||||||
|
enabled: true
|
||||||
|
host: {{ .Values.managementService.ingress.host | quote }}
|
||||||
|
clusterIssuer: {{ .Values.managementService.ingress.clusterIssuer | quote }}
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
# Ingress for kmsvc REST API — routes to OAuth2-Proxy
|
||||||
|
# TLS terminated here; oauth2-proxy handles OIDC auth
|
||||||
|
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: Ingress
|
||||||
|
metadata:
|
||||||
|
name: kmsvc
|
||||||
|
namespace: sqs
|
||||||
|
spec:
|
||||||
|
ingressClassName: nginx
|
||||||
|
tls:
|
||||||
|
- secretName: kmsvc-tls
|
||||||
|
hosts:
|
||||||
|
- kmsvc.riotpiao.homelab.com
|
||||||
|
rules:
|
||||||
|
- host: kmsvc.riotpiao.homelab.com
|
||||||
|
http:
|
||||||
|
paths:
|
||||||
|
- path: /
|
||||||
|
pathType: Prefix
|
||||||
|
backend:
|
||||||
|
service:
|
||||||
|
name: oauth2-proxy
|
||||||
|
port:
|
||||||
|
number: 4180
|
||||||
@@ -0,0 +1,106 @@
|
|||||||
|
# OAuth2-Proxy for kmsvc REST API
|
||||||
|
# Protects gRPC-gateway (REST) endpoint with Authentik OIDC
|
||||||
|
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: oauth2-proxy
|
||||||
|
namespace: sqs
|
||||||
|
|
||||||
|
---
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: oauth2-proxy
|
||||||
|
namespace: sqs
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: oauth2-proxy
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: oauth2-proxy
|
||||||
|
annotations:
|
||||||
|
secret.reloader.stakater.com/reload: "kmsvc-oidc"
|
||||||
|
spec:
|
||||||
|
serviceAccountName: oauth2-proxy
|
||||||
|
containers:
|
||||||
|
- name: oauth2-proxy
|
||||||
|
image: quay.io/oauth2-proxy/oauth2-proxy:v7.5.1
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
containerPort: 4180
|
||||||
|
protocol: TCP
|
||||||
|
env:
|
||||||
|
- name: OAUTH2_PROXY_PROVIDER
|
||||||
|
value: "oidc"
|
||||||
|
- name: OAUTH2_PROXY_OIDC_ISSUER_URL
|
||||||
|
value: "https://authentik.riotpiao.homelab.com/application/o/kmsvc/"
|
||||||
|
- name: OAUTH2_PROXY_CLIENT_ID
|
||||||
|
value: "kmsvc"
|
||||||
|
- name: OAUTH2_PROXY_CLIENT_SECRET
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: kmsvc-oidc
|
||||||
|
key: clientSecret
|
||||||
|
- name: OAUTH2_PROXY_COOKIE_SECRET
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: kmsvc-oidc
|
||||||
|
key: cookieSecret
|
||||||
|
- name: OAUTH2_PROXY_REDIRECT_URL
|
||||||
|
value: "https://kmsvc.riotpiao.homelab.com/oauth2/callback"
|
||||||
|
- name: OAUTH2_PROXY_UPSTREAM
|
||||||
|
value: "http://kmsvc-management-service:8080"
|
||||||
|
- name: OAUTH2_PROXY_COOKIE_SECURE
|
||||||
|
value: "true"
|
||||||
|
- name: OAUTH2_PROXY_COOKIE_HTTPONLY
|
||||||
|
value: "true"
|
||||||
|
- name: OAUTH2_PROXY_COOKIE_SAMESITE
|
||||||
|
value: "Lax"
|
||||||
|
- name: OAUTH2_PROXY_EMAIL_DOMAIN
|
||||||
|
value: "*"
|
||||||
|
- name: OAUTH2_PROXY_SKIP_AUTH_REGEX
|
||||||
|
value: "^/health"
|
||||||
|
- name: OAUTH2_PROXY_PASS_AUTHORIZATION_HEADER
|
||||||
|
value: "true"
|
||||||
|
- name: OAUTH2_PROXY_REVERSE_PROXY
|
||||||
|
value: "true"
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 128Mi
|
||||||
|
limits:
|
||||||
|
cpu: 200m
|
||||||
|
memory: 256Mi
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /ping
|
||||||
|
port: http
|
||||||
|
initialDelaySeconds: 10
|
||||||
|
periodSeconds: 10
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /ping
|
||||||
|
port: http
|
||||||
|
initialDelaySeconds: 5
|
||||||
|
periodSeconds: 5
|
||||||
|
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: oauth2-proxy
|
||||||
|
namespace: sqs
|
||||||
|
spec:
|
||||||
|
type: ClusterIP
|
||||||
|
ports:
|
||||||
|
- port: 4180
|
||||||
|
targetPort: http
|
||||||
|
protocol: TCP
|
||||||
|
name: http
|
||||||
|
selector:
|
||||||
|
app: oauth2-proxy
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
apiVersion: kmsvc.io/v1
|
||||||
|
kind: Queue
|
||||||
|
metadata:
|
||||||
|
name: orders-fifo
|
||||||
|
namespace: sqs
|
||||||
|
spec:
|
||||||
|
fifoQueue: true
|
||||||
|
visibilityTimeoutSeconds: 30
|
||||||
|
messageRetentionPeriodSeconds: 345600
|
||||||
|
maxReceiveCount: 5
|
||||||
|
deadLetterTargetQueue: orders-fifo-dlq
|
||||||
|
delaySeconds: 0
|
||||||
|
partitionsPerShard: 6
|
||||||
|
minShards: 1
|
||||||
|
maxShards: 8
|
||||||
|
shardSplitThresholdBytesPerSec: 5242880
|
||||||
|
shardSplitCooldownSeconds: 300
|
||||||
|
---
|
||||||
|
apiVersion: kmsvc.io/v1
|
||||||
|
kind: Queue
|
||||||
|
metadata:
|
||||||
|
name: orders-fifo-dlq
|
||||||
|
namespace: sqs
|
||||||
|
spec:
|
||||||
|
fifoQueue: true
|
||||||
|
isDLQ: true
|
||||||
|
visibilityTimeoutSeconds: 30
|
||||||
|
messageRetentionPeriodSeconds: 1209600
|
||||||
|
maxReceiveCount: 5
|
||||||
|
partitionsPerShard: 6
|
||||||
|
minShards: 1
|
||||||
|
maxShards: 1
|
||||||
@@ -0,0 +1,237 @@
|
|||||||
|
# Phase 2: Namespace-Scoped Auto-Provisioning Testing Guide
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
Phase 2 implements **namespace-scoped automatic TemporalWorker provisioning** (Kafka broker model).
|
||||||
|
|
||||||
|
One TemporalWorker per Temporal namespace processes ALL task queues in that namespace. When multiple Queues share the same `temporal.io/namespace` label, they trigger creation of a single TemporalWorker that handles all of them.
|
||||||
|
|
||||||
|
```
|
||||||
|
Queues (labeled temporal.io/namespace: "production")
|
||||||
|
├── orders-fifo
|
||||||
|
├── payments
|
||||||
|
└── notifications
|
||||||
|
↓
|
||||||
|
queue-operator creates 1 TemporalWorker (worker-production)
|
||||||
|
↓
|
||||||
|
TemporalWorker controller creates 1 Deployment
|
||||||
|
↓
|
||||||
|
Worker pod(s) connect to Temporal namespace "production"
|
||||||
|
↓
|
||||||
|
Process ALL task queues in that namespace (scale horizontally by replicas)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Implementation Changes
|
||||||
|
|
||||||
|
### 1. TemporalWorker CRD (`apis/kmsvc/v1/temporalworker_types.go`)
|
||||||
|
- New Kubernetes resource type to manage namespace-scoped workers
|
||||||
|
- Fields: Namespace (required), Image, Replicas, Resources, NodeSelector, Affinity, Tolerations
|
||||||
|
- Status: Phase (Pending/Ready/Failed), Replicas, ReadyReplicas, Conditions
|
||||||
|
- Model: 1 TemporalWorker per Temporal namespace (not per queue)
|
||||||
|
|
||||||
|
### 2. QueueReconciler Extension (`internal/operator/queue_controller.go`)
|
||||||
|
- New method: `reconcileTemporalWorker()`
|
||||||
|
- Logic: If Queue has `temporal.io/namespace` label, create TemporalWorker for that namespace
|
||||||
|
- Idempotent: multiple queues with same namespace label create same TemporalWorker (no duplicates)
|
||||||
|
|
||||||
|
### 3. TemporalWorkerReconciler (`internal/operator/temporal_worker_controller.go`)
|
||||||
|
- New controller watching TemporalWorker objects
|
||||||
|
- Creates/updates Kubernetes Deployment with:
|
||||||
|
- Pod spec: container image, env vars (TEMPORAL_FRONTEND_ADDRESS, TEMPORAL_TASK_QUEUE)
|
||||||
|
- Replicas, resources, node selector, affinity, tolerations from TemporalWorker spec
|
||||||
|
- Updates TemporalWorker status with deployment replica counts and phase
|
||||||
|
|
||||||
|
### 4. Operator Main (`cmd/queue-operator/main.go`)
|
||||||
|
- Registers TemporalWorker CRD in scheme
|
||||||
|
- Registers TemporalWorkerReconciler controller
|
||||||
|
- Controller watches TemporalWorker objects; owns Deployment objects
|
||||||
|
|
||||||
|
## Testing Procedure
|
||||||
|
|
||||||
|
### Prerequisites
|
||||||
|
- kmsvc queue-operator must be running (built and deployed)
|
||||||
|
- Temporal cluster must be ready (temporal-frontend service available at `temporal-frontend.temporal.svc.cluster.local:7233`)
|
||||||
|
- story-crater-backend Docker image must exist (used as default worker image)
|
||||||
|
|
||||||
|
### Step 1: Build and Deploy kmsvc Operator
|
||||||
|
```bash
|
||||||
|
cd /Users/rockliang/workplace/kmsvc-manage
|
||||||
|
make build # builds queue-operator binary
|
||||||
|
make docker-build # builds Docker image
|
||||||
|
make deploy # deploys to cluster (requires Helm chart)
|
||||||
|
```
|
||||||
|
|
||||||
|
Or manually:
|
||||||
|
```bash
|
||||||
|
cd /Users/rockliang/workplace/kmsvc-manage
|
||||||
|
go build -o bin/queue-operator ./cmd/queue-operator
|
||||||
|
kubectl apply -f k8s/queue-operator-rbac.yaml
|
||||||
|
kubectl apply -f k8s/queue-operator-deployment.yaml
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 2: Create Queues with Temporal Namespace Labels
|
||||||
|
```bash
|
||||||
|
kubectl apply -f /Users/rockliang/workplace/homelab/k8s/temporal/queues/example-queue.yaml
|
||||||
|
```
|
||||||
|
|
||||||
|
Verify Queues are Ready:
|
||||||
|
```bash
|
||||||
|
kubectl get queue -n sqs -l temporal.io/namespace=production
|
||||||
|
kubectl describe queue -n sqs story-crater-tasks
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected:
|
||||||
|
```
|
||||||
|
NAME FIFO PHASE AGE
|
||||||
|
story-crater-tasks false Ready 5s
|
||||||
|
story-crater-notifications false Ready 5s
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 3: Verify TemporalWorker CRD Auto-Created (1 per namespace)
|
||||||
|
```bash
|
||||||
|
kubectl get temporalworker -n temporal
|
||||||
|
kubectl describe temporalworker -n temporal worker-production
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected:
|
||||||
|
```
|
||||||
|
NAME PHASE READY DESIRED AGE
|
||||||
|
worker-production Pending 0 1 5s
|
||||||
|
```
|
||||||
|
|
||||||
|
Only ONE TemporalWorker for all queues in "production" namespace!
|
||||||
|
|
||||||
|
### Step 4: Verify Deployment Auto-Created
|
||||||
|
```bash
|
||||||
|
kubectl get deploy -n temporal -l app.kubernetes.io/managed-by=kmsvc-temporal-operator
|
||||||
|
kubectl get pods -n temporal -l app.kubernetes.io/instance=worker-production
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected:
|
||||||
|
```
|
||||||
|
NAME READY UP-TO-DATE AVAILABLE AGE
|
||||||
|
worker-production 1/1 1 1 10s
|
||||||
|
|
||||||
|
NAME READY STATUS RESTARTS AGE
|
||||||
|
worker-production-5f8b4c... 1/1 Running 0 10s
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 5: Verify Worker Connected to Temporal Namespace
|
||||||
|
Check Temporal UI for namespace "production":
|
||||||
|
```bash
|
||||||
|
open https://temporal.riotpiao.homelab.com/namespaces/production/task-queues
|
||||||
|
```
|
||||||
|
|
||||||
|
Look for all task queues with worker count > 0:
|
||||||
|
- `story-crater-tasks`
|
||||||
|
- `story-crater-notifications`
|
||||||
|
- (worker processes all of them)
|
||||||
|
|
||||||
|
Or via CLI:
|
||||||
|
```bash
|
||||||
|
kubectl port-forward -n temporal svc/temporal-frontend 7233 &
|
||||||
|
curl http://localhost:7233/api/v1/task-queues?namespace=production
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 6: Verify TemporalWorker Status Updated
|
||||||
|
```bash
|
||||||
|
kubectl get temporalworker -n temporal
|
||||||
|
kubectl describe temporalworker -n temporal worker-production
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected:
|
||||||
|
```
|
||||||
|
NAME PHASE READY DESIRED AGE
|
||||||
|
worker-production Ready 1 1 15s
|
||||||
|
|
||||||
|
Status:
|
||||||
|
Phase: Ready
|
||||||
|
Ready Replicas: 1
|
||||||
|
Replicas: 1
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 7: Test Namespace-Level Scaling
|
||||||
|
Create more queues in the same namespace:
|
||||||
|
```yaml
|
||||||
|
apiVersion: kmsvc.io/v1
|
||||||
|
kind: Queue
|
||||||
|
metadata:
|
||||||
|
name: story-crater-llm-processing
|
||||||
|
namespace: sqs
|
||||||
|
labels:
|
||||||
|
temporal.io/namespace: "production" # same namespace
|
||||||
|
```
|
||||||
|
|
||||||
|
Verify: No new TemporalWorker created (same worker handles all 3 queues):
|
||||||
|
```bash
|
||||||
|
kubectl get temporalworker -n temporal # still just 1 worker-production
|
||||||
|
kubectl get deploy -n temporal worker-production # same deployment
|
||||||
|
```
|
||||||
|
|
||||||
|
Worker auto-discovers new task queue in namespace and processes it.
|
||||||
|
|
||||||
|
### Step 8: Test Cascading Deletion
|
||||||
|
Delete a Queue; worker should remain (other queues still need it):
|
||||||
|
```bash
|
||||||
|
kubectl delete queue -n sqs story-crater-notifications
|
||||||
|
```
|
||||||
|
|
||||||
|
Verify:
|
||||||
|
```bash
|
||||||
|
kubectl get temporalworker -n temporal # worker-production still exists
|
||||||
|
kubectl get pods -n temporal worker-production # still running
|
||||||
|
```
|
||||||
|
|
||||||
|
Delete all queues in namespace:
|
||||||
|
```bash
|
||||||
|
kubectl delete queue -n sqs -l temporal.io/namespace=production
|
||||||
|
```
|
||||||
|
|
||||||
|
Verify: TemporalWorker now has no owner (not cascade-deleted; manual cleanup needed):
|
||||||
|
```bash
|
||||||
|
kubectl get temporalworker -n temporal # worker-production still there (manual cleanup)
|
||||||
|
kubectl delete temporalworker -n temporal worker-production # cleanup manually
|
||||||
|
```
|
||||||
|
|
||||||
|
## Debugging
|
||||||
|
|
||||||
|
### Queue stuck in Pending
|
||||||
|
Check queue-operator logs:
|
||||||
|
```bash
|
||||||
|
kubectl logs -n sqs deploy/kmsvc-queue-operator -f
|
||||||
|
kubectl logs -n sqs deploy/kmsvc-queue-operator --tail=50 | grep -i error
|
||||||
|
```
|
||||||
|
|
||||||
|
### TemporalWorker not created
|
||||||
|
- Verify Queue has the label: `kubectl get queue -o yaml | grep temporal.io`
|
||||||
|
- Check queue-operator logs for "reconcileTemporalWorker" errors
|
||||||
|
|
||||||
|
### Deployment not created
|
||||||
|
- Check TemporalWorker controller logs: `kubectl logs -n sqs deploy/kmsvc-queue-operator -f`
|
||||||
|
- Verify TemporalWorker exists: `kubectl get temporalworker -n temporal`
|
||||||
|
- Check Deployment errors: `kubectl describe deploy -n temporal worker-story-crater-tasks`
|
||||||
|
|
||||||
|
### Worker not showing in Temporal UI
|
||||||
|
- Check pod logs: `kubectl logs -n temporal deploy/worker-story-crater-tasks`
|
||||||
|
- Verify env vars: `kubectl set env pod -n temporal <pod-name> --list | grep TEMPORAL`
|
||||||
|
- Test connectivity: `kubectl exec -n temporal <pod-name> -- nc -zv temporal-frontend.temporal.svc.cluster.local 7233`
|
||||||
|
|
||||||
|
## Next Steps
|
||||||
|
|
||||||
|
Once Phase 2 is working:
|
||||||
|
1. **Phase 3 (Future):** Implement autoscaling based on queue depth metrics
|
||||||
|
2. **Production Hardening:**
|
||||||
|
- Add QueueRef validation (ensure Queue exists in sqs namespace)
|
||||||
|
- Add image validation/defaults from ConfigMap
|
||||||
|
- Add worker readiness probe configuration
|
||||||
|
- Add graceful shutdown/drain behavior
|
||||||
|
|
||||||
|
## Files Modified/Created
|
||||||
|
|
||||||
|
| File | Change |
|
||||||
|
|------|--------|
|
||||||
|
| `apis/kmsvc/v1/temporalworker_types.go` | NEW: CRD type definitions |
|
||||||
|
| `internal/operator/queue_controller.go` | MODIFIED: Added reconcileTemporalWorker() |
|
||||||
|
| `internal/operator/temporal_worker_controller.go` | NEW: TemporalWorker → Deployment reconciler |
|
||||||
|
| `cmd/queue-operator/main.go` | MODIFIED: Register TemporalWorker CRD + controller |
|
||||||
|
| `k8s/temporal/queues/example-queue.yaml` | NEW: Example Queue with label |
|
||||||
@@ -0,0 +1,220 @@
|
|||||||
|
# Temporal OAuth2-Proxy Setup (Authentik OIDC)
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
Protects Temporal UI with Authentik OIDC authentication. Traffic flow:
|
||||||
|
|
||||||
|
```
|
||||||
|
Browser → Ingress (TLS) → oauth2-proxy (OIDC check) → temporal-web (internal)
|
||||||
|
↓
|
||||||
|
Redirects to Authentik login
|
||||||
|
↓
|
||||||
|
JWT cookie issued
|
||||||
|
↓
|
||||||
|
Forwards to temporal-web
|
||||||
|
```
|
||||||
|
|
||||||
|
## Prerequisites
|
||||||
|
|
||||||
|
✅ Authentik OIDC provider `temporal` already exists with:
|
||||||
|
- Client ID: `temporal`
|
||||||
|
- Client Secret: stored in Kubernetes secret `temporal-oidc` (key: `clientSecret`)
|
||||||
|
- Redirect URI: `https://temporal.riotpiao.homelab.com/oauth2/callback`
|
||||||
|
|
||||||
|
## Secrets
|
||||||
|
|
||||||
|
The `temporal-oidc` secret must contain:
|
||||||
|
|
||||||
|
| Key | Value | Source |
|
||||||
|
|-----|-------|--------|
|
||||||
|
| `clientSecret` | OAuth2 client secret from Authentik | Authentik → Applications → temporal |
|
||||||
|
| `cookieSecret` | Session encryption key (base64 32-byte) | Generate: `openssl rand -base64 32` |
|
||||||
|
|
||||||
|
### Check existing secret:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
kubectl get secret -n temporal temporal-oidc
|
||||||
|
kubectl describe secret -n temporal temporal-oidc
|
||||||
|
```
|
||||||
|
|
||||||
|
### If missing, create it:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Get client secret from Authentik UI
|
||||||
|
# Applications → temporal → copy "Client Secret"
|
||||||
|
CLIENT_SECRET="..."
|
||||||
|
|
||||||
|
# Generate cookie secret
|
||||||
|
COOKIE_SECRET=$(openssl rand -base64 32)
|
||||||
|
|
||||||
|
# Create secret
|
||||||
|
kubectl create secret generic temporal-oidc \
|
||||||
|
-n temporal \
|
||||||
|
--from-literal=clientSecret="${CLIENT_SECRET}" \
|
||||||
|
--from-literal=cookieSecret="${COOKIE_SECRET}"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Deployment Steps
|
||||||
|
|
||||||
|
### Step 1: Apply OAuth2-Proxy Manifests
|
||||||
|
```bash
|
||||||
|
kubectl apply -f k8s/temporal/oauth2-proxy.yaml
|
||||||
|
```
|
||||||
|
|
||||||
|
Verify:
|
||||||
|
```bash
|
||||||
|
kubectl get deploy -n temporal oauth2-proxy
|
||||||
|
kubectl logs -n temporal deploy/oauth2-proxy
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected log:
|
||||||
|
```
|
||||||
|
[<timestamp>] [oauthproxy.go:...] Listening on 0.0.0.0:4180
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 2: Apply OAuth2-Proxy Ingress
|
||||||
|
```bash
|
||||||
|
kubectl apply -f k8s/temporal/temporal-ingress-oauth2.yaml
|
||||||
|
```
|
||||||
|
|
||||||
|
Verify:
|
||||||
|
```bash
|
||||||
|
kubectl get ingress -n temporal
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected:
|
||||||
|
```
|
||||||
|
NAME CLASS HOSTS ADDRESS PORTS AGE
|
||||||
|
temporal nginx temporal.riotpiao.homelab.com ... 80, 443 10s
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 3: Test Access
|
||||||
|
|
||||||
|
1. **Open Temporal UI (unauthenticated):**
|
||||||
|
```bash
|
||||||
|
open https://temporal.riotpiao.homelab.com
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: Redirects to Authentik login page
|
||||||
|
|
||||||
|
2. **Login with Authentik credentials**
|
||||||
|
- Username/email
|
||||||
|
- Password
|
||||||
|
- Should redirect back to `temporal.riotpiao.homelab.com` and display UI
|
||||||
|
|
||||||
|
3. **Verify auth:**
|
||||||
|
```bash
|
||||||
|
# Check for oauth2_proxy cookie
|
||||||
|
curl -v https://temporal.riotpiao.homelab.com 2>&1 | grep -i cookie
|
||||||
|
```
|
||||||
|
|
||||||
|
4. **Check oauth2-proxy logs:**
|
||||||
|
```bash
|
||||||
|
kubectl logs -n temporal deploy/oauth2-proxy -f
|
||||||
|
```
|
||||||
|
|
||||||
|
Look for:
|
||||||
|
```
|
||||||
|
[timestamp] [auth_test.go:...] Authentication successful
|
||||||
|
```
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
### Redirect URI mismatch
|
||||||
|
Error in oauth2-proxy logs:
|
||||||
|
```
|
||||||
|
redirect_uri_mismatch: The redirect_uri does not match the one registered in Authentik
|
||||||
|
```
|
||||||
|
|
||||||
|
Fix:
|
||||||
|
- Verify Authentik application (Applications → temporal) has redirect URI: `https://temporal.riotpiao.homelab.com/oauth2/callback`
|
||||||
|
- Ensure HTTPS (not HTTP)
|
||||||
|
|
||||||
|
### Missing secret
|
||||||
|
Error:
|
||||||
|
```
|
||||||
|
clientSecret: key not found in temporal-oidc secret
|
||||||
|
```
|
||||||
|
|
||||||
|
Fix:
|
||||||
|
```bash
|
||||||
|
kubectl get secret -n temporal temporal-oidc -o yaml
|
||||||
|
# If missing, create per "Secrets" section above
|
||||||
|
```
|
||||||
|
|
||||||
|
### Cookie secret expiration
|
||||||
|
OAuth2-Proxy won't start if `cookieSecret` is empty or invalid.
|
||||||
|
|
||||||
|
Fix:
|
||||||
|
```bash
|
||||||
|
COOKIE_SECRET=$(openssl rand -base64 32)
|
||||||
|
kubectl patch secret temporal-oidc -n temporal \
|
||||||
|
-p "{\"data\":{\"cookieSecret\":\"$(echo -n $COOKIE_SECRET | base64)\"}}}"
|
||||||
|
kubectl rollout restart deploy/oauth2-proxy -n temporal
|
||||||
|
```
|
||||||
|
|
||||||
|
### oauth2-proxy crashes with "connection refused"
|
||||||
|
Error in logs:
|
||||||
|
```
|
||||||
|
upstream connect error or disconnect/reset before headers
|
||||||
|
```
|
||||||
|
|
||||||
|
Likely cause: `temporal-web` service not accessible.
|
||||||
|
|
||||||
|
Check:
|
||||||
|
```bash
|
||||||
|
kubectl get svc -n temporal temporal-web
|
||||||
|
kubectl exec -n temporal deploy/oauth2-proxy -- curl http://temporal-web:8080
|
||||||
|
```
|
||||||
|
|
||||||
|
## File Structure
|
||||||
|
|
||||||
|
```
|
||||||
|
k8s/temporal/
|
||||||
|
├── oauth2-proxy.yaml # oauth2-proxy Deployment + Service + SA
|
||||||
|
├── temporal-ingress-oauth2.yaml # Ingress routing to oauth2-proxy
|
||||||
|
├── oauth2-proxy-values.yaml # Helm values (reference only)
|
||||||
|
└── temporal-values.yaml # Modified: ingress.enabled=false
|
||||||
|
```
|
||||||
|
|
||||||
|
## Next: Add to Helmfile
|
||||||
|
|
||||||
|
If integrating with helmfile.yaml.gotmpl:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
releases:
|
||||||
|
- name: temporal
|
||||||
|
# ... existing config ...
|
||||||
|
hooks:
|
||||||
|
postSync:
|
||||||
|
- events: ["success"]
|
||||||
|
showlogs: true
|
||||||
|
command: "sh"
|
||||||
|
args:
|
||||||
|
- -c
|
||||||
|
- |
|
||||||
|
kubectl apply -f k8s/temporal/oauth2-proxy.yaml
|
||||||
|
kubectl apply -f k8s/temporal/temporal-ingress-oauth2.yaml
|
||||||
|
```
|
||||||
|
|
||||||
|
Or add separate releases:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
- name: oauth2-proxy-temporal
|
||||||
|
namespace: temporal
|
||||||
|
chart: oauth2-proxy/oauth2-proxy
|
||||||
|
version: "6.x.x"
|
||||||
|
values:
|
||||||
|
- k8s/temporal/oauth2-proxy-values.yaml
|
||||||
|
set:
|
||||||
|
- name: config.clientSecret
|
||||||
|
value: "{{ (env "TEMPORAL_OIDC_CLIENT_SECRET") }}"
|
||||||
|
- name: config.cookieSecret
|
||||||
|
value: "{{ (env "TEMPORAL_OIDC_COOKIE_SECRET") }}"
|
||||||
|
```
|
||||||
|
|
||||||
|
Then add to `.env`:
|
||||||
|
```bash
|
||||||
|
TEMPORAL_OIDC_CLIENT_SECRET=<from Authentik>
|
||||||
|
TEMPORAL_OIDC_COOKIE_SECRET=$(openssl rand -base64 32)
|
||||||
|
```
|
||||||
@@ -0,0 +1,102 @@
|
|||||||
|
# Elasticsearch 7.17.0 for Temporal visibility store
|
||||||
|
# Deployed to worker nodes (not control plane to save CP resources for LLM work)
|
||||||
|
# 2Gi heap + 4Gi memory limit for stable operation
|
||||||
|
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: elasticsearch-config
|
||||||
|
namespace: temporal
|
||||||
|
data:
|
||||||
|
elasticsearch.yml: |
|
||||||
|
cluster.name: temporal-elasticsearch
|
||||||
|
node.name: temporal-elasticsearch-0
|
||||||
|
discovery.type: single-node
|
||||||
|
network.host: 0.0.0.0
|
||||||
|
http.host: 0.0.0.0
|
||||||
|
|
||||||
|
---
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: temporal-elasticsearch
|
||||||
|
namespace: temporal
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: temporal-elasticsearch
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: temporal-elasticsearch
|
||||||
|
spec:
|
||||||
|
affinity:
|
||||||
|
nodeAffinity:
|
||||||
|
requiredDuringSchedulingIgnoredDuringExecution:
|
||||||
|
nodeSelectorTerms:
|
||||||
|
- matchExpressions:
|
||||||
|
- key: node-role.kubernetes.io/worker
|
||||||
|
operator: Exists
|
||||||
|
containers:
|
||||||
|
- name: elasticsearch
|
||||||
|
image: docker.elastic.co/elasticsearch/elasticsearch:7.17.0
|
||||||
|
env:
|
||||||
|
- name: discovery.type
|
||||||
|
value: single-node
|
||||||
|
- name: "ES_JAVA_OPTS"
|
||||||
|
value: "-Xms2g -Xmx2g"
|
||||||
|
ports:
|
||||||
|
- containerPort: 9200
|
||||||
|
name: http
|
||||||
|
- containerPort: 9300
|
||||||
|
name: transport
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /_cluster/health
|
||||||
|
port: 9200
|
||||||
|
initialDelaySeconds: 180
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 10
|
||||||
|
failureThreshold: 5
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /_cluster/health?local=true
|
||||||
|
port: 9200
|
||||||
|
initialDelaySeconds: 150
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 10
|
||||||
|
failureThreshold: 5
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 2Gi
|
||||||
|
limits:
|
||||||
|
cpu: 2000m
|
||||||
|
memory: 4Gi
|
||||||
|
volumeMounts:
|
||||||
|
- name: config
|
||||||
|
mountPath: /usr/share/elasticsearch/config/elasticsearch.yml
|
||||||
|
subPath: elasticsearch.yml
|
||||||
|
volumes:
|
||||||
|
- name: config
|
||||||
|
configMap:
|
||||||
|
name: elasticsearch-config
|
||||||
|
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: temporal-elasticsearch
|
||||||
|
namespace: temporal
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
app: temporal-elasticsearch
|
||||||
|
ports:
|
||||||
|
- port: 9200
|
||||||
|
targetPort: 9200
|
||||||
|
name: http
|
||||||
|
- port: 9300
|
||||||
|
targetPort: 9300
|
||||||
|
name: transport
|
||||||
|
type: ClusterIP
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
# OAuth2-Proxy for Temporal UI — protects with Authentik OIDC
|
||||||
|
# Deployed via Helm: oauth2-proxy/oauth2-proxy chart
|
||||||
|
|
||||||
|
config:
|
||||||
|
clientID: temporal
|
||||||
|
clientSecret: "" # injected from temporal-oidc secret
|
||||||
|
cookieSecret: "" # generated; helm --set will override
|
||||||
|
configFile: ""
|
||||||
|
|
||||||
|
auth:
|
||||||
|
enabled: true
|
||||||
|
|
||||||
|
extraArgs:
|
||||||
|
- --provider=oidc
|
||||||
|
- --oidc-issuer-url=https://authentik.riotpiao.homelab.com/application/o/temporal/
|
||||||
|
- --redirect-url=https://temporal.riotpiao.homelab.com/oauth2/callback
|
||||||
|
- --upstream=http://temporal-web:8080
|
||||||
|
- --cookie-secure=true
|
||||||
|
- --cookie-httponly=true
|
||||||
|
- --cookie-samesite=Lax
|
||||||
|
- --email-domain=*
|
||||||
|
- --skip-auth-regex=^/health
|
||||||
|
- --pass-authorization-header=true
|
||||||
|
- --skip-auth-preflight=true
|
||||||
|
|
||||||
|
service:
|
||||||
|
type: ClusterIP
|
||||||
|
port: 4180
|
||||||
|
targetPort: 4180
|
||||||
|
|
||||||
|
ingress:
|
||||||
|
enabled: false # we'll keep temporal's ingress, just route to oauth2-proxy
|
||||||
|
|
||||||
|
replicaCount: 1
|
||||||
|
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 128Mi
|
||||||
|
limits:
|
||||||
|
cpu: 200m
|
||||||
|
memory: 256Mi
|
||||||
|
|
||||||
|
podAnnotations:
|
||||||
|
secret.reloader.stakater.com/reload: "temporal-oidc"
|
||||||
@@ -0,0 +1,108 @@
|
|||||||
|
# OAuth2-Proxy deployment for Temporal UI
|
||||||
|
# Requires: temporal-oidc secret with clientSecret and cookieSecret
|
||||||
|
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: oauth2-proxy
|
||||||
|
namespace: temporal
|
||||||
|
|
||||||
|
---
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: oauth2-proxy
|
||||||
|
namespace: temporal
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: oauth2-proxy
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: oauth2-proxy
|
||||||
|
annotations:
|
||||||
|
secret.reloader.stakater.com/reload: "temporal-oidc"
|
||||||
|
spec:
|
||||||
|
serviceAccountName: oauth2-proxy
|
||||||
|
containers:
|
||||||
|
- name: oauth2-proxy
|
||||||
|
image: quay.io/oauth2-proxy/oauth2-proxy:v7.5.1
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
containerPort: 4180
|
||||||
|
protocol: TCP
|
||||||
|
env:
|
||||||
|
- name: OAUTH2_PROXY_PROVIDER
|
||||||
|
value: "oidc"
|
||||||
|
- name: OAUTH2_PROXY_OIDC_ISSUER_URL
|
||||||
|
value: "https://authentik.riotpiao.homelab.com/application/o/temporal/"
|
||||||
|
- name: OAUTH2_PROXY_CLIENT_ID
|
||||||
|
value: "temporal"
|
||||||
|
- name: OAUTH2_PROXY_CLIENT_SECRET
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: temporal-oidc
|
||||||
|
key: clientSecret
|
||||||
|
- name: OAUTH2_PROXY_COOKIE_SECRET
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: temporal-oidc
|
||||||
|
key: cookieSecret
|
||||||
|
- name: OAUTH2_PROXY_REDIRECT_URL
|
||||||
|
value: "https://temporal.riotpiao.homelab.com/oauth2/callback"
|
||||||
|
- name: OAUTH2_PROXY_UPSTREAM
|
||||||
|
value: "http://temporal-web:8080"
|
||||||
|
- name: OAUTH2_PROXY_COOKIE_SECURE
|
||||||
|
value: "true"
|
||||||
|
- name: OAUTH2_PROXY_COOKIE_HTTPONLY
|
||||||
|
value: "true"
|
||||||
|
- name: OAUTH2_PROXY_COOKIE_SAMESITE
|
||||||
|
value: "Lax"
|
||||||
|
- name: OAUTH2_PROXY_EMAIL_DOMAIN
|
||||||
|
value: "*"
|
||||||
|
- name: OAUTH2_PROXY_SKIP_AUTH_REGEX
|
||||||
|
value: "^/health"
|
||||||
|
- name: OAUTH2_PROXY_PASS_AUTHORIZATION_HEADER
|
||||||
|
value: "true"
|
||||||
|
- name: OAUTH2_PROXY_SKIP_AUTH_PREFLIGHT
|
||||||
|
value: "true"
|
||||||
|
- name: OAUTH2_PROXY_REVERSE_PROXY
|
||||||
|
value: "true"
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 128Mi
|
||||||
|
limits:
|
||||||
|
cpu: 200m
|
||||||
|
memory: 256Mi
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /ping
|
||||||
|
port: http
|
||||||
|
initialDelaySeconds: 10
|
||||||
|
periodSeconds: 10
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /ping
|
||||||
|
port: http
|
||||||
|
initialDelaySeconds: 5
|
||||||
|
periodSeconds: 5
|
||||||
|
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: oauth2-proxy
|
||||||
|
namespace: temporal
|
||||||
|
spec:
|
||||||
|
type: ClusterIP
|
||||||
|
ports:
|
||||||
|
- port: 4180
|
||||||
|
targetPort: http
|
||||||
|
protocol: TCP
|
||||||
|
name: http
|
||||||
|
selector:
|
||||||
|
app: oauth2-proxy
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
# Example Queues for "production" Temporal namespace
|
||||||
|
# When applied, queue-operator creates 1 TemporalWorker (worker-production)
|
||||||
|
# that processes ALL queues in the "production" namespace
|
||||||
|
---
|
||||||
|
apiVersion: kmsvc.io/v1
|
||||||
|
kind: Queue
|
||||||
|
metadata:
|
||||||
|
name: story-crater-tasks
|
||||||
|
namespace: sqs
|
||||||
|
labels:
|
||||||
|
temporal.io/namespace: "production"
|
||||||
|
spec:
|
||||||
|
fifoQueue: false
|
||||||
|
visibilityTimeoutSeconds: 30
|
||||||
|
messageRetentionPeriodSeconds: 345600
|
||||||
|
maxReceiveCount: 5
|
||||||
|
partitionsPerShard: 6
|
||||||
|
minShards: 1
|
||||||
|
maxShards: 8
|
||||||
|
shardSplitThresholdBytesPerSec: 5242880
|
||||||
|
|
||||||
|
---
|
||||||
|
apiVersion: kmsvc.io/v1
|
||||||
|
kind: Queue
|
||||||
|
metadata:
|
||||||
|
name: story-crater-notifications
|
||||||
|
namespace: sqs
|
||||||
|
labels:
|
||||||
|
temporal.io/namespace: "production"
|
||||||
|
spec:
|
||||||
|
fifoQueue: false
|
||||||
|
visibilityTimeoutSeconds: 60
|
||||||
|
messageRetentionPeriodSeconds: 345600
|
||||||
|
maxReceiveCount: 3
|
||||||
|
partitionsPerShard: 3
|
||||||
|
minShards: 1
|
||||||
|
maxShards: 4
|
||||||
|
shardSplitThresholdBytesPerSec: 2621440
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
# Ingress for Temporal UI — routes to OAuth2-Proxy, which proxies to temporal-web
|
||||||
|
# TLS terminated here; oauth2-proxy handles OIDC auth before forwarding to backend
|
||||||
|
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: Ingress
|
||||||
|
metadata:
|
||||||
|
name: temporal
|
||||||
|
namespace: temporal
|
||||||
|
annotations:
|
||||||
|
cert-manager.io/cluster-issuer: "letsencrypt-prod"
|
||||||
|
spec:
|
||||||
|
ingressClassName: nginx
|
||||||
|
tls:
|
||||||
|
- secretName: temporal-tls
|
||||||
|
hosts:
|
||||||
|
- temporal.riotpiao.homelab.com
|
||||||
|
rules:
|
||||||
|
- host: temporal.riotpiao.homelab.com
|
||||||
|
http:
|
||||||
|
paths:
|
||||||
|
- path: /
|
||||||
|
pathType: Prefix
|
||||||
|
backend:
|
||||||
|
service:
|
||||||
|
name: oauth2-proxy
|
||||||
|
port:
|
||||||
|
number: 4180
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
# k8s/temporal/temporal-values.yaml
|
||||||
|
# Temporal — workflow engine for story-crater backend async task orchestration.
|
||||||
|
# Chart: temporal/temporal from https://go.temporal.io/helm-charts
|
||||||
|
#
|
||||||
|
# Uses Cassandra for default store (workflow history/events)
|
||||||
|
# Uses Elasticsearch for visibility store (namespace/workflow queries)
|
||||||
|
# This is the chart's native, well-tested configuration.
|
||||||
|
|
||||||
|
# ── Datastores configuration ────
|
||||||
|
# Disable auto-deployed PostgreSQL (we use external ddb for other services)
|
||||||
|
postgresql:
|
||||||
|
enabled: false
|
||||||
|
|
||||||
|
# Enable Elasticsearch for visibility store (deployed to worker node, 2Gi/4Gi memory)
|
||||||
|
elasticsearch:
|
||||||
|
enabled: true
|
||||||
|
scheme: http
|
||||||
|
host: temporal-elasticsearch
|
||||||
|
port: 9200
|
||||||
|
version: v7
|
||||||
|
logLevel: error
|
||||||
|
auth:
|
||||||
|
enabled: false
|
||||||
|
indices:
|
||||||
|
visibility: temporal_visibility_v1
|
||||||
|
|
||||||
|
# Cassandra enabled for template validation; server.config overrides with actual hosts
|
||||||
|
# Schema job template requires cassandra config to exist at top level
|
||||||
|
cassandra:
|
||||||
|
enabled: true
|
||||||
|
replicas: 3
|
||||||
|
cluster:
|
||||||
|
seedSize: 1
|
||||||
|
port: 9042
|
||||||
|
|
||||||
|
# ── Disable schema auto-setup (will initialize manually) ─────────
|
||||||
|
jobs:
|
||||||
|
autoSetup:
|
||||||
|
enabled: false
|
||||||
|
|
||||||
|
# ── Temporal server config (Cassandra + Elasticsearch persistence) ──────────────────────────────
|
||||||
|
server:
|
||||||
|
replicaCount: 1
|
||||||
|
jobService:
|
||||||
|
enabled: false
|
||||||
|
config:
|
||||||
|
logLevel: "info"
|
||||||
|
persistence:
|
||||||
|
defaultStore: default
|
||||||
|
visibilityStore: visibility
|
||||||
|
numHistoryShards: 512
|
||||||
|
datastores:
|
||||||
|
default:
|
||||||
|
# Cassandra for workflow history and events
|
||||||
|
driver: cassandra
|
||||||
|
cassandra:
|
||||||
|
hosts: "temporal-cassandra"
|
||||||
|
port: 9042
|
||||||
|
keyspace: temporal
|
||||||
|
user: user
|
||||||
|
password: "" # Cassandra auth disabled in deployment
|
||||||
|
replicationFactor: 3
|
||||||
|
consistency:
|
||||||
|
default:
|
||||||
|
consistency: local_quorum
|
||||||
|
serialConsistency: local_serial
|
||||||
|
service:
|
||||||
|
type: ClusterIP
|
||||||
|
|
||||||
|
# ── Temporal Web UI ────────────────────────────────────────────────────────
|
||||||
|
web:
|
||||||
|
replicaCount: 1
|
||||||
|
service:
|
||||||
|
type: ClusterIP
|
||||||
|
|
||||||
|
# ── Ingress ────────────────────────────────────────────────────────
|
||||||
|
# Note: ingress is disabled here. Instead, we route via oauth2-proxy.
|
||||||
|
# The ingress is applied separately as k8s/temporal/temporal-ingress-oauth2.yaml
|
||||||
|
# which terminates TLS and routes to oauth2-proxy service.
|
||||||
|
ingress:
|
||||||
|
enabled: false
|
||||||
|
|
||||||
|
# ── Monitoring ────────────────────────────────────────────────────────
|
||||||
|
prometheus:
|
||||||
|
enabled: false
|
||||||
Reference in New Issue
Block a user