From 1c02e2b831479caaa79eccad0ac4040f0067c948 Mon Sep 17 00:00:00 2001 From: Story Crater Bot <19826264+Riotpiaole@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:17:42 -0700 Subject: [PATCH] k8s/messaging: add kafka kmsvc and temporal workflows - Kafka 3-broker cluster (RF=3, min-ISR=2) - kmsvc SQS-like API on Kafka - Redis dedup (standalone, can extend to HA) - Temporal workflow orchestration (Cassandra backend) --- k8s/sqs/argocd/apps/00-strimzi-operator.yaml | 25 ++ k8s/sqs/argocd/apps/01-kafka-cluster.yaml | 31 ++ k8s/sqs/argocd/apps/02-redis.yaml | 43 +++ k8s/sqs/argocd/apps/03-queue-crd.yaml | 30 ++ .../argocd/apps/04-management-service.yaml | 37 +++ k8s/sqs/argocd/project.yaml | 23 ++ k8s/sqs/argocd/root.yaml | 22 ++ k8s/sqs/charts/kafka-cluster/Chart.yaml | 5 + .../templates/kafka-cluster.yaml | 30 ++ .../templates/kafka-nodepool.yaml | 39 +++ .../kafka-cluster/templates/storageclass.yaml | 19 ++ k8s/sqs/charts/kafka-cluster/values.yaml | 26 ++ k8s/sqs/charts/management-service/Chart.yaml | 5 + .../templates/configmap.yaml | 12 + .../templates/deployment.yaml | 42 +++ .../management-service/templates/hpa.yaml | 27 ++ .../templates/ingress-grpc.yaml | 27 ++ .../management-service/templates/ingress.yaml | 26 ++ .../management-service/templates/service.yaml | 16 + k8s/sqs/charts/management-service/values.yaml | 50 ++++ k8s/sqs/charts/queue-crd/Chart.yaml | 5 + k8s/sqs/charts/queue-crd/templates/crd.yaml | 274 ++++++++++++++++++ .../templates/operator-deployment.yaml | 37 +++ k8s/sqs/charts/queue-crd/templates/rbac.yaml | 42 +++ k8s/sqs/charts/queue-crd/values.yaml | 26 ++ k8s/sqs/environments/homelab.yaml | 29 ++ k8s/sqs/helmfile.yaml.gotmpl | 99 +++++++ k8s/sqs/kmsvc-ingress-oauth2.yaml | 25 ++ k8s/sqs/oauth2-proxy.yaml | 106 +++++++ k8s/sqs/queues/orders-fifo.yaml | 32 ++ k8s/temporal/PHASE_2_TESTING.md | 237 +++++++++++++++ k8s/temporal/TEMPORAL_OAUTH2_SETUP.md | 220 ++++++++++++++ k8s/temporal/elasticsearch.yaml | 102 +++++++ k8s/temporal/oauth2-proxy-values.yaml | 45 +++ k8s/temporal/oauth2-proxy.yaml | 108 +++++++ k8s/temporal/queues/example-queue.yaml | 38 +++ k8s/temporal/temporal-ingress-oauth2.yaml | 27 ++ k8s/temporal/temporal-values.yaml | 85 ++++++ 38 files changed, 2072 insertions(+) create mode 100644 k8s/sqs/argocd/apps/00-strimzi-operator.yaml create mode 100644 k8s/sqs/argocd/apps/01-kafka-cluster.yaml create mode 100644 k8s/sqs/argocd/apps/02-redis.yaml create mode 100644 k8s/sqs/argocd/apps/03-queue-crd.yaml create mode 100644 k8s/sqs/argocd/apps/04-management-service.yaml create mode 100644 k8s/sqs/argocd/project.yaml create mode 100644 k8s/sqs/argocd/root.yaml create mode 100644 k8s/sqs/charts/kafka-cluster/Chart.yaml create mode 100644 k8s/sqs/charts/kafka-cluster/templates/kafka-cluster.yaml create mode 100644 k8s/sqs/charts/kafka-cluster/templates/kafka-nodepool.yaml create mode 100644 k8s/sqs/charts/kafka-cluster/templates/storageclass.yaml create mode 100644 k8s/sqs/charts/kafka-cluster/values.yaml create mode 100644 k8s/sqs/charts/management-service/Chart.yaml create mode 100644 k8s/sqs/charts/management-service/templates/configmap.yaml create mode 100644 k8s/sqs/charts/management-service/templates/deployment.yaml create mode 100644 k8s/sqs/charts/management-service/templates/hpa.yaml create mode 100644 k8s/sqs/charts/management-service/templates/ingress-grpc.yaml create mode 100644 k8s/sqs/charts/management-service/templates/ingress.yaml create mode 100644 k8s/sqs/charts/management-service/templates/service.yaml create mode 100644 k8s/sqs/charts/management-service/values.yaml create mode 100644 k8s/sqs/charts/queue-crd/Chart.yaml create mode 100644 k8s/sqs/charts/queue-crd/templates/crd.yaml create mode 100644 k8s/sqs/charts/queue-crd/templates/operator-deployment.yaml create mode 100644 k8s/sqs/charts/queue-crd/templates/rbac.yaml create mode 100644 k8s/sqs/charts/queue-crd/values.yaml create mode 100644 k8s/sqs/environments/homelab.yaml create mode 100644 k8s/sqs/helmfile.yaml.gotmpl create mode 100644 k8s/sqs/kmsvc-ingress-oauth2.yaml create mode 100644 k8s/sqs/oauth2-proxy.yaml create mode 100644 k8s/sqs/queues/orders-fifo.yaml create mode 100644 k8s/temporal/PHASE_2_TESTING.md create mode 100644 k8s/temporal/TEMPORAL_OAUTH2_SETUP.md create mode 100644 k8s/temporal/elasticsearch.yaml create mode 100644 k8s/temporal/oauth2-proxy-values.yaml create mode 100644 k8s/temporal/oauth2-proxy.yaml create mode 100644 k8s/temporal/queues/example-queue.yaml create mode 100644 k8s/temporal/temporal-ingress-oauth2.yaml create mode 100644 k8s/temporal/temporal-values.yaml diff --git a/k8s/sqs/argocd/apps/00-strimzi-operator.yaml b/k8s/sqs/argocd/apps/00-strimzi-operator.yaml new file mode 100644 index 0000000..48d92d5 --- /dev/null +++ b/k8s/sqs/argocd/apps/00-strimzi-operator.yaml @@ -0,0 +1,25 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: strimzi-operator + namespace: cicd + annotations: + argocd.argoproj.io/sync-wave: "0" +spec: + project: kmsvc + source: + repoURL: https://strimzi.io/charts/ + chart: strimzi-kafka-operator + targetRevision: 0.46.0 + helm: + values: | + watchNamespaces: ["sqs"] + destination: + server: https://kubernetes.default.svc + namespace: sqs + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true diff --git a/k8s/sqs/argocd/apps/01-kafka-cluster.yaml b/k8s/sqs/argocd/apps/01-kafka-cluster.yaml new file mode 100644 index 0000000..a27d542 --- /dev/null +++ b/k8s/sqs/argocd/apps/01-kafka-cluster.yaml @@ -0,0 +1,31 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: kafka-cluster + namespace: cicd + annotations: + argocd.argoproj.io/sync-wave: "1" +spec: + project: kmsvc + source: + repoURL: https://forgejo.riotpiao.homelab.com/rock/kafaka-management-service.git + targetRevision: main + path: k8s/charts/kafka-cluster + helm: + values: | + namespace: sqs + nodePool: + replicas: 3 + storage: + class: longhorn + sizeGi: 50 + resources: + memory: 5Gi + cpu: "2" + destination: + server: https://kubernetes.default.svc + namespace: sqs + syncPolicy: + automated: + prune: true + selfHeal: true diff --git a/k8s/sqs/argocd/apps/02-redis.yaml b/k8s/sqs/argocd/apps/02-redis.yaml new file mode 100644 index 0000000..e1076a3 --- /dev/null +++ b/k8s/sqs/argocd/apps/02-redis.yaml @@ -0,0 +1,43 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: kmsvc-redis + namespace: cicd + annotations: + argocd.argoproj.io/sync-wave: "1" +spec: + project: kmsvc + source: + repoURL: https://charts.bitnami.com/bitnami + chart: redis + targetRevision: 20.6.0 + helm: + values: | + architecture: standalone + # docker.io/bitnami stopped publishing version-pinned tags; bitnamilegacy + # mirrors them for free. allowInsecureImages silences the chart's + # container-image allowlist check, which doesn't know about that mirror. + global: + security: + allowInsecureImages: true + image: + repository: bitnamilegacy/redis + auth: + enabled: false + master: + persistence: + enabled: true + storageClass: longhorn + size: 2Gi + resources: + limits: + memory: 1Gi + requests: + memory: 1Gi + destination: + server: https://kubernetes.default.svc + namespace: sqs + syncPolicy: + automated: + prune: true + selfHeal: true diff --git a/k8s/sqs/argocd/apps/03-queue-crd.yaml b/k8s/sqs/argocd/apps/03-queue-crd.yaml new file mode 100644 index 0000000..495d309 --- /dev/null +++ b/k8s/sqs/argocd/apps/03-queue-crd.yaml @@ -0,0 +1,30 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: queue-crd + namespace: cicd + annotations: + argocd.argoproj.io/sync-wave: "2" +spec: + project: kmsvc + source: + repoURL: https://forgejo.riotpiao.homelab.com/rock/kafaka-management-service.git + targetRevision: main + path: k8s/charts/queue-crd + helm: + values: | + namespace: sqs + kafkaBrokers: "kmsvc-kafka-bootstrap.sqs.svc.cluster.local:9092" + redisAddr: "kmsvc-redis-master.sqs.svc.cluster.local:6379" + image: + repository: forgejo.riotpiao.homelab.com/rock/kafka-management-service-queue-operator + # CI (.forgejo/workflows/release.yaml) writes the released git tag + # here and pushes the commit -- ArgoCD picks it up on its next sync. + tag: latest + destination: + server: https://kubernetes.default.svc + namespace: sqs + syncPolicy: + automated: + prune: true + selfHeal: true diff --git a/k8s/sqs/argocd/apps/04-management-service.yaml b/k8s/sqs/argocd/apps/04-management-service.yaml new file mode 100644 index 0000000..459c8ee --- /dev/null +++ b/k8s/sqs/argocd/apps/04-management-service.yaml @@ -0,0 +1,37 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: management-service + namespace: cicd + annotations: + argocd.argoproj.io/sync-wave: "2" +spec: + project: kmsvc + source: + repoURL: https://forgejo.riotpiao.homelab.com/rock/kafaka-management-service.git + targetRevision: main + path: k8s/charts/management-service + helm: + values: | + namespace: sqs + image: + repository: forgejo.riotpiao.homelab.com/rock/kafka-management-service + # CI (.forgejo/workflows/release.yaml) writes the released git tag + # here and pushes the commit -- ArgoCD picks it up on its next sync. + tag: latest + env: + kafkaBrokers: "kmsvc-kafka-bootstrap.sqs.svc.cluster.local:9092" + redisAddr: "kmsvc-redis-master.sqs.svc.cluster.local:6379" + authentikIssuerURL: "https://authentik.riotpiao.homelab.com/application/o/kafaka/" + authentikAudience: "QI0gPtR99ar8VvhK8Tqox4SDkTKzbNU7lbgwBNSc" + ingress: + enabled: true + host: kmsvc.riotpiao.homelab.com + clusterIssuer: homelab-ca + destination: + server: https://kubernetes.default.svc + namespace: sqs + syncPolicy: + automated: + prune: true + selfHeal: true diff --git a/k8s/sqs/argocd/project.yaml b/k8s/sqs/argocd/project.yaml new file mode 100644 index 0000000..062b2d6 --- /dev/null +++ b/k8s/sqs/argocd/project.yaml @@ -0,0 +1,23 @@ +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: kmsvc + namespace: cicd +spec: + description: Kafka Management Service (design.md) -- Strimzi/Kafka, Redis, queue-operator, message-plane server + sourceRepos: + - https://forgejo.riotpiao.homelab.com/rock/kafaka-management-service.git + - https://strimzi.io/charts/ + - https://charts.bitnami.com/bitnami + destinations: + - namespace: sqs + server: https://kubernetes.default.svc + - namespace: cicd + server: https://kubernetes.default.svc + clusterResourceWhitelist: + - group: "apiextensions.k8s.io" + kind: CustomResourceDefinition + - group: "rbac.authorization.k8s.io" + kind: ClusterRole + - group: "rbac.authorization.k8s.io" + kind: ClusterRoleBinding diff --git a/k8s/sqs/argocd/root.yaml b/k8s/sqs/argocd/root.yaml new file mode 100644 index 0000000..240b1b0 --- /dev/null +++ b/k8s/sqs/argocd/root.yaml @@ -0,0 +1,22 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: kmsvc-root + namespace: cicd +spec: + project: kmsvc + source: + repoURL: https://forgejo.riotpiao.homelab.com/rock/kafaka-management-service.git + targetRevision: main + path: k8s/argocd/apps + directory: + recurse: false + destination: + server: https://kubernetes.default.svc + namespace: cicd + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true diff --git a/k8s/sqs/charts/kafka-cluster/Chart.yaml b/k8s/sqs/charts/kafka-cluster/Chart.yaml new file mode 100644 index 0000000..a3670d0 --- /dev/null +++ b/k8s/sqs/charts/kafka-cluster/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: kafka-cluster +description: Strimzi Kafka/KafkaNodePool CRs for the kmsvc Kafka cluster (design.md §7) +type: application +version: 0.1.0 diff --git a/k8s/sqs/charts/kafka-cluster/templates/kafka-cluster.yaml b/k8s/sqs/charts/kafka-cluster/templates/kafka-cluster.yaml new file mode 100644 index 0000000..0076fa9 --- /dev/null +++ b/k8s/sqs/charts/kafka-cluster/templates/kafka-cluster.yaml @@ -0,0 +1,30 @@ +apiVersion: kafka.strimzi.io/v1beta2 +kind: Kafka +metadata: + name: {{ .Values.clusterName }} + namespace: {{ .Values.namespace }} + annotations: + strimzi.io/node-pools: enabled + strimzi.io/kraft: enabled +spec: + kafka: + version: 4.0.0 + metadataVersion: 4.0-IV3 + listeners: + - name: plain + port: 9092 + type: internal + tls: false + - name: tls + port: 9093 + type: internal + tls: true + config: + default.replication.factor: {{ .Values.kafka.replicationFactor }} + min.insync.replicas: {{ .Values.kafka.minInsyncReplicas }} + offsets.topic.replication.factor: {{ .Values.kafka.replicationFactor }} + transaction.state.log.replication.factor: {{ .Values.kafka.replicationFactor }} + transaction.state.log.min.isr: {{ .Values.kafka.minInsyncReplicas }} + entityOperator: + topicOperator: {} + userOperator: {} diff --git a/k8s/sqs/charts/kafka-cluster/templates/kafka-nodepool.yaml b/k8s/sqs/charts/kafka-cluster/templates/kafka-nodepool.yaml new file mode 100644 index 0000000..3fbc23c --- /dev/null +++ b/k8s/sqs/charts/kafka-cluster/templates/kafka-nodepool.yaml @@ -0,0 +1,39 @@ +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaNodePool +metadata: + name: {{ .Values.clusterName }}-pool + namespace: {{ .Values.namespace }} + labels: + strimzi.io/cluster: {{ .Values.clusterName }} +spec: + replicas: {{ .Values.nodePool.replicas }} + roles: + - controller + - broker + storage: + type: persistent-claim + size: {{ .Values.nodePool.storage.sizeGi }}Gi + class: {{ .Values.nodePool.storage.class }} + deleteClaim: false + resources: + limits: + memory: {{ .Values.nodePool.resources.memory }} + cpu: {{ .Values.nodePool.resources.cpu | quote }} + requests: + memory: {{ .Values.nodePool.resources.memory }} + cpu: {{ .Values.nodePool.resources.cpu | quote }} + template: + pod: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: {{ .Values.nodePool.antiAffinityTopologyKey }} + labelSelector: + matchLabels: + strimzi.io/cluster: {{ .Values.clusterName }} + kafkaContainer: + env: + - name: KAFKA_HEAP_OPTS + value: {{ .Values.nodePool.heapOpts | quote }} diff --git a/k8s/sqs/charts/kafka-cluster/templates/storageclass.yaml b/k8s/sqs/charts/kafka-cluster/templates/storageclass.yaml new file mode 100644 index 0000000..a821726 --- /dev/null +++ b/k8s/sqs/charts/kafka-cluster/templates/storageclass.yaml @@ -0,0 +1,19 @@ +{{- if eq .Values.nodePool.storage.class "longhorn-kafka" }} +# The default "longhorn" StorageClass requests 3 replicas across 3 zone-labeled +# nodes (az-a/az-b/az-c). With Longhorn's zone-aware anti-affinity, replicas +# spread 1-per-zone for durability. +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: longhorn-kafka +provisioner: driver.longhorn.io +allowVolumeExpansion: true +reclaimPolicy: Delete +volumeBindingMode: Immediate +parameters: + numberOfReplicas: "3" + staleReplicaTimeout: "30" + fromBackup: "" + fsType: "ext4" + dataLocality: "disabled" +{{- end }} diff --git a/k8s/sqs/charts/kafka-cluster/values.yaml b/k8s/sqs/charts/kafka-cluster/values.yaml new file mode 100644 index 0000000..187c591 --- /dev/null +++ b/k8s/sqs/charts/kafka-cluster/values.yaml @@ -0,0 +1,26 @@ +clusterName: kmsvc +namespace: sqs + +nodePool: + replicas: 3 + storage: + class: longhorn-kafka + # Longhorn's per-node scheduling budget on the current 2-node cluster has + # only ~36Gi of headroom left (other PVCs already reserve the rest), and + # each node hosts one replica of all 3 broker volumes -- so 3 * sizeGi + # must fit in that headroom. Revisit once the 3rd node joins. + sizeGi: 10 + resources: + memory: 5Gi + cpu: "2" + heapOpts: "-Xms2g -Xmx2g" + # design.md §7: 3 real zones now exist (talos-cp-1=az-a, talos-worker-1=az-b, + # talos-worker-2=az-c), so anti-affinity keys off zone instead of hostname — + # spreads the 3 broker pods one-per-zone/one-per-node (equivalent today, + # but zone is the correct long-term key if a node ever gets replaced within + # the same zone). + antiAffinityTopologyKey: topology.kubernetes.io/zone + +kafka: + replicationFactor: 3 + minInsyncReplicas: 2 diff --git a/k8s/sqs/charts/management-service/Chart.yaml b/k8s/sqs/charts/management-service/Chart.yaml new file mode 100644 index 0000000..cf785e9 --- /dev/null +++ b/k8s/sqs/charts/management-service/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: management-service +description: kmsvc message-plane gRPC+REST server (design.md §1, §7a, §9) +type: application +version: 0.1.0 diff --git a/k8s/sqs/charts/management-service/templates/configmap.yaml b/k8s/sqs/charts/management-service/templates/configmap.yaml new file mode 100644 index 0000000..28770b6 --- /dev/null +++ b/k8s/sqs/charts/management-service/templates/configmap.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: management-service-config + namespace: {{ .Values.namespace }} +data: + KMSVC_KAFKA_BROKERS: {{ .Values.env.kafkaBrokers | quote }} + KMSVC_REDIS_ADDR: {{ .Values.env.redisAddr | quote }} + KMSVC_AUTHENTIK_ISSUER_URL: {{ .Values.env.authentikIssuerURL | quote }} + KMSVC_AUTHENTIK_AUDIENCE: {{ .Values.env.authentikAudience | quote }} + KMSVC_GRPC_LISTEN_ADDR: ":{{ .Values.grpcPort }}" + KMSVC_HTTP_LISTEN_ADDR: ":{{ .Values.httpPort }}" diff --git a/k8s/sqs/charts/management-service/templates/deployment.yaml b/k8s/sqs/charts/management-service/templates/deployment.yaml new file mode 100644 index 0000000..a5af206 --- /dev/null +++ b/k8s/sqs/charts/management-service/templates/deployment.yaml @@ -0,0 +1,42 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: management-service + namespace: {{ .Values.namespace }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: management-service + template: + metadata: + labels: + app: management-service + spec: + containers: + - name: management-service + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: grpc + containerPort: {{ .Values.grpcPort }} + - name: http + containerPort: {{ .Values.httpPort }} + env: + - name: GOMEMLIMIT + value: {{ .Values.goMemLimit | quote }} + envFrom: + - configMapRef: + name: management-service-config + resources: + {{- toYaml .Values.resources | nindent 12 }} + readinessProbe: + tcpSocket: + port: {{ .Values.httpPort }} + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + tcpSocket: + port: {{ .Values.httpPort }} + initialDelaySeconds: 10 + periodSeconds: 20 diff --git a/k8s/sqs/charts/management-service/templates/hpa.yaml b/k8s/sqs/charts/management-service/templates/hpa.yaml new file mode 100644 index 0000000..e6a4627 --- /dev/null +++ b/k8s/sqs/charts/management-service/templates/hpa.yaml @@ -0,0 +1,27 @@ +{{- if .Values.hpa.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: management-service + namespace: {{ .Values.namespace }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: management-service + minReplicas: {{ .Values.hpa.minReplicas }} + maxReplicas: {{ .Values.hpa.maxReplicas }} + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.hpa.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.hpa.targetMemoryUtilizationPercentage }} +{{- end }} diff --git a/k8s/sqs/charts/management-service/templates/ingress-grpc.yaml b/k8s/sqs/charts/management-service/templates/ingress-grpc.yaml new file mode 100644 index 0000000..8dcdcc6 --- /dev/null +++ b/k8s/sqs/charts/management-service/templates/ingress-grpc.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.ingress.enabled .Values.ingress.grpcEnabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: management-service-grpc + namespace: {{ .Values.namespace }} + annotations: + cert-manager.io/cluster-issuer: {{ .Values.ingress.clusterIssuer }} + nginx.ingress.kubernetes.io/backend-protocol: "GRPC" +spec: + ingressClassName: {{ .Values.ingress.className }} + tls: + - hosts: + - {{ .Values.ingress.host }} + secretName: {{ .Values.ingress.tlsSecretName }} + rules: + - host: {{ .Values.ingress.host }} + http: + paths: + - path: {{ .Values.ingress.grpcPathPrefix }} + pathType: Prefix + backend: + service: + name: management-service + port: + number: {{ .Values.grpcPort }} +{{- end }} diff --git a/k8s/sqs/charts/management-service/templates/ingress.yaml b/k8s/sqs/charts/management-service/templates/ingress.yaml new file mode 100644 index 0000000..18790e2 --- /dev/null +++ b/k8s/sqs/charts/management-service/templates/ingress.yaml @@ -0,0 +1,26 @@ +{{- if .Values.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: management-service + namespace: {{ .Values.namespace }} + annotations: + cert-manager.io/cluster-issuer: {{ .Values.ingress.clusterIssuer }} +spec: + ingressClassName: {{ .Values.ingress.className }} + tls: + - hosts: + - {{ .Values.ingress.host }} + secretName: {{ .Values.ingress.tlsSecretName }} + rules: + - host: {{ .Values.ingress.host }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: management-service + port: + number: {{ .Values.httpPort }} +{{- end }} diff --git a/k8s/sqs/charts/management-service/templates/service.yaml b/k8s/sqs/charts/management-service/templates/service.yaml new file mode 100644 index 0000000..7bfa7c0 --- /dev/null +++ b/k8s/sqs/charts/management-service/templates/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: management-service + namespace: {{ .Values.namespace }} +spec: + selector: + app: management-service + ports: + - name: grpc + port: {{ .Values.grpcPort }} + targetPort: {{ .Values.grpcPort }} + - name: http + port: {{ .Values.httpPort }} + targetPort: {{ .Values.httpPort }} + type: ClusterIP diff --git a/k8s/sqs/charts/management-service/values.yaml b/k8s/sqs/charts/management-service/values.yaml new file mode 100644 index 0000000..4399d7f --- /dev/null +++ b/k8s/sqs/charts/management-service/values.yaml @@ -0,0 +1,50 @@ +namespace: sqs +replicaCount: 3 + +image: + repository: forgejo.riotpiao.homelab.com/rock/kafka-management-service + tag: latest + pullPolicy: Always + +grpcPort: 9090 +httpPort: 8080 + +env: + kafkaBrokers: "kmsvc-kafka-bootstrap.sqs.svc.cluster.local:9092" + redisAddr: "kmsvc-redis-master.sqs.svc.cluster.local:6379" + authentikIssuerURL: "" + authentikAudience: "" + +resources: + requests: + cpu: 200m + memory: 256Mi + limits: + cpu: "1" + memory: 512Mi + +# Go's GC only reacts to GOGC by default and has no idea about the cgroup +# memory limit above -- it'll happily grow heap until the kernel OOMKills it. +# Setting GOMEMLIMIT to ~90% of the container limit makes the GC self-throttle +# before that happens. Keep this in sync with resources.limits.memory. +goMemLimit: "460MiB" + +hpa: + enabled: true + minReplicas: 3 + maxReplicas: 9 + targetCPUUtilizationPercentage: 70 + targetMemoryUtilizationPercentage: 80 + +ingress: + enabled: true + className: nginx + clusterIssuer: homelab-ca + host: kmsvc.riotpiao.homelab.com + tlsSecretName: kmsvc-tls + # kmsvc-cli connects via gRPC directly to --server/KMSVC_SERVER (default + # kmsvc.riotpiao.homelab.com:443, see kmsvc-cli's README), so raw gRPC needs an + # external path too — scoped to the gRPC service's own path prefix on the + # same host/port, rather than opening the whole host to gRPC passthrough. + grpcEnabled: true + grpcPathPrefix: /kafkamgmt.v1.QueueService/ diff --git a/k8s/sqs/charts/queue-crd/Chart.yaml b/k8s/sqs/charts/queue-crd/Chart.yaml new file mode 100644 index 0000000..50763c6 --- /dev/null +++ b/k8s/sqs/charts/queue-crd/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: queue-crd +description: Queue CRD definition + queue-operator Deployment/RBAC (design.md §2a) +type: application +version: 0.1.0 diff --git a/k8s/sqs/charts/queue-crd/templates/crd.yaml b/k8s/sqs/charts/queue-crd/templates/crd.yaml new file mode 100644 index 0000000..df684c8 --- /dev/null +++ b/k8s/sqs/charts/queue-crd/templates/crd.yaml @@ -0,0 +1,274 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.21.0 + name: queues.kmsvc.io +spec: + group: kmsvc.io + names: + kind: Queue + listKind: QueueList + plural: queues + shortNames: + - queue + - queues + singular: queue + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.fifoQueue + name: FIFO + type: boolean + - jsonPath: .status.phase + name: Phase + type: string + name: v1 + schema: + openAPIV3Schema: + description: Queue is the Schema for the queues API — see design.md §2a. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: QueueSpec defines the desired state of a Queue (design.md + §2a). + properties: + deadLetterTargetQueue: + description: |- + DeadLetterTargetQueue is the name of another Queue to route exhausted + messages to. Must not point at itself or at another DLQ (design.md §5). + type: string + delaySeconds: + description: DelaySeconds is the default delivery delay applied to + sent messages. + format: int32 + maximum: 900 + minimum: 0 + type: integer + fifoQueue: + default: false + description: FIFOQueue enables per-MessageGroupId ordering and deduplication + semantics. + type: boolean + isDLQ: + description: |- + IsDLQ marks this queue as itself a dead-letter queue, used to enforce + the no-DLQ-chaining validation rule in design.md §5. + type: boolean + maxReceiveCount: + default: 5 + description: |- + MaxReceiveCount is how many times a message may be redelivered before + being routed to DeadLetterTargetQueue. + format: int32 + minimum: 1 + type: integer + maxShards: + default: 8 + description: MaxShards is the ceiling on shard count the operator + may split up to (design.md §2c). + format: int32 + minimum: 1 + type: integer + messageRetentionPeriodSeconds: + default: 345600 + description: MessageRetentionPeriodSeconds maps to the underlying + Kafka topic's retention.ms. + format: int32 + maximum: 1209600 + minimum: 60 + type: integer + minShards: + default: 1 + description: MinShards is the floor on shard count; the operator never + merges below this. + format: int32 + minimum: 1 + type: integer + partitionsPerShard: + default: 6 + description: PartitionsPerShard is the Kafka partition count on each + shard's topic. + format: int32 + minimum: 1 + type: integer + shardSplitCooldownSeconds: + default: 300 + description: |- + ShardSplitCooldownSeconds is the minimum age a shard must reach before it + is eligible to be split again, preventing rapid re-splitting of a child + that hasn't yet absorbed its share of traffic. + format: int32 + minimum: 0 + type: integer + shardSplitThresholdBytesPerSec: + default: 5242880 + description: |- + ShardSplitThresholdBytesPerSec is the sustained per-shard throughput that + triggers a split into two child shards (design.md §2c). + format: int64 + minimum: 1 + type: integer + visibilityTimeoutSeconds: + default: 30 + description: |- + VisibilityTimeoutSeconds is how long a received-but-unacked message stays + invisible to other consumers before being redelivered. + format: int32 + maximum: 43200 + minimum: 0 + type: integer + type: object + status: + description: QueueStatus defines the observed state of a Queue. + properties: + conditions: + description: Conditions hold detailed status information. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + phase: + description: Phase is the current reconciliation phase. + enum: + - Pending + - Ready + - Failed + type: string + shards: + description: |- + Shards lists every shard backing this queue, active or draining + (design.md §2a/§2c). + items: + description: ShardStatus describes one shard backing a Queue (design.md + §2a/§2c). + properties: + availabilityZones: + description: |- + AvailabilityZones lists the topology.kubernetes.io/zone values of every + node currently hosting a Kafka replica of this shard's topic, resolved + from the broker pods' node placement each reconcile. Empty until the + first successful resolution (e.g. node lookup failed transiently). + items: + type: string + type: array + createdAt: + description: |- + CreatedAt timestamps when this shard was created, used to enforce + ShardSplitCooldownSeconds. + format: date-time + type: string + hashRangeEnd: + format: int64 + type: integer + hashRangeStart: + description: |- + HashRangeStart/HashRangeEnd define the [start, end) murmur2 hash range + this shard owns over the 32-bit key space. Stored as int64 (not uint32) + because controller-gen maps Go uint32 to OpenAPI format:int32, whose max + (2147483647) is smaller than FullHashRangeEnd (0xFFFFFFFF) and the + apiserver rejects the status update. + format: int64 + type: integer + id: + description: ID is the shard's identifier, used in its topic + name (kmsvc.{queue}.shard-{id}). + type: string + parentId: + description: |- + ParentID is the shard ID this shard was split from, empty for the + original shard-0. + type: string + phase: + description: Phase is this shard's lifecycle state. + enum: + - Active + - Closing + - Closed + type: string + topic: + description: Topic is the underlying Kafka topic name for this + shard. + type: string + required: + - hashRangeEnd + - hashRangeStart + - id + - phase + - topic + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/k8s/sqs/charts/queue-crd/templates/operator-deployment.yaml b/k8s/sqs/charts/queue-crd/templates/operator-deployment.yaml new file mode 100644 index 0000000..22c9a57 --- /dev/null +++ b/k8s/sqs/charts/queue-crd/templates/operator-deployment.yaml @@ -0,0 +1,37 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: queue-operator + namespace: {{ .Values.namespace }} +spec: + replicas: 1 + selector: + matchLabels: + app: queue-operator + template: + metadata: + labels: + app: queue-operator + spec: + serviceAccountName: queue-operator + containers: + - name: queue-operator + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: KMSVC_KAFKA_BROKERS + value: {{ .Values.kafkaBrokers | quote }} + - name: KMSVC_REDIS_ADDR + value: {{ .Values.redisAddr | quote }} + - name: GOMEMLIMIT + value: {{ .Values.goMemLimit | quote }} + - name: KMSVC_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: KMSVC_KAFKA_CLUSTER_NAME + value: {{ .Values.kafkaClusterName | quote }} + - name: KMSVC_KAFKA_POOL_NAME + value: {{ .Values.kafkaPoolName | quote }} + resources: + {{- toYaml .Values.resources | nindent 12 }} diff --git a/k8s/sqs/charts/queue-crd/templates/rbac.yaml b/k8s/sqs/charts/queue-crd/templates/rbac.yaml new file mode 100644 index 0000000..39a46d7 --- /dev/null +++ b/k8s/sqs/charts/queue-crd/templates/rbac.yaml @@ -0,0 +1,42 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: queue-operator + namespace: {{ .Values.namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: queue-operator +rules: + - apiGroups: ["kmsvc.io"] + resources: ["queues"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["kmsvc.io"] + resources: ["queues/status"] + verbs: ["get", "update", "patch"] + - apiGroups: ["kmsvc.io"] + resources: ["queues/finalizers"] + verbs: ["update"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] + - apiGroups: [""] + resources: ["pods", "nodes"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: queue-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: queue-operator +subjects: + - kind: ServiceAccount + name: queue-operator + namespace: {{ .Values.namespace }} diff --git a/k8s/sqs/charts/queue-crd/values.yaml b/k8s/sqs/charts/queue-crd/values.yaml new file mode 100644 index 0000000..6828ba8 --- /dev/null +++ b/k8s/sqs/charts/queue-crd/values.yaml @@ -0,0 +1,26 @@ +namespace: sqs + +image: + repository: forgejo.riotpiao.homelab.com/rock/kafka-management-service-queue-operator + tag: latest + pullPolicy: Always + +kafkaBrokers: "kmsvc-kafka-bootstrap.sqs.svc.cluster.local:9092" +redisAddr: "kmsvc-redis-master.sqs.svc.cluster.local:6379" + +# Must match kafka-cluster chart's clusterName/derived pool name -- used to +# resolve "--" broker pod names +# for AZ-aware Queue status (design.md §2a). +kafkaClusterName: kmsvc +kafkaPoolName: kmsvc-pool + +resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + +# See management-service/values.yaml's goMemLimit comment -- same reasoning. +goMemLimit: "230MiB" diff --git a/k8s/sqs/environments/homelab.yaml b/k8s/sqs/environments/homelab.yaml new file mode 100644 index 0000000..774f724 --- /dev/null +++ b/k8s/sqs/environments/homelab.yaml @@ -0,0 +1,29 @@ +# design.md §7b: cluster-specific values for the homelab environment. +# No secrets here — Authentik client secret etc. flow through the existing +# Vault/talos-cli pattern, referenced at deploy time, not inlined. +namespace: sqs + +kafkaCluster: + nodePool: + replicas: 3 + storage: + # longhorn-kafka now uses numberOfReplicas: 3 across 3 zone-labeled nodes. + class: longhorn-kafka + # Per-node headroom: with 3 nodes and existing PVCs, estimate ~100+ Gi total + # available. Each node hosts one replica of all 3 broker volumes, so 3 * + # sizeGi must fit. Monitor usage during Kafka deployment. + sizeGi: 10 + resources: + memory: 5Gi + cpu: "2" + +redis: + storageClass: longhorn + memoryLimit: 1Gi + +managementService: + ingress: + host: kmsvc.riotpiao.homelab.com + clusterIssuer: homelab-ca + authentikIssuerURL: "https://authentik.riotpiao.homelab.com/application/o/kafaka/" + authentikAudience: "QI0gPtR99ar8VvhK8Tqox4SDkTKzbNU7lbgwBNSc" diff --git a/k8s/sqs/helmfile.yaml.gotmpl b/k8s/sqs/helmfile.yaml.gotmpl new file mode 100644 index 0000000..27e4655 --- /dev/null +++ b/k8s/sqs/helmfile.yaml.gotmpl @@ -0,0 +1,99 @@ +environments: + default: + values: + - environments/homelab.yaml + homelab: + values: + - environments/homelab.yaml +--- +helmDefaults: + wait: true + timeout: 600 + +repositories: + - name: strimzi + url: https://strimzi.io/charts/ + - name: bitnami + url: https://charts.bitnami.com/bitnami + +releases: + - name: strimzi-operator + namespace: {{ .Values.namespace }} + chart: strimzi/strimzi-kafka-operator + version: 0.46.0 + values: + - watchNamespaces: ["{{ .Values.namespace }}"] + + - name: kafka-cluster + namespace: {{ .Values.namespace }} + chart: charts/kafka-cluster + needs: + - {{ .Values.namespace }}/strimzi-operator + values: + - namespace: {{ .Values.namespace }} + nodePool: + replicas: {{ .Values.kafkaCluster.nodePool.replicas }} + storage: + class: {{ .Values.kafkaCluster.nodePool.storage.class }} + sizeGi: {{ .Values.kafkaCluster.nodePool.storage.sizeGi }} + resources: + memory: {{ .Values.kafkaCluster.nodePool.resources.memory }} + cpu: {{ .Values.kafkaCluster.nodePool.resources.cpu | quote }} + + - name: kmsvc-redis + namespace: {{ .Values.namespace }} + chart: bitnami/redis + version: 20.6.0 + values: + - architecture: standalone + # Bitnami stopped publishing version-pinned tags under docker.io/bitnami + # (only `latest` remains there); bitnamilegacy/* mirrors the old + # versioned tags for free, so pin there instead of floating on `latest`. + # The chart's container-image allowlist check doesn't know about the + # legacy mirror, hence allowInsecureImages. + global: + security: + allowInsecureImages: true + image: + repository: bitnamilegacy/redis + auth: + enabled: false + master: + persistence: + enabled: true + storageClass: {{ .Values.redis.storageClass }} + size: 2Gi + resources: + limits: + memory: {{ .Values.redis.memoryLimit }} + requests: + memory: {{ .Values.redis.memoryLimit }} + + - name: queue-crd + namespace: {{ .Values.namespace }} + chart: charts/queue-crd + needs: + - {{ .Values.namespace }}/kafka-cluster + - {{ .Values.namespace }}/kmsvc-redis + values: + - namespace: {{ .Values.namespace }} + kafkaBrokers: "kmsvc-kafka-bootstrap.{{ .Values.namespace }}.svc.cluster.local:9092" + redisAddr: "kmsvc-redis-master.{{ .Values.namespace }}.svc.cluster.local:6379" + + - name: management-service + namespace: {{ .Values.namespace }} + chart: charts/management-service + needs: + - {{ .Values.namespace }}/kafka-cluster + - {{ .Values.namespace }}/kmsvc-redis + values: + - namespace: {{ .Values.namespace }} + env: + kafkaBrokers: "kmsvc-kafka-bootstrap.{{ .Values.namespace }}.svc.cluster.local:9092" + redisAddr: "kmsvc-redis-master.{{ .Values.namespace }}.svc.cluster.local:6379" + authentikIssuerURL: {{ .Values.managementService.authentikIssuerURL | quote }} + authentikAudience: {{ .Values.managementService.authentikAudience | quote }} + ingress: + enabled: true + host: {{ .Values.managementService.ingress.host | quote }} + clusterIssuer: {{ .Values.managementService.ingress.clusterIssuer | quote }} diff --git a/k8s/sqs/kmsvc-ingress-oauth2.yaml b/k8s/sqs/kmsvc-ingress-oauth2.yaml new file mode 100644 index 0000000..0398197 --- /dev/null +++ b/k8s/sqs/kmsvc-ingress-oauth2.yaml @@ -0,0 +1,25 @@ +# Ingress for kmsvc REST API — routes to OAuth2-Proxy +# TLS terminated here; oauth2-proxy handles OIDC auth + +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: kmsvc + namespace: sqs +spec: + ingressClassName: nginx + tls: + - secretName: kmsvc-tls + hosts: + - kmsvc.riotpiao.homelab.com + rules: + - host: kmsvc.riotpiao.homelab.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: oauth2-proxy + port: + number: 4180 diff --git a/k8s/sqs/oauth2-proxy.yaml b/k8s/sqs/oauth2-proxy.yaml new file mode 100644 index 0000000..0dcbe8e --- /dev/null +++ b/k8s/sqs/oauth2-proxy.yaml @@ -0,0 +1,106 @@ +# OAuth2-Proxy for kmsvc REST API +# Protects gRPC-gateway (REST) endpoint with Authentik OIDC + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: oauth2-proxy + namespace: sqs + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: oauth2-proxy + namespace: sqs +spec: + replicas: 1 + selector: + matchLabels: + app: oauth2-proxy + template: + metadata: + labels: + app: oauth2-proxy + annotations: + secret.reloader.stakater.com/reload: "kmsvc-oidc" + spec: + serviceAccountName: oauth2-proxy + containers: + - name: oauth2-proxy + image: quay.io/oauth2-proxy/oauth2-proxy:v7.5.1 + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 4180 + protocol: TCP + env: + - name: OAUTH2_PROXY_PROVIDER + value: "oidc" + - name: OAUTH2_PROXY_OIDC_ISSUER_URL + value: "https://authentik.riotpiao.homelab.com/application/o/kmsvc/" + - name: OAUTH2_PROXY_CLIENT_ID + value: "kmsvc" + - name: OAUTH2_PROXY_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: kmsvc-oidc + key: clientSecret + - name: OAUTH2_PROXY_COOKIE_SECRET + valueFrom: + secretKeyRef: + name: kmsvc-oidc + key: cookieSecret + - name: OAUTH2_PROXY_REDIRECT_URL + value: "https://kmsvc.riotpiao.homelab.com/oauth2/callback" + - name: OAUTH2_PROXY_UPSTREAM + value: "http://kmsvc-management-service:8080" + - name: OAUTH2_PROXY_COOKIE_SECURE + value: "true" + - name: OAUTH2_PROXY_COOKIE_HTTPONLY + value: "true" + - name: OAUTH2_PROXY_COOKIE_SAMESITE + value: "Lax" + - name: OAUTH2_PROXY_EMAIL_DOMAIN + value: "*" + - name: OAUTH2_PROXY_SKIP_AUTH_REGEX + value: "^/health" + - name: OAUTH2_PROXY_PASS_AUTHORIZATION_HEADER + value: "true" + - name: OAUTH2_PROXY_REVERSE_PROXY + value: "true" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 200m + memory: 256Mi + livenessProbe: + httpGet: + path: /ping + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /ping + port: http + initialDelaySeconds: 5 + periodSeconds: 5 + +--- +apiVersion: v1 +kind: Service +metadata: + name: oauth2-proxy + namespace: sqs +spec: + type: ClusterIP + ports: + - port: 4180 + targetPort: http + protocol: TCP + name: http + selector: + app: oauth2-proxy diff --git a/k8s/sqs/queues/orders-fifo.yaml b/k8s/sqs/queues/orders-fifo.yaml new file mode 100644 index 0000000..a58b487 --- /dev/null +++ b/k8s/sqs/queues/orders-fifo.yaml @@ -0,0 +1,32 @@ +apiVersion: kmsvc.io/v1 +kind: Queue +metadata: + name: orders-fifo + namespace: sqs +spec: + fifoQueue: true + visibilityTimeoutSeconds: 30 + messageRetentionPeriodSeconds: 345600 + maxReceiveCount: 5 + deadLetterTargetQueue: orders-fifo-dlq + delaySeconds: 0 + partitionsPerShard: 6 + minShards: 1 + maxShards: 8 + shardSplitThresholdBytesPerSec: 5242880 + shardSplitCooldownSeconds: 300 +--- +apiVersion: kmsvc.io/v1 +kind: Queue +metadata: + name: orders-fifo-dlq + namespace: sqs +spec: + fifoQueue: true + isDLQ: true + visibilityTimeoutSeconds: 30 + messageRetentionPeriodSeconds: 1209600 + maxReceiveCount: 5 + partitionsPerShard: 6 + minShards: 1 + maxShards: 1 diff --git a/k8s/temporal/PHASE_2_TESTING.md b/k8s/temporal/PHASE_2_TESTING.md new file mode 100644 index 0000000..5c5a0eb --- /dev/null +++ b/k8s/temporal/PHASE_2_TESTING.md @@ -0,0 +1,237 @@ +# Phase 2: Namespace-Scoped Auto-Provisioning Testing Guide + +## Overview + +Phase 2 implements **namespace-scoped automatic TemporalWorker provisioning** (Kafka broker model). + +One TemporalWorker per Temporal namespace processes ALL task queues in that namespace. When multiple Queues share the same `temporal.io/namespace` label, they trigger creation of a single TemporalWorker that handles all of them. + +``` +Queues (labeled temporal.io/namespace: "production") + ├── orders-fifo + ├── payments + └── notifications + ↓ +queue-operator creates 1 TemporalWorker (worker-production) + ↓ +TemporalWorker controller creates 1 Deployment + ↓ +Worker pod(s) connect to Temporal namespace "production" + ↓ +Process ALL task queues in that namespace (scale horizontally by replicas) +``` + +## Implementation Changes + +### 1. TemporalWorker CRD (`apis/kmsvc/v1/temporalworker_types.go`) +- New Kubernetes resource type to manage namespace-scoped workers +- Fields: Namespace (required), Image, Replicas, Resources, NodeSelector, Affinity, Tolerations +- Status: Phase (Pending/Ready/Failed), Replicas, ReadyReplicas, Conditions +- Model: 1 TemporalWorker per Temporal namespace (not per queue) + +### 2. QueueReconciler Extension (`internal/operator/queue_controller.go`) +- New method: `reconcileTemporalWorker()` +- Logic: If Queue has `temporal.io/namespace` label, create TemporalWorker for that namespace +- Idempotent: multiple queues with same namespace label create same TemporalWorker (no duplicates) + +### 3. TemporalWorkerReconciler (`internal/operator/temporal_worker_controller.go`) +- New controller watching TemporalWorker objects +- Creates/updates Kubernetes Deployment with: + - Pod spec: container image, env vars (TEMPORAL_FRONTEND_ADDRESS, TEMPORAL_TASK_QUEUE) + - Replicas, resources, node selector, affinity, tolerations from TemporalWorker spec +- Updates TemporalWorker status with deployment replica counts and phase + +### 4. Operator Main (`cmd/queue-operator/main.go`) +- Registers TemporalWorker CRD in scheme +- Registers TemporalWorkerReconciler controller +- Controller watches TemporalWorker objects; owns Deployment objects + +## Testing Procedure + +### Prerequisites +- kmsvc queue-operator must be running (built and deployed) +- Temporal cluster must be ready (temporal-frontend service available at `temporal-frontend.temporal.svc.cluster.local:7233`) +- story-crater-backend Docker image must exist (used as default worker image) + +### Step 1: Build and Deploy kmsvc Operator +```bash +cd /Users/rockliang/workplace/kmsvc-manage +make build # builds queue-operator binary +make docker-build # builds Docker image +make deploy # deploys to cluster (requires Helm chart) +``` + +Or manually: +```bash +cd /Users/rockliang/workplace/kmsvc-manage +go build -o bin/queue-operator ./cmd/queue-operator +kubectl apply -f k8s/queue-operator-rbac.yaml +kubectl apply -f k8s/queue-operator-deployment.yaml +``` + +### Step 2: Create Queues with Temporal Namespace Labels +```bash +kubectl apply -f /Users/rockliang/workplace/homelab/k8s/temporal/queues/example-queue.yaml +``` + +Verify Queues are Ready: +```bash +kubectl get queue -n sqs -l temporal.io/namespace=production +kubectl describe queue -n sqs story-crater-tasks +``` + +Expected: +``` +NAME FIFO PHASE AGE +story-crater-tasks false Ready 5s +story-crater-notifications false Ready 5s +``` + +### Step 3: Verify TemporalWorker CRD Auto-Created (1 per namespace) +```bash +kubectl get temporalworker -n temporal +kubectl describe temporalworker -n temporal worker-production +``` + +Expected: +``` +NAME PHASE READY DESIRED AGE +worker-production Pending 0 1 5s +``` + +Only ONE TemporalWorker for all queues in "production" namespace! + +### Step 4: Verify Deployment Auto-Created +```bash +kubectl get deploy -n temporal -l app.kubernetes.io/managed-by=kmsvc-temporal-operator +kubectl get pods -n temporal -l app.kubernetes.io/instance=worker-production +``` + +Expected: +``` +NAME READY UP-TO-DATE AVAILABLE AGE +worker-production 1/1 1 1 10s + +NAME READY STATUS RESTARTS AGE +worker-production-5f8b4c... 1/1 Running 0 10s +``` + +### Step 5: Verify Worker Connected to Temporal Namespace +Check Temporal UI for namespace "production": +```bash +open https://temporal.riotpiao.homelab.com/namespaces/production/task-queues +``` + +Look for all task queues with worker count > 0: +- `story-crater-tasks` +- `story-crater-notifications` +- (worker processes all of them) + +Or via CLI: +```bash +kubectl port-forward -n temporal svc/temporal-frontend 7233 & +curl http://localhost:7233/api/v1/task-queues?namespace=production +``` + +### Step 6: Verify TemporalWorker Status Updated +```bash +kubectl get temporalworker -n temporal +kubectl describe temporalworker -n temporal worker-production +``` + +Expected: +``` +NAME PHASE READY DESIRED AGE +worker-production Ready 1 1 15s + +Status: + Phase: Ready + Ready Replicas: 1 + Replicas: 1 +``` + +### Step 7: Test Namespace-Level Scaling +Create more queues in the same namespace: +```yaml +apiVersion: kmsvc.io/v1 +kind: Queue +metadata: + name: story-crater-llm-processing + namespace: sqs + labels: + temporal.io/namespace: "production" # same namespace +``` + +Verify: No new TemporalWorker created (same worker handles all 3 queues): +```bash +kubectl get temporalworker -n temporal # still just 1 worker-production +kubectl get deploy -n temporal worker-production # same deployment +``` + +Worker auto-discovers new task queue in namespace and processes it. + +### Step 8: Test Cascading Deletion +Delete a Queue; worker should remain (other queues still need it): +```bash +kubectl delete queue -n sqs story-crater-notifications +``` + +Verify: +```bash +kubectl get temporalworker -n temporal # worker-production still exists +kubectl get pods -n temporal worker-production # still running +``` + +Delete all queues in namespace: +```bash +kubectl delete queue -n sqs -l temporal.io/namespace=production +``` + +Verify: TemporalWorker now has no owner (not cascade-deleted; manual cleanup needed): +```bash +kubectl get temporalworker -n temporal # worker-production still there (manual cleanup) +kubectl delete temporalworker -n temporal worker-production # cleanup manually +``` + +## Debugging + +### Queue stuck in Pending +Check queue-operator logs: +```bash +kubectl logs -n sqs deploy/kmsvc-queue-operator -f +kubectl logs -n sqs deploy/kmsvc-queue-operator --tail=50 | grep -i error +``` + +### TemporalWorker not created +- Verify Queue has the label: `kubectl get queue -o yaml | grep temporal.io` +- Check queue-operator logs for "reconcileTemporalWorker" errors + +### Deployment not created +- Check TemporalWorker controller logs: `kubectl logs -n sqs deploy/kmsvc-queue-operator -f` +- Verify TemporalWorker exists: `kubectl get temporalworker -n temporal` +- Check Deployment errors: `kubectl describe deploy -n temporal worker-story-crater-tasks` + +### Worker not showing in Temporal UI +- Check pod logs: `kubectl logs -n temporal deploy/worker-story-crater-tasks` +- Verify env vars: `kubectl set env pod -n temporal --list | grep TEMPORAL` +- Test connectivity: `kubectl exec -n temporal -- nc -zv temporal-frontend.temporal.svc.cluster.local 7233` + +## Next Steps + +Once Phase 2 is working: +1. **Phase 3 (Future):** Implement autoscaling based on queue depth metrics +2. **Production Hardening:** + - Add QueueRef validation (ensure Queue exists in sqs namespace) + - Add image validation/defaults from ConfigMap + - Add worker readiness probe configuration + - Add graceful shutdown/drain behavior + +## Files Modified/Created + +| File | Change | +|------|--------| +| `apis/kmsvc/v1/temporalworker_types.go` | NEW: CRD type definitions | +| `internal/operator/queue_controller.go` | MODIFIED: Added reconcileTemporalWorker() | +| `internal/operator/temporal_worker_controller.go` | NEW: TemporalWorker → Deployment reconciler | +| `cmd/queue-operator/main.go` | MODIFIED: Register TemporalWorker CRD + controller | +| `k8s/temporal/queues/example-queue.yaml` | NEW: Example Queue with label | diff --git a/k8s/temporal/TEMPORAL_OAUTH2_SETUP.md b/k8s/temporal/TEMPORAL_OAUTH2_SETUP.md new file mode 100644 index 0000000..df335cb --- /dev/null +++ b/k8s/temporal/TEMPORAL_OAUTH2_SETUP.md @@ -0,0 +1,220 @@ +# Temporal OAuth2-Proxy Setup (Authentik OIDC) + +## Overview + +Protects Temporal UI with Authentik OIDC authentication. Traffic flow: + +``` +Browser → Ingress (TLS) → oauth2-proxy (OIDC check) → temporal-web (internal) + ↓ + Redirects to Authentik login + ↓ + JWT cookie issued + ↓ + Forwards to temporal-web +``` + +## Prerequisites + +✅ Authentik OIDC provider `temporal` already exists with: +- Client ID: `temporal` +- Client Secret: stored in Kubernetes secret `temporal-oidc` (key: `clientSecret`) +- Redirect URI: `https://temporal.riotpiao.homelab.com/oauth2/callback` + +## Secrets + +The `temporal-oidc` secret must contain: + +| Key | Value | Source | +|-----|-------|--------| +| `clientSecret` | OAuth2 client secret from Authentik | Authentik → Applications → temporal | +| `cookieSecret` | Session encryption key (base64 32-byte) | Generate: `openssl rand -base64 32` | + +### Check existing secret: + +```bash +kubectl get secret -n temporal temporal-oidc +kubectl describe secret -n temporal temporal-oidc +``` + +### If missing, create it: + +```bash +# Get client secret from Authentik UI +# Applications → temporal → copy "Client Secret" +CLIENT_SECRET="..." + +# Generate cookie secret +COOKIE_SECRET=$(openssl rand -base64 32) + +# Create secret +kubectl create secret generic temporal-oidc \ + -n temporal \ + --from-literal=clientSecret="${CLIENT_SECRET}" \ + --from-literal=cookieSecret="${COOKIE_SECRET}" +``` + +## Deployment Steps + +### Step 1: Apply OAuth2-Proxy Manifests +```bash +kubectl apply -f k8s/temporal/oauth2-proxy.yaml +``` + +Verify: +```bash +kubectl get deploy -n temporal oauth2-proxy +kubectl logs -n temporal deploy/oauth2-proxy +``` + +Expected log: +``` +[] [oauthproxy.go:...] Listening on 0.0.0.0:4180 +``` + +### Step 2: Apply OAuth2-Proxy Ingress +```bash +kubectl apply -f k8s/temporal/temporal-ingress-oauth2.yaml +``` + +Verify: +```bash +kubectl get ingress -n temporal +``` + +Expected: +``` +NAME CLASS HOSTS ADDRESS PORTS AGE +temporal nginx temporal.riotpiao.homelab.com ... 80, 443 10s +``` + +### Step 3: Test Access + +1. **Open Temporal UI (unauthenticated):** + ```bash + open https://temporal.riotpiao.homelab.com + ``` + + Expected: Redirects to Authentik login page + +2. **Login with Authentik credentials** + - Username/email + - Password + - Should redirect back to `temporal.riotpiao.homelab.com` and display UI + +3. **Verify auth:** + ```bash + # Check for oauth2_proxy cookie + curl -v https://temporal.riotpiao.homelab.com 2>&1 | grep -i cookie + ``` + +4. **Check oauth2-proxy logs:** + ```bash + kubectl logs -n temporal deploy/oauth2-proxy -f + ``` + + Look for: + ``` + [timestamp] [auth_test.go:...] Authentication successful + ``` + +## Troubleshooting + +### Redirect URI mismatch +Error in oauth2-proxy logs: +``` +redirect_uri_mismatch: The redirect_uri does not match the one registered in Authentik +``` + +Fix: +- Verify Authentik application (Applications → temporal) has redirect URI: `https://temporal.riotpiao.homelab.com/oauth2/callback` +- Ensure HTTPS (not HTTP) + +### Missing secret +Error: +``` +clientSecret: key not found in temporal-oidc secret +``` + +Fix: +```bash +kubectl get secret -n temporal temporal-oidc -o yaml +# If missing, create per "Secrets" section above +``` + +### Cookie secret expiration +OAuth2-Proxy won't start if `cookieSecret` is empty or invalid. + +Fix: +```bash +COOKIE_SECRET=$(openssl rand -base64 32) +kubectl patch secret temporal-oidc -n temporal \ + -p "{\"data\":{\"cookieSecret\":\"$(echo -n $COOKIE_SECRET | base64)\"}}}" +kubectl rollout restart deploy/oauth2-proxy -n temporal +``` + +### oauth2-proxy crashes with "connection refused" +Error in logs: +``` +upstream connect error or disconnect/reset before headers +``` + +Likely cause: `temporal-web` service not accessible. + +Check: +```bash +kubectl get svc -n temporal temporal-web +kubectl exec -n temporal deploy/oauth2-proxy -- curl http://temporal-web:8080 +``` + +## File Structure + +``` +k8s/temporal/ +├── oauth2-proxy.yaml # oauth2-proxy Deployment + Service + SA +├── temporal-ingress-oauth2.yaml # Ingress routing to oauth2-proxy +├── oauth2-proxy-values.yaml # Helm values (reference only) +└── temporal-values.yaml # Modified: ingress.enabled=false +``` + +## Next: Add to Helmfile + +If integrating with helmfile.yaml.gotmpl: + +```yaml +releases: + - name: temporal + # ... existing config ... + hooks: + postSync: + - events: ["success"] + showlogs: true + command: "sh" + args: + - -c + - | + kubectl apply -f k8s/temporal/oauth2-proxy.yaml + kubectl apply -f k8s/temporal/temporal-ingress-oauth2.yaml +``` + +Or add separate releases: + +```yaml + - name: oauth2-proxy-temporal + namespace: temporal + chart: oauth2-proxy/oauth2-proxy + version: "6.x.x" + values: + - k8s/temporal/oauth2-proxy-values.yaml + set: + - name: config.clientSecret + value: "{{ (env "TEMPORAL_OIDC_CLIENT_SECRET") }}" + - name: config.cookieSecret + value: "{{ (env "TEMPORAL_OIDC_COOKIE_SECRET") }}" +``` + +Then add to `.env`: +```bash +TEMPORAL_OIDC_CLIENT_SECRET= +TEMPORAL_OIDC_COOKIE_SECRET=$(openssl rand -base64 32) +``` diff --git a/k8s/temporal/elasticsearch.yaml b/k8s/temporal/elasticsearch.yaml new file mode 100644 index 0000000..5c73b1f --- /dev/null +++ b/k8s/temporal/elasticsearch.yaml @@ -0,0 +1,102 @@ +# Elasticsearch 7.17.0 for Temporal visibility store +# Deployed to worker nodes (not control plane to save CP resources for LLM work) +# 2Gi heap + 4Gi memory limit for stable operation + +apiVersion: v1 +kind: ConfigMap +metadata: + name: elasticsearch-config + namespace: temporal +data: + elasticsearch.yml: | + cluster.name: temporal-elasticsearch + node.name: temporal-elasticsearch-0 + discovery.type: single-node + network.host: 0.0.0.0 + http.host: 0.0.0.0 + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: temporal-elasticsearch + namespace: temporal +spec: + replicas: 1 + selector: + matchLabels: + app: temporal-elasticsearch + template: + metadata: + labels: + app: temporal-elasticsearch + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/worker + operator: Exists + containers: + - name: elasticsearch + image: docker.elastic.co/elasticsearch/elasticsearch:7.17.0 + env: + - name: discovery.type + value: single-node + - name: "ES_JAVA_OPTS" + value: "-Xms2g -Xmx2g" + ports: + - containerPort: 9200 + name: http + - containerPort: 9300 + name: transport + livenessProbe: + httpGet: + path: /_cluster/health + port: 9200 + initialDelaySeconds: 180 + periodSeconds: 10 + timeoutSeconds: 10 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /_cluster/health?local=true + port: 9200 + initialDelaySeconds: 150 + periodSeconds: 10 + timeoutSeconds: 10 + failureThreshold: 5 + resources: + requests: + cpu: 500m + memory: 2Gi + limits: + cpu: 2000m + memory: 4Gi + volumeMounts: + - name: config + mountPath: /usr/share/elasticsearch/config/elasticsearch.yml + subPath: elasticsearch.yml + volumes: + - name: config + configMap: + name: elasticsearch-config + +--- +apiVersion: v1 +kind: Service +metadata: + name: temporal-elasticsearch + namespace: temporal +spec: + selector: + app: temporal-elasticsearch + ports: + - port: 9200 + targetPort: 9200 + name: http + - port: 9300 + targetPort: 9300 + name: transport + type: ClusterIP diff --git a/k8s/temporal/oauth2-proxy-values.yaml b/k8s/temporal/oauth2-proxy-values.yaml new file mode 100644 index 0000000..1b21a59 --- /dev/null +++ b/k8s/temporal/oauth2-proxy-values.yaml @@ -0,0 +1,45 @@ +# OAuth2-Proxy for Temporal UI — protects with Authentik OIDC +# Deployed via Helm: oauth2-proxy/oauth2-proxy chart + +config: + clientID: temporal + clientSecret: "" # injected from temporal-oidc secret + cookieSecret: "" # generated; helm --set will override + configFile: "" + +auth: + enabled: true + +extraArgs: + - --provider=oidc + - --oidc-issuer-url=https://authentik.riotpiao.homelab.com/application/o/temporal/ + - --redirect-url=https://temporal.riotpiao.homelab.com/oauth2/callback + - --upstream=http://temporal-web:8080 + - --cookie-secure=true + - --cookie-httponly=true + - --cookie-samesite=Lax + - --email-domain=* + - --skip-auth-regex=^/health + - --pass-authorization-header=true + - --skip-auth-preflight=true + +service: + type: ClusterIP + port: 4180 + targetPort: 4180 + +ingress: + enabled: false # we'll keep temporal's ingress, just route to oauth2-proxy + +replicaCount: 1 + +resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 200m + memory: 256Mi + +podAnnotations: + secret.reloader.stakater.com/reload: "temporal-oidc" diff --git a/k8s/temporal/oauth2-proxy.yaml b/k8s/temporal/oauth2-proxy.yaml new file mode 100644 index 0000000..a63fc59 --- /dev/null +++ b/k8s/temporal/oauth2-proxy.yaml @@ -0,0 +1,108 @@ +# OAuth2-Proxy deployment for Temporal UI +# Requires: temporal-oidc secret with clientSecret and cookieSecret + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: oauth2-proxy + namespace: temporal + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: oauth2-proxy + namespace: temporal +spec: + replicas: 1 + selector: + matchLabels: + app: oauth2-proxy + template: + metadata: + labels: + app: oauth2-proxy + annotations: + secret.reloader.stakater.com/reload: "temporal-oidc" + spec: + serviceAccountName: oauth2-proxy + containers: + - name: oauth2-proxy + image: quay.io/oauth2-proxy/oauth2-proxy:v7.5.1 + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 4180 + protocol: TCP + env: + - name: OAUTH2_PROXY_PROVIDER + value: "oidc" + - name: OAUTH2_PROXY_OIDC_ISSUER_URL + value: "https://authentik.riotpiao.homelab.com/application/o/temporal/" + - name: OAUTH2_PROXY_CLIENT_ID + value: "temporal" + - name: OAUTH2_PROXY_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: temporal-oidc + key: clientSecret + - name: OAUTH2_PROXY_COOKIE_SECRET + valueFrom: + secretKeyRef: + name: temporal-oidc + key: cookieSecret + - name: OAUTH2_PROXY_REDIRECT_URL + value: "https://temporal.riotpiao.homelab.com/oauth2/callback" + - name: OAUTH2_PROXY_UPSTREAM + value: "http://temporal-web:8080" + - name: OAUTH2_PROXY_COOKIE_SECURE + value: "true" + - name: OAUTH2_PROXY_COOKIE_HTTPONLY + value: "true" + - name: OAUTH2_PROXY_COOKIE_SAMESITE + value: "Lax" + - name: OAUTH2_PROXY_EMAIL_DOMAIN + value: "*" + - name: OAUTH2_PROXY_SKIP_AUTH_REGEX + value: "^/health" + - name: OAUTH2_PROXY_PASS_AUTHORIZATION_HEADER + value: "true" + - name: OAUTH2_PROXY_SKIP_AUTH_PREFLIGHT + value: "true" + - name: OAUTH2_PROXY_REVERSE_PROXY + value: "true" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 200m + memory: 256Mi + livenessProbe: + httpGet: + path: /ping + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /ping + port: http + initialDelaySeconds: 5 + periodSeconds: 5 + +--- +apiVersion: v1 +kind: Service +metadata: + name: oauth2-proxy + namespace: temporal +spec: + type: ClusterIP + ports: + - port: 4180 + targetPort: http + protocol: TCP + name: http + selector: + app: oauth2-proxy diff --git a/k8s/temporal/queues/example-queue.yaml b/k8s/temporal/queues/example-queue.yaml new file mode 100644 index 0000000..794ee41 --- /dev/null +++ b/k8s/temporal/queues/example-queue.yaml @@ -0,0 +1,38 @@ +# Example Queues for "production" Temporal namespace +# When applied, queue-operator creates 1 TemporalWorker (worker-production) +# that processes ALL queues in the "production" namespace +--- +apiVersion: kmsvc.io/v1 +kind: Queue +metadata: + name: story-crater-tasks + namespace: sqs + labels: + temporal.io/namespace: "production" +spec: + fifoQueue: false + visibilityTimeoutSeconds: 30 + messageRetentionPeriodSeconds: 345600 + maxReceiveCount: 5 + partitionsPerShard: 6 + minShards: 1 + maxShards: 8 + shardSplitThresholdBytesPerSec: 5242880 + +--- +apiVersion: kmsvc.io/v1 +kind: Queue +metadata: + name: story-crater-notifications + namespace: sqs + labels: + temporal.io/namespace: "production" +spec: + fifoQueue: false + visibilityTimeoutSeconds: 60 + messageRetentionPeriodSeconds: 345600 + maxReceiveCount: 3 + partitionsPerShard: 3 + minShards: 1 + maxShards: 4 + shardSplitThresholdBytesPerSec: 2621440 diff --git a/k8s/temporal/temporal-ingress-oauth2.yaml b/k8s/temporal/temporal-ingress-oauth2.yaml new file mode 100644 index 0000000..32ed340 --- /dev/null +++ b/k8s/temporal/temporal-ingress-oauth2.yaml @@ -0,0 +1,27 @@ +# Ingress for Temporal UI — routes to OAuth2-Proxy, which proxies to temporal-web +# TLS terminated here; oauth2-proxy handles OIDC auth before forwarding to backend + +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: temporal + namespace: temporal + annotations: + cert-manager.io/cluster-issuer: "letsencrypt-prod" +spec: + ingressClassName: nginx + tls: + - secretName: temporal-tls + hosts: + - temporal.riotpiao.homelab.com + rules: + - host: temporal.riotpiao.homelab.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: oauth2-proxy + port: + number: 4180 diff --git a/k8s/temporal/temporal-values.yaml b/k8s/temporal/temporal-values.yaml new file mode 100644 index 0000000..3d22de7 --- /dev/null +++ b/k8s/temporal/temporal-values.yaml @@ -0,0 +1,85 @@ +# k8s/temporal/temporal-values.yaml +# Temporal — workflow engine for story-crater backend async task orchestration. +# Chart: temporal/temporal from https://go.temporal.io/helm-charts +# +# Uses Cassandra for default store (workflow history/events) +# Uses Elasticsearch for visibility store (namespace/workflow queries) +# This is the chart's native, well-tested configuration. + +# ── Datastores configuration ──── +# Disable auto-deployed PostgreSQL (we use external ddb for other services) +postgresql: + enabled: false + +# Enable Elasticsearch for visibility store (deployed to worker node, 2Gi/4Gi memory) +elasticsearch: + enabled: true + scheme: http + host: temporal-elasticsearch + port: 9200 + version: v7 + logLevel: error + auth: + enabled: false + indices: + visibility: temporal_visibility_v1 + +# Cassandra enabled for template validation; server.config overrides with actual hosts +# Schema job template requires cassandra config to exist at top level +cassandra: + enabled: true + replicas: 3 + cluster: + seedSize: 1 + port: 9042 + +# ── Disable schema auto-setup (will initialize manually) ───────── +jobs: + autoSetup: + enabled: false + +# ── Temporal server config (Cassandra + Elasticsearch persistence) ────────────────────────────── +server: + replicaCount: 1 + jobService: + enabled: false + config: + logLevel: "info" + persistence: + defaultStore: default + visibilityStore: visibility + numHistoryShards: 512 + datastores: + default: + # Cassandra for workflow history and events + driver: cassandra + cassandra: + hosts: "temporal-cassandra" + port: 9042 + keyspace: temporal + user: user + password: "" # Cassandra auth disabled in deployment + replicationFactor: 3 + consistency: + default: + consistency: local_quorum + serialConsistency: local_serial + service: + type: ClusterIP + +# ── Temporal Web UI ──────────────────────────────────────────────────────── +web: + replicaCount: 1 + service: + type: ClusterIP + +# ── Ingress ──────────────────────────────────────────────────────── +# Note: ingress is disabled here. Instead, we route via oauth2-proxy. +# The ingress is applied separately as k8s/temporal/temporal-ingress-oauth2.yaml +# which terminates TLS and routes to oauth2-proxy service. +ingress: + enabled: false + +# ── Monitoring ──────────────────────────────────────────────────────── +prometheus: + enabled: false