feat: queue-operator auto-registers Temporal namespace before creating TemporalWorker
A Queue's temporal.io/namespace label was trusted as-is -- if the referenced Temporal namespace was never registered (or typo'd), the failure only surfaced as a worker pod silently polling a namespace that doesn't exist. Now reconcileTemporalWorker calls RegisterNamespace (idempotent, ignores AlreadyExists) via a direct WorkflowService gRPC client before creating the TemporalWorker, so namespace and worker always come into existence together. Also grant queue-operator's ClusterRole create/delete on temporalworkers (previously missing, causing forbidden errors on the create-then-delete path).
This commit is contained in:
@@ -0,0 +1,41 @@
|
||||
package operator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// fakeTemporal is an in-memory TemporalNamespaceRegisterer for reconciler
|
||||
// tests — avoids needing a real Temporal frontend just to exercise reconcile
|
||||
// logic.
|
||||
type fakeTemporal struct {
|
||||
mu sync.Mutex
|
||||
registered map[string]int
|
||||
err error
|
||||
}
|
||||
|
||||
func newFakeTemporal() *fakeTemporal {
|
||||
return &fakeTemporal{registered: map[string]int{}}
|
||||
}
|
||||
|
||||
func (f *fakeTemporal) RegisterNamespace(_ context.Context, namespace string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.err != nil {
|
||||
return f.err
|
||||
}
|
||||
f.registered[namespace]++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeTemporal) setErr(err error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.err = err
|
||||
}
|
||||
|
||||
func (f *fakeTemporal) count(namespace string) int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.registered[namespace]
|
||||
}
|
||||
@@ -36,10 +36,11 @@ const minInsyncReplicas = 2
|
||||
|
||||
// QueueReconciler reconciles Queue objects (design.md §2a).
|
||||
type QueueReconciler struct {
|
||||
Client client.Client
|
||||
Admin TopicAdmin
|
||||
Redis *goredis.Client
|
||||
Now func() time.Time
|
||||
Client client.Client
|
||||
Admin TopicAdmin
|
||||
Redis *goredis.Client
|
||||
Now func() time.Time
|
||||
Temporal TemporalNamespaceRegisterer
|
||||
|
||||
// Zones resolves shard topics' broker placement to availability zones
|
||||
// (design.md §2a AZ-awareness). Nil disables zone annotation entirely --
|
||||
@@ -248,6 +249,10 @@ func (r *QueueReconciler) reconcileTemporalWorker(ctx context.Context, queue *km
|
||||
return fmt.Errorf("invalid kubernetes name %q: %w", workerName, err)
|
||||
}
|
||||
|
||||
if err := r.Temporal.RegisterNamespace(ctx, namespace); err != nil {
|
||||
return fmt.Errorf("register temporal namespace %s: %w", namespace, err)
|
||||
}
|
||||
|
||||
replicas := int32(1)
|
||||
workerNamespace := getEnvOrDefault("KMSVC_TEMPORAL_NAMESPACE", "temporal")
|
||||
workerImage := getEnvOrDefault("KMSVC_TEMPORAL_WORKER_IMAGE", "story-crater-backend:latest")
|
||||
|
||||
@@ -2,6 +2,7 @@ package operator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -52,6 +53,12 @@ func newTestSchemeWithAppsV1(t *testing.T) *runtime.Scheme {
|
||||
}
|
||||
|
||||
func newTestReconciler(t *testing.T, objs ...client.Object) (*QueueReconciler, *fakeAdmin) {
|
||||
t.Helper()
|
||||
r, admin, _ := newTestReconcilerWithTemporal(t, objs...)
|
||||
return r, admin
|
||||
}
|
||||
|
||||
func newTestReconcilerWithTemporal(t *testing.T, objs ...client.Object) (*QueueReconciler, *fakeAdmin, *fakeTemporal) {
|
||||
t.Helper()
|
||||
scheme := newTestScheme(t)
|
||||
cl := fake.NewClientBuilder().
|
||||
@@ -60,12 +67,14 @@ func newTestReconciler(t *testing.T, objs ...client.Object) (*QueueReconciler, *
|
||||
WithObjects(objs...).
|
||||
Build()
|
||||
admin := newFakeAdmin()
|
||||
temporal := newFakeTemporal()
|
||||
return &QueueReconciler{
|
||||
Client: cl,
|
||||
Admin: admin,
|
||||
Redis: newTestRedis(t),
|
||||
Now: time.Now,
|
||||
}, admin
|
||||
Client: cl,
|
||||
Admin: admin,
|
||||
Redis: newTestRedis(t),
|
||||
Now: time.Now,
|
||||
Temporal: temporal,
|
||||
}, admin, temporal
|
||||
}
|
||||
|
||||
func baseQueue(name string) *kmsvcv1.Queue {
|
||||
@@ -288,7 +297,7 @@ func TestReconcileDrainsClosingShardWhenLagZero(t *testing.T) {
|
||||
func TestReconcileTemporalWorkerCreatesWhenLabelPresent(t *testing.T) {
|
||||
queue := baseQueue("orders")
|
||||
queue.Labels = map[string]string{"temporal.io/namespace": "default"}
|
||||
r, _ := newTestReconciler(t, queue)
|
||||
r, _, temporal := newTestReconcilerWithTemporal(t, queue)
|
||||
ctx := context.Background()
|
||||
|
||||
if err := r.Reconcile(ctx, "", "orders"); err != nil {
|
||||
@@ -302,6 +311,42 @@ func TestReconcileTemporalWorkerCreatesWhenLabelPresent(t *testing.T) {
|
||||
if worker.Spec.Namespace != "default" {
|
||||
t.Errorf("worker namespace = %q, want default", worker.Spec.Namespace)
|
||||
}
|
||||
if got := temporal.count("default"); got != 1 {
|
||||
t.Errorf("RegisterNamespace(%q) called %d times, want 1", "default", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileTemporalWorkerRegistersNamespaceBeforeCreating(t *testing.T) {
|
||||
queue := baseQueue("orders")
|
||||
queue.Labels = map[string]string{"temporal.io/namespace": "checkout"}
|
||||
r, _, temporal := newTestReconcilerWithTemporal(t, queue)
|
||||
ctx := context.Background()
|
||||
|
||||
if err := r.Reconcile(ctx, "", "orders"); err != nil {
|
||||
t.Fatalf("Reconcile: %v", err)
|
||||
}
|
||||
if got := temporal.count("checkout"); got != 1 {
|
||||
t.Errorf("RegisterNamespace(%q) called %d times, want 1", "checkout", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileTemporalWorkerFailsWhenNamespaceRegistrationFails(t *testing.T) {
|
||||
queue := baseQueue("orders")
|
||||
queue.Labels = map[string]string{"temporal.io/namespace": "checkout"}
|
||||
r, _, temporal := newTestReconcilerWithTemporal(t, queue)
|
||||
temporal.setErr(fmt.Errorf("frontend unreachable"))
|
||||
ctx := context.Background()
|
||||
|
||||
err := r.Reconcile(ctx, "", "orders")
|
||||
if err == nil {
|
||||
t.Fatal("expected Reconcile to fail when namespace registration fails")
|
||||
}
|
||||
|
||||
var worker kmsvcv1.TemporalWorker
|
||||
getErr := r.Client.Get(ctx, client.ObjectKey{Name: "worker-checkout", Namespace: "temporal"}, &worker)
|
||||
if getErr == nil {
|
||||
t.Error("expected no TemporalWorker to be created when namespace registration fails")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileTemporalWorkerValidatesNamespaceLabel(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
package operator
|
||||
|
||||
import "context"
|
||||
|
||||
// TemporalNamespaceRegisterer registers a Temporal namespace. reconcileTemporalWorker
|
||||
// calls this before creating a TemporalWorker so a Queue's temporal.io/namespace
|
||||
// label always has a real namespace behind it — previously that label was
|
||||
// trusted as-is, and a typo'd or never-registered namespace would only
|
||||
// surface as a silently-stuck worker pod polling a namespace that doesn't
|
||||
// exist.
|
||||
type TemporalNamespaceRegisterer interface {
|
||||
// RegisterNamespace registers namespace, treating AlreadyExists as success.
|
||||
RegisterNamespace(ctx context.Context, namespace string) error
|
||||
}
|
||||
Reference in New Issue
Block a user