// Package kafka implements the queue-name <-> Kafka-topic mapping and // topic admin operations described in design.md §6. package kafka import ( "fmt" ) const ( topicPrefix = "kmsvc." fifoSuffix = ".fifo" dlqSuffix = ".dlq" shardInfix = ".shard-" DefaultPartitionCount = 6 // FullHashRangeEnd is the exclusive upper bound of the 32-bit shard key // space; a queue's first shard owns [0, FullHashRangeEnd). FullHashRangeEnd uint32 = 0xFFFFFFFF ) // ShardTopicName returns the Kafka topic name for one shard of a queue, per // design.md §6: kmsvc.{queueName}.shard-{id} (standard), // kmsvc.{queueName}.fifo.shard-{id} (FIFO). func ShardTopicName(queueName string, fifo bool, shardID string) string { base := topicPrefix + queueName if fifo { base += fifoSuffix } return base + shardInfix + shardID } // DLQShardTopicName returns the DLQ topic name for one shard of a queue's // DLQ, per design.md §6: kmsvc.{queueName}.dlq.shard-{id} / .fifo.dlq.shard-{id}. func DLQShardTopicName(queueName string, fifo bool, shardID string) string { base := topicPrefix + queueName if fifo { base += fifoSuffix } return base + dlqSuffix + shardInfix + shardID } // ConsumerGroup returns the single Kafka consumer-group name shared by every // message-plane replica consuming a queue's shards, used both for normal // consumption and by the queue-operator's drain check (design.md §2c, §9). func ConsumerGroup(queueName string) string { return "kmsvc-consumer-" + queueName } // murmur2 mirrors Kafka's default partitioner hash (murmur2), used so that // FIFO partition assignment here matches what a native Kafka producer would // compute for the same key, per design.md §6. func murmur2(data []byte) uint32 { const ( seed uint32 = 0x9747b28c m uint32 = 0x5bd1e995 r = 24 ) length := len(data) h := seed ^ uint32(length) four := length / 4 for i := 0; i < four; i++ { i4 := i * 4 k := uint32(data[i4]&0xff) | (uint32(data[i4+1]&0xff) << 8) | (uint32(data[i4+2]&0xff) << 16) | (uint32(data[i4+3]&0xff) << 24) k *= m k ^= k >> r k *= m h *= m h ^= k } switch length & 3 { case 3: h ^= uint32(data[(length&^3)+2]&0xff) << 16 fallthrough case 2: h ^= uint32(data[(length&^3)+1]&0xff) << 8 fallthrough case 1: h ^= uint32(data[length&^3] & 0xff) h *= m } h ^= h >> 13 h *= m h ^= h >> 15 return h } // toPositive mirrors Kafka's Utils.toPositive, masking the sign bit so the // result is usable as an unsigned partition index. func toPositive(n uint32) uint32 { return n & 0x7fffffff } // HashKey returns the murmur2 hash of a routing key (MessageGroupId for FIFO // queues, a random UUID for standard queues) into the shard key space used // for both shard selection and within-shard partitioning (design.md §2c, §6). func HashKey(key string) uint32 { return toPositive(murmur2([]byte(key))) } // Shard is the subset of a Queue's status.shards entry needed for routing. // Phase mirrors apis/kmsvc/v1.ShardPhase as a plain string to avoid this // package depending on the CRD API package. type Shard struct { ID string Topic string HashRangeStart uint32 HashRangeEnd uint32 Phase string } // ActiveShards filters to shards eligible to receive newly-sent messages // (design.md §2c: a `Closing` shard keeps being consumed/drained but stops // being a write target). func ActiveShards(shards []Shard) []Shard { active := make([]Shard, 0, len(shards)) for _, s := range shards { if s.Phase == "" || s.Phase == "Active" { active = append(active, s) } } return active } // SelectShard returns the shard whose hash range contains routingKey's hash, // per design.md §2c. Callers doing write-path routing should pass // ActiveShards(shards) so messages never target a `Closing` shard; the // shards passed in must cover the full key space with no gaps for this to // always find a match. func SelectShard(shards []Shard, routingKey string) (Shard, bool) { h := HashKey(routingKey) for _, s := range shards { if h >= s.HashRangeStart && h < s.HashRangeEnd { return s, true } } return Shard{}, false } // SplitHashRange returns the midpoint of [start, end), the boundary between // the two child shards created by a split (design.md §2c). func SplitHashRange(start, end uint32) uint32 { return start + (end-start)/2 } // PartitionWithinShard returns the partition a message lands on within its // shard's topic. For FIFO queues routingKey is the MessageGroupId, ensuring // all messages for a group are ordered on one partition within that shard // (design.md §3, §6); for standard queues routingKey is a random per-message // value, so traffic just spreads evenly. func PartitionWithinShard(routingKey string, partitionsPerShard int32) int32 { if partitionsPerShard <= 0 { partitionsPerShard = DefaultPartitionCount } return int32(HashKey(routingKey) % uint32(partitionsPerShard)) } // ValidateNoDLQCycle enforces design.md §5's DLQ-loop guard: a queue's // dead-letter target must not be itself, and a queue marked as a DLQ must // not itself have a dead-letter target (no DLQ chains). func ValidateNoDLQCycle(queueName string, isDLQ bool, deadLetterTarget string) error { if deadLetterTarget == "" { return nil } if deadLetterTarget == queueName { return fmt.Errorf("queue %q cannot set its own dead-letter target", queueName) } if isDLQ { return fmt.Errorf("DLQ queue %q cannot itself have a dead-letter target (no DLQ chains)", queueName) } return nil }