117 lines
3.7 KiB
Go
117 lines
3.7 KiB
Go
// Package reaper implements the goroutine-based redelivery/DLQ sweep from
|
|
// design.md §5: a per-queue ticker scans vis_index for expired in-flight
|
|
// messages and drives each through the atomic reap.lua check-and-act.
|
|
package reaper
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"time"
|
|
|
|
goredis "github.com/redis/go-redis/v9"
|
|
|
|
"forgejo.riotpiao.com/rock/kmsvc-manage/internal/core/queue"
|
|
kmsvcredis "forgejo.riotpiao.com/rock/kmsvc-manage/internal/redis"
|
|
)
|
|
|
|
const (
|
|
defaultSweepInterval = 5 * time.Second
|
|
sweepBatchSize = 100
|
|
)
|
|
|
|
// DLQSender is the subset of queue.SendMessageService a Reaper needs to
|
|
// route a maxed-out message to its queue's dead-letter target. Abstracted so
|
|
// tests can substitute a fake without a real Kafka broker.
|
|
type DLQSender interface {
|
|
SendMessage(ctx context.Context, in queue.SendMessageInput) (queue.SendMessageOutput, error)
|
|
}
|
|
|
|
// Reaper sweeps one or more queues' vis_index ZSETs for expired in-flight
|
|
// messages. Safe to run concurrently from multiple replicas against the same
|
|
// queue: reap.lua's check-and-act is atomic per receipt handle, so only the
|
|
// first caller for a given expired entry gets a non-"gone" outcome.
|
|
type Reaper struct {
|
|
Redis *goredis.Client
|
|
DLQSender DLQSender
|
|
|
|
// Interval is the sleep between sweep ticks for Run; defaults to 5s.
|
|
Interval time.Duration
|
|
}
|
|
|
|
// Sweep runs one pass over queueName's vis_index, returning the number of
|
|
// expired entries it took action on (redelivered or DLQ-routed; entries
|
|
// another replica already won the race for don't count).
|
|
func (r *Reaper) Sweep(ctx context.Context, queueName string) (int, error) {
|
|
meta, ok, err := kmsvcredis.GetQueueMeta(ctx, r.Redis, queueName)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("reaper sweep %s: %w", queueName, err)
|
|
}
|
|
if !ok {
|
|
return 0, nil
|
|
}
|
|
|
|
handles, err := kmsvcredis.ExpiredVisIndexEntries(ctx, r.Redis, queueName, sweepBatchSize)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("reaper sweep %s: %w", queueName, err)
|
|
}
|
|
|
|
swept := 0
|
|
for _, handle := range handles {
|
|
result, err := kmsvcredis.Reap(ctx, r.Redis, queueName, handle, meta.MaxReceiveCount)
|
|
if err != nil {
|
|
return swept, fmt.Errorf("reaper sweep %s: %w", queueName, err)
|
|
}
|
|
|
|
switch result.Outcome {
|
|
case kmsvcredis.ReapOutcomeGone:
|
|
continue
|
|
case kmsvcredis.ReapOutcomeDLQ:
|
|
if err := r.routeToDLQ(ctx, meta, result); err != nil {
|
|
return swept, fmt.Errorf("reaper sweep %s: %w", queueName, err)
|
|
}
|
|
swept++
|
|
case kmsvcredis.ReapOutcomeRedeliver:
|
|
swept++
|
|
}
|
|
}
|
|
return swept, nil
|
|
}
|
|
|
|
// routeToDLQ produces a maxed-out message onto its queue's configured
|
|
// dead-letter queue. A queue with no DLQ configured just drops the message
|
|
// here, matching reap.lua having already deleted all of its state.
|
|
func (r *Reaper) routeToDLQ(ctx context.Context, meta kmsvcredis.QueueMeta, result kmsvcredis.ReapResult) error {
|
|
if meta.DLQQueueName == "" || r.DLQSender == nil {
|
|
return nil
|
|
}
|
|
_, err := r.DLQSender.SendMessage(ctx, queue.SendMessageInput{
|
|
QueueName: meta.DLQQueueName,
|
|
Body: result.Body,
|
|
MessageGroupID: result.GroupID,
|
|
})
|
|
return err
|
|
}
|
|
|
|
// Run sweeps queueName on a ticker until ctx is done. Intended to be started
|
|
// as its own goroutine per queue (design.md §5); sweep errors are logged-ish
|
|
// via the returned channel-free design (transient Redis/Kafka hiccups are
|
|
// retried on the next tick rather than stopping the loop).
|
|
func (r *Reaper) Run(ctx context.Context, queueName string, onSweepError func(error)) {
|
|
interval := r.Interval
|
|
if interval <= 0 {
|
|
interval = defaultSweepInterval
|
|
}
|
|
ticker := time.NewTicker(interval)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
if _, err := r.Sweep(ctx, queueName); err != nil && onSweepError != nil {
|
|
onSweepError(err)
|
|
}
|
|
}
|
|
}
|
|
}
|