feat: phase 8 serviceadapter crd rollout (32/33 tasks)

2026-08-26 13:47:36 -07:00
parent 63893d41a5
commit 425611ec42
85 changed files with 4238 additions and 5702 deletions
@@ -0,0 +1,135 @@
+package resilience
+
+import (
+	"context"
+	"math/rand"
+	"net/http"
+	"time"
+)
+
+// RetryConfig holds retry settings.
+type RetryConfig struct {
+	// MaxAttempts is the maximum number of attempts (includes initial).
+	MaxAttempts int
+	// InitialBackoff is the initial backoff duration.
+	InitialBackoff time.Duration
+	// MaxBackoff is the maximum backoff duration.
+	MaxBackoff time.Duration
+	// BackoffMultiplier is the exponential backoff multiplier.
+	BackoffMultiplier float64
+}
+
+// DefaultRetryConfig provides sensible defaults.
+func DefaultRetryConfig() *RetryConfig {
+	return &RetryConfig{
+		MaxAttempts:       3,
+		InitialBackoff:    100 * time.Millisecond,
+		MaxBackoff:        2 * time.Second,
+		BackoffMultiplier: 2.0,
+	}
+}
+
+// RetryFunc executes a function with blind retry on 5xx.
+// Returns the response and any error from the function itself (not retry logic).
+type RetryFunc func(ctx context.Context, attempt int) (*http.Response, error)
+
+// DoRetry executes the function with exponential backoff on 5xx responses.
+// Returns the final response (could be 5xx if all retries exhausted) and any error.
+func DoRetry(ctx context.Context, cfg *RetryConfig, fn RetryFunc) (*http.Response, error) {
+	if cfg == nil {
+		cfg = DefaultRetryConfig()
+	}
+
+	var lastResp *http.Response
+	var lastErr error
+
+	for attempt := 0; attempt < cfg.MaxAttempts; attempt++ {
+		// Check context before attempting
+		select {
+		case <-ctx.Done():
+			if lastResp != nil {
+				lastResp.Body.Close()
+			}
+			return nil, ctx.Err()
+		default:
+		}
+
+		resp, err := fn(ctx, attempt)
+		if err != nil {
+			lastErr = err
+			// Don't retry on network errors in the retry loop itself
+			// Let caller decide if those should be retried
+			return nil, err
+		}
+
+		// Success (not 5xx)
+		if resp.StatusCode < 500 {
+			return resp, nil
+		}
+
+		// 5xx — close and retry
+		if lastResp != nil {
+			lastResp.Body.Close()
+		}
+		lastResp = resp
+
+		// If this was the last attempt, return the 5xx response
+		if attempt == cfg.MaxAttempts-1 {
+			return resp, nil
+		}
+
+		// Calculate backoff with jitter
+		backoff := calculateBackoff(attempt, cfg)
+		select {
+		case <-ctx.Done():
+			resp.Body.Close()
+			return nil, ctx.Err()
+		case <-time.After(backoff):
+			// Continue to next attempt
+		}
+	}
+
+	return lastResp, lastErr
+}
+
+// calculateBackoff computes exponential backoff with jitter.
+func calculateBackoff(attempt int, cfg *RetryConfig) time.Duration {
+	// Exponential: initial * (multiplier ^ attempt)
+	backoff := time.Duration(float64(cfg.InitialBackoff) * (pow(cfg.BackoffMultiplier, float64(attempt))))
+
+	// Cap at max
+	if backoff > cfg.MaxBackoff {
+		backoff = cfg.MaxBackoff
+	}
+
+	// Add jitter: ±20%
+	jitterRange := backoff / 5
+	if jitterRange <= 0 {
+		return backoff
+	}
+
+	jitter := time.Duration(rand.Int63n(int64(2 * jitterRange)) - int64(jitterRange))
+
+	return backoff + jitter
+}
+
+func pow(base, exp float64) float64 {
+	result := 1.0
+	for i := 0; i < int(exp); i++ {
+		result *= base
+	}
+	return result
+}
+
+// RetryPolicy determines whether to retry based on response and config.
+type RetryPolicy struct {
+	Retryable bool // Whether this adapter allows retries
+}
+
+// ShouldRetry determines if a response should be retried.
+func (p *RetryPolicy) ShouldRetry(resp *http.Response) bool {
+	if !p.Retryable {
+		return false
+	}
+	return resp != nil && resp.StatusCode >= 500
+}
@@ -0,0 +1,158 @@
+package resilience
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestRetryOnSuccess(t *testing.T) {
+	cfg := &RetryConfig{MaxAttempts: 3}
+	attempts := 0
+
+	resp, err := DoRetry(context.Background(), cfg, func(ctx context.Context, attempt int) (*http.Response, error) {
+		attempts++
+		return &http.Response{
+			StatusCode: 200,
+			Body:       io.NopCloser(strings.NewReader("ok")),
+		}, nil
+	})
+
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if attempts != 1 {
+		t.Errorf("expected 1 attempt on success, got %d", attempts)
+	}
+	if resp.StatusCode != 200 {
+		t.Errorf("expected status 200, got %d", resp.StatusCode)
+	}
+	resp.Body.Close()
+}
+
+func TestRetryOn5xx(t *testing.T) {
+	cfg := &RetryConfig{
+		MaxAttempts:       3,
+		InitialBackoff:    10 * time.Millisecond,
+		MaxBackoff:        50 * time.Millisecond,
+		BackoffMultiplier: 2.0,
+	}
+	attempts := 0
+
+	resp, err := DoRetry(context.Background(), cfg, func(ctx context.Context, attempt int) (*http.Response, error) {
+		attempts++
+		if attempt < 2 {
+			// First two attempts return 503
+			return &http.Response{
+				StatusCode: 503,
+				Body:       io.NopCloser(strings.NewReader("unavailable")),
+			}, nil
+		}
+		// Third attempt succeeds
+		return &http.Response{
+			StatusCode: 200,
+			Body:       io.NopCloser(strings.NewReader("ok")),
+		}, nil
+	})
+
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if attempts != 3 {
+		t.Errorf("expected 3 attempts (2 retries), got %d", attempts)
+	}
+	if resp.StatusCode != 200 {
+		t.Errorf("expected status 200, got %d", resp.StatusCode)
+	}
+	resp.Body.Close()
+}
+
+func TestRetryExhaustion(t *testing.T) {
+	cfg := &RetryConfig{
+		MaxAttempts:    2,
+		InitialBackoff: 10 * time.Millisecond,
+		MaxBackoff:     50 * time.Millisecond,
+	}
+	attempts := 0
+
+	resp, err := DoRetry(context.Background(), cfg, func(ctx context.Context, attempt int) (*http.Response, error) {
+		attempts++
+		// Always return 503
+		return &http.Response{
+			StatusCode: 503,
+			Body:       io.NopCloser(strings.NewReader("unavailable")),
+		}, nil
+	})
+
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if attempts != 2 {
+		t.Errorf("expected 2 attempts (max), got %d", attempts)
+	}
+	if resp.StatusCode != 503 {
+		t.Errorf("expected status 503, got %d", resp.StatusCode)
+	}
+	resp.Body.Close()
+}
+
+func TestRetryWithContext(t *testing.T) {
+	cfg := &RetryConfig{MaxAttempts: 10}
+	attempts := 0
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// Cancel after a short delay
+	go func() {
+		time.Sleep(50 * time.Millisecond)
+		cancel()
+	}()
+
+	resp, err := DoRetry(ctx, cfg, func(ctx context.Context, attempt int) (*http.Response, error) {
+		attempts++
+		time.Sleep(30 * time.Millisecond)
+		return &http.Response{
+			StatusCode: 503,
+			Body:       io.NopCloser(strings.NewReader("unavailable")),
+		}, nil
+	})
+
+	if err != context.Canceled {
+		t.Errorf("expected context.Canceled error, got: %v", err)
+	}
+	if resp != nil {
+		resp.Body.Close()
+	}
+	// Should have fewer than all attempts due to cancellation
+	if attempts >= 10 {
+		t.Errorf("expected fewer than 10 attempts due to cancellation, got %d", attempts)
+	}
+}
+
+func TestRetryPolicyShouldRetry(t *testing.T) {
+	policy := &RetryPolicy{Retryable: true}
+
+	resp503 := &http.Response{StatusCode: 503}
+	if !policy.ShouldRetry(resp503) {
+		t.Errorf("expected to retry on 503")
+	}
+
+	resp200 := &http.Response{StatusCode: 200}
+	if policy.ShouldRetry(resp200) {
+		t.Errorf("expected not to retry on 200")
+	}
+
+	resp404 := &http.Response{StatusCode: 404}
+	if policy.ShouldRetry(resp404) {
+		t.Errorf("expected not to retry on 404")
+	}
+
+	policyNoRetry := &RetryPolicy{Retryable: false}
+	if policyNoRetry.ShouldRetry(resp503) {
+		t.Errorf("expected not to retry when retryable=false")
+	}
+}