Test
60f9ca2b1d
- Add internal/recovery package with comprehensive error recovery infrastructure - Implement RetryPolicy with exponential backoff - Three predefined policies: DefaultRetryPolicy, ActivityRetryPolicy, LLMActivityRetryPolicy - Integrate with Temporal SDK via ToTemporalRetryPolicy() - Implement DeadletterQueue for tracking permanently failed activities - Thread-safe deadletter operations with JSON persistence - Mark items as recoverable or non-recoverable - Support batch retrieval of recoverable items - Implement CheckpointManager for periodic state snapshots - Track workflow stages and task lifecycle (completed/pending/failed) - Persist checkpoints to enable recovery after crashes - Add OrchestratorWorkflowWithRecovery demonstrating recovery patterns - Structured logging at each workflow step - Retry policies applied to all activity types - Extended ActivityTuning with retry configuration fields Test Coverage: - 8/8 retry policy tests passing - 10/10 deadletter queue tests passing - 10/10 checkpoint manager tests passing - 40 total recovery tests, all passing - All existing tests continue to pass Key Features: - Exponential backoff prevents thundering herd - Deadletter audit trail with timestamps - Checkpoint interval configurable (30s default) - Thread-safe concurrent access - No external dependencies added Closes T1.1
83 lines
2.2 KiB
Go
83 lines
2.2 KiB
Go
package recovery
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
)
|
|
|
|
func TestDefaultRetryPolicy(t *testing.T) {
|
|
policy := DefaultRetryPolicy()
|
|
assert.NotNil(t, policy)
|
|
assert.Equal(t, time.Second, policy.InitialInterval)
|
|
assert.Equal(t, time.Minute, policy.MaximumInterval)
|
|
assert.Equal(t, 2.0, policy.BackoffCoefficient)
|
|
assert.Equal(t, int32(5), policy.MaximumAttempts)
|
|
}
|
|
|
|
func TestActivityRetryPolicy(t *testing.T) {
|
|
policy := ActivityRetryPolicy()
|
|
assert.NotNil(t, policy)
|
|
assert.Equal(t, 2*time.Second, policy.InitialInterval)
|
|
assert.Equal(t, 5*time.Minute, policy.MaximumInterval)
|
|
assert.Equal(t, 2.0, policy.BackoffCoefficient)
|
|
assert.Equal(t, int32(3), policy.MaximumAttempts)
|
|
}
|
|
|
|
func TestLLMActivityRetryPolicy(t *testing.T) {
|
|
policy := LLMActivityRetryPolicy()
|
|
assert.NotNil(t, policy)
|
|
assert.Equal(t, 5*time.Second, policy.InitialInterval)
|
|
assert.Equal(t, 10*time.Minute, policy.MaximumInterval)
|
|
assert.Equal(t, 1.5, policy.BackoffCoefficient)
|
|
assert.Equal(t, int32(5), policy.MaximumAttempts)
|
|
}
|
|
|
|
func TestToTemporalRetryPolicy(t *testing.T) {
|
|
policy := DefaultRetryPolicy()
|
|
temporal := policy.ToTemporalRetryPolicy()
|
|
assert.NotNil(t, temporal)
|
|
assert.Equal(t, time.Second, temporal.InitialInterval)
|
|
assert.Equal(t, time.Minute, temporal.MaximumInterval)
|
|
assert.Equal(t, 2.0, temporal.BackoffCoefficient)
|
|
assert.Equal(t, int32(5), temporal.MaximumAttempts)
|
|
}
|
|
|
|
func TestNilRetryPolicyToTemporal(t *testing.T) {
|
|
var policy *RetryPolicy
|
|
temporal := policy.ToTemporalRetryPolicy()
|
|
assert.Nil(t, temporal)
|
|
}
|
|
|
|
func TestIsRetryableError(t *testing.T) {
|
|
// Nil error is not retryable
|
|
assert.False(t, IsRetryableError(nil))
|
|
|
|
// Generic errors are retryable
|
|
assert.True(t, IsRetryableError(assert.AnError))
|
|
}
|
|
|
|
func TestRetryCount(t *testing.T) {
|
|
rc := RetryCount{Current: 0, Maximum: 3}
|
|
|
|
assert.True(t, rc.CanRetry())
|
|
|
|
rc.Increment()
|
|
assert.Equal(t, 1, rc.Current)
|
|
assert.True(t, rc.CanRetry())
|
|
|
|
rc.Increment()
|
|
rc.Increment()
|
|
assert.Equal(t, 3, rc.Current)
|
|
assert.False(t, rc.CanRetry())
|
|
}
|
|
|
|
func TestRetryCountUnlimited(t *testing.T) {
|
|
rc := RetryCount{Current: 100, Maximum: 0}
|
|
assert.True(t, rc.CanRetry())
|
|
|
|
rc.Increment()
|
|
assert.True(t, rc.CanRetry())
|
|
}
|