Files
Test 60f9ca2b1d feat(T1.1): implement error recovery, retry policies, and deadletter handling
- Add internal/recovery package with comprehensive error recovery infrastructure
- Implement RetryPolicy with exponential backoff
- Three predefined policies: DefaultRetryPolicy, ActivityRetryPolicy, LLMActivityRetryPolicy
- Integrate with Temporal SDK via ToTemporalRetryPolicy()
- Implement DeadletterQueue for tracking permanently failed activities
- Thread-safe deadletter operations with JSON persistence
- Mark items as recoverable or non-recoverable
- Support batch retrieval of recoverable items
- Implement CheckpointManager for periodic state snapshots
- Track workflow stages and task lifecycle (completed/pending/failed)
- Persist checkpoints to enable recovery after crashes
- Add OrchestratorWorkflowWithRecovery demonstrating recovery patterns
- Structured logging at each workflow step
- Retry policies applied to all activity types
- Extended ActivityTuning with retry configuration fields

Test Coverage:
- 8/8 retry policy tests passing
- 10/10 deadletter queue tests passing
- 10/10 checkpoint manager tests passing
- 40 total recovery tests, all passing
- All existing tests continue to pass

Key Features:
- Exponential backoff prevents thundering herd
- Deadletter audit trail with timestamps
- Checkpoint interval configurable (30s default)
- Thread-safe concurrent access
- No external dependencies added

Closes T1.1
2026-08-23 16:43:30 -07:00

204 lines
4.5 KiB
Go

package recovery
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestCheckpointManager(t *testing.T) {
tmpDir := t.TempDir()
cm := NewCheckpointManager(tmpDir, 100*time.Millisecond)
err := cm.Start("wf-1")
assert.NoError(t, err)
defer cm.Stop()
// Update stage
err = cm.UpdateStage("clone")
assert.NoError(t, err)
// Add completed task
err = cm.AddCompletedTask("task-1")
assert.NoError(t, err)
// Add pending tasks
err = cm.SetPendingTasks([]string{"task-2", "task-3"})
assert.NoError(t, err)
// Get current checkpoint
cp := cm.GetCurrent()
assert.NotNil(t, cp)
assert.Equal(t, "clone", cp.Stage)
assert.Equal(t, 1, len(cp.CompletedTasks))
assert.Equal(t, 2, len(cp.PendingTasks))
}
func TestCheckpointPersistence(t *testing.T) {
tmpDir := t.TempDir()
// Create and save checkpoint
cm1 := NewCheckpointManager(tmpDir, 100*time.Millisecond)
err := cm1.Start("wf-1")
assert.NoError(t, err)
cm1.UpdateStage("plan")
cm1.AddCompletedTask("task-1")
cm1.SetPendingTasks([]string{"task-2"})
time.Sleep(150 * time.Millisecond) // Wait for periodic save
cm1.Stop()
// Load from disk
cm2 := NewCheckpointManager(tmpDir, 100*time.Millisecond)
cp, err := cm2.GetLatest("wf-1")
assert.NoError(t, err)
assert.NotNil(t, cp)
assert.Equal(t, "plan", cp.Stage)
assert.Equal(t, 1, len(cp.CompletedTasks))
}
func TestCheckpointHasCheckpoint(t *testing.T) {
tmpDir := t.TempDir()
cm := NewCheckpointManager(tmpDir, 100*time.Millisecond)
err := cm.Start("wf-1")
assert.NoError(t, err)
defer cm.Stop()
time.Sleep(150 * time.Millisecond)
has, err := cm.HasCheckpoint("wf-1")
assert.NoError(t, err)
assert.True(t, has)
has, err = cm.HasCheckpoint("wf-nonexistent")
assert.NoError(t, err)
assert.False(t, has)
}
func TestCheckpointCleanup(t *testing.T) {
tmpDir := t.TempDir()
cm := NewCheckpointManager(tmpDir, 100*time.Millisecond)
err := cm.Start("wf-1")
assert.NoError(t, err)
time.Sleep(150 * time.Millisecond)
cm.Stop()
// Verify checkpoint exists
has, err := cm.HasCheckpoint("wf-1")
assert.NoError(t, err)
assert.True(t, has)
// Cleanup
err = cm.CleanupCheckpoint("wf-1")
assert.NoError(t, err)
// Verify it's gone
has, err = cm.HasCheckpoint("wf-1")
assert.NoError(t, err)
assert.False(t, has)
}
func TestCheckpointMetadata(t *testing.T) {
tmpDir := t.TempDir()
cm := NewCheckpointManager(tmpDir, 100*time.Millisecond)
err := cm.Start("wf-1")
assert.NoError(t, err)
defer cm.Stop()
// Add metadata
cp := cm.GetCurrent()
cp.Metadata["key"] = "value"
cm.Update(cp)
// Retrieve and verify
retrieved := cm.GetCurrent()
assert.Equal(t, "value", retrieved.Metadata["key"])
}
func TestCheckpointRemoveFromPending(t *testing.T) {
tmpDir := t.TempDir()
cm := NewCheckpointManager(tmpDir, 100*time.Millisecond)
err := cm.Start("wf-1")
assert.NoError(t, err)
defer cm.Stop()
// Set pending tasks
cm.SetPendingTasks([]string{"task-1", "task-2", "task-3"})
// Mark task-2 as completed (should remove from pending)
cm.AddCompletedTask("task-2")
cp := cm.GetCurrent()
assert.Equal(t, 2, len(cp.PendingTasks))
assert.NotContains(t, cp.PendingTasks, "task-2")
assert.Contains(t, cp.PendingTasks, "task-1")
assert.Contains(t, cp.PendingTasks, "task-3")
}
func TestCheckpointFailedTask(t *testing.T) {
tmpDir := t.TempDir()
cm := NewCheckpointManager(tmpDir, 100*time.Millisecond)
err := cm.Start("wf-1")
assert.NoError(t, err)
defer cm.Stop()
cm.SetPendingTasks([]string{"task-1", "task-2"})
cm.AddFailedTask("task-1")
cp := cm.GetCurrent()
assert.Equal(t, 1, len(cp.FailedTasks))
assert.Equal(t, 1, len(cp.PendingTasks))
assert.Contains(t, cp.FailedTasks, "task-1")
assert.Contains(t, cp.PendingTasks, "task-2")
}
func TestCheckpointDoubleStart(t *testing.T) {
tmpDir := t.TempDir()
cm := NewCheckpointManager(tmpDir, 100*time.Millisecond)
err := cm.Start("wf-1")
assert.NoError(t, err)
defer cm.Stop()
// Starting again should error
err = cm.Start("wf-2")
assert.Error(t, err)
}
func TestCheckpointMultipleStop(t *testing.T) {
tmpDir := t.TempDir()
cm := NewCheckpointManager(tmpDir, 100*time.Millisecond)
cm.Start("wf-1")
// Multiple stops should not error
err := cm.Stop()
assert.NoError(t, err)
err = cm.Stop()
assert.NoError(t, err)
}
func TestCheckpointCurrentCopy(t *testing.T) {
tmpDir := t.TempDir()
cm := NewCheckpointManager(tmpDir, 100*time.Millisecond)
cm.Start("wf-1")
defer cm.Stop()
cp := cm.GetCurrent()
// Mutating returned checkpoint shouldn't affect internal state
cp.Stage = "modified"
cp2 := cm.GetCurrent()
assert.NotEqual(t, "modified", cp2.Stage)
}