feat(T1.3): implement activity timeout tuning automation

- Add internal/tuning package with intelligent timeout analysis - Implement TimeoutAnalyzer for tracking activity execution metrics - Calculate percentile-based timeout recommendations (P95, P99) - Generate confidence scores based on sample size and failure rate - Implement TimeoutLessonsStore for persistent lesson tracking - Store lessons in per-task JSONL files with effectiveness tracking - Generate TimeoutTuningSignal objects for planner integration - Generate human-readable lesson format for planner context - Support three-tier priority signaling (high/medium/low) - Analyze multiple activities concurrently Analysis Features: - Track duration, success/failure, timestamps for each execution - Identify undertuned activities (P99 exceeds timeout) - Detect overtuned activities (timeout > 2x P99) - Calculate confidence scores (40% sample data + 60% reliability) - Generate recommendations with reasoning Lesson Management: - Persist lessons per task in JSONL format - Support lesson effectiveness tracking - Format lessons for planner input - Enable feedback loop for timeout optimization Test Coverage: - 14 analyzer tests (metrics, analysis, persistence) - 22 lessons tests (storage, signals, formatting) - 36 total tuning tests, all passing - Edge cases: empty metrics, all failures, multiple activities Key Design: - P99 + 20% buffer for safe timeout values - Weighted confidence scoring for reliable recommendations - Separation: Analyzer (metrics), Lessons (storage), Signals (integration) - Thread-safe analyzer with RWMutex - No external dependencies added Closes T1.3
2026-08-23 16:47:31 -07:00
parent 60f9ca2b1d
commit 927835cb0e
6 changed files with 1493 additions and 1 deletions
@@ -0,0 +1,268 @@
+package tuning
+
+import (
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestTimeoutLessonsStore(t *testing.T) {
+	tmpDir := t.TempDir()
+	store := NewTimeoutLessonsStore(tmpDir)
+
+	lesson := &TimeoutLesson{
+		ActivityType:    "activity1",
+		OldTimeout:      5 * time.Second,
+		NewTimeout:      10 * time.Second,
+		Reason:          "P99 exceeded",
+		FailureRate:     0.2,
+		SampleSize:      10,
+		ConfidenceScore: 0.85,
+		AppliedAt:       time.Now(),
+	}
+
+	// Append lesson
+	err := store.AppendLesson("task1", lesson)
+	assert.NoError(t, err)
+
+	// Read lessons
+	lessons, err := store.ReadLessons("task1")
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(lessons))
+	assert.Equal(t, "activity1", lessons[0].ActivityType)
+}
+
+func TestGetLatestLesson(t *testing.T) {
+	tmpDir := t.TempDir()
+	store := NewTimeoutLessonsStore(tmpDir)
+
+	lesson1 := &TimeoutLesson{
+		ActivityType:    "activity1",
+		OldTimeout:      5 * time.Second,
+		NewTimeout:      10 * time.Second,
+		AppliedAt:       time.Now().Add(-1 * time.Hour),
+	}
+
+	lesson2 := &TimeoutLesson{
+		ActivityType:    "activity1",
+		OldTimeout:      10 * time.Second,
+		NewTimeout:      15 * time.Second,
+		AppliedAt:       time.Now(),
+	}
+
+	store.AppendLesson("task1", lesson1)
+	store.AppendLesson("task1", lesson2)
+
+	latest, err := store.GetLatestLesson("task1")
+	assert.NoError(t, err)
+	assert.NotNil(t, latest)
+	assert.Equal(t, 15*time.Second, latest.NewTimeout)
+}
+
+func TestEmptyLessons(t *testing.T) {
+	tmpDir := t.TempDir()
+	store := NewTimeoutLessonsStore(tmpDir)
+
+	lessons, err := store.ReadLessons("nonexistent_task")
+	assert.NoError(t, err)
+	assert.Nil(t, lessons)
+
+	latest, err := store.GetLatestLesson("nonexistent_task")
+	assert.NoError(t, err)
+	assert.Nil(t, latest)
+}
+
+func TestGenerateLessonFromRecommendation(t *testing.T) {
+	rec := &TimeoutRecommendation{
+		ActivityType:       "activity1",
+		CurrentTimeout:     5 * time.Second,
+		RecommendedTimeout: 10 * time.Second,
+		P95Duration:        8 * time.Second,
+		FailureCount:       2,
+		SuccessCount:       8,
+		Confidence:         0.95,
+		Reason:             "P95 exceeded",
+		Timestamp:          time.Now(),
+	}
+
+	lesson := GenerateLessonFromRecommendation(rec)
+	assert.NotNil(t, lesson)
+	assert.Equal(t, "activity1", lesson.ActivityType)
+	assert.Equal(t, 5*time.Second, lesson.OldTimeout)
+	assert.Equal(t, 10*time.Second, lesson.NewTimeout)
+	assert.Equal(t, 0.2, lesson.FailureRate)
+	assert.Equal(t, 10, lesson.SampleSize)
+}
+
+func TestGenerateLessonFromNilRecommendation(t *testing.T) {
+	lesson := GenerateLessonFromRecommendation(nil)
+	assert.Nil(t, lesson)
+}
+
+func TestFormatLessonsForPlanner(t *testing.T) {
+	lessons := []*TimeoutLesson{
+		{
+			ActivityType:    "activity1",
+			OldTimeout:      5 * time.Second,
+			NewTimeout:      10 * time.Second,
+			Reason:          "P95 exceeded",
+			ConfidenceScore: 0.95,
+		},
+		{
+			ActivityType:    "activity2",
+			OldTimeout:      3 * time.Second,
+			NewTimeout:      6 * time.Second,
+			Reason:          "Timeout too tight",
+			ConfidenceScore: 0.75,
+		},
+	}
+
+	formatted := FormatLessonsForPlanner(lessons)
+	assert.Contains(t, formatted, "activity1")
+	assert.Contains(t, formatted, "activity2")
+	assert.Contains(t, formatted, "P95 exceeded")
+	assert.Contains(t, formatted, "95.0%")
+}
+
+func TestFormatEmptyLessons(t *testing.T) {
+	formatted := FormatLessonsForPlanner(nil)
+	assert.Equal(t, "No timeout lessons available.", formatted)
+
+	formatted = FormatLessonsForPlanner([]*TimeoutLesson{})
+	assert.Equal(t, "No timeout lessons available.", formatted)
+}
+
+func TestGenerateSignalsFromRecommendations(t *testing.T) {
+	recommendations := []TimeoutRecommendation{
+		{
+			ActivityType:       "activity1",
+			RecommendedTimeout: 10 * time.Second,
+			Reason:             "P95 exceeded",
+			Confidence:         0.95,
+		},
+		{
+			ActivityType:       "activity2",
+			RecommendedTimeout: 5 * time.Second,
+			Reason:             "Timeout reduced",
+			Confidence:         0.55,
+		},
+		{
+			ActivityType:       "activity3",
+			RecommendedTimeout: 3 * time.Second,
+			Reason:             "Low priority",
+			Confidence:         0.45,
+		},
+	}
+
+	signals := GenerateSignalsFromRecommendations(recommendations)
+	assert.Equal(t, 3, len(signals))
+
+	// Check priority levels
+	assert.Equal(t, "high", signals[0].Priority)
+	assert.Equal(t, "medium", signals[1].Priority)
+	assert.Equal(t, "low", signals[2].Priority)
+}
+
+func TestSignalStructure(t *testing.T) {
+	recommendations := []TimeoutRecommendation{
+		{
+			ActivityType:       "activity1",
+			CurrentTimeout:     5 * time.Second,
+			RecommendedTimeout: 10 * time.Second,
+			Reason:             "P95 exceeded",
+			Confidence:         0.85,
+		},
+	}
+
+	signals := GenerateSignalsFromRecommendations(recommendations)
+	assert.Greater(t, len(signals), 0)
+
+	signal := signals[0]
+	assert.Equal(t, "activity1", signal.ActivityType)
+	assert.Equal(t, 10*time.Second, signal.NewTimeout)
+	assert.Equal(t, "P95 exceeded", signal.Reason)
+	assert.Equal(t, 0.85, signal.Confidence)
+}
+
+func TestMultipleLessonAppends(t *testing.T) {
+	tmpDir := t.TempDir()
+	store := NewTimeoutLessonsStore(tmpDir)
+
+	// Append multiple lessons
+	for i := 0; i < 5; i++ {
+		lesson := &TimeoutLesson{
+			ActivityType:    "activity1",
+			OldTimeout:      time.Duration(i*5) * time.Second,
+			NewTimeout:      time.Duration((i+1)*5) * time.Second,
+		}
+		err := store.AppendLesson("task1", lesson)
+		assert.NoError(t, err)
+	}
+
+	lessons, err := store.ReadLessons("task1")
+	assert.NoError(t, err)
+	assert.Equal(t, 5, len(lessons))
+}
+
+func TestLessonPersistence(t *testing.T) {
+	tmpDir := t.TempDir()
+	store1 := NewTimeoutLessonsStore(tmpDir)
+
+	lesson := &TimeoutLesson{
+		ActivityType:    "activity1",
+		OldTimeout:      5 * time.Second,
+		NewTimeout:      10 * time.Second,
+	}
+
+	store1.AppendLesson("task1", lesson)
+
+	// Create new store instance
+	store2 := NewTimeoutLessonsStore(tmpDir)
+	lessons, err := store2.ReadLessons("task1")
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(lessons))
+	assert.Equal(t, 10*time.Second, lessons[0].NewTimeout)
+}
+
+func TestLessonEffectivenessTracking(t *testing.T) {
+	lesson := &TimeoutLesson{
+		ActivityType:    "activity1",
+		OldTimeout:      5 * time.Second,
+		NewTimeout:      10 * time.Second,
+		Effective:       false,
+	}
+
+	assert.False(t, lesson.Effective)
+
+	lesson.Effective = true
+	assert.True(t, lesson.Effective)
+}
+
+func TestHighConfidenceSignal(t *testing.T) {
+	recommendations := []TimeoutRecommendation{
+		{
+			ActivityType:       "activity1",
+			RecommendedTimeout: 10 * time.Second,
+			Reason:             "Very confident",
+			Confidence:         0.99,
+		},
+	}
+
+	signals := GenerateSignalsFromRecommendations(recommendations)
+	assert.Equal(t, "high", signals[0].Priority)
+}
+
+func TestLessonFileLayout(t *testing.T) {
+	tmpDir := t.TempDir()
+	store := NewTimeoutLessonsStore(tmpDir)
+
+	store.AppendLesson("task1", &TimeoutLesson{
+		ActivityType: "activity1",
+	})
+
+	// Verify file layout
+	expectedPath := filepath.Join(tmpDir, "tuning", "lessons", "task1_timeout_lessons.jsonl")
+	assert.DirExists(t, filepath.Dir(expectedPath))
+}