feat(T1.3): implement activity timeout tuning automation
- Add internal/tuning package with intelligent timeout analysis - Implement TimeoutAnalyzer for tracking activity execution metrics - Calculate percentile-based timeout recommendations (P95, P99) - Generate confidence scores based on sample size and failure rate - Implement TimeoutLessonsStore for persistent lesson tracking - Store lessons in per-task JSONL files with effectiveness tracking - Generate TimeoutTuningSignal objects for planner integration - Generate human-readable lesson format for planner context - Support three-tier priority signaling (high/medium/low) - Analyze multiple activities concurrently Analysis Features: - Track duration, success/failure, timestamps for each execution - Identify undertuned activities (P99 exceeds timeout) - Detect overtuned activities (timeout > 2x P99) - Calculate confidence scores (40% sample data + 60% reliability) - Generate recommendations with reasoning Lesson Management: - Persist lessons per task in JSONL format - Support lesson effectiveness tracking - Format lessons for planner input - Enable feedback loop for timeout optimization Test Coverage: - 14 analyzer tests (metrics, analysis, persistence) - 22 lessons tests (storage, signals, formatting) - 36 total tuning tests, all passing - Edge cases: empty metrics, all failures, multiple activities Key Design: - P99 + 20% buffer for safe timeout values - Weighted confidence scoring for reliable recommendations - Separation: Analyzer (metrics), Lessons (storage), Signals (integration) - Thread-safe analyzer with RWMutex - No external dependencies added Closes T1.3
This commit is contained in:
Test
@@ -0,0 +1,278 @@
|
||||
package tuning
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestTimeoutAnalyzer(t *testing.T) {
|
||||
ta := NewTimeoutAnalyzer(t.TempDir())
|
||||
|
||||
// Record some metrics
|
||||
ta.RecordExecution("activity1", 1*time.Second, true, nil)
|
||||
ta.RecordExecution("activity1", 2*time.Second, true, nil)
|
||||
ta.RecordExecution("activity1", 3*time.Second, true, nil)
|
||||
|
||||
assert.Equal(t, 3, ta.GetMetricsCount())
|
||||
}
|
||||
|
||||
func TestAnalyzeMetrics(t *testing.T) {
|
||||
ta := NewTimeoutAnalyzer(t.TempDir())
|
||||
|
||||
// Record metrics with P95 around 9s
|
||||
for i := 1; i <= 20; i++ {
|
||||
duration := time.Duration(i) * time.Second
|
||||
ta.RecordExecution("activity1", duration, true, nil)
|
||||
}
|
||||
|
||||
currentTimeouts := map[string]time.Duration{
|
||||
"activity1": 5 * time.Second,
|
||||
}
|
||||
|
||||
recommendations, err := ta.Analyze(currentTimeouts)
|
||||
assert.NoError(t, err)
|
||||
assert.Greater(t, len(recommendations), 0)
|
||||
|
||||
rec := recommendations[0]
|
||||
assert.Equal(t, "activity1", rec.ActivityType)
|
||||
assert.Equal(t, 5*time.Second, rec.CurrentTimeout)
|
||||
assert.Greater(t, rec.RecommendedTimeout, rec.CurrentTimeout)
|
||||
}
|
||||
|
||||
func TestAnalyzeWithFailures(t *testing.T) {
|
||||
ta := NewTimeoutAnalyzer(t.TempDir())
|
||||
|
||||
// Record some failures
|
||||
for i := 0; i < 5; i++ {
|
||||
ta.RecordExecution("slow_activity", 10*time.Second, false, assert.AnError)
|
||||
}
|
||||
|
||||
currentTimeouts := map[string]time.Duration{
|
||||
"slow_activity": 5 * time.Second,
|
||||
}
|
||||
|
||||
recommendations, err := ta.Analyze(currentTimeouts)
|
||||
assert.NoError(t, err)
|
||||
|
||||
if len(recommendations) > 0 {
|
||||
rec := recommendations[0]
|
||||
assert.Equal(t, 5, rec.FailureCount)
|
||||
assert.Greater(t, rec.RecommendedTimeout, rec.CurrentTimeout)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCalculatePercentile(t *testing.T) {
|
||||
durations := []time.Duration{
|
||||
1 * time.Second,
|
||||
2 * time.Second,
|
||||
3 * time.Second,
|
||||
4 * time.Second,
|
||||
5 * time.Second,
|
||||
6 * time.Second,
|
||||
7 * time.Second,
|
||||
8 * time.Second,
|
||||
9 * time.Second,
|
||||
10 * time.Second,
|
||||
}
|
||||
|
||||
p95 := calculatePercentile(durations, 0.95)
|
||||
assert.NotZero(t, p95)
|
||||
assert.LessOrEqual(t, p95, 10*time.Second)
|
||||
|
||||
p99 := calculatePercentile(durations, 0.99)
|
||||
assert.NotZero(t, p99)
|
||||
assert.GreaterOrEqual(t, p99, p95)
|
||||
}
|
||||
|
||||
func TestCalculateAverage(t *testing.T) {
|
||||
durations := []time.Duration{
|
||||
1 * time.Second,
|
||||
2 * time.Second,
|
||||
3 * time.Second,
|
||||
}
|
||||
|
||||
avg := calculateAverage(durations)
|
||||
assert.Equal(t, 2*time.Second, avg)
|
||||
}
|
||||
|
||||
func TestCalculateConfidence(t *testing.T) {
|
||||
// Perfect success
|
||||
conf := calculateConfidence(100, 0)
|
||||
assert.Equal(t, 1.0, conf)
|
||||
|
||||
// 50% success
|
||||
conf = calculateConfidence(50, 50)
|
||||
assert.Greater(t, conf, 0.0)
|
||||
assert.Less(t, conf, 1.0)
|
||||
|
||||
// All failures
|
||||
conf = calculateConfidence(0, 100)
|
||||
assert.Less(t, conf, 1.0)
|
||||
}
|
||||
|
||||
func TestGroupMetricsByActivity(t *testing.T) {
|
||||
ta := NewTimeoutAnalyzer(t.TempDir())
|
||||
|
||||
ta.RecordExecution("activity1", 1*time.Second, true, nil)
|
||||
ta.RecordExecution("activity1", 2*time.Second, true, nil)
|
||||
ta.RecordExecution("activity2", 3*time.Second, true, nil)
|
||||
|
||||
groups := ta.groupMetricsByActivity()
|
||||
assert.Equal(t, 2, len(groups))
|
||||
assert.Equal(t, 2, len(groups["activity1"]))
|
||||
assert.Equal(t, 1, len(groups["activity2"]))
|
||||
}
|
||||
|
||||
func TestClearMetrics(t *testing.T) {
|
||||
ta := NewTimeoutAnalyzer(t.TempDir())
|
||||
|
||||
ta.RecordExecution("activity1", 1*time.Second, true, nil)
|
||||
assert.Equal(t, 1, ta.GetMetricsCount())
|
||||
|
||||
ta.ClearMetrics()
|
||||
assert.Equal(t, 0, ta.GetMetricsCount())
|
||||
}
|
||||
|
||||
func TestGetRecommendations(t *testing.T) {
|
||||
ta := NewTimeoutAnalyzer(t.TempDir())
|
||||
|
||||
ta.RecordExecution("activity1", 1*time.Second, true, nil)
|
||||
ta.RecordExecution("activity1", 2*time.Second, true, nil)
|
||||
|
||||
currentTimeouts := map[string]time.Duration{
|
||||
"activity1": 5 * time.Second,
|
||||
}
|
||||
|
||||
ta.Analyze(currentTimeouts)
|
||||
recs := ta.GetRecommendations()
|
||||
assert.IsType(t, make(map[string]*TimeoutRecommendation), recs)
|
||||
}
|
||||
|
||||
func TestRecommendationStructure(t *testing.T) {
|
||||
ta := NewTimeoutAnalyzer(t.TempDir())
|
||||
|
||||
// Record consistent executions
|
||||
for i := 0; i < 10; i++ {
|
||||
ta.RecordExecution("activity1", 5*time.Second, true, nil)
|
||||
}
|
||||
|
||||
currentTimeouts := map[string]time.Duration{
|
||||
"activity1": 2 * time.Second, // Too tight
|
||||
}
|
||||
|
||||
recommendations, err := ta.Analyze(currentTimeouts)
|
||||
assert.NoError(t, err)
|
||||
|
||||
if len(recommendations) > 0 {
|
||||
rec := recommendations[0]
|
||||
assert.NotEmpty(t, rec.ActivityType)
|
||||
assert.NotZero(t, rec.CurrentTimeout)
|
||||
assert.NotZero(t, rec.P95Duration)
|
||||
assert.Greater(t, rec.SuccessCount, 0)
|
||||
assert.NotEmpty(t, rec.Reason)
|
||||
assert.Greater(t, rec.Confidence, 0.0)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipleActivities(t *testing.T) {
|
||||
ta := NewTimeoutAnalyzer(t.TempDir())
|
||||
|
||||
// Record metrics for multiple activities
|
||||
for i := 0; i < 10; i++ {
|
||||
ta.RecordExecution("fast_activity", time.Duration(i+1)*time.Second, true, nil)
|
||||
ta.RecordExecution("slow_activity", time.Duration(i+10)*time.Second, true, nil)
|
||||
}
|
||||
|
||||
currentTimeouts := map[string]time.Duration{
|
||||
"fast_activity": 3 * time.Second,
|
||||
"slow_activity": 5 * time.Second,
|
||||
}
|
||||
|
||||
recommendations, err := ta.Analyze(currentTimeouts)
|
||||
assert.NoError(t, err)
|
||||
assert.Greater(t, len(recommendations), 0)
|
||||
|
||||
// Check that we get recommendations for both activities
|
||||
hasSlowActivity := false
|
||||
for _, rec := range recommendations {
|
||||
if rec.ActivityType == "slow_activity" {
|
||||
hasSlowActivity = true
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.True(t, hasSlowActivity)
|
||||
}
|
||||
|
||||
func TestEmptyMetrics(t *testing.T) {
|
||||
ta := NewTimeoutAnalyzer(t.TempDir())
|
||||
|
||||
currentTimeouts := map[string]time.Duration{
|
||||
"activity1": 5 * time.Second,
|
||||
}
|
||||
|
||||
recommendations, err := ta.Analyze(currentTimeouts)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 0, len(recommendations))
|
||||
}
|
||||
|
||||
func TestAllFailures(t *testing.T) {
|
||||
ta := NewTimeoutAnalyzer(t.TempDir())
|
||||
|
||||
// Record only failures
|
||||
for i := 0; i < 5; i++ {
|
||||
ta.RecordExecution("activity1", 1*time.Second, false, assert.AnError)
|
||||
}
|
||||
|
||||
currentTimeouts := map[string]time.Duration{
|
||||
"activity1": 5 * time.Second,
|
||||
}
|
||||
|
||||
recommendations, err := ta.Analyze(currentTimeouts)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Should recommend increase despite no successes
|
||||
if len(recommendations) > 0 {
|
||||
rec := recommendations[0]
|
||||
assert.Equal(t, 5, rec.FailureCount)
|
||||
assert.Equal(t, 0, rec.SuccessCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSaveAndLoadMetrics(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
ta1 := NewTimeoutAnalyzer(tmpDir)
|
||||
|
||||
// Record and save
|
||||
ta1.RecordExecution("activity1", 1*time.Second, true, nil)
|
||||
ta1.RecordExecution("activity1", 2*time.Second, true, nil)
|
||||
|
||||
err := ta1.SaveMetrics()
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Load in new analyzer
|
||||
ta2 := NewTimeoutAnalyzer(tmpDir)
|
||||
err = ta2.LoadMetrics()
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Equal(t, ta1.GetMetricsCount(), ta2.GetMetricsCount())
|
||||
}
|
||||
|
||||
func TestSaveRecommendations(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
ta := NewTimeoutAnalyzer(tmpDir)
|
||||
|
||||
recommendations := []TimeoutRecommendation{
|
||||
{
|
||||
ActivityType: "activity1",
|
||||
CurrentTimeout: 5 * time.Second,
|
||||
RecommendedTimeout: 10 * time.Second,
|
||||
Confidence: 0.95,
|
||||
Timestamp: time.Now(),
|
||||
},
|
||||
}
|
||||
|
||||
err := ta.SaveRecommendations(recommendations)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
Reference in New Issue
Block a user