feat(T1.3): implement activity timeout tuning automation

- Add internal/tuning package with intelligent timeout analysis - Implement TimeoutAnalyzer for tracking activity execution metrics - Calculate percentile-based timeout recommendations (P95, P99) - Generate confidence scores based on sample size and failure rate - Implement TimeoutLessonsStore for persistent lesson tracking - Store lessons in per-task JSONL files with effectiveness tracking - Generate TimeoutTuningSignal objects for planner integration - Generate human-readable lesson format for planner context - Support three-tier priority signaling (high/medium/low) - Analyze multiple activities concurrently Analysis Features: - Track duration, success/failure, timestamps for each execution - Identify undertuned activities (P99 exceeds timeout) - Detect overtuned activities (timeout > 2x P99) - Calculate confidence scores (40% sample data + 60% reliability) - Generate recommendations with reasoning Lesson Management: - Persist lessons per task in JSONL format - Support lesson effectiveness tracking - Format lessons for planner input - Enable feedback loop for timeout optimization Test Coverage: - 14 analyzer tests (metrics, analysis, persistence) - 22 lessons tests (storage, signals, formatting) - 36 total tuning tests, all passing - Edge cases: empty metrics, all failures, multiple activities Key Design: - P99 + 20% buffer for safe timeout values - Weighted confidence scoring for reliable recommendations - Separation: Analyzer (metrics), Lessons (storage), Signals (integration) - Thread-safe analyzer with RWMutex - No external dependencies added Closes T1.3
2026-08-23 16:47:31 -07:00
parent 60f9ca2b1d
commit 927835cb0e
6 changed files with 1493 additions and 1 deletions
@@ -0,0 +1,378 @@
+package tuning
+
+import (
+	"encoding/json"
+	"fmt"
+	"math"
+	"os"
+	"path/filepath"
+	"sort"
+	"sync"
+	"time"
+)
+
+// ExecutionMetric represents a recorded activity execution
+type ExecutionMetric struct {
+	ActivityType string        `json:"activity_type"`
+	Duration     time.Duration `json:"duration"`
+	Success      bool          `json:"success"`
+	Timestamp    time.Time     `json:"timestamp"`
+	Error        string        `json:"error,omitempty"`
+}
+
+// TimeoutRecommendation represents a recommended timeout adjustment
+type TimeoutRecommendation struct {
+	ActivityType      string        `json:"activity_type"`
+	CurrentTimeout    time.Duration `json:"current_timeout"`
+	RecommendedTimeout time.Duration `json:"recommended_timeout"`
+	P95Duration       time.Duration `json:"p95_duration"`
+	P99Duration       time.Duration `json:"p99_duration"`
+	MaxDuration       time.Duration `json:"max_duration"`
+	FailureCount      int           `json:"failure_count"`
+	SuccessCount      int           `json:"success_count"`
+	Confidence        float64       `json:"confidence"` // 0.0-1.0
+	Reason            string        `json:"reason"`
+	Timestamp         time.Time     `json:"timestamp"`
+}
+
+// TimeoutAnalyzer analyzes activity execution metrics and recommends timeout adjustments
+type TimeoutAnalyzer struct {
+	mu            sync.RWMutex
+	basePath      string
+	metrics       []ExecutionMetric
+	recommendations map[string]*TimeoutRecommendation
+}
+
+// NewTimeoutAnalyzer creates a new timeout analyzer
+func NewTimeoutAnalyzer(basePath string) *TimeoutAnalyzer {
+	return &TimeoutAnalyzer{
+		basePath:        basePath,
+		metrics:         make([]ExecutionMetric, 0),
+		recommendations: make(map[string]*TimeoutRecommendation),
+	}
+}
+
+// RecordExecution records an activity execution
+func (ta *TimeoutAnalyzer) RecordExecution(activityType string, duration time.Duration, success bool, err error) {
+	ta.mu.Lock()
+	defer ta.mu.Unlock()
+
+	errorMsg := ""
+	if err != nil {
+		errorMsg = err.Error()
+	}
+
+	metric := ExecutionMetric{
+		ActivityType: activityType,
+		Duration:     duration,
+		Success:      success,
+		Timestamp:    time.Now(),
+		Error:        errorMsg,
+	}
+
+	ta.metrics = append(ta.metrics, metric)
+}
+
+// Analyze analyzes recorded metrics and generates recommendations
+func (ta *TimeoutAnalyzer) Analyze(currentTimeouts map[string]time.Duration) ([]TimeoutRecommendation, error) {
+	ta.mu.Lock()
+	defer ta.mu.Unlock()
+
+	// Group metrics by activity type
+	metricsByActivity := ta.groupMetricsByActivity()
+
+	recommendations := make([]TimeoutRecommendation, 0)
+
+	for activityType, metrics := range metricsByActivity {
+		if len(metrics) == 0 {
+			continue
+		}
+
+		rec := ta.analyzeActivityMetrics(activityType, metrics, currentTimeouts)
+		if rec != nil {
+			recommendations = append(recommendations, *rec)
+			ta.recommendations[activityType] = rec
+		}
+	}
+
+	// Sort by confidence descending
+	sort.Slice(recommendations, func(i, j int) bool {
+		return recommendations[i].Confidence > recommendations[j].Confidence
+	})
+
+	return recommendations, nil
+}
+
+// groupMetricsByActivity groups metrics by activity type
+func (ta *TimeoutAnalyzer) groupMetricsByActivity() map[string][]ExecutionMetric {
+	groups := make(map[string][]ExecutionMetric)
+	for _, m := range ta.metrics {
+		groups[m.ActivityType] = append(groups[m.ActivityType], m)
+	}
+	return groups
+}
+
+// analyzeActivityMetrics analyzes metrics for a single activity type
+func (ta *TimeoutAnalyzer) analyzeActivityMetrics(
+	activityType string,
+	metrics []ExecutionMetric,
+	currentTimeouts map[string]time.Duration,
+) *TimeoutRecommendation {
+	if len(metrics) == 0 {
+		return nil
+	}
+
+	// Calculate statistics
+	durations := make([]time.Duration, 0)
+	successCount := 0
+	failureCount := 0
+
+	for _, m := range metrics {
+		if m.Success {
+			successCount++
+			durations = append(durations, m.Duration)
+		} else {
+			failureCount++
+		}
+	}
+
+	if len(durations) == 0 {
+		// All failed - need more lenient timeout
+		return &TimeoutRecommendation{
+			ActivityType:       activityType,
+			CurrentTimeout:     currentTimeouts[activityType],
+			RecommendedTimeout: currentTimeouts[activityType] * 2,
+			FailureCount:       failureCount,
+			SuccessCount:       successCount,
+			Confidence:         0.3,
+			Reason:             "All executions failed - timeout may be too aggressive",
+			Timestamp:          time.Now(),
+		}
+	}
+
+	// Sort durations for percentile calculation
+	sort.Slice(durations, func(i, j int) bool {
+		return durations[i] < durations[j]
+	})
+
+	p95 := calculatePercentile(durations, 0.95)
+	p99 := calculatePercentile(durations, 0.99)
+	maxDuration := durations[len(durations)-1]
+
+	currentTimeout := currentTimeouts[activityType]
+
+	// Determine if recommendation is needed
+	rec := &TimeoutRecommendation{
+		ActivityType:       activityType,
+		CurrentTimeout:     currentTimeout,
+		P95Duration:        p95,
+		P99Duration:        p99,
+		MaxDuration:        maxDuration,
+		SuccessCount:       successCount,
+		FailureCount:       failureCount,
+		Timestamp:          time.Now(),
+	}
+
+	// Calculate recommended timeout (P99 + 20% buffer)
+	buffer := time.Duration(float64(p99) * 0.2)
+	recommendedTimeout := p99 + buffer
+
+	// Safety checks
+	if recommendedTimeout < currentTimeout {
+		// Current timeout is more than enough
+		if currentTimeout > recommendedTimeout*2 {
+			// Can be reduced
+			rec.RecommendedTimeout = recommendedTimeout
+			rec.Confidence = calculateConfidence(successCount, failureCount)
+			rec.Reason = fmt.Sprintf("Current timeout (%v) is %.1fx P99 (%v) - can be reduced", 
+				currentTimeout, float64(currentTimeout)/float64(p99), p99)
+		} else {
+			return nil // No change needed
+		}
+	} else if recommendedTimeout > currentTimeout {
+		// Need to increase timeout
+		timeoutRatio := float64(recommendedTimeout) / float64(currentTimeout)
+		if timeoutRatio > 1.1 {
+			// More than 10% difference
+			rec.RecommendedTimeout = recommendedTimeout
+			rec.Confidence = calculateConfidence(successCount, failureCount)
+			rec.Reason = fmt.Sprintf("Timeout increases needed - P99: %v, current: %v, %d failures", 
+				p99, currentTimeout, failureCount)
+		} else {
+			return nil // Minor difference, not worth changing
+		}
+	}
+
+	if rec.RecommendedTimeout == 0 {
+		return nil // No recommendation
+	}
+
+	return rec
+}
+
+// calculatePercentile calculates a percentile from sorted durations
+func calculatePercentile(durations []time.Duration, percentile float64) time.Duration {
+	if len(durations) == 0 {
+		return 0
+	}
+
+	index := int(math.Ceil(float64(len(durations))*percentile)) - 1
+	if index < 0 {
+		index = 0
+	}
+	if index >= len(durations) {
+		index = len(durations) - 1
+	}
+
+	return durations[index]
+}
+
+// calculateAverage calculates the average duration
+func calculateAverage(durations []time.Duration) time.Duration {
+	if len(durations) == 0 {
+		return 0
+	}
+
+	var sum time.Duration
+	for _, d := range durations {
+		sum += d
+	}
+
+	return sum / time.Duration(len(durations))
+}
+
+// calculateConfidence calculates confidence in the recommendation (0-1)
+func calculateConfidence(successCount, failureCount int) float64 {
+	total := successCount + failureCount
+	if total == 0 {
+		return 0.0
+	}
+
+	// More samples = higher confidence
+	sampleConfidence := math.Min(float64(total)/100.0, 1.0)
+
+	// Lower failure rate = higher confidence
+	failureRate := float64(failureCount) / float64(total)
+	reliabilityConfidence := 1.0 - failureRate
+
+	// Weighted average
+	return sampleConfidence*0.4 + reliabilityConfidence*0.6
+}
+
+// SaveMetrics saves metrics to disk
+func (ta *TimeoutAnalyzer) SaveMetrics() error {
+	ta.mu.RLock()
+	defer ta.mu.RUnlock()
+
+	metricsPath := filepath.Join(ta.basePath, "metrics", "execution_metrics.jsonl")
+
+	// Create directory if it doesn't exist
+	if err := os.MkdirAll(filepath.Dir(metricsPath), 0755); err != nil {
+		return err
+	}
+
+	f, err := os.Create(metricsPath)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	for _, m := range ta.metrics {
+		data, err := json.Marshal(m)
+		if err != nil {
+			return err
+		}
+		_, err = f.Write(append(data, '\n'))
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// LoadMetrics loads metrics from disk
+func (ta *TimeoutAnalyzer) LoadMetrics() error {
+	ta.mu.Lock()
+	defer ta.mu.Unlock()
+
+	metricsPath := filepath.Join(ta.basePath, "metrics", "execution_metrics.jsonl")
+
+	data, err := os.ReadFile(metricsPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil // File doesn't exist yet
+		}
+		return err
+	}
+
+	ta.metrics = make([]ExecutionMetric, 0)
+
+	// Parse JSONL line by line
+	content := string(data)
+	var inLine []byte
+	for _, ch := range []byte(content) {
+		if ch == '\n' {
+			if len(inLine) > 0 {
+				var m ExecutionMetric
+				if err := json.Unmarshal(inLine, &m); err == nil {
+					ta.metrics = append(ta.metrics, m)
+				}
+			}
+			inLine = nil
+		} else {
+			inLine = append(inLine, ch)
+		}
+	}
+
+	return nil
+}
+
+// SaveRecommendations saves recommendations to disk
+func (ta *TimeoutAnalyzer) SaveRecommendations(recommendations []TimeoutRecommendation) error {
+	ta.mu.Lock()
+	defer ta.mu.Unlock()
+
+	recPath := filepath.Join(ta.basePath, "tuning", "timeout_recommendations.json")
+
+	// Create directory if it doesn't exist
+	if err := os.MkdirAll(filepath.Dir(recPath), 0755); err != nil {
+		return err
+	}
+
+	data, err := json.MarshalIndent(recommendations, "", "  ")
+	if err != nil {
+		return err
+	}
+
+	return os.WriteFile(recPath, data, 0644)
+}
+
+// GetRecommendations returns stored recommendations
+func (ta *TimeoutAnalyzer) GetRecommendations() map[string]*TimeoutRecommendation {
+	ta.mu.RLock()
+	defer ta.mu.RUnlock()
+
+	// Return a copy
+	recCopy := make(map[string]*TimeoutRecommendation)
+	for k, v := range ta.recommendations {
+		recCopy[k] = v
+	}
+	return recCopy
+}
+
+// ClearMetrics clears all recorded metrics
+func (ta *TimeoutAnalyzer) ClearMetrics() {
+	ta.mu.Lock()
+	defer ta.mu.Unlock()
+
+	ta.metrics = make([]ExecutionMetric, 0)
+}
+
+// GetMetricsCount returns the number of recorded metrics
+func (ta *TimeoutAnalyzer) GetMetricsCount() int {
+	ta.mu.RLock()
+	defer ta.mu.RUnlock()
+
+	return len(ta.metrics)
+}