package tuning import ( "encoding/json" "fmt" "math" "os" "path/filepath" "sort" "sync" "time" ) // ExecutionMetric represents a recorded activity execution type ExecutionMetric struct { ActivityType string `json:"activity_type"` Duration time.Duration `json:"duration"` Success bool `json:"success"` Timestamp time.Time `json:"timestamp"` Error string `json:"error,omitempty"` } // TimeoutRecommendation represents a recommended timeout adjustment type TimeoutRecommendation struct { ActivityType string `json:"activity_type"` CurrentTimeout time.Duration `json:"current_timeout"` RecommendedTimeout time.Duration `json:"recommended_timeout"` P95Duration time.Duration `json:"p95_duration"` P99Duration time.Duration `json:"p99_duration"` MaxDuration time.Duration `json:"max_duration"` FailureCount int `json:"failure_count"` SuccessCount int `json:"success_count"` Confidence float64 `json:"confidence"` // 0.0-1.0 Reason string `json:"reason"` Timestamp time.Time `json:"timestamp"` } // TimeoutAnalyzer analyzes activity execution metrics and recommends timeout adjustments type TimeoutAnalyzer struct { mu sync.RWMutex basePath string metrics []ExecutionMetric recommendations map[string]*TimeoutRecommendation } // NewTimeoutAnalyzer creates a new timeout analyzer func NewTimeoutAnalyzer(basePath string) *TimeoutAnalyzer { return &TimeoutAnalyzer{ basePath: basePath, metrics: make([]ExecutionMetric, 0), recommendations: make(map[string]*TimeoutRecommendation), } } // RecordExecution records an activity execution func (ta *TimeoutAnalyzer) RecordExecution(activityType string, duration time.Duration, success bool, err error) { ta.mu.Lock() defer ta.mu.Unlock() errorMsg := "" if err != nil { errorMsg = err.Error() } metric := ExecutionMetric{ ActivityType: activityType, Duration: duration, Success: success, Timestamp: time.Now(), Error: errorMsg, } ta.metrics = append(ta.metrics, metric) } // Analyze analyzes recorded metrics and generates recommendations func (ta *TimeoutAnalyzer) Analyze(currentTimeouts map[string]time.Duration) ([]TimeoutRecommendation, error) { ta.mu.Lock() defer ta.mu.Unlock() // Group metrics by activity type metricsByActivity := ta.groupMetricsByActivity() recommendations := make([]TimeoutRecommendation, 0) for activityType, metrics := range metricsByActivity { if len(metrics) == 0 { continue } rec := ta.analyzeActivityMetrics(activityType, metrics, currentTimeouts) if rec != nil { recommendations = append(recommendations, *rec) ta.recommendations[activityType] = rec } } // Sort by confidence descending sort.Slice(recommendations, func(i, j int) bool { return recommendations[i].Confidence > recommendations[j].Confidence }) return recommendations, nil } // groupMetricsByActivity groups metrics by activity type func (ta *TimeoutAnalyzer) groupMetricsByActivity() map[string][]ExecutionMetric { groups := make(map[string][]ExecutionMetric) for _, m := range ta.metrics { groups[m.ActivityType] = append(groups[m.ActivityType], m) } return groups } // analyzeActivityMetrics analyzes metrics for a single activity type func (ta *TimeoutAnalyzer) analyzeActivityMetrics( activityType string, metrics []ExecutionMetric, currentTimeouts map[string]time.Duration, ) *TimeoutRecommendation { if len(metrics) == 0 { return nil } // Calculate statistics durations := make([]time.Duration, 0) successCount := 0 failureCount := 0 for _, m := range metrics { if m.Success { successCount++ durations = append(durations, m.Duration) } else { failureCount++ } } if len(durations) == 0 { // All failed - need more lenient timeout return &TimeoutRecommendation{ ActivityType: activityType, CurrentTimeout: currentTimeouts[activityType], RecommendedTimeout: currentTimeouts[activityType] * 2, FailureCount: failureCount, SuccessCount: successCount, Confidence: 0.3, Reason: "All executions failed - timeout may be too aggressive", Timestamp: time.Now(), } } // Sort durations for percentile calculation sort.Slice(durations, func(i, j int) bool { return durations[i] < durations[j] }) p95 := calculatePercentile(durations, 0.95) p99 := calculatePercentile(durations, 0.99) maxDuration := durations[len(durations)-1] currentTimeout := currentTimeouts[activityType] // Determine if recommendation is needed rec := &TimeoutRecommendation{ ActivityType: activityType, CurrentTimeout: currentTimeout, P95Duration: p95, P99Duration: p99, MaxDuration: maxDuration, SuccessCount: successCount, FailureCount: failureCount, Timestamp: time.Now(), } // Calculate recommended timeout (P99 + 20% buffer) buffer := time.Duration(float64(p99) * 0.2) recommendedTimeout := p99 + buffer // Safety checks if recommendedTimeout < currentTimeout { // Current timeout is more than enough if currentTimeout > recommendedTimeout*2 { // Can be reduced rec.RecommendedTimeout = recommendedTimeout rec.Confidence = calculateConfidence(successCount, failureCount) rec.Reason = fmt.Sprintf("Current timeout (%v) is %.1fx P99 (%v) - can be reduced", currentTimeout, float64(currentTimeout)/float64(p99), p99) } else { return nil // No change needed } } else if recommendedTimeout > currentTimeout { // Need to increase timeout timeoutRatio := float64(recommendedTimeout) / float64(currentTimeout) if timeoutRatio > 1.1 { // More than 10% difference rec.RecommendedTimeout = recommendedTimeout rec.Confidence = calculateConfidence(successCount, failureCount) rec.Reason = fmt.Sprintf("Timeout increases needed - P99: %v, current: %v, %d failures", p99, currentTimeout, failureCount) } else { return nil // Minor difference, not worth changing } } if rec.RecommendedTimeout == 0 { return nil // No recommendation } return rec } // calculatePercentile calculates a percentile from sorted durations func calculatePercentile(durations []time.Duration, percentile float64) time.Duration { if len(durations) == 0 { return 0 } index := int(math.Ceil(float64(len(durations))*percentile)) - 1 if index < 0 { index = 0 } if index >= len(durations) { index = len(durations) - 1 } return durations[index] } // calculateAverage calculates the average duration func calculateAverage(durations []time.Duration) time.Duration { if len(durations) == 0 { return 0 } var sum time.Duration for _, d := range durations { sum += d } return sum / time.Duration(len(durations)) } // calculateConfidence calculates confidence in the recommendation (0-1) func calculateConfidence(successCount, failureCount int) float64 { total := successCount + failureCount if total == 0 { return 0.0 } // More samples = higher confidence sampleConfidence := math.Min(float64(total)/100.0, 1.0) // Lower failure rate = higher confidence failureRate := float64(failureCount) / float64(total) reliabilityConfidence := 1.0 - failureRate // Weighted average return sampleConfidence*0.4 + reliabilityConfidence*0.6 } // SaveMetrics saves metrics to disk func (ta *TimeoutAnalyzer) SaveMetrics() error { ta.mu.RLock() defer ta.mu.RUnlock() metricsPath := filepath.Join(ta.basePath, "metrics", "execution_metrics.jsonl") // Create directory if it doesn't exist if err := os.MkdirAll(filepath.Dir(metricsPath), 0755); err != nil { return err } f, err := os.Create(metricsPath) if err != nil { return err } defer f.Close() for _, m := range ta.metrics { data, err := json.Marshal(m) if err != nil { return err } _, err = f.Write(append(data, '\n')) if err != nil { return err } } return nil } // LoadMetrics loads metrics from disk func (ta *TimeoutAnalyzer) LoadMetrics() error { ta.mu.Lock() defer ta.mu.Unlock() metricsPath := filepath.Join(ta.basePath, "metrics", "execution_metrics.jsonl") data, err := os.ReadFile(metricsPath) if err != nil { if os.IsNotExist(err) { return nil // File doesn't exist yet } return err } ta.metrics = make([]ExecutionMetric, 0) // Parse JSONL line by line content := string(data) var inLine []byte for _, ch := range []byte(content) { if ch == '\n' { if len(inLine) > 0 { var m ExecutionMetric if err := json.Unmarshal(inLine, &m); err == nil { ta.metrics = append(ta.metrics, m) } } inLine = nil } else { inLine = append(inLine, ch) } } return nil } // SaveRecommendations saves recommendations to disk func (ta *TimeoutAnalyzer) SaveRecommendations(recommendations []TimeoutRecommendation) error { ta.mu.Lock() defer ta.mu.Unlock() recPath := filepath.Join(ta.basePath, "tuning", "timeout_recommendations.json") // Create directory if it doesn't exist if err := os.MkdirAll(filepath.Dir(recPath), 0755); err != nil { return err } data, err := json.MarshalIndent(recommendations, "", " ") if err != nil { return err } return os.WriteFile(recPath, data, 0644) } // GetRecommendations returns stored recommendations func (ta *TimeoutAnalyzer) GetRecommendations() map[string]*TimeoutRecommendation { ta.mu.RLock() defer ta.mu.RUnlock() // Return a copy recCopy := make(map[string]*TimeoutRecommendation) for k, v := range ta.recommendations { recCopy[k] = v } return recCopy } // ClearMetrics clears all recorded metrics func (ta *TimeoutAnalyzer) ClearMetrics() { ta.mu.Lock() defer ta.mu.Unlock() ta.metrics = make([]ExecutionMetric, 0) } // GetMetricsCount returns the number of recorded metrics func (ta *TimeoutAnalyzer) GetMetricsCount() int { ta.mu.RLock() defer ta.mu.RUnlock() return len(ta.metrics) }