feat(T1.3): implement activity timeout tuning automation
- Add internal/tuning package with intelligent timeout analysis - Implement TimeoutAnalyzer for tracking activity execution metrics - Calculate percentile-based timeout recommendations (P95, P99) - Generate confidence scores based on sample size and failure rate - Implement TimeoutLessonsStore for persistent lesson tracking - Store lessons in per-task JSONL files with effectiveness tracking - Generate TimeoutTuningSignal objects for planner integration - Generate human-readable lesson format for planner context - Support three-tier priority signaling (high/medium/low) - Analyze multiple activities concurrently Analysis Features: - Track duration, success/failure, timestamps for each execution - Identify undertuned activities (P99 exceeds timeout) - Detect overtuned activities (timeout > 2x P99) - Calculate confidence scores (40% sample data + 60% reliability) - Generate recommendations with reasoning Lesson Management: - Persist lessons per task in JSONL format - Support lesson effectiveness tracking - Format lessons for planner input - Enable feedback loop for timeout optimization Test Coverage: - 14 analyzer tests (metrics, analysis, persistence) - 22 lessons tests (storage, signals, formatting) - 36 total tuning tests, all passing - Edge cases: empty metrics, all failures, multiple activities Key Design: - P99 + 20% buffer for safe timeout values - Weighted confidence scoring for reliable recommendations - Separation: Analyzer (metrics), Lessons (storage), Signals (integration) - Thread-safe analyzer with RWMutex - No external dependencies added Closes T1.3
This commit is contained in:
Test
@@ -0,0 +1,197 @@
|
||||
package tuning
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TimeoutLesson represents a learned timeout recommendation
|
||||
type TimeoutLesson struct {
|
||||
ActivityType string `json:"activity_type"`
|
||||
OldTimeout time.Duration `json:"old_timeout"`
|
||||
NewTimeout time.Duration `json:"new_timeout"`
|
||||
Reason string `json:"reason"`
|
||||
FailureRate float64 `json:"failure_rate"`
|
||||
SampleSize int `json:"sample_size"`
|
||||
ConfidenceScore float64 `json:"confidence_score"`
|
||||
AppliedAt time.Time `json:"applied_at"`
|
||||
Effective bool `json:"effective"` // Whether recommendation helped
|
||||
}
|
||||
|
||||
// TimeoutLessonsStore manages timeout lessons for task-specific tuning
|
||||
type TimeoutLessonsStore struct {
|
||||
basePath string
|
||||
}
|
||||
|
||||
// NewTimeoutLessonsStore creates a new timeout lessons store
|
||||
func NewTimeoutLessonsStore(basePath string) *TimeoutLessonsStore {
|
||||
return &TimeoutLessonsStore{
|
||||
basePath: basePath,
|
||||
}
|
||||
}
|
||||
|
||||
// AppendLesson appends a timeout lesson to the lessons file
|
||||
func (tls *TimeoutLessonsStore) AppendLesson(taskID string, lesson *TimeoutLesson) error {
|
||||
lessonsDir := filepath.Join(tls.basePath, "tuning", "lessons")
|
||||
|
||||
// Create directory if it doesn't exist
|
||||
if err := os.MkdirAll(lessonsDir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create lessons directory: %w", err)
|
||||
}
|
||||
|
||||
lessonsFile := filepath.Join(lessonsDir, fmt.Sprintf("%s_timeout_lessons.jsonl", taskID))
|
||||
|
||||
// Marshal lesson to JSON
|
||||
data, err := json.Marshal(lesson)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal lesson: %w", err)
|
||||
}
|
||||
|
||||
// Append to file
|
||||
f, err := os.OpenFile(lessonsFile, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open lessons file: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
_, err = f.Write(append(data, '\n'))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to write lesson: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReadLessons reads all timeout lessons for a task
|
||||
func (tls *TimeoutLessonsStore) ReadLessons(taskID string) ([]*TimeoutLesson, error) {
|
||||
lessonsFile := filepath.Join(tls.basePath, "tuning", "lessons", fmt.Sprintf("%s_timeout_lessons.jsonl", taskID))
|
||||
|
||||
// If file doesn't exist, return empty list
|
||||
if _, err := os.Stat(lessonsFile); os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(lessonsFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read lessons file: %w", err)
|
||||
}
|
||||
|
||||
var lessons []*TimeoutLesson
|
||||
content := string(data)
|
||||
|
||||
// Parse JSONL line by line
|
||||
var inLine []byte
|
||||
for _, ch := range []byte(content) {
|
||||
if ch == '\n' {
|
||||
if len(inLine) > 0 {
|
||||
var lesson TimeoutLesson
|
||||
if err := json.Unmarshal(inLine, &lesson); err == nil {
|
||||
lessons = append(lessons, &lesson)
|
||||
}
|
||||
}
|
||||
inLine = nil
|
||||
} else {
|
||||
inLine = append(inLine, ch)
|
||||
}
|
||||
}
|
||||
|
||||
return lessons, nil
|
||||
}
|
||||
|
||||
// GetLatestLesson returns the most recent timeout lesson for a task
|
||||
func (tls *TimeoutLessonsStore) GetLatestLesson(taskID string) (*TimeoutLesson, error) {
|
||||
lessons, err := tls.ReadLessons(taskID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(lessons) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return lessons[len(lessons)-1], nil
|
||||
}
|
||||
|
||||
// GenerateLessonFromRecommendation creates a lesson from a timeout recommendation
|
||||
func GenerateLessonFromRecommendation(rec *TimeoutRecommendation) *TimeoutLesson {
|
||||
if rec == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
failureRate := 0.0
|
||||
if rec.SuccessCount+rec.FailureCount > 0 {
|
||||
failureRate = float64(rec.FailureCount) / float64(rec.SuccessCount+rec.FailureCount)
|
||||
}
|
||||
|
||||
return &TimeoutLesson{
|
||||
ActivityType: rec.ActivityType,
|
||||
OldTimeout: rec.CurrentTimeout,
|
||||
NewTimeout: rec.RecommendedTimeout,
|
||||
Reason: rec.Reason,
|
||||
FailureRate: failureRate,
|
||||
SampleSize: rec.SuccessCount + rec.FailureCount,
|
||||
ConfidenceScore: rec.Confidence,
|
||||
AppliedAt: time.Now(),
|
||||
Effective: false, // To be determined after next run
|
||||
}
|
||||
}
|
||||
|
||||
// FormatLessonsForPlanner formats timeout lessons for planner input
|
||||
func FormatLessonsForPlanner(lessons []*TimeoutLesson) string {
|
||||
if len(lessons) == 0 {
|
||||
return "No timeout lessons available."
|
||||
}
|
||||
|
||||
output := "Recent timeout lessons learned:\n"
|
||||
for i, lesson := range lessons {
|
||||
output += fmt.Sprintf(
|
||||
"\n[Lesson %d] %s:\n Old Timeout: %v → New Timeout: %v\n Reason: %s\n Confidence: %.1f%%\n",
|
||||
i+1,
|
||||
lesson.ActivityType,
|
||||
lesson.OldTimeout,
|
||||
lesson.NewTimeout,
|
||||
lesson.Reason,
|
||||
lesson.ConfidenceScore*100,
|
||||
)
|
||||
}
|
||||
|
||||
return output
|
||||
}
|
||||
|
||||
// TimeoutTuningSignal represents a signal to update timeout tuning
|
||||
type TimeoutTuningSignal struct {
|
||||
ActivityType string `json:"activity_type"`
|
||||
NewTimeout time.Duration `json:"new_timeout"`
|
||||
Reason string `json:"reason"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
Priority string `json:"priority"` // "low", "medium", "high"
|
||||
}
|
||||
|
||||
// GenerateSignalsFromRecommendations generates tuning signals from recommendations
|
||||
func GenerateSignalsFromRecommendations(recommendations []TimeoutRecommendation) []TimeoutTuningSignal {
|
||||
signals := make([]TimeoutTuningSignal, 0)
|
||||
|
||||
for _, rec := range recommendations {
|
||||
priority := "low"
|
||||
if rec.Confidence > 0.7 {
|
||||
priority = "high"
|
||||
} else if rec.Confidence > 0.5 {
|
||||
priority = "medium"
|
||||
}
|
||||
|
||||
signal := TimeoutTuningSignal{
|
||||
ActivityType: rec.ActivityType,
|
||||
NewTimeout: rec.RecommendedTimeout,
|
||||
Reason: rec.Reason,
|
||||
Confidence: rec.Confidence,
|
||||
Priority: priority,
|
||||
}
|
||||
|
||||
signals = append(signals, signal)
|
||||
}
|
||||
|
||||
return signals
|
||||
}
|
||||
Reference in New Issue
Block a user