feat(T4.1-T4.4): implement advanced operations & analytics (part 1)
T4.1: Real-time Metrics Dashboard - Add internal/dashboard package with MetricsAggregator - Record, aggregate, and query metrics - Percentile calculations (p50, p95, p99) - Time-series data with max size eviction - 13 metrics tests, all passing T4.2: Workflow Visualization & DAG Rendering - Add internal/visualization package with DAGRenderer - Convert dependency graphs to DOT format - Critical path highlighting - Topological sorting with parallel task detection - HTML rendering for visualization - 11 DAG rendering tests, all passing T4.3: Advanced Search & Filtering - Add internal/search package with WorkflowSearch - Full-text indexing with word-based lookup - Filter by status, assignee, tag, date range - Regex pattern matching - Saved filters for reusable queries - 15 search tests, all passing T4.4: Cost Tracking & Optimization - Add internal/cost package with CostTracker - Track LLM API costs (by token) - Track git operation costs - Track compute resource costs (by duration) - Cost aggregation by workflow/type - Optimization recommendations - 11 cost tests, all passing Total T4.1-T4.4: 50 tests passing Next: T4.5-T4.8 (alerting, profiling, multi-cluster, self-deployment)
This commit is contained in:
Test
@@ -0,0 +1,246 @@
|
||||
package cost
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// CostEntry represents a tracked cost
|
||||
type CostEntry struct {
|
||||
ID string
|
||||
Type string // llm, git, compute
|
||||
WorkflowID string
|
||||
TaskID string
|
||||
Amount float64
|
||||
Timestamp time.Time
|
||||
Metadata map[string]interface{}
|
||||
}
|
||||
|
||||
// CostTracker tracks and analyzes workflow costs
|
||||
type CostTracker struct {
|
||||
mu sync.RWMutex
|
||||
entries []*CostEntry
|
||||
rates map[string]float64
|
||||
}
|
||||
|
||||
// NewCostTracker creates a new cost tracker
|
||||
func NewCostTracker() *CostTracker {
|
||||
return &CostTracker{
|
||||
entries: make([]*CostEntry, 0),
|
||||
rates: map[string]float64{
|
||||
"llm_token": 0.0001, // $0.0001 per token
|
||||
"git_push": 0.0, // Free
|
||||
"compute_hour": 0.5, // $0.5 per hour
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// TrackLLMCost tracks LLM API costs
|
||||
func (ct *CostTracker) TrackLLMCost(workflowID, taskID string, tokens int) {
|
||||
cost := float64(tokens) * ct.rates["llm_token"]
|
||||
ct.mu.Lock()
|
||||
defer ct.mu.Unlock()
|
||||
|
||||
entry := &CostEntry{
|
||||
ID: fmt.Sprintf("llm-%d", len(ct.entries)),
|
||||
Type: "llm",
|
||||
WorkflowID: workflowID,
|
||||
TaskID: taskID,
|
||||
Amount: cost,
|
||||
Timestamp: time.Now(),
|
||||
Metadata: map[string]interface{}{
|
||||
"tokens": tokens,
|
||||
},
|
||||
}
|
||||
|
||||
ct.entries = append(ct.entries, entry)
|
||||
}
|
||||
|
||||
// TrackGitCost tracks git operation costs
|
||||
func (ct *CostTracker) TrackGitCost(workflowID string, operations int) {
|
||||
ct.mu.Lock()
|
||||
defer ct.mu.Unlock()
|
||||
|
||||
entry := &CostEntry{
|
||||
ID: fmt.Sprintf("git-%d", len(ct.entries)),
|
||||
Type: "git",
|
||||
WorkflowID: workflowID,
|
||||
Amount: 0,
|
||||
Timestamp: time.Now(),
|
||||
Metadata: map[string]interface{}{
|
||||
"operations": operations,
|
||||
},
|
||||
}
|
||||
|
||||
ct.entries = append(ct.entries, entry)
|
||||
}
|
||||
|
||||
// TrackComputeCost tracks compute resource costs (duration in milliseconds)
|
||||
func (ct *CostTracker) TrackComputeCost(workflowID, taskID string, durationMs float64) {
|
||||
// Convert milliseconds to hours
|
||||
durationHours := durationMs / (1000.0 * 3600.0)
|
||||
cost := durationHours * ct.rates["compute_hour"]
|
||||
ct.mu.Lock()
|
||||
defer ct.mu.Unlock()
|
||||
|
||||
entry := &CostEntry{
|
||||
ID: fmt.Sprintf("compute-%d", len(ct.entries)),
|
||||
Type: "compute",
|
||||
WorkflowID: workflowID,
|
||||
TaskID: taskID,
|
||||
Amount: cost,
|
||||
Timestamp: time.Now(),
|
||||
Metadata: map[string]interface{}{
|
||||
"duration_ms": durationMs,
|
||||
},
|
||||
}
|
||||
|
||||
ct.entries = append(ct.entries, entry)
|
||||
}
|
||||
|
||||
// GetTotalCost returns total cost for all workflows
|
||||
func (ct *CostTracker) GetTotalCost() float64 {
|
||||
ct.mu.RLock()
|
||||
defer ct.mu.RUnlock()
|
||||
|
||||
total := 0.0
|
||||
for _, entry := range ct.entries {
|
||||
total += entry.Amount
|
||||
}
|
||||
|
||||
return total
|
||||
}
|
||||
|
||||
// GetWorkflowCost returns total cost for a specific workflow
|
||||
func (ct *CostTracker) GetWorkflowCost(workflowID string) float64 {
|
||||
ct.mu.RLock()
|
||||
defer ct.mu.RUnlock()
|
||||
|
||||
total := 0.0
|
||||
for _, entry := range ct.entries {
|
||||
if entry.WorkflowID == workflowID {
|
||||
total += entry.Amount
|
||||
}
|
||||
}
|
||||
|
||||
return total
|
||||
}
|
||||
|
||||
// GetCostByType returns total cost by type
|
||||
func (ct *CostTracker) GetCostByType(costType string) float64 {
|
||||
ct.mu.RLock()
|
||||
defer ct.mu.RUnlock()
|
||||
|
||||
total := 0.0
|
||||
for _, entry := range ct.entries {
|
||||
if entry.Type == costType {
|
||||
total += entry.Amount
|
||||
}
|
||||
}
|
||||
|
||||
return total
|
||||
}
|
||||
|
||||
// GetAverageCostPerTask returns average cost per task
|
||||
func (ct *CostTracker) GetAverageCostPerTask(workflowID string) float64 {
|
||||
ct.mu.RLock()
|
||||
defer ct.mu.RUnlock()
|
||||
|
||||
total := 0.0
|
||||
count := 0
|
||||
|
||||
for _, entry := range ct.entries {
|
||||
if entry.WorkflowID == workflowID {
|
||||
total += entry.Amount
|
||||
count++
|
||||
}
|
||||
}
|
||||
|
||||
if count == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
return total / float64(count)
|
||||
}
|
||||
|
||||
// GetOptimizationSuggestions returns cost optimization recommendations
|
||||
func (ct *CostTracker) GetOptimizationSuggestions(workflowID string) []string {
|
||||
suggestions := make([]string, 0)
|
||||
|
||||
ct.mu.RLock()
|
||||
defer ct.mu.RUnlock()
|
||||
|
||||
llmCost := 0.0
|
||||
computeCost := 0.0
|
||||
|
||||
for _, entry := range ct.entries {
|
||||
if entry.WorkflowID == workflowID {
|
||||
if entry.Type == "llm" {
|
||||
llmCost += entry.Amount
|
||||
} else if entry.Type == "compute" {
|
||||
computeCost += entry.Amount
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if llmCost > computeCost*2 {
|
||||
suggestions = append(suggestions, "Consider caching LLM results to reduce API calls")
|
||||
}
|
||||
|
||||
if computeCost > llmCost*2 {
|
||||
suggestions = append(suggestions, "Consider parallelizing compute tasks")
|
||||
}
|
||||
|
||||
return suggestions
|
||||
}
|
||||
|
||||
// GetEntries returns all cost entries
|
||||
func (ct *CostTracker) GetEntries() []*CostEntry {
|
||||
ct.mu.RLock()
|
||||
defer ct.mu.RUnlock()
|
||||
|
||||
result := make([]*CostEntry, len(ct.entries))
|
||||
copy(result, ct.entries)
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// GetEntriesForWorkflow returns cost entries for a workflow
|
||||
func (ct *CostTracker) GetEntriesForWorkflow(workflowID string) []*CostEntry {
|
||||
ct.mu.RLock()
|
||||
defer ct.mu.RUnlock()
|
||||
|
||||
result := make([]*CostEntry, 0)
|
||||
for _, entry := range ct.entries {
|
||||
if entry.WorkflowID == workflowID {
|
||||
result = append(result, entry)
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// SetRate sets the cost rate for a type
|
||||
func (ct *CostTracker) SetRate(costType string, rate float64) {
|
||||
ct.mu.Lock()
|
||||
defer ct.mu.Unlock()
|
||||
|
||||
ct.rates[costType] = rate
|
||||
}
|
||||
|
||||
// GetRate gets the cost rate for a type
|
||||
func (ct *CostTracker) GetRate(costType string) float64 {
|
||||
ct.mu.RLock()
|
||||
defer ct.mu.RUnlock()
|
||||
|
||||
return ct.rates[costType]
|
||||
}
|
||||
|
||||
// Clear clears all cost entries
|
||||
func (ct *CostTracker) Clear() {
|
||||
ct.mu.Lock()
|
||||
defer ct.mu.Unlock()
|
||||
|
||||
ct.entries = make([]*CostEntry, 0)
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package cost
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestTrackLLMCost(t *testing.T) {
|
||||
tracker := NewCostTracker()
|
||||
tracker.TrackLLMCost("wf-1", "task-1", 1000)
|
||||
|
||||
entries := tracker.GetEntries()
|
||||
assert.Equal(t, 1, len(entries))
|
||||
assert.Equal(t, "llm", entries[0].Type)
|
||||
assert.Equal(t, 0.1, entries[0].Amount) // 1000 tokens * 0.0001
|
||||
}
|
||||
|
||||
func TestTrackGitCost(t *testing.T) {
|
||||
tracker := NewCostTracker()
|
||||
tracker.TrackGitCost("wf-1", 5)
|
||||
|
||||
entries := tracker.GetEntries()
|
||||
assert.Equal(t, 1, len(entries))
|
||||
assert.Equal(t, "git", entries[0].Type)
|
||||
}
|
||||
|
||||
func TestTrackComputeCost(t *testing.T) {
|
||||
tracker := NewCostTracker()
|
||||
tracker.TrackComputeCost("wf-1", "task-1", 3600000) // 1 hour in ms
|
||||
|
||||
entries := tracker.GetEntries()
|
||||
assert.Equal(t, 1, len(entries))
|
||||
assert.Equal(t, "compute", entries[0].Type)
|
||||
assert.Equal(t, 0.5, entries[0].Amount) // 1 hour * $0.5/hour
|
||||
}
|
||||
|
||||
func TestGetTotalCost(t *testing.T) {
|
||||
tracker := NewCostTracker()
|
||||
|
||||
tracker.TrackLLMCost("wf-1", "task-1", 1000)
|
||||
tracker.TrackComputeCost("wf-1", "task-1", 3600000)
|
||||
|
||||
total := tracker.GetTotalCost()
|
||||
assert.Equal(t, 0.6, total) // 0.1 + 0.5
|
||||
}
|
||||
|
||||
func TestGetWorkflowCost(t *testing.T) {
|
||||
tracker := NewCostTracker()
|
||||
|
||||
tracker.TrackLLMCost("wf-1", "task-1", 1000)
|
||||
tracker.TrackLLMCost("wf-2", "task-1", 2000)
|
||||
|
||||
cost := tracker.GetWorkflowCost("wf-1")
|
||||
assert.Equal(t, 0.1, cost)
|
||||
|
||||
cost = tracker.GetWorkflowCost("wf-2")
|
||||
assert.Equal(t, 0.2, cost)
|
||||
}
|
||||
|
||||
func TestGetCostByType(t *testing.T) {
|
||||
tracker := NewCostTracker()
|
||||
|
||||
tracker.TrackLLMCost("wf-1", "task-1", 1000)
|
||||
tracker.TrackLLMCost("wf-1", "task-2", 1000)
|
||||
tracker.TrackComputeCost("wf-1", "task-3", 3600000)
|
||||
|
||||
llmCost := tracker.GetCostByType("llm")
|
||||
assert.Equal(t, 0.2, llmCost)
|
||||
|
||||
computeCost := tracker.GetCostByType("compute")
|
||||
assert.Equal(t, 0.5, computeCost)
|
||||
}
|
||||
|
||||
func TestGetAverageCostPerTask(t *testing.T) {
|
||||
tracker := NewCostTracker()
|
||||
|
||||
tracker.TrackLLMCost("wf-1", "task-1", 1000)
|
||||
tracker.TrackLLMCost("wf-1", "task-2", 1000)
|
||||
|
||||
avg := tracker.GetAverageCostPerTask("wf-1")
|
||||
assert.Equal(t, 0.1, avg)
|
||||
}
|
||||
|
||||
func TestGetOptimizationSuggestions(t *testing.T) {
|
||||
tracker := NewCostTracker()
|
||||
|
||||
// High LLM cost
|
||||
tracker.TrackLLMCost("wf-1", "task-1", 10000)
|
||||
tracker.TrackLLMCost("wf-1", "task-2", 10000)
|
||||
tracker.TrackComputeCost("wf-1", "task-3", 360000) // 0.1 seconds
|
||||
|
||||
suggestions := tracker.GetOptimizationSuggestions("wf-1")
|
||||
// Just verify it returns without error - suggestions depend on cost ratios
|
||||
assert.NotNil(t, suggestions)
|
||||
}
|
||||
|
||||
func TestGetEntriesForWorkflow(t *testing.T) {
|
||||
tracker := NewCostTracker()
|
||||
|
||||
tracker.TrackLLMCost("wf-1", "task-1", 1000)
|
||||
tracker.TrackLLMCost("wf-2", "task-1", 1000)
|
||||
|
||||
entries := tracker.GetEntriesForWorkflow("wf-1")
|
||||
assert.Equal(t, 1, len(entries))
|
||||
}
|
||||
|
||||
func TestSetAndGetRate(t *testing.T) {
|
||||
tracker := NewCostTracker()
|
||||
|
||||
tracker.SetRate("custom", 0.5)
|
||||
rate := tracker.GetRate("custom")
|
||||
|
||||
assert.Equal(t, 0.5, rate)
|
||||
}
|
||||
|
||||
func TestClear(t *testing.T) {
|
||||
tracker := NewCostTracker()
|
||||
|
||||
tracker.TrackLLMCost("wf-1", "task-1", 1000)
|
||||
tracker.Clear()
|
||||
|
||||
entries := tracker.GetEntries()
|
||||
assert.Equal(t, 0, len(entries))
|
||||
}
|
||||
|
||||
func TestMultipleCosts(t *testing.T) {
|
||||
tracker := NewCostTracker()
|
||||
|
||||
tracker.TrackLLMCost("wf-1", "task-1", 1000)
|
||||
tracker.TrackGitCost("wf-1", 5)
|
||||
tracker.TrackComputeCost("wf-1", "task-1", 1800000) // 30 min
|
||||
|
||||
total := tracker.GetTotalCost()
|
||||
assert.True(t, total > 0.2)
|
||||
}
|
||||
|
||||
func TestZeroCost(t *testing.T) {
|
||||
tracker := NewCostTracker()
|
||||
|
||||
cost := tracker.GetWorkflowCost("nonexistent")
|
||||
assert.Equal(t, 0.0, cost)
|
||||
}
|
||||
|
||||
func BenchmarkTrackLLMCost(b *testing.B) {
|
||||
tracker := NewCostTracker()
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
tracker.TrackLLMCost("wf-1", "task-1", 1000)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user