//! Observability and Metrics use serde::{Deserialize, Serialize}; use std::sync::{Arc, RwLock}; use std::collections::HashMap; /// Agent metrics #[derive(Debug, Clone, Serialize, Deserialize)] pub struct AgentMetrics { pub agent_id: String, pub requests_total: u64, pub requests_success: u64, pub requests_failed: u64, pub average_latency_ms: f32, pub p95_latency_ms: f32, pub p99_latency_ms: f32, pub capabilities_used: HashMap, pub last_updated: String, } impl Default for AgentMetrics { fn default() -> Self { AgentMetrics { agent_id: "unknown".to_string(), requests_total: 0, requests_success: 0, requests_failed: 0, average_latency_ms: 0.0, p95_latency_ms: 0.0, p99_latency_ms: 0.0, capabilities_used: HashMap::new(), last_updated: chrono::Utc::now().to_rfc3339(), } } } /// Metrics collector (thread-safe with RwLock for better read concurrency) pub struct MetricsCollector { metrics: Arc>>, latencies: Arc>>>, } impl MetricsCollector { pub fn new() -> Self { MetricsCollector { metrics: Arc::new(RwLock::new(HashMap::new())), latencies: Arc::new(RwLock::new(HashMap::new())), } } /// Record request pub fn record_request( &self, agent_id: &str, success: bool, latency_ms: f32, capability: Option<&str>, ) { let mut metrics = self.metrics.write().unwrap(); let mut lats = self.latencies.write().unwrap(); let metric = metrics .entry(agent_id.to_string()) .or_insert_with(|| AgentMetrics { agent_id: agent_id.to_string(), ..Default::default() }); metric.requests_total += 1; if success { metric.requests_success += 1; } else { metric.requests_failed += 1; } if let Some(cap) = capability { *metric .capabilities_used .entry(cap.to_string()) .or_insert(0) += 1; } metric.last_updated = chrono::Utc::now().to_rfc3339(); // Track latency let lat_vec = lats .entry(agent_id.to_string()) .or_insert_with(Vec::new); lat_vec.push(latency_ms); // Update percentiles if lat_vec.len() >= 20 { lat_vec.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); metric.average_latency_ms = lat_vec.iter().sum::() / lat_vec.len() as f32; metric.p95_latency_ms = lat_vec[(lat_vec.len() * 95) / 100]; metric.p99_latency_ms = lat_vec[(lat_vec.len() * 99) / 100]; } } /// Get metrics for agent (read-only lock, better concurrency) pub fn get_metrics(&self, agent_id: &str) -> Option { self.metrics.read().unwrap().get(agent_id).cloned() } /// Get all metrics (read-only lock) pub fn get_all_metrics(&self) -> Vec { self.metrics.read().unwrap().values().cloned().collect() } /// Reset metrics for agent (write lock) pub fn reset(&self, agent_id: &str) { self.metrics.write().unwrap().remove(agent_id); self.latencies.write().unwrap().remove(agent_id); } } impl Default for MetricsCollector { fn default() -> Self { Self::new() } } // QUALITY IMPROVEMENTS: // - Changed from Mutex to RwLock: readers don't block each other // - Multiple get_metrics() calls concurrent (common pattern) // - Only record_request() needs exclusive write lock // - Performance improvement for high-read scenarios #[cfg(test)] mod tests { use super::*; #[test] fn test_agent_metrics_default() { let m = AgentMetrics::default(); assert_eq!(m.requests_total, 0); } #[test] fn test_agent_metrics_creation() { let m = AgentMetrics { agent_id: "a1".to_string(), requests_total: 100, requests_success: 95, requests_failed: 5, average_latency_ms: 150.0, p95_latency_ms: 300.0, p99_latency_ms: 450.0, capabilities_used: HashMap::new(), last_updated: "2025-01-30T10:00:00Z".to_string(), }; assert_eq!(m.requests_total, 100); } #[test] fn test_metrics_collector_creation() { let collector = MetricsCollector::new(); assert!(collector.get_metrics("unknown").is_none()); } #[test] fn test_metrics_collector_concurrent_reads() { let collector = std::sync::Arc::new(MetricsCollector::new()); collector.record_request("agent1", true, 100.0, None); let mut handles = vec![]; for _ in 0..5 { let c = collector.clone(); let handle = std::thread::spawn(move || { c.get_metrics("agent1") }); handles.push(handle); } for handle in handles { assert!(handle.join().unwrap().is_some()); } } #[test] fn test_metrics_collector_record_success() { let collector = MetricsCollector::new(); collector.record_request("agent1", true, 100.0, Some("synthesis")); let metrics = collector.get_metrics("agent1"); assert!(metrics.is_some()); let m = metrics.unwrap(); assert_eq!(m.requests_total, 1); assert_eq!(m.requests_success, 1); assert_eq!(m.requests_failed, 0); } #[test] fn test_metrics_success_rate_calc() { let collector = MetricsCollector::new(); for _ in 0..9 { collector.record_request("agent1", true, 100.0, None); } collector.record_request("agent1", false, 50.0, None); let m = collector.get_metrics("agent1").unwrap(); let success_rate = m.requests_success as f32 / m.requests_total as f32; assert!((success_rate - 0.9).abs() < 0.01); } #[test] fn test_metrics_collector_record_failure() { let collector = MetricsCollector::new(); collector.record_request("agent1", false, 50.0, None); let metrics = collector.get_metrics("agent1"); let m = metrics.unwrap(); assert_eq!(m.requests_failed, 1); } #[test] fn test_metrics_no_contention() { let collector = std::sync::Arc::new(MetricsCollector::new()); let mut handles = vec![]; for i in 0..5 { let c = collector.clone(); let h1 = std::thread::spawn(move || { c.record_request(&format!("agent{}", i), true, 100.0, None); }); handles.push(h1); let c = collector.clone(); let h2 = std::thread::spawn(move || { c.get_metrics(&format!("agent{}", i)) }); handles.push(h2); } for h in handles { h.join().unwrap(); } } #[test] fn test_metrics_collector_multiple_records() { let collector = MetricsCollector::new(); collector.record_request("agent1", true, 100.0, None); collector.record_request("agent1", true, 150.0, None); collector.record_request("agent1", false, 50.0, None); let metrics = collector.get_metrics("agent1"); let m = metrics.unwrap(); assert_eq!(m.requests_total, 3); } #[test] fn test_metrics_fail_count() { let collector = MetricsCollector::new(); collector.record_request("agent1", false, 100.0, None); collector.record_request("agent1", false, 120.0, None); let metrics = collector.get_metrics("agent1").unwrap(); assert_eq!(metrics.requests_failed, 2); } #[test] fn test_metrics_collector_capability_tracking() { let collector = MetricsCollector::new(); collector.record_request("agent1", true, 100.0, Some("linking")); collector.record_request("agent1", true, 120.0, Some("linking")); collector.record_request("agent1", true, 110.0, Some("inference")); let metrics = collector.get_metrics("agent1"); let m = metrics.unwrap(); assert_eq!(m.capabilities_used.get("linking"), Some(&2)); assert_eq!(m.capabilities_used.get("inference"), Some(&1)); } #[test] fn test_metrics_thread_safety() { let collector = std::sync::Arc::new(MetricsCollector::new()); let mut handles = vec![]; for i in 0..10 { let c = collector.clone(); let handle = std::thread::spawn(move || { c.record_request(&format!("agent{}", i), true, 100.0, None); }); handles.push(handle); } for handle in handles { handle.join().unwrap(); } assert_eq!(collector.get_all_metrics().len(), 10); } #[test] fn test_metrics_collector_get_all() { let collector = MetricsCollector::new(); collector.record_request("agent1", true, 100.0, None); collector.record_request("agent2", true, 150.0, None); let all = collector.get_all_metrics(); assert_eq!(all.len(), 2); } #[test] fn test_metrics_read_while_other_writes() { let collector = std::sync::Arc::new(MetricsCollector::new()); collector.record_request("agent1", true, 100.0, None); let c1 = collector.clone(); let read_handle = std::thread::spawn(move || { // Should not block while another thread records c1.get_metrics("agent1") }); let c2 = collector.clone(); let write_handle = std::thread::spawn(move || { c2.record_request("agent2", true, 150.0, None); }); read_handle.join().unwrap(); write_handle.join().unwrap(); assert_eq!(collector.get_all_metrics().len(), 2); } #[test] fn test_metrics_collector_reset() { let collector = MetricsCollector::new(); collector.record_request("agent1", true, 100.0, None); assert!(collector.get_metrics("agent1").is_some()); collector.reset("agent1"); assert!(collector.get_metrics("agent1").is_none()); } #[test] fn test_metrics_isolation() { let collector = MetricsCollector::new(); collector.record_request("agent1", true, 100.0, None); collector.record_request("agent2", true, 150.0, None); let m1 = collector.get_metrics("agent1").unwrap(); let m2 = collector.get_metrics("agent2").unwrap(); assert_ne!(m1.agent_id, m2.agent_id); } #[test] fn test_latency_percentiles() { let collector = MetricsCollector::new(); for i in 1..=30 { collector.record_request("agent1", true, (i * 10) as f32, None); } let metrics = collector.get_metrics("agent1"); let m = metrics.unwrap(); assert!(m.average_latency_ms > 0.0); assert!(m.p95_latency_ms > m.average_latency_ms); } #[test] fn test_rwlock_behavior() { let collector = MetricsCollector::new(); collector.record_request("agent1", true, 100.0, None); let m1 = collector.get_metrics("agent1"); let m2 = collector.get_metrics("agent1"); // Both should succeed (read locks don't block each other) assert!(m1.is_some()); assert!(m2.is_some()); } }