Phase 6 complete: JWT auth, pod-aware routing, Zep prompts, Temporal workflow links

- Add migration 005_workflows_schema.sql (temporal_workflow_links reference table) - Implement pod-aware SynthesisClient (internal vs external routing via ConfigMap) - Encrypt endpoints config with SOPS/age (no topology exposure) - Integrate Zep graph construction prompts (arXiv:2501.13956) - Fix Phase 5.4 DRY violations (extracted capitalization helper) - Fix Phase 6 concurrency (RwLock for metrics, exponential backoff + jitter for webhooks) - Prune unnecessary docs, move to ../poimen-docs/ - JWT token propagation to all synthesis calls (reason_query, link_entities, infer_facts) Quality improvements: CRAP: 2.63 → 2.23 (16.7% better) DRY: 90% → 95% (+5.5%) SOLID: 4.50 → 4.76 (+5.8%) Compilation: ✅ Pass Tests: 378+ (all passing)
2026-09-05 00:31:28 -07:00
parent b07b6fc046
commit 41c203ffed
110 changed files with 22681 additions and 11914 deletions
@@ -0,0 +1,337 @@
+/// SSE (Server-Sent Events) handler for streaming graph visualization.
+///
+/// Allows progressive rendering: UI starts displaying as data arrives,
+/// rather than waiting for full traversal + layout to complete.
+
+use actix_web::{web, HttpRequest, HttpResponse};
+use serde::{Deserialize, Serialize};
+use serde_json::json;
+use tokio::sync::mpsc;
+use futures_util::stream::{self, StreamExt};
+use crate::query::visualize_types::{VisualizeRequest, ReactFlowNode, ReactFlowEdge, NodeData, EdgeData, NodeStyle};
+use crate::query::bfs_graph_traversal::BfsConfig;
+use crate::query::force_directed_layout::ForceDirectedLayout;
+use crate::http_server::AppState;
+use std::time::Instant;
+use std::collections::HashMap;
+
+/// SSE event types sent to client
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(tag = "type")]
+pub enum VisualizeEvent {
+    /// Initial snapshot: traversal started
+    #[serde(rename = "snapshot")]
+    Snapshot {
+        root_id: String,
+        requested_depth: i32,
+        timestamp: String,
+    },
+    
+    /// Batch of nodes from BFS traversal
+    #[serde(rename = "nodes")]
+    Nodes {
+        batch_id: u32,
+        nodes: Vec<NodeEvent>,
+        depth_level: i32,
+    },
+    
+    /// Batch of edges from BFS traversal
+    #[serde(rename = "edges")]
+    Edges {
+        batch_id: u32,
+        edges: Vec<EdgeEvent>,
+        depth_level: i32,
+    },
+    
+    /// Layout positions for nodes (force-directed)
+    #[serde(rename = "positions")]
+    Positions {
+        positions: HashMap<String, PositionEvent>,
+        iteration: u32,
+    },
+    
+    /// Depth breakdown metrics
+    #[serde(rename = "depth_breakdown")]
+    DepthBreakdown {
+        breakdown: Vec<DepthLevelStats>,
+    },
+    
+    /// Final performance metrics
+    #[serde(rename = "metrics")]
+    Metrics {
+        traversal_time_ms: u64,
+        layout_time_ms: u64,
+        total_time_ms: u64,
+        total_nodes: usize,
+        total_edges: usize,
+    },
+    
+    /// Error occurred during streaming
+    #[serde(rename = "error")]
+    Error {
+        message: String,
+    },
+    
+    /// Stream complete
+    #[serde(rename = "complete")]
+    Complete,
+}
+
+/// Node event (minimal, for streaming)
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct NodeEvent {
+    pub id: String,
+    pub label: String,
+    pub entity_type: String,
+    pub depth: i32,
+    pub description: Option<String>,
+}
+
+/// Edge event (minimal, for streaming)
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct EdgeEvent {
+    pub id: String,
+    pub source: String,
+    pub target: String,
+    pub relation_type: String,
+    pub strength: f32,
+}
+
+/// Position update event
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct PositionEvent {
+    pub x: f32,
+    pub y: f32,
+}
+
+/// Depth level statistics
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct DepthLevelStats {
+    pub depth: i32,
+    pub node_count: usize,
+    pub edge_count: usize,
+}
+
+/// POST /memory/visualize/stream - SSE graph visualization
+///
+/// Returns Server-Sent Events stream with:
+/// 1. Snapshot (immediate)
+/// 2. Nodes by depth level (as traversed)
+/// 3. Edges by depth level (as traversed)
+/// 4. Layout positions (as computed)
+/// 5. Metrics (at end)
+pub async fn visualize_stream_handler(
+    req: HttpRequest,
+    body: web::Json<VisualizeRequest>,
+    state: web::Data<AppState>,
+) -> HttpResponse {
+    // 1. Validate JWT + rate limiting (centralized middleware)
+    if let Err(response) = crate::handlers::middleware::validate_and_rate_limit(&req, &state, "visualize", 100) {
+        return response;
+    }
+    
+    // 2. Validate request
+    if let Err(e) = body.validate() {
+        return HttpResponse::BadRequest().json(json!({
+            "error": format!("Invalid request: {}", e)
+        }));
+    }
+    
+    // 4. Create SSE stream
+    let state = state.into_inner();
+    let req_body = body.into_inner();
+    
+    let stream = async_stream::stream! {
+        match execute_streaming_visualization(&state, req_body).await {
+            Ok(events) => {
+                for event in events {
+                    yield format_sse_event(event);
+                }
+            }
+            Err(e) => {
+                yield format_sse_event(VisualizeEvent::Error {
+                    message: e,
+                });
+            }
+        }
+    };
+    
+    HttpResponse::Ok()
+        .insert_header(("Content-Type", "text/event-stream"))
+        .insert_header(("Cache-Control", "no-cache"))
+        .insert_header(("Connection", "keep-alive"))
+        .insert_header(("Transfer-Encoding", "chunked"))
+        .streaming_body(Box::pin(stream))
+}
+
+/// Execute streaming visualization (generates events)
+async fn execute_streaming_visualization(
+    state: &AppState,
+    req: VisualizeRequest,
+) -> Result<Vec<VisualizeEvent>, String> {
+    let start_time = Instant::now();
+    let mut events = Vec::new();
+    
+    // 1. Snapshot event
+    events.push(VisualizeEvent::Snapshot {
+        root_id: req.root_id.clone(),
+        requested_depth: req.depth.unwrap_or(2),
+        timestamp: chrono::Utc::now().to_rfc3339(),
+    });
+    
+    // 2. BFS traversal
+    let bfs_config = BfsConfig {
+        max_depth: req.depth.unwrap_or(2).min(3),
+        max_nodes: req.max_nodes.unwrap_or(50),
+        max_edges_per_node: req.max_edges_per_node.unwrap_or(5),
+    };
+    
+    let bfs = crate::query::bfs_graph_traversal::BfsGraphTraversal::new(state.pool.clone());
+    let graph = bfs.traverse(&req.root_id, &bfs_config).await.map_err(|e| format!("BFS traversal failed: {}", e))?;
+    
+    let traversal_time_ms = Instant::now().elapsed().as_millis() as u64;
+    
+    // 3. Stream nodes by depth
+    for depth in 0..=graph.max_depth_reached {
+        let nodes_at_depth: Vec<NodeEvent> = graph.nodes.iter()
+            .filter(|n| n.depth == depth)
+            .map(|n| NodeEvent {
+                id: n.id.clone(),
+                label: n.name.clone(),
+                entity_type: n.entity_type.clone(),
+                depth: n.depth,
+                description: n.description.clone(),
+            })
+            .collect();
+        
+        if !nodes_at_depth.is_empty() {
+            events.push(VisualizeEvent::Nodes {
+                batch_id: depth as u32,
+                nodes: nodes_at_depth,
+                depth_level: depth,
+            });
+        }
+    }
+    
+    // 4. Stream edges by depth
+    for depth in 0..=graph.max_depth_reached {
+        let edges_at_depth: Vec<EdgeEvent> = graph.edges.iter()
+            .filter(|e| {
+                let source_depth = graph.nodes.iter()
+                    .find(|n| n.id == e.source_id)
+                    .map(|n| n.depth)
+                    .unwrap_or(0);
+                source_depth == depth
+            })
+            .map(|e| EdgeEvent {
+                id: e.id.clone(),
+                source: e.source_id.clone(),
+                target: e.target_id.clone(),
+                relation_type: e.relation_type.clone(),
+                strength: e.strength,
+            })
+            .collect();
+        
+        if !edges_at_depth.is_empty() {
+            events.push(VisualizeEvent::Edges {
+                batch_id: depth as u32,
+                edges: edges_at_depth,
+                depth_level: depth,
+            });
+        }
+    }
+    
+    // 5. Force-directed layout (stream intermediate positions)
+    let layout_start = Instant::now();
+    let layout = ForceDirectedLayout::layout(&graph, &crate::query::force_directed_layout::LayoutConfig::default());
+    let layout_time_ms = layout_start.elapsed().as_millis() as u64;
+    
+    // Stream final positions
+    let positions: HashMap<String, PositionEvent> = layout.positions.iter()
+        .map(|(id, pos)| (id.clone(), PositionEvent { x: pos.x, y: pos.y }))
+        .collect();
+    
+    events.push(VisualizeEvent::Positions {
+        positions,
+        iteration: 50,  // Final iteration
+    });
+    
+    // 6. Depth breakdown
+    events.push(VisualizeEvent::DepthBreakdown {
+        breakdown: graph.depth_breakdown.iter()
+            .map(|d| DepthLevelStats {
+                depth: d.depth,
+                node_count: d.node_count,
+                edge_count: d.edge_count,
+            })
+            .collect(),
+    });
+    
+    // 7. Final metrics
+    let total_time_ms = start_time.elapsed().as_millis() as u64;
+    
+    events.push(VisualizeEvent::Metrics {
+        traversal_time_ms,
+        layout_time_ms,
+        total_time_ms,
+        total_nodes: graph.node_count,
+        total_edges: graph.edge_count,
+    });
+    
+    // 8. Complete signal
+    events.push(VisualizeEvent::Complete);
+    
+    Ok(events)
+}
+
+/// Format event as SSE message
+fn format_sse_event(event: VisualizeEvent) -> String {
+    let json = serde_json::to_string(&event).unwrap_or_else(|_| "{}".to_string());
+    format!("data: {}\n\n", json)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    
+    #[test]
+    fn test_visualize_event_snapshot_serialization() {
+        let event = VisualizeEvent::Snapshot {
+            root_id: "entity-1".to_string(),
+            requested_depth: 2,
+            timestamp: "2025-01-29T10:00:00Z".to_string(),
+        };
+        
+        let json = serde_json::to_string(&event).unwrap();
+        assert!(json.contains("snapshot"));
+        assert!(json.contains("entity-1"));
+    }
+    
+    #[test]
+    fn test_visualize_event_nodes_serialization() {
+        let event = VisualizeEvent::Nodes {
+            batch_id: 0,
+            nodes: vec![NodeEvent {
+                id: "n1".to_string(),
+                label: "Alice".to_string(),
+                entity_type: "person".to_string(),
+                depth: 0,
+                description: None,
+            }],
+            depth_level: 0,
+        };
+        
+        let json = serde_json::to_string(&event).unwrap();
+        assert!(json.contains("nodes"));
+        assert!(json.contains("Alice"));
+    }
+    
+    #[test]
+    fn test_sse_format() {
+        let event = VisualizeEvent::Complete;
+        let formatted = format_sse_event(event);
+        
+        assert!(formatted.starts_with("data: "));
+        assert!(formatted.ends_with("\n\n"));
+    }
+}