feat: database layer + canvas validator/converter + LLM inference activities

- Add pkg/db models and CRUD methods for workflows - Add internal/routing canvas validator (DAG check, connectivity) - Add internal/routing canvas converter (Canvas → WorkflowSpec) - Register LLMInferenceActivity and LLMBatchInferenceActivity - Update api/server and cmd/server with database integration - Add K8s environment variable support - Update activity knowledge base with LLM activities - Add .env.example configuration template
2026-09-05 00:43:30 -07:00
parent 924aa398b6
commit 0da90fdd7a
13 changed files with 2183 additions and 7 deletions
@@ -473,10 +473,130 @@
        "dependencies": [],
        "notes": "Must run before LLM Router to provide auth token. Call early in workflow."
      }
+    },
+    {
+      "name": "LLMInferenceActivity",
+      "description": "Call LLM API with custom prompt and get response text",
+      "category": "llm",
+      "inputs": {
+        "model": {
+          "type": "string",
+          "description": "Model ID (reasoning, ornith:35b, ornith:13b, qwen2.5:3b)",
+          "required": true,
+          "examples": ["reasoning", "ornith:35b"]
+        },
+        "system_prompt": {
+          "type": "string",
+          "description": "System instruction for the model",
+          "required": false,
+          "default": ""
+        },
+        "user_prompt": {
+          "type": "string",
+          "description": "User message to send to the model",
+          "required": true
+        },
+        "temperature": {
+          "type": "number",
+          "description": "Sampling temperature (0.0-1.0, higher=more creative)",
+          "required": false,
+          "default": 0.7
+        },
+        "max_tokens": {
+          "type": "integer",
+          "description": "Maximum tokens in response",
+          "required": false
+        }
+      },
+      "outputs": {
+        "response": {
+          "type": "string",
+          "description": "LLM response text"
+        },
+        "model": {
+          "type": "string",
+          "description": "Model used for inference"
+        },
+        "stop_reason": {
+          "type": "string",
+          "description": "Why inference stopped (stop_sequence, length, etc)"
+        },
+        "tokens_used": {
+          "type": "integer",
+          "description": "Total tokens consumed"
+        }
+      },
+      "constraints": {
+        "defaultTimeout": "120s",
+        "isFlaky": true,
+        "recommendedRetries": 2,
+        "retryBackoff": 2.0,
+        "dependencies": [],
+        "notes": "API-dependent. Network flaky. Use for single prompts. See LLMBatchInferenceActivity for multiple."
+      }
+    },
+    {
+      "name": "LLMBatchInferenceActivity",
+      "description": "Call LLM API multiple times sequentially with different prompts",
+      "category": "llm",
+      "inputs": {
+        "model": {
+          "type": "string",
+          "description": "Model ID (reasoning, ornith:35b, ornith:13b, qwen2.5:3b)",
+          "required": true
+        },
+        "system_prompt": {
+          "type": "string",
+          "description": "System instruction (same for all prompts)",
+          "required": false
+        },
+        "prompts": {
+          "type": "array",
+          "description": "List of user prompts to process",
+          "required": true,
+          "items": {
+            "type": "string"
+          }
+        },
+        "temperature": {
+          "type": "number",
+          "description": "Sampling temperature (0.0-1.0)",
+          "required": false,
+          "default": 0.7
+        }
+      },
+      "outputs": {
+        "responses": {
+          "type": "array",
+          "description": "List of LLM responses (parallel to input prompts)",
+          "items": {
+            "type": "string"
+          }
+        },
+        "model": {
+          "type": "string",
+          "description": "Model used"
+        },
+        "errors": {
+          "type": "array",
+          "description": "Error messages for failed prompts",
+          "items": {
+            "type": "string"
+          }
+        }
+      },
+      "constraints": {
+        "defaultTimeout": "600s",
+        "isFlaky": true,
+        "recommendedRetries": 1,
+        "retryBackoff": 2.0,
+        "dependencies": [],
+        "notes": "Sequential processing of multiple prompts. Use for batch analysis, summarization, etc."
+      }
    }
  ],
  "metadata": {
-    "totalActivities": 10,
+    "totalActivities": 12,
    "lastUpdated": "2025-08-31T00:00:00Z",
    "categories": {
      "repository": 1,
@@ -488,7 +608,8 @@
      "approval": 1,
      "storage": 1,
      "memory": 1,
-      "authentication": 1
+      "authentication": 1,
+      "llm": 2
    }
  }
 }
@@ -0,0 +1,178 @@
+package routing
+
+import (
+	"fmt"
+
+	"github.com/rockliang/poimen/workflows/pkg/db"
+)
+
+// CanvasConverter converts visual canvas to executable WorkflowSpec
+type CanvasConverter struct {
+	validator *CanvasValidator
+}
+
+// NewCanvasConverter creates a converter
+func NewCanvasConverter() *CanvasConverter {
+	return &CanvasConverter{
+		validator: NewCanvasValidator(),
+	}
+}
+
+// CanvasToWorkflowSpec converts canvas to WorkflowSpec
+func (cc *CanvasConverter) CanvasToWorkflowSpec(canvas *db.Canvas) (*WorkflowSpec, error) {
+	// Validate first
+	if err := cc.validator.ValidateCanvas(canvas); err != nil {
+		return nil, fmt.Errorf("canvas validation failed: %w", err)
+	}
+
+	// Get topological order
+	sortedNodes, err := cc.validator.TopoSort(canvas.Nodes, canvas.Edges)
+	if err != nil {
+		return nil, fmt.Errorf("topological sort failed: %w", err)
+	}
+
+	// Build states from sorted nodes
+	states := []State{}
+	nodeToState := make(map[string]int) // node ID to state index
+
+	for i, node := range sortedNodes {
+		state := cc.nodeToState(node, canvas.Edges)
+		states = append(states, state)
+		nodeToState[node.ID] = i
+	}
+
+	// Wire up transitions
+	for i, node := range sortedNodes {
+		outgoing := cc.getOutgoingEdges(node.ID, canvas.Edges)
+
+		if len(outgoing) == 0 {
+			// Last state - no transitions
+			continue
+		}
+
+		if len(outgoing) == 1 {
+			// Single outgoing edge
+			targetNode := outgoing[0]
+			targetIdx := nodeToState[targetNode]
+			if targetIdx > i {
+				states[i].Next = states[targetIdx].Name
+			}
+		} else {
+			// Multiple outgoing edges - parallel
+			states[i].Type = "Parallel"
+			branches := []interface{}{}
+			for _, targetNode := range outgoing {
+				branches = append(branches, map[string]string{
+					"state": states[nodeToState[targetNode]].Name,
+				})
+			}
+			if states[i].Branches == nil {
+				states[i].Branches = branches
+			}
+		}
+	}
+
+	spec := &WorkflowSpec{
+		Name:   canvas.Name,
+		Input:  map[string]interface{}{},
+		States: states,
+	}
+
+	return spec, nil
+}
+
+// nodeToState converts a canvas node to a workflow state
+func (cc *CanvasConverter) nodeToState(node db.WorkflowNode, edges []db.WorkflowEdge) State {
+	// Map node type to activity name
+	activityName := cc.mapActivityType(node.Type)
+
+	state := State{
+		Name:       node.ID,
+		Type:       TaskActivity,
+		Activity:   activityName,
+		Retry:      &RetryPolicy{MaxAttempts: 3, BackoffSeconds: 2},
+		Timeout:    "300s",
+		Parameters: node.Data,
+	}
+
+	return state
+}
+
+// mapActivityType maps canvas activity type to Poimen activity
+func (cc *CanvasConverter) mapActivityType(canvasType string) string {
+	typeMap := map[string]string{
+		"clone-repo":          "CloneRepoActivity",
+		"analyze-code":        "AnalyzeCodeActivity",
+		"security-scan":       "SecurityScanActivity",
+		"generate-report":     "GenerateReportActivity",
+		"deployment-precheck": "DeploymentPreCheckActivity",
+		"notify-status":       "NotifyStatusActivity",
+		"approve-workflow":    "ApproveWorkflowActivity",
+		"archive-results":     "ArchiveResultsActivity",
+		"retrieve-memory":     "RetrieveMemoryActivity",
+		"assume-role":         "AssumeRoleActivity",
+		"llm-inference":       "LLMInferenceActivity",
+		"llm-batch-inference": "LLMBatchInferenceActivity",
+	}
+
+	if mapped, ok := typeMap[canvasType]; ok {
+		return mapped
+	}
+
+	return canvasType // fallback to type as-is
+}
+
+// getOutgoingEdges returns target node IDs for a given source node
+func (cc *CanvasConverter) getOutgoingEdges(nodeID string, edges []db.WorkflowEdge) []string {
+	targets := []string{}
+	seen := make(map[string]bool)
+
+	for _, edge := range edges {
+		if edge.Source == nodeID && !seen[edge.Target] {
+			targets = append(targets, edge.Target)
+			seen[edge.Target] = true
+		}
+	}
+
+	return targets
+}
+
+// CanvasToExecutionPlan converts canvas to sequential activity list
+func (cc *CanvasConverter) CanvasToExecutionPlan(canvas *db.Canvas) ([]ExecutionStep, error) {
+	// Validate first
+	if err := cc.validator.ValidateCanvas(canvas); err != nil {
+		return nil, fmt.Errorf("canvas validation failed: %w", err)
+	}
+
+	// Get topological order
+	sortedNodes, err := cc.validator.TopoSort(canvas.Nodes, canvas.Edges)
+	if err != nil {
+		return nil, fmt.Errorf("topological sort failed: %w", err)
+	}
+
+	steps := []ExecutionStep{}
+	for i, node := range sortedNodes {
+		step := ExecutionStep{
+			Index:        i,
+			NodeID:       node.ID,
+			ActivityName: cc.mapActivityType(node.Type),
+			Label:        node.Label,
+			Parameters:   node.Data,
+			Timeout:      "300s",
+		}
+		steps = append(steps, step)
+	}
+
+	return steps, nil
+}
+
+// ExecutionStep represents one activity in execution plan
+type ExecutionStep struct {
+	Index        int                   `json:"index"`
+	NodeID       string                `json:"node_id"`
+	ActivityName string                `json:"activity_name"`
+	Label        string                `json:"label"`
+	Parameters   map[string]interface{} `json:"parameters"`
+	Timeout      string                `json:"timeout"`
+	DependsOn    []int                 `json:"depends_on,omitempty"` // Indices of predecessor steps
+}
@@ -0,0 +1,316 @@
+package routing
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/rockliang/poimen/workflows/pkg/db"
+)
+
+// CanvasValidator validates React Flow canvas (nodes + edges)
+type CanvasValidator struct {
+	activityRegistry map[string]bool
+}
+
+// NewCanvasValidator creates validator with activity registry
+func NewCanvasValidator() *CanvasValidator {
+	return &CanvasValidator{
+		activityRegistry: map[string]bool{
+			"clone-repo":              true,
+			"analyze-code":            true,
+			"security-scan":           true,
+			"generate-report":         true,
+			"deployment-precheck":     true,
+			"notify-status":           true,
+			"approve-workflow":        true,
+			"archive-results":         true,
+			"retrieve-memory":         true,
+			"assume-role":             true,
+			"llm-inference":           true,
+			"llm-batch-inference":     true,
+		},
+	}
+}
+
+// ValidateCanvas checks canvas structure, connectivity, and DAG
+func (cv *CanvasValidator) ValidateCanvas(canvas *db.Canvas) error {
+	if canvas == nil {
+		return fmt.Errorf("canvas is nil")
+	}
+
+	if len(canvas.Nodes) == 0 {
+		return fmt.Errorf("canvas has no nodes")
+	}
+
+	// Step 1: Validate nodes
+	if err := cv.validateNodes(canvas.Nodes); err != nil {
+		return fmt.Errorf("node validation failed: %w", err)
+	}
+
+	// Step 2: Validate edges
+	if err := cv.validateEdges(canvas.Nodes, canvas.Edges); err != nil {
+		return fmt.Errorf("edge validation failed: %w", err)
+	}
+
+	// Step 3: Check for cycles (must be DAG)
+	if err := cv.detectCycles(canvas.Nodes, canvas.Edges); err != nil {
+		return fmt.Errorf("cycle detected: %w", err)
+	}
+
+	// Step 4: Check connectivity (all nodes reachable from start)
+	if err := cv.validateConnectivity(canvas.Nodes, canvas.Edges); err != nil {
+		return fmt.Errorf("connectivity check failed: %w", err)
+	}
+
+	return nil
+}
+
+// validateNodes checks each node has required fields and valid type
+func (cv *CanvasValidator) validateNodes(nodes []db.WorkflowNode) error {
+	if len(nodes) == 0 {
+		return fmt.Errorf("no nodes in canvas")
+	}
+
+	nodeIds := make(map[string]bool)
+
+	for i, node := range nodes {
+		// Check required fields
+		if node.ID == "" {
+			return fmt.Errorf("node[%d] has empty ID", i)
+		}
+
+		if nodeIds[node.ID] {
+			return fmt.Errorf("node[%d] has duplicate ID: %s", i, node.ID)
+		}
+		nodeIds[node.ID] = true
+
+		if node.Label == "" {
+			return fmt.Errorf("node[%d] (%s) has empty label", i, node.ID)
+		}
+
+		if node.Position == nil {
+			return fmt.Errorf("node[%d] (%s) has no position", i, node.ID)
+		}
+
+		// Check activity type (if present)
+		if node.Type != "" && !cv.activityRegistry[strings.ToLower(node.Type)] {
+			return fmt.Errorf("node[%d] (%s) has unknown activity type: %s", i, node.ID, node.Type)
+		}
+
+		// Check data structure
+		if node.Data == nil {
+			return fmt.Errorf("node[%d] (%s) has no data", i, node.ID)
+		}
+	}
+
+	return nil
+}
+
+// validateEdges checks edges reference valid nodes
+func (cv *CanvasValidator) validateEdges(nodes []db.WorkflowNode, edges []db.WorkflowEdge) error {
+	nodeIds := make(map[string]bool)
+	for _, node := range nodes {
+		nodeIds[node.ID] = true
+	}
+
+	for i, edge := range edges {
+		// Check required fields
+		if edge.Source == "" {
+			return fmt.Errorf("edge[%d] has empty source", i)
+		}
+
+		if edge.Target == "" {
+			return fmt.Errorf("edge[%d] has empty target", i)
+		}
+
+		// Check source node exists
+		if !nodeIds[edge.Source] {
+			return fmt.Errorf("edge[%d] references unknown source node: %s", i, edge.Source)
+		}
+
+		// Check target node exists
+		if !nodeIds[edge.Target] {
+			return fmt.Errorf("edge[%d] references unknown target node: %s", i, edge.Target)
+		}
+
+		// Check self-loops (discouraged but allow for now)
+		if edge.Source == edge.Target {
+			// Could warn here but not fail
+		}
+	}
+
+	return nil
+}
+
+// detectCycles checks for cycles in the DAG (must be acyclic)
+func (cv *CanvasValidator) detectCycles(nodes []db.WorkflowNode, edges []db.WorkflowEdge) error {
+	// Build adjacency list
+	graph := make(map[string][]string)
+	inDegree := make(map[string]int)
+
+	for _, node := range nodes {
+		graph[node.ID] = []string{}
+		inDegree[node.ID] = 0
+	}
+
+	for _, edge := range edges {
+		graph[edge.Source] = append(graph[edge.Source], edge.Target)
+		inDegree[edge.Target]++
+	}
+
+	// Kahn's algorithm: topological sort
+	queue := []string{}
+	for _, node := range nodes {
+		if inDegree[node.ID] == 0 {
+			queue = append(queue, node.ID)
+		}
+	}
+
+	processed := 0
+	for len(queue) > 0 {
+		// Dequeue
+		current := queue[0]
+		queue = queue[1:]
+		processed++
+
+		// Visit neighbors
+		for _, neighbor := range graph[current] {
+			inDegree[neighbor]--
+			if inDegree[neighbor] == 0 {
+				queue = append(queue, neighbor)
+			}
+		}
+	}
+
+	// If we didn't process all nodes, there's a cycle
+	if processed != len(nodes) {
+		return fmt.Errorf("graph has cycle (processed %d/%d nodes)", processed, len(nodes))
+	}
+
+	return nil
+}
+
+// validateConnectivity checks all nodes are reachable from start nodes
+func (cv *CanvasValidator) validateConnectivity(nodes []db.WorkflowNode, edges []db.WorkflowEdge) error {
+	if len(nodes) == 0 {
+		return nil
+	}
+
+	// Build adjacency list
+	graph := make(map[string][]string)
+	inDegree := make(map[string]int)
+
+	for _, node := range nodes {
+		graph[node.ID] = []string{}
+		inDegree[node.ID] = 0
+	}
+
+	for _, edge := range edges {
+		graph[edge.Source] = append(graph[edge.Source], edge.Target)
+		inDegree[edge.Target]++
+	}
+
+	// Find start nodes (in-degree 0)
+	startNodes := []string{}
+	for _, node := range nodes {
+		if inDegree[node.ID] == 0 {
+			startNodes = append(startNodes, node.ID)
+		}
+	}
+
+	if len(startNodes) == 0 {
+		return fmt.Errorf("no start nodes found (all nodes have incoming edges)")
+	}
+
+	// BFS from all start nodes
+	visited := make(map[string]bool)
+	queue := startNodes
+
+	for len(queue) > 0 {
+		// Dequeue
+		current := queue[0]
+		queue = queue[1:]
+
+		if visited[current] {
+			continue
+		}
+		visited[current] = true
+
+		// Visit neighbors
+		for _, neighbor := range graph[current] {
+			if !visited[neighbor] {
+				queue = append(queue, neighbor)
+			}
+		}
+	}
+
+	// Check all nodes were visited
+	if len(visited) != len(nodes) {
+		unreached := []string{}
+		for _, node := range nodes {
+			if !visited[node.ID] {
+				unreached = append(unreached, node.ID)
+			}
+		}
+		return fmt.Errorf("unreachable nodes: %v", unreached)
+	}
+
+	return nil
+}
+
+// TopoSort returns nodes in topological order (execution order)
+func (cv *CanvasValidator) TopoSort(nodes []db.WorkflowNode, edges []db.WorkflowEdge) ([]db.WorkflowNode, error) {
+	if len(nodes) == 0 {
+		return []db.WorkflowNode{}, nil
+	}
+
+	// Build adjacency list and in-degree map
+	graph := make(map[string][]string)
+	inDegree := make(map[string]int)
+	nodeMap := make(map[string]db.WorkflowNode)
+
+	for _, node := range nodes {
+		graph[node.ID] = []string{}
+		inDegree[node.ID] = 0
+		nodeMap[node.ID] = node
+	}
+
+	for _, edge := range edges {
+		graph[edge.Source] = append(graph[edge.Source], edge.Target)
+		inDegree[edge.Target]++
+	}
+
+	// Kahn's algorithm
+	queue := []string{}
+	for _, node := range nodes {
+		if inDegree[node.ID] == 0 {
+			queue = append(queue, node.ID)
+		}
+	}
+
+	result := []db.WorkflowNode{}
+	processed := make(map[string]bool)
+
+	for len(queue) > 0 {
+		// Dequeue
+		current := queue[0]
+		queue = queue[1:]
+
+		result = append(result, nodeMap[current])
+		processed[current] = true
+
+		// Visit neighbors
+		for _, neighbor := range graph[current] {
+			inDegree[neighbor]--
+			if inDegree[neighbor] == 0 {
+				queue = append(queue, neighbor)
+			}
+		}
+	}
+
+	if len(result) != len(nodes) {
+		return nil, fmt.Errorf("topological sort failed: graph has cycle")
+	}
+
+	return result, nil
+}