feat: implement proper orchestrator workflow with reconciliation loop
ci / test (push) Failing after 1m2s
ci / test (push) Failing after 1m2s
Rewrite OrchestratorWorkflow as true reconciliation loop: - PlanningActivity decides what tasks to dispatch - Fan-out TaskUnit workflows for parallel execution - Each TaskUnit runs Implementer → Test → Judge → Commit - Judge reviews code quality, retries on failure with lessons - Fan-in waits for all TaskUnits - Board update and squash merge on success - continue-as-new for long-running workflows - Proper error handling and signal support Key changes: - statemachine/orchestrator.go: Reconciliation loop (Plan → Dispatch → Review → Repeat) - statemachine/taskunit.go: Task execution with retry loop & judge review - statemachine/types.go: Updated TaskUnitInput/Output for new workflow - cmd/worker/main.go: Register RunIntegrationTestActivity - action/integration.go: Renamed from integration_test.go (fix Go build issue) Models: - Planner: reasoning (OpenAI-compatible from local LLM API) - Judge: reasoning (reviews diff + tests, gates success) - Implementer: ornith:35b (executes tasks) Verification: go build ./cmd/worker ./cmd/starter ✓
This commit is contained in:
+94
-121
@@ -4,161 +4,134 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"go.temporal.io/sdk/temporal"
|
||||
"go.temporal.io/sdk/workflow"
|
||||
)
|
||||
|
||||
// TaskUnitWorkflow executes a single task with retry loops, timeout escalation, and lessons injection.
|
||||
// TaskUnitWorkflow executes a single task with retries and judge review.
|
||||
// Flow: Worktree → Implementer (retry) → Test → Judge → Commit or Retry
|
||||
func TaskUnitWorkflow(ctx workflow.Context, in TaskUnitInput) (TaskUnitOutput, error) {
|
||||
// Initialize output
|
||||
logger := workflow.GetLogger(ctx)
|
||||
output := TaskUnitOutput{
|
||||
TaskID: in.TaskID,
|
||||
Verdict: "fail",
|
||||
Status: "failed",
|
||||
Reason: "",
|
||||
Branch: fmt.Sprintf("task/%s", in.TaskID),
|
||||
Changes: "",
|
||||
}
|
||||
|
||||
// 1. Add worktree for isolated work
|
||||
activityOpts := workflow.ActivityOptions{
|
||||
StartToCloseTimeout: 30 * time.Minute,
|
||||
ScheduleToCloseTimeout: 35 * time.Minute,
|
||||
}
|
||||
actCtx := workflow.WithActivityOptions(ctx, activityOpts)
|
||||
|
||||
// Add worktree
|
||||
var worktreePath string
|
||||
wtErr := workflow.ExecuteActivity(
|
||||
ctx,
|
||||
"GitWorktreeAddActivity",
|
||||
map[string]interface{}{
|
||||
"RepoPath": in.TargetRepoPath,
|
||||
"TaskID": in.TaskID,
|
||||
},
|
||||
).Get(ctx, &worktreePath)
|
||||
if wtErr != nil {
|
||||
output.Critique = fmt.Sprintf("Failed to create worktree: %v", wtErr)
|
||||
if err := workflow.ExecuteActivity(actCtx, "GitWorktreeAddActivity", map[string]interface{}{
|
||||
"RepoPath": in.TargetRepoPath,
|
||||
"TaskID": in.TaskID,
|
||||
}).Get(ctx, &worktreePath); err != nil {
|
||||
output.Reason = fmt.Sprintf("worktree add failed: %v", err)
|
||||
return output, nil
|
||||
}
|
||||
|
||||
// 2. Retry loop with separate timeout and judge attempt tracking
|
||||
timeoutAttempt := 1
|
||||
for judgeAttempt := 1; judgeAttempt <= in.MaxJudgeRetries; judgeAttempt++ {
|
||||
// Calculate timeouts for this attempt
|
||||
baseTimeout := in.BaseTimeout * time.Duration(timeoutAttempt)
|
||||
heartbeatTimeout := baseTimeout / 4
|
||||
logger.Info("task unit started", "task", in.TaskID, "worktree", worktreePath)
|
||||
|
||||
// Prepare activity options with escalating timeout
|
||||
ao := workflow.ActivityOptions{
|
||||
ScheduleToCloseTimeout: baseTimeout,
|
||||
StartToCloseTimeout: baseTimeout,
|
||||
HeartbeatTimeout: heartbeatTimeout,
|
||||
RetryPolicy: &temporal.RetryPolicy{
|
||||
MaximumAttempts: 1, // We manage retries in this loop
|
||||
},
|
||||
// Retry loop: implementer + test + judge
|
||||
maxRetries := 3
|
||||
var lessons string
|
||||
|
||||
for attempt := 1; attempt <= maxRetries; attempt++ {
|
||||
logger.Info("attempt", "task", in.TaskID, "attempt", attempt)
|
||||
|
||||
// Call ImplementerActivity with escalating timeout
|
||||
timeoutMultiplier := int64(attempt)
|
||||
implOpts := workflow.ActivityOptions{
|
||||
StartToCloseTimeout: time.Duration(timeoutMultiplier*30) * time.Minute,
|
||||
ScheduleToCloseTimeout: time.Duration(timeoutMultiplier*35) * time.Minute,
|
||||
}
|
||||
ctxWithOptions := workflow.WithActivityOptions(ctx, ao)
|
||||
implCtx := workflow.WithActivityOptions(ctx, implOpts)
|
||||
|
||||
// Call implementer activity
|
||||
var implOutput map[string]interface{}
|
||||
implErr := workflow.ExecuteActivity(
|
||||
ctxWithOptions,
|
||||
"ImplementerActivity",
|
||||
map[string]interface{}{
|
||||
"TaskID": in.TaskID,
|
||||
"WorktreePath": worktreePath,
|
||||
"Prompt": in.ImplementerSpec,
|
||||
},
|
||||
).Get(ctx, &implOutput)
|
||||
implErr := workflow.ExecuteActivity(implCtx, "ImplementerActivity", map[string]interface{}{
|
||||
"Config": in.Config,
|
||||
"TaskID": in.TaskID,
|
||||
"WorktreePath": worktreePath,
|
||||
"Lessons": lessons,
|
||||
}).Get(ctx, &implOutput)
|
||||
|
||||
// Check if it's a timeout error
|
||||
if implErr != nil && isStartToCloseTimeout(implErr) {
|
||||
// Timeout: escalate and retry without consuming judge attempt
|
||||
timeoutAttempt++
|
||||
judgeAttempt-- // Don't consume a judge retry on timeout
|
||||
continue
|
||||
}
|
||||
if implErr != nil {
|
||||
output.Critique = fmt.Sprintf("Implementer failed: %v", implErr)
|
||||
if attempt < maxRetries {
|
||||
logger.Info("implementer failed, will retry", "task", in.TaskID, "attempt", attempt, "error", implErr)
|
||||
continue
|
||||
}
|
||||
output.Reason = fmt.Sprintf("implementer exhausted after %d attempts: %v", maxRetries, implErr)
|
||||
return output, nil
|
||||
}
|
||||
|
||||
// Call judge activity
|
||||
judgeTimeout := time.Minute * 5
|
||||
judgeCtx := workflow.WithActivityOptions(ctx, workflow.ActivityOptions{
|
||||
ScheduleToCloseTimeout: judgeTimeout,
|
||||
StartToCloseTimeout: judgeTimeout,
|
||||
})
|
||||
var judgeOutput map[string]interface{}
|
||||
judgeErr := workflow.ExecuteActivity(
|
||||
judgeCtx,
|
||||
"JudgeActivity",
|
||||
map[string]interface{}{
|
||||
"TaskID": in.TaskID,
|
||||
"WorktreePath": worktreePath,
|
||||
"Prompt": in.JudgeSpec,
|
||||
},
|
||||
).Get(ctx, &judgeOutput)
|
||||
if judgeErr != nil {
|
||||
output.Critique = fmt.Sprintf("Judge error: %v", judgeErr)
|
||||
return output, nil
|
||||
}
|
||||
|
||||
// Check judge verdict
|
||||
verdict := ""
|
||||
if judgeOutput != nil {
|
||||
if v, ok := judgeOutput["Verdict"].(string); ok {
|
||||
verdict = v
|
||||
// Run integration test
|
||||
var testOutput map[string]interface{}
|
||||
if err := workflow.ExecuteActivity(actCtx, "RunIntegrationTestActivity", map[string]interface{}{
|
||||
"WorktreePath": worktreePath,
|
||||
"TestCmd": "go test ./...",
|
||||
}).Get(ctx, &testOutput); err != nil {
|
||||
logger.Info("test failed", "task", in.TaskID, "error", err)
|
||||
testOutput = map[string]interface{}{
|
||||
"success": false,
|
||||
"logs": fmt.Sprintf("test error: %v", err),
|
||||
}
|
||||
}
|
||||
|
||||
// Get diff
|
||||
var diff string
|
||||
if err := workflow.ExecuteActivity(actCtx, "GitDiffActivity", map[string]interface{}{
|
||||
"WorktreePath": worktreePath,
|
||||
}).Get(ctx, &diff); err != nil {
|
||||
logger.Info("diff failed", "task", in.TaskID, "error", err)
|
||||
}
|
||||
|
||||
// Call JudgeActivity
|
||||
var judgeOutput map[string]interface{}
|
||||
if err := workflow.ExecuteActivity(actCtx, "JudgeActivity", map[string]interface{}{
|
||||
"Config": in.Config,
|
||||
"Diff": diff,
|
||||
"IntegrationTestLogs": fmt.Sprintf("%v", testOutput),
|
||||
}).Get(ctx, &judgeOutput); err != nil {
|
||||
logger.Info("judge failed", "task", in.TaskID, "error", err)
|
||||
}
|
||||
|
||||
verdict, _ := judgeOutput["Verdict"].(string)
|
||||
critique, _ := judgeOutput["Critique"].(string)
|
||||
|
||||
if verdict == "pass" {
|
||||
// Commit in worktree
|
||||
commitErr := workflow.ExecuteActivity(
|
||||
ctx,
|
||||
"GitCommitActivity",
|
||||
map[string]interface{}{
|
||||
"WorktreePath": worktreePath,
|
||||
"Message": fmt.Sprintf("%s: implementation", in.TaskID),
|
||||
},
|
||||
).Get(ctx, nil)
|
||||
if commitErr != nil {
|
||||
output.Critique = fmt.Sprintf("Commit failed: %v", commitErr)
|
||||
// Commit and success
|
||||
if err := workflow.ExecuteActivity(actCtx, "GitCommitActivity", map[string]interface{}{
|
||||
"WorktreePath": worktreePath,
|
||||
"Message": fmt.Sprintf("%s: implementation", in.TaskID),
|
||||
}).Get(ctx, nil); err != nil {
|
||||
output.Reason = fmt.Sprintf("commit failed: %v", err)
|
||||
return output, nil
|
||||
}
|
||||
|
||||
// Success!
|
||||
output.Verdict = "pass"
|
||||
output.Branch = "task/" + in.TaskID
|
||||
output.Status = "success"
|
||||
output.Changes = fmt.Sprintf("completed in %d attempt(s)", attempt)
|
||||
logger.Info("task success", "task", in.TaskID, "attempt", attempt)
|
||||
return output, nil
|
||||
}
|
||||
|
||||
// Judge failed: update lessons and retry
|
||||
critique := ""
|
||||
if judgeOutput != nil {
|
||||
if c, ok := judgeOutput["Critique"].(string); ok {
|
||||
critique = c
|
||||
}
|
||||
// Judge failed: append to lessons and retry
|
||||
if attempt < maxRetries {
|
||||
lessons = fmt.Sprintf("%s\nAttempt %d critique: %s", lessons, attempt, critique)
|
||||
logger.Info("judge rejected, appending to lessons and retrying", "task", in.TaskID, "attempt", attempt)
|
||||
continue
|
||||
}
|
||||
|
||||
updateErr := workflow.ExecuteActivity(
|
||||
ctx,
|
||||
"UpdateLessonsActivity",
|
||||
map[string]interface{}{
|
||||
"TargetRepoPath": in.TargetRepoPath,
|
||||
"TaskID": in.TaskID,
|
||||
"Attempt": judgeAttempt,
|
||||
"Critique": critique,
|
||||
},
|
||||
).Get(ctx, nil)
|
||||
if updateErr != nil {
|
||||
output.Critique = fmt.Sprintf("Failed to update lessons: %v", updateErr)
|
||||
return output, nil
|
||||
}
|
||||
|
||||
// Continue to next judge attempt with lessons injected
|
||||
// Exhausted retries after judge failures
|
||||
output.Reason = fmt.Sprintf("judge rejected after %d attempts. Last critique: %s", maxRetries, critique)
|
||||
logger.Info("judge exhausted retries", "task", in.TaskID)
|
||||
return output, nil
|
||||
}
|
||||
|
||||
// Retries exhausted
|
||||
output.Verdict = "fail"
|
||||
output.Critique = fmt.Sprintf("Exhausted %d judge retries", in.MaxJudgeRetries)
|
||||
output.Reason = "exhausted all retry attempts"
|
||||
return output, nil
|
||||
}
|
||||
|
||||
// isStartToCloseTimeout checks if an error is a StartToCloseTimeout error
|
||||
func isStartToCloseTimeout(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
return fmt.Sprint(err) == "context deadline exceeded"
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user