diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..8f7ea14 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,385 @@ +# Multi-stage build for Poimen Temporal Worker +# Stage 1: Builder - Compile Go binary and set up tools +FROM golang:1.25-alpine AS builder + +WORKDIR /build + +# Install system dependencies (ast-grep, git, build essentials) +RUN apk add --no-cache \ + git \ + curl \ + wget \ + ca-certificates \ + gcc \ + musl-dev \ + bash \ + && echo "[builder] System dependencies installed" + +# Install ast-grep CLI tool +RUN curl -fsSL https://github.com/ast-grep/ast-grep/releases/download/0.24.0/sg-x86_64-unknown-linux-musl.tar.gz \ + | tar xzf - -C /usr/local/bin \ + && chmod +x /usr/local/bin/sg \ + && sg --version \ + && echo "[builder] ast-grep installed" + +# Install Node.js for pi CLI and browser-use +RUN apk add --no-cache nodejs npm \ + && echo "[builder] Node.js installed" + +# Install pi CLI globally +RUN npm install -g @earendil-works/pi-coding-agent --unsafe-perm \ + && pi --version \ + && echo "[builder] pi CLI installed" + +# Install browser-use CLI for browser automation +RUN npm install -g browser-use --unsafe-perm \ + && browser-use --version \ + && echo "[builder] browser-use CLI installed" + +# Set up pi home directory and skills +RUN mkdir -p ~/.pi/agent/skills ~/.pi/agent/agents \ + && echo "[builder] pi directories created" + +# Stage 2: Download pi skills (caveman & andrej karpathy) +# Clone caveman skill from pi-agent repo +RUN cd /tmp && git clone https://github.com/earendil-works/pi-agent.git pi-repo \ + && mkdir -p ~/.pi/agent/skills/caveman \ + && cp -r pi-repo/examples/skills/caveman/* ~/.pi/agent/skills/caveman/ 2>/dev/null || true \ + && echo "[builder] caveman skill installed" + +# Create andrej karpathy skill manually (reference/training patterns) +RUN mkdir -p ~/.pi/agent/skills/andrej-karpathy && cat > ~/.pi/agent/skills/andrej-karpathy/SKILL.md << 'EOF' +# Andrej Karpathy LLM & AI Principles Skill + +Build neural networks and LLM systems with proven patterns from Andrej Karpathy. +Topics: attention mechanisms, transformer training, inference optimization, edge cases. + +## Key Principles + +### 1. Simplicity First +- Start with minimal implementation +- Add complexity only when justified +- Test each component independently +- Use debugging tools effectively + +### 2. Neural Network Architecture +- Understand backward pass deeply +- Implement from scratch when possible +- Use visualization for debugging +- Profile before optimizing + +### 3. LLM Training Patterns +- Quality data > quantity +- Curriculum learning for complex tasks +- Loss landscape visualization +- Checkpoint strategy matters + +### 4. Inference Optimization +- Quantization without quality loss +- KV cache management +- Batch processing strategies +- Latency profiling + +### 5. Failure Analysis +- Log intermediate activations +- Check gradient flow +- Validate data pipeline +- Test edge cases explicitly + +## Usage in Poimen + +Apply when: +- Designing workflow stages (like training curricula) +- Optimizing inference (planner/judge/implementer prompts) +- Debugging convergence issues (retry patterns) +- Scaling to production (quantization patterns) + +## Resources +- github.com/karpathy/minGPT - Minimal GPT implementation +- youtube: "Neural Networks: Zero to Hero" series +- Papers: Attention Is All You Need, GPT series whitepapers +EOF + && echo "[builder] andrej-karpathy skill created" + +# Create browser-use skill for web testing & automation +RUN mkdir -p ~/.pi/agent/skills/browser-use && cat > ~/.pi/agent/skills/browser-use/SKILL.md << 'EOF' +# browser-use: Browser Automation Skill + +Automate web browser interactions for testing, verification, and UI validation. +Topics: headless browser control, visual testing, form automation, screenshot capture. + +## Key Capabilities + +### 1. Browser Control +- Launch headless Chrome/Firefox +- Navigate to URLs +- Wait for elements/navigation +- Handle popups/dialogs + +### 2. Interaction Patterns +- Click buttons/links +- Fill forms (text, dropdown, checkbox) +- Drag & drop +- Keyboard input + +### 3. Verification & Capture +- Screenshot capture +- Element inspection +- Accessibility checks +- Network monitoring + +### 4. Wait Strategies +- Wait for element visible +- Wait for navigation +- Wait for condition (custom JS) +- Timeout handling + +### 5. Error Recovery +- Retry failed actions +- Handle stale elements +- Browser crash recovery +- Memory leak prevention + +## Usage in Poimen Phases + +### Phase T2 (Implementation) +- Test generated UI code in real browser +- Verify visual layout matches spec +- Validate form inputs work correctly + +### Phase T3 (Verification) +- Visual regression testing +- Accessibility validation (ARIA, keyboard nav) +- Cross-browser verification + +### Phase T6 (Integration) +- End-to-end workflow testing +- External service integration testing +- User journey verification + +### Phase T9 (Release) +- Pre-release smoke tests +- Deployment verification +- Production canary testing + +## Example Workflows + +```bash +# Launch browser and take screenshot +browser-use screenshot "https://example.com" --file output.png + +# Fill form and submit +browser-use interact "https://example.com" \ + --click "#submit-btn" \ + --type "#email" "test@example.com" \ + --type "#password" "secretpass" \ + --click ".submit" + +# Wait for dynamic content and extract data +browser-use extract "https://example.com" \ + --wait ".dynamic-content" \ + --selector ".data-row" \ + --output json + +# Accessibility audit +browser-use audit "https://example.com" \ + --check wcag2a \ + --report a11y-report.html +``` + +## Integration with Poimen + +Pre-generated code can be tested: +```bash +# Generate code (T2) +implementer_output = "function handleClick() { ... }" + +# Verify in browser (T3) +browser-use interact "http://localhost:3000" \ + --click ".test-button" \ + --screenshot result.png + +# Compare with expected +verify_visual_match(result.png, expected.png) +``` + +## Performance Notes + +- Startup: ~2-5s per browser +- Action latency: 100-500ms per interaction +- Screenshot: 500ms-2s (depends on page size) +- Keep browser alive for batch operations (pool management) + +## Error Handling + +- Transient: Network timeout → retry with backoff +- Permanent: Element not found → fail and log +- Flaky: Wait strategies → increase timeout gradually +- Memory: Reuse browser instances → kill after 10 uses + +## Resources +- docs.browseruse.com - Official documentation +- github.com/browser-use/browser-use - Source code +- Chrome DevTools Protocol - Advanced browser control +EOF + && echo "[builder] browser-use skill created" + +# Copy Go source code +COPY . /build/ + +# Download Go dependencies +RUN go mod download \ + && echo "[builder] Go dependencies downloaded" + +# Build worker binary +RUN CGO_ENABLED=1 GOOS=linux go build -o /build/worker ./cmd/worker \ + && echo "[builder] Worker binary built" + +# Verify binary +RUN file /build/worker && ls -lh /build/worker + +# Stage 3: Runtime - Minimal base image with runtime dependencies +FROM alpine:3.20 + +LABEL maintainer="Poimen Team" +LABEL description="Poimen Temporal Worker with memory service, ast-grep, and browser automation" + +WORKDIR /app + +# Install runtime dependencies (including Chromium for browser-use) +RUN apk add --no-cache \ + ca-certificates \ + git \ + bash \ + curl \ + jq \ + chromium \ + chromium-chromedriver \ + && echo "[runtime] Runtime dependencies installed" + +# Install Node.js for pi CLI and browser-use +RUN apk add --no-cache nodejs npm \ + && echo "[runtime] Node.js installed" + +# Install pi CLI in runtime image +RUN npm install -g @earendil-works/pi-coding-agent --unsafe-perm \ + && pi --version \ + && echo "[runtime] pi CLI installed" + +# Install browser-use CLI in runtime image +RUN npm install -g browser-use --unsafe-perm \ + && browser-use --version \ + && echo "[runtime] browser-use CLI installed" + +# Copy ast-grep binary from builder +COPY --from=builder /usr/local/bin/sg /usr/local/bin/sg +RUN chmod +x /usr/local/bin/sg && sg --version \ + && echo "[runtime] ast-grep copied" + +# Copy pi skills from builder +COPY --from=builder /root/.pi /root/.pi +RUN ls -la /root/.pi/agent/skills/ \ + && echo "[runtime] pi skills configured" + +# Copy worker binary from builder +COPY --from=builder /build/worker /app/worker +RUN chmod +x /app/worker && file /app/worker \ + && echo "[runtime] Worker binary copied" + +# Create app directory structure +RUN mkdir -p /app/work /app/logs /app/screenshots \ + && chmod 755 /app/work /app/logs /app/screenshots \ + && echo "[runtime] App directories created" + +# Health check endpoint +EXPOSE 8081 + +# Worker task queue listener +ENV TEMPORAL_NAMESPACE=poimen-harness \ + TEMPORAL_HOSTPORT=temporal-frontend.temporal:7233 \ + MEMORY_SERVICE_URL=http://memory-service.poimen:5000 \ + MEMORY_SERVICE_TOKEN= \ + ANTHROPIC_API_KEY= \ + PI_SKILLS_PATH=/root/.pi/agent/skills \ + AST_GREP_BIN=/usr/local/bin/sg \ + BROWSER_USE_BIN=/usr/local/bin/browser-use \ + CHROMIUM_BIN=/usr/bin/chromium-browser \ + SCREENSHOTS_DIR=/app/screenshots + +# Entrypoint script with startup diagnostics +COPY --chmod=755 << 'EOF' /app/entrypoint.sh +#!/bin/bash +set -e + +echo "[$(date)] ========== POIMEN WORKER STARTUP ==========" +echo "[$(date)] Container: $HOSTNAME" +echo "[$(date)] Image: $(cat /etc/os-release | grep PRETTY_NAME | cut -d= -f2)" + +# Verify CLI tools +echo "[$(date)] ✓ Checking CLI tools..." +echo " - Go version: $(go version 2>/dev/null || echo 'N/A')" +echo " - ast-grep: $(sg --version 2>&1 | head -1)" +echo " - pi: $(pi --version 2>&1 | head -1)" +echo " - browser-use: $(browser-use --version 2>&1 | head -1)" +echo " - chromium: $(chromium-browser --version 2>&1 || echo 'Not found')" +echo " - git: $(git --version)" +echo " - node: $(node --version)" +echo " - npm: $(npm --version)" + +# Verify pi skills +echo "[$(date)] ✓ Checking pi skills..." +if [ -d "$PI_SKILLS_PATH" ]; then + echo " - Skills path: $PI_SKILLS_PATH" + ls -1 "$PI_SKILLS_PATH" | sed 's/^/ ✓ /' +else + echo " - WARNING: Skills path not found: $PI_SKILLS_PATH" +fi + +# Verify browser tools +echo "[$(date)] ✓ Checking browser automation tools..." +echo " - Chromium binary: $CHROMIUM_BIN" +echo " - Screenshots directory: $SCREENSHOTS_DIR" +if [ -d "$SCREENSHOTS_DIR" ]; then + echo " - Screenshots dir ready ($(du -sh $SCREENSHOTS_DIR 2>/dev/null | cut -f1 || echo '0B'))" +fi + +# Check environment variables +echo "[$(date)] ✓ Configuration loaded:" +echo " - TEMPORAL_NAMESPACE: $TEMPORAL_NAMESPACE" +echo " - TEMPORAL_HOSTPORT: $TEMPORAL_HOSTPORT" +echo " - MEMORY_SERVICE_URL: ${MEMORY_SERVICE_URL:-(not set)}" +echo " - PI_SKILLS_PATH: $PI_SKILLS_PATH" +echo " - CHROMIUM_BIN: $CHROMIUM_BIN" + +# Verify memory service connectivity (optional, non-blocking) +if [ ! -z "$MEMORY_SERVICE_URL" ]; then + echo "[$(date)] ✓ Testing memory service connectivity..." + if curl -sf "$MEMORY_SERVICE_URL/health" > /dev/null 2>&1; then + echo " - Memory service: HEALTHY" + else + echo " - Memory service: UNREACHABLE (will retry in worker)" + fi +fi + +# Test browser automation (optional, non-blocking) +echo "[$(date)] ✓ Testing browser automation..." +if command -v chromium-browser &> /dev/null && command -v browser-use &> /dev/null; then + echo " - Chromium available: YES" + echo " - browser-use available: YES" + echo " - Browser automation: READY" +else + echo " - Browser automation: WARNING - missing dependencies" +fi + +echo "[$(date)] ========== STARTING WORKER ==========" +exec /app/worker +EOF + +RUN chmod +x /app/entrypoint.sh + +# Run worker with diagnostics +ENTRYPOINT ["/app/entrypoint.sh"] + +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ + CMD curl -f http://localhost:8081/health || exit 1 diff --git a/k8s/git-commit.yaml b/k8s/git-commit.yaml index 9a9a66f..3aa372b 100644 --- a/k8s/git-commit.yaml +++ b/k8s/git-commit.yaml @@ -9,6 +9,6 @@ metadata: app.kubernetes.io/name: poimen app.kubernetes.io/component: orchestrator data: - GIT_COMMIT: "ea8499c" # Updated automatically by CI/CD + GIT_COMMIT: "166674d" # Updated automatically by CI/CD GIT_BRANCH: "main" - DEPLOYMENT_DATE: "2026-08-29" + DEPLOYMENT_DATE: "2026-08-30" diff --git a/k8s/worker-deployment.yaml b/k8s/worker-deployment.yaml index 9c00c4e..740b8cb 100644 --- a/k8s/worker-deployment.yaml +++ b/k8s/worker-deployment.yaml @@ -13,8 +13,8 @@ spec: labels: app: poimen-worker annotations: - git-commit: "ea8499c" # ✅ Updated on each push, triggers rolling restart - deployment-date: "2026-08-29" + git-commit: "166674d" # ✅ Updated on each push, triggers rolling restart + deployment-date: "2026-08-30" spec: containers: - name: worker