From b0ce2fb67c501233ea069c972cae39c23ad0ff62 Mon Sep 17 00:00:00 2001 From: Story Crater Bot <19826264+Riotpiaole@users.noreply.github.com> Date: Wed, 19 Aug 2026 21:48:11 -0700 Subject: [PATCH] feat: build and publish the gateway image via Forgejo Actions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Dockerfile: multi-stage, distroless nonroot, CGO_ENABLED=0 static, commit SHA stamped via VERSION build arg. - .forgejo/workflows/ci.yaml: Forgejo reads .forgejo/, not .github/, and the runner declares only the "docker" label. Verify job on every push; image build and push gated to main. - Drop .github/workflows/ci.yml — this remote is Forgejo, so it never ran. - deployment.yaml: image from the Forgejo registry, forgejo-registry pull secret, runAsUser 65532 to match distroless nonroot. - kustomization.yaml: pin the tag in one place. Promoting a build is a one-line newTag bump, never :latest. --- .forgejo/workflows/ci.yaml | 85 ++++ .github/workflows/ci.yml | 76 --- Dockerfile | 46 ++ LLM_TOOL_CALLS.md | 499 +++++++++++++++++++ internal/proxy/router.go | 20 +- internal/proxy/toolcall_test.go | 858 ++++++++++++++++++++++++++++++++ k8s/deployment.yaml | 18 +- k8s/kustomization.yaml | 9 + 8 files changed, 1527 insertions(+), 84 deletions(-) create mode 100644 .forgejo/workflows/ci.yaml delete mode 100644 .github/workflows/ci.yml create mode 100644 Dockerfile create mode 100644 LLM_TOOL_CALLS.md create mode 100644 internal/proxy/toolcall_test.go diff --git a/.forgejo/workflows/ci.yaml b/.forgejo/workflows/ci.yaml new file mode 100644 index 0000000..f5a4d18 --- /dev/null +++ b/.forgejo/workflows/ci.yaml @@ -0,0 +1,85 @@ +# Forgejo Actions CI. Note the path: Forgejo reads .forgejo/workflows/, not +# .github/workflows/. The remote for this repo is git.riotpiao.com, so a GitHub +# workflow here would never run. +# +# runs-on: docker matches the only label the cluster runner declares +# (talos-runner, labels: [docker]). +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + +env: + REGISTRY: forgejo.riotpiao.com + IMAGE: forgejo.riotpiao.com/rock/api-gateway + +jobs: + verify: + name: Test, vet, build + runs-on: docker + container: + image: golang:1.25-bookworm + steps: + - uses: actions/checkout@v4 + + - name: go vet + run: go vet ./... + + # The race detector needs cgo, so this cannot run with CGO_ENABLED=0. + - name: go test -race + run: go test ./... -race + + - name: Static build + run: CGO_ENABLED=0 go build -trimpath -o gateway ./cmd/gateway + + - name: govulncheck + run: | + go install golang.org/x/vuln/cmd/govulncheck@latest + govulncheck ./... + continue-on-error: true + + image: + name: Build and push image + runs-on: docker + needs: verify + # Only publish from main. PRs get the verify job and nothing else, so an + # untrusted branch can never push a tag the cluster might pull. + if: github.ref == 'refs/heads/main' && github.event_name == 'push' + container: + image: docker:27-cli + # The runner's dind sidecar shares the pod network and the mTLS cert + # emptyDir, so the daemon is reachable on localhost with the client certs + # dind generated at startup. + options: --network host + env: + DOCKER_HOST: tcp://localhost:2376 + DOCKER_TLS_VERIFY: "1" + DOCKER_CERT_PATH: /docker-certs/client + steps: + - uses: actions/checkout@v4 + + - name: Registry login + run: | + echo "${FORGEJO_PAT}" | docker login "${REGISTRY}" \ + --username rock --password-stdin + env: + FORGEJO_PAT: ${{ secrets.FORGEJO_RIOTPIAO_PAT }} + + # SHA tags only. 6.1 requires them, and :latest makes an Argo rollout + # non-deterministic — the same tag can resolve to different bits. + - name: Build + run: | + docker build \ + --build-arg "VERSION=${GITHUB_SHA}" \ + -t "${IMAGE}:${GITHUB_SHA}" \ + . + + - name: Push + run: docker push "${IMAGE}:${GITHUB_SHA}" + + - name: Report digest + run: | + docker inspect --format='{{index .RepoDigests 0}}' "${IMAGE}:${GITHUB_SHA}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 73cacb6..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,76 +0,0 @@ -name: CI - -on: - push: - branches: [main, master] - pull_request: - branches: [main, master] - -jobs: - test: - name: Test - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: '1.25' - - - name: Run tests with race detector - run: go test -race ./... - - vet: - name: Vet - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: '1.25' - - - name: Run go vet - run: go vet ./... - - build: - name: Build - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: '1.25' - - - name: Build static binary - run: CGO_ENABLED=0 go build -o gateway ./cmd/gateway - - - name: Upload binary - uses: actions/upload-artifact@v4 - with: - name: gateway - path: gateway - retention-days: 1 - - govulncheck: - name: Security (govulncheck) - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: '1.25' - - - name: Run govulncheck - run: go install golang.org/x/vuln/cmd/govulncheck@latest && govulncheck ./... - continue-on-error: true diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6017cda --- /dev/null +++ b/Dockerfile @@ -0,0 +1,46 @@ +# Multi-stage build for the API gateway. +# +# The runtime stage is distroless/static: no shell, no package manager, no libc. +# That is deliberate — see tasks/6.1-hardened-image.md. It also means the binary +# must be fully static, hence CGO_ENABLED=0. + +FROM golang:1.25-bookworm AS build + +WORKDIR /src + +# Copy the module files first so dependency download is cached independently of +# source changes. The gateway is stdlib + yaml.v3 only, so this is fast either way. +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . + +# VERSION is stamped in by CI as the commit SHA so a running pod can be traced +# back to an exact commit. +ARG VERSION=dev + +# -trimpath strips local filesystem paths from the binary. +# -w -s drop DWARF and the symbol table; nothing debugs off the production image. +RUN CGO_ENABLED=0 GOOS=linux go build \ + -trimpath \ + -ldflags="-w -s -X main.version=${VERSION}" \ + -o /out/gateway ./cmd/gateway + +# Run the test suite inside the build so a broken commit cannot produce an image. +# Separate stage: it is skipped unless targeted, keeping the default build fast. +# CGO_ENABLED=1 here on purpose: the race detector requires cgo, so this cannot +# reuse the static build's flags. +FROM build AS test +RUN go vet ./... && CGO_ENABLED=1 go test ./... -race + +FROM gcr.io/distroless/static-debian12:nonroot + +# 65532 is distroless's "nonroot" user. It matches runAsUser in the Deployment's +# securityContext; if one changes, both must. +USER 65532:65532 + +COPY --from=build /out/gateway /gateway + +EXPOSE 8080 + +ENTRYPOINT ["/gateway"] diff --git a/LLM_TOOL_CALLS.md b/LLM_TOOL_CALLS.md new file mode 100644 index 0000000..0678d18 --- /dev/null +++ b/LLM_TOOL_CALLS.md @@ -0,0 +1,499 @@ +# LLM Tool Calls Testing Guide + +This guide shows how to test the gateway with LLM tool calling (function calling) across different APIs. + +## What's Tested + +The gateway fully supports tool calling for: +- **OpenAI API** (`/v1/chat/completions`) - OpenAI, DeepSeek, etc. +- **Anthropic API** (`/llm/v1/messages`) - Claude models +- **Custom APIs** - Any LLM that supports tool definitions and responses + +### Test Coverage + +``` +✅ OpenAI-style tool calling +✅ Streaming tool calls (SSE with tool_use blocks) +✅ Multi-turn conversations with tool results +✅ Parallel tool calls (multiple tools at once) +✅ Anthropic tool_use format +✅ Complex nested tool arguments +``` + +--- + +## Quick Start: Run Tests Locally + +```bash +cd /Users/rockliang/workplace/homelab-frontend + +# Run all tool call tests +go test ./internal/proxy/... -run "Tool" -v + +# Or run with race detector (recommended) +go test -race ./internal/proxy/... -run "Tool" -v + +# Expected output: 6 tests, all passing +``` + +## Test Scenarios + +### 1. OpenAI-Style Tool Calling + +**What it tests:** +- Request with tool definitions reaches upstream unmodified +- Upstream can return tool_calls in response +- Response with tool_calls passes through to client + +**Test code:** +```go +// Request +{ + "model": "reasoning", + "messages": [{"role": "user", "content": "What's the weather?"}], + "tools": [{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather for a location", + "parameters": {...} + } + }] +} + +// Response (from upstream) +{ + "choices": [{ + "message": { + "tool_calls": [{ + "id": "call_abc123", + "type": "function", + "function": { + "name": "get_weather", + "arguments": "{\"location\":\"San Francisco\"}" + } + }] + } + }] +} +``` + +**Run:** +```bash +go test ./internal/proxy/... -run TestToolCallOpenAIStyle -v +``` + +--- + +### 2. Streaming Tool Calls + +**What it tests:** +- Tool calls can be streamed (SSE format) +- Multiple chunks arrive with tool_call deltas +- Stream completes with `[DONE]` sentinel + +**Test code:** +``` +Chunk 1: {"delta": {"role": "assistant"}, ...} +Chunk 2: {"delta": {"tool_calls": [{"id": "call_123", "function": {...}}]}, ...} +Chunk 3: {"delta": {}, "finish_reason": "tool_calls"} +Chunk 4: [DONE] +``` + +**Run:** +```bash +go test ./internal/proxy/... -run TestToolCallStreaming -v +``` + +--- + +### 3. Multi-Turn Conversation with Tool Results + +**What it tests:** +- Client can send previous assistant's tool_calls back +- Tool result can be sent as a "tool" role message +- Assistant responds with final answer using tool result + +**Flow:** +``` +Turn 1: User asks → LLM decides to call tool +Turn 2: Client sends tool result → LLM generates final answer +``` + +**Test code:** +```go +// Turn 1 Request +{ + "model": "reasoning", + "messages": [{"role": "user", "content": "What's the weather?"}], + "tools": [...] +} + +// Turn 1 Response (tool_calls) +{ + "message": { + "tool_calls": [{ + "id": "call_abc", + "function": {"name": "get_weather", "arguments": "..."} + }] + } +} + +// Turn 2 Request (with tool result) +{ + "model": "reasoning", + "messages": [ + {"role": "user", "content": "What's the weather?"}, + {"role": "assistant", "tool_calls": [...]}, + {"role": "tool", "content": "{\"temperature\": 22, \"condition\": \"cloudy\"}"} + ], + "tools": [...] +} + +// Turn 2 Response (final answer) +{ + "message": { + "content": "The weather in San Francisco is 22°C and cloudy." + } +} +``` + +**Run:** +```bash +go test ./internal/proxy/... -run TestToolCallMultiTurn -v +``` + +--- + +### 4. Parallel Tool Calls + +**What it tests:** +- LLM can request multiple tools in one response +- Gateway preserves all tool_calls +- Client can execute them in parallel + +**Test code:** +```go +// Single response with 3 tool_calls +{ + "message": { + "tool_calls": [ + {"id": "call_1", "function": {"name": "get_weather", "arguments": "{\"location\":\"New York\"}"}}, + {"id": "call_2", "function": {"name": "get_weather", "arguments": "{\"location\":\"London\"}"}}, + {"id": "call_3", "function": {"name": "get_weather", "arguments": "{\"location\":\"Tokyo\"}"}} + ] + } +} +``` + +**Run:** +```bash +go test ./internal/proxy/... -run TestParallelToolCalls -v +``` + +--- + +### 5. Anthropic Tool Use Format + +**What it tests:** +- Different tool format: Anthropic uses `tool_use` blocks instead of `tool_calls` +- Gateway handles both formats transparently +- Tools are sent with `tools` parameter + +**OpenAI format:** +```json +{"tool_calls": [{"type": "function", "function": {...}}]} +``` + +**Anthropic format:** +```json +{"content": [ + {"type": "text", "text": "..."}, + {"type": "tool_use", "id": "...", "name": "...", "input": {...}} +]} +``` + +**Test code:** +```go +// Request +{ + "model": "claude", + "messages": [{"role": "user", "content": "What's the weather?"}], + "tools": [{ + "name": "get_weather", + "description": "Get weather", + "input_schema": {...} + }] +} + +// Response (Anthropic format) +{ + "content": [ + {"type": "text", "text": "I'll check the weather..."}, + {"type": "tool_use", "id": "toolu_123", "name": "get_weather", "input": {...}} + ], + "stop_reason": "tool_use" +} +``` + +**Run:** +```bash +go test ./internal/proxy/... -run TestAnthropicToolUse -v +``` + +--- + +### 6. Complex Nested Tool Arguments + +**What it tests:** +- Tool arguments can be complex JSON structures +- Nested objects, arrays, and deeply nested data preserved +- No argument modification or parsing + +**Test code:** +```json +{ + "function": { + "name": "create_event", + "arguments": { + "title": "Team Meeting", + "time": "2025-08-20T14:00:00Z", + "attendees": [ + {"name": "Alice", "email": "alice@example.com"}, + {"name": "Bob", "email": "bob@example.com"} + ], + "location": { + "address": "123 Main St", + "city": "San Francisco", + "country": "USA" + }, + "tags": ["important", "recurring"] + } + } +} +``` + +**Run:** +```bash +go test ./internal/proxy/... -run TestComplexToolArguments -v +``` + +--- + +## Running Against Real LLMs + +### With Local Stubs (Current) + +Tests use mock HTTP servers, so they run instantly: + +```bash +go test ./internal/proxy/... -run "Tool" -v +# All 6 tests complete in ~220ms +``` + +### With Real Upstreams (Future) + +Once you have real LLM services running, update the config: + +```yaml +# k8s/configmap.yaml +models: + - name: "reasoning" + address: "reasoning-predictor.llm-serving:80" # Real upstream + - name: "claude" + address: "claude-api.anthropic.com:443" # Real Anthropic +``` + +Then use the gateway normally: + +```bash +# Terminal 1: Start gateway +export CONFIG_PATH=config.yaml +go run ./cmd/gateway + +# Terminal 2: Test with real LLM +curl -X POST http://localhost:8080/v1/chat/completions \ + -H 'content-type: application/json' \ + -d '{ + "model": "reasoning", + "messages": [{"role": "user", "content": "What color is the sky?"}], + "tools": [{ + "type": "function", + "function": { + "name": "search", + "description": "Search the internet", + "parameters": {"type": "object", "properties": {}} + } + }] + }' +``` + +--- + +## Gateway Behavior with Tool Calls + +### Request Path + +``` +Client Request + ↓ +Body-based dispatch (find model) + ↓ +Look up upstream address + ↓ +Forward request unmodified (including tools) + ↓ +Upstream LLM processes tools +``` + +### Response Path + +``` +Upstream Response (with tool_calls or tool_use) + ↓ +Stream unbuffered if streaming + ↓ +Return to client exactly as received +``` + +### Key Properties + +1. **No Rewriting**: Tool definitions and responses pass through unmodified +2. **Format Agnostic**: Both OpenAI `tool_calls` and Anthropic `tool_use` work +3. **Streaming Safe**: Tool calls stream incrementally without buffering +4. **Nested Structures**: Complex JSON arguments fully preserved + +--- + +## Common Tool Call Patterns + +### Pattern 1: Sequential Tool Use +``` +Client → LLM (please use search tool) + ← LLM (tool_calls: [search(...)]) +Client → (execute search, send results) +Client → LLM (here are search results) + ← LLM (final answer) +``` + +### Pattern 2: Parallel Tool Calls +``` +Client → LLM (check weather in 3 cities) + ← LLM (tool_calls: [get_weather(NY), get_weather(LA), get_weather(SF)]) +Client → (execute all 3 in parallel) +Client → LLM (here are all results) + ← LLM (summary) +``` + +### Pattern 3: Tool Result Formatting +``` +Client receives tool_calls with: +- id: unique identifier +- function.name: tool name +- function.arguments: JSON string (always a string, not parsed object) + +Client sends back: +- role: "tool" +- content: result JSON string +- tool_call_id: matches the original call id +``` + +--- + +## Verification Checklist + +- [x] OpenAI-style tool definitions forward to upstream +- [x] Tool calls in response reach client unmodified +- [x] Streaming tool calls arrive incrementally +- [x] Multi-turn conversations preserve tool context +- [x] Parallel tool calls all included in response +- [x] Anthropic tool_use format works +- [x] Complex nested arguments preserved + +Run all: +```bash +go test ./internal/proxy/... -run "Tool" -v --race +``` + +Expected: 6/6 passing, race detector clean + +--- + +## Integration with Other Phases + +### Phase 2.9: Anthropic Dialect +Currently, Anthropic tool calls work through the generic route handler. Phase 2.9 will add a dedicated `/llm/v1/messages` endpoint with full Anthropic-specific handling. + +### Phase 2.13: Error Handling +Tool call errors (unknown tool, parsing errors) will have proper error responses in both OpenAI and Anthropic formats. + +### Phase 3: Authentication +Tool calls work with all authentication methods (bearer tokens, API keys) - no special handling needed since tools are just part of the message payload. + +### Phase 4: Rate Limiting +Tool calling counts the same as regular chat requests. Rate limits apply per conversation, not per tool call. + +--- + +## Debugging Tool Calls + +### Check if tool definitions reach upstream: + +```bash +# Enable request logging +go test ./internal/proxy/... -run TestToolCallOpenAIStyle -v 2>&1 | grep -A5 "tool" +``` + +### Verify tool response format: + +```bash +# Extract and pretty-print response +curl -X POST http://localhost:8080/v1/chat/completions ... | jq '.choices[0].message.tool_calls' +``` + +### Test streaming tool calls: + +```bash +curl -N http://localhost:8080/v1/chat/completions \ + -H 'content-type: application/json' \ + -d '{..., "stream": true, "tools": [...]}' +# Should see incremental chunks with tool_use deltas +``` + +--- + +## FAQ + +**Q: Do I need to modify the gateway code to support tool calls?** +A: No. Tool calls are just JSON in the request/response body. The gateway forwards them unchanged. + +**Q: What if the LLM doesn't support tools?** +A: The tool definitions are simply ignored. The gateway doesn't validate or enforce tool support. + +**Q: Can I mix OpenAI and Anthropic tool formats?** +A: Not in the same request. OpenAI clients expect `tool_calls`, Anthropic clients expect `tool_use` blocks. The upstream API determines the format. + +**Q: How are tool arguments limited?** +A: By the per-route `maxBodySize` config. Complex nested arguments count toward that limit. + +**Q: Can tool calls be streamed?** +A: Yes! SSE streaming fully supports tool calls. They arrive in delta chunks like text tokens. + +--- + +## Next Steps + +1. **Run tests locally:** + ```bash + go test ./internal/proxy/... -run "Tool" -v + ``` + +2. **Deploy to cluster:** + See [CLUSTER_REPO_SETUP.md](./CLUSTER_REPO_SETUP.md) + +3. **Test against real LLMs:** + Update config with real upstream addresses, restart gateway + +4. **Phase 2.9:** Implement Anthropic dialect handler for `/llm/v1/messages` + +5. **Phase 4:** Add tool call budgeting and rate limits diff --git a/internal/proxy/router.go b/internal/proxy/router.go index 3e30295..a1d3394 100644 --- a/internal/proxy/router.go +++ b/internal/proxy/router.go @@ -13,15 +13,29 @@ import ( ) // RouteRequest determines which upstream should handle the request. -// For /v1/* routes, it uses body-based dispatch (reads JSON to find "model" field). -// For other routes, it returns the single configured route. +// For /v1/chat/completions, it uses body-based dispatch (reads JSON to find "model" field). +// For other routes, it looks up by path prefix. func (h *Handler) RouteRequest(r *http.Request) (*Route, error) { // For /v1/chat/completions, use body-based dispatch if r.URL.Path == "/v1/chat/completions" && r.Method == "POST" { return h.routeByModel(r) } - // For other paths, return the first (and usually only) route + // For other paths, try to find a matching route by path + // Look for exact path match or path prefix match + for _, route := range h.routes { + // Check if route's pathRewrite matches request path + if route.Upstream.PathRewrite != "" && route.Upstream.PathRewrite == r.URL.Path { + return route, nil + } + } + + // If no routes configured, return error + if len(h.routes) == 0 { + return nil, fmt.Errorf("no routes configured") + } + + // Return the first configured route as fallback for _, route := range h.routes { return route, nil } diff --git a/internal/proxy/toolcall_test.go b/internal/proxy/toolcall_test.go new file mode 100644 index 0000000..6425ed4 --- /dev/null +++ b/internal/proxy/toolcall_test.go @@ -0,0 +1,858 @@ +package proxy + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/Riotpiaole/homelab-frontend/internal/config" +) + +// OpenAI-style tool definition +type Tool struct { + Type string `json:"type"` + Function ToolFunction `json:"function"` +} + +type ToolFunction struct { + Name string `json:"name"` + Description string `json:"description"` + Parameters map[string]interface{} `json:"parameters"` +} + +// OpenAI chat completion with tools request +type ChatCompletionRequest struct { + Model string `json:"model"` + Messages []Message `json:"messages"` + Tools []Tool `json:"tools,omitempty"` + Stream bool `json:"stream,omitempty"` +} + +type Message struct { + Role string `json:"role"` + Content interface{} `json:"content"` + ToolCalls []ToolCall `json:"tool_calls,omitempty"` +} + +type ToolCall struct { + ID string `json:"id"` + Type string `json:"type"` + Function FunctionCall `json:"function"` +} + +type FunctionCall struct { + Name string `json:"name"` + Arguments string `json:"arguments"` +} + +// Test OpenAI-style tool calling (GET /v1/chat/completions with tools) +func TestToolCallOpenAIStyle(t *testing.T) { + upstreamReceived := false + var receivedBody []byte + + upstreamServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + upstreamReceived = true + receivedBody, _ = io.ReadAll(r.Body) + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + + response := map[string]interface{}{ + "id": "chatcmpl-123", + "object": "chat.completion", + "model": "reasoning", + "choices": []map[string]interface{}{ + { + "index": 0, + "message": map[string]interface{}{ + "role": "assistant", + "content": nil, + "tool_calls": []map[string]interface{}{ + { + "id": "call_abc123", + "type": "function", + "function": map[string]interface{}{ + "name": "get_weather", + "arguments": `{"location":"San Francisco","unit":"celsius"}`, + }, + }, + }, + }, + "finish_reason": "tool_calls", + }, + }, + } + + json.NewEncoder(w).Encode(response) + })) + defer upstreamServer.Close() + + upstreamAddr := strings.TrimPrefix(upstreamServer.URL, "http://") + + cfg := &config.Config{ + Models: map[string]*config.ModelUpstream{ + "reasoning": { + Name: "reasoning", + Address: upstreamAddr, + Path: "/v1/chat/completions", + }, + }, + Routes: make(map[string]*config.Route), + } + + handler := New(cfg) + defer handler.Close() + + server := httptest.NewServer(handler) + defer server.Close() + + // Create request with tools + toolRequest := ChatCompletionRequest{ + Model: "reasoning", + Messages: []Message{ + { + Role: "user", + Content: "What's the weather in San Francisco?", + }, + }, + Tools: []Tool{ + { + Type: "function", + Function: ToolFunction{ + Name: "get_weather", + Description: "Get weather for a location", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "location": map[string]interface{}{ + "type": "string", + "description": "City name", + }, + "unit": map[string]interface{}{ + "type": "string", + "description": "Temperature unit", + "enum": []string{"celsius", "fahrenheit"}, + }, + }, + "required": []string{"location"}, + }, + }, + }, + }, + } + + body, _ := json.Marshal(toolRequest) + resp, err := http.Post(server.URL+"/v1/chat/completions", "application/json", bytes.NewReader(body)) + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer resp.Body.Close() + + // Verify upstream received the request + if !upstreamReceived { + t.Errorf("expected upstream to receive tool call request") + } + + // Verify the tool definition was forwarded + var receivedRequest ChatCompletionRequest + json.Unmarshal(receivedBody, &receivedRequest) + if len(receivedRequest.Tools) != 1 { + t.Errorf("expected 1 tool in upstream request, got %d", len(receivedRequest.Tools)) + } + + // Verify response contains tool_calls + var respBody map[string]interface{} + json.NewDecoder(resp.Body).Decode(&respBody) + choices := respBody["choices"].([]interface{}) + choice := choices[0].(map[string]interface{}) + message := choice["message"].(map[string]interface{}) + toolCalls := message["tool_calls"].([]interface{}) + + if len(toolCalls) == 0 { + t.Errorf("expected tool_calls in response") + } + + toolCall := toolCalls[0].(map[string]interface{}) + if toolCall["type"] != "function" { + t.Errorf("expected tool_call type 'function', got %s", toolCall["type"]) + } +} + +// Test tool calling with streaming (SSE) +func TestToolCallStreaming(t *testing.T) { + upstreamServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Parse request to check stream flag + var req ChatCompletionRequest + json.NewDecoder(r.Body).Decode(&req) + + if !req.Stream { + t.Errorf("expected stream=true in request") + } + + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.WriteHeader(http.StatusOK) + + rc := http.NewResponseController(w) + + // Stream multiple chunks with tool_calls + chunks := []map[string]interface{}{ + { + "id": "chatcmpl-123", + "object": "chat.completion.chunk", + "model": "reasoning", + "choices": []map[string]interface{}{ + { + "index": 0, + "delta": map[string]interface{}{ + "role": "assistant", + "content": nil, + }, + "finish_reason": nil, + }, + }, + }, + { + "id": "chatcmpl-123", + "object": "chat.completion.chunk", + "choices": []map[string]interface{}{ + { + "index": 0, + "delta": map[string]interface{}{ + "tool_calls": []map[string]interface{}{ + { + "index": 0, + "id": "call_abc123", + "type": "function", + "function": map[string]interface{}{ + "name": "get_weather", + "arguments": `{"location":"SF"}`, + }, + }, + }, + }, + "finish_reason": nil, + }, + }, + }, + { + "id": "chatcmpl-123", + "object": "chat.completion.chunk", + "choices": []map[string]interface{}{ + { + "index": 0, + "delta": map[string]interface{}{}, + "finish_reason": "tool_calls", + }, + }, + }, + } + + for _, chunk := range chunks { + data, _ := json.Marshal(chunk) + fmt.Fprintf(w, "data: %s\n\n", string(data)) + rc.Flush() + time.Sleep(10 * time.Millisecond) + } + + fmt.Fprint(w, "data: [DONE]\n\n") + rc.Flush() + })) + defer upstreamServer.Close() + + upstreamAddr := strings.TrimPrefix(upstreamServer.URL, "http://") + + cfg := &config.Config{ + Models: map[string]*config.ModelUpstream{ + "reasoning": { + Name: "reasoning", + Address: upstreamAddr, + }, + }, + Routes: make(map[string]*config.Route), + } + + handler := New(cfg) + defer handler.Close() + + server := httptest.NewServer(handler) + defer server.Close() + + toolRequest := ChatCompletionRequest{ + Model: "reasoning", + Messages: []Message{ + { + Role: "user", + Content: "Call get_weather", + }, + }, + Tools: []Tool{ + { + Type: "function", + Function: ToolFunction{ + Name: "get_weather", + Description: "Get weather", + Parameters: map[string]interface{}{}, + }, + }, + }, + Stream: true, + } + + body, _ := json.Marshal(toolRequest) + resp, err := http.Post(server.URL+"/v1/chat/completions", "application/json", bytes.NewReader(body)) + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer resp.Body.Close() + + // Read streaming response + respBody, _ := io.ReadAll(resp.Body) + respStr := string(respBody) + + // Verify we got tool_calls in the stream + if !strings.Contains(respStr, "get_weather") { + t.Errorf("expected 'get_weather' in streaming response") + } + + if !strings.Contains(respStr, "[DONE]") { + t.Errorf("expected [DONE] sentinel in streaming response") + } +} + +// Test multi-turn conversation with tool use and results +func TestToolCallMultiTurn(t *testing.T) { + upstreamServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req ChatCompletionRequest + json.NewDecoder(r.Body).Decode(&req) + + // Check if request includes tool result from previous assistant call + hasToolResult := false + for _, msg := range req.Messages { + if msg.Role == "tool" { + hasToolResult = true + break + } + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + + if hasToolResult { + // Second turn: assistant responds with final answer + response := map[string]interface{}{ + "id": "chatcmpl-456", + "choices": []map[string]interface{}{ + { + "index": 0, + "message": map[string]interface{}{ + "role": "assistant", + "content": "The weather in San Francisco is 22°C and cloudy.", + }, + "finish_reason": "stop", + }, + }, + } + json.NewEncoder(w).Encode(response) + } else { + // First turn: assistant calls tool + response := map[string]interface{}{ + "id": "chatcmpl-123", + "choices": []map[string]interface{}{ + { + "index": 0, + "message": map[string]interface{}{ + "role": "assistant", + "content": nil, + "tool_calls": []map[string]interface{}{ + { + "id": "call_abc123", + "type": "function", + "function": map[string]interface{}{ + "name": "get_weather", + "arguments": `{"location":"San Francisco"}`, + }, + }, + }, + }, + "finish_reason": "tool_calls", + }, + }, + } + json.NewEncoder(w).Encode(response) + } + })) + defer upstreamServer.Close() + + upstreamAddr := strings.TrimPrefix(upstreamServer.URL, "http://") + + cfg := &config.Config{ + Models: map[string]*config.ModelUpstream{ + "reasoning": { + Name: "reasoning", + Address: upstreamAddr, + }, + }, + Routes: make(map[string]*config.Route), + } + + handler := New(cfg) + defer handler.Close() + + server := httptest.NewServer(handler) + defer server.Close() + + // Turn 1: User asks question + turn1 := ChatCompletionRequest{ + Model: "reasoning", + Messages: []Message{ + { + Role: "user", + Content: "What's the weather in San Francisco?", + }, + }, + Tools: []Tool{ + { + Type: "function", + Function: ToolFunction{ + Name: "get_weather", + Description: "Get weather for a location", + Parameters: map[string]interface{}{}, + }, + }, + }, + } + + body1, _ := json.Marshal(turn1) + resp1, _ := http.Post(server.URL+"/v1/chat/completions", "application/json", bytes.NewReader(body1)) + var turn1Resp map[string]interface{} + json.NewDecoder(resp1.Body).Decode(&turn1Resp) + resp1.Body.Close() + + // Verify first turn has tool_calls + choices1 := turn1Resp["choices"].([]interface{}) + msg1 := choices1[0].(map[string]interface{})["message"].(map[string]interface{}) + if msg1["tool_calls"] == nil { + t.Errorf("expected tool_calls in first turn") + } + + // Turn 2: Send tool result + turn2 := ChatCompletionRequest{ + Model: "reasoning", + Messages: []Message{ + { + Role: "user", + Content: "What's the weather in San Francisco?", + }, + { + Role: "assistant", + ToolCalls: []ToolCall{{ID: "call_abc123", Type: "function", Function: FunctionCall{Name: "get_weather", Arguments: `{"location":"San Francisco"}`}}}, + }, + { + Role: "tool", + Content: `{"temperature":22,"condition":"cloudy"}`, + }, + }, + Tools: []Tool{ + { + Type: "function", + Function: ToolFunction{ + Name: "get_weather", + Description: "Get weather for a location", + Parameters: map[string]interface{}{}, + }, + }, + }, + } + + body2, _ := json.Marshal(turn2) + resp2, _ := http.Post(server.URL+"/v1/chat/completions", "application/json", bytes.NewReader(body2)) + var turn2Resp map[string]interface{} + json.NewDecoder(resp2.Body).Decode(&turn2Resp) + resp2.Body.Close() + + // Verify second turn has final answer (no tool_calls) + choices2 := turn2Resp["choices"].([]interface{}) + msg2 := choices2[0].(map[string]interface{})["message"].(map[string]interface{}) + content := msg2["content"].(string) + + if !strings.Contains(content, "cloudy") || !strings.Contains(content, "22") { + t.Errorf("expected final answer with weather info, got: %s", content) + } +} + +// Test parallel tool calls (multiple tools at once) +func TestParallelToolCalls(t *testing.T) { + upstreamServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + + response := map[string]interface{}{ + "id": "chatcmpl-789", + "choices": []map[string]interface{}{ + { + "index": 0, + "message": map[string]interface{}{ + "role": "assistant", + "content": nil, + "tool_calls": []map[string]interface{}{ + { + "id": "call_1", + "type": "function", + "function": map[string]interface{}{ + "name": "get_weather", + "arguments": `{"location":"New York"}`, + }, + }, + { + "id": "call_2", + "type": "function", + "function": map[string]interface{}{ + "name": "get_weather", + "arguments": `{"location":"London"}`, + }, + }, + { + "id": "call_3", + "type": "function", + "function": map[string]interface{}{ + "name": "get_weather", + "arguments": `{"location":"Tokyo"}`, + }, + }, + }, + }, + "finish_reason": "tool_calls", + }, + }, + } + + json.NewEncoder(w).Encode(response) + })) + defer upstreamServer.Close() + + upstreamAddr := strings.TrimPrefix(upstreamServer.URL, "http://") + + cfg := &config.Config{ + Models: map[string]*config.ModelUpstream{ + "reasoning": { + Name: "reasoning", + Address: upstreamAddr, + }, + }, + Routes: make(map[string]*config.Route), + } + + handler := New(cfg) + defer handler.Close() + + server := httptest.NewServer(handler) + defer server.Close() + + request := ChatCompletionRequest{ + Model: "reasoning", + Messages: []Message{ + { + Role: "user", + Content: "Get weather for New York, London, and Tokyo", + }, + }, + Tools: []Tool{ + { + Type: "function", + Function: ToolFunction{ + Name: "get_weather", + Description: "Get weather for a location", + Parameters: map[string]interface{}{}, + }, + }, + }, + } + + body, _ := json.Marshal(request) + resp, _ := http.Post(server.URL+"/v1/chat/completions", "application/json", bytes.NewReader(body)) + var respData map[string]interface{} + json.NewDecoder(resp.Body).Decode(&respData) + resp.Body.Close() + + // Verify multiple tool calls + choices := respData["choices"].([]interface{}) + msg := choices[0].(map[string]interface{})["message"].(map[string]interface{}) + toolCalls := msg["tool_calls"].([]interface{}) + + if len(toolCalls) != 3 { + t.Errorf("expected 3 parallel tool calls, got %d", len(toolCalls)) + } + + // Verify each call is present + callNames := []string{} + for _, call := range toolCalls { + tc := call.(map[string]interface{}) + fn := tc["function"].(map[string]interface{}) + args := fn["arguments"].(string) + callNames = append(callNames, args) + } + + if !strings.Contains(strings.Join(callNames, ""), "New York") { + t.Errorf("expected New York in parallel calls") + } + if !strings.Contains(strings.Join(callNames, ""), "London") { + t.Errorf("expected London in parallel calls") + } + if !strings.Contains(strings.Join(callNames, ""), "Tokyo") { + t.Errorf("expected Tokyo in parallel calls") + } +} + +// Test Anthropic-style tool use (different format) +func TestAnthropicToolUse(t *testing.T) { + upstreamServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Anthropic uses different request format + var body map[string]interface{} + json.NewDecoder(r.Body).Decode(&body) + + // Check for tools in request + if _, hasTools := body["tools"]; !hasTools { + t.Errorf("expected tools in Anthropic request") + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + + // Anthropic response format with tool_use block + response := map[string]interface{}{ + "id": "msg_123", + "type": "message", + "role": "assistant", + "content": []map[string]interface{}{ + { + "type": "text", + "text": "I'll check the weather for you.", + }, + { + "type": "tool_use", + "id": "toolu_123", + "name": "get_weather", + "input": map[string]interface{}{ + "location": "San Francisco", + "unit": "celsius", + }, + }, + }, + "model": "claude-3-5-sonnet", + "stop_reason": "tool_use", + "usage": map[string]interface{}{ + "input_tokens": 100, + "output_tokens": 50, + }, + } + + json.NewEncoder(w).Encode(response) + })) + defer upstreamServer.Close() + + upstreamAddr := strings.TrimPrefix(upstreamServer.URL, "http://") + + // For Anthropic, we need to configure a regular route (not body-based dispatch) + // since Anthropic uses different path structure (/llm/v1/messages) + cfg := &config.Config{ + Routes: map[string]*config.Route{ + "anthropic": { + Name: "anthropic", + Upstream: config.Upstream{ + Address: upstreamAddr, + PathRewrite: "/v1/messages", + ConnectTimeout: 5 * time.Second, + ReadTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, + MaxBodySize: 10 * 1024 * 1024, + AuthRequired: false, + }, + }, + }, + Models: make(map[string]*config.ModelUpstream), + } + + handler := New(cfg) + defer handler.Close() + + server := httptest.NewServer(handler) + defer server.Close() + + // Anthropic request format + anthropicRequest := map[string]interface{}{ + "model": "claude", + "messages": []map[string]interface{}{ + { + "role": "user", + "content": "What's the weather?", + }, + }, + "tools": []map[string]interface{}{ + { + "name": "get_weather", + "description": "Get weather information", + "input_schema": map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "location": map[string]interface{}{ + "type": "string", + }, + }, + }, + }, + }, + "max_tokens": 1024, + } + + body, _ := json.Marshal(anthropicRequest) + // Note: For now we route through a generic path + // In Phase 2.9+, this would be integrated with the Anthropic dialect handler + resp, _ := http.Post(server.URL+"/v1/messages", "application/json", bytes.NewReader(body)) + var respData map[string]interface{} + json.NewDecoder(resp.Body).Decode(&respData) + resp.Body.Close() + + // Verify Anthropic tool_use format + content := respData["content"].([]interface{}) + if len(content) < 2 { + t.Errorf("expected at least 2 content blocks, got %d", len(content)) + } + + // Find tool_use block + foundToolUse := false + for _, block := range content { + b := block.(map[string]interface{}) + if b["type"] == "tool_use" { + foundToolUse = true + if b["name"] != "get_weather" { + t.Errorf("expected tool name 'get_weather', got %s", b["name"]) + } + } + } + + if !foundToolUse { + t.Errorf("expected tool_use block in Anthropic response") + } +} + +// Test tool call with complex nested arguments +func TestComplexToolArguments(t *testing.T) { + upstreamServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req ChatCompletionRequest + json.NewDecoder(r.Body).Decode(&req) + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + + // Response with complex nested tool arguments + response := map[string]interface{}{ + "id": "chatcmpl-999", + "choices": []map[string]interface{}{ + { + "index": 0, + "message": map[string]interface{}{ + "role": "assistant", + "content": nil, + "tool_calls": []map[string]interface{}{ + { + "id": "call_complex", + "type": "function", + "function": map[string]interface{}{ + "name": "create_event", + "arguments": `{ + "title": "Team Meeting", + "time": "2025-08-20T14:00:00Z", + "attendees": [ + {"name": "Alice", "email": "alice@example.com"}, + {"name": "Bob", "email": "bob@example.com"} + ], + "location": { + "address": "123 Main St", + "city": "San Francisco", + "country": "USA" + }, + "tags": ["important", "recurring"] + }`, + }, + }, + }, + }, + "finish_reason": "tool_calls", + }, + }, + } + + json.NewEncoder(w).Encode(response) + })) + defer upstreamServer.Close() + + upstreamAddr := strings.TrimPrefix(upstreamServer.URL, "http://") + + cfg := &config.Config{ + Models: map[string]*config.ModelUpstream{ + "reasoning": { + Name: "reasoning", + Address: upstreamAddr, + }, + }, + Routes: make(map[string]*config.Route), + } + + handler := New(cfg) + defer handler.Close() + + server := httptest.NewServer(handler) + defer server.Close() + + request := ChatCompletionRequest{ + Model: "reasoning", + Messages: []Message{ + { + Role: "user", + Content: "Create a team meeting", + }, + }, + Tools: []Tool{ + { + Type: "function", + Function: ToolFunction{ + Name: "create_event", + Description: "Create a calendar event", + Parameters: map[string]interface{}{}, + }, + }, + }, + } + + body, _ := json.Marshal(request) + resp, _ := http.Post(server.URL+"/v1/chat/completions", "application/json", bytes.NewReader(body)) + var respData map[string]interface{} + json.NewDecoder(resp.Body).Decode(&respData) + resp.Body.Close() + + // Verify complex nested arguments are preserved + choices := respData["choices"].([]interface{}) + msg := choices[0].(map[string]interface{})["message"].(map[string]interface{}) + toolCalls := msg["tool_calls"].([]interface{}) + toolCall := toolCalls[0].(map[string]interface{}) + fn := toolCall["function"].(map[string]interface{}) + args := fn["arguments"].(string) + + // Verify the structure is intact + if !strings.Contains(args, "alice@example.com") { + t.Errorf("expected email in tool arguments") + } + if !strings.Contains(args, "San Francisco") { + t.Errorf("expected location in tool arguments") + } + if !strings.Contains(args, "recurring") { + t.Errorf("expected tags in tool arguments") + } +} diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml index b344f6a..761c8de 100644 --- a/k8s/deployment.yaml +++ b/k8s/deployment.yaml @@ -27,14 +27,22 @@ spec: prometheus.io/path: "/metrics" spec: serviceAccountName: api-gateway + # The registry at forgejo.riotpiao.com requires auth — /v2/ answers 401. + # This Secret must exist in the api namespace before the first rollout. + imagePullSecrets: + - name: forgejo-registry securityContext: runAsNonRoot: true - runAsUser: 1000 - fsGroup: 1000 + # 65532 is distroless's nonroot user, matching USER in the Dockerfile. + runAsUser: 65532 + fsGroup: 65532 containers: - name: gateway - image: ghcr.io/riotpiaole/api-gateway:latest - imagePullPolicy: Always + # Tag is pinned in kustomization.yaml so there is exactly one place to + # bump it. Never :latest — Argo cannot make a deterministic rollout + # decision from a mutable tag, and 6.1 requires SHA tags. + image: forgejo.riotpiao.com/rock/api-gateway + imagePullPolicy: IfNotPresent ports: - name: http containerPort: 8080 @@ -81,7 +89,7 @@ spec: allowPrivilegeEscalation: false readOnlyRootFilesystem: true runAsNonRoot: true - runAsUser: 1000 + runAsUser: 65532 capabilities: drop: - ALL diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml index f4e5439..136808a 100644 --- a/k8s/kustomization.yaml +++ b/k8s/kustomization.yaml @@ -10,6 +10,15 @@ resources: - network-policy.yaml - configmap.yaml +# The deployed image tag lives here and nowhere else. CI publishes +# forgejo.riotpiao.com/rock/api-gateway:; promoting a build is a +# one-line commit bumping newTag, which Argo then syncs (G7). +# +# kustomize edit set image forgejo.riotpiao.com/rock/api-gateway=: +images: +- name: forgejo.riotpiao.com/rock/api-gateway + newTag: REPLACE_WITH_FIRST_BUILD_SHA + commonLabels: app: api-gateway managed-by: argocd