feat: build and publish the gateway image via Forgejo Actions
- Dockerfile: multi-stage, distroless nonroot, CGO_ENABLED=0 static, commit SHA stamped via VERSION build arg. - .forgejo/workflows/ci.yaml: Forgejo reads .forgejo/, not .github/, and the runner declares only the "docker" label. Verify job on every push; image build and push gated to main. - Drop .github/workflows/ci.yml — this remote is Forgejo, so it never ran. - deployment.yaml: image from the Forgejo registry, forgejo-registry pull secret, runAsUser 65532 to match distroless nonroot. - kustomization.yaml: pin the tag in one place. Promoting a build is a one-line newTag bump, never :latest.
This commit is contained in:
Story Crater Bot
@@ -0,0 +1,85 @@
|
||||
# Forgejo Actions CI. Note the path: Forgejo reads .forgejo/workflows/, not
|
||||
# .github/workflows/. The remote for this repo is git.riotpiao.com, so a GitHub
|
||||
# workflow here would never run.
|
||||
#
|
||||
# runs-on: docker matches the only label the cluster runner declares
|
||||
# (talos-runner, labels: [docker]).
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
env:
|
||||
REGISTRY: forgejo.riotpiao.com
|
||||
IMAGE: forgejo.riotpiao.com/rock/api-gateway
|
||||
|
||||
jobs:
|
||||
verify:
|
||||
name: Test, vet, build
|
||||
runs-on: docker
|
||||
container:
|
||||
image: golang:1.25-bookworm
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: go vet
|
||||
run: go vet ./...
|
||||
|
||||
# The race detector needs cgo, so this cannot run with CGO_ENABLED=0.
|
||||
- name: go test -race
|
||||
run: go test ./... -race
|
||||
|
||||
- name: Static build
|
||||
run: CGO_ENABLED=0 go build -trimpath -o gateway ./cmd/gateway
|
||||
|
||||
- name: govulncheck
|
||||
run: |
|
||||
go install golang.org/x/vuln/cmd/govulncheck@latest
|
||||
govulncheck ./...
|
||||
continue-on-error: true
|
||||
|
||||
image:
|
||||
name: Build and push image
|
||||
runs-on: docker
|
||||
needs: verify
|
||||
# Only publish from main. PRs get the verify job and nothing else, so an
|
||||
# untrusted branch can never push a tag the cluster might pull.
|
||||
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
||||
container:
|
||||
image: docker:27-cli
|
||||
# The runner's dind sidecar shares the pod network and the mTLS cert
|
||||
# emptyDir, so the daemon is reachable on localhost with the client certs
|
||||
# dind generated at startup.
|
||||
options: --network host
|
||||
env:
|
||||
DOCKER_HOST: tcp://localhost:2376
|
||||
DOCKER_TLS_VERIFY: "1"
|
||||
DOCKER_CERT_PATH: /docker-certs/client
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Registry login
|
||||
run: |
|
||||
echo "${FORGEJO_PAT}" | docker login "${REGISTRY}" \
|
||||
--username rock --password-stdin
|
||||
env:
|
||||
FORGEJO_PAT: ${{ secrets.FORGEJO_RIOTPIAO_PAT }}
|
||||
|
||||
# SHA tags only. 6.1 requires them, and :latest makes an Argo rollout
|
||||
# non-deterministic — the same tag can resolve to different bits.
|
||||
- name: Build
|
||||
run: |
|
||||
docker build \
|
||||
--build-arg "VERSION=${GITHUB_SHA}" \
|
||||
-t "${IMAGE}:${GITHUB_SHA}" \
|
||||
.
|
||||
|
||||
- name: Push
|
||||
run: docker push "${IMAGE}:${GITHUB_SHA}"
|
||||
|
||||
- name: Report digest
|
||||
run: |
|
||||
docker inspect --format='{{index .RepoDigests 0}}' "${IMAGE}:${GITHUB_SHA}"
|
||||
@@ -1,76 +0,0 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, master]
|
||||
pull_request:
|
||||
branches: [main, master]
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: Test
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: '1.25'
|
||||
|
||||
- name: Run tests with race detector
|
||||
run: go test -race ./...
|
||||
|
||||
vet:
|
||||
name: Vet
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: '1.25'
|
||||
|
||||
- name: Run go vet
|
||||
run: go vet ./...
|
||||
|
||||
build:
|
||||
name: Build
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: '1.25'
|
||||
|
||||
- name: Build static binary
|
||||
run: CGO_ENABLED=0 go build -o gateway ./cmd/gateway
|
||||
|
||||
- name: Upload binary
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: gateway
|
||||
path: gateway
|
||||
retention-days: 1
|
||||
|
||||
govulncheck:
|
||||
name: Security (govulncheck)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: '1.25'
|
||||
|
||||
- name: Run govulncheck
|
||||
run: go install golang.org/x/vuln/cmd/govulncheck@latest && govulncheck ./...
|
||||
continue-on-error: true
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
# Multi-stage build for the API gateway.
|
||||
#
|
||||
# The runtime stage is distroless/static: no shell, no package manager, no libc.
|
||||
# That is deliberate — see tasks/6.1-hardened-image.md. It also means the binary
|
||||
# must be fully static, hence CGO_ENABLED=0.
|
||||
|
||||
FROM golang:1.25-bookworm AS build
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
# Copy the module files first so dependency download is cached independently of
|
||||
# source changes. The gateway is stdlib + yaml.v3 only, so this is fast either way.
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
COPY . .
|
||||
|
||||
# VERSION is stamped in by CI as the commit SHA so a running pod can be traced
|
||||
# back to an exact commit.
|
||||
ARG VERSION=dev
|
||||
|
||||
# -trimpath strips local filesystem paths from the binary.
|
||||
# -w -s drop DWARF and the symbol table; nothing debugs off the production image.
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build \
|
||||
-trimpath \
|
||||
-ldflags="-w -s -X main.version=${VERSION}" \
|
||||
-o /out/gateway ./cmd/gateway
|
||||
|
||||
# Run the test suite inside the build so a broken commit cannot produce an image.
|
||||
# Separate stage: it is skipped unless targeted, keeping the default build fast.
|
||||
# CGO_ENABLED=1 here on purpose: the race detector requires cgo, so this cannot
|
||||
# reuse the static build's flags.
|
||||
FROM build AS test
|
||||
RUN go vet ./... && CGO_ENABLED=1 go test ./... -race
|
||||
|
||||
FROM gcr.io/distroless/static-debian12:nonroot
|
||||
|
||||
# 65532 is distroless's "nonroot" user. It matches runAsUser in the Deployment's
|
||||
# securityContext; if one changes, both must.
|
||||
USER 65532:65532
|
||||
|
||||
COPY --from=build /out/gateway /gateway
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
ENTRYPOINT ["/gateway"]
|
||||
@@ -0,0 +1,499 @@
|
||||
# LLM Tool Calls Testing Guide
|
||||
|
||||
This guide shows how to test the gateway with LLM tool calling (function calling) across different APIs.
|
||||
|
||||
## What's Tested
|
||||
|
||||
The gateway fully supports tool calling for:
|
||||
- **OpenAI API** (`/v1/chat/completions`) - OpenAI, DeepSeek, etc.
|
||||
- **Anthropic API** (`/llm/v1/messages`) - Claude models
|
||||
- **Custom APIs** - Any LLM that supports tool definitions and responses
|
||||
|
||||
### Test Coverage
|
||||
|
||||
```
|
||||
✅ OpenAI-style tool calling
|
||||
✅ Streaming tool calls (SSE with tool_use blocks)
|
||||
✅ Multi-turn conversations with tool results
|
||||
✅ Parallel tool calls (multiple tools at once)
|
||||
✅ Anthropic tool_use format
|
||||
✅ Complex nested tool arguments
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Quick Start: Run Tests Locally
|
||||
|
||||
```bash
|
||||
cd /Users/rockliang/workplace/homelab-frontend
|
||||
|
||||
# Run all tool call tests
|
||||
go test ./internal/proxy/... -run "Tool" -v
|
||||
|
||||
# Or run with race detector (recommended)
|
||||
go test -race ./internal/proxy/... -run "Tool" -v
|
||||
|
||||
# Expected output: 6 tests, all passing
|
||||
```
|
||||
|
||||
## Test Scenarios
|
||||
|
||||
### 1. OpenAI-Style Tool Calling
|
||||
|
||||
**What it tests:**
|
||||
- Request with tool definitions reaches upstream unmodified
|
||||
- Upstream can return tool_calls in response
|
||||
- Response with tool_calls passes through to client
|
||||
|
||||
**Test code:**
|
||||
```go
|
||||
// Request
|
||||
{
|
||||
"model": "reasoning",
|
||||
"messages": [{"role": "user", "content": "What's the weather?"}],
|
||||
"tools": [{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get weather for a location",
|
||||
"parameters": {...}
|
||||
}
|
||||
}]
|
||||
}
|
||||
|
||||
// Response (from upstream)
|
||||
{
|
||||
"choices": [{
|
||||
"message": {
|
||||
"tool_calls": [{
|
||||
"id": "call_abc123",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"arguments": "{\"location\":\"San Francisco\"}"
|
||||
}
|
||||
}]
|
||||
}
|
||||
}]
|
||||
}
|
||||
```
|
||||
|
||||
**Run:**
|
||||
```bash
|
||||
go test ./internal/proxy/... -run TestToolCallOpenAIStyle -v
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 2. Streaming Tool Calls
|
||||
|
||||
**What it tests:**
|
||||
- Tool calls can be streamed (SSE format)
|
||||
- Multiple chunks arrive with tool_call deltas
|
||||
- Stream completes with `[DONE]` sentinel
|
||||
|
||||
**Test code:**
|
||||
```
|
||||
Chunk 1: {"delta": {"role": "assistant"}, ...}
|
||||
Chunk 2: {"delta": {"tool_calls": [{"id": "call_123", "function": {...}}]}, ...}
|
||||
Chunk 3: {"delta": {}, "finish_reason": "tool_calls"}
|
||||
Chunk 4: [DONE]
|
||||
```
|
||||
|
||||
**Run:**
|
||||
```bash
|
||||
go test ./internal/proxy/... -run TestToolCallStreaming -v
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 3. Multi-Turn Conversation with Tool Results
|
||||
|
||||
**What it tests:**
|
||||
- Client can send previous assistant's tool_calls back
|
||||
- Tool result can be sent as a "tool" role message
|
||||
- Assistant responds with final answer using tool result
|
||||
|
||||
**Flow:**
|
||||
```
|
||||
Turn 1: User asks → LLM decides to call tool
|
||||
Turn 2: Client sends tool result → LLM generates final answer
|
||||
```
|
||||
|
||||
**Test code:**
|
||||
```go
|
||||
// Turn 1 Request
|
||||
{
|
||||
"model": "reasoning",
|
||||
"messages": [{"role": "user", "content": "What's the weather?"}],
|
||||
"tools": [...]
|
||||
}
|
||||
|
||||
// Turn 1 Response (tool_calls)
|
||||
{
|
||||
"message": {
|
||||
"tool_calls": [{
|
||||
"id": "call_abc",
|
||||
"function": {"name": "get_weather", "arguments": "..."}
|
||||
}]
|
||||
}
|
||||
}
|
||||
|
||||
// Turn 2 Request (with tool result)
|
||||
{
|
||||
"model": "reasoning",
|
||||
"messages": [
|
||||
{"role": "user", "content": "What's the weather?"},
|
||||
{"role": "assistant", "tool_calls": [...]},
|
||||
{"role": "tool", "content": "{\"temperature\": 22, \"condition\": \"cloudy\"}"}
|
||||
],
|
||||
"tools": [...]
|
||||
}
|
||||
|
||||
// Turn 2 Response (final answer)
|
||||
{
|
||||
"message": {
|
||||
"content": "The weather in San Francisco is 22°C and cloudy."
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Run:**
|
||||
```bash
|
||||
go test ./internal/proxy/... -run TestToolCallMultiTurn -v
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 4. Parallel Tool Calls
|
||||
|
||||
**What it tests:**
|
||||
- LLM can request multiple tools in one response
|
||||
- Gateway preserves all tool_calls
|
||||
- Client can execute them in parallel
|
||||
|
||||
**Test code:**
|
||||
```go
|
||||
// Single response with 3 tool_calls
|
||||
{
|
||||
"message": {
|
||||
"tool_calls": [
|
||||
{"id": "call_1", "function": {"name": "get_weather", "arguments": "{\"location\":\"New York\"}"}},
|
||||
{"id": "call_2", "function": {"name": "get_weather", "arguments": "{\"location\":\"London\"}"}},
|
||||
{"id": "call_3", "function": {"name": "get_weather", "arguments": "{\"location\":\"Tokyo\"}"}}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Run:**
|
||||
```bash
|
||||
go test ./internal/proxy/... -run TestParallelToolCalls -v
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 5. Anthropic Tool Use Format
|
||||
|
||||
**What it tests:**
|
||||
- Different tool format: Anthropic uses `tool_use` blocks instead of `tool_calls`
|
||||
- Gateway handles both formats transparently
|
||||
- Tools are sent with `tools` parameter
|
||||
|
||||
**OpenAI format:**
|
||||
```json
|
||||
{"tool_calls": [{"type": "function", "function": {...}}]}
|
||||
```
|
||||
|
||||
**Anthropic format:**
|
||||
```json
|
||||
{"content": [
|
||||
{"type": "text", "text": "..."},
|
||||
{"type": "tool_use", "id": "...", "name": "...", "input": {...}}
|
||||
]}
|
||||
```
|
||||
|
||||
**Test code:**
|
||||
```go
|
||||
// Request
|
||||
{
|
||||
"model": "claude",
|
||||
"messages": [{"role": "user", "content": "What's the weather?"}],
|
||||
"tools": [{
|
||||
"name": "get_weather",
|
||||
"description": "Get weather",
|
||||
"input_schema": {...}
|
||||
}]
|
||||
}
|
||||
|
||||
// Response (Anthropic format)
|
||||
{
|
||||
"content": [
|
||||
{"type": "text", "text": "I'll check the weather..."},
|
||||
{"type": "tool_use", "id": "toolu_123", "name": "get_weather", "input": {...}}
|
||||
],
|
||||
"stop_reason": "tool_use"
|
||||
}
|
||||
```
|
||||
|
||||
**Run:**
|
||||
```bash
|
||||
go test ./internal/proxy/... -run TestAnthropicToolUse -v
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 6. Complex Nested Tool Arguments
|
||||
|
||||
**What it tests:**
|
||||
- Tool arguments can be complex JSON structures
|
||||
- Nested objects, arrays, and deeply nested data preserved
|
||||
- No argument modification or parsing
|
||||
|
||||
**Test code:**
|
||||
```json
|
||||
{
|
||||
"function": {
|
||||
"name": "create_event",
|
||||
"arguments": {
|
||||
"title": "Team Meeting",
|
||||
"time": "2025-08-20T14:00:00Z",
|
||||
"attendees": [
|
||||
{"name": "Alice", "email": "[email protected]"},
|
||||
{"name": "Bob", "email": "[email protected]"}
|
||||
],
|
||||
"location": {
|
||||
"address": "123 Main St",
|
||||
"city": "San Francisco",
|
||||
"country": "USA"
|
||||
},
|
||||
"tags": ["important", "recurring"]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Run:**
|
||||
```bash
|
||||
go test ./internal/proxy/... -run TestComplexToolArguments -v
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Running Against Real LLMs
|
||||
|
||||
### With Local Stubs (Current)
|
||||
|
||||
Tests use mock HTTP servers, so they run instantly:
|
||||
|
||||
```bash
|
||||
go test ./internal/proxy/... -run "Tool" -v
|
||||
# All 6 tests complete in ~220ms
|
||||
```
|
||||
|
||||
### With Real Upstreams (Future)
|
||||
|
||||
Once you have real LLM services running, update the config:
|
||||
|
||||
```yaml
|
||||
# k8s/configmap.yaml
|
||||
models:
|
||||
- name: "reasoning"
|
||||
address: "reasoning-predictor.llm-serving:80" # Real upstream
|
||||
- name: "claude"
|
||||
address: "claude-api.anthropic.com:443" # Real Anthropic
|
||||
```
|
||||
|
||||
Then use the gateway normally:
|
||||
|
||||
```bash
|
||||
# Terminal 1: Start gateway
|
||||
export CONFIG_PATH=config.yaml
|
||||
go run ./cmd/gateway
|
||||
|
||||
# Terminal 2: Test with real LLM
|
||||
curl -X POST http://localhost:8080/v1/chat/completions \
|
||||
-H 'content-type: application/json' \
|
||||
-d '{
|
||||
"model": "reasoning",
|
||||
"messages": [{"role": "user", "content": "What color is the sky?"}],
|
||||
"tools": [{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "search",
|
||||
"description": "Search the internet",
|
||||
"parameters": {"type": "object", "properties": {}}
|
||||
}
|
||||
}]
|
||||
}'
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Gateway Behavior with Tool Calls
|
||||
|
||||
### Request Path
|
||||
|
||||
```
|
||||
Client Request
|
||||
↓
|
||||
Body-based dispatch (find model)
|
||||
↓
|
||||
Look up upstream address
|
||||
↓
|
||||
Forward request unmodified (including tools)
|
||||
↓
|
||||
Upstream LLM processes tools
|
||||
```
|
||||
|
||||
### Response Path
|
||||
|
||||
```
|
||||
Upstream Response (with tool_calls or tool_use)
|
||||
↓
|
||||
Stream unbuffered if streaming
|
||||
↓
|
||||
Return to client exactly as received
|
||||
```
|
||||
|
||||
### Key Properties
|
||||
|
||||
1. **No Rewriting**: Tool definitions and responses pass through unmodified
|
||||
2. **Format Agnostic**: Both OpenAI `tool_calls` and Anthropic `tool_use` work
|
||||
3. **Streaming Safe**: Tool calls stream incrementally without buffering
|
||||
4. **Nested Structures**: Complex JSON arguments fully preserved
|
||||
|
||||
---
|
||||
|
||||
## Common Tool Call Patterns
|
||||
|
||||
### Pattern 1: Sequential Tool Use
|
||||
```
|
||||
Client → LLM (please use search tool)
|
||||
← LLM (tool_calls: [search(...)])
|
||||
Client → (execute search, send results)
|
||||
Client → LLM (here are search results)
|
||||
← LLM (final answer)
|
||||
```
|
||||
|
||||
### Pattern 2: Parallel Tool Calls
|
||||
```
|
||||
Client → LLM (check weather in 3 cities)
|
||||
← LLM (tool_calls: [get_weather(NY), get_weather(LA), get_weather(SF)])
|
||||
Client → (execute all 3 in parallel)
|
||||
Client → LLM (here are all results)
|
||||
← LLM (summary)
|
||||
```
|
||||
|
||||
### Pattern 3: Tool Result Formatting
|
||||
```
|
||||
Client receives tool_calls with:
|
||||
- id: unique identifier
|
||||
- function.name: tool name
|
||||
- function.arguments: JSON string (always a string, not parsed object)
|
||||
|
||||
Client sends back:
|
||||
- role: "tool"
|
||||
- content: result JSON string
|
||||
- tool_call_id: matches the original call id
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Verification Checklist
|
||||
|
||||
- [x] OpenAI-style tool definitions forward to upstream
|
||||
- [x] Tool calls in response reach client unmodified
|
||||
- [x] Streaming tool calls arrive incrementally
|
||||
- [x] Multi-turn conversations preserve tool context
|
||||
- [x] Parallel tool calls all included in response
|
||||
- [x] Anthropic tool_use format works
|
||||
- [x] Complex nested arguments preserved
|
||||
|
||||
Run all:
|
||||
```bash
|
||||
go test ./internal/proxy/... -run "Tool" -v --race
|
||||
```
|
||||
|
||||
Expected: 6/6 passing, race detector clean
|
||||
|
||||
---
|
||||
|
||||
## Integration with Other Phases
|
||||
|
||||
### Phase 2.9: Anthropic Dialect
|
||||
Currently, Anthropic tool calls work through the generic route handler. Phase 2.9 will add a dedicated `/llm/v1/messages` endpoint with full Anthropic-specific handling.
|
||||
|
||||
### Phase 2.13: Error Handling
|
||||
Tool call errors (unknown tool, parsing errors) will have proper error responses in both OpenAI and Anthropic formats.
|
||||
|
||||
### Phase 3: Authentication
|
||||
Tool calls work with all authentication methods (bearer tokens, API keys) - no special handling needed since tools are just part of the message payload.
|
||||
|
||||
### Phase 4: Rate Limiting
|
||||
Tool calling counts the same as regular chat requests. Rate limits apply per conversation, not per tool call.
|
||||
|
||||
---
|
||||
|
||||
## Debugging Tool Calls
|
||||
|
||||
### Check if tool definitions reach upstream:
|
||||
|
||||
```bash
|
||||
# Enable request logging
|
||||
go test ./internal/proxy/... -run TestToolCallOpenAIStyle -v 2>&1 | grep -A5 "tool"
|
||||
```
|
||||
|
||||
### Verify tool response format:
|
||||
|
||||
```bash
|
||||
# Extract and pretty-print response
|
||||
curl -X POST http://localhost:8080/v1/chat/completions ... | jq '.choices[0].message.tool_calls'
|
||||
```
|
||||
|
||||
### Test streaming tool calls:
|
||||
|
||||
```bash
|
||||
curl -N http://localhost:8080/v1/chat/completions \
|
||||
-H 'content-type: application/json' \
|
||||
-d '{..., "stream": true, "tools": [...]}'
|
||||
# Should see incremental chunks with tool_use deltas
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## FAQ
|
||||
|
||||
**Q: Do I need to modify the gateway code to support tool calls?**
|
||||
A: No. Tool calls are just JSON in the request/response body. The gateway forwards them unchanged.
|
||||
|
||||
**Q: What if the LLM doesn't support tools?**
|
||||
A: The tool definitions are simply ignored. The gateway doesn't validate or enforce tool support.
|
||||
|
||||
**Q: Can I mix OpenAI and Anthropic tool formats?**
|
||||
A: Not in the same request. OpenAI clients expect `tool_calls`, Anthropic clients expect `tool_use` blocks. The upstream API determines the format.
|
||||
|
||||
**Q: How are tool arguments limited?**
|
||||
A: By the per-route `maxBodySize` config. Complex nested arguments count toward that limit.
|
||||
|
||||
**Q: Can tool calls be streamed?**
|
||||
A: Yes! SSE streaming fully supports tool calls. They arrive in delta chunks like text tokens.
|
||||
|
||||
---
|
||||
|
||||
## Next Steps
|
||||
|
||||
1. **Run tests locally:**
|
||||
```bash
|
||||
go test ./internal/proxy/... -run "Tool" -v
|
||||
```
|
||||
|
||||
2. **Deploy to cluster:**
|
||||
See [CLUSTER_REPO_SETUP.md](./CLUSTER_REPO_SETUP.md)
|
||||
|
||||
3. **Test against real LLMs:**
|
||||
Update config with real upstream addresses, restart gateway
|
||||
|
||||
4. **Phase 2.9:** Implement Anthropic dialect handler for `/llm/v1/messages`
|
||||
|
||||
5. **Phase 4:** Add tool call budgeting and rate limits
|
||||
@@ -13,15 +13,29 @@ import (
|
||||
)
|
||||
|
||||
// RouteRequest determines which upstream should handle the request.
|
||||
// For /v1/* routes, it uses body-based dispatch (reads JSON to find "model" field).
|
||||
// For other routes, it returns the single configured route.
|
||||
// For /v1/chat/completions, it uses body-based dispatch (reads JSON to find "model" field).
|
||||
// For other routes, it looks up by path prefix.
|
||||
func (h *Handler) RouteRequest(r *http.Request) (*Route, error) {
|
||||
// For /v1/chat/completions, use body-based dispatch
|
||||
if r.URL.Path == "/v1/chat/completions" && r.Method == "POST" {
|
||||
return h.routeByModel(r)
|
||||
}
|
||||
|
||||
// For other paths, return the first (and usually only) route
|
||||
// For other paths, try to find a matching route by path
|
||||
// Look for exact path match or path prefix match
|
||||
for _, route := range h.routes {
|
||||
// Check if route's pathRewrite matches request path
|
||||
if route.Upstream.PathRewrite != "" && route.Upstream.PathRewrite == r.URL.Path {
|
||||
return route, nil
|
||||
}
|
||||
}
|
||||
|
||||
// If no routes configured, return error
|
||||
if len(h.routes) == 0 {
|
||||
return nil, fmt.Errorf("no routes configured")
|
||||
}
|
||||
|
||||
// Return the first configured route as fallback
|
||||
for _, route := range h.routes {
|
||||
return route, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,858 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Riotpiaole/homelab-frontend/internal/config"
|
||||
)
|
||||
|
||||
// OpenAI-style tool definition
|
||||
type Tool struct {
|
||||
Type string `json:"type"`
|
||||
Function ToolFunction `json:"function"`
|
||||
}
|
||||
|
||||
type ToolFunction struct {
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
Parameters map[string]interface{} `json:"parameters"`
|
||||
}
|
||||
|
||||
// OpenAI chat completion with tools request
|
||||
type ChatCompletionRequest struct {
|
||||
Model string `json:"model"`
|
||||
Messages []Message `json:"messages"`
|
||||
Tools []Tool `json:"tools,omitempty"`
|
||||
Stream bool `json:"stream,omitempty"`
|
||||
}
|
||||
|
||||
type Message struct {
|
||||
Role string `json:"role"`
|
||||
Content interface{} `json:"content"`
|
||||
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
|
||||
}
|
||||
|
||||
type ToolCall struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Function FunctionCall `json:"function"`
|
||||
}
|
||||
|
||||
type FunctionCall struct {
|
||||
Name string `json:"name"`
|
||||
Arguments string `json:"arguments"`
|
||||
}
|
||||
|
||||
// Test OpenAI-style tool calling (GET /v1/chat/completions with tools)
|
||||
func TestToolCallOpenAIStyle(t *testing.T) {
|
||||
upstreamReceived := false
|
||||
var receivedBody []byte
|
||||
|
||||
upstreamServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
upstreamReceived = true
|
||||
receivedBody, _ = io.ReadAll(r.Body)
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
||||
response := map[string]interface{}{
|
||||
"id": "chatcmpl-123",
|
||||
"object": "chat.completion",
|
||||
"model": "reasoning",
|
||||
"choices": []map[string]interface{}{
|
||||
{
|
||||
"index": 0,
|
||||
"message": map[string]interface{}{
|
||||
"role": "assistant",
|
||||
"content": nil,
|
||||
"tool_calls": []map[string]interface{}{
|
||||
{
|
||||
"id": "call_abc123",
|
||||
"type": "function",
|
||||
"function": map[string]interface{}{
|
||||
"name": "get_weather",
|
||||
"arguments": `{"location":"San Francisco","unit":"celsius"}`,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"finish_reason": "tool_calls",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
json.NewEncoder(w).Encode(response)
|
||||
}))
|
||||
defer upstreamServer.Close()
|
||||
|
||||
upstreamAddr := strings.TrimPrefix(upstreamServer.URL, "http://")
|
||||
|
||||
cfg := &config.Config{
|
||||
Models: map[string]*config.ModelUpstream{
|
||||
"reasoning": {
|
||||
Name: "reasoning",
|
||||
Address: upstreamAddr,
|
||||
Path: "/v1/chat/completions",
|
||||
},
|
||||
},
|
||||
Routes: make(map[string]*config.Route),
|
||||
}
|
||||
|
||||
handler := New(cfg)
|
||||
defer handler.Close()
|
||||
|
||||
server := httptest.NewServer(handler)
|
||||
defer server.Close()
|
||||
|
||||
// Create request with tools
|
||||
toolRequest := ChatCompletionRequest{
|
||||
Model: "reasoning",
|
||||
Messages: []Message{
|
||||
{
|
||||
Role: "user",
|
||||
Content: "What's the weather in San Francisco?",
|
||||
},
|
||||
},
|
||||
Tools: []Tool{
|
||||
{
|
||||
Type: "function",
|
||||
Function: ToolFunction{
|
||||
Name: "get_weather",
|
||||
Description: "Get weather for a location",
|
||||
Parameters: map[string]interface{}{
|
||||
"type": "object",
|
||||
"properties": map[string]interface{}{
|
||||
"location": map[string]interface{}{
|
||||
"type": "string",
|
||||
"description": "City name",
|
||||
},
|
||||
"unit": map[string]interface{}{
|
||||
"type": "string",
|
||||
"description": "Temperature unit",
|
||||
"enum": []string{"celsius", "fahrenheit"},
|
||||
},
|
||||
},
|
||||
"required": []string{"location"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
body, _ := json.Marshal(toolRequest)
|
||||
resp, err := http.Post(server.URL+"/v1/chat/completions", "application/json", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatalf("request failed: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Verify upstream received the request
|
||||
if !upstreamReceived {
|
||||
t.Errorf("expected upstream to receive tool call request")
|
||||
}
|
||||
|
||||
// Verify the tool definition was forwarded
|
||||
var receivedRequest ChatCompletionRequest
|
||||
json.Unmarshal(receivedBody, &receivedRequest)
|
||||
if len(receivedRequest.Tools) != 1 {
|
||||
t.Errorf("expected 1 tool in upstream request, got %d", len(receivedRequest.Tools))
|
||||
}
|
||||
|
||||
// Verify response contains tool_calls
|
||||
var respBody map[string]interface{}
|
||||
json.NewDecoder(resp.Body).Decode(&respBody)
|
||||
choices := respBody["choices"].([]interface{})
|
||||
choice := choices[0].(map[string]interface{})
|
||||
message := choice["message"].(map[string]interface{})
|
||||
toolCalls := message["tool_calls"].([]interface{})
|
||||
|
||||
if len(toolCalls) == 0 {
|
||||
t.Errorf("expected tool_calls in response")
|
||||
}
|
||||
|
||||
toolCall := toolCalls[0].(map[string]interface{})
|
||||
if toolCall["type"] != "function" {
|
||||
t.Errorf("expected tool_call type 'function', got %s", toolCall["type"])
|
||||
}
|
||||
}
|
||||
|
||||
// Test tool calling with streaming (SSE)
|
||||
func TestToolCallStreaming(t *testing.T) {
|
||||
upstreamServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Parse request to check stream flag
|
||||
var req ChatCompletionRequest
|
||||
json.NewDecoder(r.Body).Decode(&req)
|
||||
|
||||
if !req.Stream {
|
||||
t.Errorf("expected stream=true in request")
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
||||
rc := http.NewResponseController(w)
|
||||
|
||||
// Stream multiple chunks with tool_calls
|
||||
chunks := []map[string]interface{}{
|
||||
{
|
||||
"id": "chatcmpl-123",
|
||||
"object": "chat.completion.chunk",
|
||||
"model": "reasoning",
|
||||
"choices": []map[string]interface{}{
|
||||
{
|
||||
"index": 0,
|
||||
"delta": map[string]interface{}{
|
||||
"role": "assistant",
|
||||
"content": nil,
|
||||
},
|
||||
"finish_reason": nil,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"id": "chatcmpl-123",
|
||||
"object": "chat.completion.chunk",
|
||||
"choices": []map[string]interface{}{
|
||||
{
|
||||
"index": 0,
|
||||
"delta": map[string]interface{}{
|
||||
"tool_calls": []map[string]interface{}{
|
||||
{
|
||||
"index": 0,
|
||||
"id": "call_abc123",
|
||||
"type": "function",
|
||||
"function": map[string]interface{}{
|
||||
"name": "get_weather",
|
||||
"arguments": `{"location":"SF"}`,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"finish_reason": nil,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"id": "chatcmpl-123",
|
||||
"object": "chat.completion.chunk",
|
||||
"choices": []map[string]interface{}{
|
||||
{
|
||||
"index": 0,
|
||||
"delta": map[string]interface{}{},
|
||||
"finish_reason": "tool_calls",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, chunk := range chunks {
|
||||
data, _ := json.Marshal(chunk)
|
||||
fmt.Fprintf(w, "data: %s\n\n", string(data))
|
||||
rc.Flush()
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
|
||||
fmt.Fprint(w, "data: [DONE]\n\n")
|
||||
rc.Flush()
|
||||
}))
|
||||
defer upstreamServer.Close()
|
||||
|
||||
upstreamAddr := strings.TrimPrefix(upstreamServer.URL, "http://")
|
||||
|
||||
cfg := &config.Config{
|
||||
Models: map[string]*config.ModelUpstream{
|
||||
"reasoning": {
|
||||
Name: "reasoning",
|
||||
Address: upstreamAddr,
|
||||
},
|
||||
},
|
||||
Routes: make(map[string]*config.Route),
|
||||
}
|
||||
|
||||
handler := New(cfg)
|
||||
defer handler.Close()
|
||||
|
||||
server := httptest.NewServer(handler)
|
||||
defer server.Close()
|
||||
|
||||
toolRequest := ChatCompletionRequest{
|
||||
Model: "reasoning",
|
||||
Messages: []Message{
|
||||
{
|
||||
Role: "user",
|
||||
Content: "Call get_weather",
|
||||
},
|
||||
},
|
||||
Tools: []Tool{
|
||||
{
|
||||
Type: "function",
|
||||
Function: ToolFunction{
|
||||
Name: "get_weather",
|
||||
Description: "Get weather",
|
||||
Parameters: map[string]interface{}{},
|
||||
},
|
||||
},
|
||||
},
|
||||
Stream: true,
|
||||
}
|
||||
|
||||
body, _ := json.Marshal(toolRequest)
|
||||
resp, err := http.Post(server.URL+"/v1/chat/completions", "application/json", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatalf("request failed: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Read streaming response
|
||||
respBody, _ := io.ReadAll(resp.Body)
|
||||
respStr := string(respBody)
|
||||
|
||||
// Verify we got tool_calls in the stream
|
||||
if !strings.Contains(respStr, "get_weather") {
|
||||
t.Errorf("expected 'get_weather' in streaming response")
|
||||
}
|
||||
|
||||
if !strings.Contains(respStr, "[DONE]") {
|
||||
t.Errorf("expected [DONE] sentinel in streaming response")
|
||||
}
|
||||
}
|
||||
|
||||
// Test multi-turn conversation with tool use and results
|
||||
func TestToolCallMultiTurn(t *testing.T) {
|
||||
upstreamServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var req ChatCompletionRequest
|
||||
json.NewDecoder(r.Body).Decode(&req)
|
||||
|
||||
// Check if request includes tool result from previous assistant call
|
||||
hasToolResult := false
|
||||
for _, msg := range req.Messages {
|
||||
if msg.Role == "tool" {
|
||||
hasToolResult = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
||||
if hasToolResult {
|
||||
// Second turn: assistant responds with final answer
|
||||
response := map[string]interface{}{
|
||||
"id": "chatcmpl-456",
|
||||
"choices": []map[string]interface{}{
|
||||
{
|
||||
"index": 0,
|
||||
"message": map[string]interface{}{
|
||||
"role": "assistant",
|
||||
"content": "The weather in San Francisco is 22°C and cloudy.",
|
||||
},
|
||||
"finish_reason": "stop",
|
||||
},
|
||||
},
|
||||
}
|
||||
json.NewEncoder(w).Encode(response)
|
||||
} else {
|
||||
// First turn: assistant calls tool
|
||||
response := map[string]interface{}{
|
||||
"id": "chatcmpl-123",
|
||||
"choices": []map[string]interface{}{
|
||||
{
|
||||
"index": 0,
|
||||
"message": map[string]interface{}{
|
||||
"role": "assistant",
|
||||
"content": nil,
|
||||
"tool_calls": []map[string]interface{}{
|
||||
{
|
||||
"id": "call_abc123",
|
||||
"type": "function",
|
||||
"function": map[string]interface{}{
|
||||
"name": "get_weather",
|
||||
"arguments": `{"location":"San Francisco"}`,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"finish_reason": "tool_calls",
|
||||
},
|
||||
},
|
||||
}
|
||||
json.NewEncoder(w).Encode(response)
|
||||
}
|
||||
}))
|
||||
defer upstreamServer.Close()
|
||||
|
||||
upstreamAddr := strings.TrimPrefix(upstreamServer.URL, "http://")
|
||||
|
||||
cfg := &config.Config{
|
||||
Models: map[string]*config.ModelUpstream{
|
||||
"reasoning": {
|
||||
Name: "reasoning",
|
||||
Address: upstreamAddr,
|
||||
},
|
||||
},
|
||||
Routes: make(map[string]*config.Route),
|
||||
}
|
||||
|
||||
handler := New(cfg)
|
||||
defer handler.Close()
|
||||
|
||||
server := httptest.NewServer(handler)
|
||||
defer server.Close()
|
||||
|
||||
// Turn 1: User asks question
|
||||
turn1 := ChatCompletionRequest{
|
||||
Model: "reasoning",
|
||||
Messages: []Message{
|
||||
{
|
||||
Role: "user",
|
||||
Content: "What's the weather in San Francisco?",
|
||||
},
|
||||
},
|
||||
Tools: []Tool{
|
||||
{
|
||||
Type: "function",
|
||||
Function: ToolFunction{
|
||||
Name: "get_weather",
|
||||
Description: "Get weather for a location",
|
||||
Parameters: map[string]interface{}{},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
body1, _ := json.Marshal(turn1)
|
||||
resp1, _ := http.Post(server.URL+"/v1/chat/completions", "application/json", bytes.NewReader(body1))
|
||||
var turn1Resp map[string]interface{}
|
||||
json.NewDecoder(resp1.Body).Decode(&turn1Resp)
|
||||
resp1.Body.Close()
|
||||
|
||||
// Verify first turn has tool_calls
|
||||
choices1 := turn1Resp["choices"].([]interface{})
|
||||
msg1 := choices1[0].(map[string]interface{})["message"].(map[string]interface{})
|
||||
if msg1["tool_calls"] == nil {
|
||||
t.Errorf("expected tool_calls in first turn")
|
||||
}
|
||||
|
||||
// Turn 2: Send tool result
|
||||
turn2 := ChatCompletionRequest{
|
||||
Model: "reasoning",
|
||||
Messages: []Message{
|
||||
{
|
||||
Role: "user",
|
||||
Content: "What's the weather in San Francisco?",
|
||||
},
|
||||
{
|
||||
Role: "assistant",
|
||||
ToolCalls: []ToolCall{{ID: "call_abc123", Type: "function", Function: FunctionCall{Name: "get_weather", Arguments: `{"location":"San Francisco"}`}}},
|
||||
},
|
||||
{
|
||||
Role: "tool",
|
||||
Content: `{"temperature":22,"condition":"cloudy"}`,
|
||||
},
|
||||
},
|
||||
Tools: []Tool{
|
||||
{
|
||||
Type: "function",
|
||||
Function: ToolFunction{
|
||||
Name: "get_weather",
|
||||
Description: "Get weather for a location",
|
||||
Parameters: map[string]interface{}{},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
body2, _ := json.Marshal(turn2)
|
||||
resp2, _ := http.Post(server.URL+"/v1/chat/completions", "application/json", bytes.NewReader(body2))
|
||||
var turn2Resp map[string]interface{}
|
||||
json.NewDecoder(resp2.Body).Decode(&turn2Resp)
|
||||
resp2.Body.Close()
|
||||
|
||||
// Verify second turn has final answer (no tool_calls)
|
||||
choices2 := turn2Resp["choices"].([]interface{})
|
||||
msg2 := choices2[0].(map[string]interface{})["message"].(map[string]interface{})
|
||||
content := msg2["content"].(string)
|
||||
|
||||
if !strings.Contains(content, "cloudy") || !strings.Contains(content, "22") {
|
||||
t.Errorf("expected final answer with weather info, got: %s", content)
|
||||
}
|
||||
}
|
||||
|
||||
// Test parallel tool calls (multiple tools at once)
|
||||
func TestParallelToolCalls(t *testing.T) {
|
||||
upstreamServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
||||
response := map[string]interface{}{
|
||||
"id": "chatcmpl-789",
|
||||
"choices": []map[string]interface{}{
|
||||
{
|
||||
"index": 0,
|
||||
"message": map[string]interface{}{
|
||||
"role": "assistant",
|
||||
"content": nil,
|
||||
"tool_calls": []map[string]interface{}{
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": map[string]interface{}{
|
||||
"name": "get_weather",
|
||||
"arguments": `{"location":"New York"}`,
|
||||
},
|
||||
},
|
||||
{
|
||||
"id": "call_2",
|
||||
"type": "function",
|
||||
"function": map[string]interface{}{
|
||||
"name": "get_weather",
|
||||
"arguments": `{"location":"London"}`,
|
||||
},
|
||||
},
|
||||
{
|
||||
"id": "call_3",
|
||||
"type": "function",
|
||||
"function": map[string]interface{}{
|
||||
"name": "get_weather",
|
||||
"arguments": `{"location":"Tokyo"}`,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"finish_reason": "tool_calls",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
json.NewEncoder(w).Encode(response)
|
||||
}))
|
||||
defer upstreamServer.Close()
|
||||
|
||||
upstreamAddr := strings.TrimPrefix(upstreamServer.URL, "http://")
|
||||
|
||||
cfg := &config.Config{
|
||||
Models: map[string]*config.ModelUpstream{
|
||||
"reasoning": {
|
||||
Name: "reasoning",
|
||||
Address: upstreamAddr,
|
||||
},
|
||||
},
|
||||
Routes: make(map[string]*config.Route),
|
||||
}
|
||||
|
||||
handler := New(cfg)
|
||||
defer handler.Close()
|
||||
|
||||
server := httptest.NewServer(handler)
|
||||
defer server.Close()
|
||||
|
||||
request := ChatCompletionRequest{
|
||||
Model: "reasoning",
|
||||
Messages: []Message{
|
||||
{
|
||||
Role: "user",
|
||||
Content: "Get weather for New York, London, and Tokyo",
|
||||
},
|
||||
},
|
||||
Tools: []Tool{
|
||||
{
|
||||
Type: "function",
|
||||
Function: ToolFunction{
|
||||
Name: "get_weather",
|
||||
Description: "Get weather for a location",
|
||||
Parameters: map[string]interface{}{},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
body, _ := json.Marshal(request)
|
||||
resp, _ := http.Post(server.URL+"/v1/chat/completions", "application/json", bytes.NewReader(body))
|
||||
var respData map[string]interface{}
|
||||
json.NewDecoder(resp.Body).Decode(&respData)
|
||||
resp.Body.Close()
|
||||
|
||||
// Verify multiple tool calls
|
||||
choices := respData["choices"].([]interface{})
|
||||
msg := choices[0].(map[string]interface{})["message"].(map[string]interface{})
|
||||
toolCalls := msg["tool_calls"].([]interface{})
|
||||
|
||||
if len(toolCalls) != 3 {
|
||||
t.Errorf("expected 3 parallel tool calls, got %d", len(toolCalls))
|
||||
}
|
||||
|
||||
// Verify each call is present
|
||||
callNames := []string{}
|
||||
for _, call := range toolCalls {
|
||||
tc := call.(map[string]interface{})
|
||||
fn := tc["function"].(map[string]interface{})
|
||||
args := fn["arguments"].(string)
|
||||
callNames = append(callNames, args)
|
||||
}
|
||||
|
||||
if !strings.Contains(strings.Join(callNames, ""), "New York") {
|
||||
t.Errorf("expected New York in parallel calls")
|
||||
}
|
||||
if !strings.Contains(strings.Join(callNames, ""), "London") {
|
||||
t.Errorf("expected London in parallel calls")
|
||||
}
|
||||
if !strings.Contains(strings.Join(callNames, ""), "Tokyo") {
|
||||
t.Errorf("expected Tokyo in parallel calls")
|
||||
}
|
||||
}
|
||||
|
||||
// Test Anthropic-style tool use (different format)
|
||||
func TestAnthropicToolUse(t *testing.T) {
|
||||
upstreamServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Anthropic uses different request format
|
||||
var body map[string]interface{}
|
||||
json.NewDecoder(r.Body).Decode(&body)
|
||||
|
||||
// Check for tools in request
|
||||
if _, hasTools := body["tools"]; !hasTools {
|
||||
t.Errorf("expected tools in Anthropic request")
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
||||
// Anthropic response format with tool_use block
|
||||
response := map[string]interface{}{
|
||||
"id": "msg_123",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": []map[string]interface{}{
|
||||
{
|
||||
"type": "text",
|
||||
"text": "I'll check the weather for you.",
|
||||
},
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_123",
|
||||
"name": "get_weather",
|
||||
"input": map[string]interface{}{
|
||||
"location": "San Francisco",
|
||||
"unit": "celsius",
|
||||
},
|
||||
},
|
||||
},
|
||||
"model": "claude-3-5-sonnet",
|
||||
"stop_reason": "tool_use",
|
||||
"usage": map[string]interface{}{
|
||||
"input_tokens": 100,
|
||||
"output_tokens": 50,
|
||||
},
|
||||
}
|
||||
|
||||
json.NewEncoder(w).Encode(response)
|
||||
}))
|
||||
defer upstreamServer.Close()
|
||||
|
||||
upstreamAddr := strings.TrimPrefix(upstreamServer.URL, "http://")
|
||||
|
||||
// For Anthropic, we need to configure a regular route (not body-based dispatch)
|
||||
// since Anthropic uses different path structure (/llm/v1/messages)
|
||||
cfg := &config.Config{
|
||||
Routes: map[string]*config.Route{
|
||||
"anthropic": {
|
||||
Name: "anthropic",
|
||||
Upstream: config.Upstream{
|
||||
Address: upstreamAddr,
|
||||
PathRewrite: "/v1/messages",
|
||||
ConnectTimeout: 5 * time.Second,
|
||||
ReadTimeout: 30 * time.Second,
|
||||
WriteTimeout: 30 * time.Second,
|
||||
MaxBodySize: 10 * 1024 * 1024,
|
||||
AuthRequired: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
Models: make(map[string]*config.ModelUpstream),
|
||||
}
|
||||
|
||||
handler := New(cfg)
|
||||
defer handler.Close()
|
||||
|
||||
server := httptest.NewServer(handler)
|
||||
defer server.Close()
|
||||
|
||||
// Anthropic request format
|
||||
anthropicRequest := map[string]interface{}{
|
||||
"model": "claude",
|
||||
"messages": []map[string]interface{}{
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's the weather?",
|
||||
},
|
||||
},
|
||||
"tools": []map[string]interface{}{
|
||||
{
|
||||
"name": "get_weather",
|
||||
"description": "Get weather information",
|
||||
"input_schema": map[string]interface{}{
|
||||
"type": "object",
|
||||
"properties": map[string]interface{}{
|
||||
"location": map[string]interface{}{
|
||||
"type": "string",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"max_tokens": 1024,
|
||||
}
|
||||
|
||||
body, _ := json.Marshal(anthropicRequest)
|
||||
// Note: For now we route through a generic path
|
||||
// In Phase 2.9+, this would be integrated with the Anthropic dialect handler
|
||||
resp, _ := http.Post(server.URL+"/v1/messages", "application/json", bytes.NewReader(body))
|
||||
var respData map[string]interface{}
|
||||
json.NewDecoder(resp.Body).Decode(&respData)
|
||||
resp.Body.Close()
|
||||
|
||||
// Verify Anthropic tool_use format
|
||||
content := respData["content"].([]interface{})
|
||||
if len(content) < 2 {
|
||||
t.Errorf("expected at least 2 content blocks, got %d", len(content))
|
||||
}
|
||||
|
||||
// Find tool_use block
|
||||
foundToolUse := false
|
||||
for _, block := range content {
|
||||
b := block.(map[string]interface{})
|
||||
if b["type"] == "tool_use" {
|
||||
foundToolUse = true
|
||||
if b["name"] != "get_weather" {
|
||||
t.Errorf("expected tool name 'get_weather', got %s", b["name"])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !foundToolUse {
|
||||
t.Errorf("expected tool_use block in Anthropic response")
|
||||
}
|
||||
}
|
||||
|
||||
// Test tool call with complex nested arguments
|
||||
func TestComplexToolArguments(t *testing.T) {
|
||||
upstreamServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var req ChatCompletionRequest
|
||||
json.NewDecoder(r.Body).Decode(&req)
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
||||
// Response with complex nested tool arguments
|
||||
response := map[string]interface{}{
|
||||
"id": "chatcmpl-999",
|
||||
"choices": []map[string]interface{}{
|
||||
{
|
||||
"index": 0,
|
||||
"message": map[string]interface{}{
|
||||
"role": "assistant",
|
||||
"content": nil,
|
||||
"tool_calls": []map[string]interface{}{
|
||||
{
|
||||
"id": "call_complex",
|
||||
"type": "function",
|
||||
"function": map[string]interface{}{
|
||||
"name": "create_event",
|
||||
"arguments": `{
|
||||
"title": "Team Meeting",
|
||||
"time": "2025-08-20T14:00:00Z",
|
||||
"attendees": [
|
||||
{"name": "Alice", "email": "[email protected]"},
|
||||
{"name": "Bob", "email": "[email protected]"}
|
||||
],
|
||||
"location": {
|
||||
"address": "123 Main St",
|
||||
"city": "San Francisco",
|
||||
"country": "USA"
|
||||
},
|
||||
"tags": ["important", "recurring"]
|
||||
}`,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"finish_reason": "tool_calls",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
json.NewEncoder(w).Encode(response)
|
||||
}))
|
||||
defer upstreamServer.Close()
|
||||
|
||||
upstreamAddr := strings.TrimPrefix(upstreamServer.URL, "http://")
|
||||
|
||||
cfg := &config.Config{
|
||||
Models: map[string]*config.ModelUpstream{
|
||||
"reasoning": {
|
||||
Name: "reasoning",
|
||||
Address: upstreamAddr,
|
||||
},
|
||||
},
|
||||
Routes: make(map[string]*config.Route),
|
||||
}
|
||||
|
||||
handler := New(cfg)
|
||||
defer handler.Close()
|
||||
|
||||
server := httptest.NewServer(handler)
|
||||
defer server.Close()
|
||||
|
||||
request := ChatCompletionRequest{
|
||||
Model: "reasoning",
|
||||
Messages: []Message{
|
||||
{
|
||||
Role: "user",
|
||||
Content: "Create a team meeting",
|
||||
},
|
||||
},
|
||||
Tools: []Tool{
|
||||
{
|
||||
Type: "function",
|
||||
Function: ToolFunction{
|
||||
Name: "create_event",
|
||||
Description: "Create a calendar event",
|
||||
Parameters: map[string]interface{}{},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
body, _ := json.Marshal(request)
|
||||
resp, _ := http.Post(server.URL+"/v1/chat/completions", "application/json", bytes.NewReader(body))
|
||||
var respData map[string]interface{}
|
||||
json.NewDecoder(resp.Body).Decode(&respData)
|
||||
resp.Body.Close()
|
||||
|
||||
// Verify complex nested arguments are preserved
|
||||
choices := respData["choices"].([]interface{})
|
||||
msg := choices[0].(map[string]interface{})["message"].(map[string]interface{})
|
||||
toolCalls := msg["tool_calls"].([]interface{})
|
||||
toolCall := toolCalls[0].(map[string]interface{})
|
||||
fn := toolCall["function"].(map[string]interface{})
|
||||
args := fn["arguments"].(string)
|
||||
|
||||
// Verify the structure is intact
|
||||
if !strings.Contains(args, "[email protected]") {
|
||||
t.Errorf("expected email in tool arguments")
|
||||
}
|
||||
if !strings.Contains(args, "San Francisco") {
|
||||
t.Errorf("expected location in tool arguments")
|
||||
}
|
||||
if !strings.Contains(args, "recurring") {
|
||||
t.Errorf("expected tags in tool arguments")
|
||||
}
|
||||
}
|
||||
+13
-5
@@ -27,14 +27,22 @@ spec:
|
||||
prometheus.io/path: "/metrics"
|
||||
spec:
|
||||
serviceAccountName: api-gateway
|
||||
# The registry at forgejo.riotpiao.com requires auth — /v2/ answers 401.
|
||||
# This Secret must exist in the api namespace before the first rollout.
|
||||
imagePullSecrets:
|
||||
- name: forgejo-registry
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
fsGroup: 1000
|
||||
# 65532 is distroless's nonroot user, matching USER in the Dockerfile.
|
||||
runAsUser: 65532
|
||||
fsGroup: 65532
|
||||
containers:
|
||||
- name: gateway
|
||||
image: ghcr.io/riotpiaole/api-gateway:latest
|
||||
imagePullPolicy: Always
|
||||
# Tag is pinned in kustomization.yaml so there is exactly one place to
|
||||
# bump it. Never :latest — Argo cannot make a deterministic rollout
|
||||
# decision from a mutable tag, and 6.1 requires SHA tags.
|
||||
image: forgejo.riotpiao.com/rock/api-gateway
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
@@ -81,7 +89,7 @@ spec:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
runAsUser: 65532
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
|
||||
@@ -10,6 +10,15 @@ resources:
|
||||
- network-policy.yaml
|
||||
- configmap.yaml
|
||||
|
||||
# The deployed image tag lives here and nowhere else. CI publishes
|
||||
# forgejo.riotpiao.com/rock/api-gateway:<commit-sha>; promoting a build is a
|
||||
# one-line commit bumping newTag, which Argo then syncs (G7).
|
||||
#
|
||||
# kustomize edit set image forgejo.riotpiao.com/rock/api-gateway=:<sha>
|
||||
images:
|
||||
- name: forgejo.riotpiao.com/rock/api-gateway
|
||||
newTag: REPLACE_WITH_FIRST_BUILD_SHA
|
||||
|
||||
commonLabels:
|
||||
app: api-gateway
|
||||
managed-by: argocd
|
||||
|
||||
Reference in New Issue
Block a user