Compare commits
13
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c1a720687e | ||
|
|
152e4259ae | ||
|
|
5c30fd4fb7 | ||
|
|
3f89511bdd | ||
|
|
3edcb10310 | ||
|
|
df553cc70d | ||
|
|
f32ff08365 | ||
|
|
c44a9b174f | ||
|
|
f1030aae82 | ||
|
|
dc97afb0ef | ||
|
|
eb4c2ff13d | ||
|
|
4463508f29 | ||
|
|
11ca4a5902 |
+32
-42
@@ -1,5 +1,3 @@
|
||||
# Single pipeline: verify → build → push.
|
||||
# One workflow per push, one concurrency group per branch.
|
||||
name: CI
|
||||
|
||||
on:
|
||||
@@ -8,53 +6,40 @@ on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
concurrency:
|
||||
group: ci-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
REGISTRY: forgejo.riotpiao.com
|
||||
IMAGE: forgejo.riotpiao.com/rock/api-gateway
|
||||
IMAGE: forgejo.riotpiao.com/rock/homelab-frontend
|
||||
|
||||
jobs:
|
||||
verify:
|
||||
name: Vet, test, build
|
||||
test:
|
||||
name: Test
|
||||
runs-on: golang
|
||||
container:
|
||||
image: golang:1.26-bookworm
|
||||
steps:
|
||||
- name: install node (required by JS-based actions)
|
||||
run: apt-get update && apt-get install -y --no-install-recommends nodejs ca-certificates git
|
||||
- name: Install Node.js for actions runtime
|
||||
run: apt-get update && apt-get install -y nodejs
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: go vet
|
||||
- name: Go vet
|
||||
run: go vet ./...
|
||||
|
||||
- name: go test -race
|
||||
run: go test ./... -race
|
||||
- name: Go test
|
||||
run: go test ./...
|
||||
|
||||
- name: Static build (smoke)
|
||||
run: CGO_ENABLED=0 go build -trimpath -o gateway ./cmd/gateway
|
||||
|
||||
push:
|
||||
name: Build and push image
|
||||
needs: verify
|
||||
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
||||
build-push:
|
||||
name: Build & Push Image
|
||||
needs: test
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
runs-on: golang
|
||||
container:
|
||||
image: docker:27-cli
|
||||
volumes:
|
||||
- /docker-certs/client:/docker-certs/client:ro
|
||||
env:
|
||||
DOCKER_HOST: tcp://localhost:2376
|
||||
DOCKER_TLS_VERIFY: "1"
|
||||
DOCKER_CERT_PATH: /docker-certs/client
|
||||
steps:
|
||||
- name: install node (required by JS-based actions)
|
||||
run: apk add --no-cache nodejs git
|
||||
- name: Install Node.js and Docker
|
||||
run: |
|
||||
apt-get update
|
||||
apt-get install -y nodejs docker.io
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Get short SHA
|
||||
id: sha
|
||||
@@ -64,20 +49,25 @@ jobs:
|
||||
|
||||
- name: Registry login
|
||||
run: |
|
||||
echo "${REGISTRY_PAT}" | docker login "${REGISTRY}" \
|
||||
--username rock --password-stdin
|
||||
echo "${REGISTRY_TOKEN}" | docker login "${REGISTRY}" \
|
||||
--username "${REGISTRY_USER}" --password-stdin
|
||||
env:
|
||||
REGISTRY_PAT: ${{ secrets.REGISTRY_PAT }}
|
||||
REGISTRY_USER: ${{ secrets.FORGEJO_REGISTRY_USER }}
|
||||
REGISTRY_TOKEN: ${{ secrets.FORGEJO_REGISTRY_TOKEN }}
|
||||
|
||||
- name: Build image
|
||||
- name: Build Docker image
|
||||
run: |
|
||||
docker build \
|
||||
--build-arg "VERSION=${{ steps.sha.outputs.short_sha }}" \
|
||||
docker build --no-cache \
|
||||
-t "${IMAGE}:${{ steps.sha.outputs.short_sha }}" \
|
||||
-t "${IMAGE}:latest" \
|
||||
-f Dockerfile \
|
||||
.
|
||||
|
||||
- name: Push image
|
||||
- name: Push Docker image
|
||||
run: |
|
||||
docker push "${IMAGE}:${{ steps.sha.outputs.short_sha }}"
|
||||
docker push "${IMAGE}:latest"
|
||||
echo "✓ Image pushed: ${IMAGE}:${{ steps.sha.outputs.short_sha }}"
|
||||
|
||||
- name: Prune unused images
|
||||
run: docker image prune -a --force 2>&1 | tail -3 || true
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
creation_rules:
|
||||
- path_regex: \.enc\.ya?ml$
|
||||
age: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
|
||||
@@ -783,3 +783,402 @@ curl -X POST https://api.riotpiao.com/workflow \
|
||||
- **Readiness:** `curl https://api.riotpiao.com/readyz`
|
||||
- **Models:** `curl https://api.riotpiao.com/v1/models`
|
||||
- **Logs:** `kubectl -n api logs deployment/homelab-frontend`
|
||||
|
||||
---
|
||||
|
||||
## LLM Inference in Workflows
|
||||
|
||||
The Poimen workflows system includes built-in LLM inference activities that call `/v1/chat/completions` via the gateway.
|
||||
|
||||
### LLMInferenceActivity
|
||||
|
||||
Single-prompt LLM inference within a workflow.
|
||||
|
||||
**Workflow Definition (Canvas Node):**
|
||||
```json
|
||||
{
|
||||
"id": "llm-node-1",
|
||||
"type": "llm-inference",
|
||||
"label": "Analyze Code with LLM",
|
||||
"data": {
|
||||
"model": "reasoning",
|
||||
"system_prompt": "You are a code analysis expert. Provide detailed feedback.",
|
||||
"user_prompt": "Analyze this code for security issues: {{ previous_output.code }}",
|
||||
"temperature": 0.7,
|
||||
"max_tokens": 2048,
|
||||
"auth_token": "{{ user.jwt_token }}"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Fields:**
|
||||
- `model` (required): Model ID (reasoning, ornith:35b, ornith:13b, qwen2.5:3b)
|
||||
- `system_prompt`: System instruction for the model
|
||||
- `user_prompt` (required): User message to send
|
||||
- `temperature`: Sampling temperature (0.0-1.0, default 0.7)
|
||||
- `max_tokens`: Maximum output tokens
|
||||
- `auth_token` (optional): JWT token for authenticated endpoints (propagates as Authorization: Bearer header)
|
||||
|
||||
**Backend Implementation:**
|
||||
The LLMInferenceActivity in the workflows backend automatically:
|
||||
1. Substitutes template variables (e.g., `{{ previous_output.code }}`)
|
||||
2. Calls `/v1/chat/completions` with the resolved prompt
|
||||
3. Returns the LLM response as activity output
|
||||
4. Retries on transient failures (up to 3 attempts)
|
||||
5. Timeouts after 120 seconds
|
||||
|
||||
**Output:**
|
||||
```json
|
||||
{
|
||||
"response": "The code has several security vulnerabilities...",
|
||||
"model": "reasoning",
|
||||
"stop_reason": "stop_sequence",
|
||||
"tokens_used": 450
|
||||
}
|
||||
```
|
||||
|
||||
**Supported Models:**
|
||||
- `reasoning` — DeepSeek-R1-Distill (best for complex analysis)
|
||||
- `ornith:35b` — Ollama 35B
|
||||
- `ornith:13b` — Ollama 13B
|
||||
- `qwen2.5:3b` — Qwen 2.5 3B
|
||||
|
||||
---
|
||||
|
||||
### LLMBatchInferenceActivity
|
||||
|
||||
Multiple-prompt LLM inference (sequential processing).
|
||||
|
||||
**Workflow Definition:**
|
||||
```json
|
||||
{
|
||||
"id": "llm-batch-1",
|
||||
"type": "llm-batch-inference",
|
||||
"label": "Batch Code Review",
|
||||
"data": {
|
||||
"model": "reasoning",
|
||||
"system_prompt": "Review each code snippet and provide feedback.",
|
||||
"prompts": [
|
||||
"Review snippet 1: {{ files[0].content }}",
|
||||
"Review snippet 2: {{ files[1].content }}",
|
||||
"Review snippet 3: {{ files[2].content }}"
|
||||
],
|
||||
"auth_token": "{{ user.jwt_token }}"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Fields:**
|
||||
- `model` (required): Model ID
|
||||
- `system_prompt`: System instruction (same for all prompts)
|
||||
- `prompts` (required): List of user prompts to process
|
||||
- `temperature`: Sampling temperature (0.0-1.0)
|
||||
- `auth_token` (optional): JWT token for authenticated endpoints (propagates as Authorization: Bearer header)
|
||||
|
||||
**Output:**
|
||||
```json
|
||||
{
|
||||
"responses": [
|
||||
"Snippet 1 review...",
|
||||
"Snippet 2 review...",
|
||||
"Snippet 3 review..."
|
||||
],
|
||||
"model": "reasoning",
|
||||
"errors": []
|
||||
}
|
||||
```
|
||||
|
||||
**Typical Use Cases:**
|
||||
- Batch code review across multiple files
|
||||
- Parallel document summarization
|
||||
- Comparative analysis of alternatives
|
||||
- Policy compliance checking
|
||||
|
||||
---
|
||||
|
||||
### Workflow Integration Examples
|
||||
|
||||
**1. Code Analysis Workflow**
|
||||
```
|
||||
Clone Repo → Analyze Code → LLM Security Review → Generate Report → Notify
|
||||
```
|
||||
|
||||
**2. Document Processing**
|
||||
```
|
||||
Retrieve Documents → Embed + Index → LLM Summarize (batch) → Archive
|
||||
```
|
||||
|
||||
**3. Multi-Stage Review**
|
||||
```
|
||||
Retrieve Memory → LLM Context Extraction → Route to Activity A/B/C → Notify
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Authentication & Authorization
|
||||
|
||||
JWT tokens can be passed to LLM inference activities and are automatically propagated to the LLM API endpoint.
|
||||
|
||||
**Token Flow:**
|
||||
```
|
||||
Workflow Canvas
|
||||
↓ (auth_token field)
|
||||
Poimen Workflow Executor
|
||||
↓ (passed to LLMInferenceActivity)
|
||||
Activity calls LLM client
|
||||
↓ (adds "Authorization: Bearer {token}" header)
|
||||
homelab-frontend proxy
|
||||
↓ (preserves Authorization header)
|
||||
LLM Backend (reasoning/ollama/etc)
|
||||
↓ (validates token)
|
||||
Response returned
|
||||
```
|
||||
|
||||
**Example: Passing User Token from RetrieveMemory Activity**
|
||||
```json
|
||||
{
|
||||
"id": "flow-1",
|
||||
"type": "retrieve-memory",
|
||||
"label": "Get User Context",
|
||||
"data": {...}
|
||||
}
|
||||
→
|
||||
{
|
||||
"id": "llm-1",
|
||||
"type": "llm-inference",
|
||||
"label": "Analyze with User's Token",
|
||||
"data": {
|
||||
"model": "reasoning",
|
||||
"user_prompt": "...",
|
||||
"auth_token": "{{ previous_output.user_token }}"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Token Validation:**
|
||||
- Tokens are validated by homelab-frontend proxy (checks signature, expiration)
|
||||
- Only valid tokens are propagated to LLM backend
|
||||
- Invalid tokens result in 401 Unauthorized error
|
||||
- Missing token (if required) results in 401 Unauthorized
|
||||
|
||||
**Note:** The `auth_token` field is optional. If omitted, the LLM API is called without authentication (public endpoints only).
|
||||
|
||||
---
|
||||
|
||||
### CanvasReasonerActivity
|
||||
|
||||
Auto-suggest workflow connections using LLM reasoning. When you drop new activities onto the canvas, this activity analyzes them and suggests logical connections based on input/output compatibility and workflow patterns.
|
||||
|
||||
**Workflow Definition:**
|
||||
```json
|
||||
{
|
||||
"id": "canvas-reason-1",
|
||||
"type": "canvas-reasoner",
|
||||
"label": "Auto-Connect Activities",
|
||||
"data": {
|
||||
"nodes": "{{ workflow.nodes }}",
|
||||
"edges": "{{ workflow.edges }}",
|
||||
"preserve_existing": true,
|
||||
"auth_token": "{{ user.jwt_token }}"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Use Cases:**
|
||||
- New nodes added to canvas → automatically suggest connections
|
||||
- Validate workflow design → LLM reasoning explains connections
|
||||
- Redesign workflow → suggest optimal activity sequence
|
||||
- Data flow analysis → ensure proper input/output matching
|
||||
|
||||
**How It Works:**
|
||||
1. Analyzes all node types and their configurations
|
||||
2. Reviews existing edges (if preserving)
|
||||
3. Uses reasoning model to infer logical connections
|
||||
4. Returns suggested edges with confidence score
|
||||
5. Includes reasoning explanation
|
||||
|
||||
**Output Example:**
|
||||
```json
|
||||
{
|
||||
"suggested_edges": [
|
||||
{"source": "clone-1", "target": "analyze-1"},
|
||||
{"source": "analyze-1", "target": "security-scan-1"},
|
||||
{"source": "security-scan-1", "target": "report-1"}
|
||||
],
|
||||
"reasoning": "Clone repository first, analyze code, perform security scan, generate report. Standard code review workflow.",
|
||||
"confidence": 0.92
|
||||
}
|
||||
```
|
||||
|
||||
**Fields:**
|
||||
- `nodes` (required): Canvas nodes to analyze
|
||||
- `edges` (required): Current edges
|
||||
- `preserve_existing` (optional, default true): Keep existing edges and only suggest new ones
|
||||
- `auth_token` (optional): JWT for LLM reasoning calls
|
||||
|
||||
**Confidence Scores:**
|
||||
- 0.9-1.0: High confidence (common patterns)
|
||||
- 0.7-0.9: Medium confidence (reasonable connections)
|
||||
- 0.5-0.7: Low confidence (multiple valid approaches)
|
||||
- <0.5: Unsure (manual review recommended)
|
||||
|
||||
**Integration Example:**
|
||||
```
|
||||
User drops 3 new nodes on canvas
|
||||
↓
|
||||
Workflow calls CanvasReasonerActivity
|
||||
↓
|
||||
LLM analyzes schemas: Clone (out: path,commit) → Analyze (in: path) → Report (in: metrics)
|
||||
↓
|
||||
Compatibility checker validates edges
|
||||
↓
|
||||
Returns:
|
||||
- Suggested edges (Clone → Analyze → Report)
|
||||
- Incompatible edges (Report → Approve [terminal sink])
|
||||
- Disconnected nodes (if any isolated nodes)
|
||||
- User alerts explaining issues
|
||||
↓
|
||||
Frontend shows:
|
||||
✅ Green edges (compatible)
|
||||
❌ Red warnings (incompatible)
|
||||
🔌 Yellow badges (disconnected)
|
||||
↓
|
||||
User approves compatible edges, fixes/removes incompatible ones
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Handling Incompatible Connections
|
||||
|
||||
When two activities **cannot** be connected, the response includes detailed incompatibility data:
|
||||
|
||||
**Example: Terminal Activity Blocking Connection**
|
||||
```json
|
||||
{
|
||||
"suggested_edges": [
|
||||
{"source": "clone-1", "target": "analyze-1"}
|
||||
],
|
||||
"incompatible_edges": [
|
||||
{
|
||||
"source": "security-scan-1",
|
||||
"target": "approve-1",
|
||||
"reason": "ApproveWorkflowActivity accepts no inputs (terminal sink activity)",
|
||||
"source_needs": "to output: issues, metrics, severity",
|
||||
"target_needs": "none (approval only blocks workflow)",
|
||||
"suggestion": "ApproveWorkflowActivity must be the final step. Place it after Report generates summary."
|
||||
}
|
||||
],
|
||||
"user_alerts": [
|
||||
"⚠️ security-scan-1 → approve-1: ApproveWorkflowActivity is terminal (no inputs). Place it at the end of the workflow."
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Example: Type Mismatch**
|
||||
```json
|
||||
{
|
||||
"incompatible_edges": [
|
||||
{
|
||||
"source": "llm-inference-1",
|
||||
"target": "deployment-check-1",
|
||||
"reason": "Output type mismatch: string ≠ object",
|
||||
"source_needs": "outputs: response (string)",
|
||||
"target_needs": "inputs: deployment_plan (object)",
|
||||
"suggestion": "Insert LLM transformer node to convert string response → deployment_plan object"
|
||||
}
|
||||
],
|
||||
"user_alerts": [
|
||||
"⚠️ llm-inference-1 → deployment-check-1: Type mismatch (string ≠ object). Use LLM transformation node to map outputs."
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Example: Disconnected Nodes**
|
||||
```json
|
||||
{
|
||||
"disconnected_nodes": ["security-scan-1", "notify-1"],
|
||||
"user_alerts": [
|
||||
"🔌 Node 'SecurityScan' has no connections. Connect it or remove from canvas.",
|
||||
"🔌 Node 'Notify' has no incoming edges. Check if it should receive data."
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Incompatibility Warning Schema
|
||||
|
||||
```json
|
||||
{
|
||||
"source": "string - source node ID",
|
||||
"target": "string - target node ID",
|
||||
"reason": "string - why connection fails",
|
||||
"source_needs": "string - what source outputs",
|
||||
"target_needs": "string - what target requires",
|
||||
"suggestion": "string - how to fix it"
|
||||
}
|
||||
```
|
||||
|
||||
### Common Incompatibility Reasons
|
||||
|
||||
| Reason | Example | Solution |
|
||||
|--------|---------|----------|
|
||||
| **Terminal Activity** | Notify → CloneRepo | Can't output from sink (terminal) |
|
||||
| **Type Mismatch** | string → object | Use LLM transformer node |
|
||||
| **No Outputs** | Notification has no outputs | Terminal activities can't be sources |
|
||||
| **No Inputs** | Approval has no inputs | Terminal activities can't accept data |
|
||||
| **Semantic Mismatch** | Approval → Analysis | Doesn't make logical sense |
|
||||
|
||||
### Frontend Alert Display
|
||||
|
||||
**Sidebar UI:**
|
||||
```
|
||||
🚨 Connection Issues (3)
|
||||
|
||||
⚠️ CloneRepo-1 → Approve-1
|
||||
Terminal sink can't receive inputs
|
||||
[Fix] [Ignore] [Remove Node]
|
||||
|
||||
⚠️ LLMInference-1 → DeploymentCheck-1
|
||||
Type mismatch: string → object
|
||||
[Add Transformer] [Manual Map]
|
||||
|
||||
🔌 SecurityScan-1 (isolated)
|
||||
No connections detected
|
||||
[Connect] [Remove]
|
||||
```
|
||||
|
||||
**Canvas Visual Feedback:**
|
||||
- ❌ Incompatible suggested edges appear as **red dashed lines** (don't auto-add)
|
||||
- ⚠️ Disconnected nodes show **yellow border** with icon
|
||||
- ✅ Compatible edges appear as **green solid lines** (safe to accept)
|
||||
|
||||
---
|
||||
|
||||
### Error Handling
|
||||
|
||||
If LLM inference fails:
|
||||
- First activity retry (2-second backoff)
|
||||
- Second activity retry (4-second backoff)
|
||||
- Third activity retry (8-second backoff)
|
||||
- If all retries fail, workflow records error and proceeds to next activity (or fails if terminal)
|
||||
|
||||
**Common Failure Scenarios:**
|
||||
- Network timeout: `connection refused` (retry automatically)
|
||||
- Model not found: `unknown model: xyz` (terminal error)
|
||||
- Rate limited: HTTP 429 (retry with exponential backoff)
|
||||
- Prompt too long: `context length exceeded` (terminal error)
|
||||
|
||||
---
|
||||
|
||||
### Performance & Cost
|
||||
|
||||
- Single prompt inference: ~100-500ms (model-dependent)
|
||||
- Batch processing: Serial (not parallel), ~100-500ms per prompt
|
||||
- Model inference costs: Free (on-premise Ollama/Reasoning models)
|
||||
- Token counting: Provided in response for quota tracking
|
||||
|
||||
**Optimization Tips:**
|
||||
- Use `ornith:13b` or `qwen2.5:3b` for faster inference
|
||||
- Use `reasoning` only for complex analysis that needs reasoning
|
||||
- Cache frequently-used prompts at workflow level
|
||||
- Use batch activity for multiple similar prompts (better throughput)
|
||||
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
# SLA: API Gateway & Platform Services
|
||||
|
||||
## API Gateway (api.riotpiao.com)
|
||||
|
||||
### Availability
|
||||
|
||||
| Target | Measurement | Alert |
|
||||
|--------|------------|-------|
|
||||
| 99.9% uptime | `probe_success{instance=~".*api.riotpiao.com.*"}` | `APIGatewayProbeDown` fires after 2m down |
|
||||
| Monthly budget: 43.8 min downtime | 7-day SLO: `avg_over_time(probe_success[7d]) * 100` | |
|
||||
| Zero ready pods = critical | `sum(kube_pod_status_ready{namespace="api"}) == 0` | `APIGatewayDown` fires after 1m |
|
||||
|
||||
### Latency
|
||||
|
||||
Baselines measured from 200-request canary run against live cluster.
|
||||
SLA set at ~2x measured p99 for headroom.
|
||||
|
||||
| Endpoint | Measured p50 | Measured p99 | SLA (p95) | SLA (p99) | Alert |
|
||||
|----------|-------------|-------------|-----------|-----------|-------|
|
||||
| LLM Chat (qwen) | 514ms | 609ms | <1s | <2s | `APIGatewayLatencyHigh` |
|
||||
| LLM Chat (reasoning) | 300ms | 328ms | <1s | <2s | `APIGatewayLatencyHigh` |
|
||||
| LLM Chat (ornith:35b) | 1.2s | 1.2s | <3s | <5s | `APIGatewayLatencyCritical` |
|
||||
| LLM Chat (streaming) | 569ms | 628ms | <1s | <2s | `APIGatewayLatencyHigh` |
|
||||
| Embeddings | 189ms | 287ms | <500ms | <1s | `APIGatewayLatencyHigh` |
|
||||
| Rerank | 106ms | 218ms | <500ms | <1s | `APIGatewayLatencyHigh` |
|
||||
| Models list | 68ms | 277ms | <300ms | <500ms | `APIGatewayLatencyHigh` |
|
||||
| Auth rejection | 69ms | 87ms | <200ms | <500ms | (no alert, expected fast) |
|
||||
|
||||
### Error Rate
|
||||
|
||||
| Target | Measurement | Alert |
|
||||
|--------|------------|-------|
|
||||
| 5xx < 1% | `nginx_ingress_controller_requests{status=~"5.."}` / total | `APIGateway5xxErrorRate` fires after 5m >1% |
|
||||
| Total errors < 10% | 4xx + 5xx / total | `APIGatewayHighErrorRate` fires after 10m >10% |
|
||||
|
||||
---
|
||||
|
||||
## LLM Serving (llm-serving namespace)
|
||||
|
||||
| Target | Measurement | Alert |
|
||||
|--------|------------|-------|
|
||||
| All predictors running | replicas ready == desired per deployment | `LLMPredictorDown` fires after 5m |
|
||||
| Zero LLM pods = critical | `sum(ready{namespace="llm-serving"}) == 0` | `LLMServingDown` fires after 2m |
|
||||
| No restart storms | restart count in 15m | `LLMPredictorRestarted` on any restart |
|
||||
|
||||
---
|
||||
|
||||
## Cluster Infrastructure
|
||||
|
||||
### Node Health
|
||||
|
||||
| Target | Measurement | Alert |
|
||||
|--------|------------|-------|
|
||||
| All nodes Ready | `kube_node_status_condition` | `NodeNotReady` fires after 2m |
|
||||
| CPU < 90% sustained | `node_cpu_seconds_total` | `NodeHighCPU` fires after 15m |
|
||||
| Memory < 90% sustained | `node_memory_MemAvailable_bytes` | `NodeHighMemory` fires after 15m |
|
||||
| Disk < 85% | `node_filesystem_avail_bytes` | `NodeDiskFull` fires after 5m (critical) |
|
||||
|
||||
### Pod Health
|
||||
|
||||
| Target | Measurement | Alert |
|
||||
|--------|------------|-------|
|
||||
| No pods pending > 10m | `kube_pod_status_phase{phase="Pending"}` | `PodStuckPending` |
|
||||
| No CrashLoopBackOff > 5m | `kube_pod_container_status_waiting_reason` | `PodCrashLooping` (critical) |
|
||||
| OOMKilled < 3/hour | `kube_pod_container_status_last_terminated_reason` | `OOMKilledSpike` |
|
||||
| No restart storms | >5 restarts in 15m | `ContainerRestartStorm` |
|
||||
|
||||
### Jobs
|
||||
|
||||
| Target | Measurement | Alert |
|
||||
|--------|------------|-------|
|
||||
| No failed jobs | `kube_job_status_failed > 0` | `JobFailed` fires after 5m |
|
||||
| No stuck jobs > 2h | `kube_job_status_active` + age | `JobStuckRunning` |
|
||||
| CronJobs on schedule | last_schedule vs next_schedule | `CronJobMissedSchedule` fires after 10m |
|
||||
|
||||
### Storage
|
||||
|
||||
| Target | Measurement | Alert |
|
||||
|--------|------------|-------|
|
||||
| Longhorn drives healthy | `longhorn_disk_health` | `LonghornDriveOffline` fires after 5m (critical) |
|
||||
|
||||
### DNS
|
||||
|
||||
| Target | Measurement | Alert |
|
||||
|--------|------------|-------|
|
||||
| CoreDNS SERVFAIL < 0.5/s | `coredns_dns_responses_total{rcode="SERVFAIL"}` | `CoreDNSErrorSpike` fires after 5m |
|
||||
|
||||
### Probes
|
||||
|
||||
| Target | Measurement | Alert |
|
||||
|--------|------------|-------|
|
||||
| All service probes passing | `probe_success` | `ServiceProbeDown` fires after 3m (critical) |
|
||||
| Probe latency < 2s | `probe_duration_seconds` | `ServiceProbeSlow` fires after 5m |
|
||||
| Certs valid > 14 days | `certmanager_certificate_expiration_timestamp_seconds` | `CertificateExpiringSoon` |
|
||||
|
||||
---
|
||||
|
||||
## Alert Severity Levels
|
||||
|
||||
| Severity | Meaning | Response Time |
|
||||
|----------|---------|--------------|
|
||||
| **critical** | Service down or data loss risk. Immediate impact on users. | Investigate within 15 min |
|
||||
| **warning** | Degraded performance or resource pressure. No immediate outage. | Investigate within 4 hours |
|
||||
|
||||
### Critical Alerts (require immediate action)
|
||||
|
||||
- `APIGatewayDown` — zero gateway pods
|
||||
- `LLMServingDown` — zero LLM pods
|
||||
- `NodeNotReady` — node lost
|
||||
- `PodCrashLooping` — service crashing repeatedly
|
||||
- `NodeDiskFull` — disk > 85%
|
||||
- `LonghornDriveOffline` — storage unhealthy
|
||||
- `ServiceProbeDown` — external service unreachable
|
||||
- `APIGateway5xxErrorRate` — 5xx > 1%
|
||||
- `APIGatewayLatencyCritical` — p99 > 5s
|
||||
|
||||
---
|
||||
|
||||
## Current Alert Status
|
||||
|
||||
Alerts firing after deployment:
|
||||
|
||||
| Alert | State | Root Cause |
|
||||
|-------|-------|-----------|
|
||||
| `APIGatewayProbeDown` | pending | Blackbox probe for api-gateway not yet active (pod restart needed) |
|
||||
| `PodStuckPending` | pending | `sms/macos-bluebubbles` pending 22d (scheduling constraint) |
|
||||
| `PodCrashLooping` | pending | `iam/authentik-provision` job in Error state |
|
||||
| `DeploymentReplicasUnavailable` | pending | Same root causes above |
|
||||
| `ServiceProbeDown` | pending | api-gateway probe target not in blackbox yet |
|
||||
|
||||
None are false positives. All reflect real cluster state.
|
||||
+53
-17
@@ -3,6 +3,7 @@ package auth
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -10,21 +11,33 @@ import (
|
||||
"github.com/golang-jwt/jwt/v5"
|
||||
)
|
||||
|
||||
// isValidIssuer checks if issuer is from Authentik (any provider/app).
|
||||
// Accepts: https://authentik.riotpiao.com/application/o/{provider}/
|
||||
func isValidIssuer(iss string) bool {
|
||||
return strings.Contains(iss, "authentik.riotpiao.com/application/o/") &&
|
||||
strings.HasSuffix(iss, "/")
|
||||
}
|
||||
|
||||
// Validator validates JWTs against Authentik JWKS.
|
||||
// Supports multi-issuer: any Authentik service account provider is accepted
|
||||
// (portfolio-agent, memory-agent, api-gw, etc.) because all share the same
|
||||
// JWKS signing key.
|
||||
type Validator struct {
|
||||
issuer string
|
||||
audience string
|
||||
issuer string // Not used for validation (kept for logging); issuer regex check is sufficient
|
||||
audience string // Not used for validation; any audience from valid Authentik issuer is accepted
|
||||
jwksURL string
|
||||
jwks *keyfunc.JWKS
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// NewValidator creates a new JWT validator for a service.
|
||||
// issuer and audience params are deprecated (ignored for validation) but kept
|
||||
// for backward compatibility. Multi-issuer validation via isValidIssuer() is used instead.
|
||||
// JWKS fetching is lazy (deferred until first validation).
|
||||
func NewValidator(issuer, audience, jwksURL string) *Validator {
|
||||
return &Validator{
|
||||
issuer: issuer,
|
||||
audience: audience,
|
||||
issuer: issuer, // deprecated param, kept for compat
|
||||
audience: audience, // deprecated param, kept for compat
|
||||
jwksURL: jwksURL,
|
||||
jwks: nil, // Lazy-loaded on first use
|
||||
}
|
||||
@@ -107,32 +120,56 @@ func (v *Validator) ValidateBearerToken(authHeader string) (jwt.MapClaims, error
|
||||
}
|
||||
}
|
||||
|
||||
// Check iss (issuer)
|
||||
if iss, ok := claims["iss"].(string); !ok || iss != v.issuer {
|
||||
return nil, fmt.Errorf("invalid issuer: expected %s, got %s", v.issuer, iss)
|
||||
// Check iss (issuer) - accept any Authentik provider issuer
|
||||
// (portfolio-agent, memory-agent, api-gw, etc.)
|
||||
// All use same signing key so JWKS validation is sufficient
|
||||
if iss, ok := claims["iss"].(string); !ok {
|
||||
return nil, fmt.Errorf("missing issuer claim")
|
||||
} else if !isValidIssuer(iss) {
|
||||
return nil, fmt.Errorf("invalid issuer: %s", iss)
|
||||
}
|
||||
|
||||
// Check aud (audience)
|
||||
if aud, ok := claims["aud"].(string); !ok || aud != v.audience {
|
||||
return nil, fmt.Errorf("invalid audience: expected %s, got %s", v.audience, aud)
|
||||
// Check aud (audience) - accept any Authentik-provided audience
|
||||
// since all Authentik service accounts use the same signing key.
|
||||
// The issuer check above is sufficient to ensure JWT came from Authentik.
|
||||
if aud, ok := claims["aud"].(string); !ok {
|
||||
return nil, fmt.Errorf("missing audience claim")
|
||||
} else if aud == "" {
|
||||
return nil, fmt.Errorf("empty audience claim")
|
||||
}
|
||||
// Note: Not hardcoding expected audience. Any audience from a valid Authentik
|
||||
// issuer is accepted, since all service accounts are under the same trust boundary.
|
||||
|
||||
return claims, nil
|
||||
}
|
||||
|
||||
// CheckPermissions checks if claims contain required permission(s).
|
||||
// Checks both "permissions" claim (for users) and "roles" claim (for service accounts).
|
||||
// Returns true if any required permission is found or wildcard "*" exists.
|
||||
func (v *Validator) CheckPermissions(claims jwt.MapClaims, required ...string) bool {
|
||||
permsIface, ok := claims["permissions"]
|
||||
if !ok {
|
||||
return false
|
||||
// Try permissions claim first (for user tokens)
|
||||
if permsIface, ok := claims["permissions"]; ok {
|
||||
if perms, ok := permsIface.([]interface{}); ok {
|
||||
if v.checkPermList(perms, required...) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
perms, ok := permsIface.([]interface{})
|
||||
if !ok {
|
||||
return false
|
||||
// Fall back to roles claim (for service account tokens)
|
||||
if rolesIface, ok := claims["roles"]; ok {
|
||||
if roles, ok := rolesIface.([]interface{}); ok {
|
||||
if v.checkPermList(roles, required...) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// checkPermList is a helper that checks a permission/role list.
|
||||
func (v *Validator) checkPermList(perms []interface{}, required ...string) bool {
|
||||
for _, perm := range perms {
|
||||
permStr, ok := perm.(string)
|
||||
if !ok {
|
||||
@@ -147,7 +184,6 @@ func (v *Validator) CheckPermissions(claims jwt.MapClaims, required ...string) b
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
|
||||
@@ -323,7 +323,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
writeProblemDetail(w, http.StatusForbidden,
|
||||
"https://api.example.com/problems/forbidden",
|
||||
"Forbidden",
|
||||
fmt.Sprintf("JWT validation failed: %v", err),
|
||||
"JWT validation failed",
|
||||
nil)
|
||||
logging.Errorf("auth failed", err, map[string]string{
|
||||
"path": r.URL.Path,
|
||||
|
||||
+2
-2
@@ -109,8 +109,8 @@ spec:
|
||||
- ALL
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: api-gateway-config
|
||||
secret:
|
||||
secretName: api-gateway-config
|
||||
affinity:
|
||||
podAntiAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: api-gateway-config
|
||||
namespace: api
|
||||
labels:
|
||||
app: api-gateway
|
||||
type: Opaque
|
||||
stringData:
|
||||
config.yaml: |
|
||||
auth:
|
||||
enabled: true
|
||||
issuer: "https://authentik.riotpiao.com/application/o/api-gw/"
|
||||
audience: "api-gw"
|
||||
jwksUrl: "http://authentik-server.iam.svc.cluster.local/application/o/api-gw/jwks/"
|
||||
requiredCapability: "llm:inference"
|
||||
routes: []
|
||||
models:
|
||||
- name: "reasoning"
|
||||
address: "reasoning-predictor.llm-serving:80"
|
||||
path: "/v1/chat/completions"
|
||||
- name: "ornith:35b"
|
||||
address: "ornith-predictor.llm-serving:80"
|
||||
path: "/v1/chat/completions"
|
||||
- name: "qwen2.5:3b-instruct"
|
||||
address: "ornith-predictor.llm-serving:80"
|
||||
path: "/v1/chat/completions"
|
||||
- name: "nomic-ai/nomic-embed-text-v2-moe"
|
||||
address: "embeddings-predictor.llm-serving:80"
|
||||
path: "/v1/embeddings"
|
||||
- name: "BAAI/bge-reranker-base"
|
||||
address: "reranker-predictor.llm-serving:80"
|
||||
path: "/v1/rerank"
|
||||
adapters:
|
||||
- serviceName: sqs
|
||||
upstream:
|
||||
url: http://management-service.sqs.svc.cluster.local:9090
|
||||
timeoutSeconds: 30
|
||||
auth:
|
||||
required: true
|
||||
resources:
|
||||
- name: send-message
|
||||
methods:
|
||||
- verb: POST
|
||||
upstreamPath: /sqs/send
|
||||
- name: receive-message
|
||||
methods:
|
||||
- verb: POST
|
||||
upstreamPath: /sqs/receive
|
||||
- name: list-queues
|
||||
methods:
|
||||
- verb: GET
|
||||
upstreamPath: /sqs/queues
|
||||
- serviceName: workflow
|
||||
upstream:
|
||||
url: grpc://temporal-frontend.temporal.svc.cluster.local:7233
|
||||
timeoutSeconds: 60
|
||||
auth:
|
||||
required: false
|
||||
resources:
|
||||
- name: execute
|
||||
methods:
|
||||
- verb: POST
|
||||
upstreamPath: /temporal.api.workflowservice.v1.WorkflowService/ExecuteWorkflow
|
||||
- name: describe
|
||||
methods:
|
||||
- verb: GET
|
||||
upstreamPath: /temporal.api.workflowservice.v1.WorkflowService/DescribeWorkflowExecution
|
||||
- name: list
|
||||
methods:
|
||||
- verb: GET
|
||||
upstreamPath: /temporal.api.workflowservice.v1.WorkflowService/ListWorkflowExecutions
|
||||
- serviceName: memory
|
||||
upstream:
|
||||
url: http://poimen-memory.poimen.svc.cluster.local:8080
|
||||
timeoutSeconds: 30
|
||||
auth:
|
||||
required: false
|
||||
resources:
|
||||
- name: query
|
||||
methods:
|
||||
- verb: POST
|
||||
upstreamPath: /memory/query
|
||||
- name: ingest
|
||||
methods:
|
||||
- verb: POST
|
||||
upstreamPath: /memory/ingest
|
||||
- name: skills
|
||||
methods:
|
||||
- verb: GET
|
||||
upstreamPath: /memory/skills
|
||||
- serviceName: s3
|
||||
upstream:
|
||||
url: http://minio.storage.svc.cluster.local:9000
|
||||
timeoutSeconds: 30
|
||||
auth:
|
||||
required: false
|
||||
resources:
|
||||
- name: list-objects
|
||||
methods:
|
||||
- verb: GET
|
||||
upstreamPath: /
|
||||
- name: get-object
|
||||
methods:
|
||||
- verb: GET
|
||||
upstreamPath: /
|
||||
- name: put-object
|
||||
methods:
|
||||
- verb: PUT
|
||||
upstreamPath: /
|
||||
- serviceName: iam
|
||||
upstream:
|
||||
url: http://authentik-server.iam.svc.cluster.local:80
|
||||
timeoutSeconds: 30
|
||||
auth:
|
||||
required: false
|
||||
resources:
|
||||
- name: list-roles
|
||||
methods:
|
||||
- verb: GET
|
||||
upstreamPath: /api/v3/roles
|
||||
- name: list-users
|
||||
methods:
|
||||
- verb: GET
|
||||
upstreamPath: /api/v3/users
|
||||
- name: create-role
|
||||
methods:
|
||||
- verb: POST
|
||||
upstreamPath: /api/v3/roles
|
||||
@@ -8,7 +8,7 @@ resources:
|
||||
- service.yaml
|
||||
- deployment.yaml
|
||||
- network-policy.yaml
|
||||
- configmap.yaml
|
||||
- gateway-config-secret.enc.yaml
|
||||
|
||||
# The deployed image tag lives here and nowhere else. CI publishes
|
||||
# forgejo.riotpiao.com/rock/api-gateway:<commit-sha> and tags it as :latest on main.
|
||||
|
||||
Reference in New Issue
Block a user