From 96ae855d35edfeda2ddd98b0af6461f43c45f11d Mon Sep 17 00:00:00 2001 From: rock Date: Sun, 30 Aug 2026 18:02:25 -0700 Subject: [PATCH] fix: default auth to Bearer token (riotpiao gateway uses JWT now) --- crates/mem-llm/src/chat.rs | 15 +- docs/LIFECYCLE.md | 643 +++++++++++++++++++++++++++++++++++++ 2 files changed, 646 insertions(+), 12 deletions(-) create mode 100644 docs/LIFECYCLE.md diff --git a/crates/mem-llm/src/chat.rs b/crates/mem-llm/src/chat.rs index f71daac..e0fc6b6 100644 --- a/crates/mem-llm/src/chat.rs +++ b/crates/mem-llm/src/chat.rs @@ -38,18 +38,9 @@ impl AuthMode { if api_key.is_empty() { return Self::None; } - // OpenRouter, OpenAI, Anthropic, Together, etc. use Bearer - if base_url.contains("openrouter.ai") - || base_url.contains("api.openai.com") - || base_url.contains("api.anthropic.com") - || base_url.contains("api.together.xyz") - || base_url.contains("api.groq.com") - { - Self::Bearer - } else { - // Default: apikey header (riotpiao gateway) - Self::ApiKeyHeader - } + // Bearer is the standard for OpenAI-compatible APIs + // including riotpiao gateway (JWT/OAuth), OpenRouter, OpenAI, etc. + Self::Bearer } } diff --git a/docs/LIFECYCLE.md b/docs/LIFECYCLE.md new file mode 100644 index 0000000..5cba8f6 --- /dev/null +++ b/docs/LIFECYCLE.md @@ -0,0 +1,643 @@ +# Document Lifecycle — Control-Plane Flow + +This document traces the complete lifecycle of a single piece of knowledge from ingestion to retrieval, showing every component it touches and every decision made along the way. + +--- + +## Example: "Rust Ownership & Borrowing" knowledge file + +``` +rust-ownership.md (source) → mem learn CLI → POST /memory/learn → chunk → +gated loop (evaluate + compact) → embed → pgvector → searchable via /memory/query +``` + +--- + +## Phase 1: Submission + +**Actor:** Agent or human operator +**Entry point:** `mem learn` CLI or `POST /memory/learn` + +``` +┌─────────────────────────────────────────────────────┐ +│ rust-ownership.md │ +│ │ +│ # Rust Ownership │ +│ ## Borrowing Rules │ +│ - &T immutable borrow, &mut T mutable borrow │ +│ - Cannot have &mut while & exists │ +│ ## Lifetimes │ +│ - 'a annotations tell compiler how long refs live │ +│ ## Smart Pointers │ +│ - Box heap allocation, single ownership │ +│ - Arc atomic reference counting, thread-safe │ +└─────────────────────────────────────────────────────┘ + │ + ▼ + ┌──────────────────┐ + │ mem learn CLI │ + │ │ + │ Reads file │ + │ Validates .md │ + │ Sends to API │ + └────────┬─────────┘ + │ + HTTP POST /memory/learn + { + "project": "knowledge", + "text": "", + "query": "What are key facts in rust-ownership?", + "model": "qwen2.5:3b-instruct", + "memory_budget": 4096 + } + │ + ▼ +``` + +## Phase 2: Authentication & Rate Limiting + +**Component:** `http_server.rs` → `jwt_validator.rs` → `rate_limiter.rs` + +``` + ┌──────────────────────┐ + │ Auth Gate │ + │ │ + │ JWT token valid? │──── no ──→ 401 Unauthorized + │ Has memory:write? │──── no ──→ 403 Forbidden + │ Rate limit ok? │──── no ──→ 429 Too Many Requests + │ (100 ingest/hr) │ + └──────────┬───────────┘ + │ yes + ▼ +``` + +## Phase 3: Chunking + +**Component:** `chunk_markdown_text()` in `http_server.rs` + +The raw markdown is split into chunks on `## ` heading boundaries. Each chunk stays under `chunk_size` (default 2000 chars). + +``` + ┌──────────────────────────────────┐ + │ Chunker │ + │ │ + │ Split on ## headings │ + │ Hard-split if > chunk_size │ + │ │ + │ Input: 1 file (3200 chars) │ + │ Output: 4 chunks │ + └──────────┬───────────────────────┘ + │ + ┌────────────┼────────────┬──────────────┐ + ▼ ▼ ▼ ▼ + ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ + │ Chunk 1 │ │ Chunk 2 │ │ Chunk 3 │ │ Chunk 4 │ + │ # Title │ │ ## Borrow│ │ ## Life │ │ ## Smart │ + │ + intro │ │ Rules │ │ times │ │ Pointers │ + │ 400 ch │ │ 600 ch │ │ 500 ch │ │ 700 ch │ + └──────────┘ └──────────┘ └──────────┘ └──────────┘ + │ │ │ │ + └────────────┴────────────┴──────────────┘ + │ + ▼ +``` + +## Phase 4: Gated Loop (The Core) + +**Component:** `gated_loop.rs` → LLM (`qwen2.5:3b-instruct`) + +This is where evaluation and compaction happen. The LLM sees each chunk alongside the current memory and decides: +- **Update gate** (`yes|no`): Does this chunk contain new knowledge? +- **Candidate memory** (`...`): Rewritten memory incorporating the chunk +- **Exit gate** (`continue|end`): Should we stop scanning? + +``` + ┌─────────────────────────────────────────────────────┐ + │ GATED LOOP │ + │ │ + │ Memory = "" (empty at start) │ + │ Query = "What are key facts in rust-ownership?" │ + │ │ + │ ┌─────────────────────────────────────────────┐ │ + │ │ Turn 1: Chunk 1 (# Title + intro) │ │ + │ │ │ │ + │ │ LLM receives: │ │ + │ │ System: "You are evaluating evidence..." │ │ + │ │ User: "Memory: (empty) │ │ + │ │ Chunk: # Rust Ownership..." │ │ + │ │ │ │ + │ │ LLM responds: │ │ + │ │ Title only, no substance │ │ + │ │ no │ │ + │ │ │ │ + │ │ continue │ │ + │ │ │ │ + │ │ Decision: REJECTED (no new knowledge) │ │ + │ │ Memory: "" (unchanged) │ │ + │ └─────────────────────────────────────────────┘ │ + │ │ │ + │ ▼ │ + │ ┌─────────────────────────────────────────────┐ │ + │ │ Turn 2: Chunk 2 (## Borrowing Rules) │ │ + │ │ │ │ + │ │ LLM responds: │ │ + │ │ Core ownership rules, new │ │ + │ │ yes │ │ + │ │ │ │ + │ │ Rust borrowing: &T immutable, &mut T │ │ + │ │ mutable. Cannot hold &mut while & alive │ │ + │ │ │ │ + │ │ continue │ │ + │ │ │ │ + │ │ Decision: ACCEPTED │ │ + │ │ Memory: "Rust borrowing: &T immutable..." │ │ + │ └─────────────────────────────────────────────┘ │ + │ │ │ + │ ▼ │ + │ ┌─────────────────────────────────────────────┐ │ + │ │ Turn 3: Chunk 3 (## Lifetimes) │ │ + │ │ │ │ + │ │ LLM receives current memory + new chunk │ │ + │ │ │ │ + │ │ LLM responds: │ │ + │ │ New info, adds to borrowing│ │ + │ │ yes │ │ + │ │ │ │ + │ │ Rust ownership: &T immutable borrow, │ │ + │ │ &mut T mutable. Cannot coexist. │ │ + │ │ Lifetimes: 'a annotations for ref │ │ + │ │ duration. Elision: single input → output│ │ + │ │ │ │ + │ │ continue │ │ + │ │ │ │ + │ │ Decision: ACCEPTED + COMPACTED │ │ + │ │ Memory: merged borrowing + lifetimes │ │ + │ │ (Note: LLM rewrote, not appended) │ │ + │ └─────────────────────────────────────────────┘ │ + │ │ │ + │ ▼ │ + │ ┌─────────────────────────────────────────────┐ │ + │ │ Turn 4: Chunk 4 (## Smart Pointers) │ │ + │ │ │ │ + │ │ Decision: ACCEPTED + COMPACTED │ │ + │ │ Memory: ownership + lifetimes + pointers │ │ + │ │ (Still within memory_budget: 4096 tokens) │ │ + │ └─────────────────────────────────────────────┘ │ + │ │ + │ Result: │ + │ chunks_seen: 4 │ + │ chunks_used: 3 (chunk 1 rejected) │ + │ final_memory: "Rust ownership: &T immutable │ + │ borrow, &mut T mutable. Cannot coexist. │ + │ Lifetimes: 'a annotations... Box heap, │ + │ Arc thread-safe shared ownership..." │ + │ │ + └──────────────────────┬──────────────────────────────┘ + │ + ▼ +``` + +### Pluggable LLM Architecture + +The gated loop accepts any `LlmClient` trait implementation. The current design +uses a single model for both gate and rewrite. The target design splits them: + +``` +┌─────────────────────────────────────────────────────────────────────┐ +│ GATED LOOP (pluggable LLMs) │ +│ │ +│ LoopConfig { │ +│ gate_model: "qwen2.5:3b-instruct" ← fast, cheap │ +│ rewrite_model: "ornith:35b" ← accurate, expensive │ +│ embed_model: "nomic-embed-text-v2" ← fixed 768-dim │ +│ } │ +│ │ +│ For each chunk: │ +│ │ +│ ┌──────────┐ ┌───────────────────────────────────────────┐ │ +│ │ Chunk │────▶│ GATE LLM (3B, fast) │ │ +│ │ + M_t-1 │ │ │ │ +│ └──────────┘ │ Prompt: "Does this chunk add new │ │ +│ │ knowledge not already in memory?" │ │ +│ │ │ │ +│ │ Output: yes|no │ │ +│ │ continue|end │ │ +│ │ │ │ +│ │ Cost: ~200 tokens in, ~20 tokens out │ │ +│ │ Latency: ~100ms │ │ +│ └──────────┬──────────┬─────────────────────┘ │ +│ │ │ │ +│ yes │ │ no │ +│ ▼ ▼ │ +│ ┌─────────────┐ ┌──────────┐ │ +│ │ REWRITE LLM │ │ SKIP │ │ +│ │ (35B, smart)│ │ │ │ +│ │ │ │ M_t = M │ │ +│ │ Prompt: │ │ (no │ │ +│ │ "Rewrite │ │ change) │ │ +│ │ memory │ └──────────┘ │ +│ │ merging │ │ +│ │ old + new. │ │ +│ │ Keep all │ │ +│ │ facts. │ │ +│ │ Stay under │ │ +│ │ budget." │ │ +│ │ │ │ +│ │ Cost: ~2K │ │ +│ │ tokens in, │ │ +│ │ ~1K out │ │ +│ │ Latency: │ │ +│ │ ~2-5s │ │ +│ └──────┬──────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────┐ │ +│ │ Budget │ │ +│ │ Check │ │ +│ │ │ │ +│ │ len(M_new) │ │ +│ │ > budget? │ │ +│ └──┬──────┬───┘ │ +│ no │ │ yes │ +│ ▼ ▼ │ +│ ┌─────────┐ ┌──────────┐ │ +│ │ Accept │ │ Reject │ │ +│ │ M_t = │ │ Budget │ │ +│ │ M_new │ │ exceeded │ │ +│ └─────────┘ └──────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────────┘ +``` + +#### Cost Model (per file, ~5 chunks) + +| Step | Model | Calls | Tokens/call | Total tokens | +|---|---|---|---|---| +| Gate (all chunks) | 3B | 5 | ~220 | ~1,100 | +| Rewrite (accepted only, ~3) | 35B | 3 | ~3,000 | ~9,000 | +| Embed (final memory) | nomic-embed | 1 | ~500 | ~500 | + +Rejected chunks only cost a gate call (~100ms). The expensive rewrite +only runs when the gate accepts. For a typical file where 60% of chunks +are accepted, this saves ~40% of 35B inference cost vs single-model. + +#### Trait Interface + +```rust +/// Pluggable LLM backend for the gated loop. +pub trait LlmClient: Send + Sync { + fn complete_blocking( + &self, system: &str, user: &str, max_tokens: usize + ) -> Result; +} + +/// Split-model client: fast gate + accurate rewrite. +pub struct SplitModelClient { + gate: Box, // qwen2.5:3b — decides yes/no + rewrite: Box, // ornith:35b — produces compacted memory +} + +/// Single-model client (current default). +pub struct SingleModelClient { + llm: ChatClient, // same model for gate + rewrite +} +``` + +#### API Request (pluggable models) + +```json +POST /memory/learn +{ + "project": "knowledge", + "text": "## Rust Ownership\n...", + "gate_model": "qwen2.5:3b-instruct", + "rewrite_model": "ornith:35b", + "memory_budget": 4096 +} +``` + +If `rewrite_model` is omitted, falls back to `gate_model` (single-model mode, +current behavior). This keeps the API backward-compatible. + +### Supported Providers + +The `ChatClient` uses OpenAI-compatible `/v1/chat/completions` format. +Auth mode is auto-detected from the base URL: + +``` +┌──────────────────────────────────────────────────────────────────┐ +│ Provider Support │ +│ │ +│ ┌─────────────────┐ ┌──────────────────┐ │ +│ │ riotpiao gateway│ │ OpenRouter │ │ +│ │ │ │ │ │ +│ │ base: api.riot │ │ base: openrouter │ │ +│ │ piao.com/v1 │ │ .ai/api/v1 │ │ +│ │ auth: apikey: │ │ auth: Bearer │ │ +│ │ header │ │ header │ │ +│ │ models: │ │ models: │ │ +│ │ qwen2.5:3b │ │ openai/gpt-4o │ │ +│ │ ornith:35b │ │ anthropic/ │ │ +│ │ reasoning │ │ claude-sonnet │ │ +│ └────────┬────────┘ │ google/gemini │ │ +│ │ └────────┬─────────┘ │ +│ │ │ │ +│ ▼ ▼ │ +│ ┌──────────────────────────────────────────┐ │ +│ │ ChatClient (OpenAI-compatible) │ │ +│ │ │ │ +│ │ POST {base_url}/chat/completions │ │ +│ │ { │ │ +│ │ "model": "...", │ │ +│ │ "messages": [{role, content}, ...], │ │ +│ │ "max_tokens": N │ │ +│ │ } │ │ +│ │ │ │ +│ │ Auth auto-detected from URL: │ │ +│ │ *openrouter.ai → Bearer │ │ +│ │ *api.openai.com → Bearer │ │ +│ │ *api.riotpiao.com → apikey header │ │ +│ │ *localhost → None │ │ +│ └──────────────────────────────────────────┘ │ +│ │ +│ ┌─────────────────┐ ┌──────────────────┐ │ +│ │ OpenAI │ │ Ollama (local) │ │ +│ │ │ │ │ │ +│ │ base: api.open │ │ base: localhost │ │ +│ │ ai.com/v1 │ │ :11434/v1 │ │ +│ │ auth: Bearer │ │ auth: None │ │ +│ │ models: │ │ models: │ │ +│ │ gpt-4o │ │ llama3.1 │ │ +│ │ gpt-4o-mini │ │ qwen2.5:3b │ │ +│ └─────────────────┘ └──────────────────┘ │ +│ │ +└──────────────────────────────────────────────────────────────────┘ +``` + +#### Environment Configuration + +```bash +# riotpiao gateway (default) +export LLM_API_BASE=https://api.riotpiao.com/v1 +export LLM_API_KEY=your-gateway-key + +# OpenRouter (access to GPT-4o, Claude, Gemini, etc.) +export LLM_API_BASE=https://openrouter.ai/api/v1 +export LLM_API_KEY=sk-or-v1-... + +# OpenAI direct +export LLM_API_BASE=https://api.openai.com/v1 +export LLM_API_KEY=sk-... + +# Ollama local (no auth) +export LLM_API_BASE=http://localhost:11434/v1 +export LLM_API_KEY= +``` + +#### Mixed Provider Example (gate local, rewrite cloud) + +```json +POST /memory/learn +{ + "project": "knowledge", + "text": "...", + "gate_model": "qwen2.5:3b-instruct", + "gate_base_url": "https://api.riotpiao.com/v1", + "rewrite_model": "anthropic/claude-sonnet-4", + "rewrite_base_url": "https://openrouter.ai/api/v1" +} +``` + +Gate runs on local 3B (free, fast), rewrite uses Claude via OpenRouter +(accurate, paid per token). Only accepted chunks hit the cloud. + +### Why This Prevents Unbounded Growth + +The key insight: **the LLM rewrites memory on every accepted chunk, not appends.** Turn 3 doesn't add lifetimes after borrowing — it produces a new memory that integrates both. If the same file is ingested again, most chunks will be rejected (`no`) because the knowledge already exists in memory. + +The `memory_budget` parameter (default 4096 tokens) enforces a hard cap. If the LLM's candidate exceeds it, the chunk triggers `BudgetExceeded` and is skipped. + +## Phase 5: Embedding + +**Component:** `EmbeddingsClient` → `nomic-ai/nomic-embed-text-v2-moe` + +The compacted final memory is embedded into a 768-dimensional vector. + +``` + ┌──────────────────────────────────────────┐ + │ Embedding │ + │ │ + │ Input: "Rust ownership: &T immutable │ + │ borrow, &mut T mutable. Cannot..." │ + │ │ + │ Model: nomic-embed-text-v2-moe │ + │ Output: [0.035, 0.022, -0.018, ...] │ + │ 768 dimensions │ + │ │ + └──────────────────┬───────────────────────┘ + │ + ▼ +``` + +## Phase 6: Storage + +**Component:** `pgvector` (CNPG cluster, 2 replicas) + +The compacted memory + embedding is stored. SHA256 dedup ensures re-ingesting the same content updates rather than duplicates. + +``` + ┌──────────────────────────────────────────┐ + │ pgvector (memory_chunks) │ + │ │ + │ INSERT INTO memory_chunks │ + │ (id, project, level, text, │ + │ embedding, sha256, source) │ + │ ON CONFLICT (sha256) │ + │ DO UPDATE SET text, embedding │ + │ │ + │ ┌────────────────────────────────────┐ │ + │ │ id: uuid-abc-123 │ │ + │ │ project: knowledge │ │ + │ │ level: L1 │ │ + │ │ text: "Rust ownership: ..." │ │ + │ │ embedding: [0.035, 0.022, ...] │ │ + │ │ sha256: de12cd34ef56... │ │ + │ │ source: learn://knowledge │ │ + │ └────────────────────────────────────┘ │ + │ │ + └──────────────────┬───────────────────────┘ + │ + ▼ +``` + +## Phase 7: Response + +**Component:** `http_server.rs` + +``` + ┌──────────────────────────────────────────┐ + │ HTTP Response │ + │ │ + │ 200 OK │ + │ { │ + │ "project": "knowledge", │ + │ "status": "completed", │ + │ "chunks_seen": 4, │ + │ "chunks_used": 3, │ + │ "memory": "Rust ownership: &T...", │ + │ "memory_tokens": 156, │ + │ "stored": true, │ + │ "model": "qwen2.5:3b-instruct" │ + │ } │ + └──────────────────────────────────────────┘ +``` + +--- + +## Phase 8: Retrieval (Later) + +**Actor:** Agent querying memory during a task +**Entry point:** `POST /memory/query` or `POST /memory/context` + +``` +Agent: "How does Rust handle shared ownership across threads?" + │ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ Retrieval Pipeline │ +│ │ +│ 1. Embed query → [0.041, -0.012, ...] (768-dim) │ +│ │ +│ 2. Parallel search: │ +│ ├─ Semantic: pgvector cosine similarity │ +│ │ SELECT * FROM memory_chunks │ +│ │ ORDER BY embedding <=> query_vec │ +│ │ → score: 0.89 │ +│ │ │ +│ └─ Lexical: OpenSearch BM25 │ +│ multi_match: "shared ownership threads" │ +│ → score: 0.76 │ +│ │ +│ 3. RRF Fusion: 0.6 × semantic + 0.4 × lexical │ +│ → fused_score: 0.84 │ +│ │ +│ 4. Return: │ +│ { │ +│ "score": 0.84, │ +│ "text": "Rust ownership: ... Arc atomic │ +│ reference counting, thread-safe shared ownership" │ +│ } │ +│ │ +└─────────────────────────────────────────────────────────────┘ +``` + +--- + +## Re-Ingestion (Idempotent) + +When the same file is ingested again: + +``` +rust-ownership.md (unchanged) → POST /memory/learn → chunk → gated loop: + + Turn 1: "# Rust Ownership" → no (title, no substance) + Turn 2: "## Borrowing" → no (already in memory) + Turn 3: "## Lifetimes" → no (already in memory) + Turn 4: "## Smart Pointers"→ no (already in memory) + + Result: chunks_seen=4, chunks_used=0 + Memory: unchanged, nothing new stored +``` + +When the file is updated with new content: + +``` +rust-ownership.md (added ## Async section) → POST /memory/learn → gated loop: + + Turn 1-4: existing sections → no (known) + Turn 5: "## Async/Await" → yes (new!) + Memory rewritten: ownership + lifetimes + pointers + async + + Result: chunks_seen=5, chunks_used=1 + pgvector: ON CONFLICT (sha256) DO UPDATE → old record replaced +``` + +--- + +## Complete Data Flow Diagram + +``` + ┌──────────┐ + │ Source │ + │ (.md) │ + └────┬─────┘ + │ + ┌────▼─────┐ + │ mem learn│ CLI + │ (or any │ agent) + └────┬─────┘ + │ POST /memory/learn + ┌────▼─────────────────────┐ + │ Memory Service (k8s) │ + │ │ + │ ┌─────────┐ │ + │ │ Auth │ JWT/rate │ + │ └────┬────┘ │ + │ ▼ │ + │ ┌─────────┐ │ + │ │ Chunker │ split on ## │ + │ └────┬────┘ │ + │ ▼ │ + │ ┌──────────────┐ │ + │ │ Gated Loop │ │ + │ │ │ │ + │ │ chunk ──→ LLM ──→ │ + │ │ evaluate + compact │ + │ │ (update gate) │ + │ │ │ │ + │ │ Memory rewritten │ + │ │ each accepted turn │ + │ └──────┬───────┘ │ + │ ▼ │ + │ ┌──────────────┐ │ + │ │ Embed │ │ + │ │ 768-dim vec │ │ + │ └──────┬───────┘ │ + │ ▼ │ + │ ┌──────────────┐ │ + │ │ pgvector │ │ + │ │ UPSERT │ │ + │ │ (sha256 │ │ + │ │ dedup) │ │ + │ └──────────────┘ │ + │ │ + └──────────────────────────┘ + │ + ┌─────────▼───────────┐ + │ Query Path │ + │ │ + │ /memory/query │ + │ /memory/context │ + │ │ + │ embed query → │ + │ pgvector search + │ + │ OpenSearch BM25 → │ + │ RRF fusion → │ + │ ranked results │ + └─────────────────────┘ +``` + +--- + +## Key Properties + +| Property | Mechanism | +|---|---| +| **No unbounded growth** | Gated loop rewrites memory, doesn't append. `memory_budget` caps size. | +| **Deduplication** | SHA256 on compacted memory. `ON CONFLICT DO UPDATE`. | +| **Idempotent re-ingest** | Same content → LLM rejects all chunks → nothing changes. | +| **Quality gate** | LLM decides what's worth keeping. Title-only chunks rejected. | +| **Compaction built-in** | Every accepted chunk triggers a rewrite that merges + trims. | +| **Searchable** | Embedded in pgvector + indexed in OpenSearch. Hybrid retrieval. | +| **Auditable** | JSONL event log records all gated loop decisions (immutable). | +| **Model-agnostic** | `model` parameter selects evaluator LLM per request. |