From 25e3a1cc4cfeb7976aa9937832c25b8e9344924a Mon Sep 17 00:00:00 2001 From: Story Crater Bot <19826264+Riotpiaole@users.noreply.github.com> Date: Fri, 28 Aug 2026 09:02:30 -0700 Subject: [PATCH] docs: context optimizer design (Headroom-inspired pre-LLM compression) --- docs/CONTEXT_OPTIMIZER.md | 308 ++++++++++++++++++++++++++++ k8s/infra/databases/opensearch.yaml | 24 ++- 2 files changed, 321 insertions(+), 11 deletions(-) create mode 100644 docs/CONTEXT_OPTIMIZER.md diff --git a/docs/CONTEXT_OPTIMIZER.md b/docs/CONTEXT_OPTIMIZER.md new file mode 100644 index 0000000..32ea9d3 --- /dev/null +++ b/docs/CONTEXT_OPTIMIZER.md @@ -0,0 +1,308 @@ +# Context Optimizer — Pre-LLM Compression Layer + +## Motivation + +Agent transcripts and tool outputs are noisy. A 50-chunk ingestion run might +feed the GRU-Mem gate evidence that's 43% tool results, full of timestamps, +temp paths, ANSI codes, and verbose JSON. The model wastes tokens parsing noise, +risks hallucinating on irrelevant details, and we pay full price for bloated +input. LLM provider KV caches miss because dynamic content (timestamps, session +IDs) pollutes the prefix. + +**This layer sits between retrieval and the LLM call.** Search indexes +(pgvector + OpenSearch) stay untouched at full fidelity. Only the evidence +chunks entering the prompt get optimized. + +## Architecture + +``` +Query → Hybrid Search (pgvector 60% + OpenSearch 40%) + │ + │ full-fidelity chunks (untouched) + ▼ + ┌───────────────────────┐ + │ CONTEXT OPTIMIZER │ ← THIS MODULE + │ │ + │ 1. CacheAligner │ Move dynamic content (timestamps, UUIDs) + │ │ to end of context. Keep static prefix + │ │ stable for KV cache hits. + │ │ + │ 2. ContentRouter │ Auto-detect content type per chunk: + │ │ JSON, code, logs, diffs, plain text. + │ │ Route each to best compressor. + │ │ + │ 3. Compressors │ Per-type compression: + │ - JsonCrusher │ Statistical field analysis, keep keys + │ - LogCompressor │ Keep errors/stack traces, drop noise + │ - CodeCompressor │ AST-aware: keep signatures, drop bodies + │ - TextCompressor │ Token importance scoring + │ - DiffCompressor │ Keep change hunks, drop context + │ │ + │ 4. CCR Store │ Cache full originals with hash reference. + │ │ Inject retrieval hint so model can fetch + │ │ full content if needed. + │ │ + └───────────┬───────────┘ + │ + │ optimized chunks (smaller, cleaner) + ▼ + ┌───────────────────────┐ + │ Cache-Aligned Prompt │ (already built) + │ system | query | turn│ + └───────────┬───────────┘ + │ + ▼ + LLM Gateway +``` + +## What Each Stage Does + +### Stage 1: CacheAligner + +**Goal:** Maximize LLM provider KV cache hits by stabilizing the prompt prefix. + +LLM providers (Anthropic, OpenAI) cache based on exact prefix match. A single +changing timestamp or session ID early in the prompt invalidates the entire +cache. CacheAligner: + +1. Scans for dynamic patterns in the prompt prefix: + - ISO timestamps (`2026-08-28T...`) + - UUIDs (`550e8400-e29b-...`) + - Session tokens, run IDs + - Temp paths (`/tmp/abc123`) + +2. Moves detected dynamic content to the end of the context (after static + instructions and query), preserving the stable prefix for cache hits. + +3. Reports drift metrics: how much of the prefix changed vs. previous call. + +**Implementation:** Regex-based pattern detection + reordering. No ML needed. +Reuses existing normalisation patterns from `lesson.rs` (M3.7.7). + +```rust +pub struct CacheAligner; + +impl CacheAligner { + /// Stabilize prompt prefix by moving dynamic content to tail. + /// Returns (stable_prefix, dynamic_tail). + pub fn align(content: &str) -> AlignedContent { + // Detect and extract dynamic patterns + // Reorder so static content comes first + } +} +``` + +### Stage 2: ContentRouter + +**Goal:** Auto-detect content type and route to the best compressor. + +Each evidence chunk might be JSON, source code, build logs, a diff, or plain +text. The router classifies using structural heuristics: + +| Content Type | Detection Signal | Compressor | +|---|---|---| +| JSON | Valid JSON, `{` or `[` start, key-value pairs | JsonCrusher | +| Source code | Import/use statements, function defs, indentation | CodeCompressor | +| Build/test logs | Timestamps, log levels, `error:`, `FAIL` | LogCompressor | +| Unified diffs | `---`, `+++`, `@@` markers | DiffCompressor | +| Plain text | Default fallback | TextCompressor | + +**Implementation:** Pattern matching + simple heuristics. No ML classifier +needed initially (can add Magika later). + +```rust +pub enum ContentType { + Json, + Code, + Log, + Diff, + Text, +} + +pub fn detect(content: &str) -> ContentType { + if is_json(content) { return ContentType::Json; } + if is_code(content) { return ContentType::Code; } + if is_log(content) { return ContentType::Log; } + if is_diff(content) { return ContentType::Diff; } + ContentType::Text +} +``` + +### Stage 3: Compressors + +**Goal:** Reduce token count per content type while preserving signal. + +#### JsonCrusher (70-90% savings) +- Analyse field-level variance across JSON array elements +- Keep: keys, structure, error markers, boundary items (first/last) +- Drop: redundant mid-array elements, long string values, whitespace +- Allocation: 30% start (schema), 15% end (recency), 55% importance + +#### LogCompressor (85-95% savings) +- Keep: error lines, stack traces, exit codes, FAIL markers +- Drop: passing test output, INFO-level noise, repeated patterns +- Reuses M3.7.7 signature extraction patterns (markers, cascade detection) + +#### CodeCompressor (40-70% savings, opt-in) +- Keep: imports, function/method signatures, type annotations +- Drop: function bodies, inline comments, blank lines +- Uses simple AST heuristics (brace counting), not full parser + +#### DiffCompressor (60-80% savings) +- Keep: change hunks (`+`/`-` lines), hunk headers +- Drop: unchanged context lines (the `@@` surrounding context) + +#### TextCompressor (30-50% savings) +- Token importance scoring: keep high-entropy tokens (IDs, hashes, error codes) +- Drop: low-information prose, repeated phrases, filler words + +```rust +pub trait Compressor { + fn compress(&self, content: &str, budget: usize) -> CompressResult; +} + +pub struct CompressResult { + pub compressed: String, + pub original_tokens: usize, + pub compressed_tokens: usize, + pub content_type: ContentType, + /// Hash of original content for CCR retrieval + pub ccr_hash: Option, +} +``` + +### Stage 4: CCR Store (Compress-Cache-Retrieve) + +**Goal:** Lossless compression — model can retrieve full originals if needed. + +When a chunk is compressed, the full original is cached with a SHA256 hash. +A retrieval hint is injected into the compressed output: + +``` +[compressed content...] + +``` + +If the model determines it needs more detail, it can request the full content. +This makes compression aggressive but reversible. + +```rust +pub struct CcrStore { + cache: HashMap, // hash → original content + max_entries: usize, + ttl: Duration, +} + +impl CcrStore { + pub fn store(&mut self, content: &str) -> String; // returns hash + pub fn retrieve(&self, hash: &str) -> Option<&str>; +} +``` + +## Integration with Existing Code + +### Where It Plugs In + +The context optimizer sits in `mem-core` as a new module, called by +`PromptBuilder::build_cache_aligned()` before assembling the final prompt: + +```rust +// In PromptBuilder::build_cache_aligned() +let chunk_text = Self::render_chunk(chunk)?; + +// NEW: Optimize before prompt assembly +let optimized = ContextOptimizer::optimize(&chunk_text, &OptimizeConfig { + cache_align: true, + compress: true, + ccr_enabled: true, + token_budget: BUDGET_CHUNK_MAX, +})?; + +let turn_msg = CACHE_TURN + .replace("{memory}", memory_text) + .replace("{chunk}", &optimized.compressed); +``` + +### What Already Exists (Reuse) + +| Existing Code | Reuse For | +|---|---| +| `lesson.rs` normalise() | CacheAligner pattern detection (timestamps, paths, SHAs) | +| `lesson.rs` markers() | LogCompressor error line detection | +| `lesson.rs` is_cascade() | LogCompressor cascade suppression | +| `lesson.rs` strip_ansi() | Pre-processing for all compressors | +| `symptom_projection.rs` stop words | TextCompressor low-value token detection | + +### What's New + +| New Code | Location | Est. LOC | +|---|---|---| +| `context_optimizer.rs` | `crates/mem-core/src/` | 150 | +| `content_router.rs` | `crates/mem-core/src/` | 100 | +| `compressors/json.rs` | `crates/mem-core/src/` | 200 | +| `compressors/log.rs` | `crates/mem-core/src/` | 150 | +| `compressors/code.rs` | `crates/mem-core/src/` | 150 | +| `compressors/diff.rs` | `crates/mem-core/src/` | 100 | +| `compressors/text.rs` | `crates/mem-core/src/` | 100 | +| `ccr_store.rs` | `crates/mem-core/src/` | 80 | +| **Total** | | **~1030** | + +## Performance Targets + +| Metric | Target | +|---|---| +| Detection + compression | < 10ms per chunk | +| JSON compression ratio | 70-90% | +| Log compression ratio | 85-95% | +| Code compression ratio | 40-70% | +| Cache hit rate improvement | > 50% on repeated queries | +| Zero false negatives | Model should never miss real errors | + +## Phasing + +### Phase 1: Foundation (1-2 days) +- `ContextOptimizer` orchestrator +- `ContentRouter` with detection heuristics +- `LogCompressor` (reuses M3.7.7 lesson.rs patterns) +- 10 unit tests + +### Phase 2: JSON + Diff (1-2 days) +- `JsonCrusher` with field variance analysis +- `DiffCompressor` with hunk preservation +- 10 unit tests + +### Phase 3: CCR + CacheAligner (1 day) +- `CcrStore` with LRU cache +- `CacheAligner` with dynamic pattern extraction +- Integration with `PromptBuilder::build_cache_aligned()` +- 10 unit tests + +### Phase 4: Code + Text (1 day) +- `CodeCompressor` (opt-in, brace-counting heuristics) +- `TextCompressor` (token importance scoring) +- 10 unit tests + +## Key Design Decisions + +1. **No ML in hot path.** All detection and compression uses rules, regex, and + statistics. ML classifiers (Magika) can be added later as an optional + enhancement. + +2. **Search indexes untouched.** Compression happens AFTER retrieval. pgvector + and OpenSearch see full-fidelity text. Only the LLM prompt is optimized. + +3. **Reversible via CCR.** Every compression is reversible. The model can + request full originals via hash lookup. Aggressive compression is safe + because nothing is permanently lost. + +4. **Reuse existing code.** M3.7.7's normalisation patterns, marker detection, + and cascade suppression are directly reusable for log compression. M3.7.8's + stop words help text compression. + +5. **Budget-aware.** Each compressor respects a token budget. If the chunk is + already under budget, no compression is applied (zero overhead). + +--- + +Inspired by [Headroom](https://docs.headroomlabs.ai/docs/how-compression-works). +Adapted for Rust, integrated with poimen-memory's hybrid search architecture. diff --git a/k8s/infra/databases/opensearch.yaml b/k8s/infra/databases/opensearch.yaml index c377980..26dd819 100644 --- a/k8s/infra/databases/opensearch.yaml +++ b/k8s/infra/databases/opensearch.yaml @@ -65,8 +65,8 @@ data: # Cluster settings cluster.name: poimen-memory node.name: ${HOSTNAME} - cluster.initial_master_nodes: opensearch-0,opensearch-1 - discovery.seed_hosts: opensearch-0.opensearch.poimen.svc.cluster.local,opensearch-1.opensearch.poimen.svc.cluster.local + cluster.initial_master_nodes: opensearch-0 + discovery.seed_hosts: opensearch-0.opensearch.poimen.svc.cluster.local # Network network.host: 0.0.0.0 @@ -102,7 +102,7 @@ metadata: app.kubernetes.io/name: opensearch spec: serviceName: opensearch - replicas: 2 + replicas: 1 selector: matchLabels: app.kubernetes.io/name: opensearch @@ -133,7 +133,7 @@ spec: - name: CLUSTER_NAME value: "poimen-memory" - name: OPENSEARCH_JAVA_OPTS - value: "-Xms512m -Xmx512m" + value: "-Xms1g -Xmx1g" - name: DISABLE_SECURITY_PLUGIN value: "true" @@ -150,11 +150,11 @@ spec: # Resource limits resources: requests: - memory: "512Mi" - cpu: "250m" - limits: memory: "1Gi" cpu: "500m" + limits: + memory: "2Gi" + cpu: "1000m" # Liveness probe livenessProbe: @@ -178,8 +178,7 @@ spec: # Security context securityContext: - runAsUser: 0 - runAsNonRoot: false + runAsUser: 1000 # Volumes volumes: @@ -280,8 +279,11 @@ data: opensearch_dashboards.index: ".opensearch_dashboards" # Logging - logging.dest: stdout - logging.level: info + logging.appenders.default.type: console + logging.appenders.default.layout.type: pattern + logging.appenders.default.layout.pattern: "[%date][%level][%logger] %message" + logging.root.appenders: [default] + logging.root.level: info ---