From e62860d232491ecef1c5fea6e6a43a98df15d03c Mon Sep 17 00:00:00 2001 From: rock Date: Tue, 15 Sep 2026 00:07:04 +0900 Subject: [PATCH] chore: remove progress markdown files (track via Forgejo issues only) --- FIXME_CRITICAL.md | 263 -------------------------------------- MONITORING_AGENT_TASKS.md | 217 ------------------------------- STATUS_CURRENT.md | 191 --------------------------- 3 files changed, 671 deletions(-) delete mode 100644 FIXME_CRITICAL.md delete mode 100644 MONITORING_AGENT_TASKS.md delete mode 100644 STATUS_CURRENT.md diff --git a/FIXME_CRITICAL.md b/FIXME_CRITICAL.md deleted file mode 100644 index ca07a79..0000000 --- a/FIXME_CRITICAL.md +++ /dev/null @@ -1,263 +0,0 @@ -# CRITICAL FIXES NEEDED - Poimen Memory Service - -## STATUS: Service Non-Functional ❌ - -**Root Issues Blocking Service**: -1. ✅ HTTP handler deadlock fixed (schema init error handling) -2. ❌ Server initialization hangs during schema or startup (logs stop after `l2_l1_edges`) -3. ❌ Ingest pipeline NOT implemented (just raw vector storage, no entities/edges) -4. ❌ Temporal schema missing (no t_valid, t_invalid, version tracking) -5. ❌ GRM gate not integrated (no memorability scores, confidence) -6. ❌ Query doesn't use knowledge graph (just vector search) -7. ❌ Compaction disabled -8. ❌ Verification gates missing - ---- - -## STEP 1: Fix Server Startup Hang ⚠️ - -**Current Issue**: Server hangs during initialization after schema creation. - -**Suspected causes**: -- OptimizerServiceBuilder.build() getting stuck -- AccessGuard creation blocking -- Background task spawning deadlock - -**Fix**: -```rust -// In http_server.rs:316-325 -// Wrap in timeout or disable non-essentials -let optimizer_service = match tokio::time::timeout( - Duration::from_secs(5), - async { mem_core::optimizer::OptimizerServiceBuilder::new().build() } -).await { - Ok(Ok(service)) => Some(Arc::new(service)), - _ => { - tracing::warn!("Optimizer initialization skipped (timeout or error)"); - None - } -}; -``` - -**Test**: `./target/release/mem serve --port 9999` should reach "Starting HTTP server" within 10s - ---- - -## STEP 2: Implement Ingest Pipeline (HIGH PRIORITY) - -**Current Implementation** (`ingest_worker.rs`): -```rust -// Just stores raw chunks + embeddings -store_chunk_l0(&l0_chunk) -store_memory_l1(&l1_memory, &embedding) -``` - -**Expected Implementation**: -```rust -// 1. Extract entities (entity_extractor) -let entities = entity_extractor.extract(&content).await?; - -// 2. Extract facts + edges (fact_extractor) -let facts = fact_extractor.extract(&content, entities).await?; - -// 3. Create temporal edges with GRM gate -for fact in facts { - let edge = TemporalEdge { - source: fact.source_entity, - target: fact.target_entity, - relation: fact.relation, - fact: fact.text, - t_valid: now(), - t_invalid: None, - confidence: grm_gate.score(&fact)?, // ← GRM gate - version: 1, - }; - edge_repo.insert(&edge).await?; -} - -// 4. Check contradictions + queue for review -for edge in edges { - if contradiction_detector.detect(&edge, existing_edges)? { - review_queue.enqueue(&edge).await?; - } -} -``` - -**Files to modify**: -- `crates/mem-cli/src/ingest_worker.rs` (core ingest logic) -- `crates/mem-ingest/src/ingest_pipeline.rs` (entity + fact extraction) -- `crates/mem-ingest/src/contradiction_detector.rs` (pre-filter + review) - ---- - -## STEP 3: Update Storage Schema (MEDIUM PRIORITY) - -**Missing fields**: -```sql -ALTER TABLE memories_l1 ADD COLUMN ( - t_valid TIMESTAMP NOT NULL DEFAULT NOW(), - t_invalid TIMESTAMP, - confidence FLOAT DEFAULT 0.5, - version INT DEFAULT 1, - memorability_score INT, - contribution_date TIMESTAMP -); - -ALTER TABLE l1_l0_edges MODIFY TO ( - l1_id UUID, - l0_id UUID, - relation_type VARCHAR, - fact TEXT, - t_valid TIMESTAMP DEFAULT NOW(), - t_invalid TIMESTAMP, - confidence FLOAT, - contradiction_flag BOOL DEFAULT FALSE, - review_queue_id UUID, - version INT DEFAULT 1, - PRIMARY KEY (l1_id, l0_id, version) -); -``` - -**Migration script**: `crates/mem-store/migrations/003_temporal_grm_schema.sql` - ---- - -## STEP 4: Wire Query Handler to Knowledge Graph (MEDIUM PRIORITY) - -**Current** (`query_handler` in http_server.rs): -```rust -async fn query_handler(...) -> HttpResponse { - // Just semantic search - let results = vector_search(query)?; - HttpResponse::Ok().json(results) -} -``` - -**Expected**: -```rust -async fn query_handler(query: QueryRequest) -> HttpResponse { - // 1. Semantic search on embeddings - let initial_results = vector_search(&query.text)?; - - // 2. Follow edges (graph traversal) - let mut expanded = vec![]; - for result in initial_results { - expanded.push(result); - // Get related entities via edges - let related = edge_repo.find_by_source(&result.entity_id).await?; - expanded.extend(related); - } - - // 3. Apply temporal filters - expanded.retain(|e| e.t_valid <= now() && (e.t_invalid.is_none() || e.t_invalid > now())); - - // 4. Sort by confidence + recency - expanded.sort_by(|a, b| { - b.confidence.partial_cmp(&a.confidence) - .then_with(|| b.t_valid.cmp(&a.t_valid)) - }); - - // 5. Apply compaction/cache alignment - for item in &mut expanded { - item.text = optimizer.compress(item.text)?; - } - - HttpResponse::Ok().json(MemoryResponse { - entities: expanded, - confidence_scores: compute_scores(&expanded), - }) -} -``` - ---- - -## STEP 5: Enable Compaction Endpoint (LOW PRIORITY) - -**Current**: Code exists but never called. - -**Fix**: Add K8s CronJob that calls `POST /memory/compact` daily: -```yaml -apiVersion: batch/v1 -kind: CronJob -metadata: - name: memory-compaction -spec: - schedule: "0 2 * * *" # 2 AM UTC - jobTemplate: - spec: - template: - spec: - containers: - - name: compact - image: bitnami/curl:latest - command: - - curl - - -X POST - - -H "Authorization: Bearer $ADMIN_TOKEN" - - http://poimen-memory:8080/memory/compact - restartPolicy: OnFailure -``` - ---- - -## STEP 6: Add Verification Gates (LOW PRIORITY) - -**Missing**: `GET /memory/verify` endpoint that checks M1.8, M2.8, M3.7, M8.9 gates - ---- - -## IMPLEMENTATION ORDER - -1. **FIX STARTUP** (1 hour) → Get server running -2. **INGEST PIPELINE** (3 hours) → Wire entity + fact extraction -3. **TEMPORAL SCHEMA** (1 hour) → Add missing columns -4. **QUERY HANDLER** (2 hours) → Implement graph traversal -5. **COMPACTION** (1 hour) → Add CronJob -6. **GATES** (2 hours) → Quality verification - -**Total**: ~10 hours to full working system - ---- - -## TEST PLAN - -```bash -# 1. Server starts -curl http://localhost:9999/health -# Expected: {"status":"ok","uptime_seconds":N} - -# 2. Ingest works -curl -X POST http://localhost:9999/memory/ingest \ - -H "Content-Type: application/json" \ - -d '{"project":"test","source":"test://1","ingest_id":"i1","records":[{"role":"user","text":"Hello world","timestamp":"2026-01-08T16:00:00Z","source_position":0}]}' -# Expected: {"ingest_id":"i1","status":"pending",...} - -# 3. Query returns entities with edges -curl -X POST http://localhost:9999/memory/query \ - -H "Content-Type: application/json" \ - -d '{"project":"test","query":"hello"}' -# Expected: {"results":[{"type":"entity","name":"...","edges":[...]}]} - -# 4. Temporal filtering works -curl http://localhost:9999/memory/query?project=test&temporal_floor=2026-01-01 - -# 5. Compaction works -curl -X POST http://localhost:9999/memory/compact -# Expected: {"phase":"completed","records_deduplicated":N} -``` - ---- - -## FILES MODIFIED SO FAR - -✅ `crates/mem-cli/src/http_server.rs` - Added error handling for schema init - ---- - -## NEXT SESSION TODO - -- [ ] Fix server startup hang (debug OptimizerService) -- [ ] Implement ingest_worker to call entity_extractor + fact_extractor -- [ ] Add temporal columns to schema -- [ ] Update query_handler to traverse edges -- [ ] Test end-to-end with sample data diff --git a/MONITORING_AGENT_TASKS.md b/MONITORING_AGENT_TASKS.md deleted file mode 100644 index bb234e8..0000000 --- a/MONITORING_AGENT_TASKS.md +++ /dev/null @@ -1,217 +0,0 @@ -# Monitoring Agent: Implementation Tasks - -**Milestone**: `monitoring-agent` -**Status**: 🔧 Not started -**Duration**: 4-6 weeks -**Effort**: ~1,500 LOC - ---- - -## Phase 1: Temporal Setup (3-5 days) - -### Task 1.1: Deploy Temporal Server in K8s -- [ ] StatefulSet configuration (persistence) -- [ ] PostgreSQL event log backend -- [ ] ElasticSearch for visibility -- [ ] K8s manifests in `k8s/temporal/` -- [ ] Health checks + readiness probes -- **Effort**: 150 LOC | **Time**: 2 days -- **Dependencies**: None -- **Blocks**: Phase 2 - -### Task 1.2: Add Temporal SDK to Rust Project -- [ ] Add `temporal-rust-sdk` to `Cargo.toml` -- [ ] Create `crates/mem-temporal/` workspace crate -- [ ] Worker registration + gRPC connection -- [ ] Activity executor setup -- [ ] Workflow executor setup -- **Effort**: 200 LOC | **Time**: 1 day -- **Dependencies**: 1.1 -- **Blocks**: Phase 2 - -### Task 1.3: Temporal Configuration + Secrets -- [ ] Environment variables (TEMPORAL_HOST, TEMPORAL_NAMESPACE) -- [ ] Worker identity configuration -- [ ] Task queue setup (synthesis-queue, compaction-queue) -- **Effort**: 50 LOC | **Time**: 4 hours -- **Dependencies**: 1.1, 1.2 -- **Blocks**: Phase 2 - ---- - -## Phase 2: Agent Workflows (1-2 weeks) - -### Task 2.1: Synthesis Workflow Definition -- [ ] `crates/mem-temporal/src/workflows/synthesis_workflow.rs` -- [ ] Workflow orchestration logic -- [ ] Activity composition (health check → synthesis → logging → metrics) -- [ ] Retry policies (exponential backoff, max 5 retries) -- [ ] Heartbeat configuration (every 10s) -- **Effort**: 200 LOC | **Time**: 3 days -- **Dependencies**: 1.2, 1.3 -- **Blocks**: 2.3, 2.4 - -### Task 2.2: Synthesis Activities (5 activities) -- [ ] `MonitorMemoryHealth` activity - - GET /health check - - Latency measurement - - Failure detection - -- [ ] `ExecuteSynthesis` activity - - POST /memory/synthesize call - - LLM integration - - Heartbeat emission - -- [ ] `LogSynthesisResult` activity - - POST /memory/ingest (audit) - - Temporal audit trail - -- [ ] `UpdateCacheMetrics` activity - - Metric recording - - Performance tracking - -- [ ] `CoordinateCompaction` activity - - Signal to compaction agent - - Readiness check - -- **Effort**: 250 LOC | **Time**: 4 days -- **Dependencies**: 2.1 -- **Blocks**: 2.3 - -### Task 2.3: Compaction Workflow Definition -- [ ] `crates/mem-temporal/src/workflows/compaction_workflow.rs` -- [ ] 4-stage orchestration (identify → dedup → gc → invalidate) -- [ ] Failure handling + rollback strategy -- **Effort**: 150 LOC | **Time**: 2 days -- **Dependencies**: 1.2, 1.3 -- **Blocks**: 2.4 - -### Task 2.4: Compaction Activities (4 activities) -- [ ] `IdentifyDuplicates` activity -- [ ] `DeduplicateEdges` activity -- [ ] `GarbageCollection` activity -- [ ] `InvalidateCache` activity -- **Effort**: 200 LOC | **Time**: 3 days -- **Dependencies**: 2.3 -- **Blocks**: Integration tests - -### Task 2.5: Worker + Task Queue Registration -- [ ] Activity worker setup -- [ ] Workflow worker setup -- [ ] Task queue polling -- [ ] Namespace configuration -- **Effort**: 100 LOC | **Time**: 1 day -- **Dependencies**: 2.1-2.4 -- **Blocks**: Phase 3 - ---- - -## Phase 3: Agent Self-Awareness (2-3 weeks) - -### Task 3.1: AGENT_PROMPT Entity Type -- [ ] Schema: New entity type in memory_entity -- [ ] Repository: `synthesis_cache_repo.rs` (get_agent_prompt) -- [ ] Migration: Add to entity type enum -- [ ] Activity: Load prompt on agent startup -- **Effort**: 100 LOC | **Time**: 1 day -- **Dependencies**: Memory service -- **Blocks**: 3.2 - -### Task 3.2: AGENT_SKILL Linking -- [ ] Edge type: agent → skill relationships -- [ ] Repository methods: link_agent_to_skill, get_agent_skills -- [ ] Confidence tracking per skill -- [ ] Success rate calculation -- **Effort**: 80 LOC | **Time**: 1 day -- **Dependencies**: 3.1 -- **Blocks**: 3.4 - -### Task 3.3: AGENT_PERFORMANCE Metrics -- [ ] Entity type: Temporal metrics -- [ ] Repository: Store + query metrics -- [ ] Activity: Log performance data post-execution -- [ ] Time window filtering (last_7_days, last_30_days) -- **Effort**: 120 LOC | **Time**: 2 days -- **Dependencies**: 3.1 -- **Blocks**: 3.4 - -### Task 3.4: Agent Decision Tracking + Learning -- [ ] Edge type: agent_decision_outcome -- [ ] Decision logging (parameter, value, confidence before) -- [ ] Outcome recording (result, metric) -- [ ] Confidence evolution (update after outcome) -- [ ] Learning loop in agent code -- **Effort**: 200 LOC | **Time**: 3 days -- **Dependencies**: 3.1-3.3 -- **Blocks**: 3.5 - -### Task 3.5: Agent Audit Trail Integration -- [ ] Dual audit: Temporal history + Memory entities -- [ ] Query interface for reviewers -- [ ] Temporal CLI integration -- [ ] Retention policy (365 days) -- **Effort**: 100 LOC | **Time**: 1 day -- **Dependencies**: 3.1-3.4 -- **Blocks**: Testing - ---- - -## Testing & Documentation - -### Task 4.1: Integration Tests -- [ ] Workflow execution end-to-end -- [ ] Activity retry behavior -- [ ] Heartbeat detection -- [ ] Failure recovery -- [ ] State replay on restart -- **Effort**: 300 LOC | **Time**: 3 days -- **Dependencies**: Phase 2 complete -- **Blocks**: Integration - -### Task 4.2: Monitoring & Observability -- [ ] Temporal UI setup (temporal.riotpiao.com) -- [ ] Prometheus metrics export -- [ ] Alerting rules (workflow timeout, activity failure) -- [ ] Grafana dashboards -- **Effort**: 150 LOC | **Time**: 2 days -- **Dependencies**: Phase 1 complete -- **Blocks**: Production - -### Task 4.3: Documentation -- [ ] Agent architecture diagram -- [ ] Workflow execution flow -- [ ] Operational runbook -- [ ] Troubleshooting guide -- **Effort**: 50 LOC | **Time**: 1 day -- **Dependencies**: All phases -- **Blocks**: Release - ---- - -## Credentials Status - -✅ **SOPS Encrypted**: `k8s/app/memory-agent-secrets.enc.yaml` -- CLIENT_ID: `memory-agent` -- CLIENT_SECRET: Encrypted -- TOKEN_URL: `https://authentik.riotpiao.com/application/o/token/` -- AUTHENTIK_ISSUER: `https://authentik.riotpiao.com/application/o/memory-agent/` - -✅ **JWT Auth Verified**: `memory-agent` credentials working -- Test result: Token obtained successfully -- Expiry: 1 hour (3600s) -- Scopes: Default (sufficient for LLM operations) - ---- - -## Timeline - -``` -Week 1 (Phase 1): Temporal setup -Week 2-3 (Phase 2): Agent workflows -Week 4-5 (Phase 3): Self-awareness -Week 6 (Testing + Docs): Integration + release -``` - -**Start Date**: TBD -**Target End Date**: TBD (+4-6 weeks) - diff --git a/STATUS_CURRENT.md b/STATUS_CURRENT.md deleted file mode 100644 index 8c3f471..0000000 --- a/STATUS_CURRENT.md +++ /dev/null @@ -1,191 +0,0 @@ -# Current Status - Poimen Memory Service (2026-01-08) - -## ✅ COMPLETED THIS SESSION - -### 1. Removed AccessGuard RBAC (Blocker Issue #1) -- ❌ ~~AccessGuard initialization~~ REMOVED -- ❌ ~~RBAC checks in handlers~~ REMOVED -- ❌ ~~Permission-based access control~~ DEFERRED -- ✅ Code now compiles with `cargo build --release` -- ✅ Binary created: `target/release/mem` - -### 2. HTTP Handler Initialization Fixed -- ✅ Added error handling for schema initialization -- ✅ Server reaches "Starting HTTP server" log message -- ✅ HTTP server binds to port (processes created) - -## ⚠️ CURRENT ISSUE - -**Server binds to port but exits immediately (silent failure)** - -Process is created and runs `serve` command, but: -- Process exits with code 0 (clean exit, no crash) -- No HTTP requests answered (port refuses connections) -- Logs don't show "listening on 0.0.0.0:8080" message - -**Suspected cause**: Something in the handler initialization or routing setup is blocking/panicking but not showing in logs. - -## 🔧 DEBUGGING STEPS NEEDED - -1. Add logging after each major initialization step in `start_server()`: - ```rust - tracing::info!("About to create AppState"); - let state = web::Data::new(AppState { ... }); - tracing::info!("AppState created"); - - tracing::info!("About to create HttpServer"); - HttpServer::new(move || { ... }) - tracing::info!("HttpServer created, about to bind"); - - .bind(("0.0.0.0", port))? - tracing::info!("Bound to port {}", port); - - .run() - tracing::info!("About to run()"); - .await?; - tracing::info!("Server running"); - ``` - -2. Run with `RUST_BACKTRACE=1` to see panics -3. Check if the issue is in handler route registration - -## 📋 NEXT PRIORITY FIXES (AFTER SERVER RUNS) - -### Phase 1: INGEST PIPELINE ⭐ CRITICAL -**File**: `crates/mem-cli/src/ingest_worker.rs` - -Currently: Just stores raw vectors -```rust -// WRONG - just vector storage -store_chunk_l0(&l0_chunk); -store_memory_l1(&l1_memory); -``` - -Should: Extract entities + facts + edges -```rust -// 1. Extract entities -let entities = entity_extractor.extract(&content).await?; - -// 2. Extract facts/relationships -let facts = fact_extractor.extract(&content, &entities).await?; - -// 3. Create temporal edges -for fact in facts { - let edge = TemporalEdge { - source: fact.source_entity, - target: fact.target_entity, - relation: fact.relation, - fact: fact.text, - t_valid: now(), - t_invalid: None, - confidence: 0.8, // GRM gate score - version: 1, - }; - edge_repo.insert(&edge).await?; -} - -// 4. Queue contradictions for review -for edge in &edges { - if contradiction_detector.detect(edge, existing_edges)? { - review_queue.enqueue(edge).await?; - } -} -``` - -### Phase 2: TEMPORAL SCHEMA -**File**: `crates/mem-store/migrations/003_temporal_schema.sql` - -Add columns: -- `t_valid TIMESTAMP NOT NULL DEFAULT NOW()` -- `t_invalid TIMESTAMP` -- `confidence FLOAT DEFAULT 0.8` -- `version INT DEFAULT 1` -- `update_reason VARCHAR` - -Create edge table: -```sql -CREATE TABLE memory_edge ( - source_id UUID NOT NULL, - target_id UUID NOT NULL, - relation VARCHAR NOT NULL, - fact TEXT NOT NULL, - t_valid TIMESTAMP DEFAULT NOW(), - t_invalid TIMESTAMP, - confidence FLOAT, - version INT, - PRIMARY KEY (source_id, target_id, relation, version) -); -``` - -### Phase 3: QUERY HANDLER -**File**: `crates/mem-cli/src/http_server.rs` - -Change `query_handler()` from vector-only to graph-aware: -```rust -// 1. Vector search -let results = semantic_search(query)?; - -// 2. Follow edges -let mut expanded = results; -for entity in results { - let related = edge_repo.find_by_source(&entity.id).await?; - expanded.extend(related); -} - -// 3. Apply temporal filter -expanded.retain(|e| is_valid_at_time(e, now())); - -// 4. Sort by confidence + recency -expanded.sort_by_key(|e| (-e.confidence, -e.t_valid)); - -// 5. Return -HttpResponse::Ok().json(expanded) -``` - -### Phase 4: END-TO-END TESTING -```bash -# 1. Ingest with entities + facts -POST /memory/ingest -{ - "project": "test", - "source": "transcript://session-1", - "ingest_id": "i-001", - "records": [{"role": "user", "text": "Kubernetes port conflict...", ...}] -} -# Expected: {"ingest_id":"i-001","status":"pending"} - -# 2. Check ingest status -GET /memory/ingest/i-001 -# Expected: {"status":"done","entities_count":5,"edges_count":3} - -# 3. Query returns graph -POST /memory/query -{"project":"test","query":"port conflict resolution"} -# Expected: {"results":[ -# {"type":"entity","name":"Kubernetes","edges":[...]}, -# {"type":"entity","name":"Port","edges":[...]}, -# {"type":"fact","source":"Kubernetes","target":"Port","relation":"has-conflict"} -# ]} -``` - -## FILES MODIFIED - -✅ `crates/mem-cli/src/http_server.rs` - Removed RBAC, added error handling -✅ Created `STATUS_CURRENT.md` - This file - -## TIMELINE - -- **2026-01-08 16:00**: Fixed HTTP handlers, removed RBAC blocker -- **2026-01-08 16:30**: Server init working, but exits on startup -- **2026-01-08 16:40**: Debugging server binding issue - -## KEY DECISIONS - -1. **RBAC deferred**: MVP focuses on core ingest/query, auth added later -2. **Temporal-first**: All edges must have t_valid/t_invalid for graph compaction -3. **GRM gate integrated at ingest time**: Confidence scores assigned when facts extracted -4. **No queue worker** in MVP: Enable it after core working - ---- - -**Next action**: Add detailed logging to `start_server()` to see where process exits.