docs: add concrete implementation details to RAG/RBAC design
Each phase now includes: - Exact code locations (which crates/files) - Function signatures and method stubs - Unit tests with expected behavior - Integration tests for end-to-end verification - Homelab vault structure (test data) - Performance benchmarks and targets - Verification checklists Phases 1-7 now actionable: 1. Wiki-link graph indexing (parser + repo + SQL schema) 2. Multi-scope TF-IDF (global + project-local + chunk metadata) 3. Hybrid retrieval (wiki-scoped router + RRF fusion) 4. LLM call optimization (chunk selector with budget) 5. Chunk metadata extraction (heading + key terms + category) 6. Cache alignment (locality-aware wiki traversal) 7. OIDC + RBAC (JWT parsing + policy engine + audit logging) End-to-end test scenario provided.
This commit is contained in:
@@ -649,6 +649,607 @@ impl CacheAlignedTraversal {
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
## Implementation Details & Testing Strategy
|
||||||
|
|
||||||
|
### Phase 1: Wiki-Link Graph Indexing
|
||||||
|
|
||||||
|
**Code Location:**
|
||||||
|
```
|
||||||
|
crates/mem-ingest/src/
|
||||||
|
├── wiki_link_parser.rs (NEW: extract [[links]] from markdown)
|
||||||
|
├── wiki_link_index.rs (NEW: build graph, traverse)
|
||||||
|
└── wiki_link_repo.rs (NEW: Postgres storage)
|
||||||
|
|
||||||
|
crates/mem-store/src/
|
||||||
|
└── event_log.rs (EXTEND: add wiki_link_indexed event)
|
||||||
|
|
||||||
|
k8s/
|
||||||
|
└── migrations/
|
||||||
|
└── wiki_links.sql (NEW: schema for wiki_links table)
|
||||||
|
```
|
||||||
|
|
||||||
|
**Implementation Tasks:**
|
||||||
|
|
||||||
|
1. **Wiki-Link Parser** (`wiki_link_parser.rs`)
|
||||||
|
```rust
|
||||||
|
pub struct WikiLinkParser;
|
||||||
|
impl WikiLinkParser {
|
||||||
|
// Extract [[target]] from markdown
|
||||||
|
pub fn parse_links(content: &str) -> Vec<WikiLink>
|
||||||
|
|
||||||
|
// Infer link type: memory | skill | shared
|
||||||
|
fn infer_link_type(target: &str) -> LinkType
|
||||||
|
|
||||||
|
// Resolve relative paths: [[debugging.md]] → poimen/tools/debugging.md
|
||||||
|
fn resolve_path(target: &str, source_dir: &Path) -> PathBuf
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
2. **Wiki-Link Repository** (`wiki_link_repo.rs`)
|
||||||
|
```rust
|
||||||
|
pub struct WikiLinkRepo;
|
||||||
|
impl WikiLinkRepo {
|
||||||
|
// Bulk insert/upsert wiki links
|
||||||
|
pub async fn upsert_links(project: &str, links: Vec<WikiLink>) -> Result<u32>
|
||||||
|
|
||||||
|
// Query: all reachable docs from a project
|
||||||
|
pub async fn reachable_docs(project: &str) -> Result<Vec<String>>
|
||||||
|
|
||||||
|
// Query: backlinks to a doc (who links to this?)
|
||||||
|
pub async fn backlinks(project: &str, path: &str) -> Result<Vec<String>>
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
3. **Schema** (`wiki_links.sql`)
|
||||||
|
```sql
|
||||||
|
CREATE TABLE wiki_links (
|
||||||
|
id BIGSERIAL PRIMARY KEY,
|
||||||
|
project VARCHAR(256),
|
||||||
|
source_path VARCHAR(1024),
|
||||||
|
target_path VARCHAR(1024),
|
||||||
|
link_type VARCHAR(32), -- 'memory' | 'skill' | 'shared'
|
||||||
|
resolved_path VARCHAR(1024), -- fully qualified path
|
||||||
|
created_at TIMESTAMP,
|
||||||
|
UNIQUE(project, source_path, target_path),
|
||||||
|
INDEX(project, source_path),
|
||||||
|
INDEX(project, target_path)
|
||||||
|
);
|
||||||
|
```
|
||||||
|
|
||||||
|
**Testing:**
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// tests/it_wiki_link_parser.rs
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_parse_wiki_links_basic() {
|
||||||
|
let content = r#"
|
||||||
|
## Borrowing
|
||||||
|
See [[lifetimes.md]] for more.
|
||||||
|
Also check [[../../shared/skills/SKILL-ownership]]
|
||||||
|
"#;
|
||||||
|
|
||||||
|
let links = WikiLinkParser::parse_links(content);
|
||||||
|
assert_eq!(links.len(), 2);
|
||||||
|
assert!(links[0].target_path.contains("lifetimes"));
|
||||||
|
assert!(links[1].target_path.contains("SKILL-ownership"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_wiki_link_resolution() {
|
||||||
|
let parser = WikiLinkParser::new("poimen/tools");
|
||||||
|
let resolved = parser.resolve_path("[[../concepts/design-patterns.md]]");
|
||||||
|
assert_eq!(resolved, Path::new("poimen/concepts/design-patterns.md"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_reachable_docs() {
|
||||||
|
// Insert test links
|
||||||
|
let repo = WikiLinkRepo::new(db.clone());
|
||||||
|
repo.upsert_links("poimen", vec![
|
||||||
|
WikiLink { source: "tools/kubectl.md", target: "debugging.md", ... },
|
||||||
|
WikiLink { source: "debugging.md", target: "../../shared/skills/SKILL-k8s" },
|
||||||
|
]).await?;
|
||||||
|
|
||||||
|
// Query reachable from tools/kubectl.md
|
||||||
|
let reachable = repo.reachable_docs("poimen").await?;
|
||||||
|
assert!(reachable.contains(&"tools/kubectl.md".to_string()));
|
||||||
|
assert!(reachable.contains(&"debugging.md".to_string()));
|
||||||
|
assert!(reachable.contains(&"shared/skills/SKILL-k8s".to_string()));
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Homelab Structure (test vault):**
|
||||||
|
|
||||||
|
```
|
||||||
|
homelab/vault/
|
||||||
|
projects/
|
||||||
|
poimen/
|
||||||
|
_access.yaml
|
||||||
|
index.md
|
||||||
|
"[[tools/kubectl.md]]"
|
||||||
|
"[[memories/ownership.md]]"
|
||||||
|
tools/
|
||||||
|
kubectl.md
|
||||||
|
"[[../debugging/pod-crashes.md]]"
|
||||||
|
"[[../../shared/skills/SKILL-kubernetes-debugging]]"
|
||||||
|
debugging/
|
||||||
|
pod-crashes.md
|
||||||
|
"[[../../memories/ownership.md]]"
|
||||||
|
memories/
|
||||||
|
ownership.md
|
||||||
|
"[[../tools/kubectl.md]]"
|
||||||
|
|
||||||
|
rust-guide/
|
||||||
|
_access.yaml
|
||||||
|
index.md
|
||||||
|
"[[memories/ownership.md]]"
|
||||||
|
memories/
|
||||||
|
ownership.md
|
||||||
|
"[[../../shared/concepts/design-patterns.md]]"
|
||||||
|
|
||||||
|
shared/
|
||||||
|
concepts/
|
||||||
|
design-patterns.md
|
||||||
|
skills/
|
||||||
|
SKILL-kubernetes-debugging/
|
||||||
|
_access.yaml
|
||||||
|
SKILL.md
|
||||||
|
"[[../../projects/poimen/tools/kubectl.md]]"
|
||||||
|
```
|
||||||
|
|
||||||
|
**Verification Checklist:**
|
||||||
|
- [ ] Parser correctly extracts all [[link]] types
|
||||||
|
- [ ] Path resolution handles relative paths (../../../)
|
||||||
|
- [ ] Reachable docs includes transitive links (A→B→C)
|
||||||
|
- [ ] Backlinks correct (reverse graph)
|
||||||
|
- [ ] Wiki-links survive Postgres round-trip
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Phase 2: Multi-Scope TF-IDF Indexing
|
||||||
|
|
||||||
|
**Code Location:**
|
||||||
|
```
|
||||||
|
crates/mem-core/src/
|
||||||
|
├── tfidf/
|
||||||
|
│ ├── mod.rs (NEW: module root)
|
||||||
|
│ ├── global_index.rs (NEW: global term vocab)
|
||||||
|
│ ├── project_index.rs (NEW: project-local TF-IDF)
|
||||||
|
│ └── chunk_metadata.rs (NEW: heading+term extraction)
|
||||||
|
└── optimizer/
|
||||||
|
└── tfidf_scorer.rs (NEW: TF-IDF scoring)
|
||||||
|
|
||||||
|
k8s/migrations/
|
||||||
|
└── tfidf_index.sql (NEW: schema for term stats)
|
||||||
|
```
|
||||||
|
|
||||||
|
**Implementation Tasks:**
|
||||||
|
|
||||||
|
1. **Global Index Builder** (`global_index.rs`)
|
||||||
|
```rust
|
||||||
|
pub struct GlobalTfIdfIndex {
|
||||||
|
vocabulary: BTreeMap<String, TermStats>,
|
||||||
|
idf_cache: Arc<RwLock<...>>,
|
||||||
|
}
|
||||||
|
impl GlobalTfIdfIndex {
|
||||||
|
pub async fn build_from_vault(vault_root: &Path) -> Result<Self>
|
||||||
|
pub async fn compute_idf(&self, term: &str) -> f32
|
||||||
|
pub async fn reload() -> Result<()> // hot-reload after ingest
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
2. **Project Index** (`project_index.rs`)
|
||||||
|
```rust
|
||||||
|
pub struct ProjectTfIdfIndex {
|
||||||
|
project: String,
|
||||||
|
doc_freqs: HashMap<String, f32>, // doc_id → freq
|
||||||
|
idf: BTreeMap<String, f32>, // term → IDF
|
||||||
|
}
|
||||||
|
impl ProjectTfIdfIndex {
|
||||||
|
pub async fn build(project: &str, vault_root: &Path) -> Result<Self>
|
||||||
|
pub fn score_chunk(&self, query: &str, doc_id: &str) -> f32
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
3. **Chunk Metadata Extractor** (`chunk_metadata.rs`)
|
||||||
|
```rust
|
||||||
|
pub struct ChunkMetadata {
|
||||||
|
heading: String,
|
||||||
|
first_sentence: String,
|
||||||
|
key_terms: Vec<(String, f32)>, // term, TF
|
||||||
|
category: String, // error | solution | tool | concept
|
||||||
|
}
|
||||||
|
impl ChunkMetadata {
|
||||||
|
pub fn extract(content: &str) -> Result<Self>
|
||||||
|
pub fn extract_key_terms(heading: &str, first_line: &str) -> Vec<String>
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Testing:**
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// tests/it_tfidf_indexing.rs
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_global_idf_computation() {
|
||||||
|
let docs = vec![
|
||||||
|
"kubernetes pod error CrashLoopBackOff",
|
||||||
|
"kubernetes pod debugging guide",
|
||||||
|
"docker container error",
|
||||||
|
];
|
||||||
|
|
||||||
|
let index = GlobalTfIdfIndex::from_docs(&docs).await?;
|
||||||
|
|
||||||
|
// "kubernetes" appears in 2/3 docs → IDF = log(3/2) ≈ 0.176
|
||||||
|
// "error" appears in 2/3 docs → IDF ≈ 0.176
|
||||||
|
// "CrashLoopBackOff" appears in 1/3 docs → IDF = log(3/1) ≈ 1.099 (high!)
|
||||||
|
|
||||||
|
assert!(index.compute_idf("kubernetes") < 0.3);
|
||||||
|
assert!(index.compute_idf("CrashLoopBackOff") > 0.9);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_project_scoped_scoring() {
|
||||||
|
let vault = setup_test_vault().await?;
|
||||||
|
let project_idx = ProjectTfIdfIndex::build("poimen", &vault).await?;
|
||||||
|
|
||||||
|
let query = "kubernetes pod crash debugging";
|
||||||
|
let doc1_score = project_idx.score_chunk(query, "tools/kubectl.md");
|
||||||
|
let doc2_score = project_idx.score_chunk(query, "memories/ownership.md");
|
||||||
|
|
||||||
|
// kubectl.md should rank higher (more pod-related terms)
|
||||||
|
assert!(doc1_score > doc2_score);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_chunk_metadata_extraction() {
|
||||||
|
let content = r#"
|
||||||
|
## Pod Crashes in CrashLoopBackOff
|
||||||
|
When a Kubernetes pod enters CrashLoopBackOff status,
|
||||||
|
it means the container is crashing repeatedly.
|
||||||
|
"#;
|
||||||
|
|
||||||
|
let metadata = ChunkMetadata::extract(content)?;
|
||||||
|
assert_eq!(metadata.heading, "## Pod Crashes in CrashLoopBackOff");
|
||||||
|
assert!(metadata.category == "error");
|
||||||
|
assert!(metadata.key_terms.iter().any(|(t, _)| t == "CrashLoopBackOff"));
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Verification Checklist:**
|
||||||
|
- [ ] Global IDF correctly computed (rare terms high, common low)
|
||||||
|
- [ ] Project-local TF-IDF differs from global (project-specific vocab)
|
||||||
|
- [ ] Chunk metadata correctly extracts heading + key terms
|
||||||
|
- [ ] Category inference (error/solution/tool) accurate
|
||||||
|
- [ ] TF-IDF scores consistent across Postgres round-trips
|
||||||
|
- [ ] Benchmark: TF-IDF scoring < 10ms per query on 1000-doc project
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Phase 3: Hybrid Retrieval
|
||||||
|
|
||||||
|
**Code Location:**
|
||||||
|
```
|
||||||
|
crates/mem-cli/src/
|
||||||
|
├── query_router.rs (EXTEND: wiki-scoped routing)
|
||||||
|
├── retrieval_plan.rs (NEW: three-stage plan execution)
|
||||||
|
└── authorized_query_router.rs (NEW: RBAC filtering)
|
||||||
|
|
||||||
|
tests/
|
||||||
|
└── it_wiki_graph_retrieval.rs (NEW: end-to-end tests)
|
||||||
|
```
|
||||||
|
|
||||||
|
**Implementation Tasks:**
|
||||||
|
|
||||||
|
1. **Query Router with Wiki Scope** (`query_router.rs`)
|
||||||
|
```rust
|
||||||
|
pub async fn route_query(
|
||||||
|
&self,
|
||||||
|
query: &str,
|
||||||
|
project: &str, // wiki scope defined here
|
||||||
|
) -> Result<RetrievalPlan> {
|
||||||
|
// 1. Get reachable docs from wiki-link graph
|
||||||
|
let scoped_docs = wiki_index.reachable_docs(project).await?;
|
||||||
|
|
||||||
|
// 2. TF-IDF pre-filter on scoped docs
|
||||||
|
let tfidf_candidates = project_tfidf
|
||||||
|
.score_chunks(query, &scoped_docs)
|
||||||
|
.into_iter()
|
||||||
|
.filter(|(_, score)| score > &0.1)
|
||||||
|
.take(20)
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
|
// 3. Semantic search (only on TF-IDF candidates)
|
||||||
|
let semantic_results = semantic_search(query, &tfidf_candidates).await?;
|
||||||
|
|
||||||
|
// 4. RRF Fusion
|
||||||
|
let fused = rrf_fusion(&tfidf_candidates, &semantic_results)?;
|
||||||
|
|
||||||
|
Ok(RetrievalPlan { candidates: fused })
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
2. **Retrieval Plan Executor** (`retrieval_plan.rs`)
|
||||||
|
```rust
|
||||||
|
pub struct RetrievalPlan {
|
||||||
|
candidates: Vec<(String, f32)>, // doc_id, fused_score
|
||||||
|
project: String,
|
||||||
|
search_strategy: String,
|
||||||
|
}
|
||||||
|
impl RetrievalPlan {
|
||||||
|
pub async fn fetch_chunks(&self, budget: usize) -> Result<Vec<Chunk>>
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Testing:**
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// tests/it_wiki_graph_retrieval.rs
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_wiki_scoped_retrieval() {
|
||||||
|
setup_test_vault_with_links().await?;
|
||||||
|
|
||||||
|
let router = QueryRouter::new(wiki_index, tfidf_idx, embeddings);
|
||||||
|
let plan = router.route_query(
|
||||||
|
"how to fix pod crashes",
|
||||||
|
"poimen"
|
||||||
|
).await?;
|
||||||
|
|
||||||
|
// Should only include docs reachable from poimen/index.md
|
||||||
|
for (doc_id, _) in &plan.candidates {
|
||||||
|
assert!(
|
||||||
|
doc_id.starts_with("poimen/") || doc_id.starts_with("shared/"),
|
||||||
|
"Doc {} outside project scope",
|
||||||
|
doc_id
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_rrf_fusion_scoring() {
|
||||||
|
let tfidf_results = vec![
|
||||||
|
("doc1".to_string(), 0.9),
|
||||||
|
("doc2".to_string(), 0.7),
|
||||||
|
];
|
||||||
|
let semantic_results = vec![
|
||||||
|
("doc2".to_string(), 0.95),
|
||||||
|
("doc1".to_string(), 0.6),
|
||||||
|
];
|
||||||
|
|
||||||
|
let fused = rrf_fusion(&tfidf_results, &semantic_results)?;
|
||||||
|
|
||||||
|
// doc2 should rank higher (high semantic + ok TF-IDF)
|
||||||
|
assert_eq!(fused[0].0, "doc2");
|
||||||
|
assert!(fused[0].1 > fused[1].1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_llm_call_reduction() {
|
||||||
|
let query = "kubernetes debugging";
|
||||||
|
let candidates = router.route_query(query, "poimen").await?.candidates;
|
||||||
|
|
||||||
|
// Before optimization: 100+ candidates passed to LLM
|
||||||
|
// After: only top-10 selected
|
||||||
|
let selected = ChunkSelector::select(candidates, budget=4096).await?;
|
||||||
|
|
||||||
|
assert!(selected.len() <= 10);
|
||||||
|
assert!(selected.len() > 0);
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Homelab Benchmark:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Load test vault into Postgres + OpenSearch
|
||||||
|
cargo test --test it_wiki_graph_retrieval -- --nocapture --test-threads=1
|
||||||
|
|
||||||
|
# Metrics to log:
|
||||||
|
# - Retrieval latency (wiki-nav + TF-IDF + semantic)
|
||||||
|
# - LLM call reduction % (original vs optimized)
|
||||||
|
# - Chunk accuracy (top-5 results match expected docs)
|
||||||
|
```
|
||||||
|
|
||||||
|
**Verification Checklist:**
|
||||||
|
- [ ] Wiki scoping correctly filters candidates
|
||||||
|
- [ ] TF-IDF pre-filter reduces search space 80%+
|
||||||
|
- [ ] RRF fusion improves ranking vs semantic-only
|
||||||
|
- [ ] Retrieval latency < 500ms (wiki + TF-IDF + semantic)
|
||||||
|
- [ ] Selected chunks fit budget (< max_tokens)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Phase 4-6: Chunk Metadata, Cache Alignment, Testing
|
||||||
|
|
||||||
|
[Abbreviated for space; similar pattern to above]
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Phase 7: OIDC + RBAC Implementation
|
||||||
|
|
||||||
|
**Code Location:**
|
||||||
|
```
|
||||||
|
crates/mem-cli/src/
|
||||||
|
├── rbac/
|
||||||
|
│ ├── mod.rs
|
||||||
|
│ ├── oidc_claims.rs (NEW: JWT parsing)
|
||||||
|
│ ├── access_engine.rs (NEW: policy evaluation)
|
||||||
|
│ ├── vault_policy_loader.rs (NEW: load from Vault)
|
||||||
|
│ └── audit_log.rs (NEW: decision logging)
|
||||||
|
└── http_server.rs (EXTEND: add RBAC middleware)
|
||||||
|
|
||||||
|
k8s/migrations/
|
||||||
|
└── rbac.sql (NEW: rbac_audit_log table)
|
||||||
|
```
|
||||||
|
|
||||||
|
**Implementation Tasks:**
|
||||||
|
|
||||||
|
1. **OIDC Claims Extractor** (`oidc_claims.rs`)
|
||||||
|
```rust
|
||||||
|
pub struct OidcClaims {
|
||||||
|
pub sub: String,
|
||||||
|
pub groups: Vec<String>,
|
||||||
|
pub roles: Vec<String>,
|
||||||
|
pub permissions: Vec<String>,
|
||||||
|
}
|
||||||
|
impl OidcClaims {
|
||||||
|
pub fn from_jwt(token: &str, jwks: &JWKS) -> Result<Self>
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
2. **RBAC Engine** (`access_engine.rs`)
|
||||||
|
```rust
|
||||||
|
pub struct RbacEngine {
|
||||||
|
policy_cache: Arc<RwLock<PolicyCache>>,
|
||||||
|
}
|
||||||
|
impl RbacEngine {
|
||||||
|
pub async fn check_access(
|
||||||
|
&self,
|
||||||
|
claims: &OidcClaims,
|
||||||
|
resource_type: &str, // "project" | "skill"
|
||||||
|
resource_name: &str,
|
||||||
|
action: &str, // "read" | "write"
|
||||||
|
) -> Result<bool> { ... }
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Testing:**
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// tests/it_rbac_engine.rs
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_oidc_jwt_parsing() {
|
||||||
|
let token = create_test_jwt(vec!["platform-team"], "charlie");
|
||||||
|
let claims = OidcClaims::from_jwt(&token, &test_jwks())?;
|
||||||
|
|
||||||
|
assert_eq!(claims.sub, "charlie");
|
||||||
|
assert!(claims.groups.contains(&"platform-team".to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_rbac_project_access_group() {
|
||||||
|
let mut engine = RbacEngine::new(vault_client);
|
||||||
|
let claims = OidcClaims {
|
||||||
|
sub: "charlie".to_string(),
|
||||||
|
groups: vec!["platform-team".to_string()],
|
||||||
|
roles: vec!["viewer".to_string()],
|
||||||
|
permissions: vec!["memory:read".to_string()],
|
||||||
|
};
|
||||||
|
|
||||||
|
// Policy: access_level="group", allowed_groups=[platform-team]
|
||||||
|
let allowed = engine.check_access(&claims, "project", "poimen", "read").await?;
|
||||||
|
assert!(allowed);
|
||||||
|
|
||||||
|
// Audit log check
|
||||||
|
let audit = engine.last_decision_log()?;
|
||||||
|
assert_eq!(audit.decision, "allow");
|
||||||
|
assert_eq!(audit.reason, "in_allowed_group");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_rbac_project_access_denied() {
|
||||||
|
let claims = OidcClaims {
|
||||||
|
sub: "alice".to_string(),
|
||||||
|
groups: vec!["data-team".to_string()], // NOT in allowed_groups
|
||||||
|
roles: vec!["viewer".to_string()],
|
||||||
|
permissions: vec![],
|
||||||
|
};
|
||||||
|
|
||||||
|
let allowed = engine.check_access(&claims, "project", "poimen", "read").await?;
|
||||||
|
assert!(!allowed);
|
||||||
|
|
||||||
|
let audit = engine.last_decision_log()?;
|
||||||
|
assert_eq!(audit.decision, "deny");
|
||||||
|
assert_eq!(audit.reason, "not_in_allowed_groups");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_rbac_skill_filtering_in_retrieval() {
|
||||||
|
let claims = OidcClaims {
|
||||||
|
sub: "charlie".to_string(),
|
||||||
|
groups: vec!["platform-team".to_string()],
|
||||||
|
roles: vec![],
|
||||||
|
permissions: vec![],
|
||||||
|
};
|
||||||
|
|
||||||
|
let router = AuthorizedQueryRouter::new(query_router, rbac_engine);
|
||||||
|
let plan = router.route_query(
|
||||||
|
"kubernetes debugging",
|
||||||
|
"poimen",
|
||||||
|
&claims
|
||||||
|
).await?;
|
||||||
|
|
||||||
|
// SKILL-private-debug (owner=ml-team) should be filtered out
|
||||||
|
for (doc_id, _) in &plan.candidates {
|
||||||
|
if doc_id.contains("SKILL-private") {
|
||||||
|
panic!("Private skill leaked to unauthorized user");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Homelab Setup for RBAC Testing:**
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
# homelab/vault/projects/poimen/_access.yaml
|
||||||
|
project: poimen
|
||||||
|
owner_group: platform-team
|
||||||
|
access_level: group
|
||||||
|
allowed_groups: [platform-team, devops-team]
|
||||||
|
required_role: null
|
||||||
|
|
||||||
|
# homelab/vault/shared/skills/SKILL-private-debug/_access.yaml
|
||||||
|
skill: SKILL-private-debug
|
||||||
|
owner_group: ml-team
|
||||||
|
access_level: private
|
||||||
|
allowed_groups: []
|
||||||
|
required_role: null
|
||||||
|
|
||||||
|
# Authentik test users (via API)
|
||||||
|
- charlie: groups=[platform-team, devops-team], roles=[viewer]
|
||||||
|
- alice: groups=[data-team], roles=[viewer]
|
||||||
|
- bob: groups=[ml-team], roles=[editor]
|
||||||
|
```
|
||||||
|
|
||||||
|
**Verification Checklist:**
|
||||||
|
- [ ] JWT validation rejects expired/invalid tokens
|
||||||
|
- [ ] OIDC claims correctly parsed from token
|
||||||
|
- [ ] Project access check respects access_level (public/group/private)
|
||||||
|
- [ ] Skill filtering removes unauthorized results
|
||||||
|
- [ ] Audit log records all decisions (allow/deny)
|
||||||
|
- [ ] Policy hot-reload works without service restart
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## End-to-End Test Scenario
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1. Set up homelab vault structure
|
||||||
|
mkdir -p homelab/vault/{projects/poimen,shared/skills}
|
||||||
|
cp test_vault_structure.sh homelab/vault/
|
||||||
|
./test_vault_structure.sh
|
||||||
|
|
||||||
|
# 2. Load vault into Postgres + OpenSearch
|
||||||
|
cargo run --bin mem ingest --project poimen --vault homelab/vault
|
||||||
|
|
||||||
|
# 3. Create test users in Authentik
|
||||||
|
curl -X POST http://authentik:9000/api/v3/core/users/ \
|
||||||
|
-H "Authorization: Bearer <token>" \
|
||||||
|
-d '{"username": "charlie", "groups": ["platform-team", "devops-team"]}'
|
||||||
|
|
||||||
|
# 4. Test wiki-graph retrieval
|
||||||
|
curl -X POST http://localhost:8080/memory/query \
|
||||||
|
-H "Authorization: Bearer $(get_jwt charlie)" \
|
||||||
|
-d '{"project": "poimen", "query": "fix pod crash"}'
|
||||||
|
# Expected: results from poimen + shared skills, filtered by RBAC
|
||||||
|
|
||||||
|
# 5. Run integration tests
|
||||||
|
cargo test --test it_wiki_graph_retrieval -- --nocapture
|
||||||
|
cargo test --test it_rbac_engine -- --nocapture
|
||||||
|
|
||||||
|
# 6. Benchmark retrieval performance
|
||||||
|
cargo bench --bench wiki_graph_retrieval
|
||||||
|
# Expected: wiki-scoped retrieval 70-80% faster than full-vault search
|
||||||
|
```
|
||||||
|
|
||||||
## Implementation Order
|
## Implementation Order
|
||||||
|
|
||||||
1. **Phase 1** — Wiki-Link Graph Indexing
|
1. **Phase 1** — Wiki-Link Graph Indexing
|
||||||
@@ -681,7 +1282,7 @@ impl CacheAlignedTraversal {
|
|||||||
- Wiki-link ordering by cache locality
|
- Wiki-link ordering by cache locality
|
||||||
- Monitor KV cache hit ratio
|
- Monitor KV cache hit ratio
|
||||||
|
|
||||||
7. **Phase 7** — RBAC (Role-Based Access Control)
|
7. **Phase 7** — OIDC + RBAC (Universal Auth + Policy Enforcement)
|
||||||
- Project + skill ownership model
|
- Project + skill ownership model
|
||||||
- Access level enforcement (private/group/public)
|
- Access level enforcement (private/group/public)
|
||||||
- RBAC check in retrieval pipeline
|
- RBAC check in retrieval pipeline
|
||||||
|
|||||||
Reference in New Issue
Block a user