refactor: replace Obsidian projector with standalone service (ppatlabs/obsidian)
This commit is contained in:
Story Crater Bot
@@ -0,0 +1,495 @@
|
||||
use anyhow::{anyhow, Result};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use serde::{Serialize, Deserialize};
|
||||
|
||||
use mem_store::{PgRepo, Level};
|
||||
|
||||
/// Memory record from log
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MemoryRecord {
|
||||
pub level: String, // "L0", "L1", "L2"
|
||||
pub project: String,
|
||||
pub query_id: Option<String>, // Some for L0/L1, None for L2
|
||||
pub text: String, // Final memory text
|
||||
pub updated: String, // ISO8601 timestamp from run
|
||||
pub run_id: String,
|
||||
pub t: i32, // Timestamp/sequence
|
||||
pub source: Option<String>, // "pi", "claude", "transcript" for L0
|
||||
pub chunks_seen: Option<i32>,
|
||||
pub chunks_used: Option<i32>,
|
||||
pub parents: Vec<MemoryParent>, // Provenance (L0 chunks, L1 references)
|
||||
pub gate: Option<bool>, // M1.8 evidence gate
|
||||
}
|
||||
|
||||
/// Parent reference for provenance
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MemoryParent {
|
||||
pub source: String, // "pi", "claude", etc.
|
||||
pub t: i32, // Timestamp/sequence
|
||||
pub text: String, // Parent text content
|
||||
pub description: Option<String>, // e.g., "chunk 66 — Kong body buffer"
|
||||
}
|
||||
|
||||
/// Verification options
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct VerifyOpts {
|
||||
pub project: String,
|
||||
pub check_db: bool,
|
||||
pub check_log: bool,
|
||||
pub log_dir: Option<PathBuf>,
|
||||
pub format: OutputFormat,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum OutputFormat {
|
||||
Text,
|
||||
Json,
|
||||
}
|
||||
|
||||
/// Violation of an invariant
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct Violation {
|
||||
pub invariant: u32,
|
||||
pub description: String,
|
||||
pub sha: Option<String>,
|
||||
pub level: Option<String>,
|
||||
pub run_id: Option<String>,
|
||||
pub log_line: Option<usize>,
|
||||
}
|
||||
|
||||
/// Verification result
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct VerificationResult {
|
||||
pub project: String,
|
||||
pub clean: bool,
|
||||
pub violations: Vec<Violation>,
|
||||
pub total_violations: usize,
|
||||
}
|
||||
|
||||
pub struct Verifier {
|
||||
repo: PgRepo,
|
||||
}
|
||||
|
||||
impl Verifier {
|
||||
pub async fn new(db_url: &str) -> Result<Self> {
|
||||
let repo = PgRepo::connect(db_url).await?;
|
||||
Ok(Self { repo })
|
||||
}
|
||||
|
||||
/// Run all verifications
|
||||
pub async fn verify(&self, opts: VerifyOpts) -> Result<VerificationResult> {
|
||||
let log_dir = opts.log_dir.unwrap_or_else(|| PathBuf::from("log"));
|
||||
let mut violations = Vec::new();
|
||||
|
||||
// Read log
|
||||
let (memories, _) = Self::read_log(&log_dir, &opts.project).await?;
|
||||
|
||||
// Check database if requested
|
||||
if opts.check_db {
|
||||
let db_violations = self.check_database(&opts.project, &memories).await?;
|
||||
violations.extend(db_violations);
|
||||
}
|
||||
|
||||
// Check log if requested
|
||||
if opts.check_log {
|
||||
let log_violations = Self::check_log_invariants(&memories);
|
||||
violations.extend(log_violations);
|
||||
}
|
||||
|
||||
// Deduplicate violations
|
||||
violations.sort_by_key(|v| (v.invariant, v.sha.clone().unwrap_or_default()));
|
||||
violations.dedup_by_key(|v| (v.invariant, v.sha.clone().unwrap_or_default()));
|
||||
|
||||
let clean = violations.is_empty();
|
||||
let total_violations = violations.len();
|
||||
|
||||
Ok(VerificationResult {
|
||||
project: opts.project,
|
||||
clean,
|
||||
violations,
|
||||
total_violations,
|
||||
})
|
||||
}
|
||||
|
||||
/// Check invariants against the database
|
||||
async fn check_database(
|
||||
&self,
|
||||
project: &str,
|
||||
memories: &[MemoryRecord],
|
||||
) -> Result<Vec<Violation>> {
|
||||
let mut violations = Vec::new();
|
||||
|
||||
// Get all nodes in database for this project
|
||||
let nodes = self.repo.list_nodes(project).await?;
|
||||
let node_shas: HashSet<_> = nodes.iter().map(|n| n.sha256.clone()).collect();
|
||||
|
||||
// Get all edges in database
|
||||
let edges = self.repo.list_edges(project).await?;
|
||||
|
||||
// Build parent map from memories
|
||||
let mut memory_parents: HashMap<String, Vec<String>> = HashMap::new();
|
||||
let mut memory_levels: HashMap<String, String> = HashMap::new();
|
||||
let mut evidence_shas: HashSet<String> = HashSet::new();
|
||||
let mut memory_shas_with_evidence: HashSet<String> = HashSet::new();
|
||||
|
||||
for memory in memories {
|
||||
let sha = Self::memory_sha(&memory.text);
|
||||
let level_str = &memory.level;
|
||||
memory_levels.insert(sha.clone(), level_str.clone());
|
||||
|
||||
let parents: Vec<String> = memory
|
||||
.parents
|
||||
.iter()
|
||||
.map(|p| Self::memory_sha(&p.text))
|
||||
.collect();
|
||||
|
||||
if !parents.is_empty() {
|
||||
memory_shas_with_evidence.insert(sha.clone());
|
||||
}
|
||||
|
||||
for parent_sha in &parents {
|
||||
evidence_shas.insert(parent_sha.clone());
|
||||
}
|
||||
|
||||
memory_parents.insert(sha, parents);
|
||||
}
|
||||
|
||||
// Invariant 2: Every parent sha in edges exists as a node
|
||||
for edge in &edges {
|
||||
if !node_shas.contains(&edge.parent_sha) {
|
||||
violations.push(Violation {
|
||||
invariant: 2,
|
||||
description: format!(
|
||||
"Parent sha {} referenced in edge but not found as node",
|
||||
&edge.parent_sha
|
||||
),
|
||||
sha: Some(edge.parent_sha.clone()),
|
||||
level: None,
|
||||
run_id: None,
|
||||
log_line: None,
|
||||
});
|
||||
}
|
||||
if !node_shas.contains(&edge.child_sha) {
|
||||
violations.push(Violation {
|
||||
invariant: 2,
|
||||
description: format!(
|
||||
"Child sha {} referenced in edge but not found as node",
|
||||
&edge.child_sha
|
||||
),
|
||||
sha: Some(edge.child_sha.clone()),
|
||||
level: None,
|
||||
run_id: None,
|
||||
log_line: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Invariant 6: Check level consistency in database (L1 parents are L0, L2 parents are L1)
|
||||
for node in &nodes {
|
||||
if node.level == "L1" {
|
||||
for edge in &edges {
|
||||
if edge.child_sha == node.sha256 {
|
||||
if let Some(parent_node) = nodes.iter().find(|n| n.sha256 == edge.parent_sha) {
|
||||
if parent_node.level != "L0" {
|
||||
violations.push(Violation {
|
||||
invariant: 6,
|
||||
description: format!(
|
||||
"L1 node {} has parent with level {} (expected L0)",
|
||||
&node.sha256, &parent_node.level
|
||||
),
|
||||
sha: Some(node.sha256.clone()),
|
||||
level: Some("L1".to_string()),
|
||||
run_id: node.run_id.clone(),
|
||||
log_line: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if node.level == "L2" {
|
||||
for edge in &edges {
|
||||
if edge.child_sha == node.sha256 {
|
||||
if let Some(parent_node) = nodes.iter().find(|n| n.sha256 == edge.parent_sha) {
|
||||
if parent_node.level != "L1" {
|
||||
violations.push(Violation {
|
||||
invariant: 6,
|
||||
description: format!(
|
||||
"L2 node {} has parent with level {} (expected L1)",
|
||||
&node.sha256, &parent_node.level
|
||||
),
|
||||
sha: Some(node.sha256.clone()),
|
||||
level: Some("L2".to_string()),
|
||||
run_id: node.run_id.clone(),
|
||||
log_line: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(violations)
|
||||
}
|
||||
|
||||
/// Check invariants against the log
|
||||
fn check_log_invariants(memories: &[MemoryRecord]) -> Vec<Violation> {
|
||||
let mut violations = Vec::new();
|
||||
|
||||
// Build maps
|
||||
let mut memory_map: HashMap<String, &MemoryRecord> = HashMap::new();
|
||||
let mut memory_parents: HashMap<String, Vec<String>> = HashMap::new();
|
||||
let mut evidence_shas: HashSet<String> = HashSet::new();
|
||||
let mut level_map: HashMap<String, String> = HashMap::new();
|
||||
let mut evidence_gate_count = 0;
|
||||
let mut evidence_records = 0;
|
||||
|
||||
for (line_num, memory) in memories.iter().enumerate() {
|
||||
let sha = Self::memory_sha(&memory.text);
|
||||
memory_map.insert(sha.clone(), memory);
|
||||
level_map.insert(sha.clone(), memory.level.clone());
|
||||
|
||||
let parents: Vec<String> = memory
|
||||
.parents
|
||||
.iter()
|
||||
.map(|p| Self::memory_sha(&p.text))
|
||||
.collect();
|
||||
|
||||
memory_parents.insert(sha, parents.clone());
|
||||
|
||||
// Count evidence and gates
|
||||
for parent_sha in &parents {
|
||||
evidence_shas.insert(parent_sha.clone());
|
||||
evidence_records += 1;
|
||||
}
|
||||
|
||||
if memory.gate == Some(true) {
|
||||
evidence_gate_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Invariant 1: Every L1 has at least one L0 parent
|
||||
for (sha, memory) in &memory_map {
|
||||
if memory.level == "L1" {
|
||||
if let Some(parents) = memory_parents.get(sha) {
|
||||
if parents.is_empty() {
|
||||
violations.push(Violation {
|
||||
invariant: 1,
|
||||
description: "L1 memory has no parents (evidence)".to_string(),
|
||||
sha: Some(sha.clone()),
|
||||
level: Some("L1".to_string()),
|
||||
run_id: memory.run_id.clone(),
|
||||
log_line: None,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
violations.push(Violation {
|
||||
invariant: 1,
|
||||
description: "L1 memory not found in parent map".to_string(),
|
||||
sha: Some(sha.clone()),
|
||||
level: Some("L1".to_string()),
|
||||
run_id: memory.run_id.clone(),
|
||||
log_line: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Invariant 2: Every parent sha resolves to a memory that exists
|
||||
for (sha, parents) in &memory_parents {
|
||||
for parent_sha in parents {
|
||||
if !memory_map.contains_key(parent_sha) {
|
||||
violations.push(Violation {
|
||||
invariant: 2,
|
||||
description: format!("Parent sha {} not found in log", parent_sha),
|
||||
sha: Some(parent_sha.clone()),
|
||||
level: None,
|
||||
run_id: None,
|
||||
log_line: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Invariant 3: Every evidence sha appears as a parent of at least one memory
|
||||
for evidence_sha in &evidence_shas {
|
||||
let mut is_cited = false;
|
||||
for (sha, parents) in &memory_parents {
|
||||
if parents.contains(evidence_sha) {
|
||||
is_cited = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if !is_cited {
|
||||
violations.push(Violation {
|
||||
invariant: 3,
|
||||
description: format!("Evidence sha {} is not cited by any memory", evidence_sha),
|
||||
sha: Some(evidence_sha.clone()),
|
||||
level: None,
|
||||
run_id: None,
|
||||
log_line: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Invariant 4: evidence count equals gate.update == true count
|
||||
if evidence_records != evidence_gate_count {
|
||||
violations.push(Violation {
|
||||
invariant: 4,
|
||||
description: format!(
|
||||
"Evidence count {} does not match update-gate count {}",
|
||||
evidence_records, evidence_gate_count
|
||||
),
|
||||
sha: None,
|
||||
level: None,
|
||||
run_id: None,
|
||||
log_line: None,
|
||||
});
|
||||
}
|
||||
|
||||
// Invariant 5: No cycles
|
||||
for (sha, parents) in &memory_parents {
|
||||
if let Some(cycle) = Self::detect_cycle(sha, parents, &memory_parents) {
|
||||
violations.push(Violation {
|
||||
invariant: 5,
|
||||
description: format!("Cycle detected: {} → {}", sha, cycle),
|
||||
sha: Some(sha.clone()),
|
||||
level: None,
|
||||
run_id: None,
|
||||
log_line: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Invariant 6: Level consistency in log (L1 parents are L0, L2 parents are L1)
|
||||
for (sha, memory) in &memory_map {
|
||||
if memory.level == "L1" {
|
||||
if let Some(parents) = memory_parents.get(sha) {
|
||||
for parent_sha in parents {
|
||||
if let Some(parent_level) = level_map.get(parent_sha) {
|
||||
if parent_level != "L0" {
|
||||
violations.push(Violation {
|
||||
invariant: 6,
|
||||
description: format!(
|
||||
"L1 memory has parent with level {} (expected L0)",
|
||||
parent_level
|
||||
),
|
||||
sha: Some(sha.clone()),
|
||||
level: Some("L1".to_string()),
|
||||
run_id: memory.run_id.clone(),
|
||||
log_line: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if memory.level == "L2" {
|
||||
if let Some(parents) = memory_parents.get(sha) {
|
||||
for parent_sha in parents {
|
||||
if let Some(parent_level) = level_map.get(parent_sha) {
|
||||
if parent_level != "L1" {
|
||||
violations.push(Violation {
|
||||
invariant: 6,
|
||||
description: format!(
|
||||
"L2 memory has parent with level {} (expected L1)",
|
||||
parent_level
|
||||
),
|
||||
sha: Some(sha.clone()),
|
||||
level: Some("L2".to_string()),
|
||||
run_id: memory.run_id.clone(),
|
||||
log_line: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
violations
|
||||
}
|
||||
|
||||
/// Detect cycle in parent graph
|
||||
fn detect_cycle(
|
||||
node: &str,
|
||||
_parents: &[String],
|
||||
all_parents: &HashMap<String, Vec<String>>,
|
||||
) -> Option<String> {
|
||||
let mut visited = HashSet::new();
|
||||
let mut rec_stack = HashSet::new();
|
||||
|
||||
fn dfs(
|
||||
node: &str,
|
||||
all_parents: &HashMap<String, Vec<String>>,
|
||||
visited: &mut HashSet<String>,
|
||||
rec_stack: &mut HashSet<String>,
|
||||
) -> Option<String> {
|
||||
visited.insert(node.to_string());
|
||||
rec_stack.insert(node.to_string());
|
||||
|
||||
if let Some(parents) = all_parents.get(node) {
|
||||
for parent in parents {
|
||||
if !visited.contains(parent) {
|
||||
if let Some(cycle) = dfs(parent, all_parents, visited, rec_stack) {
|
||||
return Some(cycle);
|
||||
}
|
||||
} else if rec_stack.contains(parent) {
|
||||
return Some(parent.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
rec_stack.remove(node);
|
||||
None
|
||||
}
|
||||
|
||||
dfs(node, all_parents, &mut visited, &mut rec_stack)
|
||||
}
|
||||
|
||||
/// Read log directory
|
||||
async fn read_log(
|
||||
log_dir: &Path,
|
||||
project: &str,
|
||||
) -> Result<(Vec<MemoryRecord>, usize)> {
|
||||
let mut memories = Vec::new();
|
||||
let mut total_lines = 0;
|
||||
|
||||
if !log_dir.exists() {
|
||||
return Ok((memories, total_lines));
|
||||
}
|
||||
|
||||
let entries = fs::read_dir(log_dir)?;
|
||||
for entry in entries {
|
||||
let entry = entry?;
|
||||
let path = entry.path();
|
||||
|
||||
if path.extension().map(|e| e == "jsonl").unwrap_or(false) {
|
||||
let content = fs::read_to_string(&path)?;
|
||||
for line in content.lines() {
|
||||
total_lines += 1;
|
||||
if line.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Ok(record) = serde_json::from_str::<MemoryRecord>(line) {
|
||||
if record.project == project {
|
||||
memories.push(record);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok((memories, total_lines))
|
||||
}
|
||||
|
||||
/// Compute memory sha256
|
||||
fn memory_sha(text: &str) -> String {
|
||||
use sha2::Digest;
|
||||
let mut hasher = sha2::Sha256::new();
|
||||
hasher.update(text.as_bytes());
|
||||
format!("{:x}", hasher.finalize())
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user