refactor: replace Obsidian projector with standalone service (ppatlabs/obsidian)
This commit is contained in:
Story Crater Bot
@@ -8,6 +8,7 @@ pub mod jwt_validator;
|
||||
pub mod opensearch_client;
|
||||
pub mod query_optimizer;
|
||||
pub mod hybrid_query_worker;
|
||||
pub mod verify;
|
||||
|
||||
pub use endpoints::{IngestQueue, IngestRequest, JobStatus};
|
||||
pub use ingest_worker::IngestWorker;
|
||||
|
||||
@@ -6,6 +6,7 @@ mod query_worker;
|
||||
mod rate_limiter;
|
||||
mod idempotency;
|
||||
mod jwt_validator;
|
||||
mod verify;
|
||||
|
||||
use clap::{Parser, Subcommand};
|
||||
use mem_chunk::token_counter::CharsOverFourCounter;
|
||||
@@ -113,6 +114,28 @@ enum Commands {
|
||||
#[arg(long)]
|
||||
database_url: Option<String>,
|
||||
},
|
||||
|
||||
/// Verify edge closure and graph integrity
|
||||
Verify {
|
||||
/// Project name
|
||||
#[arg(long, value_name = "PROJECT")]
|
||||
project: String,
|
||||
/// Check database (default: true)
|
||||
#[arg(long, default_value_t = true)]
|
||||
db: bool,
|
||||
/// Check log (default: true)
|
||||
#[arg(long, default_value_t = true)]
|
||||
log: bool,
|
||||
/// Log directory
|
||||
#[arg(long)]
|
||||
log_dir: Option<PathBuf>,
|
||||
/// Output format (text, json)
|
||||
#[arg(long, default_value = "text")]
|
||||
format: String,
|
||||
/// Database URL
|
||||
#[arg(long)]
|
||||
database_url: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
@@ -160,6 +183,14 @@ async fn main() -> anyhow::Result<()> {
|
||||
let database_url = database_url.unwrap_or_else(|| std::env::var("DATABASE_URL").unwrap_or_else(|_| "postgresql://app:poimen@localhost:5432/memory".to_string()));
|
||||
http_server::start_server(port, api_key, &database_url).await?
|
||||
}
|
||||
Commands::Verify { project, db, log, log_dir, format: fmt, database_url } => {
|
||||
let database_url = database_url.unwrap_or_else(|| std::env::var("DATABASE_URL").unwrap_or_else(|_| "postgresql://app:poimen@localhost:5432/memory".to_string()));
|
||||
let output_format = match fmt.as_str() {
|
||||
"json" => verify::OutputFormat::Json,
|
||||
_ => verify::OutputFormat::Text,
|
||||
};
|
||||
cmd_verify(&project, db, log, log_dir, output_format, &database_url).await?
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -274,3 +305,54 @@ async fn cmd_ingest(
|
||||
println!("Done.");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn cmd_verify(
|
||||
project: &str,
|
||||
check_db: bool,
|
||||
check_log: bool,
|
||||
log_dir: Option<PathBuf>,
|
||||
format: verify::OutputFormat,
|
||||
database_url: &str,
|
||||
) -> anyhow::Result<()> {
|
||||
let opts = verify::VerifyOpts {
|
||||
project: project.to_string(),
|
||||
check_db,
|
||||
check_log,
|
||||
log_dir,
|
||||
format,
|
||||
};
|
||||
|
||||
let verifier = verify::Verifier::new(database_url).await?;
|
||||
let result = verifier.verify(opts).await?;
|
||||
|
||||
match format {
|
||||
verify::OutputFormat::Json => {
|
||||
println!("{}", serde_json::to_string_pretty(&result)?);
|
||||
}
|
||||
verify::OutputFormat::Text => {
|
||||
println!("Project: {}", result.project);
|
||||
println!("Status: {}", if result.clean { "✓ CLEAN" } else { "✗ VIOLATIONS" });
|
||||
println!("Total violations: {}", result.total_violations);
|
||||
|
||||
if !result.violations.is_empty() {
|
||||
println!("\nViolations:");
|
||||
for v in &result.violations {
|
||||
println!(
|
||||
" Invariant {}: {} (sha: {}, level: {})",
|
||||
v.invariant,
|
||||
v.description,
|
||||
v.sha.as_deref().unwrap_or("N/A"),
|
||||
v.level.as_deref().unwrap_or("N/A")
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Exit with non-zero if there are violations
|
||||
if !result.clean {
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -0,0 +1,495 @@
|
||||
use anyhow::{anyhow, Result};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use serde::{Serialize, Deserialize};
|
||||
|
||||
use mem_store::{PgRepo, Level};
|
||||
|
||||
/// Memory record from log
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MemoryRecord {
|
||||
pub level: String, // "L0", "L1", "L2"
|
||||
pub project: String,
|
||||
pub query_id: Option<String>, // Some for L0/L1, None for L2
|
||||
pub text: String, // Final memory text
|
||||
pub updated: String, // ISO8601 timestamp from run
|
||||
pub run_id: String,
|
||||
pub t: i32, // Timestamp/sequence
|
||||
pub source: Option<String>, // "pi", "claude", "transcript" for L0
|
||||
pub chunks_seen: Option<i32>,
|
||||
pub chunks_used: Option<i32>,
|
||||
pub parents: Vec<MemoryParent>, // Provenance (L0 chunks, L1 references)
|
||||
pub gate: Option<bool>, // M1.8 evidence gate
|
||||
}
|
||||
|
||||
/// Parent reference for provenance
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MemoryParent {
|
||||
pub source: String, // "pi", "claude", etc.
|
||||
pub t: i32, // Timestamp/sequence
|
||||
pub text: String, // Parent text content
|
||||
pub description: Option<String>, // e.g., "chunk 66 — Kong body buffer"
|
||||
}
|
||||
|
||||
/// Verification options
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct VerifyOpts {
|
||||
pub project: String,
|
||||
pub check_db: bool,
|
||||
pub check_log: bool,
|
||||
pub log_dir: Option<PathBuf>,
|
||||
pub format: OutputFormat,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum OutputFormat {
|
||||
Text,
|
||||
Json,
|
||||
}
|
||||
|
||||
/// Violation of an invariant
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct Violation {
|
||||
pub invariant: u32,
|
||||
pub description: String,
|
||||
pub sha: Option<String>,
|
||||
pub level: Option<String>,
|
||||
pub run_id: Option<String>,
|
||||
pub log_line: Option<usize>,
|
||||
}
|
||||
|
||||
/// Verification result
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct VerificationResult {
|
||||
pub project: String,
|
||||
pub clean: bool,
|
||||
pub violations: Vec<Violation>,
|
||||
pub total_violations: usize,
|
||||
}
|
||||
|
||||
pub struct Verifier {
|
||||
repo: PgRepo,
|
||||
}
|
||||
|
||||
impl Verifier {
|
||||
pub async fn new(db_url: &str) -> Result<Self> {
|
||||
let repo = PgRepo::connect(db_url).await?;
|
||||
Ok(Self { repo })
|
||||
}
|
||||
|
||||
/// Run all verifications
|
||||
pub async fn verify(&self, opts: VerifyOpts) -> Result<VerificationResult> {
|
||||
let log_dir = opts.log_dir.unwrap_or_else(|| PathBuf::from("log"));
|
||||
let mut violations = Vec::new();
|
||||
|
||||
// Read log
|
||||
let (memories, _) = Self::read_log(&log_dir, &opts.project).await?;
|
||||
|
||||
// Check database if requested
|
||||
if opts.check_db {
|
||||
let db_violations = self.check_database(&opts.project, &memories).await?;
|
||||
violations.extend(db_violations);
|
||||
}
|
||||
|
||||
// Check log if requested
|
||||
if opts.check_log {
|
||||
let log_violations = Self::check_log_invariants(&memories);
|
||||
violations.extend(log_violations);
|
||||
}
|
||||
|
||||
// Deduplicate violations
|
||||
violations.sort_by_key(|v| (v.invariant, v.sha.clone().unwrap_or_default()));
|
||||
violations.dedup_by_key(|v| (v.invariant, v.sha.clone().unwrap_or_default()));
|
||||
|
||||
let clean = violations.is_empty();
|
||||
let total_violations = violations.len();
|
||||
|
||||
Ok(VerificationResult {
|
||||
project: opts.project,
|
||||
clean,
|
||||
violations,
|
||||
total_violations,
|
||||
})
|
||||
}
|
||||
|
||||
/// Check invariants against the database
|
||||
async fn check_database(
|
||||
&self,
|
||||
project: &str,
|
||||
memories: &[MemoryRecord],
|
||||
) -> Result<Vec<Violation>> {
|
||||
let mut violations = Vec::new();
|
||||
|
||||
// Get all nodes in database for this project
|
||||
let nodes = self.repo.list_nodes(project).await?;
|
||||
let node_shas: HashSet<_> = nodes.iter().map(|n| n.sha256.clone()).collect();
|
||||
|
||||
// Get all edges in database
|
||||
let edges = self.repo.list_edges(project).await?;
|
||||
|
||||
// Build parent map from memories
|
||||
let mut memory_parents: HashMap<String, Vec<String>> = HashMap::new();
|
||||
let mut memory_levels: HashMap<String, String> = HashMap::new();
|
||||
let mut evidence_shas: HashSet<String> = HashSet::new();
|
||||
let mut memory_shas_with_evidence: HashSet<String> = HashSet::new();
|
||||
|
||||
for memory in memories {
|
||||
let sha = Self::memory_sha(&memory.text);
|
||||
let level_str = &memory.level;
|
||||
memory_levels.insert(sha.clone(), level_str.clone());
|
||||
|
||||
let parents: Vec<String> = memory
|
||||
.parents
|
||||
.iter()
|
||||
.map(|p| Self::memory_sha(&p.text))
|
||||
.collect();
|
||||
|
||||
if !parents.is_empty() {
|
||||
memory_shas_with_evidence.insert(sha.clone());
|
||||
}
|
||||
|
||||
for parent_sha in &parents {
|
||||
evidence_shas.insert(parent_sha.clone());
|
||||
}
|
||||
|
||||
memory_parents.insert(sha, parents);
|
||||
}
|
||||
|
||||
// Invariant 2: Every parent sha in edges exists as a node
|
||||
for edge in &edges {
|
||||
if !node_shas.contains(&edge.parent_sha) {
|
||||
violations.push(Violation {
|
||||
invariant: 2,
|
||||
description: format!(
|
||||
"Parent sha {} referenced in edge but not found as node",
|
||||
&edge.parent_sha
|
||||
),
|
||||
sha: Some(edge.parent_sha.clone()),
|
||||
level: None,
|
||||
run_id: None,
|
||||
log_line: None,
|
||||
});
|
||||
}
|
||||
if !node_shas.contains(&edge.child_sha) {
|
||||
violations.push(Violation {
|
||||
invariant: 2,
|
||||
description: format!(
|
||||
"Child sha {} referenced in edge but not found as node",
|
||||
&edge.child_sha
|
||||
),
|
||||
sha: Some(edge.child_sha.clone()),
|
||||
level: None,
|
||||
run_id: None,
|
||||
log_line: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Invariant 6: Check level consistency in database (L1 parents are L0, L2 parents are L1)
|
||||
for node in &nodes {
|
||||
if node.level == "L1" {
|
||||
for edge in &edges {
|
||||
if edge.child_sha == node.sha256 {
|
||||
if let Some(parent_node) = nodes.iter().find(|n| n.sha256 == edge.parent_sha) {
|
||||
if parent_node.level != "L0" {
|
||||
violations.push(Violation {
|
||||
invariant: 6,
|
||||
description: format!(
|
||||
"L1 node {} has parent with level {} (expected L0)",
|
||||
&node.sha256, &parent_node.level
|
||||
),
|
||||
sha: Some(node.sha256.clone()),
|
||||
level: Some("L1".to_string()),
|
||||
run_id: node.run_id.clone(),
|
||||
log_line: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if node.level == "L2" {
|
||||
for edge in &edges {
|
||||
if edge.child_sha == node.sha256 {
|
||||
if let Some(parent_node) = nodes.iter().find(|n| n.sha256 == edge.parent_sha) {
|
||||
if parent_node.level != "L1" {
|
||||
violations.push(Violation {
|
||||
invariant: 6,
|
||||
description: format!(
|
||||
"L2 node {} has parent with level {} (expected L1)",
|
||||
&node.sha256, &parent_node.level
|
||||
),
|
||||
sha: Some(node.sha256.clone()),
|
||||
level: Some("L2".to_string()),
|
||||
run_id: node.run_id.clone(),
|
||||
log_line: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(violations)
|
||||
}
|
||||
|
||||
/// Check invariants against the log
|
||||
fn check_log_invariants(memories: &[MemoryRecord]) -> Vec<Violation> {
|
||||
let mut violations = Vec::new();
|
||||
|
||||
// Build maps
|
||||
let mut memory_map: HashMap<String, &MemoryRecord> = HashMap::new();
|
||||
let mut memory_parents: HashMap<String, Vec<String>> = HashMap::new();
|
||||
let mut evidence_shas: HashSet<String> = HashSet::new();
|
||||
let mut level_map: HashMap<String, String> = HashMap::new();
|
||||
let mut evidence_gate_count = 0;
|
||||
let mut evidence_records = 0;
|
||||
|
||||
for (line_num, memory) in memories.iter().enumerate() {
|
||||
let sha = Self::memory_sha(&memory.text);
|
||||
memory_map.insert(sha.clone(), memory);
|
||||
level_map.insert(sha.clone(), memory.level.clone());
|
||||
|
||||
let parents: Vec<String> = memory
|
||||
.parents
|
||||
.iter()
|
||||
.map(|p| Self::memory_sha(&p.text))
|
||||
.collect();
|
||||
|
||||
memory_parents.insert(sha, parents.clone());
|
||||
|
||||
// Count evidence and gates
|
||||
for parent_sha in &parents {
|
||||
evidence_shas.insert(parent_sha.clone());
|
||||
evidence_records += 1;
|
||||
}
|
||||
|
||||
if memory.gate == Some(true) {
|
||||
evidence_gate_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Invariant 1: Every L1 has at least one L0 parent
|
||||
for (sha, memory) in &memory_map {
|
||||
if memory.level == "L1" {
|
||||
if let Some(parents) = memory_parents.get(sha) {
|
||||
if parents.is_empty() {
|
||||
violations.push(Violation {
|
||||
invariant: 1,
|
||||
description: "L1 memory has no parents (evidence)".to_string(),
|
||||
sha: Some(sha.clone()),
|
||||
level: Some("L1".to_string()),
|
||||
run_id: memory.run_id.clone(),
|
||||
log_line: None,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
violations.push(Violation {
|
||||
invariant: 1,
|
||||
description: "L1 memory not found in parent map".to_string(),
|
||||
sha: Some(sha.clone()),
|
||||
level: Some("L1".to_string()),
|
||||
run_id: memory.run_id.clone(),
|
||||
log_line: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Invariant 2: Every parent sha resolves to a memory that exists
|
||||
for (sha, parents) in &memory_parents {
|
||||
for parent_sha in parents {
|
||||
if !memory_map.contains_key(parent_sha) {
|
||||
violations.push(Violation {
|
||||
invariant: 2,
|
||||
description: format!("Parent sha {} not found in log", parent_sha),
|
||||
sha: Some(parent_sha.clone()),
|
||||
level: None,
|
||||
run_id: None,
|
||||
log_line: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Invariant 3: Every evidence sha appears as a parent of at least one memory
|
||||
for evidence_sha in &evidence_shas {
|
||||
let mut is_cited = false;
|
||||
for (sha, parents) in &memory_parents {
|
||||
if parents.contains(evidence_sha) {
|
||||
is_cited = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if !is_cited {
|
||||
violations.push(Violation {
|
||||
invariant: 3,
|
||||
description: format!("Evidence sha {} is not cited by any memory", evidence_sha),
|
||||
sha: Some(evidence_sha.clone()),
|
||||
level: None,
|
||||
run_id: None,
|
||||
log_line: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Invariant 4: evidence count equals gate.update == true count
|
||||
if evidence_records != evidence_gate_count {
|
||||
violations.push(Violation {
|
||||
invariant: 4,
|
||||
description: format!(
|
||||
"Evidence count {} does not match update-gate count {}",
|
||||
evidence_records, evidence_gate_count
|
||||
),
|
||||
sha: None,
|
||||
level: None,
|
||||
run_id: None,
|
||||
log_line: None,
|
||||
});
|
||||
}
|
||||
|
||||
// Invariant 5: No cycles
|
||||
for (sha, parents) in &memory_parents {
|
||||
if let Some(cycle) = Self::detect_cycle(sha, parents, &memory_parents) {
|
||||
violations.push(Violation {
|
||||
invariant: 5,
|
||||
description: format!("Cycle detected: {} → {}", sha, cycle),
|
||||
sha: Some(sha.clone()),
|
||||
level: None,
|
||||
run_id: None,
|
||||
log_line: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Invariant 6: Level consistency in log (L1 parents are L0, L2 parents are L1)
|
||||
for (sha, memory) in &memory_map {
|
||||
if memory.level == "L1" {
|
||||
if let Some(parents) = memory_parents.get(sha) {
|
||||
for parent_sha in parents {
|
||||
if let Some(parent_level) = level_map.get(parent_sha) {
|
||||
if parent_level != "L0" {
|
||||
violations.push(Violation {
|
||||
invariant: 6,
|
||||
description: format!(
|
||||
"L1 memory has parent with level {} (expected L0)",
|
||||
parent_level
|
||||
),
|
||||
sha: Some(sha.clone()),
|
||||
level: Some("L1".to_string()),
|
||||
run_id: memory.run_id.clone(),
|
||||
log_line: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if memory.level == "L2" {
|
||||
if let Some(parents) = memory_parents.get(sha) {
|
||||
for parent_sha in parents {
|
||||
if let Some(parent_level) = level_map.get(parent_sha) {
|
||||
if parent_level != "L1" {
|
||||
violations.push(Violation {
|
||||
invariant: 6,
|
||||
description: format!(
|
||||
"L2 memory has parent with level {} (expected L1)",
|
||||
parent_level
|
||||
),
|
||||
sha: Some(sha.clone()),
|
||||
level: Some("L2".to_string()),
|
||||
run_id: memory.run_id.clone(),
|
||||
log_line: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
violations
|
||||
}
|
||||
|
||||
/// Detect cycle in parent graph
|
||||
fn detect_cycle(
|
||||
node: &str,
|
||||
_parents: &[String],
|
||||
all_parents: &HashMap<String, Vec<String>>,
|
||||
) -> Option<String> {
|
||||
let mut visited = HashSet::new();
|
||||
let mut rec_stack = HashSet::new();
|
||||
|
||||
fn dfs(
|
||||
node: &str,
|
||||
all_parents: &HashMap<String, Vec<String>>,
|
||||
visited: &mut HashSet<String>,
|
||||
rec_stack: &mut HashSet<String>,
|
||||
) -> Option<String> {
|
||||
visited.insert(node.to_string());
|
||||
rec_stack.insert(node.to_string());
|
||||
|
||||
if let Some(parents) = all_parents.get(node) {
|
||||
for parent in parents {
|
||||
if !visited.contains(parent) {
|
||||
if let Some(cycle) = dfs(parent, all_parents, visited, rec_stack) {
|
||||
return Some(cycle);
|
||||
}
|
||||
} else if rec_stack.contains(parent) {
|
||||
return Some(parent.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
rec_stack.remove(node);
|
||||
None
|
||||
}
|
||||
|
||||
dfs(node, all_parents, &mut visited, &mut rec_stack)
|
||||
}
|
||||
|
||||
/// Read log directory
|
||||
async fn read_log(
|
||||
log_dir: &Path,
|
||||
project: &str,
|
||||
) -> Result<(Vec<MemoryRecord>, usize)> {
|
||||
let mut memories = Vec::new();
|
||||
let mut total_lines = 0;
|
||||
|
||||
if !log_dir.exists() {
|
||||
return Ok((memories, total_lines));
|
||||
}
|
||||
|
||||
let entries = fs::read_dir(log_dir)?;
|
||||
for entry in entries {
|
||||
let entry = entry?;
|
||||
let path = entry.path();
|
||||
|
||||
if path.extension().map(|e| e == "jsonl").unwrap_or(false) {
|
||||
let content = fs::read_to_string(&path)?;
|
||||
for line in content.lines() {
|
||||
total_lines += 1;
|
||||
if line.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Ok(record) = serde_json::from_str::<MemoryRecord>(line) {
|
||||
if record.project == project {
|
||||
memories.push(record);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok((memories, total_lines))
|
||||
}
|
||||
|
||||
/// Compute memory sha256
|
||||
fn memory_sha(text: &str) -> String {
|
||||
use sha2::Digest;
|
||||
let mut hasher = sha2::Sha256::new();
|
||||
hasher.update(text.as_bytes());
|
||||
format!("{:x}", hasher.finalize())
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user