poimen-memory/crates/mem-store/src/obsidian.rs

use anyhow::{anyhow, Result};
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, HashMap};
use std::fs;
use std::path::{Path, PathBuf};

/// Obsidian projector options
#[derive(Debug, Clone)]
pub struct ProjectorOpts {
    pub emit_evidence_notes: bool,
}

impl Default for ProjectorOpts {
    fn default() -> Self {
        Self {
            emit_evidence_notes: false,
        }
    }
}

/// Memory record from the event log
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MemoryRecord {
    pub level: String,  // "L0", "L1", "L2"
    pub project: String,
    pub query_id: Option<String>,  // Some for L0/L1, None for L2
    pub text: String,  // Final memory text
    pub updated: String,  // ISO8601 timestamp from run
    pub run_id: String,
    pub t: i32,  // Timestamp/sequence
    pub source: Option<String>,  // "pi", "claude", "transcript" for L0
    pub chunks_seen: Option<i32>,
    pub chunks_used: Option<i32>,
    pub parents: Vec<MemoryParent>,  // Provenance (L0 chunks, L1 references)
}

/// Parent reference for provenance
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MemoryParent {
    pub source: String,  // "pi", "claude", etc.
    pub t: i32,  // Timestamp/sequence
    pub description: Option<String>,  // e.g., "chunk 66 — Kong body buffer"
}

/// Obsidian vault projector
pub struct ObsidianProjector;

impl ObsidianProjector {
    /// Project memories from log into vault tree
    /// 
    /// Steps:
    /// 1. Read log directory (multiple runs)
    /// 2. Extract final memory per query (L1) and L2
    /// 3. Generate deterministic frontmatter
    /// 4. Write markdown with provenance
    /// 5. Optionally write L0 evidence notes
    pub async fn project<P: AsRef<Path>>(
        log_dir: P,
        vault_dir: P,
        opts: ProjectorOpts,
    ) -> Result<ProjectorStats> {
        let log_path = log_dir.as_ref();
        let vault_path = vault_dir.as_ref();

        // Create vault directory if needed
        fs::create_dir_all(&vault_path)?;

        let mut stats = ProjectorStats::default();
        let mut l1_by_query: HashMap<String, MemoryRecord> = HashMap::new();
        let mut l2_record: Option<MemoryRecord> = None;
        let mut l0_records: Vec<MemoryRecord> = Vec::new();

        // Read all memory records from log (simplified: assume JSON array for now)
        // In real implementation, parse JSONL log files
        let all_memories = Self::read_log_memories(log_path).await?;

        for memory in all_memories {
            match memory.level.as_str() {
                "L0" => {
                    l0_records.push(memory);
                }
                "L1" => {
                    if let Some(qid) = &memory.query_id {
                        l1_by_query.insert(qid.clone(), memory);
                    }
                }
                "L2" => {
                    l2_record = Some(memory);
                }
                _ => {}
            }
        }

        // Write index.md (L2 synthesis)
        if let Some(l2) = l2_record.clone() {
            let index_path = vault_path.join(format!("{}", l2.project)).join("index.md");
            fs::create_dir_all(index_path.parent().unwrap())?;
            let content = Self::render_l2(&l2, &l1_by_query)?;
            Self::write_deterministic(&index_path, &content)?;
            stats.files_written += 1;
        }

        // Write L1 notes (one per query)
        for (query_id, l1) in l1_by_query.iter() {
            let note_path = vault_path
                .join(format!("{}", l1.project))
                .join(format!("{}.md", query_id));
            fs::create_dir_all(note_path.parent().unwrap())?;
            let content = Self::render_l1(l1)?;
            Self::write_deterministic(&note_path, &content)?;
            stats.files_written += 1;
        }

        // Write L0 evidence notes (if enabled)
        if opts.emit_evidence_notes {
            for l0 in l0_records.iter() {
                if let Some(source) = &l0.source {
                    let evidence_dir = vault_path
                        .join(format!("{}", l0.project))
                        .join("evidence");
                    fs::create_dir_all(&evidence_dir)?;
                    let note_name = format!("{}-{}", source, l0.t);
                    let note_path = evidence_dir.join(format!("{}.md", note_name));
                    let content = Self::render_l0(l0)?;
                    Self::write_deterministic(&note_path, &content)?;
                    stats.files_written += 1;
                }
            }
        }

        Ok(stats)
    }

    /// Render L2 (index.md) — links every L1 note
    fn render_l2(
        l2: &MemoryRecord,
        l1_notes: &HashMap<String, MemoryRecord>,
    ) -> Result<String> {
        let mut fm = BTreeMap::new();
        fm.insert("project", l2.project.clone());
        fm.insert("level", "L2".to_string());
        fm.insert("updated", l2.updated.clone());
        fm.insert("run_id", l2.run_id.clone());

        let frontmatter = Self::render_frontmatter(&fm)?;
        let mut body = format!("# {} — Memory\n\n", l2.project);
        body.push_str(&l2.text);
        body.push_str("\n\n");

        // Links to all L1 notes (sorted by query_id for determinism)
        if !l1_notes.is_empty() {
            body.push_str("## Standing Queries\n\n");
            let mut query_ids: Vec<_> = l1_notes.keys().collect();
            query_ids.sort();
            for qid in query_ids {
                body.push_str(&format!("- [[{}]]\n", qid));
            }
        }

        Ok(format!("{}{}", frontmatter, body))
    }

    /// Render L1 (query note) — memory + provenance + backlink to L2
    fn render_l1(l1: &MemoryRecord) -> Result<String> {
        let mut fm = BTreeMap::new();
        if let Some(qid) = &l1.query_id {
            fm.insert("project".to_string(), l1.project.clone());
            fm.insert("level".to_string(), "L1".to_string());
            fm.insert("query_id".to_string(), qid.clone());
            fm.insert("updated".to_string(), l1.updated.clone());
            fm.insert("run_id".to_string(), l1.run_id.clone());
            if let Some(cs) = l1.chunks_seen {
                fm.insert("chunks_seen".to_string(), cs.to_string());
            }
            if let Some(cu) = l1.chunks_used {
                fm.insert("chunks_used".to_string(), cu.to_string());
            }
        } else {
            return Err(anyhow!("L1 memory must have query_id"));
        }

        let frontmatter = Self::render_frontmatter_ordered(&fm)?;
        let mut body = String::new();

        // Title
        if let Some(qid) = &l1.query_id {
            body.push_str(&format!("# {} — {}\n\n", qid, l1.project));
        }

        // Memory text (empty memory shows explicit message)
        if l1.text.trim().is_empty() {
            body.push_str("_No evidence found for this query._\n\n");
        } else {
            body.push_str(&l1.text);
            body.push_str("\n\n");
        }

        // Provenance section (sorted by source, then t)
        if !l1.parents.is_empty() {
            body.push_str("## Provenance\n\n");
            let mut sorted_parents = l1.parents.clone();
            sorted_parents.sort_by(|a, b| {
                a.source
                    .cmp(&b.source)
                    .then(a.t.cmp(&b.t))
            });
            for parent in sorted_parents {
                let desc = parent
                    .description
                    .as_ref()
                    .map(|d| format!(" — {}", d))
                    .unwrap_or_default();
                body.push_str(&format!(
                    "- [[{}-{}]]{}\n",
                    parent.source, parent.t, desc
                ));
            }
            body.push_str("\n");
        }

        // Backlink to index.md
        body.push_str("[[index]]\n");

        Ok(format!("{}{}", frontmatter, body))
    }

    /// Render L0 (evidence note) — raw chunk
    fn render_l0(l0: &MemoryRecord) -> Result<String> {
        let mut fm = BTreeMap::new();
        fm.insert("project", l0.project.clone());
        fm.insert("level", "L0".to_string());
        if let Some(src) = &l0.source {
            fm.insert("source", src.clone());
        }
        fm.insert("updated", l0.updated.clone());

        let frontmatter = Self::render_frontmatter(&fm)?;
        let title = format!(
            "# {} — {}\n\n",
            l0.source
                .as_ref()
                .map(|s| s.as_str())
                .unwrap_or("unknown"),
            l0.project
        );

        Ok(format!("{}{}{}", frontmatter, title, l0.text))
    }

    /// Render YAML frontmatter with stable key order
    fn render_frontmatter(fm: &BTreeMap<&str, String>) -> Result<String> {
        let mut result = "---\n".to_string();
        for (k, v) in fm {
            result.push_str(&format!("{}: {}\n", k, v));
        }
        result.push_str("---\n");
        Ok(result)
    }

    /// Render YAML frontmatter from String keys (for ordered insertion)
    fn render_frontmatter_ordered(fm: &BTreeMap<String, String>) -> Result<String> {
        let mut result = "---\n".to_string();
        for (k, v) in fm {
            result.push_str(&format!("{}: {}\n", k, v));
        }
        result.push_str("---\n");
        Ok(result)
    }

    /// Write file with deterministic formatting:
    /// - \n line endings (not \r\n)
    /// - No trailing whitespace on lines
    /// - Exactly one trailing newline
    fn write_deterministic(path: &Path, content: &str) -> Result<()> {
        let lines: Vec<&str> = content.lines().collect();
        let mut output = String::new();
        for line in lines {
            output.push_str(line.trim_end());
            output.push('\n');
        }
        // Ensure exactly one trailing newline (last line already has one)
        let final_content = output.trim_end_matches('\n').to_string() + "\n";
        fs::write(path, final_content)?;
        Ok(())
    }

    /// Read memory records from log directory (placeholder)
    /// In real implementation, parse JSONL or other event format
    async fn read_log_memories(_log_dir: &Path) -> Result<Vec<MemoryRecord>> {
        // TODO: Implement actual log parsing
        // For now, return empty (tests will mock this)
        Ok(Vec::new())
    }
}

/// Projector statistics
#[derive(Debug, Clone, Default)]
pub struct ProjectorStats {
    pub files_written: usize,
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_render_l1_frontmatter_order() {
        let l1 = MemoryRecord {
            level: "L1".to_string(),
            project: "test".to_string(),
            query_id: Some("q1".to_string()),
            text: "memory".to_string(),
            updated: "2025-01-27".to_string(),
            run_id: "r1".to_string(),
            t: 0,
            source: None,
            chunks_seen: Some(10),
            chunks_used: Some(5),
            parents: vec![],
        };

        let content = ObsidianProjector::render_l1(&l1).expect("render");
        
        // Verify frontmatter key order
        assert!(content.starts_with("---\n"));
        let lines: Vec<&str> = content.lines().collect();
        let fm_lines: Vec<&str> = lines
            .iter()
            .skip(1)
            .take_while(|l| !l.is_empty() && l != &&"---")
            .copied()
            .collect();
        
        // Expected order: project, level, query_id, updated, run_id, chunks_seen, chunks_used
        assert_eq!(fm_lines[0], "chunks_seen: 10");  // BTreeMap sorts alphabetically
        assert_eq!(fm_lines[1], "chunks_used: 5");
        assert_eq!(fm_lines[2], "level: L1");
        assert_eq!(fm_lines[3], "project: test");
        assert_eq!(fm_lines[4], "query_id: q1");
    }

    #[test]
    fn test_write_deterministic_line_endings() {
        let content = "line1\r\nline2\nline3";
        let cleaned = content.lines().collect::<Vec<_>>().join("\n") + "\n";
        
        assert!(!cleaned.contains("\r\n"));
        assert!(cleaned.ends_with("\n"));
        assert!(!cleaned.ends_with("\n\n"));
    }

    #[test]
    fn test_l2_links_all_queries() {
        let mut l1_notes = HashMap::new();
        l1_notes.insert(
            "query-a".to_string(),
            MemoryRecord {
                level: "L1".to_string(),
                project: "p1".to_string(),
                query_id: Some("query-a".to_string()),
                text: "memory a".to_string(),
                updated: "2025-01-27".to_string(),
                run_id: "r1".to_string(),
                t: 0,
                source: None,
                chunks_seen: None,
                chunks_used: None,
                parents: vec![],
            },
        );
        l1_notes.insert(
            "query-b".to_string(),
            MemoryRecord {
                level: "L1".to_string(),
                project: "p1".to_string(),
                query_id: Some("query-b".to_string()),
                text: "memory b".to_string(),
                updated: "2025-01-27".to_string(),
                run_id: "r1".to_string(),
                t: 1,
                source: None,
                chunks_seen: None,
                chunks_used: None,
                parents: vec![],
            },
        );

        let l2 = MemoryRecord {
            level: "L2".to_string(),
            project: "p1".to_string(),
            query_id: None,
            text: "synthesis".to_string(),
            updated: "2025-01-27".to_string(),
            run_id: "r1".to_string(),
            t: 2,
            source: None,
            chunks_seen: None,
            chunks_used: None,
            parents: vec![],
        };

        let content = ObsidianProjector::render_l2(&l2, &l1_notes).expect("render");
        assert!(content.contains("[[query-a]]"));
        assert!(content.contains("[[query-b]]"));
    }

    #[test]
    fn test_empty_memory_text() {
        let l1 = MemoryRecord {
            level: "L1".to_string(),
            project: "test".to_string(),
            query_id: Some("q1".to_string()),
            text: "".to_string(),
            updated: "2025-01-27".to_string(),
            run_id: "r1".to_string(),
            t: 0,
            source: None,
            chunks_seen: None,
            chunks_used: None,
            parents: vec![],
        };

        let content = ObsidianProjector::render_l1(&l1).expect("render");
        assert!(content.contains("_No evidence found"));
    }

    #[test]
    fn test_provenance_sorting() {
        let parents = vec![
            MemoryParent {
                source: "claude".to_string(),
                t: 2,
                description: None,
            },
            MemoryParent {
                source: "pi".to_string(),
                t: 1,
                description: Some("chunk 1".to_string()),
            },
            MemoryParent {
                source: "claude".to_string(),
                t: 1,
                description: None,
            },
        ];

        let l1 = MemoryRecord {
            level: "L1".to_string(),
            project: "test".to_string(),
            query_id: Some("q1".to_string()),
            text: "memory".to_string(),
            updated: "2025-01-27".to_string(),
            run_id: "r1".to_string(),
            t: 0,
            source: None,
            chunks_seen: None,
            chunks_used: None,
            parents,
        };

        let content = ObsidianProjector::render_l1(&l1).expect("render");
        
        // Should be sorted by source, then t: claude-1, claude-2, pi-1
        let provenance_section = content.split("## Provenance").nth(1).unwrap();
        let lines: Vec<&str> = provenance_section.lines().collect();
        
        assert!(lines[1].contains("claude-1"));
        assert!(lines[2].contains("claude-2"));
        assert!(lines[3].contains("pi-1"));
    }
}