poimen-memory/crates/mem-ingest/src/obsidian_ref_source.rs

//! M3.6.2 — ObsidianRefSource: Reference document ingestion from Obsidian vault
//!
//! Fetches documents from Obsidian REST API, chunks via M3.6.1 heading-boundary logic,
//! and emits Reference records for indexing in Postgres + OpenSearch.
//! 
//! No vault projection: Obsidian remains source of truth for rebuilds.

use anyhow::Result;
use mem_chunk::record_source::RecordSource;
use mem_core::Record;
use futures::stream::Stream;

/// Reference record metadata
#[derive(Debug, Clone)]
pub struct RefMetadata {
    pub obsidian_path: String,      // e.g., "docs/kubectl.md"
    pub heading_path: String,       // e.g., "kubectl.md > Common Issues"
    pub doc_sha: String,            // SHA256 of entire document
    pub chunk_sha: String,          // SHA256 of this chunk
}

/// Obsidian REST API client
pub struct ObsidianClient {
    base_url: String,
}

impl ObsidianClient {
    pub fn new(base_url: String) -> Self {
        Self { base_url }
    }

    /// List all markdown files in vault
    pub async fn list_files(&self) -> Result<Vec<String>> {
        // TODO: Call Obsidian REST API
        // GET {base_url}/api/vault/listFiles
        // Returns: Vec<String> with .md file paths
        Ok(vec![])
    }

    /// Read file contents from vault
    pub async fn read_file(&self, _path: &str) -> Result<String> {
        // TODO: Call Obsidian REST API
        // GET {base_url}/api/vault/readFile?path={path}
        // Returns: file contents
        Ok(String::new())
    }
}

/// ObsidianRefSource: Fetches & chunks reference documents from Obsidian vault
pub struct ObsidianRefSource {
    client: ObsidianClient,
    project: String,
    allowed_paths: Vec<String>,  // e.g., ["docs/", "reference/"]
}

impl ObsidianRefSource {
    pub fn new(
        obsidian_url: String,
        project: String,
        allowed_paths: Vec<String>,
    ) -> Self {
        let client = ObsidianClient::new(obsidian_url);
        Self {
            client,
            project,
            allowed_paths,
        }
    }

    /// Check if a file path is allowed (matches configured prefixes)
    fn is_allowed_path(&self, path: &str) -> bool {
        self.allowed_paths.iter().any(|prefix| path.starts_with(prefix))
    }

    /// Chunk reference document via heading-boundary logic
    fn chunk_document(&self, path: &str, content: &str) -> Vec<Record> {
        // TODO: Apply M3.6.1 heading-boundary chunking
        // - Split by headings
        // - Compute chunk hashes (sha256)
        // - Build breadcrumb paths (Heading > Subheading > Section)
        // - Yield Record for each chunk with level="R"
        
        vec![]
    }
}

impl RecordSource for ObsidianRefSource {
    fn records(self) -> Box<dyn Stream<Item = Result<Record, String>> + Unpin> {
        // TODO: Implement async streaming
        // 1. Call client.list_files()
        // 2. Filter by allowed_paths
        // 3. For each file: client.read_file() -> chunk_document()
        // 4. Yield records with:
        //    - level: "R"
        //    - source: "obsidian://vault/{path}"
        //    - kind: None (reference docs have no kind)
        //    - no query_id (R answers no standing question)
        
        Box::new(futures::stream::empty())
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_is_allowed_path() {
        let source = ObsidianRefSource::new(
            "http://obsidian:8080".to_string(),
            "test".to_string(),
            vec!["docs/".to_string(), "reference/".to_string()],
        );

        assert!(source.is_allowed_path("docs/kubectl.md"));
        assert!(source.is_allowed_path("reference/networking.md"));
        assert!(!source.is_allowed_path("private/secret.md"));
    }

    #[test]
    fn test_obsidian_client_creation() {
        let client = ObsidianClient::new("http://obsidian:8080".to_string());
        assert_eq!(client.base_url, "http://obsidian:8080");
    }

    #[tokio::test]
    async fn test_obsidian_ref_source_creation() {
        let source = ObsidianRefSource::new(
            "http://obsidian:8080".to_string(),
            "test".to_string(),
            vec!["docs/".to_string()],
        );

        assert_eq!(source.project, "test");
        assert_eq!(source.allowed_paths.len(), 1);
    }

    #[test]
    #[ignore]  // TODO: Implement M3.6.1 heading-boundary chunking
    fn test_chunk_document() {
        let source = ObsidianRefSource::new(
            "http://obsidian:8080".to_string(),
            "test".to_string(),
            vec!["docs/".to_string()],
        );

        let content = "# Main\n\nSection 1\n\n## Sub\n\nSection 2";
        let chunks = source.chunk_document("docs/test.md", content);

        // Should split by headings
        assert!(chunks.len() > 0);
    }

    #[test]
    fn test_reference_record_format() {
        // Record should have:
        // - level: "R"
        // - source: "obsidian://vault/path"
        // - no query_id
        // - no kind
        // - breadcrumb with heading path
        
        let expected_source = "obsidian://poimen-vault/docs/kubectl.md";
        assert!(expected_source.starts_with("obsidian://"));
    }
}