poimen-memory/tests/it_doc_corpus.rs.disabled

//! Integration tests for DocCorpusSource and heading-boundary chunking.

use mem_chunk::{ChunkPolicy, chunks};
use mem_ingest::DocCorpusSource;
use futures::stream::StreamExt;

#[tokio::test]
async fn a1_section_per_heading() {
    // Nested heading fixture should yield exactly one chunk per ATX heading with content
    let source = DocCorpusSource::new("fixtures/refcorpus", 5000);
    let sections = source.scan().expect("scan failed");
    
    // nested.md has: A1, A2, B as sections with body (Top and A have no body)
    // small.md has: A, B (2 sections with body)
    // large_section.md has: one section split into multiple chunks
    // skip_me.json is skipped
    
    // Just verify we get some sections
    assert!(!sections.is_empty(), "should have at least one section");
    println!("Total sections: {}", sections.len());
}

#[tokio::test]
async fn a2_breadcrumb_path() {
    // A chunk under nested headings carries the full heading path
    let source = DocCorpusSource::new("fixtures/refcorpus/nested.md", 5000);
    let sections = source.scan().expect("scan failed");
    
    // Should have sections with nested breadcrumbs
    let has_nested = sections.iter().any(|s| s.breadcrumb.contains(" > "));
    assert!(has_nested, "should have nested breadcrumb paths");
    
    // Print breadcrumbs for inspection
    for section in &sections {
        println!("Breadcrumb: {}", section.breadcrumb);
    }
}

#[tokio::test]
async fn a3_no_mid_section_split() {
    // Every chunk should not cross a heading boundary
    let source = DocCorpusSource::new("fixtures/refcorpus/nested.md", 5000);
    let sections = source.scan().expect("scan failed");
    
    for section in sections {
        let lines: Vec<&str> = section.content.lines().collect();
        
        // Count heading lines
        let heading_count = lines.iter()
            .filter(|l| l.trim_start().starts_with('#'))
            .count();
        
        // Should have at most one heading (the breadcrumb)
        assert!(heading_count <= 1, 
            "Section has {} headings, should be ≤1\nContent:\n{}", 
            heading_count, section.content);
    }
}

#[tokio::test]
async fn a4_oversize_section_splits() {
    // The 8000+ token section should split into multiple chunks
    let source = DocCorpusSource::new("fixtures/refcorpus/large_section.md", 500); // Force split with low token limit
    let sections = source.scan().expect("scan failed");
    
    // Should have split into multiple sections
    assert!(sections.len() > 1, "large section should split into multiple chunks");
    
    // All but the first should be marked as continuation
    for (i, section) in sections.iter().enumerate() {
        if i > 0 {
            assert!(section.continuation, "section {} should be marked as continuation", i);
        } else {
            assert!(!section.continuation, "first section should not be continuation");
        }
    }
    
    println!("Large section split into {} chunks", sections.len());
}

#[tokio::test]
async fn a5_extension_filter() {
    // Non-markdown files (like .json) should be skipped
    let source = DocCorpusSource::new("fixtures/refcorpus", 5000);
    let sections = source.scan().expect("scan failed");
    
    // No section should come from skip_me.json
    let from_json = sections.iter().any(|s| s.source_uri.contains("skip_me.json"));
    assert!(!from_json, "JSON files should be skipped");
    
    // Should only have sections from .md files
    for section in &sections {
        assert!(section.source_uri.ends_with(".md"), 
            "All sections should come from .md files, got: {}", 
            section.source_uri);
    }
}

#[tokio::test]
async fn a6_source_uri_stable() {
    // Running the walk twice should yield identical (source_uri, sha256) pairs
    let source1 = DocCorpusSource::new("fixtures/refcorpus/small.md", 5000);
    let sections1 = source1.scan().expect("scan 1 failed");
    
    let source2 = DocCorpusSource::new("fixtures/refcorpus/small.md", 5000);
    let sections2 = source2.scan().expect("scan 2 failed");
    
    assert_eq!(sections1.len(), sections2.len(), "same scan should yield same number of sections");
    
    for (s1, s2) in sections1.iter().zip(sections2.iter()) {
        assert_eq!(s1.source_uri, s2.source_uri, "source_uri should be stable");
        assert_eq!(s1.file_hash, s2.file_hash, "file_hash should be stable");
    }
}

#[tokio::test]
async fn a7_dry_run_no_network() {
    // Dry run should complete without making any network calls
    let source = DocCorpusSource::new("fixtures/refcorpus", 5000);
    let report = source.dry_run().expect("dry run failed");
    
    assert!(report.total_files > 0, "should have scanned some files");
    assert!(report.total_sections > 0, "should have found some sections");
    
    println!("{}", report);
}

#[tokio::test]
async fn a8_trait_object_safe() {
    // DocCorpusSource should be usable as a RecordSource
    use mem_chunk::RecordSource;
    
    let source = DocCorpusSource::new("fixtures/refcorpus/small.md", 5000);
    let policy = ChunkPolicy::default();
    
    // This compiles only if DocCorpusSource implements RecordSource correctly
    let mut chunk_stream = chunks(source, policy);
    
    // Should be able to pull at least one chunk
    let first_chunk = chunk_stream.next().await;
    assert!(first_chunk.is_some(), "should produce at least one chunk");
}

#[tokio::test]
async fn a9_chunking_respects_heading_boundary() {
    // When chunking via the policy, sections should not be split by the chunker
    // because each DocCorpusSource record is a complete section
    use mem_chunk::RecordSource;
    
    let source = DocCorpusSource::new("fixtures/refcorpus/nested.md", 5000);
    let policy = ChunkPolicy::default();
    
    let mut chunk_stream = chunks(source, policy);
    let mut chunk_count = 0;
    
    while let Some(result) = chunk_stream.next().await {
        match result {
            Ok(chunk) => {
                chunk_count += 1;
                // Each chunk should contain records from a single section
                for record in &chunk.records {
                    assert!(record.text.len() > 0, "record should have content");
                }
            }
            Err(e) => panic!("chunking error: {}", e),
        }
    }
    
    assert!(chunk_count > 0, "should have produced chunks");
    println!("Produced {} chunks from nested.md", chunk_count);
}