feat: production ingest test suite with detailed logging
Add comprehensive E2E test scripts and logging for production testing: - test_prod_ingest_real.sh: Full ingest test against K8s cluster with api-gw - apply_migrations.sh: Manual database schema migration (backup method) - collect_prod_logs.sh: Pod log collection before/after tests - run_production_test.sh: Orchestrates full test + log collection - tests/integration_ingest_with_gw.rs: Integration test with embeddings - tests/unit_ingest_logging.rs: Unit tests for extraction pipeline Enhanced logging in ingest_worker.rs: - Per-record event tracking (extraction, save) - Entity and edge operation logging - Error accumulation and reporting - Structured logging for observability Production testing identified root cause: - Ingest + embedding pipeline working correctly - Entity extraction functional - Database schema missing (migration not applied) - Logs clearly show: relation "memory_entity" does not exist Next: Trigger DB Migration workflow in Forgejo Actions to apply crates/mem-store/migrations/*.sql files.
This commit is contained in:
@@ -0,0 +1,221 @@
|
||||
//! Unit test: Ingest pipeline with detailed error logging
|
||||
//!
|
||||
//! Tests extraction pipeline in isolation without requiring HTTP server or embeddings.
|
||||
//! Useful for debugging extraction errors.
|
||||
//!
|
||||
//! Usage:
|
||||
//! ```
|
||||
//! RUST_LOG=debug,mem_ingest=debug cargo test --test unit_ingest_logging -- --nocapture
|
||||
//! ```
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use mem_ingest::ingest_pipeline::{IngestPipeline, Episode};
|
||||
use mem_ingest::entity_extractor::WikiLinkFallbackExtractor;
|
||||
use mem_ingest::fact_extractor::SimpleFactExtractor;
|
||||
use mem_ingest::contradiction_detector::ContradictionHandler;
|
||||
use std::sync::Arc;
|
||||
|
||||
fn init_logging() {
|
||||
let _ = tracing_subscriber::fmt()
|
||||
.with_max_level(tracing::Level::DEBUG)
|
||||
.with_writer(std::io::stderr)
|
||||
.try_init();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_wiki_link_extraction() {
|
||||
init_logging();
|
||||
|
||||
println!("\n[TEST] Wiki link extraction with logging\n");
|
||||
|
||||
let entity_extractor = Arc::new(WikiLinkFallbackExtractor);
|
||||
let fact_extractor = Arc::new(SimpleFactExtractor);
|
||||
let contradiction_detector = Arc::new(ContradictionHandler::default());
|
||||
|
||||
let pipeline = IngestPipeline::new(
|
||||
entity_extractor,
|
||||
fact_extractor,
|
||||
contradiction_detector,
|
||||
);
|
||||
|
||||
let episode = Episode {
|
||||
id: "test-1".to_string(),
|
||||
project_id: "test-project".to_string(),
|
||||
text: "Kubernetes [[Docker]] is a [[Container]] orchestration platform. It works with [[Go]] programs."
|
||||
.to_string(),
|
||||
wiki_links: vec!["Docker".to_string(), "Container".to_string(), "Go".to_string()],
|
||||
};
|
||||
|
||||
tracing::info!(
|
||||
target: "test",
|
||||
episode_id = %episode.id,
|
||||
wiki_links = ?episode.wiki_links,
|
||||
"Starting pipeline ingest"
|
||||
);
|
||||
|
||||
match pipeline.ingest(&episode).await {
|
||||
Ok(result) => {
|
||||
tracing::info!(
|
||||
target: "test",
|
||||
entities = result.entities.len(),
|
||||
edges = result.edges.len(),
|
||||
reviews = result.reviews.len(),
|
||||
"Pipeline succeeded"
|
||||
);
|
||||
|
||||
println!("✓ Extracted {} entities", result.entities.len());
|
||||
for entity in &result.entities {
|
||||
println!(" - {} ({}): {}", entity.name, entity.entity_type.as_str(), entity.summary.as_deref().unwrap_or(""));
|
||||
}
|
||||
|
||||
println!("✓ Extracted {} edges", result.edges.len());
|
||||
for edge in &result.edges {
|
||||
println!(" - {} --[{}]--> {}", edge.source_entity_id, edge.relation_type, edge.target_entity_id);
|
||||
}
|
||||
|
||||
assert!(result.entities.len() > 0, "Should extract entities");
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!(
|
||||
target: "test",
|
||||
error = %e,
|
||||
"Pipeline failed"
|
||||
);
|
||||
panic!("Pipeline failed: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_extraction_error_logging() {
|
||||
init_logging();
|
||||
|
||||
println!("\n[TEST] Pipeline error handling with logging\n");
|
||||
|
||||
let entity_extractor = Arc::new(WikiLinkFallbackExtractor);
|
||||
let fact_extractor = Arc::new(SimpleFactExtractor);
|
||||
let contradiction_detector = Arc::new(ContradictionHandler::default());
|
||||
|
||||
let pipeline = IngestPipeline::new(
|
||||
entity_extractor,
|
||||
fact_extractor,
|
||||
contradiction_detector,
|
||||
);
|
||||
|
||||
// Episode with problematic content (empty, or only whitespace)
|
||||
let episode = Episode {
|
||||
id: "test-empty".to_string(),
|
||||
project_id: "test-project".to_string(),
|
||||
text: "".to_string(),
|
||||
wiki_links: vec![],
|
||||
};
|
||||
|
||||
tracing::info!(
|
||||
target: "test",
|
||||
episode_id = %episode.id,
|
||||
text_len = episode.text.len(),
|
||||
"Processing empty episode"
|
||||
);
|
||||
|
||||
match pipeline.ingest(&episode).await {
|
||||
Ok(result) => {
|
||||
tracing::info!(
|
||||
target: "test",
|
||||
entities = result.entities.len(),
|
||||
edges = result.edges.len(),
|
||||
"Empty episode processed (no error expected)"
|
||||
);
|
||||
println!("✓ Empty episode handled gracefully");
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!(
|
||||
target: "test",
|
||||
error = %e,
|
||||
"Empty episode caused error"
|
||||
);
|
||||
// Empty is OK for some extractors
|
||||
println!("⚠ Empty episode error (may be expected): {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_multiple_records_error_accumulation() {
|
||||
init_logging();
|
||||
|
||||
println!("\n[TEST] Processing multiple records and logging errors\n");
|
||||
|
||||
let entity_extractor = Arc::new(WikiLinkFallbackExtractor);
|
||||
let fact_extractor = Arc::new(SimpleFactExtractor);
|
||||
let contradiction_detector = Arc::new(ContradictionHandler::default());
|
||||
|
||||
let pipeline = IngestPipeline::new(
|
||||
entity_extractor,
|
||||
fact_extractor,
|
||||
contradiction_detector,
|
||||
);
|
||||
|
||||
let records = vec![
|
||||
("Kubernetes [[Docker]] is a container orchestrator", "wiki/k8s"),
|
||||
("Docker [[Linux]] containers enable microservices", "wiki/docker"),
|
||||
("", "wiki/empty"),
|
||||
("Go [[Concurrency]] is powerful for backend services", "wiki/go"),
|
||||
];
|
||||
|
||||
let mut success_count = 0;
|
||||
let mut error_count = 0;
|
||||
|
||||
for (idx, (text, source)) in records.iter().enumerate() {
|
||||
let episode = Episode {
|
||||
id: format!("record-{}", idx),
|
||||
project_id: "test-project".to_string(),
|
||||
text: text.to_string(),
|
||||
wiki_links: vec![],
|
||||
};
|
||||
|
||||
tracing::info!(
|
||||
target: "test",
|
||||
record_idx = idx,
|
||||
source = source,
|
||||
text_len = text.len(),
|
||||
"Processing record"
|
||||
);
|
||||
|
||||
match pipeline.ingest(&episode).await {
|
||||
Ok(result) => {
|
||||
tracing::debug!(
|
||||
target: "test",
|
||||
record_idx = idx,
|
||||
entities = result.entities.len(),
|
||||
edges = result.edges.len(),
|
||||
"Record succeeded"
|
||||
);
|
||||
println!(" ✓ Record {}: {} entities, {} edges", idx, result.entities.len(), result.edges.len());
|
||||
success_count += 1;
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
target: "test",
|
||||
record_idx = idx,
|
||||
error = %e,
|
||||
source = source,
|
||||
"Record failed"
|
||||
);
|
||||
println!(" ✗ Record {}: {}", idx, e);
|
||||
error_count += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println!("\nSummary: {} success, {} errors", success_count, error_count);
|
||||
|
||||
tracing::info!(
|
||||
target: "test",
|
||||
total_records = records.len(),
|
||||
success = success_count,
|
||||
errors = error_count,
|
||||
"Batch processing complete"
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user