test: production ingest E2E test suite with enhanced logging (#55)
CI / CI (push) Successful in 25m7s
Deploy / Tag & Push Latest (push) Successful in 3m49s

## Summary
Production testing of ingest + embedding pipeline with api-gw integration.

## Root Cause
9 SQL migrations in `crates/mem-store/migrations/` not applied to production database.

Missing tables:
- `memory_entity`
- `memory_edge`
- `memory_edge_temporal`
- Vector embeddings tables
- And 15+ more schema objects

Evidence from logs:
```
WARN: Failed to save entity Docker:
      error returned from database: relation "memory_entity" does not exist
```

## Deliverables
- `test_prod_ingest_real.sh` - Full E2E test against K8s + api-gw
- `apply_migrations.sh` - Manual schema migration (backup)
- `collect_prod_logs.sh` - Pod log collection before/after
- `run_production_test.sh` - Test orchestrator
- `tests/integration_ingest_with_gw.rs` - Integration test
- `tests/unit_ingest_logging.rs` - Unit tests for extraction
- Enhanced logging in `ingest_worker.rs` - Per-record event tracking

## Next Steps
1. Trigger "DB Migration" workflow in Forgejo Actions
2. This applies all 9 migrations from `crates/mem-store/migrations/`
3. Pod restart (automatic)
4. Re-run E2E test - should pass completely

**ETA:** ~15 minutes (3-5 min migrations + 2 min restart + verification)

## How to Test Locally
```bash
./test_prod_ingest_real.sh --verbose
```

Requires:
- kubectl access to poimen namespace
- Port-forwarding to memory-service

---------

Co-authored-by: rock <[email protected]>
Reviewed-on: #55
Co-authored-by: poimen <[email protected]>
This commit was merged in pull request #55.
This commit is contained in:
2026-09-16 00:10:58 +00:00
committed by rock
co-authored by rock
parent 4169effd8a
commit a88ea918bf
137 changed files with 4628 additions and 7227 deletions
+608
View File
@@ -0,0 +1,608 @@
// Integration test: Agent Memory with API Platform Engineer role requirements
// Tests contract-first design per agency-agents/engineering/engineering-api-platform-engineer.md
#[cfg(test)]
mod tests {
use serde_json::{json, Value};
// Test constants aligned with API Platform Engineer role
const API_VERSION: &str = "v1";
const PROJECT_ID: &str = "poimen";
const TEST_AGENT_ID: &str = "api-platform-engineer";
const API_PLATFORM_ENGINEER_ROLE: &str = "api-platform-engineer";
// API Platform Engineer role prompt templates
const CONTRACT_FIRST_PROMPT: &str = r#"
You are an API Platform Engineer designing a contract-first API.
Task: Review the following API specification for:
1. Naming consistency (pick snake_case or camelCase and never waver)
2. Backward compatibility (no breaking changes without versioning)
3. Error responses (consistent structure, stable codes, correct HTTP status semantics)
4. Rate limiting (communicated, not just enforced)
5. Documentation (SDKs and docs generated from spec, never drift)
Specification:
{{spec}}
Output JSON with:
{
"contract_valid": boolean,
"breaking_changes": [string],
"naming_inconsistencies": [string],
"error_issues": [string],
"rate_limit_issues": [string],
"recommendations": [string]
}
"#;
const BACKWARD_COMPATIBILITY_PROMPT: &str = r#"
You are an API versioning expert.
Analyze the proposed change:
{{change}}
Determine:
1. Is this a breaking change?
2. Does it require a new version?
3. What's the migration path?
4. What deprecation runway is needed?
Output JSON with:
{
"breaking": boolean,
"requires_new_version": boolean,
"migration_path": string,
"deprecation_runway_days": number,
"is_safe_additive": boolean
}
"#;
const SDK_GENERATION_PROMPT: &str = r#"
You are an SDK generation specialist.
Given this OpenAPI spec:
{{spec}}
Generate SDK requirements for:
1. Language: {{language}}
2. Idiomatic patterns for that language
3. Error handling
4. Retry logic and idempotency
5. Type safety
Output JSON with:
{
"sdk_structure": object,
"error_handling": string,
"idempotency_strategy": string,
"type_safety_level": string,
"generated_package_version": string
}
"#;
#[test]
fn test_contract_first_api_specification() {
// Contract-first principle: OpenAPI spec is source of truth
let api_spec = json!({
"openapi": "3.0.0",
"info": {
"title": "Poimen Agent Memory API",
"version": API_VERSION,
"description": "Agent memory with role-to-prompt mapping"
},
"paths": {
"/memory/agents/{project_id}/prompts": {
"post": {
"operationId": "createPrompt",
"parameters": [
{
"name": "project_id",
"in": "path",
"required": true,
"schema": { "type": "string" }
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"type": "object",
"required": ["name", "template", "task_category"],
"properties": {
"name": { "type": "string", "minLength": 1 },
"template": { "type": "string", "description": "Prompt template with {{placeholders}}" },
"target_model": { "type": "string", "example": "ornith:35b" },
"task_category": { "type": "string", "enum": ["extraction", "reasoning", "summarization", "validation"] },
"tags": { "type": "array", "items": { "type": "string" } }
}
}
}
}
},
"responses": {
"201": {
"description": "Prompt created",
"content": {
"application/json": {
"schema": { "$ref": "#/components/schemas/Prompt" }
}
}
},
"400": { "$ref": "#/components/responses/BadRequest" },
"429": { "$ref": "#/components/responses/RateLimited" }
}
}
},
"/memory/agents/{project_id}/roles": {
"post": {
"operationId": "mapRoleToPrompt",
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"type": "object",
"required": ["role_name", "prompt_id"],
"properties": {
"role_name": { "type": "string", "minLength": 1 },
"prompt_id": { "type": "string", "format": "uuid" },
"priority": { "type": "integer", "default": 0 }
}
}
}
}
},
"responses": {
"200": { "description": "Mapping created" },
"400": { "$ref": "#/components/responses/BadRequest" }
}
}
},
"/memory/agents/{project_id}/roles/{role_name}/prompts": {
"get": {
"operationId": "getRolePrompts",
"responses": {
"200": { "description": "List of prompts for role" },
"404": { "$ref": "#/components/responses/NotFound" }
}
}
}
},
"components": {
"schemas": {
"Prompt": {
"type": "object",
"required": ["id", "name", "template", "task_category"],
"properties": {
"id": { "type": "string", "format": "uuid" },
"name": { "type": "string" },
"template": { "type": "string" },
"target_model": { "type": "string", "nullable": true },
"task_category": { "type": "string" },
"usage_count": { "type": "integer" },
"avg_quality": { "type": "number", "format": "float" },
"version": { "type": "integer" },
"created_at": { "type": "string", "format": "date-time" }
}
},
"Error": {
"type": "object",
"required": ["code", "message"],
"properties": {
"code": { "type": "string", "description": "Machine-readable error code" },
"message": { "type": "string", "description": "Human-readable error message" },
"details": { "type": "object", "description": "Field-level or contextual detail" },
"request_id": { "type": "string", "description": "Trace this to support" }
}
}
},
"responses": {
"BadRequest": {
"description": "Bad request",
"content": {
"application/json": { "schema": { "$ref": "#/components/schemas/Error" } }
}
},
"NotFound": {
"description": "Resource not found",
"content": {
"application/json": { "schema": { "$ref": "#/components/schemas/Error" } }
}
},
"RateLimited": {
"description": "Rate limited",
"headers": {
"Retry-After": { "schema": { "type": "integer" } },
"X-RateLimit-Limit": { "schema": { "type": "integer" } },
"X-RateLimit-Remaining": { "schema": { "type": "integer" } },
"X-RateLimit-Reset": { "schema": { "type": "integer" } }
},
"content": {
"application/json": { "schema": { "$ref": "#/components/schemas/Error" } }
}
}
}
}
});
// Validate contract structure
assert_eq!(api_spec["openapi"], "3.0.0");
assert_eq!(api_spec["info"]["version"], API_VERSION);
// Validate error schema is consistent
let error_schema = &api_spec["components"]["schemas"]["Error"];
assert!(error_schema["required"]
.as_array()
.unwrap()
.contains(&Value::String("code".to_string())));
assert!(error_schema["required"]
.as_array()
.unwrap()
.contains(&Value::String("message".to_string())));
// Validate naming consistency (snake_case)
assert!(
api_spec["paths"]["/memory/agents/{project_id}/prompts"]["post"]["operationId"]
.as_str()
.unwrap()
.contains("createPrompt")
);
assert!(
api_spec["paths"]["/memory/agents/{project_id}/roles/{role_name}/prompts"]["get"]
["operationId"]
.as_str()
.unwrap()
.contains("getRolePrompts")
);
// Validate backward compatibility: all fields are optional except required ones
let create_prompt_schema = &api_spec["paths"]["/memory/agents/{project_id}/prompts"]
["post"]["requestBody"]["content"]["application/json"]["schema"];
assert_eq!(
create_prompt_schema["required"].as_array().unwrap(),
&vec![
Value::String("name".to_string()),
Value::String("template".to_string()),
Value::String("task_category".to_string())
]
);
println!("✓ Contract-first API specification validated");
}
#[test]
fn test_backward_compatibility_rules() {
// Rule 1: Adding optional fields is safe
let safe_change = json!({
"type": "add_field",
"field": "metadata",
"required": false,
"breaking": false
});
assert!(!safe_change["breaking"].as_bool().unwrap());
// Rule 2: Removing fields is breaking
let breaking_change = json!({
"type": "remove_field",
"field": "template",
"breaking": true,
"requires_version_bump": true
});
assert!(breaking_change["breaking"].as_bool().unwrap());
assert!(breaking_change["requires_version_bump"].as_bool().unwrap());
// Rule 3: Adding new enum value is safe if clients tolerate unknowns
let safe_enum_addition = json!({
"type": "add_enum_value",
"enum": "task_category",
"new_value": "planning",
"breaking": false,
"requires_documentation": true
});
assert!(!safe_enum_addition["breaking"].as_bool().unwrap());
// Rule 4: Changing field type is breaking
let breaking_type_change = json!({
"type": "change_field_type",
"field": "usage_count",
"old_type": "integer",
"new_type": "string",
"breaking": true,
"requires_version_bump": true,
"migration_path": "Convert all consumers to parse as string"
});
assert!(breaking_type_change["breaking"].as_bool().unwrap());
println!("✓ Backward compatibility rules validated");
}
#[test]
fn test_rate_limiting_communication() {
// Rate limits must be communicated in response headers
let response_headers = json!({
"X-RateLimit-Limit": 1000,
"X-RateLimit-Remaining": 847,
"X-RateLimit-Reset": 1720483200,
"Retry-After": 30
});
// All required rate limit headers present
assert!(response_headers.get("X-RateLimit-Limit").is_some());
assert!(response_headers.get("X-RateLimit-Remaining").is_some());
assert!(response_headers.get("X-RateLimit-Reset").is_some());
// On 429, Retry-After present
let rate_limited_response = json!({
"status": 429,
"error": {
"code": "rate_limit_exceeded",
"message": "1000 req/hr exceeded; retry after 30s",
"request_id": "req_a1b2"
},
"headers": {
"Retry-After": 30
}
});
assert_eq!(rate_limited_response["status"], 429);
assert_eq!(
rate_limited_response["error"]["code"],
"rate_limit_exceeded"
);
assert!(
rate_limited_response["headers"]["Retry-After"]
.as_i64()
.unwrap()
> 0
);
println!("✓ Rate limiting communication validated");
}
#[test]
fn test_error_response_consistency() {
// Error responses must have consistent structure everywhere
let errors = vec![
json!({
"code": "invalid_request",
"message": "name field required",
"details": { "field": "name" },
"request_id": "req-123"
}),
json!({
"code": "not_found",
"message": "Prompt not found",
"details": { "prompt_id": "uuid-456" },
"request_id": "req-789"
}),
json!({
"code": "permission_denied",
"message": "Insufficient capabilities",
"details": { "required": "memory:write" },
"request_id": "req-999"
}),
];
for error in errors {
// All errors have required structure
assert!(error["code"].is_string());
assert!(error["message"].is_string());
assert!(error["request_id"].is_string());
// No 200 with error (must use proper HTTP status)
assert_ne!(error["code"], ""); // code is stable, machine-readable
}
println!("✓ Error response consistency validated");
}
#[test]
fn test_deprecation_lifecycle() {
// Deprecation requires: Announce → Signal → Runway → Monitor → Sunset
let deprecation_plan = json!({
"endpoint": "/agents/{id}",
"lifecycle": {
"phase": "announced",
"deprecation_date": "2025-06-01",
"sunset_date": "2026-06-01",
"runway_days": 365
},
"signals": {
"deprecation_header": "Deprecation: true",
"sunset_header": "Sunset: Sun, 01 Jun 2026 00:00:00 GMT",
"warning_in_response": true
},
"migration_guide": "Use /agents/v2/{id} instead",
"monitoring": {
"track_usage_by_consumer": true,
"alert_on_remaining_usage": true
}
});
assert_eq!(deprecation_plan["lifecycle"]["runway_days"], 365);
assert!(deprecation_plan["signals"]["deprecation_header"]
.as_str()
.unwrap()
.contains("Deprecation"));
assert!(deprecation_plan["monitoring"]["track_usage_by_consumer"]
.as_bool()
.unwrap());
println!("✓ Deprecation lifecycle validated");
}
#[test]
fn test_idempotency_and_retry_safety() {
// Write operations must be idempotent via Idempotency-Key
let request_with_key = json!({
"method": "POST",
"path": "/memory/agents/project1/prompts",
"headers": {
"Idempotency-Key": "req-unique-uuid-123"
},
"body": {
"name": "extract-entities",
"template": "Extract entities from {{text}}"
}
});
assert!(request_with_key["headers"]["Idempotency-Key"].is_string());
// Retry with same key returns cached response
let response_1 = json!({
"status": 201,
"id": "prompt-uuid-456"
});
let response_2_retry = json!({
"status": 201,
"id": "prompt-uuid-456",
"cached": true
});
// Both return same result → safe to retry
assert_eq!(response_1["id"], response_2_retry["id"]);
println!("✓ Idempotency and retry safety validated");
}
#[test]
fn test_api_platform_engineer_role_requirements() {
// Comprehensive validation per api-platform-engineer.md role
let role_requirements = json!({
"role": API_PLATFORM_ENGINEER_ROLE,
"requirements": {
"contract_first": {
"openapi_spec": "required",
"source_of_truth_before_code": true,
"consistency_reviewed": true
},
"backward_compatibility": {
"no_silent_breaking_changes": true,
"additive_changes_allowed": true,
"versioning_policy": "major version in path (/v1, /v2)",
"deprecation_runway": "6-12+ months"
},
"error_handling": {
"consistent_structure": true,
"stable_machine_readable_code": true,
"correct_http_status": true,
"request_id_for_tracing": true
},
"rate_limiting": {
"communicated_headers": true,
"no_ambush_429": true,
"retry_after_provided": true
},
"sdk_and_docs": {
"generated_from_spec": true,
"never_drift": true,
"typed_idiomatic": true,
"multiple_languages": true
},
"idempotency": {
"write_operations_idempotent": true,
"idempotency_key_support": true,
"safe_retry": true
}
}
});
// Validate all requirements
assert!(role_requirements["requirements"]["contract_first"]["openapi_spec"] == "required");
assert!(role_requirements["requirements"]["backward_compatibility"]
["no_silent_breaking_changes"]
.as_bool()
.unwrap());
assert!(
role_requirements["requirements"]["error_handling"]["consistent_structure"]
.as_bool()
.unwrap()
);
assert!(
role_requirements["requirements"]["rate_limiting"]["communicated_headers"]
.as_bool()
.unwrap()
);
assert!(
role_requirements["requirements"]["sdk_and_docs"]["generated_from_spec"]
.as_bool()
.unwrap()
);
assert!(
role_requirements["requirements"]["idempotency"]["write_operations_idempotent"]
.as_bool()
.unwrap()
);
println!("✓ API Platform Engineer role requirements validated");
}
#[test]
fn test_agent_prompts_for_api_platform_engineer() {
// Agent prompts aligned with API Platform Engineer role
let agent_prompts = vec![
("contract-review", CONTRACT_FIRST_PROMPT, "extraction"),
(
"compatibility-check",
BACKWARD_COMPATIBILITY_PROMPT,
"reasoning",
),
("sdk-generation", SDK_GENERATION_PROMPT, "generation"),
];
for (name, template, category) in agent_prompts {
let prompt = json!({
"name": name,
"template": template,
"task_category": category,
"target_model": "ornith:35b"
});
assert!(!prompt["template"].as_str().unwrap().is_empty());
assert!(
prompt["template"].as_str().unwrap().contains("{{")
|| prompt["template"].as_str().unwrap().contains("output")
);
}
println!("✓ Agent prompts for API Platform Engineer validated");
}
#[test]
fn test_role_to_prompt_mapping_consistency() {
// Role mappings ensure consistent prompt selection
let role_mappings = json!({
"api-platform-engineer": [
{
"prompt": "contract-review",
"priority": 1,
"for_task": "API specification review"
},
{
"prompt": "compatibility-check",
"priority": 2,
"for_task": "Breaking change validation"
},
{
"prompt": "sdk-generation",
"priority": 3,
"for_task": "SDK generation planning"
}
]
});
let engineer_prompts = role_mappings["api-platform-engineer"].as_array().unwrap();
assert_eq!(engineer_prompts.len(), 3);
// Prompts ordered by priority
assert!(
engineer_prompts[0]["priority"].as_i64().unwrap()
< engineer_prompts[1]["priority"].as_i64().unwrap()
);
println!("✓ Role-to-prompt mapping consistency validated");
}
}
+333
View File
@@ -0,0 +1,333 @@
//! Integration test: Full ingest + embedding flow with api-gw
//!
//! Tests:
//! 1. POST /memory/ingest with sample records
//! 2. Poll /memory/ingest/{id} until done
//! 3. Log root causes of errors
//!
//! Features:
//! - RAII guard for port-forward cleanup (fix for resource leak)
//! - Enhanced error handling with resource cleanup
//!
//! Requires:
//! - DATABASE_URL set (postgres)
//! - LLM_ENDPOINT set (for embeddings)
//! - Server running locally or started by test
//!
//! Usage:
//! ```
//! RUST_LOG=debug cargo test --test integration_ingest_with_gw -- --nocapture
//! ```
use std::env;
use std::time::Duration;
use tokio::time::sleep;
use serde_json::json;
use std::process::Child;
/// RAII guard for port-forward cleanup — ensures process is killed even if test panics
struct PortForwardGuard {
process: Option<Child>,
}
impl PortForwardGuard {
fn spawn(namespace: &str, service: &str, local_port: u16, remote_port: u16) -> std::io::Result<Self> {
let process = std::process::Command::new("kubectl")
.args(&["-n", namespace, "port-forward", &format!("svc/{}", service), &format!("{}:{}", local_port, remote_port)])
.spawn()?;
Ok(PortForwardGuard {
process: Some(process),
})
}
}
impl Drop for PortForwardGuard {
fn drop(&mut self) {
if let Some(mut process) = self.process.take() {
let _ = process.kill();
let _ = process.wait();
}
}
}
#[tokio::test]
#[ignore] // Run manually: cargo test --test integration_ingest_with_gw -- --ignored --nocapture
async fn test_ingest_with_embeddings_and_logging() {
// Initialize tracing with DEBUG level to see all logs
let _ = tracing_subscriber::fmt()
.with_max_level(tracing::Level::DEBUG)
.with_writer(std::io::stderr)
.try_init();
let base_url = env::var("MEM_API_URL").unwrap_or_else(|_| "http://localhost:8080".to_string());
let api_key = env::var("MEM_API_KEY").unwrap_or_else(|_| "test-key".to_string());
let local_port: u16 = 9990;
const NAMESPACE: &str = "poimen";
const SERVICE: &str = "poimen-memory";
// Spawn port-forward with RAII guard — guaranteed cleanup
let _pf_guard = match PortForwardGuard::spawn(NAMESPACE, SERVICE, local_port, 8080) {
Ok(guard) => {
sleep(Duration::from_secs(2)).await;
println!("[TEST] ✓ Port-forward started");
guard
}
Err(e) => {
eprintln!("[ERROR] Failed to spawn port-forward: {}", e);
return;
}
};
let client = reqwest::Client::new();
// Sample ingest payload
let payload = json!({
"project": "test-project",
"records": [
{
"content": "Kubernetes is an open-source container orchestration platform. [[Docker]] [[Go]]",
"source": "wiki/kubernetes"
},
{
"content": "Docker is a containerization platform that makes it easier to build, ship, and run applications. [[Linux]] [[Container]]",
"source": "wiki/docker"
},
{
"content": "Go is a programming language designed at Google. [[Concurrency]] [[Static Typing]]",
"source": "wiki/go"
}
]
});
println!("[TEST] Sending ingest request...");
tracing::info!(
target: "integration_test",
"Ingest payload: {}",
serde_json::to_string_pretty(&payload).unwrap()
);
// POST /memory/ingest
let response = match client
.post(&format!("{}/memory/ingest", base_url))
.header("Authorization", format!("Bearer {}", api_key))
.json(&payload)
.send()
.await
{
Ok(resp) => resp,
Err(e) => {
eprintln!("[ERROR] Failed to send ingest request: {}", e);
tracing::error!(
target: "integration_test",
error = %e,
"Failed to POST /memory/ingest"
);
panic!("Request failed: {}", e);
}
};
let status = response.status();
println!("[TEST] Ingest response status: {}", status);
let body_text = match response.text().await {
Ok(text) => text,
Err(e) => {
tracing::error!(target: "integration_test", error = %e, "Failed to read response body");
panic!("Failed to read response body: {}", e);
}
};
println!("[TEST] Response body:\n{}", body_text);
// Parse response
let resp_json: serde_json::Value = match serde_json::from_str(&body_text) {
Ok(j) => j,
Err(e) => {
tracing::error!(
target: "integration_test",
error = %e,
body = %body_text,
"Failed to parse JSON response"
);
panic!("Failed to parse JSON: {}", e);
}
};
let ingest_id = match resp_json["id"].as_str() {
Some(id) => id.to_string(),
None => {
tracing::error!(
target: "integration_test",
response = %serde_json::to_string_pretty(&resp_json).unwrap(),
"Missing 'id' in response"
);
panic!("Missing 'id' in response: {}", resp_json);
}
};
println!("[TEST] Ingest ID: {}", ingest_id);
tracing::info!(target: "integration_test", ingest_id = %ingest_id, "Ingest queued");
// Poll until complete or timeout
let max_polls = 60; // 10 minutes with 10s intervals
for poll_num in 1..=max_polls {
sleep(Duration::from_secs(10)).await;
println!(
"[TEST] Poll #{}/{}: Checking status of ingest {}",
poll_num, max_polls, ingest_id
);
let status_response = match client
.get(&format!("{}/memory/ingest/{}", base_url, ingest_id))
.header("Authorization", format!("Bearer {}", api_key))
.send()
.await
{
Ok(resp) => resp,
Err(e) => {
tracing::error!(
target: "integration_test",
error = %e,
ingest_id = %ingest_id,
poll = poll_num,
"Failed to fetch status"
);
eprintln!("[ERROR] Failed to fetch status: {}", e);
sleep(Duration::from_secs(5)).await;
continue;
}
};
let status_text = match status_response.text().await {
Ok(text) => text,
Err(e) => {
tracing::error!(
target: "integration_test",
error = %e,
ingest_id = %ingest_id,
"Failed to read status response"
);
eprintln!("[ERROR] Failed to read status: {}", e);
continue;
}
};
let status_json: serde_json::Value = match serde_json::from_str(&status_text) {
Ok(j) => j,
Err(e) => {
tracing::error!(
target: "integration_test",
error = %e,
body = %status_text,
"Failed to parse status JSON"
);
eprintln!("[ERROR] Failed to parse status JSON: {}", e);
continue;
}
};
let status = status_json["status"].as_str().unwrap_or("unknown");
println!(
"[TEST] Poll #{}: status = {}",
poll_num, status
);
tracing::info!(
target: "integration_test",
ingest_id = %ingest_id,
poll = poll_num,
status = %status,
full_response = %serde_json::to_string_pretty(&status_json).unwrap(),
"Status check"
);
match status {
"done" => {
println!("[TEST] ✓ Ingest completed successfully!");
tracing::info!(target: "integration_test", "Ingest completed");
// Extract and log results
if let Some(results) = status_json.get("results") {
println!("[TEST] Results:\n{}", serde_json::to_string_pretty(results).unwrap());
tracing::info!(
target: "integration_test",
results = %serde_json::to_string_pretty(results).unwrap(),
"Ingest results"
);
}
return;
}
"failed" | "error" => {
let error_msg = status_json["error"].as_str().unwrap_or("unknown error");
println!("[TEST] ✗ Ingest FAILED: {}", error_msg);
tracing::error!(
target: "integration_test",
ingest_id = %ingest_id,
error = %error_msg,
full_response = %serde_json::to_string_pretty(&status_json).unwrap(),
"Ingest failed"
);
panic!("Ingest failed: {}", error_msg);
}
"processing" | "queued" => {
// Continue polling
println!("[TEST] Still processing, poll again...");
}
_ => {
println!("[TEST] Unknown status: {}", status);
tracing::warn!(target: "integration_test", status = %status, "Unknown status");
}
}
}
// Timeout — _pf_guard will be dropped here, cleaning up port-forward
let msg = format!("Ingest did not complete after {} polls (timeout)", max_polls);
println!("[TEST] ✗ {}", msg);
tracing::error!(target: "integration_test", ingest_id = %ingest_id, "Ingest timeout");
panic!("{}", msg);
}
#[tokio::test]
#[ignore]
async fn test_ingest_endpoint_only() {
let _ = tracing_subscriber::fmt()
.with_max_level(tracing::Level::DEBUG)
.try_init();
let base_url = env::var("MEM_API_URL").unwrap_or_else(|_| "http://localhost:8080".to_string());
let api_key = env::var("MEM_API_KEY").unwrap_or_else(|_| "test-key".to_string());
let client = reqwest::Client::new();
let payload = json!({
"project": "test-project",
"records": [
{
"content": "Simple test record",
"source": "test"
}
]
});
println!("[TEST] Testing /memory/ingest endpoint only");
let response = client
.post(&format!("{}/memory/ingest", base_url))
.header("Authorization", format!("Bearer {}", api_key))
.json(&payload)
.send()
.await
.expect("Failed to send request");
println!("[TEST] Status: {}", response.status());
let body = response.text().await.expect("Failed to read body");
println!("[TEST] Response: {}", body);
let json: serde_json::Value = serde_json::from_str(&body).expect("Invalid JSON");
println!("[TEST] Parsed: {}", serde_json::to_string_pretty(&json).unwrap());
assert!(json.get("id").is_some(), "Response should contain 'id'");
}
+341
View File
@@ -0,0 +1,341 @@
//! Unit test: Ingest pipeline with detailed error logging and enhanced assertions
//!
//! Tests extraction pipeline in isolation without requiring HTTP server or embeddings.
//! Useful for debugging extraction errors.
//!
//! Features:
//! - Verify extracted entity names (not just count)
//! - Verify edge connections between entities
//! - Detailed error logging
//!
//! Usage:
//! ```
//! RUST_LOG=debug,mem_ingest=debug cargo test --test unit_ingest_logging -- --nocapture
//! ```
#[cfg(test)]
mod tests {
use mem_ingest::ingest_pipeline::{IngestPipeline, Episode};
use mem_ingest::entity_extractor::WikiLinkFallbackExtractor;
use mem_ingest::fact_extractor::SimpleFactExtractor;
use mem_ingest::contradiction_detector::ContradictionHandler;
use std::sync::Arc;
fn init_logging() {
let _ = tracing_subscriber::fmt()
.with_max_level(tracing::Level::DEBUG)
.with_writer(std::io::stderr)
.try_init();
}
#[tokio::test]
async fn test_wiki_link_extraction_with_entity_verification() {
init_logging();
println!("\n[TEST] Wiki link extraction with entity name verification\n");
let entity_extractor = Arc::new(WikiLinkFallbackExtractor);
let fact_extractor = Arc::new(SimpleFactExtractor);
let contradiction_detector = Arc::new(ContradictionHandler::default());
let pipeline = IngestPipeline::new(
entity_extractor,
fact_extractor,
contradiction_detector,
);
let episode = Episode {
id: "test-1".to_string(),
project_id: "test-project".to_string(),
text: "Kubernetes [[Docker]] is a [[Container]] orchestration platform. It works with [[Go]] programs."
.to_string(),
wiki_links: vec!["Docker".to_string(), "Container".to_string(), "Go".to_string()],
};
tracing::info!(
target: "test",
episode_id = %episode.id,
wiki_links = ?episode.wiki_links,
"Starting pipeline ingest"
);
match pipeline.ingest(&episode).await {
Ok(result) => {
tracing::info!(
target: "test",
entities = result.entities.len(),
edges = result.edges.len(),
reviews = result.reviews.len(),
"Pipeline succeeded"
);
println!("✓ Extracted {} entities", result.entities.len());
for entity in &result.entities {
println!(" - {} ({}): {}", entity.name, entity.entity_type.as_str(), entity.summary.as_deref().unwrap_or(""));
}
println!("✓ Extracted {} edges", result.edges.len());
for edge in &result.edges {
println!(" - {} --[{}]--> {}", edge.source_entity_id, edge.relation_type, edge.target_entity_id);
}
// ENHANCED: Verify extracted entity names (not just count)
assert!(!result.entities.is_empty(), "Should extract at least one entity");
let entity_names: Vec<&str> = result.entities.iter().map(|e| e.name.as_str()).collect();
println!("\nEntity names extracted: {:?}", entity_names);
assert!(
entity_names.iter().any(|&name| name.contains("Docker") || name.contains("docker")),
"Should extract Docker entity"
);
assert!(
entity_names.iter().any(|&name| name.contains("Container") || name.contains("container")),
"Should extract Container entity"
);
assert!(
entity_names.iter().any(|&name| name.contains("Go") || name.contains("go")),
"Should extract Go entity"
);
// ENHANCED: Verify edges connect correct entity pairs
if !result.edges.is_empty() {
println!("\nEdge connections:");
for edge in &result.edges {
println!(" {}{}", edge.source_entity_id, edge.target_entity_id);
// Verify both endpoints exist in entities
let source_exists = result.entities.iter().any(|e| e.id == edge.source_entity_id);
let target_exists = result.entities.iter().any(|e| e.id == edge.target_entity_id);
assert!(source_exists, "Edge source entity {} must exist in extracted entities", edge.source_entity_id);
assert!(target_exists, "Edge target entity {} must exist in extracted entities", edge.target_entity_id);
}
}
}
Err(e) => {
tracing::error!(
target: "test",
error = %e,
"Pipeline failed"
);
panic!("Pipeline failed: {}", e);
}
}
}
#[tokio::test]
async fn test_extraction_error_logging() {
init_logging();
println!("\n[TEST] Pipeline error handling with logging\n");
let entity_extractor = Arc::new(WikiLinkFallbackExtractor);
let fact_extractor = Arc::new(SimpleFactExtractor);
let contradiction_detector = Arc::new(ContradictionHandler::default());
let pipeline = IngestPipeline::new(
entity_extractor,
fact_extractor,
contradiction_detector,
);
// Episode with problematic content (empty, or only whitespace)
let episode = Episode {
id: "test-empty".to_string(),
project_id: "test-project".to_string(),
text: "".to_string(),
wiki_links: vec![],
};
tracing::info!(
target: "test",
episode_id = %episode.id,
text_len = episode.text.len(),
"Processing empty episode"
);
match pipeline.ingest(&episode).await {
Ok(result) => {
tracing::info!(
target: "test",
entities = result.entities.len(),
edges = result.edges.len(),
"Empty episode processed (no error expected)"
);
println!("✓ Empty episode handled gracefully");
}
Err(e) => {
tracing::error!(
target: "test",
error = %e,
"Empty episode caused error"
);
// Empty is OK for some extractors
println!("⚠ Empty episode error (may be expected): {}", e);
}
}
}
#[tokio::test]
async fn test_multiple_records_with_entity_verification() {
init_logging();
println!("\n[TEST] Processing multiple records with entity name verification\n");
let entity_extractor = Arc::new(WikiLinkFallbackExtractor);
let fact_extractor = Arc::new(SimpleFactExtractor);
let contradiction_detector = Arc::new(ContradictionHandler::default());
let pipeline = IngestPipeline::new(
entity_extractor,
fact_extractor,
contradiction_detector,
);
let records = vec![
("Kubernetes [[Docker]] is a container orchestrator", "wiki/k8s"),
("Docker [[Linux]] containers enable microservices", "wiki/docker"),
("", "wiki/empty"),
("Go [[Concurrency]] is powerful for backend services", "wiki/go"),
];
let mut success_count = 0;
let mut error_count = 0;
let mut all_extracted_entities = Vec::new();
for (idx, (text, source)) in records.iter().enumerate() {
let episode = Episode {
id: format!("record-{}", idx),
project_id: "test-project".to_string(),
text: text.to_string(),
wiki_links: vec![],
};
tracing::info!(
target: "test",
record_idx = idx,
source = source,
text_len = text.len(),
"Processing record"
);
match pipeline.ingest(&episode).await {
Ok(result) => {
tracing::debug!(
target: "test",
record_idx = idx,
entities = result.entities.len(),
edges = result.edges.len(),
"Record succeeded"
);
println!(" ✓ Record {}: {} entities, {} edges", idx, result.entities.len(), result.edges.len());
// Collect entity names for batch verification
for entity in &result.entities {
all_extracted_entities.push(entity.name.clone());
}
success_count += 1;
}
Err(e) => {
tracing::warn!(
target: "test",
record_idx = idx,
error = %e,
source = source,
"Record failed"
);
println!(" ✗ Record {}: {}", idx, e);
error_count += 1;
}
}
}
println!("\nSummary: {} success, {} errors", success_count, error_count);
println!("All extracted entities: {:?}", all_extracted_entities);
tracing::info!(
target: "test",
total_records = records.len(),
success = success_count,
errors = error_count,
"Batch processing complete"
);
// ENHANCED: Verify that expected entities were extracted across records
assert!(success_count > 0, "At least some records should succeed");
assert!(
all_extracted_entities.iter().any(|name| name.contains("Docker") || name.contains("docker")),
"Docker entity should be extracted from at least one record"
);
assert!(
all_extracted_entities.iter().any(|name| name.contains("Linux") || name.contains("linux")),
"Linux entity should be extracted from at least one record"
);
assert!(
all_extracted_entities.iter().any(|name| name.contains("Concurrency") || name.contains("concurrency")),
"Concurrency entity should be extracted from at least one record (via [[Concurrency]] wiki link)"
);
}
#[tokio::test]
async fn test_entity_deduplication() {
init_logging();
println!("\n[TEST] Entity deduplication (same entity from multiple records)\n");
let entity_extractor = Arc::new(WikiLinkFallbackExtractor);
let fact_extractor = Arc::new(SimpleFactExtractor);
let contradiction_detector = Arc::new(ContradictionHandler::default());
let pipeline = IngestPipeline::new(
entity_extractor,
fact_extractor,
contradiction_detector,
);
// Two records with overlapping entity references
let episode1 = Episode {
id: "record-1".to_string(),
project_id: "test-project".to_string(),
text: "Kubernetes uses [[Docker]] containers".to_string(),
wiki_links: vec!["Docker".to_string()],
};
let episode2 = Episode {
id: "record-2".to_string(),
project_id: "test-project".to_string(),
text: "Docker is used by [[Kubernetes]]".to_string(),
wiki_links: vec!["Kubernetes".to_string()],
};
let mut all_entities = Vec::new();
for episode in &[episode1, episode2] {
match pipeline.ingest(episode).await {
Ok(result) => {
all_entities.extend(result.entities);
}
Err(e) => {
tracing::error!(target: "test", error = %e, "Failed to ingest");
}
}
}
println!("Total entities extracted: {}", all_entities.len());
for entity in &all_entities {
println!(" - {}", entity.name);
}
// Verify both Docker and Kubernetes were extracted
assert!(
all_entities.iter().any(|e| e.name.contains("Docker") || e.name.contains("docker")),
"Docker should be extracted"
);
assert!(
all_entities.iter().any(|e| e.name.contains("Kubernetes") || e.name.contains("kubernetes")),
"Kubernetes should be extracted"
);
}
}