Files
poimen-memory/crates/mem-cli/src/query/temporal_query.rs
T
poimenandrock a88ea918bf
CI / CI (push) Successful in 25m7s
Deploy / Tag & Push Latest (push) Successful in 3m49s
test: production ingest E2E test suite with enhanced logging (#55)
## Summary
Production testing of ingest + embedding pipeline with api-gw integration.

## Root Cause
9 SQL migrations in `crates/mem-store/migrations/` not applied to production database.

Missing tables:
- `memory_entity`
- `memory_edge`
- `memory_edge_temporal`
- Vector embeddings tables
- And 15+ more schema objects

Evidence from logs:
```
WARN: Failed to save entity Docker:
      error returned from database: relation "memory_entity" does not exist
```

## Deliverables
- `test_prod_ingest_real.sh` - Full E2E test against K8s + api-gw
- `apply_migrations.sh` - Manual schema migration (backup)
- `collect_prod_logs.sh` - Pod log collection before/after
- `run_production_test.sh` - Test orchestrator
- `tests/integration_ingest_with_gw.rs` - Integration test
- `tests/unit_ingest_logging.rs` - Unit tests for extraction
- Enhanced logging in `ingest_worker.rs` - Per-record event tracking

## Next Steps
1. Trigger "DB Migration" workflow in Forgejo Actions
2. This applies all 9 migrations from `crates/mem-store/migrations/`
3. Pod restart (automatic)
4. Re-run E2E test - should pass completely

**ETA:** ~15 minutes (3-5 min migrations + 2 min restart + verification)

## How to Test Locally
```bash
./test_prod_ingest_real.sh --verbose
```

Requires:
- kubectl access to poimen namespace
- Port-forwarding to memory-service

---------

Co-authored-by: rock <[email protected]>
Reviewed-on: #55
Co-authored-by: poimen <[email protected]>
2026-09-16 00:10:58 +00:00

270 lines
8.3 KiB
Rust

//! Temporal Query Support: As-Of-Date Queries
//!
//! Query memory state at a specific point in time.
//! Essential for reconstructing historical knowledge state (Zep alignment).
//!
//! CRAP: 12 (Temporal filtering logic)
//! SOLID: Single responsibility (temporal queries)
//! DRY: Reuses query types from mem_core
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
/// Temporal query configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TemporalQueryConfig {
pub enabled: bool,
pub allow_future_dates: bool, // Allow querying past future dates
pub default_to_now: bool, // If no time specified, use NOW()
pub max_lookback_days: Option<i64>, // Limit how far back to query
}
impl Default for TemporalQueryConfig {
fn default() -> Self {
Self {
enabled: true,
allow_future_dates: false,
default_to_now: true,
max_lookback_days: Some(365 * 5), // 5 years
}
}
}
/// Temporal query specification
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TemporalQuery {
/// Base query text
pub query: String,
/// Point in time to query at
pub as_of_time: DateTime<Utc>,
/// Optional: time range for temporal search
pub time_range: Option<(DateTime<Utc>, DateTime<Utc>)>,
}
/// Temporal query result
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TemporalQueryResult {
pub query: String,
pub as_of_time: DateTime<Utc>,
pub num_facts: usize,
pub valid_facts: usize, // Facts valid at as_of_time
pub invalid_facts: usize, // Facts invalid at as_of_time
pub note: String,
}
/// Temporal filter for edges
#[derive(Debug, Clone)]
pub struct TemporalFilter {
config: TemporalQueryConfig,
}
impl TemporalFilter {
pub fn new(config: TemporalQueryConfig) -> Self {
Self { config }
}
/// Validate query time
pub fn validate_query_time(&self, time: DateTime<Utc>) -> Result<(), String> {
if !self.config.enabled {
return Ok(());
}
let now = Utc::now();
// Check if querying future
if !self.config.allow_future_dates && time > now {
return Err(format!(
"Cannot query future time: {} (now: {})",
time, now
));
}
// Check lookback limit
if let Some(max_days) = self.config.max_lookback_days {
let cutoff = now - chrono::Duration::days(max_days);
if time < cutoff {
return Err(format!(
"Query time {} exceeds max lookback of {} days",
time, max_days
));
}
}
Ok(())
}
/// Check if edge is valid at point in time
/// Returns: (is_valid_at_time, is_expired_at_time)
pub fn is_edge_valid_at_time(
&self,
t_valid: Option<DateTime<Utc>>,
t_invalid: Option<DateTime<Utc>>,
query_time: DateTime<Utc>,
) -> (bool, bool) {
if !self.config.enabled {
return (true, false);
}
// Edge is valid if:
// - t_valid is None or <= query_time (became true at/before query time)
// - t_invalid is None or > query_time (didn't become false before query time)
let is_valid = (t_valid.is_none() || t_valid.unwrap() <= query_time)
&& (t_invalid.is_none() || t_invalid.unwrap() > query_time);
let is_expired = t_invalid.is_some() && t_invalid.unwrap() <= query_time;
(is_valid, is_expired)
}
/// Get SQL WHERE clause for temporal filtering
pub fn sql_where_clause(
&self,
query_time: DateTime<Utc>,
table_prefix: &str,
) -> String {
if !self.config.enabled {
return format!("{}.t_expired IS NULL", table_prefix);
}
format!(
"({p}.t_valid IS NULL OR {p}.t_valid <= '{time}') AND \
({p}.t_invalid IS NULL OR {p}.t_invalid > '{time}') AND \
{p}.t_expired IS NULL",
p = table_prefix,
time = query_time.to_rfc3339()
)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_temporal_config_defaults() {
let config = TemporalQueryConfig::default();
assert!(config.enabled);
assert!(!config.allow_future_dates);
assert!(config.default_to_now);
assert_eq!(config.max_lookback_days, Some(365 * 5));
}
#[test]
fn test_validate_query_time_now() {
let config = TemporalQueryConfig::default();
let filter = TemporalFilter::new(config);
let now = Utc::now();
assert!(filter.validate_query_time(now).is_ok());
}
#[test]
fn test_validate_query_time_past() {
let config = TemporalQueryConfig::default();
let filter = TemporalFilter::new(config);
let past = Utc::now() - chrono::Duration::days(30);
assert!(filter.validate_query_time(past).is_ok());
}
#[test]
fn test_validate_query_time_future_disallowed() {
let config = TemporalQueryConfig {
allow_future_dates: false,
..Default::default()
};
let filter = TemporalFilter::new(config);
let future = Utc::now() + chrono::Duration::days(30);
assert!(filter.validate_query_time(future).is_err());
}
#[test]
fn test_validate_query_time_future_allowed() {
let config = TemporalQueryConfig {
allow_future_dates: true,
..Default::default()
};
let filter = TemporalFilter::new(config);
let future = Utc::now() + chrono::Duration::days(30);
assert!(filter.validate_query_time(future).is_ok());
}
#[test]
fn test_is_edge_valid_at_time_current() {
let config = TemporalQueryConfig::default();
let filter = TemporalFilter::new(config);
let now = Utc::now();
let past = now - chrono::Duration::days(10);
// Edge valid from past, still active
let (is_valid, is_expired) = filter.is_edge_valid_at_time(Some(past), None, now);
assert!(is_valid);
assert!(!is_expired);
}
#[test]
fn test_is_edge_valid_at_time_expired() {
let config = TemporalQueryConfig::default();
let filter = TemporalFilter::new(config);
let now = Utc::now();
let past = now - chrono::Duration::days(10);
let future = now + chrono::Duration::days(10);
// Edge valid from past, became invalid before now
let (is_valid, is_expired) = filter.is_edge_valid_at_time(Some(past), Some(now - chrono::Duration::days(1)), now);
assert!(!is_valid);
assert!(is_expired);
}
#[test]
fn test_is_edge_valid_at_time_historical() {
let config = TemporalQueryConfig::default();
let filter = TemporalFilter::new(config);
let now = Utc::now();
let past_30 = now - chrono::Duration::days(30);
let past_10 = now - chrono::Duration::days(10);
let past_5 = now - chrono::Duration::days(5);
// Query at 30 days ago: edge didn't exist yet
let (is_valid, _) = filter.is_edge_valid_at_time(Some(past_10), Some(past_5), past_30);
assert!(!is_valid);
// Query at 8 days ago: edge was valid
let (is_valid, _) = filter.is_edge_valid_at_time(Some(past_10), Some(past_5), now - chrono::Duration::days(8));
assert!(is_valid);
}
#[test]
fn test_sql_where_clause() {
let config = TemporalQueryConfig::default();
let filter = TemporalFilter::new(config);
let now = Utc::now();
let clause = filter.sql_where_clause(now, "e");
assert!(clause.contains("e.t_valid IS NULL OR e.t_valid <="));
assert!(clause.contains("e.t_invalid IS NULL OR e.t_invalid >"));
assert!(clause.contains("e.t_expired IS NULL"));
}
#[test]
fn test_sql_where_clause_disabled() {
let config = TemporalQueryConfig {
enabled: false,
..Default::default()
};
let filter = TemporalFilter::new(config);
let now = Utc::now();
let clause = filter.sql_where_clause(now, "e");
// When disabled, only check t_expired
assert_eq!(clause, "e.t_expired IS NULL");
}
}