poimenandrock
a88ea918bf
## Summary
Production testing of ingest + embedding pipeline with api-gw integration.
## Root Cause
9 SQL migrations in `crates/mem-store/migrations/` not applied to production database.
Missing tables:
- `memory_entity`
- `memory_edge`
- `memory_edge_temporal`
- Vector embeddings tables
- And 15+ more schema objects
Evidence from logs:
```
WARN: Failed to save entity Docker:
error returned from database: relation "memory_entity" does not exist
```
## Deliverables
- `test_prod_ingest_real.sh` - Full E2E test against K8s + api-gw
- `apply_migrations.sh` - Manual schema migration (backup)
- `collect_prod_logs.sh` - Pod log collection before/after
- `run_production_test.sh` - Test orchestrator
- `tests/integration_ingest_with_gw.rs` - Integration test
- `tests/unit_ingest_logging.rs` - Unit tests for extraction
- Enhanced logging in `ingest_worker.rs` - Per-record event tracking
## Next Steps
1. Trigger "DB Migration" workflow in Forgejo Actions
2. This applies all 9 migrations from `crates/mem-store/migrations/`
3. Pod restart (automatic)
4. Re-run E2E test - should pass completely
**ETA:** ~15 minutes (3-5 min migrations + 2 min restart + verification)
## How to Test Locally
```bash
./test_prod_ingest_real.sh --verbose
```
Requires:
- kubectl access to poimen namespace
- Port-forwarding to memory-service
---------
Co-authored-by: rock <[email protected]>
Reviewed-on: #55
Co-authored-by: poimen <[email protected]>
78 lines
2.0 KiB
TOML
78 lines
2.0 KiB
TOML
[package]
|
|
name = "poimen-memory"
|
|
version = "0.1.0"
|
|
edition = "2021"
|
|
publish = false
|
|
|
|
[workspace]
|
|
members = [
|
|
".",
|
|
"crates/mem-core",
|
|
"crates/mem-chunk",
|
|
"crates/mem-llm",
|
|
"crates/mem-ingest",
|
|
"crates/mem-store",
|
|
"crates/mem-cli",
|
|
]
|
|
resolver = "2"
|
|
|
|
[workspace.dependencies]
|
|
tokio = { version = "1.35", features = ["full"] }
|
|
futures = "0.3"
|
|
serde = { version = "1.0", features = ["derive"] }
|
|
serde_json = "1.0"
|
|
serde_yaml = "0.9"
|
|
anyhow = "1.0"
|
|
thiserror = "1.0"
|
|
sha2 = "0.10"
|
|
clap = { version = "4.4", features = ["derive"] }
|
|
reqwest = { version = "0.11", features = ["json"] }
|
|
tracing = "0.1"
|
|
tracing-subscriber = "0.3"
|
|
toml = "0.8"
|
|
hex = "0.4"
|
|
time = { version = "0.3", features = ["serde", "formatting", "parsing", "macros"] }
|
|
chrono = { version = "0.4", features = ["serde"] }
|
|
tokenizers = "0.13"
|
|
once_cell = "1.19"
|
|
actix-web = "4.4"
|
|
actix-rt = "2.9"
|
|
uuid = { version = "1.6", features = ["v4", "serde"] }
|
|
async-trait = "0.1"
|
|
urlencoding = "2.1"
|
|
base64 = "0.21"
|
|
sqlx = { version = "0.7", features = ["postgres", "runtime-tokio-rustls", "chrono", "uuid", "json"] }
|
|
pgvector = { version = "0.2", features = ["sqlx"] }
|
|
jsonwebtoken = "9.2"
|
|
regex = "1.10"
|
|
|
|
[dev-dependencies]
|
|
toml = { workspace = true }
|
|
mem-core = { path = "crates/mem-core" }
|
|
mem-chunk = { path = "crates/mem-chunk" }
|
|
mem-ingest = { path = "crates/mem-ingest" }
|
|
mem-llm = { path = "crates/mem-llm" }
|
|
mem-store = { path = "crates/mem-store" }
|
|
mem-cli = { path = "crates/mem-cli" }
|
|
anyhow = { workspace = true }
|
|
serde_json = { workspace = true }
|
|
time = { workspace = true }
|
|
tokio = { workspace = true }
|
|
futures = { workspace = true }
|
|
actix-web = { workspace = true }
|
|
actix-rt = { workspace = true }
|
|
wiremock = "0.6"
|
|
chrono = { version = "0.4", features = ["serde"] }
|
|
regex = { workspace = true }
|
|
sqlx = { workspace = true }
|
|
base64 = { workspace = true }
|
|
tracing = { workspace = true }
|
|
tracing-subscriber = { workspace = true }
|
|
reqwest = { workspace = true }
|
|
uuid = { workspace = true }
|
|
|
|
[profile.release]
|
|
opt-level = 3
|
|
lto = true
|
|
codegen-units = 1
|