diff --git a/.env b/.env new file mode 100644 index 0000000..7fc2d86 --- /dev/null +++ b/.env @@ -0,0 +1,50 @@ +# Local development environment (.env file) +# Copy to .env and fill in your local/dev URLs +# .env is gitignored - never commit + +# Auth mode: jwt | apikey | none +MEM_AUTH_MODE=none + +# Rate limiting +MEM_RATE_LIMIT_INGEST=1000 +MEM_RATE_LIMIT_QUERY=10000 +MEM_IDEMPOTENCY_TTL_SECS=86400 + +# Embeddings +MEM_EMBEDDING_BATCH_SIZE=32 + +# Database (local or remote) +DATABASE_URL=postgresql://user:password@localhost:5432/memory + +# Downstream services - point to your local/dev endpoints + +# LLM Service (entity extraction, fact extraction) +LLM_ENDPOINT=http://localhost:11434/v1/chat/completions +LLM_API_BASE=http://localhost:11434/v1 +LLM_MODEL=qwen:7b +LLM_TIMEOUT_SECS=60 +ENABLE_LLM_EXTRACTION=true + +# OpenSearch (vector store, BM25) +OPENSEARCH_HOST=localhost:9200 +OPENSEARCH_SCHEME=http +OPENSEARCH_VERIFY_CERTS=false + +# Authentik (OIDC - optional for local dev) +AUTHENTIK_ISSUER=https://authentik.riotpiao.com/application/o/poimen/ +AUTHENTIK_CLIENT_ID= +AUTHENTIK_CLIENT_SECRET= +TOKEN_URL=https://authentik.riotpiao.com/application/o/token/ +AUTHENTIK_VERIFY_SSL=false + +# Temporal (workflow orchestration - future) +TEMPORAL_ENDPOINT=localhost:7233 +TEMPORAL_NAMESPACE=poimen + +# API Gateway (route optimization - future) +GATEWAY_URL=http://localhost:8080 + +# Server config +MEM_PORT=8080 +MEM_API_KEY=test-key +MEM_HOME=/tmp diff --git a/Cargo.lock b/Cargo.lock index 59f54f6..6aff59c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2053,6 +2053,7 @@ dependencies = [ "mem-ingest", "mem-llm", "mem-store", + "once_cell", "pgvector", "rand 0.8.7", "redis", diff --git a/crates/mem-cli/src/http_server.rs b/crates/mem-cli/src/http_server.rs index 3f92303..09fe3c0 100644 --- a/crates/mem-cli/src/http_server.rs +++ b/crates/mem-cli/src/http_server.rs @@ -439,29 +439,52 @@ pub async fn ingest_handler( body: web::Json, state: web::Data, ) -> HttpResponse { + use crate::metrics::*; + INGEST_REQUESTS_TOTAL.inc(); + INGEST_IN_FLIGHT.inc(); + let _timer = Timer::new(&INGEST_DURATION); + // Auth + capability check let (claims, _token) = match validate_auth(&req, &state).await { Ok(c) => c, - Err(e) => return e, + Err(e) => { + INGEST_AUTH_FAILURES.inc(); + INGEST_ERRORS_TOTAL.inc(); + INGEST_IN_FLIGHT.dec(); + return e; + } }; if !has_capability(&claims, "memory:write") { + INGEST_AUTH_FAILURES.inc(); + INGEST_ERRORS_TOTAL.inc(); + INGEST_IN_FLIGHT.dec(); return HttpResponse::Forbidden().json(json!({ "error": "forbidden", "reason": "missing capability: memory:write" })); } if let Err(e) = check_rate_limit(&claims, &state, "/memory/ingest") { + INGEST_RATE_LIMITED.inc(); + INGEST_IN_FLIGHT.dec(); return e; } // Check idempotency if let Some(cached) = state.idempotency_store.get(&body.ingest_id) { tracing::info!("Returning cached response for ingest_id: {}", body.ingest_id); + INGEST_DUPLICATES_TOTAL.inc(); + INGEST_IN_FLIGHT.dec(); return HttpResponse::Accepted().json(cached); } + let byte_count: usize = body.records.iter().map(|r| r.text.len()).sum(); + INGEST_BYTES_TOTAL.inc_by(byte_count as u64); + INGEST_RECORDS_TOTAL.inc_by(body.records.len() as u64); + // Execute ingest - execute_ingest(&state, &body).await + let resp = execute_ingest(&state, &body).await; + INGEST_IN_FLIGHT.dec(); + resp } /// Execute ingest job creation and spawn worker