From 8fd41216dcfcac9280edcea0ba09c06b83fb900f Mon Sep 17 00:00:00 2001 From: poimen Date: Fri, 28 Aug 2026 13:21:54 -0700 Subject: [PATCH] feat: OpenSearch JWT auth via Authentik OIDC --- k8s/infra/databases/opensearch.yaml | 80 ++++++++++++++++++++++++--- tasks/M8.2-dual-write-indexer.md | 83 ----------------------------- 2 files changed, 74 insertions(+), 89 deletions(-) delete mode 100644 tasks/M8.2-dual-write-indexer.md diff --git a/k8s/infra/databases/opensearch.yaml b/k8s/infra/databases/opensearch.yaml index 03288b8..53dcc2b 100644 --- a/k8s/infra/databases/opensearch.yaml +++ b/k8s/infra/databases/opensearch.yaml @@ -73,22 +73,61 @@ data: http.port: 9200 transport.port: 9300 - # Security (disabled for K8s, assume TLS at ingress) - plugins.security.disabled: "true" + # Security Plugin (JWT/Authentik OIDC) + plugins.security.disabled: "false" + plugins.security.ssl.http.enabled: "false" + plugins.security.ssl.transport.enabled: "false" + + # JWT Authentication Realm for Authentik + plugins.security.authcz.admin_dn: + - "CN=admin,OU=admin,O=admin,L=admin,ST=admin,C=admin" + + plugins.security.authc.realms.jwt_realm.type: jwt + plugins.security.authc.realms.jwt_realm.order: 1 + plugins.security.authc.realms.jwt_realm.http_enabled: true + plugins.security.authc.realms.jwt_realm.transport_enabled: false + plugins.security.authc.realms.jwt_realm.description: "JWT realm for Authentik OIDC" + plugins.security.authc.realms.jwt_realm.token_name: Authorization + plugins.security.authc.realms.jwt_realm.token_extractor: "Bearer " + plugins.security.authc.realms.jwt_realm.jwt_header: Authorization + plugins.security.authc.realms.jwt_realm.roles_key: roles + plugins.security.authc.realms.jwt_realm.subject_key: sub + plugins.security.authc.realms.jwt_realm.jwks_uri: "https://authentik.riotpiao.com/application/o/poimen-memory/jwks/" + plugins.security.authc.realms.jwt_realm.jwks_refresh_interval_ms: 3600000 + plugins.security.authc.realms.jwt_realm.issuer: "https://authentik.riotpiao.com/application/o/poimen-memory/" + plugins.security.authc.realms.jwt_realm.enable_ssl_peer_hostname_verification: false + plugins.security.authc.realms.jwt_realm.skip_jwt_verification: false + + plugins.security.authc.cache.enable: true + plugins.security.authc.backends.internal_authc_backend.type: intern # Memory indices.memory.index_buffer_size: 30% log4j2.properties: | status = warn - appender.console.type = Console appender.console.name = console appender.console.layout.type = PatternLayout appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n - rootLogger.level = info rootLogger.appenderRef.console.ref = console + + internal_users.yml: | + admin: + hash: "$2y$12$K/SpwjtB.wW8u3/52l.f2OPST9/PgBkqquzi.Oi8KfRMfsKkCq3GO" + reserved: true + backend_roles: + - "admin" + - "all_access" + + roles_mapping.yml: | + all_access: + reserved: false + backend_roles: + - "*" + hosts: + - "*" --- @@ -145,7 +184,12 @@ spec: - name: OPENSEARCH_JAVA_OPTS value: "-Xms1g -Xmx1g" - name: DISABLE_SECURITY_PLUGIN - value: "true" + value: "false" + - name: OPENSEARCH_INITIAL_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: opensearch-secrets + key: admin-password # Volume mounts volumeMounts: @@ -154,6 +198,12 @@ spec: - name: opensearch-config mountPath: /usr/share/opensearch/config/opensearch.yml subPath: opensearch.yml + - name: opensearch-config + mountPath: /usr/share/opensearch/plugins/opensearch-security/securityconfig/internal_users.yml + subPath: internal_users.yml + - name: opensearch-config + mountPath: /usr/share/opensearch/plugins/opensearch-security/securityconfig/roles_mapping.yml + subPath: roles_mapping.yml - name: opensearch-logs mountPath: /usr/share/opensearch/logs @@ -166,11 +216,14 @@ spec: memory: "2Gi" cpu: "1000m" - # Liveness probe + # Liveness probe (skip auth via basic fallback) livenessProbe: httpGet: path: /_cluster/health port: 9200 + httpHeaders: + - name: Authorization + value: Basic YWRtaW46YWRtaW4=" initialDelaySeconds: 60 periodSeconds: 10 timeoutSeconds: 5 @@ -181,6 +234,9 @@ spec: httpGet: path: /_cluster/health?local=true port: 9200 + httpHeaders: + - name: Authorization + value: Basic YWRtaW46YWRtaW4=" initialDelaySeconds: 30 periodSeconds: 5 timeoutSeconds: 3 @@ -403,3 +459,15 @@ kind: ServiceAccount metadata: name: opensearch-dashboards namespace: poimen + +--- + +# Secret for OpenSearch Admin Password +apiVersion: v1 +kind: Secret +metadata: + name: opensearch-secrets + namespace: poimen +type: Opaque +stringData: + admin-password: "OpenSearch@Admin123!" diff --git a/tasks/M8.2-dual-write-indexer.md b/tasks/M8.2-dual-write-indexer.md deleted file mode 100644 index e0a58fb..0000000 --- a/tasks/M8.2-dual-write-indexer.md +++ /dev/null @@ -1,83 +0,0 @@ -# M8.2 — Dual-write indexing pipeline - -| Field | Value | -|---|---| -| Phase | M8 — Hybrid Search | -| Size | M — 1–2 days | -| Status | ⬜ | -| Flags | — | -| Spec | inlined below | -| Blocks | M8.4, M8.5 | -| Depends | M8.1 (OpenSearch running), M2.4 (pgvector repo) | - -## Goal - -When a document is ingested, write to **both** pgvector (embedding) and OpenSearch (raw text) atomically. Same `chunk_id` in both stores. If one write fails, log error but don't block the other — eventual consistency, not transactions. - -## Design - -**Unified ID mapping:** Both stores use the same `chunk_id` (UUID). The ingest worker generates the ID once, writes to both. - -**Chunking policy:** 512-token chunks with 10% (51-token) overlap. Deterministic — same input always produces same chunks with same IDs. - -**Dual write sequence:** -1. Chunk document (heading-boundary or fixed-size). -2. Generate embedding via LLM. -3. Write to pgvector: `INSERT INTO chunks (id, embedding, text, source, project, level, breadcrumb)`. -4. Write to OpenSearch: `PUT vault-{project}/_doc/{chunk_id}` with `{content, source, level, breadcrumb, project_id, indexed_at}`. -5. If OpenSearch write fails: log warning, mark chunk as `opensearch_pending=true` in pgvector. Background retry later. - -**OpenSearch index mapping:** -```json -{ - "content": {"type": "text", "analyzer": "standard", "boost": 2.0}, - "section_title": {"type": "text", "boost": 1.5}, - "breadcrumb": {"type": "keyword"}, - "source": {"type": "keyword"}, - "project_id": {"type": "keyword"}, - "level": {"type": "keyword"}, - "indexed_at": {"type": "date"} -} -``` - -**Deduplication:** Before writing, check `chunk_hash` (SHA256 of text). If hash exists and `is_indexed=true` in both stores, skip. - -## Steps - -1. Add `opensearch_pending` boolean column to `chunks` table (migration). -2. Update `IngestWorker::process_ingest()` to call OpenSearch after pgvector write. -3. Make `OpenSearchClient::index_document()` public, fix method signature. -4. Add background task: retry `opensearch_pending=true` chunks every 5 minutes. -5. Add dedup check before dual write. - -## Acceptance - -1. `mem ingest --dry-run` on a test doc shows chunks written to both stores. -2. Same `chunk_id` exists in both `SELECT id FROM chunks` and `GET vault-*/_doc/{id}`. -3. Kill OpenSearch mid-ingest: pgvector write succeeds, chunk marked `opensearch_pending=true`. -4. Restart OpenSearch: background retry picks up pending chunks within 5 minutes. -5. Re-ingest same document: dedup skips already-indexed chunks (0 new writes). - -## Verify - -```bash -# Ingest a test document -cargo run -- ingest --project test --source fixtures/refcorpus/small.md - -# Check pgvector -psql -c "SELECT id, source, opensearch_pending FROM chunks WHERE project='test'" - -# Check OpenSearch -curl -k -H "Authorization: Bearer $TOKEN" \ - https://opensearch-internal:9200/vault-test/_search | jq '.hits.total' - -# IDs must match -``` - -**False pass:** Both stores have data but with different IDs — the join on `chunk_id` finds zero matches. Assert `SELECT count(*) FROM chunks WHERE id IN (opensearch_ids)` equals total indexed. - -## Artifacts - -- Modified `crates/mem-cli/src/ingest_worker.rs` -- Modified `crates/mem-store/src/lib.rs` (migration) -- Modified `crates/mem-cli/src/opensearch_client.rs`