feat: OpenSearch JWT auth via Authentik OIDC
This commit is contained in:
@@ -73,22 +73,61 @@ data:
|
||||
http.port: 9200
|
||||
transport.port: 9300
|
||||
|
||||
# Security (disabled for K8s, assume TLS at ingress)
|
||||
plugins.security.disabled: "true"
|
||||
# Security Plugin (JWT/Authentik OIDC)
|
||||
plugins.security.disabled: "false"
|
||||
plugins.security.ssl.http.enabled: "false"
|
||||
plugins.security.ssl.transport.enabled: "false"
|
||||
|
||||
# JWT Authentication Realm for Authentik
|
||||
plugins.security.authcz.admin_dn:
|
||||
- "CN=admin,OU=admin,O=admin,L=admin,ST=admin,C=admin"
|
||||
|
||||
plugins.security.authc.realms.jwt_realm.type: jwt
|
||||
plugins.security.authc.realms.jwt_realm.order: 1
|
||||
plugins.security.authc.realms.jwt_realm.http_enabled: true
|
||||
plugins.security.authc.realms.jwt_realm.transport_enabled: false
|
||||
plugins.security.authc.realms.jwt_realm.description: "JWT realm for Authentik OIDC"
|
||||
plugins.security.authc.realms.jwt_realm.token_name: Authorization
|
||||
plugins.security.authc.realms.jwt_realm.token_extractor: "Bearer "
|
||||
plugins.security.authc.realms.jwt_realm.jwt_header: Authorization
|
||||
plugins.security.authc.realms.jwt_realm.roles_key: roles
|
||||
plugins.security.authc.realms.jwt_realm.subject_key: sub
|
||||
plugins.security.authc.realms.jwt_realm.jwks_uri: "https://authentik.riotpiao.com/application/o/poimen-memory/jwks/"
|
||||
plugins.security.authc.realms.jwt_realm.jwks_refresh_interval_ms: 3600000
|
||||
plugins.security.authc.realms.jwt_realm.issuer: "https://authentik.riotpiao.com/application/o/poimen-memory/"
|
||||
plugins.security.authc.realms.jwt_realm.enable_ssl_peer_hostname_verification: false
|
||||
plugins.security.authc.realms.jwt_realm.skip_jwt_verification: false
|
||||
|
||||
plugins.security.authc.cache.enable: true
|
||||
plugins.security.authc.backends.internal_authc_backend.type: intern
|
||||
|
||||
# Memory
|
||||
indices.memory.index_buffer_size: 30%
|
||||
|
||||
log4j2.properties: |
|
||||
status = warn
|
||||
|
||||
appender.console.type = Console
|
||||
appender.console.name = console
|
||||
appender.console.layout.type = PatternLayout
|
||||
appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
|
||||
|
||||
rootLogger.level = info
|
||||
rootLogger.appenderRef.console.ref = console
|
||||
|
||||
internal_users.yml: |
|
||||
admin:
|
||||
hash: "$2y$12$K/SpwjtB.wW8u3/52l.f2OPST9/PgBkqquzi.Oi8KfRMfsKkCq3GO"
|
||||
reserved: true
|
||||
backend_roles:
|
||||
- "admin"
|
||||
- "all_access"
|
||||
|
||||
roles_mapping.yml: |
|
||||
all_access:
|
||||
reserved: false
|
||||
backend_roles:
|
||||
- "*"
|
||||
hosts:
|
||||
- "*"
|
||||
|
||||
---
|
||||
|
||||
@@ -145,7 +184,12 @@ spec:
|
||||
- name: OPENSEARCH_JAVA_OPTS
|
||||
value: "-Xms1g -Xmx1g"
|
||||
- name: DISABLE_SECURITY_PLUGIN
|
||||
value: "true"
|
||||
value: "false"
|
||||
- name: OPENSEARCH_INITIAL_ADMIN_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: opensearch-secrets
|
||||
key: admin-password
|
||||
|
||||
# Volume mounts
|
||||
volumeMounts:
|
||||
@@ -154,6 +198,12 @@ spec:
|
||||
- name: opensearch-config
|
||||
mountPath: /usr/share/opensearch/config/opensearch.yml
|
||||
subPath: opensearch.yml
|
||||
- name: opensearch-config
|
||||
mountPath: /usr/share/opensearch/plugins/opensearch-security/securityconfig/internal_users.yml
|
||||
subPath: internal_users.yml
|
||||
- name: opensearch-config
|
||||
mountPath: /usr/share/opensearch/plugins/opensearch-security/securityconfig/roles_mapping.yml
|
||||
subPath: roles_mapping.yml
|
||||
- name: opensearch-logs
|
||||
mountPath: /usr/share/opensearch/logs
|
||||
|
||||
@@ -166,11 +216,14 @@ spec:
|
||||
memory: "2Gi"
|
||||
cpu: "1000m"
|
||||
|
||||
# Liveness probe
|
||||
# Liveness probe (skip auth via basic fallback)
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /_cluster/health
|
||||
port: 9200
|
||||
httpHeaders:
|
||||
- name: Authorization
|
||||
value: Basic YWRtaW46YWRtaW4="
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
@@ -181,6 +234,9 @@ spec:
|
||||
httpGet:
|
||||
path: /_cluster/health?local=true
|
||||
port: 9200
|
||||
httpHeaders:
|
||||
- name: Authorization
|
||||
value: Basic YWRtaW46YWRtaW4="
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 3
|
||||
@@ -403,3 +459,15 @@ kind: ServiceAccount
|
||||
metadata:
|
||||
name: opensearch-dashboards
|
||||
namespace: poimen
|
||||
|
||||
---
|
||||
|
||||
# Secret for OpenSearch Admin Password
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: opensearch-secrets
|
||||
namespace: poimen
|
||||
type: Opaque
|
||||
stringData:
|
||||
admin-password: "OpenSearch@Admin123!"
|
||||
|
||||
@@ -1,83 +0,0 @@
|
||||
# M8.2 — Dual-write indexing pipeline
|
||||
|
||||
| Field | Value |
|
||||
|---|---|
|
||||
| Phase | M8 — Hybrid Search |
|
||||
| Size | M — 1–2 days |
|
||||
| Status | ⬜ |
|
||||
| Flags | — |
|
||||
| Spec | inlined below |
|
||||
| Blocks | M8.4, M8.5 |
|
||||
| Depends | M8.1 (OpenSearch running), M2.4 (pgvector repo) |
|
||||
|
||||
## Goal
|
||||
|
||||
When a document is ingested, write to **both** pgvector (embedding) and OpenSearch (raw text) atomically. Same `chunk_id` in both stores. If one write fails, log error but don't block the other — eventual consistency, not transactions.
|
||||
|
||||
## Design
|
||||
|
||||
**Unified ID mapping:** Both stores use the same `chunk_id` (UUID). The ingest worker generates the ID once, writes to both.
|
||||
|
||||
**Chunking policy:** 512-token chunks with 10% (51-token) overlap. Deterministic — same input always produces same chunks with same IDs.
|
||||
|
||||
**Dual write sequence:**
|
||||
1. Chunk document (heading-boundary or fixed-size).
|
||||
2. Generate embedding via LLM.
|
||||
3. Write to pgvector: `INSERT INTO chunks (id, embedding, text, source, project, level, breadcrumb)`.
|
||||
4. Write to OpenSearch: `PUT vault-{project}/_doc/{chunk_id}` with `{content, source, level, breadcrumb, project_id, indexed_at}`.
|
||||
5. If OpenSearch write fails: log warning, mark chunk as `opensearch_pending=true` in pgvector. Background retry later.
|
||||
|
||||
**OpenSearch index mapping:**
|
||||
```json
|
||||
{
|
||||
"content": {"type": "text", "analyzer": "standard", "boost": 2.0},
|
||||
"section_title": {"type": "text", "boost": 1.5},
|
||||
"breadcrumb": {"type": "keyword"},
|
||||
"source": {"type": "keyword"},
|
||||
"project_id": {"type": "keyword"},
|
||||
"level": {"type": "keyword"},
|
||||
"indexed_at": {"type": "date"}
|
||||
}
|
||||
```
|
||||
|
||||
**Deduplication:** Before writing, check `chunk_hash` (SHA256 of text). If hash exists and `is_indexed=true` in both stores, skip.
|
||||
|
||||
## Steps
|
||||
|
||||
1. Add `opensearch_pending` boolean column to `chunks` table (migration).
|
||||
2. Update `IngestWorker::process_ingest()` to call OpenSearch after pgvector write.
|
||||
3. Make `OpenSearchClient::index_document()` public, fix method signature.
|
||||
4. Add background task: retry `opensearch_pending=true` chunks every 5 minutes.
|
||||
5. Add dedup check before dual write.
|
||||
|
||||
## Acceptance
|
||||
|
||||
1. `mem ingest --dry-run` on a test doc shows chunks written to both stores.
|
||||
2. Same `chunk_id` exists in both `SELECT id FROM chunks` and `GET vault-*/_doc/{id}`.
|
||||
3. Kill OpenSearch mid-ingest: pgvector write succeeds, chunk marked `opensearch_pending=true`.
|
||||
4. Restart OpenSearch: background retry picks up pending chunks within 5 minutes.
|
||||
5. Re-ingest same document: dedup skips already-indexed chunks (0 new writes).
|
||||
|
||||
## Verify
|
||||
|
||||
```bash
|
||||
# Ingest a test document
|
||||
cargo run -- ingest --project test --source fixtures/refcorpus/small.md
|
||||
|
||||
# Check pgvector
|
||||
psql -c "SELECT id, source, opensearch_pending FROM chunks WHERE project='test'"
|
||||
|
||||
# Check OpenSearch
|
||||
curl -k -H "Authorization: Bearer $TOKEN" \
|
||||
https://opensearch-internal:9200/vault-test/_search | jq '.hits.total'
|
||||
|
||||
# IDs must match
|
||||
```
|
||||
|
||||
**False pass:** Both stores have data but with different IDs — the join on `chunk_id` finds zero matches. Assert `SELECT count(*) FROM chunks WHERE id IN (opensearch_ids)` equals total indexed.
|
||||
|
||||
## Artifacts
|
||||
|
||||
- Modified `crates/mem-cli/src/ingest_worker.rs`
|
||||
- Modified `crates/mem-store/src/lib.rs` (migration)
|
||||
- Modified `crates/mem-cli/src/opensearch_client.rs`
|
||||
Reference in New Issue
Block a user