feat: Memory Phase 3 — LLM ingest, observability, migrations #49

Open
rock wants to merge 18 commits from feat/memory-ingest-retrieval into main
2 changed files with 59 additions and 45 deletions
Showing only changes of commit 8dc774a6c8 - Show all commits
+4 -2
View File
@@ -15,8 +15,10 @@ jobs:
name: Tag & Push Latest
runs-on: rust
steps:
- name: Install Docker
run: apt-get update && apt-get install -y docker.io
- name: Install Node.js and Docker
run: |
apt-get update
apt-get install -y nodejs docker.io
- name: Checkout code
uses: actions/checkout@v4
+55 -43
View File
@@ -11,66 +11,78 @@ env:
DB_HOST: memory-db-rw.poimen.svc.cluster.local
DB_PORT: "5432"
DB_NAME: memory
MIGRATIONS_DIR: crates/mem-store/migrations
jobs:
migrate:
name: Run Migrations
runs-on: rust
steps:
- name: Install psql
run: apt-get update && apt-get install -y postgresql-client
- name: Install Node.js and psql
run: |
apt-get update
apt-get install -y nodejs postgresql-client
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 2
- name: Fetch previous migrations state
- name: Detect changed migrations
id: detect
run: |
git fetch origin main --depth=2
# List changed migration files
CHANGED=$(git diff --name-only HEAD~1 HEAD -- crates/mem-store/migrations/ || echo "")
echo "Changed migrations: $CHANGED"
echo "CHANGED_MIGRATIONS=$CHANGED" >> $GITHUB_ENV
CHANGED=$(git diff --name-only HEAD~1 HEAD -- "$MIGRATIONS_DIR"/*.sql 2>/dev/null || echo "")
if [ -n "$CHANGED" ]; then
echo "files=$CHANGED" >> $GITHUB_OUTPUT
echo "found=true" >> $GITHUB_OUTPUT
echo "Changed: $CHANGED"
else
echo "found=false" >> $GITHUB_OUTPUT
echo "No migration changes detected"
fi
- name: Run migrations
if: env.CHANGED_MIGRATIONS != ''
- name: Apply changed migrations (push)
if: github.event_name == 'push' && steps.detect.outputs.found == 'true'
env:
PGHOST: ${{ env.DB_HOST }}
PGPORT: ${{ env.DB_PORT }}
PGDATABASE: ${{ env.DB_NAME }}
PGUSER: ${{ secrets.DB_USER }}
PGPASSWORD: ${{ secrets.DB_PASSWORD }}
run: |
export PGPASSWORD="${DB_PASSWORD}"
echo "=== Running changed migrations ==="
for f in $CHANGED_MIGRATIONS; do
if [ -f "$f" ]; then
echo "--- Applying: $f ---"
psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f "$f" 2>&1
if [ $? -ne 0 ]; then
echo "ERROR: Migration $f failed!"
exit 1
fi
echo "--- OK: $f ---"
fi
for f in ${{ steps.detect.outputs.files }}; do
[ -f "$f" ] || continue
echo "=== Applying: $f ==="
psql -v ON_ERROR_STOP=1 -f "$f"
echo "=== OK ==="
done
echo "=== Verify schema ==="
psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "\dt memory*"
env:
DB_USER: ${{ secrets.DB_USER }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
- name: Run all migrations (manual trigger)
- name: Apply all migrations (dispatch)
if: github.event_name == 'workflow_dispatch'
env:
PGHOST: ${{ env.DB_HOST }}
PGPORT: ${{ env.DB_PORT }}
PGDATABASE: ${{ env.DB_NAME }}
PGUSER: ${{ secrets.DB_USER }}
PGPASSWORD: ${{ secrets.DB_PASSWORD }}
run: |
export PGPASSWORD="${DB_PASSWORD}"
echo "=== Running all migrations in order ==="
for f in $(ls crates/mem-store/migrations/*.sql | sort); do
echo "--- Applying: $f ---"
psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f "$f" 2>&1 || true
echo "--- Done: $f ---"
for f in $(ls "$MIGRATIONS_DIR"/*.sql | sort); do
echo "=== Applying: $f ==="
psql -v ON_ERROR_STOP=1 -f "$f" || true
echo "=== Done ==="
done
echo "=== Final schema ==="
psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "\dt memory*"
psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "\d memory_entity"
psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "\d memory_edge"
- name: Verify schema
env:
DB_USER: ${{ secrets.DB_USER }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
PGHOST: ${{ env.DB_HOST }}
PGPORT: ${{ env.DB_PORT }}
PGDATABASE: ${{ env.DB_NAME }}
PGUSER: ${{ secrets.DB_USER }}
PGPASSWORD: ${{ secrets.DB_PASSWORD }}
run: |
echo "=== Tables ==="
psql -c "\dt memory*"
echo "=== Entity Schema ==="
psql -c "\d memory_entity"
echo "=== Edge Schema ==="
psql -c "\d memory_edge"