poimen-memory/test_prod_ingest_real.sh

#!/bin/bash

# Production real test: Full ingest with api-gw + embedding
# Sends records through the complete pipeline and logs all errors
#
# Usage:
#   ./test_prod_ingest_real.sh [--verbose]

set -e

NAMESPACE="poimen"
SERVICE="poimen-memory"
LOCAL_PORT="9990"
TIMESTAMP=$(date +%Y%m%d-%H%M%S)
LOG_FILE="/tmp/ingest_test_${TIMESTAMP}.log"
VERBOSE="${1:-}"

{
  echo "=========================================="
  echo "Production Ingest Test: $(date)"
  echo "=========================================="
  echo ""
  echo "Namespace: $NAMESPACE"
  echo "Service: $SERVICE"
  echo "Local Port: $LOCAL_PORT"
  echo "Log: $LOG_FILE"
  echo ""

  # Start port-forward
  echo "Starting port-forward..."
  kubectl -n "$NAMESPACE" port-forward "svc/$SERVICE" "$LOCAL_PORT:8080" >/dev/null 2>&1 &
  PF_PID=$!
  
  cleanup() {
    if [ -n "$PF_PID" ]; then
      kill $PF_PID 2>/dev/null || true
      wait $PF_PID 2>/dev/null || true
    fi
  }
  trap cleanup EXIT
  
  sleep 2
  
  if ! kill -0 $PF_PID 2>/dev/null; then
    echo "✗ Port-forward failed"
    exit 1
  fi
  echo "✓ Port-forward running (PID $PF_PID)"
  echo ""
  
  # Check health
  echo "Checking /health endpoint..."
  if ! curl -s "http://localhost:$LOCAL_PORT/health" >/dev/null 2>&1; then
    echo "✗ Health check failed"
    exit 1
  fi
  echo "✓ Health check passed"
  echo ""
  
  # Prepare ingest request
  INGEST_ID="ingest-test-${TIMESTAMP}"
  
  PAYLOAD=$(cat <<'EOFPAYLOAD'
{
  "project": "production-real-test",
  "source": "integration-test",
  "ingest_id": "INGEST_ID_PLACEHOLDER",
  "records": [
    {
      "role": "user",
      "text": "Kubernetes [[Docker]] [[Linux]] is an open-source container orchestration platform. It automates many manual processes involved in deploying, managing, and scaling containerized applications.",
      "timestamp": "2026-09-14T13:00:00Z",
      "source_position": 0
    },
    {
      "role": "user",
      "text": "Docker [[Container]] [[Go]] is a containerization platform that makes it easier to build, ship, and run applications. Docker achieves high efficiency through the use of operating system-level virtualization.",
      "timestamp": "2026-09-14T13:01:00Z",
      "source_position": 1
    },
    {
      "role": "user",
      "text": "Go [[Concurrency]] [[Static Typing]] is a programming language designed at Google. It is statically typed, compiled, and known for its simplicity, concurrent programming model, and efficient execution.",
      "timestamp": "2026-09-14T13:02:00Z",
      "source_position": 2
    }
  ]
}
EOFPAYLOAD
)
  
  # Replace placeholder
  PAYLOAD="${PAYLOAD//INGEST_ID_PLACEHOLDER/$INGEST_ID}"
  
  echo "Sending ingest request..."
  if [ -n "$VERBOSE" ]; then
    echo "Payload:"
    echo "$PAYLOAD" | jq . 2>/dev/null || echo "$PAYLOAD"
    echo ""
  fi
  
  RESPONSE=$(curl -s -w "\n%{http_code}" -X POST \
    "http://localhost:$LOCAL_PORT/memory/ingest" \
    -H "Content-Type: application/json" \
    -H "Authorization: Bearer test-key" \
    -d "$PAYLOAD")
  
  HTTP_CODE=$(echo "$RESPONSE" | tail -1)
  BODY=$(echo "$RESPONSE" | head -n-1)
  
  echo "HTTP Status: $HTTP_CODE"
  if [ "$HTTP_CODE" != "202" ]; then
    echo "✗ Unexpected HTTP status"
    echo "Response: $BODY"
    exit 1
  fi
  echo "✓ Request accepted"
  echo ""
  
  if [ -n "$VERBOSE" ]; then
    echo "Response body:"
    echo "$BODY" | jq . 2>/dev/null || echo "$BODY"
    echo ""
  fi
  
  # Extract ID
  ID=$(echo "$BODY" | jq -r '.ingest_id // empty' 2>/dev/null)
  if [ -z "$ID" ]; then
    echo "✗ Missing ingest_id in response"
    echo "Response: $BODY"
    exit 1
  fi
  
  echo "Ingest ID: $ID"
  echo ""
  
  # Poll status
  echo "Polling job status..."
  echo "=========================================="
  
  MAX_POLLS=120  # 10 minutes at 5s intervals
  poll_count=0
  
  while [ $poll_count -lt $MAX_POLLS ]; do
    poll_count=$((poll_count+1))
    
    STATUS_RESP=$(curl -s "http://localhost:$LOCAL_PORT/memory/ingest/$ID" \
      -H "Authorization: Bearer test-key")
    
    STATUS=$(echo "$STATUS_RESP" | jq -r '.status // "unknown"' 2>/dev/null)
    
    printf "[%3d] %-20s" "$poll_count" "$STATUS"
    
    case "$STATUS" in
      done)
        echo " ✓"
        echo "=========================================="
        echo ""
        echo "✓ SUCCESS: Ingest completed"
        
        if [ -n "$VERBOSE" ]; then
          echo ""
          echo "Final response:"
          echo "$STATUS_RESP" | jq . 2>/dev/null || echo "$STATUS_RESP"
        fi
        exit 0
        ;;
      failed|error)
        echo " ✗"
        echo "=========================================="
        echo ""
        echo "✗ FAILED: Ingest did not complete"
        echo ""
        echo "Final response:"
        echo "$STATUS_RESP" | jq . 2>/dev/null || echo "$STATUS_RESP"
        exit 1
        ;;
      processing|queued|pending)
        echo ""
        sleep 5
        ;;
      *)
        echo " (unknown)"
        sleep 5
        ;;
    esac
  done
  
  echo "=========================================="
  echo ""
  echo "✗ TIMEOUT: Ingest did not complete after ${MAX_POLLS} polls (${poll_count}m)"
  exit 1

} 2>&1 | tee "$LOG_FILE"

echo ""
echo "Full log saved to: $LOG_FILE"