poimen-memory/collect_prod_logs.sh

#!/bin/bash

# Collect logs from production pods for debugging ingest errors
#
# Usage:
#   ./collect_prod_logs.sh before  # Capture baseline
#   ./test_production_ingest.sh    # Run test
#   ./collect_prod_logs.sh after   # Capture post-test logs
#   ./collect_prod_logs.sh analyze # Show diff + errors

set -e

NAMESPACE="poimen"
TIMESTAMP=$(date +%Y%m%d-%H%M%S)
LOG_DIR="prod_logs_${TIMESTAMP}"

case "${1:-all}" in
  before)
    echo "Collecting pre-test baseline logs..."
    mkdir -p "$LOG_DIR/before"
    
    kubectl -n "$NAMESPACE" get pods > "$LOG_DIR/before/pods.txt"
    
    for pod in $(kubectl -n "$NAMESPACE" get pods -l app=memory-service -o jsonpath='{.items[*].metadata.name}'); do
      echo "  Collecting logs from $pod..."
      kubectl -n "$NAMESPACE" logs "$pod" --all-containers=true > "$LOG_DIR/before/${pod}.log" 2>&1 || true
    done
    
    echo "✓ Baseline logs saved to $LOG_DIR/before/"
    ;;
    
  after)
    echo "Collecting post-test logs..."
    mkdir -p "$LOG_DIR/after"
    
    kubectl -n "$NAMESPACE" get pods > "$LOG_DIR/after/pods.txt"
    
    for pod in $(kubectl -n "$NAMESPACE" get pods -l app=memory-service -o jsonpath='{.items[*].metadata.name}'); do
      echo "  Collecting logs from $pod..."
      kubectl -n "$NAMESPACE" logs "$pod" --all-containers=true > "$LOG_DIR/after/${pod}.log" 2>&1 || true
    done
    
    echo "✓ Post-test logs saved to $LOG_DIR/after/"
    ;;
    
  analyze)
    if [ ! -d "$LOG_DIR/before" ] || [ ! -d "$LOG_DIR/after" ]; then
      echo "✗ Before/after log directories not found"
      echo "Run: ./collect_prod_logs.sh before && ./test_production_ingest.sh && ./collect_prod_logs.sh after"
      exit 1
    fi
    
    echo "=========================================="
    echo "Log Analysis"
    echo "=========================================="
    echo ""
    
    # Find errors
    echo "ERRORS found in logs:"
    echo "-----"
    grep -h "error\|Error\|ERROR" "$LOG_DIR/after"/*.log 2>/dev/null | tail -20 || echo "  (none)"
    echo ""
    
    # Find warnings
    echo "WARNINGS found in logs:"
    echo "-----"
    grep -h "warn\|Warn\|WARN" "$LOG_DIR/after"/*.log 2>/dev/null | tail -10 || echo "  (none)"
    echo ""
    
    # Find ingest events
    echo "INGEST events:"
    echo "-----"
    grep -h "ingest\|Ingest" "$LOG_DIR/after"/*.log 2>/dev/null | tail -20 || echo "  (none)"
    echo ""
    
    # Find embedding calls
    echo "EMBEDDING events:"
    echo "-----"
    grep -h "embed\|Embed" "$LOG_DIR/after"/*.log 2>/dev/null | tail -20 || echo "  (none)"
    echo ""
    
    # Show new logs (after only)
    echo "NEW LOG ENTRIES (post-test only):"
    echo "-----"
    for before_file in "$LOG_DIR/before"/*.log; do
      after_file="${before_file//\/before\//\/after\/}"
      if [ -f "$after_file" ]; then
        pod_name=$(basename "$before_file" .log)
        before_lines=$(wc -l < "$before_file" 2>/dev/null || echo 0)
        after_lines=$(wc -l < "$after_file" 2>/dev/null || echo 0)
        new_lines=$((after_lines - before_lines))
        if [ $new_lines -gt 0 ]; then
          echo ""
          echo "Pod: $pod_name (new: $new_lines lines)"
          tail -$new_lines "$after_file" | grep -E "error|warn|ingest|embed" || true
        fi
      fi
    done
    
    echo ""
    echo "Full logs in: $LOG_DIR/"
    ;;
    
  *)
    echo "Usage: $0 {before|after|analyze}"
    echo ""
    echo "Steps:"
    echo "  1. ./collect_prod_logs.sh before"
    echo "  2. ./test_production_ingest.sh"
    echo "  3. ./collect_prod_logs.sh after"
    echo "  4. ./collect_prod_logs.sh analyze"
    exit 1
    ;;
esac