#!/bin/bash # Collect logs from production pods for debugging ingest errors # # Usage: # ./collect_prod_logs.sh before # Capture baseline # ./test_production_ingest.sh # Run test # ./collect_prod_logs.sh after # Capture post-test logs # ./collect_prod_logs.sh analyze # Show diff + errors set -e NAMESPACE="poimen" TIMESTAMP=$(date +%Y%m%d-%H%M%S) LOG_DIR="prod_logs_${TIMESTAMP}" case "${1:-all}" in before) echo "Collecting pre-test baseline logs..." mkdir -p "$LOG_DIR/before" kubectl -n "$NAMESPACE" get pods > "$LOG_DIR/before/pods.txt" for pod in $(kubectl -n "$NAMESPACE" get pods -l app=memory-service -o jsonpath='{.items[*].metadata.name}'); do echo " Collecting logs from $pod..." kubectl -n "$NAMESPACE" logs "$pod" --all-containers=true > "$LOG_DIR/before/${pod}.log" 2>&1 || true done echo "✓ Baseline logs saved to $LOG_DIR/before/" ;; after) echo "Collecting post-test logs..." mkdir -p "$LOG_DIR/after" kubectl -n "$NAMESPACE" get pods > "$LOG_DIR/after/pods.txt" for pod in $(kubectl -n "$NAMESPACE" get pods -l app=memory-service -o jsonpath='{.items[*].metadata.name}'); do echo " Collecting logs from $pod..." kubectl -n "$NAMESPACE" logs "$pod" --all-containers=true > "$LOG_DIR/after/${pod}.log" 2>&1 || true done echo "✓ Post-test logs saved to $LOG_DIR/after/" ;; analyze) if [ ! -d "$LOG_DIR/before" ] || [ ! -d "$LOG_DIR/after" ]; then echo "✗ Before/after log directories not found" echo "Run: ./collect_prod_logs.sh before && ./test_production_ingest.sh && ./collect_prod_logs.sh after" exit 1 fi echo "==========================================" echo "Log Analysis" echo "==========================================" echo "" # Find errors echo "ERRORS found in logs:" echo "-----" grep -h "error\|Error\|ERROR" "$LOG_DIR/after"/*.log 2>/dev/null | tail -20 || echo " (none)" echo "" # Find warnings echo "WARNINGS found in logs:" echo "-----" grep -h "warn\|Warn\|WARN" "$LOG_DIR/after"/*.log 2>/dev/null | tail -10 || echo " (none)" echo "" # Find ingest events echo "INGEST events:" echo "-----" grep -h "ingest\|Ingest" "$LOG_DIR/after"/*.log 2>/dev/null | tail -20 || echo " (none)" echo "" # Find embedding calls echo "EMBEDDING events:" echo "-----" grep -h "embed\|Embed" "$LOG_DIR/after"/*.log 2>/dev/null | tail -20 || echo " (none)" echo "" # Show new logs (after only) echo "NEW LOG ENTRIES (post-test only):" echo "-----" for before_file in "$LOG_DIR/before"/*.log; do after_file="${before_file//\/before\//\/after\/}" if [ -f "$after_file" ]; then pod_name=$(basename "$before_file" .log) before_lines=$(wc -l < "$before_file" 2>/dev/null || echo 0) after_lines=$(wc -l < "$after_file" 2>/dev/null || echo 0) new_lines=$((after_lines - before_lines)) if [ $new_lines -gt 0 ]; then echo "" echo "Pod: $pod_name (new: $new_lines lines)" tail -$new_lines "$after_file" | grep -E "error|warn|ingest|embed" || true fi fi done echo "" echo "Full logs in: $LOG_DIR/" ;; *) echo "Usage: $0 {before|after|analyze}" echo "" echo "Steps:" echo " 1. ./collect_prod_logs.sh before" echo " 2. ./test_production_ingest.sh" echo " 3. ./collect_prod_logs.sh after" echo " 4. ./collect_prod_logs.sh analyze" exit 1 ;; esac