#!/bin/bash

# TPOT performance bottleneck analysis script
# Specifically designed to analyze why Go Router is twice as slow as Rust Router
#
# Usage:
#   ./scripts/analyze_tpot.sh [options]
#
# Options:
#   --duration SECONDS     CPU profile duration (default: 60)
#   --requests NUM        Number of requests (default: 100)
#   --concurrency NUM     Concurrency level (default: 20)
#   --pprof-port PORT     pprof port (default: 6060)
#   --server-url URL      Server URL (default: http://localhost:8080)

set -e

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
PROFILE_DIR="${PROJECT_ROOT}/profiles"
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
OUTPUT_DIR="${PROFILE_DIR}/tpot_analysis_${TIMESTAMP}"

# Colors
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
BLUE='\033[0;34m'
NC='\033[0m'

# Default values
DURATION=${DURATION:-60}
NUM_REQUESTS=${NUM_REQUESTS:-100}
CONCURRENCY=${CONCURRENCY:-20}
PPROF_PORT=${PPROF_PORT:-6060}
SERVER_URL=${SERVER_URL:-http://localhost:8080}

# Parse arguments
while [[ $# -gt 0 ]]; do
    case $1 in
        --duration)
            DURATION="$2"
            shift 2
            ;;
        --requests)
            NUM_REQUESTS="$2"
            shift 2
            ;;
        --concurrency)
            CONCURRENCY="$2"
            shift 2
            ;;
        --pprof-port)
            PPROF_PORT="$2"
            shift 2
            ;;
        --server-url)
            SERVER_URL="$2"
            shift 2
            ;;
        *)
            echo "Unknown option: $1"
            exit 1
            ;;
    esac
done

mkdir -p "$OUTPUT_DIR"

# Check for graphviz (optional, needed for some pprof visualizations)
HAS_GRAPHVIZ=false
if command -v dot >/dev/null 2>&1; then
    HAS_GRAPHVIZ=true
fi

echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}TPOT Performance Bottleneck Analysis${NC}"
echo -e "${BLUE}========================================${NC}"
echo ""
echo "Configuration:"
echo "  Duration:    ${DURATION}s"
echo "  Requests:   $NUM_REQUESTS"
echo "  Concurrency: $CONCURRENCY"
echo "  Server URL:  $SERVER_URL"
echo "  pprof Port:  $PPROF_PORT"
echo "  Output Dir:  $OUTPUT_DIR"
if [ "$HAS_GRAPHVIZ" = "false" ]; then
    echo ""
    echo -e "${YELLOW}Note: graphviz not found. Some pprof visualizations may not work.${NC}"
    echo -e "${YELLOW}To install graphviz:${NC}"
    echo -e "${YELLOW}  macOS:   brew install graphviz${NC}"
    echo -e "${YELLOW}  Ubuntu:  sudo apt-get install graphviz${NC}"
    echo -e "${YELLOW}  CentOS:  sudo yum install graphviz${NC}"
    echo -e "${YELLOW}Text reports will still be generated without graphviz.${NC}"
fi
echo ""

# Check if server is running
echo -e "${YELLOW}[Check] Verifying server is running...${NC}"
if ! curl -s "${SERVER_URL}/health" > /dev/null 2>&1; then
    echo -e "${RED}Error: Server not responding at ${SERVER_URL}${NC}"
    echo ""
    echo "Please start the server first with profiling enabled:"
    echo "  ./run.sh --profile --pprof-port $PPROF_PORT"
    echo "  or"
    echo "  PPROF_ENABLED=true PPROF_PORT=$PPROF_PORT make run"
    exit 1
fi
echo -e "${GREEN}✓ Server is running${NC}"
echo ""

# Check if pprof is enabled
echo -e "${YELLOW}[Check] Verifying pprof is enabled...${NC}"
if ! curl -s "http://localhost:${PPROF_PORT}/debug/pprof/" > /dev/null 2>&1; then
    echo -e "${RED}Error: pprof not accessible at http://localhost:${PPROF_PORT}/debug/pprof/${NC}"
    echo ""
    echo "Please start the server with profiling enabled:"
    echo "  ./run.sh --profile --pprof-port $PPROF_PORT"
    exit 1
fi
echo -e "${GREEN}✓ pprof is enabled${NC}"
echo ""

# ============================================
# Step 1: Collect baseline profiles
# ============================================
echo -e "${GREEN}[Step 1/8] Collecting baseline profiles...${NC}"

# Baseline memory
go tool pprof -proto -output="${OUTPUT_DIR}/heap_before.pb.gz" \
    "http://localhost:${PPROF_PORT}/debug/pprof/heap" > /dev/null 2>&1 || true

# Baseline goroutine
go tool pprof -proto -output="${OUTPUT_DIR}/goroutine_before.pb.gz" \
    "http://localhost:${PPROF_PORT}/debug/pprof/goroutine" > /dev/null 2>&1 || true

echo -e "${GREEN}✓ Baseline profiles collected${NC}"
echo ""

# ============================================
# Step 2: Start CPU profile collection
# ============================================
echo -e "${GREEN}[Step 2/8] Starting CPU profile collection (${DURATION}s)...${NC}"
go tool pprof -proto -output="${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz" \
    "http://localhost:${PPROF_PORT}/debug/pprof/profile?seconds=${DURATION}" &
CPU_PID=$!
sleep 2
echo -e "${GREEN}✓ CPU profile collection started${NC}"
echo ""

# ============================================
# Step 3: Run load test with streaming requests
# ============================================
echo -e "${GREEN}[Step 3/8] Running load test ($NUM_REQUESTS streaming requests, concurrency=$CONCURRENCY)...${NC}"

# Function to run a single streaming request
run_streaming_request() {
    local request_id=$1
    local start_time=$(date +%s)
    local start_nanos=$(date +%N 2>/dev/null || echo "000000000")

    curl -N -s -X POST "${SERVER_URL}/v1/chat/completions" \
        -H "Content-Type: application/json" \
        -d "{
            \"model\": \"default\",
            \"messages\": [{\"role\": \"user\", \"content\": \"Write a 500-word story with character dialogue and scene descriptions\"}],
            \"stream\": true,
            \"max_tokens\": 300,
            \"temperature\": 0.7
        }" > /dev/null

    local end_time=$(date +%s)
    local end_nanos=$(date +%N 2>/dev/null || echo "000000000")
    local duration=$((end_time - start_time))
    echo "$duration" >> "${OUTPUT_DIR}/request_times.txt"
}

# Run requests with controlled concurrency
# Use a temporary file to track job PIDs to avoid conflicts with CPU_PID
JOB_PIDS_FILE="${OUTPUT_DIR}/.job_pids_$$"
> "$JOB_PIDS_FILE"

for i in $(seq 1 $NUM_REQUESTS); do
    # Wait if we've reached concurrency limit
    while [ $(wc -l < "$JOB_PIDS_FILE" 2>/dev/null || echo 0) -ge $CONCURRENCY ]; do
        # Check and remove completed jobs
        while IFS= read -r pid; do
            if [ -n "$pid" ] && ! kill -0 "$pid" 2>/dev/null; then
                # Process completed, remove from file
                grep -v "^${pid}$" "$JOB_PIDS_FILE" > "${JOB_PIDS_FILE}.tmp" && \
                    mv "${JOB_PIDS_FILE}.tmp" "$JOB_PIDS_FILE" || true
            fi
        done < "$JOB_PIDS_FILE"
        sleep 0.1
    done

    # Start new request
    run_streaming_request $i &
    echo $! >> "$JOB_PIDS_FILE"

    # Progress indicator
    if [ $((i % 10)) -eq 0 ]; then
        echo "  Progress: $i/$NUM_REQUESTS requests sent..."
    fi
done

# Wait for all remaining jobs (excluding CPU_PID)
while IFS= read -r pid; do
    if [ -n "$pid" ] && [ "$pid" != "$CPU_PID" ]; then
        wait "$pid" 2>/dev/null || true
    fi
done < "$JOB_PIDS_FILE"

# Clean up
rm -f "$JOB_PIDS_FILE" "${JOB_PIDS_FILE}.tmp" 2>/dev/null || true

echo -e "${GREEN}✓ Load test completed${NC}"
echo ""

# ============================================
# Step 4: Wait for CPU profile to complete
# ============================================
echo -e "${GREEN}[Step 4/8] Waiting for CPU profile to complete...${NC}"
# Wait for the process, but handle the case where it might have already completed
if kill -0 $CPU_PID 2>/dev/null; then
    wait $CPU_PID 2>/dev/null || true
else
    # Process already completed, just wait a bit to ensure file is written
    sleep 1
fi
echo -e "${GREEN}✓ CPU profile collection completed${NC}"
echo ""

# ============================================
# Step 5: Collect final profiles
# ============================================
echo -e "${GREEN}[Step 5/8] Collecting final profiles...${NC}"

# Final memory
go tool pprof -proto -output="${OUTPUT_DIR}/heap_after.pb.gz" \
    "http://localhost:${PPROF_PORT}/debug/pprof/heap" > /dev/null 2>&1 || true

# Final goroutine
go tool pprof -proto -output="${OUTPUT_DIR}/goroutine_after.pb.gz" \
    "http://localhost:${PPROF_PORT}/debug/pprof/goroutine" > /dev/null 2>&1 || true

# Mutex profile
go tool pprof -proto -output="${OUTPUT_DIR}/mutex.pb.gz" \
    "http://localhost:${PPROF_PORT}/debug/pprof/mutex" > /dev/null 2>&1 || true

# Block profile
go tool pprof -proto -output="${OUTPUT_DIR}/block.pb.gz" \
    "http://localhost:${PPROF_PORT}/debug/pprof/block" > /dev/null 2>&1 || true

echo -e "${GREEN}✓ Final profiles collected${NC}"
echo ""

# ============================================
# Step 6: Generate analysis reports
# ============================================
echo -e "${GREEN}[Step 6/8] Generating analysis reports...${NC}"

# CPU analysis
echo "  Generating CPU reports..."
go tool pprof -top -cum "${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz" > "${OUTPUT_DIR}/01_cpu_top_cum.txt" 2>&1 || true
go tool pprof -top "${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz" > "${OUTPUT_DIR}/02_cpu_top_flat.txt" 2>&1 || true

# Memory analysis
echo "  Generating memory reports..."
if [ -f "${OUTPUT_DIR}/heap_after.pb.gz" ]; then
    go tool pprof -top -alloc_space "${OUTPUT_DIR}/heap_after.pb.gz" > "${OUTPUT_DIR}/03_memory_alloc_space.txt" 2>&1 || true
    go tool pprof -top -alloc_objects "${OUTPUT_DIR}/heap_after.pb.gz" > "${OUTPUT_DIR}/04_memory_alloc_objects.txt" 2>&1 || true
    go tool pprof -top -inuse_space "${OUTPUT_DIR}/heap_after.pb.gz" > "${OUTPUT_DIR}/05_memory_inuse_space.txt" 2>&1 || true
fi

# Memory growth
if [ -f "${OUTPUT_DIR}/heap_before.pb.gz" ] && [ -f "${OUTPUT_DIR}/heap_after.pb.gz" ]; then
    go tool pprof -top -base="${OUTPUT_DIR}/heap_before.pb.gz" \
        "${OUTPUT_DIR}/heap_after.pb.gz" > "${OUTPUT_DIR}/06_memory_growth.txt" 2>&1 || true
fi

# FFI/CGO analysis
echo "  Analyzing FFI/CGO calls..."
go tool pprof -top "${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz" 2>&1 | \
    grep -iE "(block_on|CGO|FFI|ffi|runtime\.cgo|_Cfunc)" > "${OUTPUT_DIR}/07_ffi_cgo_analysis.txt" || \
    echo "No FFI/CGO related functions found" > "${OUTPUT_DIR}/07_ffi_cgo_analysis.txt"

# JSON serialization analysis
echo "  Analyzing JSON serialization..."
go tool pprof -top "${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz" 2>&1 | \
    grep -iE "(json|Marshal|Unmarshal|Encode|Decode|sonic|jsoniter)" > "${OUTPUT_DIR}/08_json_analysis.txt" || \
    echo "No JSON related functions found" > "${OUTPUT_DIR}/08_json_analysis.txt"

# Goroutine analysis
if [ -f "${OUTPUT_DIR}/goroutine_after.pb.gz" ]; then
    echo "  Analyzing goroutines..."
    go tool pprof -top "${OUTPUT_DIR}/goroutine_after.pb.gz" > "${OUTPUT_DIR}/09_goroutine_analysis.txt" 2>&1 || true
fi

# Mutex analysis
if [ -f "${OUTPUT_DIR}/mutex.pb.gz" ]; then
    echo "  Analyzing mutex contention..."
    go tool pprof -top "${OUTPUT_DIR}/mutex.pb.gz" > "${OUTPUT_DIR}/10_mutex_analysis.txt" 2>&1 || true
fi

# Block analysis
if [ -f "${OUTPUT_DIR}/block.pb.gz" ]; then
    echo "  Analyzing blocking operations..."
    go tool pprof -top "${OUTPUT_DIR}/block.pb.gz" > "${OUTPUT_DIR}/11_block_analysis.txt" 2>&1 || true
fi

# Request timing statistics
if [ -f "${OUTPUT_DIR}/request_times.txt" ] && [ -s "${OUTPUT_DIR}/request_times.txt" ]; then
    echo "  Calculating request timing statistics..."
    {
        echo "Request Timing Statistics"
        echo "========================"
        echo ""
        echo "Total requests: $(wc -l < "${OUTPUT_DIR}/request_times.txt" | tr -d ' ')"
        echo ""
        awk '{
            sum+=$1
            sumsq+=$1*$1
            if(NR==1 || $1<min) min=$1
            if(NR==1 || $1>max) max=$1
        } END {
            if(NR > 0) {
                mean=sum/NR
                variance=(sumsq/NR - mean*mean)
                stddev=sqrt(variance)
                print "Min:    " min "s"
                print "Max:    " max "s"
                print "Mean:   " mean "s"
                print "StdDev: " stddev "s"
            }
        }' "${OUTPUT_DIR}/request_times.txt"
    } > "${OUTPUT_DIR}/12_request_timing.txt"
fi

echo -e "${GREEN}✓ Analysis reports generated${NC}"
echo ""

# ============================================
# Step 7: Generate summary report
# ============================================
echo -e "${GREEN}[Step 7/8] Generating summary report...${NC}"

SUMMARY_FILE="${OUTPUT_DIR}/00_SUMMARY.md"
cat > "$SUMMARY_FILE" <<EOF
# TPOT Performance Analysis Summary

**Analysis Date:** $(date)
**Duration:** ${DURATION}s
**Requests:** $NUM_REQUESTS
**Concurrency:** $CONCURRENCY

## Key Findings

### 1. CPU Hotspots (Top 10 Cumulative Time)

\`\`\`
$(head -15 "${OUTPUT_DIR}/01_cpu_top_cum.txt" | tail -10)
\`\`\`

### 2. CPU Hotspots (Top 10 Flat Time)

\`\`\`
$(head -15 "${OUTPUT_DIR}/02_cpu_top_flat.txt" | tail -10)
\`\`\`

### 3. FFI/CGO Overhead

\`\`\`
$(cat "${OUTPUT_DIR}/07_ffi_cgo_analysis.txt")
\`\`\`

### 4. JSON Serialization Overhead

\`\`\`
$(cat "${OUTPUT_DIR}/08_json_analysis.txt")
\`\`\`

### 5. Memory Allocation (Top 10 by Space)

\`\`\`
$(head -15 "${OUTPUT_DIR}/03_memory_alloc_space.txt" | tail -10)
\`\`\`

### 6. Memory Allocation (Top 10 by Objects)

\`\`\`
$(head -15 "${OUTPUT_DIR}/04_memory_alloc_objects.txt" | tail -10)
\`\`\`

### 7. Mutex Contention

\`\`\`
$(head -15 "${OUTPUT_DIR}/10_mutex_analysis.txt" | tail -10 2>/dev/null || echo "No significant mutex contention detected")
\`\`\`

### 8. Blocking Operations

\`\`\`
$(head -15 "${OUTPUT_DIR}/11_block_analysis.txt" | tail -10 2>/dev/null || echo "No significant blocking detected")
\`\`\`

## Performance Bottlenecks Identified

### High Priority Issues

1. **FFI/CGO Overhead**
   - Check: \`cat ${OUTPUT_DIR}/07_ffi_cgo_analysis.txt\`
   - Impact: FFI calls add overhead compared to native Rust code
   - Recommendation: Minimize FFI calls, batch operations

2. **JSON Serialization**
   - Check: \`cat ${OUTPUT_DIR}/08_json_analysis.txt\`
   - Impact: JSON marshaling/unmarshaling can be expensive
   - Recommendation: Use faster JSON library (jsoniter), reduce serialization frequency

3. **Memory Allocations**
   - Check: \`cat ${OUTPUT_DIR}/03_memory_alloc_space.txt\`
   - Impact: Frequent allocations cause GC pressure
   - Recommendation: Use object pools, pre-allocate buffers

### Medium Priority Issues

4. **Goroutine Overhead**
   - Check: \`cat ${OUTPUT_DIR}/09_goroutine_analysis.txt\`
   - Impact: Too many goroutines can cause scheduling overhead
   - Recommendation: Limit goroutine count, use worker pools

5. **Lock Contention**
   - Check: \`cat ${OUTPUT_DIR}/10_mutex_analysis.txt\`
   - Impact: Lock contention reduces parallelism
   - Recommendation: Reduce lock granularity, use lock-free structures

## Comparison with Rust Router

### Expected Differences

1. **FFI Overhead**: Go → Rust FFI calls add ~100-500ns per call
2. **GC Overhead**: Go's GC can cause pauses (usually <1ms)
3. **JSON Library**: Go's standard library is slower than Rust's serde
4. **Memory Layout**: Go's GC affects cache locality

### Optimization Opportunities

1. **Reduce FFI Calls**
   - Batch token processing
   - Use async FFI (if possible)
   - Cache frequently used FFI results

2. **Optimize JSON**
   - Use jsoniter (already implemented)
   - Pre-allocate JSON buffers
   - Reduce serialization frequency

3. **Memory Management**
   - Use sync.Pool for frequently allocated objects
   - Pre-allocate slices with known capacity
   - Avoid unnecessary string copies

4. **Concurrency**
   - Use worker pools instead of spawning goroutines per request
   - Limit concurrent FFI calls
   - Use channels efficiently

## Next Steps

1. Review detailed reports in this directory
2. Use interactive pprof: \`go tool pprof -http=:8081 ${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz\`
3. Compare with Rust router profiles (if available)
4. Implement optimizations based on findings
5. Re-run analysis to measure improvements

## Files Generated

- \`00_SUMMARY.md\` - This summary
- \`01_cpu_top_cum.txt\` - CPU top functions (cumulative)
- \`02_cpu_top_flat.txt\` - CPU top functions (flat)
- \`03_memory_alloc_space.txt\` - Memory allocation by space
- \`04_memory_alloc_objects.txt\` - Memory allocation by objects
- \`05_memory_inuse_space.txt\` - Memory in use by space
- \`06_memory_growth.txt\` - Memory growth during test
- \`07_ffi_cgo_analysis.txt\` - FFI/CGO overhead analysis
- \`08_json_analysis.txt\` - JSON serialization analysis
- \`09_goroutine_analysis.txt\` - Goroutine analysis
- \`10_mutex_analysis.txt\` - Mutex contention analysis
- \`11_block_analysis.txt\` - Blocking operations analysis
- \`12_request_timing.txt\` - Request timing statistics
- \`*.pb.gz\` - Raw profile files for interactive analysis

EOF

echo -e "${GREEN}✓ Summary report generated${NC}"
echo ""

# ============================================
# Step 8: Display summary
# ============================================
echo -e "${GREEN}[Step 8/8] Analysis Complete!${NC}"
echo ""
echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}Summary${NC}"
echo -e "${BLUE}========================================${NC}"
echo ""
echo -e "${YELLOW}Top CPU Hotspots (Cumulative):${NC}"
head -12 "${OUTPUT_DIR}/01_cpu_top_cum.txt" | tail -10
echo ""
echo -e "${YELLOW}FFI/CGO Overhead:${NC}"
cat "${OUTPUT_DIR}/07_ffi_cgo_analysis.txt"
echo ""
echo -e "${YELLOW}JSON Serialization Overhead:${NC}"
cat "${OUTPUT_DIR}/08_json_analysis.txt"
echo ""
echo -e "${YELLOW}Top Memory Allocations:${NC}"
head -12 "${OUTPUT_DIR}/03_memory_alloc_space.txt" | tail -10
echo ""
if [ -f "${OUTPUT_DIR}/12_request_timing.txt" ]; then
    echo -e "${YELLOW}Request Timing:${NC}"
    cat "${OUTPUT_DIR}/12_request_timing.txt"
    echo ""
fi
echo -e "${GREEN}========================================${NC}"
echo ""
echo -e "${BLUE}Detailed Reports:${NC}"
echo "  Summary:      cat ${OUTPUT_DIR}/00_SUMMARY.md"
echo "  CPU (cum):    cat ${OUTPUT_DIR}/01_cpu_top_cum.txt"
echo "  CPU (flat):   cat ${OUTPUT_DIR}/02_cpu_top_flat.txt"
echo "  FFI/CGO:      cat ${OUTPUT_DIR}/07_ffi_cgo_analysis.txt"
echo "  JSON:         cat ${OUTPUT_DIR}/08_json_analysis.txt"
echo "  Memory:       cat ${OUTPUT_DIR}/03_memory_alloc_space.txt"
echo ""
echo -e "${BLUE}Interactive Analysis:${NC}"
echo "  Run: go tool pprof -http=:8081 ${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz"
echo "  Then visit:"
echo "    - http://localhost:8081/ui/flamegraph (Flame Graph - no graphviz needed)"
echo "    - http://localhost:8081/ui/top (Top Functions - no graphviz needed)"
if [ "$HAS_GRAPHVIZ" = "true" ]; then
    echo "    - http://localhost:8081/ui/graph (Call Graph - requires graphviz)"
else
    echo "    - http://localhost:8081/ui/graph (Call Graph - requires graphviz, not available)"
fi
echo ""
if [ "$HAS_GRAPHVIZ" = "false" ]; then
    echo -e "${YELLOW}Note: Install graphviz to enable call graph visualization:${NC}"
    echo -e "${YELLOW}  macOS:   brew install graphviz${NC}"
    echo -e "${YELLOW}  Ubuntu:  sudo apt-get install graphviz${NC}"
    echo -e "${YELLOW}  CentOS:  sudo yum install graphviz${NC}"
    echo ""
fi
echo -e "${GREEN}All files saved to: ${OUTPUT_DIR}${NC}"
echo ""