diff --git a/docs/performance_dashboard/README.md b/docs/performance_dashboard/README.md new file mode 100644 index 000000000..857dc26a8 --- /dev/null +++ b/docs/performance_dashboard/README.md @@ -0,0 +1,147 @@ +# SGLang Performance Dashboard + +A web-based dashboard for visualizing SGLang nightly test performance metrics. + +## Features + +- **Performance Trends**: View throughput, latency, and TTFT trends over time +- **Model Comparison**: Compare performance across different models and configurations +- **Filtering**: Filter by GPU configuration, model, variant, and batch size +- **Interactive Charts**: Zoom, pan, and hover for detailed metrics +- **Run History**: View recent benchmark runs with links to GitHub Actions + +## Quick Start + +### Option 1: Run with Local Server (Recommended) + +For live data from GitHub Actions artifacts: + +```bash +# Install requirements +pip install requests + +# Run the server +python server.py --fetch-on-start + +# Visit http://localhost:8000 +``` + +The server provides: +- Automatic fetching of metrics from GitHub +- Caching to reduce API calls +- `/api/metrics` endpoint for the frontend + +### Option 2: Fetch Data Manually + +Use the fetch script to download metrics data: + +```bash +# Fetch last 30 days of metrics +python fetch_metrics.py --output metrics_data.json + +# Fetch a specific run +python fetch_metrics.py --run-id 21338741812 --output single_run.json + +# Fetch only scheduled (nightly) runs +python fetch_metrics.py --scheduled-only --days 7 +``` + +## GitHub Token + +To download artifacts from GitHub, you need authentication: + +1. **Using `gh` CLI** (recommended): + ```bash + gh auth login + ``` + +2. **Using environment variable**: + ```bash + export GITHUB_TOKEN=your_token_here + ``` + +Without a token, the dashboard will show run metadata but not detailed benchmark results. + +## Data Structure + +The metrics JSON has this structure: + +```json +{ + "run_id": "21338741812", + "run_date": "2026-01-25T22:24:02.090218+00:00", + "commit_sha": "5cdb391...", + "branch": "main", + "results": [ + { + "gpu_config": "8-gpu-h200", + "partition": 0, + "model": "deepseek-ai/DeepSeek-V3.1", + "variant": "TP8+MTP", + "benchmarks": [ + { + "batch_size": 1, + "input_len": 4096, + "output_len": 512, + "latency_ms": 2400.72, + "input_throughput": 21408.64, + "output_throughput": 231.74, + "overall_throughput": 1919.43, + "ttft_ms": 191.32, + "acc_length": 3.19 + } + ] + } + ] +} +``` + +## Deployment + +### GitHub Pages + +The dashboard can be deployed to GitHub Pages for public access: + +1. Copy the dashboard files to `docs/performance_dashboard/` +2. Enable GitHub Pages in repository settings +3. Set up a GitHub Action to periodically update metrics data + +### Self-Hosted + +For a self-hosted deployment with live data: + +1. Set up a server running `server.py` +2. Configure a cron job or systemd timer to refresh data +3. Optionally put behind nginx/caddy for SSL + +## Metrics Explained + +- **Overall Throughput**: Total tokens (input + output) processed per second +- **Input Throughput**: Input tokens processed per second (prefill speed) +- **Output Throughput**: Output tokens generated per second (decode speed) +- **Latency**: End-to-end time to complete the request +- **TTFT**: Time to First Token - time until the first output token +- **Acc Length**: Acceptance length for speculative decoding (MTP variants) + +## Contributing + +To add support for new metrics or visualizations: + +1. Update `fetch_metrics.py` if data collection needs changes +2. Modify `app.js` to add new chart types or filters +3. Update `index.html` for UI changes + +## Troubleshooting + +**No data displayed** +- Check browser console for errors +- Verify GitHub API is accessible +- Try running with `server.py --fetch-on-start` + +**API rate limits** +- Use a GitHub token for higher limits +- The server caches data for 5 minutes + +**Charts not rendering** +- Ensure Chart.js is loading from CDN +- Check for JavaScript errors in console diff --git a/docs/performance_dashboard/app.js b/docs/performance_dashboard/app.js new file mode 100644 index 000000000..d8b05c3d2 --- /dev/null +++ b/docs/performance_dashboard/app.js @@ -0,0 +1,836 @@ +// SGLang Performance Dashboard Application + +const GITHUB_REPO = 'sgl-project/sglang'; +const WORKFLOW_NAME = 'nightly-test-nvidia.yml'; +const ARTIFACT_PREFIX = 'consolidated-metrics-'; + +// Chart instances (array for batch-separated charts) +let activeCharts = []; + +// Data storage +let allMetricsData = []; +let currentModel = null; +let currentMetricType = 'throughput'; // throughput, latency, ttft, inputThroughput + +// Metric type definitions +const metricTypes = { + throughput: { label: 'Overall Throughput', unit: 'tokens/sec', field: 'throughput' }, + outputThroughput: { label: 'Output Throughput', unit: 'tokens/sec', field: 'outputThroughput' }, + inputThroughput: { label: 'Input Throughput', unit: 'tokens/sec', field: 'inputThroughput' }, + latency: { label: 'Latency', unit: 'ms', field: 'latency' }, + ttft: { label: 'Time to First Token', unit: 'ms', field: 'ttft' }, + accLength: { label: 'Accept Length', unit: 'tokens', field: 'accLength', filterInvalid: true } +}; + +// Chart.js default configuration for dark theme +Chart.defaults.color = '#8b949e'; +Chart.defaults.borderColor = '#30363d'; + +const chartColors = [ + '#58a6ff', '#3fb950', '#d29922', '#f85149', '#a371f7', + '#79c0ff', '#56d364', '#e3b341', '#ff7b72', '#bc8cff' +]; + +// Initialize the dashboard +async function init() { + try { + await loadData(); + document.getElementById('loading').style.display = 'none'; + document.getElementById('content').style.display = 'block'; + populateFilters(); + updateStats(); + updateCharts(); + updateRunsTable(); + } catch (error) { + console.error('Failed to initialize dashboard:', error); + document.getElementById('loading').style.display = 'none'; + document.getElementById('error').style.display = 'block'; + document.getElementById('error-message').textContent = error.message; + } +} + +// Load data from local server API or GitHub +async function loadData() { + // Try local server API first (if running server.py) + try { + const response = await fetch('/api/metrics'); + if (response.ok) { + const data = await response.json(); + if (data.length > 0 && data[0].results && data[0].results.length > 0) { + allMetricsData = data; + console.log(`Loaded ${data.length} records from local API`); + allMetricsData.sort((a, b) => new Date(b.run_date) - new Date(a.run_date)); + return; + } + } + } catch (error) { + console.log('Local API not available, trying GitHub API'); + } + + // Try to load from GitHub API + const runs = await fetchWorkflowRuns(); + const metricsPromises = runs.map(run => fetchMetricsForRun(run)); + const results = await Promise.allSettled(metricsPromises); + + allMetricsData = results + .filter(r => r.status === 'fulfilled' && r.value !== null) + .map(r => r.value); + + if (allMetricsData.length === 0) { + throw new Error('No metrics data available. Please run the server.py with --fetch-on-start to fetch data from GitHub.'); + } + + // Sort by date descending + allMetricsData.sort((a, b) => new Date(b.run_date) - new Date(a.run_date)); +} + +// Fetch workflow runs from GitHub API +async function fetchWorkflowRuns() { + const response = await fetch( + `https://api.github.com/repos/${GITHUB_REPO}/actions/workflows/${WORKFLOW_NAME}/runs?status=completed&per_page=30`, + { + headers: { + 'Accept': 'application/vnd.github.v3+json' + } + } + ); + + if (!response.ok) { + throw new Error(`GitHub API error: ${response.status}`); + } + + const data = await response.json(); + return data.workflow_runs || []; +} + +// Fetch metrics artifact for a specific run +async function fetchMetricsForRun(run) { + try { + // Get artifacts for this run + const artifactsResponse = await fetch( + `https://api.github.com/repos/${GITHUB_REPO}/actions/runs/${run.id}/artifacts`, + { + headers: { + 'Accept': 'application/vnd.github.v3+json' + } + } + ); + + if (!artifactsResponse.ok) return null; + + const artifactsData = await artifactsResponse.json(); + const metricsArtifact = artifactsData.artifacts.find( + a => a.name.startsWith(ARTIFACT_PREFIX) + ); + + if (!metricsArtifact) return null; + + // Note: GitHub API doesn't allow direct artifact download without authentication + // For public access, we would need to use a proxy or pre-process the data + // For now, return run metadata - in production, use a backend to fetch artifacts + return { + run_id: run.id.toString(), + run_date: run.created_at, + commit_sha: run.head_sha, + branch: run.head_branch, + artifact_id: metricsArtifact.id, + results: [] // Would be populated from artifact content + }; + } catch (error) { + console.warn(`Failed to fetch metrics for run ${run.id}:`, error); + return null; + } +} + +// Populate filter dropdowns +function populateFilters() { + const gpuConfigs = new Set(); + const models = new Set(); + const batchSizes = new Set(); + const ioLengths = new Set(); + + allMetricsData.forEach(run => { + run.results.forEach(result => { + gpuConfigs.add(result.gpu_config); + models.add(result.model); + // Try new structure first (benchmarks_by_io_len), fall back to flat benchmarks + if (result.benchmarks_by_io_len) { + Object.entries(result.benchmarks_by_io_len).forEach(([ioKey, ioData]) => { + ioLengths.add(ioKey); + ioData.benchmarks.forEach(bench => { + batchSizes.add(bench.batch_size); + }); + }); + } else if (result.benchmarks) { + result.benchmarks.forEach(bench => { + batchSizes.add(bench.batch_size); + if (bench.input_len && bench.output_len) { + ioLengths.add(`${bench.input_len}_${bench.output_len}`); + } + }); + } + }); + }); + + // No "all" option for GPU and Model - populate with first value selected + const gpuArray = Array.from(gpuConfigs).sort(); + const modelArray = Array.from(models).sort(); + + populateSelectNoAll('gpu-filter', gpuArray); + populateSelectNoAll('model-filter', modelArray); + populateSelect('batch-filter', Array.from(batchSizes).sort((a, b) => a - b)); + populateSelectWithLabels('io-len-filter', sortIoLengths(Array.from(ioLengths)), formatIoLenLabel); + + // Set initial values (first option) + if (gpuArray.length > 0) { + document.getElementById('gpu-filter').value = gpuArray[0]; + } + if (modelArray.length > 0) { + document.getElementById('model-filter').value = modelArray[0]; + currentModel = modelArray[0]; + } + + // Update variants based on selected model + updateVariantFilter(); + // Update IO length filter based on selected GPU/model + updateIoLenFilter(); + + // Create metric type tabs + createMetricTabs(); +} + +// Format input/output length key for display +function formatIoLenLabel(ioKey) { + if (!ioKey) return 'Unknown'; + const parts = ioKey.split('_'); + if (parts.length === 2) { + return `In: ${parts[0]}, Out: ${parts[1]}`; + } + return ioKey; +} + +// Sort IO length keys numerically (by input length, then output length) +function sortIoLengths(ioLengths) { + return ioLengths.filter(key => key && key.includes('_')).sort((a, b) => { + const [aIn, aOut] = a.split('_').map(Number); + const [bIn, bOut] = b.split('_').map(Number); + if (isNaN(aIn) || isNaN(bIn)) return 0; + return (aIn - bIn) || (aOut - bOut); + }); +} + +// Populate select with custom label formatting +function populateSelectWithLabels(selectId, options, labelFormatter) { + const select = document.getElementById(selectId); + options.forEach(option => { + const opt = document.createElement('option'); + opt.value = option; + opt.textContent = labelFormatter ? labelFormatter(option) : option; + select.appendChild(opt); + }); +} + +// Update IO length filter based on selected GPU and model +function updateIoLenFilter() { + const gpuFilterEl = document.getElementById('gpu-filter'); + const modelFilterEl = document.getElementById('model-filter'); + const ioLenSelect = document.getElementById('io-len-filter'); + if (!gpuFilterEl || !modelFilterEl || !ioLenSelect) return; + + const gpuFilter = gpuFilterEl.value; + const modelFilter = modelFilterEl.value; + + const ioLengths = new Set(); + + allMetricsData.forEach(run => { + run.results.forEach(result => { + if (result.gpu_config === gpuFilter && result.model === modelFilter) { + if (result.benchmarks_by_io_len) { + Object.keys(result.benchmarks_by_io_len).forEach(ioKey => { + ioLengths.add(ioKey); + }); + } else if (result.benchmarks) { + result.benchmarks.forEach(bench => { + if (bench.input_len && bench.output_len) { + ioLengths.add(`${bench.input_len}_${bench.output_len}`); + } + }); + } + } + }); + }); + + const ioLenArray = sortIoLengths(Array.from(ioLengths)); + const currentIoLen = ioLenSelect.value; + + // Clear and repopulate + ioLenSelect.innerHTML = ''; + ioLenArray.forEach(ioLen => { + const opt = document.createElement('option'); + opt.value = ioLen; + opt.textContent = formatIoLenLabel(ioLen); + ioLenSelect.appendChild(opt); + }); + + // Try to restore previous selection if still valid + if (ioLenArray.includes(currentIoLen)) { + ioLenSelect.value = currentIoLen; + } else { + ioLenSelect.value = 'all'; + } +} + +// Update variant filter based on selected GPU and model +function updateVariantFilter() { + const gpuFilter = document.getElementById('gpu-filter').value; + const modelFilter = document.getElementById('model-filter').value; + + const variants = new Set(); + + allMetricsData.forEach(run => { + run.results.forEach(result => { + if (result.gpu_config === gpuFilter && result.model === modelFilter) { + // Use 'default' for null/undefined variants + variants.add(result.variant || 'default'); + } + }); + }); + + const variantArray = Array.from(variants).sort(); + const variantSelect = document.getElementById('variant-filter'); + const currentVariant = variantSelect.value; + + // Clear and repopulate + variantSelect.innerHTML = ''; + variantArray.forEach(variant => { + const opt = document.createElement('option'); + opt.value = variant; + opt.textContent = variant; + variantSelect.appendChild(opt); + }); + + // Try to restore previous selection if still valid + if (variantArray.includes(currentVariant)) { + variantSelect.value = currentVariant; + } else { + variantSelect.value = 'all'; + } +} + +function populateSelect(selectId, options) { + const select = document.getElementById(selectId); + options.forEach(option => { + const opt = document.createElement('option'); + opt.value = option; + opt.textContent = option; + select.appendChild(opt); + }); +} + +function populateSelectNoAll(selectId, options) { + const select = document.getElementById(selectId); + // Remove the "all" option if present + while (select.options.length > 0) { + select.remove(0); + } + options.forEach(option => { + const opt = document.createElement('option'); + opt.value = option; + opt.textContent = option; + select.appendChild(opt); + }); +} + +function createMetricTabs() { + const tabsContainer = document.getElementById('metric-tabs'); + tabsContainer.innerHTML = ''; + + Object.entries(metricTypes).forEach(([key, metric], index) => { + const tab = document.createElement('div'); + tab.className = index === 0 ? 'tab active' : 'tab'; + tab.textContent = metric.label; + tab.dataset.metric = key; + tab.onclick = () => selectMetricTab(key, tab); + tabsContainer.appendChild(tab); + }); +} + +function selectMetricTab(metricKey, tabElement) { + document.querySelectorAll('.tab').forEach(t => t.classList.remove('active')); + tabElement.classList.add('active'); + currentMetricType = metricKey; + + // Update chart title + const metric = metricTypes[metricKey]; + document.getElementById('metric-title').textContent = `${metric.label} (${metric.unit})`; + + updateCharts(); +} + +// Handle model filter dropdown change +function handleModelFilterChange(model) { + currentModel = model; + // Update variant filter based on new model selection + updateVariantFilter(); + // Update IO length filter based on new model selection + updateIoLenFilter(); + updateCharts(); +} + +// Handle GPU filter change +function handleGpuFilterChange() { + // Update variant filter based on new GPU selection + updateVariantFilter(); + // Update IO length filter based on new GPU selection + updateIoLenFilter(); + updateCharts(); +} + +// Update summary stats +function updateStats() { + const statsRow = document.getElementById('stats-row'); + const latestRun = allMetricsData[0]; + + if (!latestRun) { + statsRow.innerHTML = ''; + const noDataDiv = document.createElement('div'); + noDataDiv.className = 'no-data'; + noDataDiv.textContent = 'No data available'; + statsRow.appendChild(noDataDiv); + return; + } + + const totalModels = new Set(latestRun.results.map(r => r.model)).size; + const totalBenchmarks = latestRun.results.reduce((sum, r) => { + // Count benchmarks from either structure + if (r.benchmarks_by_io_len) { + return sum + Object.values(r.benchmarks_by_io_len).reduce( + (ioSum, ioData) => ioSum + ioData.benchmarks.length, 0 + ); + } + return sum + (r.benchmarks ? r.benchmarks.length : 0); + }, 0); + + statsRow.innerHTML = ''; // Clear previous stats + + const addStat = (label, value) => { + const card = document.createElement('div'); + card.className = 'stat-card'; + const labelEl = document.createElement('div'); + labelEl.className = 'label'; + labelEl.textContent = label; + const valueEl = document.createElement('div'); + valueEl.className = 'value'; + valueEl.textContent = value; + card.appendChild(labelEl); + card.appendChild(valueEl); + statsRow.appendChild(card); + }; + + addStat('Total Runs', allMetricsData.length); + addStat('Models Tested', totalModels); + addStat('Benchmarks', totalBenchmarks); +} + +// Update charts based on current filters and selected metric type +function updateCharts() { + const gpuFilter = document.getElementById('gpu-filter').value; + const modelFilter = currentModel; + const variantFilter = document.getElementById('variant-filter').value; + const ioLenFilter = document.getElementById('io-len-filter').value; + const batchFilter = document.getElementById('batch-filter').value; + + // Prepare data for charts - grouped by batch size + const chartDataByBatch = prepareChartDataByBatch(gpuFilter, modelFilter, variantFilter, ioLenFilter, batchFilter); + + // Update chart for the selected metric type + updateMetricChart(chartDataByBatch, currentMetricType); +} + +function prepareChartData(gpuFilter, modelFilter, variantFilter, ioLenFilter, batchFilter) { + const seriesMap = new Map(); + + allMetricsData.forEach(run => { + const runDate = new Date(run.run_date); + + run.results.forEach(result => { + // Apply filters + if (result.gpu_config !== gpuFilter) return; + if (result.model !== modelFilter) return; + if (variantFilter !== 'all' && result.variant !== variantFilter) return; + + // Helper function to process a benchmark entry + const processBenchmark = (bench, ioKey) => { + if (batchFilter !== 'all' && bench.batch_size !== parseInt(batchFilter)) return; + + const ioLabel = ioKey ? `, ${formatIoLenLabel(ioKey)}` : ''; + const seriesKey = `${result.model.split('/').pop()} (${result.variant}, BS=${bench.batch_size}${ioLabel})`; + + if (!seriesMap.has(seriesKey)) { + seriesMap.set(seriesKey, { + label: seriesKey, + data: [], + model: result.model, + variant: result.variant, + batchSize: bench.batch_size, + ioKey: ioKey + }); + } + + seriesMap.get(seriesKey).data.push({ + x: runDate, + throughput: bench.overall_throughput, + outputThroughput: bench.output_throughput, + latency: bench.latency_ms, + ttft: bench.ttft_ms, + inputThroughput: bench.input_throughput, + accLength: bench.acc_length, + runId: run.run_id + }); + }; + + // Use benchmarks_by_io_len if available + if (result.benchmarks_by_io_len) { + Object.entries(result.benchmarks_by_io_len).forEach(([ioKey, ioData]) => { + if (ioLenFilter !== 'all' && ioKey !== ioLenFilter) return; + ioData.benchmarks.forEach(bench => processBenchmark(bench, ioKey)); + }); + } else if (result.benchmarks) { + result.benchmarks.forEach(bench => { + const benchIoKey = bench.input_len && bench.output_len + ? `${bench.input_len}_${bench.output_len}` + : null; + if (ioLenFilter !== 'all' && benchIoKey !== ioLenFilter) return; + processBenchmark(bench, benchIoKey); + }); + } + }); + }); + + // Sort data points by date + seriesMap.forEach(series => { + series.data.sort((a, b) => a.x - b.x); + }); + + return Array.from(seriesMap.values()); +} + +// Prepare chart data grouped by batch size - each batch size is a separate series +function prepareChartDataByBatch(gpuFilter, modelFilter, variantFilter, ioLenFilter, batchFilter) { + const batchDataMap = new Map(); // batch_size -> Map of variant -> data + + allMetricsData.forEach(run => { + const runDate = new Date(run.run_date); + + run.results.forEach(result => { + // Apply filters - GPU and Model are required (no "all" option) + if (result.gpu_config !== gpuFilter) return; + if (result.model !== modelFilter) return; + if (variantFilter !== 'all' && result.variant !== variantFilter) return; + + // Use benchmarks_by_io_len if available, otherwise fall back to flat benchmarks + if (result.benchmarks_by_io_len) { + Object.entries(result.benchmarks_by_io_len).forEach(([ioKey, ioData]) => { + // Apply IO length filter + if (ioLenFilter !== 'all' && ioKey !== ioLenFilter) return; + + ioData.benchmarks.forEach(bench => { + if (batchFilter !== 'all' && bench.batch_size !== parseInt(batchFilter)) return; + + const batchSize = bench.batch_size; + const variantLabel = result.variant || 'default'; + // Include IO length in series key when showing all lengths + const seriesKey = ioLenFilter === 'all' + ? `${variantLabel} (${formatIoLenLabel(ioKey)})` + : variantLabel; + + if (!batchDataMap.has(batchSize)) { + batchDataMap.set(batchSize, new Map()); + } + + const variantMap = batchDataMap.get(batchSize); + if (!variantMap.has(seriesKey)) { + variantMap.set(seriesKey, { + label: seriesKey, + data: [], + model: result.model, + variant: result.variant, + batchSize: batchSize, + ioKey: ioKey + }); + } + + variantMap.get(seriesKey).data.push({ + x: runDate, + throughput: bench.overall_throughput, + outputThroughput: bench.output_throughput, + latency: bench.latency_ms, + ttft: bench.ttft_ms, + inputThroughput: bench.input_throughput, + accLength: bench.acc_length, + runId: run.run_id + }); + }); + }); + } else if (result.benchmarks) { + // Fall back to flat benchmarks for backward compatibility + result.benchmarks.forEach(bench => { + // Apply IO length filter using flat structure + const benchIoKey = bench.input_len && bench.output_len + ? `${bench.input_len}_${bench.output_len}` + : null; + if (ioLenFilter !== 'all' && benchIoKey !== ioLenFilter) return; + if (batchFilter !== 'all' && bench.batch_size !== parseInt(batchFilter)) return; + + const batchSize = bench.batch_size; + const variantLabel = result.variant || 'default'; + // Include IO length in series key when showing all lengths + const seriesKey = ioLenFilter === 'all' && benchIoKey + ? `${variantLabel} (${formatIoLenLabel(benchIoKey)})` + : variantLabel; + + if (!batchDataMap.has(batchSize)) { + batchDataMap.set(batchSize, new Map()); + } + + const variantMap = batchDataMap.get(batchSize); + if (!variantMap.has(seriesKey)) { + variantMap.set(seriesKey, { + label: seriesKey, + data: [], + model: result.model, + variant: result.variant, + batchSize: batchSize, + ioKey: benchIoKey + }); + } + + variantMap.get(seriesKey).data.push({ + x: runDate, + throughput: bench.overall_throughput, + outputThroughput: bench.output_throughput, + latency: bench.latency_ms, + ttft: bench.ttft_ms, + inputThroughput: bench.input_throughput, + accLength: bench.acc_length, + runId: run.run_id + }); + }); + } + }); + }); + + // Sort data points by date and convert to array format + const result = {}; + batchDataMap.forEach((variantMap, batchSize) => { + variantMap.forEach(series => { + series.data.sort((a, b) => a.x - b.x); + }); + result[batchSize] = Array.from(variantMap.values()); + }); + + return result; +} + +// Unified chart update function for any metric type +function updateMetricChart(chartDataByBatch, metricType) { + const container = document.getElementById('charts-container'); + container.innerHTML = ''; + + // Destroy existing charts + activeCharts.forEach(chart => chart.destroy()); + activeCharts = []; + + const metric = metricTypes[metricType]; + const batchSizes = Object.keys(chartDataByBatch).sort((a, b) => parseInt(a) - parseInt(b)); + + if (batchSizes.length === 0) { + container.innerHTML = '
No data available for the selected filters
'; + return; + } + + let hasAnyData = false; + + batchSizes.forEach(batchSize => { + const chartData = chartDataByBatch[batchSize]; + + const ctx_datasets = chartData.map((series, index) => { + // Filter data points - for metrics like accLength, exclude invalid values (-1 or null) + let dataPoints = series.data.map(d => ({ x: d.x, y: d[metric.field] })); + if (metric.filterInvalid) { + dataPoints = dataPoints.filter(d => d.y != null && d.y !== -1 && d.y > 0); + } + return { + label: series.label, + data: dataPoints, + borderColor: chartColors[index % chartColors.length], + backgroundColor: chartColors[index % chartColors.length] + '20', + tension: 0.1, + fill: false + }; + }).filter(dataset => dataset.data.length > 0); // Remove empty datasets + + // Skip this batch size if no valid data + if (ctx_datasets.length === 0) { + return; + } + + hasAnyData = true; + + const chartWrapper = document.createElement('div'); + chartWrapper.className = 'batch-chart-wrapper'; + + const title = document.createElement('div'); + title.className = 'batch-chart-title'; + title.textContent = `Batch Size: ${batchSize}`; + chartWrapper.appendChild(title); + + const chartContainer = document.createElement('div'); + chartContainer.className = 'chart-container'; + const canvas = document.createElement('canvas'); + chartContainer.appendChild(canvas); + chartWrapper.appendChild(chartContainer); + container.appendChild(chartWrapper); + + const ctx = canvas.getContext('2d'); + + const chart = new Chart(ctx, { + type: 'line', + data: { datasets: ctx_datasets }, + options: getChartOptions(metric.unit) + }); + activeCharts.push(chart); + }); + + // Show message if no valid data for this metric + if (!hasAnyData) { + container.innerHTML = `
No valid ${metric.label.toLowerCase()} data available for the selected filters
`; + } +} + +function getChartOptions(yAxisLabel) { + return { + responsive: true, + maintainAspectRatio: false, + interaction: { + mode: 'index', + intersect: false + }, + plugins: { + legend: { + position: 'bottom', + labels: { + boxWidth: 12, + padding: 10, + font: { size: 11 } + } + }, + tooltip: { + backgroundColor: '#21262d', + borderColor: '#30363d', + borderWidth: 1, + titleFont: { size: 13 }, + bodyFont: { size: 12 }, + padding: 12 + } + }, + scales: { + x: { + type: 'time', + time: { + unit: 'day', + displayFormats: { + day: 'MMM d' + } + }, + grid: { + color: '#21262d' + } + }, + y: { + title: { + display: true, + text: yAxisLabel + }, + grid: { + color: '#21262d' + } + } + } + }; +} + +// Escape HTML to prevent XSS +function escapeHtml(text) { + const div = document.createElement('div'); + div.textContent = text; + return div.innerHTML; +} + +// Update runs table +function updateRunsTable() { + const tbody = document.getElementById('runs-table-body'); + tbody.innerHTML = ''; + + allMetricsData.slice(0, 10).forEach(run => { + const models = new Set(run.results.map(r => r.model.split('/').pop())); + const date = new Date(run.run_date); + + const row = document.createElement('tr'); + + // Create cells safely to prevent XSS + const dateCell = document.createElement('td'); + dateCell.textContent = `${date.toLocaleDateString()} ${date.toLocaleTimeString()}`; + + const runIdCell = document.createElement('td'); + const runLink = document.createElement('a'); + runLink.href = `https://github.com/${GITHUB_REPO}/actions/runs/${encodeURIComponent(run.run_id)}`; + runLink.target = '_blank'; + runLink.className = 'run-link'; + runLink.textContent = run.run_id; + runIdCell.appendChild(runLink); + + const commitCell = document.createElement('td'); + const commitCode = document.createElement('code'); + commitCode.textContent = run.commit_sha.substring(0, 7); + commitCell.appendChild(commitCode); + + const branchCell = document.createElement('td'); + branchCell.textContent = run.branch; + + const modelsCell = document.createElement('td'); + Array.from(models).forEach((model, index) => { + if (index > 0) modelsCell.appendChild(document.createTextNode(' ')); + const badge = document.createElement('span'); + badge.className = 'model-badge'; + badge.textContent = model; + modelsCell.appendChild(badge); + }); + + row.appendChild(dateCell); + row.appendChild(runIdCell); + row.appendChild(commitCell); + row.appendChild(branchCell); + row.appendChild(modelsCell); + + tbody.appendChild(row); + }); +} + +// Refresh data +async function refreshData() { + document.getElementById('content').style.display = 'none'; + document.getElementById('loading').style.display = 'flex'; + await init(); +} + +// Format numbers for display +function formatNumber(num) { + if (num >= 1000) { + return (num / 1000).toFixed(1) + 'k'; + } + return num.toFixed(1); +} + +// Initialize on page load +document.addEventListener('DOMContentLoaded', init); diff --git a/docs/performance_dashboard/fetch_metrics.py b/docs/performance_dashboard/fetch_metrics.py new file mode 100755 index 000000000..264e7f334 --- /dev/null +++ b/docs/performance_dashboard/fetch_metrics.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 +""" +Fetch and process SGLang nightly test metrics from GitHub Actions artifacts. + +This script fetches consolidated metrics from GitHub Actions workflow runs +and outputs them as JSON for the performance dashboard. + +Usage: + python fetch_metrics.py --output metrics_data.json + python fetch_metrics.py --output metrics_data.json --days 30 + python fetch_metrics.py --output metrics_data.json --run-id 21338741812 +""" + +import argparse +import io +import json +import os +import sys +import zipfile +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Optional + +import requests + +GITHUB_REPO = "sgl-project/sglang" +WORKFLOW_NAME = "nightly-test-nvidia.yml" +ARTIFACT_PREFIX = "consolidated-metrics-" + + +def get_github_token() -> Optional[str]: + """Get GitHub token from environment or gh CLI.""" + # Check environment variable first + token = os.environ.get("GITHUB_TOKEN") + if token: + return token + + # Try gh CLI + try: + import subprocess + + result = subprocess.run( + ["gh", "auth", "token"], + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip() + except (subprocess.CalledProcessError, FileNotFoundError): + pass + + return None + + +def get_headers(token: Optional[str]) -> dict: + """Get request headers with optional authentication.""" + headers = { + "Accept": "application/vnd.github.v3+json", + } + if token: + headers["Authorization"] = f"Bearer {token}" + return headers + + +def fetch_workflow_runs( + token: Optional[str], + days: int = 30, + event: Optional[str] = None, +) -> list: + """Fetch completed workflow runs from GitHub Actions.""" + url = f"https://api.github.com/repos/{GITHUB_REPO}/actions/workflows/{WORKFLOW_NAME}/runs" + + params = { + "status": "completed", + "per_page": 100, + } + + if event: + params["event"] = event + + response = requests.get(url, headers=get_headers(token), params=params, timeout=30) + response.raise_for_status() + + runs = response.json().get("workflow_runs", []) + + # Filter by date + cutoff = datetime.now(timezone.utc) - timedelta(days=days) + runs = [ + run + for run in runs + if datetime.fromisoformat(run["created_at"].replace("Z", "+00:00")) > cutoff + ] + + return runs + + +def fetch_run_artifacts(token: Optional[str], run_id: int) -> list: + """Fetch artifacts for a specific workflow run.""" + url = f"https://api.github.com/repos/{GITHUB_REPO}/actions/runs/{run_id}/artifacts" + + response = requests.get(url, headers=get_headers(token), timeout=30) + response.raise_for_status() + + return response.json().get("artifacts", []) + + +def download_artifact(token: Optional[str], artifact_id: int) -> Optional[bytes]: + """Download an artifact by ID.""" + if not token: + print(f"Warning: GitHub token required to download artifacts", file=sys.stderr) + return None + + url = f"https://api.github.com/repos/{GITHUB_REPO}/actions/artifacts/{artifact_id}/zip" + + headers = get_headers(token) + response = requests.get(url, headers=headers, allow_redirects=True, timeout=60) + + if response.status_code == 200: + return response.content + + print( + f"Failed to download artifact {artifact_id}: {response.status_code}", + file=sys.stderr, + ) + return None + + +def extract_metrics_from_zip(zip_content: bytes) -> Optional[dict]: + """Extract metrics JSON from a zip file.""" + try: + with zipfile.ZipFile(io.BytesIO(zip_content)) as zf: + # Find the JSON file in the archive + json_files = [f for f in zf.namelist() if f.endswith(".json")] + if not json_files: + return None + + with zf.open(json_files[0]) as f: + return json.load(f) + except (zipfile.BadZipFile, json.JSONDecodeError) as e: + print(f"Failed to extract metrics: {e}", file=sys.stderr) + return None + + +def fetch_metrics_for_run(token: Optional[str], run: dict) -> Optional[dict]: + """Fetch metrics for a single workflow run.""" + run_id = run["id"] + print(f"Fetching metrics for run {run_id}...", file=sys.stderr) + + artifacts = fetch_run_artifacts(token, run_id) + + # Find consolidated metrics artifact + metrics_artifact = None + for artifact in artifacts: + if artifact["name"].startswith(ARTIFACT_PREFIX): + metrics_artifact = artifact + break + + if not metrics_artifact: + print(f"No consolidated metrics found for run {run_id}", file=sys.stderr) + return None + + # Download and extract + zip_content = download_artifact(token, metrics_artifact["id"]) + if not zip_content: + return None + + metrics = extract_metrics_from_zip(zip_content) + if not metrics: + return None + + # Ensure required fields are present + if "run_id" not in metrics: + metrics["run_id"] = str(run_id) + if "run_date" not in metrics: + metrics["run_date"] = run["created_at"] + if "commit_sha" not in metrics: + metrics["commit_sha"] = run["head_sha"] + if "branch" not in metrics: + metrics["branch"] = run["head_branch"] + + return metrics + + +def fetch_single_run(token: Optional[str], run_id: int) -> Optional[dict]: + """Fetch metrics for a single run by ID.""" + url = f"https://api.github.com/repos/{GITHUB_REPO}/actions/runs/{run_id}" + + response = requests.get(url, headers=get_headers(token), timeout=30) + response.raise_for_status() + + run = response.json() + return fetch_metrics_for_run(token, run) + + +def main(): + parser = argparse.ArgumentParser( + description="Fetch SGLang nightly test metrics from GitHub Actions" + ) + parser.add_argument( + "--output", + "-o", + type=str, + default="metrics_data.json", + help="Output JSON file path", + ) + parser.add_argument( + "--days", + type=int, + default=30, + help="Number of days to fetch (default: 30)", + ) + parser.add_argument( + "--run-id", + type=int, + help="Fetch a specific run by ID", + ) + parser.add_argument( + "--event", + type=str, + choices=["schedule", "workflow_dispatch", "push"], + help="Filter by trigger event type", + ) + parser.add_argument( + "--scheduled-only", + action="store_true", + help="Only fetch scheduled (nightly) runs", + ) + + args = parser.parse_args() + + token = get_github_token() + if not token: + print( + "Warning: No GitHub token found. Some features may be limited.", + file=sys.stderr, + ) + print( + "Set GITHUB_TOKEN env var or login with 'gh auth login'", + file=sys.stderr, + ) + + all_metrics = [] + + if args.run_id: + # Fetch single run + metrics = fetch_single_run(token, args.run_id) + if metrics: + all_metrics.append(metrics) + else: + # Fetch multiple runs + event = "schedule" if args.scheduled_only else args.event + runs = fetch_workflow_runs(token, days=args.days, event=event) + print(f"Found {len(runs)} workflow runs", file=sys.stderr) + + for run in runs: + metrics = fetch_metrics_for_run(token, run) + if metrics: + all_metrics.append(metrics) + + # Sort by date descending + all_metrics.sort(key=lambda x: x.get("run_date", ""), reverse=True) + + # Write output + output_path = Path(args.output) + with open(output_path, "w") as f: + json.dump(all_metrics, f, indent=2) + + print(f"Wrote {len(all_metrics)} metrics records to {output_path}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/docs/performance_dashboard/index.html b/docs/performance_dashboard/index.html new file mode 100644 index 000000000..6bd63d72d --- /dev/null +++ b/docs/performance_dashboard/index.html @@ -0,0 +1,460 @@ + + + + + + SGLang Performance Dashboard + + + + + +
+
+

+ + + + + + SGLang Performance Dashboard +

+
+ + + View Workflow + +
+
+ +
+ +
+ + + + + + +
+ + + + diff --git a/docs/performance_dashboard/server.py b/docs/performance_dashboard/server.py new file mode 100755 index 000000000..0713c19f4 --- /dev/null +++ b/docs/performance_dashboard/server.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python3 +""" +Simple development server for the SGLang Performance Dashboard. + +This server: +1. Serves the static HTML/JS files +2. Provides an API endpoint to fetch metrics from GitHub +3. Caches metrics data to reduce API calls + +Usage: + python server.py + python server.py --port 8080 + python server.py --host 0.0.0.0 # Allow external access + python server.py --fetch-on-start +""" + +import argparse +import http.server +import io +import json +import os +import socketserver +import threading +import time +import zipfile +from datetime import datetime, timedelta, timezone +from pathlib import Path +from urllib.parse import urlparse + +import requests + +GITHUB_REPO = "sgl-project/sglang" +WORKFLOW_NAME = "nightly-test-nvidia.yml" +ARTIFACT_PREFIX = "consolidated-metrics-" + +# Cache for metrics data with thread-safe lock +cache_lock = threading.Lock() +metrics_cache = { + "data": [], + "last_updated": None, + "updating": False, +} + +CACHE_TTL = 300 # 5 minutes +REQUEST_TIMEOUT = 30 # seconds + + +def get_github_token(): + """Get GitHub token from environment or gh CLI.""" + token = os.environ.get("GITHUB_TOKEN") + if token: + return token + + try: + import subprocess + + result = subprocess.run( + ["gh", "auth", "token"], + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip() + except (subprocess.CalledProcessError, FileNotFoundError): + pass + + return None + + +def fetch_metrics_from_github(days=30): + """Fetch metrics from GitHub Actions artifacts.""" + token = get_github_token() + headers = {"Accept": "application/vnd.github.v3+json"} + if token: + headers["Authorization"] = f"Bearer {token}" + + # Get workflow runs - only scheduled (nightly) runs, not workflow_dispatch + url = f"https://api.github.com/repos/{GITHUB_REPO}/actions/workflows/{WORKFLOW_NAME}/runs" + params = {"status": "completed", "per_page": 50, "event": "schedule"} + + try: + response = requests.get( + url, headers=headers, params=params, timeout=REQUEST_TIMEOUT + ) + if not response.ok: + print(f"Failed to fetch workflow runs: {response.status_code}") + return [] + except requests.exceptions.RequestException as e: + print(f"Network error fetching workflow runs: {e}") + return [] + + runs = response.json().get("workflow_runs", []) + + # Filter by date + cutoff = datetime.now(timezone.utc) - timedelta(days=days) + runs = [ + run + for run in runs + if datetime.fromisoformat(run["created_at"].replace("Z", "+00:00")) > cutoff + ] + + all_metrics = [] + + for run in runs[:20]: # Limit to 20 most recent + run_id = run["id"] + + # Get artifacts + artifacts_url = f"https://api.github.com/repos/{GITHUB_REPO}/actions/runs/{run_id}/artifacts" + try: + artifacts_resp = requests.get( + artifacts_url, headers=headers, timeout=REQUEST_TIMEOUT + ) + if not artifacts_resp.ok: + continue + except requests.exceptions.RequestException as e: + print(f"Network error fetching artifacts for run {run_id}: {e}") + continue + + artifacts = artifacts_resp.json().get("artifacts", []) + + # Find consolidated metrics + for artifact in artifacts: + if artifact["name"].startswith(ARTIFACT_PREFIX): + if not token: + # Without token, we can't download - return metadata only + all_metrics.append( + { + "run_id": str(run_id), + "run_date": run["created_at"], + "commit_sha": run["head_sha"], + "branch": run["head_branch"], + "results": [], + } + ) + break + + # Download artifact + download_url = f"https://api.github.com/repos/{GITHUB_REPO}/actions/artifacts/{artifact['id']}/zip" + try: + download_resp = requests.get( + download_url, + headers=headers, + allow_redirects=True, + timeout=REQUEST_TIMEOUT, + ) + except requests.exceptions.RequestException as e: + print(f"Network error downloading artifact: {e}") + break + + if download_resp.ok: + try: + with zipfile.ZipFile(io.BytesIO(download_resp.content)) as zf: + json_files = [ + f for f in zf.namelist() if f.endswith(".json") + ] + if json_files: + with zf.open(json_files[0]) as f: + metrics = json.load(f) + # Ensure required fields + metrics.setdefault("run_id", str(run_id)) + metrics.setdefault("run_date", run["created_at"]) + metrics.setdefault("commit_sha", run["head_sha"]) + metrics.setdefault("branch", run["head_branch"]) + all_metrics.append(metrics) + except (zipfile.BadZipFile, json.JSONDecodeError) as e: + print(f"Failed to process artifact: {e}") + break + + return all_metrics + + +def update_cache_async(): + """Update the metrics cache in background with thread safety.""" + with cache_lock: + if metrics_cache["updating"]: + return + metrics_cache["updating"] = True + + try: + data = fetch_metrics_from_github() + with cache_lock: + metrics_cache["data"] = data + metrics_cache["last_updated"] = time.time() + print(f"Cache updated with {len(data)} metrics records") + finally: + with cache_lock: + metrics_cache["updating"] = False + + +class DashboardHandler(http.server.SimpleHTTPRequestHandler): + """HTTP request handler for the dashboard.""" + + def __init__(self, *args, directory=None, **kwargs): + super().__init__(*args, directory=directory, **kwargs) + + def do_GET(self): + parsed = urlparse(self.path) + + # Prevent directory traversal attacks + if ".." in parsed.path or parsed.path.startswith("//"): + self.send_error(400, "Invalid path") + return + + if parsed.path == "/api/metrics": + self.handle_metrics_api(parsed) + elif parsed.path == "/api/refresh": + self.handle_refresh_api() + else: + super().do_GET() + + def handle_metrics_api(self, parsed): + """Handle /api/metrics endpoint.""" + # Check cache with thread safety + with cache_lock: + cache_valid = ( + metrics_cache["last_updated"] + and time.time() - metrics_cache["last_updated"] < CACHE_TTL + ) + data = metrics_cache["data"].copy() + + if not cache_valid: + # Trigger background update + threading.Thread(target=update_cache_async, daemon=True).start() + + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Access-Control-Allow-Origin", "*") + self.end_headers() + self.wfile.write(json.dumps(data).encode()) + + def handle_refresh_api(self): + """Handle /api/refresh endpoint.""" + threading.Thread(target=update_cache_async, daemon=True).start() + + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Access-Control-Allow-Origin", "*") + self.end_headers() + self.wfile.write(json.dumps({"status": "refreshing"}).encode()) + + def log_message(self, format, *args): + """Custom log format.""" + print(f"[{self.log_date_time_string()}] {args[0]}") + + +def main(): + parser = argparse.ArgumentParser(description="SGLang Performance Dashboard Server") + parser.add_argument("--port", type=int, default=8000, help="Port to serve on") + parser.add_argument( + "--host", + default="127.0.0.1", + help="Host to bind to (use 0.0.0.0 for external access)", + ) + parser.add_argument( + "--fetch-on-start", action="store_true", help="Fetch metrics on startup" + ) + args = parser.parse_args() + + # Change to dashboard directory + dashboard_dir = Path(__file__).parent + os.chdir(dashboard_dir) + + if args.fetch_on_start: + print("Fetching initial metrics data...") + update_cache_async() + + handler = lambda *a, **kw: DashboardHandler(*a, directory=str(dashboard_dir), **kw) + + with socketserver.TCPServer((args.host, args.port), handler) as httpd: + print(f"Serving dashboard at http://{args.host}:{args.port}") + print("Press Ctrl+C to stop") + try: + httpd.serve_forever() + except KeyboardInterrupt: + print("\nShutting down...") + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/save_metrics.py b/scripts/ci/save_metrics.py index ac136e1b3..455c118bd 100755 --- a/scripts/ci/save_metrics.py +++ b/scripts/ci/save_metrics.py @@ -44,7 +44,11 @@ def parse_result_file(filepath: str) -> list[dict]: def transform_benchmark_result(result: dict, gpu_config: str, partition: int) -> dict: - """Transform a benchmark result to the metrics schema.""" + """Transform a benchmark result to the metrics schema. + + Note: input_len and output_len are preserved here for the flat benchmarks list, + but are also used as grouping keys in benchmarks_by_io_len. + """ # Handle None values safely for numeric conversions latency = result.get("latency") last_ttft = result.get("last_ttft") @@ -62,10 +66,20 @@ def transform_benchmark_result(result: dict, gpu_config: str, partition: int) -> } +def get_io_len_key(input_len: int, output_len: int) -> str: + """Generate a key for input/output length combination.""" + return f"{input_len}_{output_len}" + + def group_results_by_model( results: list[dict], gpu_config: str, partition: int ) -> list[dict]: - """Group benchmark results by model, variant, and server_args.""" + """Group benchmark results by model, variant, and server_args. + + Results are organized with two benchmark structures: + - benchmarks: flat list of all benchmarks (for backward compatibility) + - benchmarks_by_io_len: nested structure grouped by input/output length combinations + """ groups = {} for result in results: @@ -85,11 +99,35 @@ def group_results_by_model( "variant": variant, "server_args": server_args, "benchmarks": [], + "benchmarks_by_io_len": {}, } - groups[key]["benchmarks"].append( - transform_benchmark_result(result, gpu_config, partition) - ) + transformed = transform_benchmark_result(result, gpu_config, partition) + + # Add to flat benchmarks list (backward compatibility) + groups[key]["benchmarks"].append(transformed) + + # Add to nested benchmarks_by_io_len structure + input_len = result.get("input_len") + output_len = result.get("output_len") + if input_len is not None and output_len is not None: + io_key = get_io_len_key(input_len, output_len) + if io_key not in groups[key]["benchmarks_by_io_len"]: + groups[key]["benchmarks_by_io_len"][io_key] = { + "input_len": input_len, + "output_len": output_len, + "benchmarks": [], + } + # For the nested structure, exclude input_len and output_len from individual benchmarks + # since they're already in the parent + nested_benchmark = { + k: v + for k, v in transformed.items() + if k not in ("input_len", "output_len") + } + groups[key]["benchmarks_by_io_len"][io_key]["benchmarks"].append( + nested_benchmark + ) return list(groups.values())