diff --git a/.github/bench-dashboard/index.html b/.github/bench-dashboard/index.html index 5490d71ac..cdc162e6e 100644 --- a/.github/bench-dashboard/index.html +++ b/.github/bench-dashboard/index.html @@ -78,10 +78,80 @@ margin-top: 14px; min-height: 460px; } - #chart { + #chart, #chart-profile { width: 100% !important; height: 440px !important; } + /* HTML legend replaces Chart.js built-in legend: a full-width row + is the click target (fixing the issue where only the colour swatch + registered clicks and wrapped labels drifted onto the next line), + and the container scrolls when series count exceeds the visible + height (fixing silent series hiding by Chart.js). */ + .html-legend { + margin-top: 10px; + max-height: 180px; + overflow-y: auto; + border-top: 1px solid #e4ece8; + padding-top: 8px; + } + .html-legend-item { + display: flex; + align-items: center; + gap: 8px; + padding: 4px 8px; + border-radius: 6px; + cursor: pointer; + font-size: 0.85rem; + line-height: 1.3; + user-select: none; + /* Render elements as full-width text rows rather than + default button chrome — the buttons exist for a11y (keyboard + + screen reader support), not for visual styling. */ + width: 100%; + background: transparent; + border: 1px solid transparent; + text-align: left; + font-family: inherit; + color: inherit; + } + .html-legend-item:hover { + background: var(--accent-soft); + } + .html-legend-item:focus-visible { + outline: none; + border-color: var(--accent); + background: var(--accent-soft); + } + .html-legend-item.hidden-series .html-legend-label { + text-decoration: line-through; + opacity: 0.55; + } + .html-legend-swatch { + width: 14px; + height: 14px; + border-radius: 3px; + flex-shrink: 0; + } + .html-legend-label { + flex: 1; + overflow-wrap: anywhere; + } + .profile-toggles { + display: flex; + flex-wrap: wrap; + gap: 12px; + margin-top: 10px; + } + .profile-toggles label { + display: inline-flex; + align-items: center; + gap: 6px; + font-size: 0.88rem; + cursor: pointer; + } + .profile-toggles input[type="checkbox"] { + margin: 0; + } #status { margin-top: 8px; font-size: 0.88rem; @@ -183,9 +253,47 @@ Rust vs C FFI Relative Dashboard + + + Level Profile (speed + ratio) + + Absolute throughput and compression ratio per level for a single scenario, with + Rust and FFI overlaid. Filled-area lines make horizontal divergence (level + where Rust falls behind FFI on speed or ratio) easy to spot. + + + Target + + + Scenario + + + Stage + + + Source + + + Snapshot + + + + + Rust speed + FFI speed + Rust ratio + FFI ratio + + + + + + + + Raw timings (secondary view) @@ -228,6 +336,13 @@ Rust vs C FFI Relative Dashboard const state = { records: [], chart: null, + profileChart: null, + profileSeriesVisibility: { + rust_speed: true, + ffi_speed: true, + rust_ratio: true, + ffi_ratio: true, + }, timeIndex: new Map(), windowInitialized: false, }; @@ -245,6 +360,21 @@ Rust vs C FFI Relative Dashboard to: el("f-to"), }; + const profileSelectors = { + target: el("pf-target"), + scenario: el("pf-scenario"), + stage: el("pf-stage"), + source: el("pf-source"), + snapshot: el("pf-snapshot"), + }; + + // Sentinel for the snapshot dropdown's "latest" entry, which picks + // whichever record was generated last for each (level, metric) + // independently. Selecting a specific timestamp instead pins the + // chart to that one snapshot — useful when comparing two commits + // side-by-side via browser tabs. + const PROFILE_LATEST = "__latest__"; + function parseTimestamp(value) { const parsed = Date.parse(String(value || "")); return Number.isFinite(parsed) ? parsed : null; @@ -500,6 +630,30 @@ Rust vs C FFI Relative Dashboard return `${target} • ${scenario || "scenario"} • ${stage || "stage"}:${level || "level"} • ${source || "none"} • ${metric || "metric"}`; } + // Sign-gated regression classification. A value "outside the parity + // band" is only a regression when it moves AGAINST us: + // - compression_ratio: delta = rust_ratio / ffi_ratio; per emitter + // interpretation (run-benchmarks.sh), delta > 1 means Rust produced + // a LARGER compressed output than FFI — that's a ratio regression. + // delta < 1 is a win and must not count as "outside band". + // - throughput_bytes_per_sec: delta > 1 means Rust faster than FFI. + // delta < 1 (Rust slower) is the regression direction. + // - peak_alloc_bytes: delta > 1 means Rust allocated more than FFI — + // that's the regression direction. + // For any other / unknown metric, fall back to symmetric out-of-band + // detection so we don't silently drop signals from future metrics. + function isRegression(row) { + if (typeof row.delta_ratio !== "number") return false; + const metric = row.metric; + if (metric === "throughput_bytes_per_sec") { + return row.delta_ratio < BAND_LOW; + } + if (metric === "compression_ratio" || metric === "peak_alloc_bytes") { + return row.delta_ratio > BAND_HIGH; + } + return row.delta_ratio < BAND_LOW || row.delta_ratio > BAND_HIGH; + } + function updateStatus(filtered) { const numericRows = filtered.filter((row) => typeof row.delta_ratio === "number"); if (!numericRows.length) { @@ -522,32 +676,45 @@ Rust vs C FFI Relative Dashboard if (latestPerSeries.length === 1) { const latest = latestPerSeries[0].row; const pct = ((latest.delta_ratio - 1) * 100).toFixed(2); + const regression = isRegression(latest); const inBand = latest.delta_ratio >= BAND_LOW && latest.delta_ratio <= BAND_HIGH; + const label = regression + ? "outside parity band" + : inBand + ? "within parity band" + : "outside band (win — Rust beat FFI)"; setStatusText( - `Latest delta: ${latest.delta_ratio.toFixed(4)} (${pct}%) ${inBand ? "within" : "outside"} parity band`, - inBand ? "near-parity" : "outside-band", + `Latest delta: ${latest.delta_ratio.toFixed(4)} (${pct}%) ${label}`, + regression ? "outside-band" : "near-parity", ); return; } const perSeries = latestPerSeries.map(({ seriesId, row }) => { const pct = ((row.delta_ratio - 1) * 100).toFixed(2); + const regression = isRegression(row); const inBand = row.delta_ratio >= BAND_LOW && row.delta_ratio <= BAND_HIGH; const deviation = Math.abs(row.delta_ratio - 1); + const verdict = regression ? "outside band" : inBand ? "within band" : "win"; return { seriesId, + regression, inBand, deviation, - text: `${compactSeriesLabel(seriesId)}: ${row.delta_ratio.toFixed(4)} (${pct}%) ${inBand ? "within" : "outside"} band`, + text: `${compactSeriesLabel(seriesId)}: ${row.delta_ratio.toFixed(4)} (${pct}%) ${verdict}`, }; }); const status = el("status"); status.replaceChildren(); - const outsideBand = perSeries.filter((item) => !item.inBand); - const focus = (outsideBand.length ? outsideBand : perSeries) + const regressions = perSeries.filter((item) => item.regression); + const wins = perSeries.filter((item) => !item.regression && !item.inBand); + const focus = (regressions.length ? regressions : perSeries) .sort((a, b) => b.deviation - a.deviation) .slice(0, 8); const summary = document.createElement("div"); - summary.textContent = `Latest series: ${perSeries.length}. Outside parity band: ${outsideBand.length}. Showing top ${focus.length} by deviation.`; + summary.textContent = + `Latest series: ${perSeries.length}. Outside parity band: ${regressions.length}` + + (wins.length ? ` (notable wins not counted: ${wins.length}).` : ".") + + ` Showing top ${focus.length} by deviation.`; status.appendChild(summary); if (focus.length > 0) { const list = document.createElement("ul"); @@ -558,7 +725,65 @@ Rust vs C FFI Relative Dashboard } status.appendChild(list); } - status.className = outsideBand.length === 0 ? "near-parity" : "outside-band"; + status.className = regressions.length === 0 ? "near-parity" : "outside-band"; + } + + // Replaces Chart.js built-in legend. Each row is a full-width click + // target (fixes the bug where only the colour swatch registered + // clicks while the wrapping label drifted to the next line), the + // container scrolls when there are too many series for the visible + // area (fixes silent series hiding), and toggling visibility uses + // the same Chart.js `meta.hidden` mechanism the built-in legend + // used so dataset rendering stays identical. + // `onToggle(index, ds, nextVisible)` lets the caller route the + // visibility change through external state (e.g. the profile + // chart's checkbox-backed `profileSeriesVisibility`). When + // omitted, the legend toggles the dataset directly — matching + // the built-in Chart.js legend behaviour the delta chart relies + // on. Hidden detection uses `chart.isDatasetVisible(i)` so the + // `hidden-series` styling reflects both `meta.hidden` toggles + // (legend clicks) AND `dataset.hidden` set at construction time + // (which leaves `meta.hidden === null`). + function renderHtmlLegend(chart, container, onToggle) { + if (!chart || !container) return; + container.replaceChildren(); + const datasets = chart.data.datasets || []; + for (let i = 0; i < datasets.length; i++) { + const ds = datasets[i]; + // gives keyboard activation (Enter/Space), focus + // outline, and a button role for screen readers without any + // extra JS. aria-pressed reflects current visibility so AT + // users hear "pressed" / "not pressed" instead of guessing. + const item = document.createElement("button"); + item.type = "button"; + item.className = "html-legend-item"; + const visible = chart.isDatasetVisible(i); + item.setAttribute("aria-pressed", visible ? "true" : "false"); + if (!visible) item.classList.add("hidden-series"); + + const swatch = document.createElement("span"); + swatch.className = "html-legend-swatch"; + swatch.style.background = String(ds.borderColor || ds.backgroundColor || "#888"); + item.appendChild(swatch); + + const label = document.createElement("span"); + label.className = "html-legend-label"; + label.textContent = ds.label || `series ${i + 1}`; + item.appendChild(label); + + item.addEventListener("click", () => { + const nextVisible = !chart.isDatasetVisible(i); + if (typeof onToggle === "function") { + onToggle(i, ds, nextVisible); + return; + } + chart.setDatasetVisibility(i, nextVisible); + chart.update(); + renderHtmlLegend(chart, container, onToggle); + }); + + container.appendChild(item); + } } function renderChart(filtered) { @@ -603,11 +828,403 @@ Rust vs C FFI Relative Dashboard }, }, plugins: { - legend: { position: "bottom" }, + legend: { display: false }, }, maintainAspectRatio: false, }, }); + renderHtmlLegend(state.chart, el("legend-delta")); + } + + function profileFilter() { + return { + target: profileSelectors.target.value, + scenario: profileSelectors.scenario.value, + stage: profileSelectors.stage.value, + // Decompress stages emit per-implementation source values + // (`rust_stream`, `c_stream`) that share the same level keys. + // Without filtering by source the 4 profile lines would + // silently mix runs and render an incorrect chart, so the + // source dropdown is mandatory in the grouping key alongside + // target/scenario/stage. + source: profileSelectors.source.value, + snapshot: profileSelectors.snapshot.value || PROFILE_LATEST, + }; + } + + // Pick one record per (level, metric) for the chosen target / + // scenario / stage. When `snapshot === PROFILE_LATEST` we take + // whichever record is the most recent per (level, metric) + // independently — useful for the "current state across every + // level" view. When `snapshot` is a specific generated_at value + // (or commit sha) the chart is pinned to that exact run so the + // 4 lines describe one commit and nothing else, which is what + // you want when comparing two snapshots side-by-side. + function buildProfileData(filter) { + const rows = state.records.filter((r) => { + if ((r.target || "unknown-target") !== filter.target) return false; + if (r.scenario !== filter.scenario) return false; + if (r.stage !== filter.stage) return false; + if ((r.source || "none") !== filter.source) return false; + if (filter.snapshot !== PROFILE_LATEST) { + const label = r.generated_at || r.commit_sha || "local-run"; + if (label !== filter.snapshot) return false; + } + return true; + }); + const byLevelMetric = new Map(); + for (const row of rows) { + if (typeof row.rust_value !== "number" || typeof row.ffi_value !== "number") continue; + const key = `${row.level}|${row.metric}`; + const existing = byLevelMetric.get(key); + if (!existing || compareByGeneratedAt(row, existing) > 0) { + byLevelMetric.set(key, row); + } + } + + const levelSet = new Set(); + for (const row of byLevelMetric.values()) levelSet.add(row.level); + const levels = Array.from(levelSet) + .map((value) => ({ value, parsed: parseLevelId(value) })) + .sort((a, b) => { + if (a.parsed && b.parsed) return a.parsed.numeric - b.parsed.numeric; + if (a.parsed) return -1; + if (b.parsed) return 1; + return a.value.localeCompare(b.value); + }); + + const levelLabels = levels.map((entry) => + entry.parsed ? entry.parsed.label : entry.value, + ); + + const lookup = (level, metric, side) => { + const row = byLevelMetric.get(`${level}|${metric}`); + if (!row) return null; + // Speed records carry bytes/sec, divide to MiB/s for axis legibility. + if (metric === "throughput_bytes_per_sec") { + return side === "rust" ? row.rust_value / (1024 * 1024) : row.ffi_value / (1024 * 1024); + } + return side === "rust" ? row.rust_value : row.ffi_value; + }; + + const rustSpeed = levels.map((e) => lookup(e.value, "throughput_bytes_per_sec", "rust")); + const ffiSpeed = levels.map((e) => lookup(e.value, "throughput_bytes_per_sec", "ffi")); + const rustRatio = levels.map((e) => lookup(e.value, "compression_ratio", "rust")); + const ffiRatio = levels.map((e) => lookup(e.value, "compression_ratio", "ffi")); + + return { + levels: levelLabels, + levelCount: levels.length, + datasets: [ + { + seriesKey: "rust_speed", + label: "Rust speed (MiB/s)", + data: rustSpeed, + yAxisID: "y", + borderColor: "hsl(150 65% 38%)", + backgroundColor: "hsla(150 65% 38% / 0.3)", + }, + { + seriesKey: "ffi_speed", + label: "FFI speed (MiB/s)", + data: ffiSpeed, + yAxisID: "y", + borderColor: "hsl(205 65% 45%)", + backgroundColor: "hsla(205 65% 45% / 0.3)", + }, + { + seriesKey: "rust_ratio", + label: "Rust ratio", + data: rustRatio, + yAxisID: "y1", + borderColor: "hsl(30 75% 45%)", + backgroundColor: "hsla(30 75% 45% / 0.3)", + borderDash: [6, 4], + }, + { + seriesKey: "ffi_ratio", + label: "FFI ratio", + data: ffiRatio, + yAxisID: "y1", + borderColor: "hsl(340 65% 50%)", + backgroundColor: "hsla(340 65% 50% / 0.3)", + borderDash: [6, 4], + }, + ], + }; + } + + // Single source of truth for the chart instance: created once on + // first render, then patched in place on every selector change + // (scenario / stage / target) and on every toggle. Recreating the + // chart would drop animation continuity AND the Chart.js-internal + // visibility state — both regressions the PR's acceptance criteria + // explicitly call out ("repopulate without recreating the chart"). + function buildProfileDatasets(rawDatasets) { + return rawDatasets.map((ds) => ({ + label: ds.label, + data: ds.data, + yAxisID: ds.yAxisID, + borderColor: ds.borderColor, + backgroundColor: ds.backgroundColor, + borderDash: ds.borderDash, + borderWidth: 2, + pointRadius: 3, + spanGaps: true, + tension: 0.35, + fill: "origin", + hidden: !state.profileSeriesVisibility[ds.seriesKey], + _seriesKey: ds.seriesKey, + })); + } + + function onProfileLegendToggle(index, ds, nextVisible) { + const key = ds._seriesKey; + if (key) { + // Persist visibility across re-renders and keep the checkbox UI + // in sync — without this, clicking the legend produced a state + // mismatch with the checkbox, and a subsequent scenario switch + // would silently restore the hidden series. + state.profileSeriesVisibility[key] = nextVisible; + const checkbox = document.querySelector( + `.profile-toggles input[type=checkbox][data-series="${key}"]`, + ); + if (checkbox) checkbox.checked = nextVisible; + } + if (state.profileChart) { + state.profileChart.setDatasetVisibility(index, nextVisible); + state.profileChart.update("none"); + renderHtmlLegend( + state.profileChart, + el("legend-profile"), + onProfileLegendToggle, + ); + } + } + + function renderProfileChart() { + const filter = profileFilter(); + if (!filter.target || !filter.scenario || !filter.stage) { + el("profile-status").textContent = "Select target, scenario and stage to render the level profile."; + return; + } + const { levels, levelCount, datasets } = buildProfileData(filter); + if (!levelCount) { + if (state.profileChart) { + // No data for the chosen filter — clear the chart in place + // rather than destroying it, so the next non-empty selection + // still updates the existing instance. + state.profileChart.data.labels = []; + state.profileChart.data.datasets = []; + state.profileChart.update("none"); + } + el("legend-profile").replaceChildren(); + el("profile-status").textContent = `No level points for target=${filter.target}, scenario=${filter.scenario}, stage=${filter.stage}, source=${filter.source}.`; + return; + } + const snapshotNote = filter.snapshot === PROFILE_LATEST + ? "latest per level" + : `snapshot=${filter.snapshot}`; + el("profile-status").textContent = + `Showing ${levelCount} level point(s) for scenario=${filter.scenario}, stage=${filter.stage}, source=${filter.source} (${snapshotNote}).`; + + const chartDatasets = buildProfileDatasets(datasets); + + if (state.profileChart) { + // In-place update path: swap labels + datasets, then ask + // Chart.js to redraw without animation. This preserves the + // chart instance (tooltips, scales, animation state) across + // every target/scenario/stage switch. + state.profileChart.data.labels = levels; + state.profileChart.data.datasets = chartDatasets; + state.profileChart.update("none"); + } else { + state.profileChart = new Chart(el("chart-profile"), { + type: "line", + data: { labels: levels, datasets: chartDatasets }, + options: { + responsive: true, + maintainAspectRatio: false, + interaction: { mode: "index", intersect: false }, + scales: { + x: { + title: { display: true, text: "Compression level" }, + ticks: { autoSkip: false, maxRotation: 60, minRotation: 30 }, + }, + y: { + type: "linear", + position: "left", + title: { display: true, text: "Throughput (MiB/s)" }, + beginAtZero: true, + }, + y1: { + type: "linear", + position: "right", + title: { display: true, text: "Compression ratio" }, + grid: { drawOnChartArea: false }, + }, + }, + plugins: { + legend: { display: false }, + tooltip: { mode: "index", intersect: false }, + }, + }, + }); + } + renderHtmlLegend( + state.profileChart, + el("legend-profile"), + onProfileLegendToggle, + ); + } + + function applyProfileToggles() { + if (!state.profileChart) return; + const datasets = state.profileChart.data.datasets || []; + for (let i = 0; i < datasets.length; i++) { + const key = datasets[i]._seriesKey; + if (!key) continue; + state.profileChart.setDatasetVisibility(i, !!state.profileSeriesVisibility[key]); + } + state.profileChart.update("none"); + renderHtmlLegend( + state.profileChart, + el("legend-profile"), + onProfileLegendToggle, + ); + } + + function bindProfileControls() { + // Filter cascade: + // target/scenario/stage change → source list may change + // → snapshot list may change + // source change → snapshot list may change + // Each downstream selector is repopulated before re-rendering, + // preserving the existing selection where it remains valid. + const cascadeUpper = ["target", "scenario", "stage"]; + for (const [name, control] of Object.entries(profileSelectors)) { + if (cascadeUpper.includes(name)) { + control.addEventListener("change", () => { + repopulateSourceOptions(); + repopulateSnapshotOptions(); + renderProfileChart(); + }); + } else if (name === "source") { + control.addEventListener("change", () => { + repopulateSnapshotOptions(); + renderProfileChart(); + }); + } else { + control.addEventListener("change", renderProfileChart); + } + } + for (const cb of document.querySelectorAll(".profile-toggles input[type=checkbox]")) { + cb.addEventListener("change", (e) => { + const key = e.currentTarget.dataset.series; + if (!key) return; + state.profileSeriesVisibility[key] = e.currentTarget.checked; + applyProfileToggles(); + }); + } + } + + // Build a label per record that's stable across reloads but still + // informative: prefer the timestamp (which the dropdown sorts + // chronologically via compareUniqValues), but fall back to commit + // SHA when there's no timestamp and finally to "local-run". The + // dropdown value is this same label so the selection survives a + // filter change as long as the snapshot still produces data for + // the new filter. + function snapshotLabel(row) { + return row.generated_at || row.commit_sha || "local-run"; + } + + function repopulateSourceOptions() { + const target = profileSelectors.target.value; + const scenario = profileSelectors.scenario.value; + const stage = profileSelectors.stage.value; + const sources = uniq( + state.records + .filter((r) => + (r.target || "unknown-target") === target && + r.scenario === scenario && + r.stage === stage, + ) + .map((r) => r.source || "none"), + ); + const previous = profileSelectors.source.value; + const fragment = document.createDocumentFragment(); + for (const value of sources) { + const opt = document.createElement("option"); + opt.value = String(value); + opt.textContent = String(value); + fragment.appendChild(opt); + } + profileSelectors.source.replaceChildren(fragment); + profileSelectors.source.value = sources.includes(previous) + ? previous + : (sources.includes("none") ? "none" : (sources[0] || "none")); + } + + function repopulateSnapshotOptions() { + const filter = { + target: profileSelectors.target.value, + scenario: profileSelectors.scenario.value, + stage: profileSelectors.stage.value, + source: profileSelectors.source.value, + }; + const labels = uniq( + state.records + .filter((r) => + (r.target || "unknown-target") === filter.target && + r.scenario === filter.scenario && + r.stage === filter.stage && + (r.source || "none") === filter.source, + ) + .map(snapshotLabel), + ); + const previous = profileSelectors.snapshot.value; + const fragment = document.createDocumentFragment(); + const latestOpt = document.createElement("option"); + latestOpt.value = PROFILE_LATEST; + latestOpt.textContent = "latest"; + fragment.appendChild(latestOpt); + for (const label of labels) { + const opt = document.createElement("option"); + opt.value = String(label); + opt.textContent = String(label); + fragment.appendChild(opt); + } + profileSelectors.snapshot.replaceChildren(fragment); + profileSelectors.snapshot.value = labels.includes(previous) || previous === PROFILE_LATEST + ? previous + : PROFILE_LATEST; + } + + function renderProfileSelectors() { + const targets = uniq(state.records.map((r) => r.target || "unknown-target")); + const scenarios = uniq(state.records.map((r) => r.scenario).filter(Boolean)); + const stages = uniq(state.records.map((r) => r.stage).filter(Boolean)); + + const populate = (sel, values, preferred) => { + const fragment = document.createDocumentFragment(); + for (const value of values) { + const opt = document.createElement("option"); + opt.value = String(value); + opt.textContent = String(value); + fragment.appendChild(opt); + } + sel.replaceChildren(fragment); + if (preferred && values.includes(preferred)) sel.value = preferred; + }; + + populate(profileSelectors.target, targets, targets[0]); + populate(profileSelectors.scenario, scenarios, scenarios[0]); + // "compress" is the canonical stage for level profile (level + // sweeps only meaningfully exist on the compression side). + populate(profileSelectors.stage, stages, stages.includes("compress") ? "compress" : stages[0]); + repopulateSourceOptions(); + repopulateSnapshotOptions(); } function currentFilter() { @@ -729,6 +1346,9 @@ Rust vs C FFI Relative Dashboard selectors.metric.value = availableMetrics.includes(defaultMetric) ? defaultMetric : (availableMetrics[0] || ALWAYS); bindFilters(); rerender(); + renderProfileSelectors(); + bindProfileControls(); + renderProfileChart(); renderRawSummary(); }
+ Absolute throughput and compression ratio per level for a single scenario, with + Rust and FFI overlaid. Filled-area lines make horizontal divergence (level + where Rust falls behind FFI on speed or ratio) easy to spot. +
@@ -228,6 +336,13 @@