From 059d8573e3f8818f7c521ae2d58d3d0a1a2f72aa Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Thu, 19 Mar 2026 16:08:32 +0800 Subject: [PATCH 01/41] Migrate DiskANN benchmark pipeline from ADO to GitHub Actions - Add benchmarks.yml workflow using workflow_dispatch, comparing current branch against a configurable baseline ref - Add compare_disk_index_json_output.py to diff benchmark crate JSON outputs into a CSV suitable for benchmark_result_parse.py - Add benchmark_result_parse.py for validating results and posting PR comments - Add wikipedia-100K-disk-index.json benchmark config using the public Wikipedia-100K dataset from big-ann-benchmarks (100K Cohere embeddings, 768-dim, cosine distance) to replace internal ADO datasets --- .github/scripts/benchmark_result_parse.py | 507 ++++++++++++++++++ .../scripts/compare_disk_index_json_output.py | 258 +++++++++ .github/workflows/benchmarks.yml | 318 +++++++++++ .../wikipedia-100K-disk-index.json | 40 ++ 4 files changed, 1123 insertions(+) create mode 100644 .github/scripts/benchmark_result_parse.py create mode 100644 .github/scripts/compare_disk_index_json_output.py create mode 100644 .github/workflows/benchmarks.yml create mode 100644 diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json diff --git a/.github/scripts/benchmark_result_parse.py b/.github/scripts/benchmark_result_parse.py new file mode 100644 index 0000000000..0308b29900 --- /dev/null +++ b/.github/scripts/benchmark_result_parse.py @@ -0,0 +1,507 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT license. + +""" +Benchmark Result Parser for GitHub Actions + +Parses benchmark CSV results and validates against thresholds. +Posts comments to GitHub PRs when regressions are detected. + +Migrated from ADO: .pipelines/templates/BenchmarkResultParse.py + +Usage: + python benchmark_result_parse.py --mode pr --file results.csv + python benchmark_result_parse.py --mode aa --file results.csv --data search + +Environment Variables (for PR comments): + GITHUB_TOKEN: GitHub token for API access + GITHUB_REPOSITORY: Owner/repo (e.g., "microsoft/DiskANN") + GITHUB_PR_NUMBER: Pull request number + GITHUB_RUN_ID: Workflow run ID for linking to logs +""" + +import csv +import os +import sys +import argparse +import json +from typing import Any + +# Optional: requests for posting PR comments +try: + import requests + HAS_REQUESTS = True +except ImportError: + HAS_REQUESTS = False + + +# ============================================================================= +# Data Structures +# ============================================================================= + +# Template for full benchmark data (build + search) +DATA_TEMPLATE_FULL = { + "DiskIndexBuild-PqConstruction": { + "duration_seconds": [], + "peak_memory_usage": [] + }, + "DiskIndexBuild-InmemIndexBuild": { + "duration_seconds": [], + "peak_memory_usage": [] + }, + "search_disk_index-search_completed": { + "duration_seconds": [], + "peak_memory_usage": [] + }, + "disk_index_perf_test": { + "total_duration_seconds": [], + }, + "index-build statistics": { + "total_time": [], + "total_comparisons": [], + "search_hops": [] + }, + "search-with-L=2000-bw=4": { + "latency_95": [], + "mean_latency": [], + "mean_io_time": [], + "mean_cpus": [], + "qps": [], + "mean_ios": [], + "mean_comps": [], + "mean_hops": [], + "recall": [] + } +} + +# Template for search-only benchmark data +DATA_TEMPLATE_SEARCH = { + "search_disk_index-search_completed": { + "duration_seconds": [], + "peak_memory_usage": [] + }, + "disk_index_perf_test": { + "total_duration_seconds": [], + }, + "search-with-L=2000-bw=4": { + "latency_95": [], + "mean_latency": [], + "mean_io_time": [], + "mean_cpus": [], + "qps": [], + "mean_ios": [], + "mean_comps": [], + "mean_hops": [], + "recall": [] + } +} + +# Thresholds for benchmark values +# Format: [threshold_percentage, direction, contract_value] +# - threshold_percentage: Maximum allowed deviation percentage +# - direction: 'GT' = higher is better, 'LT' = lower is better +# - contract_value: Promised performance value (empty string if none) +# +# For 'GT' metrics (like QPS, recall): regression if value decreases beyond threshold +# For 'LT' metrics (like latency, memory): regression if value increases beyond threshold +DATA_THRESHOLDS = { + "DiskIndexBuild-PqConstruction": { + "duration_seconds": [10, 'LT', ""], + "peak_memory_usage": [10, 'LT', ""] + }, + "DiskIndexBuild-InmemIndexBuild": { + "duration_seconds": [10, 'LT', ""], + "peak_memory_usage": [10, 'LT', ""] + }, + "search_disk_index-search_completed": { + "duration_seconds": [10, 'LT', ""], + "peak_memory_usage": [10, 'LT', 1.42] + }, + "disk_index_perf_test": { + "total_duration_seconds": [10, 'LT', ""], + }, + "index-build statistics": { + "total_time": [10, 'LT', 1206], + "total_comparisons": [1, 'LT', ""], + "search_hops": [1, 'LT', ""] + }, + "search-with-L=2000-bw=4": { + "latency_95": [10, 'LT', ""], + "mean_latency": [10, 'LT', ""], + "mean_io_time": [10, 'LT', ""], + "mean_cpus": [10, 'LT', ""], + "qps": [10, 'GT', 29], + "mean_ios": [1, 'LT', 2026], + "mean_comps": [1, 'LT', 50000], + "mean_hops": [1, 'LT', ""], + "recall": [1, 'GT', 95.1] + } +} + + +# ============================================================================= +# CSV Parsing +# ============================================================================= + +def parse_csv(file_path: str, data: dict[str, dict[str, list]]) -> dict[str, dict[str, list]]: + """ + Parse benchmark CSV file and populate data structure. + + CSV format expected: + Column 0: (unused) + Column 1: Category name (e.g., "search-with-L=2000-bw=4") + Column 2: Metric name (e.g., "qps") + Column 3: Current value + Column 4: Baseline value + Column 5: Change percentage + """ + with open(file_path, 'r', encoding='utf-8') as f: + reader = csv.reader(f) + next(reader) # Skip header row + + current_key = None + for row in reader: + if len(row) < 6: + continue + + # Column 1 contains category name (only set on first row of category) + if row[1]: + current_key = row[1] + elif current_key and current_key in data: + metric_name = row[2] + if metric_name in data[current_key]: + # Append: [current_value, baseline_value, change_percentage] + data[current_key][metric_name].append(row[3]) # current + data[current_key][metric_name].append(row[4]) # baseline + data[current_key][metric_name].append(row[5]) # change % + + return data + + +def get_data_template(data_type: str) -> dict[str, dict[str, list]]: + """Get a fresh copy of the data template.""" + import copy + if data_type == 'search': + return copy.deepcopy(DATA_TEMPLATE_SEARCH) + return copy.deepcopy(DATA_TEMPLATE_FULL) + + +# ============================================================================= +# Threshold Checking +# ============================================================================= + +def get_target_change_range(threshold: float, direction: str, mode: str) -> tuple[float, float]: + """ + Calculate acceptable change range based on threshold and direction. + + Args: + threshold: Maximum allowed deviation percentage + direction: 'GT' (higher is better) or 'LT' (lower is better) + mode: 'aa' (A/A test, symmetric) or 'pr' (PR test, directional) + + Returns: + Tuple of (min_allowed, max_allowed) change percentages + """ + if mode == 'aa': + # A/A test: symmetric threshold + return (-threshold, threshold) + else: + # PR test: directional threshold + if direction == 'GT': + # Higher is better: allow any improvement, flag regressions + return (-threshold, float('inf')) + else: + # Lower is better: allow any improvement (negative change), flag increases + return (float('-inf'), threshold) + + +def format_interval(start: float, end: float) -> str: + """Format a numeric interval as a string.""" + start_str = '-inf' if start == float('-inf') else f"{start}%" + end_str = 'inf' if end == float('inf') else f"{end}%" + return f"({start_str} - {end_str})" + + +def is_change_threshold_failed(change: float, target_range: tuple[float, float]) -> bool: + """Check if the change exceeds the allowed threshold range.""" + return change < target_range[0] or change > target_range[1] + + +def is_promise_broken(current_value: float, target_value: Any, direction: str) -> tuple[bool, str]: + """ + Check if the current value violates a promised contract value. + + Returns: + Tuple of (is_broken, formatted_target_value) + """ + if target_value == "": + return False, "N/A" + + target_value = float(target_value) + + if direction == 'GT': + # Higher is better: current should be >= target + if current_value < target_value: + return True, f"> {target_value}" + else: + # Lower is better: current should be <= target + if current_value > target_value: + return True, f"< {target_value}" + + return False, str(target_value) + + +def get_outcome_message(threshold_failed: bool, promise_broken: bool) -> str: + """Generate human-readable outcome message.""" + if threshold_failed and promise_broken: + return 'Regression detected, Promise broken' + elif promise_broken: + return 'Promise broken' + elif threshold_failed: + return 'Regression detected' + return 'OK' + + +def check_thresholds( + data: dict[str, dict[str, list]], + thresholds: dict[str, dict[str, list]], + mode: str, + run_id: str | None = None +) -> tuple[bool, str]: + """ + Check all metrics against their thresholds. + + Returns: + Tuple of (has_failures, failure_report_markdown) + """ + failed_rows = [] + + for category in data: + for metric in data[category]: + # Skip metrics without thresholds defined + if category not in thresholds or metric not in thresholds[category]: + print(f"Skipping {category}/{metric} - no threshold defined") + continue + + values = data[category][metric] + if not values: + print(f"ERROR: {category}/{metric} has no data") + return True, f"Missing data for {category}/{metric}" + + # Parse values: [current, baseline, change%] + try: + value_current = float(values[0]) + value_baseline = float(values[1]) + change = float(values[2]) if values[2] else 0.0 + except (ValueError, IndexError) as e: + print(f"ERROR: Failed to parse {category}/{metric}: {e}") + return True, f"Parse error for {category}/{metric}" + + # Get threshold config + threshold_config = thresholds[category][metric] + threshold_pct = threshold_config[0] + direction = threshold_config[1] + contract_value = threshold_config[2] + + # Check thresholds + target_range = get_target_change_range(threshold_pct, direction, mode) + threshold_failed = is_change_threshold_failed(change, target_range) + promise_broken, target_formatted = is_promise_broken(value_current, contract_value, direction) + + if threshold_failed: + print(f"THRESHOLD FAILED: {category}/{metric} change={change}% allowed={format_interval(*target_range)}") + if promise_broken: + print(f"PROMISE BROKEN: {category}/{metric} value={value_current} required={target_formatted}") + + if threshold_failed or promise_broken: + outcome = get_outcome_message(threshold_failed, promise_broken) + failed_rows.append( + f"| {category}/{metric} | {value_baseline} | {value_current} | " + f"{target_formatted} | {change}% | {format_interval(*target_range)} | {outcome} |" + ) + + if failed_rows: + # Build failure report + logs_link = "" + if run_id: + repo = os.getenv('GITHUB_REPOSITORY', 'microsoft/DiskANN') + logs_link = f"https://github.com/{repo}/actions/runs/{run_id}" + + report = "### ❌ Benchmark Check Failed\n\n" + if logs_link: + report += f"Please investigate the [workflow logs]({logs_link}) to determine if the failure is due to your changes.\n\n" + + report += "| Metric | Baseline | Current | Contract | Change | Allowed | Outcome |\n" + report += "|--------|----------|---------|----------|--------|---------|--------|\n" + report += "\n".join(failed_rows) + + return True, report + + return False, "" + + +# ============================================================================= +# GitHub Integration +# ============================================================================= + +def post_github_pr_comment(comment: str) -> bool: + """ + Post a comment to a GitHub pull request. + + Requires environment variables: + GITHUB_TOKEN: Personal access token or GitHub Actions token + GITHUB_REPOSITORY: Owner/repo format + GITHUB_PR_NUMBER: Pull request number + """ + if not HAS_REQUESTS: + print("WARNING: 'requests' module not available, cannot post PR comment") + return False + + token = os.getenv('GITHUB_TOKEN') + repo = os.getenv('GITHUB_REPOSITORY') + pr_number = os.getenv('GITHUB_PR_NUMBER') + + if not all([token, repo, pr_number]): + print("WARNING: Missing GitHub environment variables for PR comment") + print(f" GITHUB_TOKEN: {'set' if token else 'missing'}") + print(f" GITHUB_REPOSITORY: {repo or 'missing'}") + print(f" GITHUB_PR_NUMBER: {pr_number or 'missing'}") + return False + + url = f"https://api.github.com/repos/{repo}/issues/{pr_number}/comments" + headers = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "X-GitHub-Api-Version": "2022-11-28" + } + body = {"body": comment} + + try: + response = requests.post(url, headers=headers, json=body, timeout=30) + response.raise_for_status() + print(f"Successfully posted comment to PR #{pr_number}") + return True + except requests.RequestException as e: + print(f"ERROR: Failed to post PR comment: {e}") + return False + + +def write_github_step_summary(content: str) -> None: + """Write content to GitHub Actions step summary.""" + summary_file = os.getenv('GITHUB_STEP_SUMMARY') + if summary_file: + with open(summary_file, 'a', encoding='utf-8') as f: + f.write(content) + f.write("\n") + + +def write_github_output(name: str, value: str) -> None: + """Write an output variable for GitHub Actions.""" + output_file = os.getenv('GITHUB_OUTPUT') + if output_file: + with open(output_file, 'a', encoding='utf-8') as f: + f.write(f"{name}={value}\n") + + +# ============================================================================= +# Main +# ============================================================================= + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description='Parse benchmark results and validate against thresholds.', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Check PR benchmark results + python benchmark_result_parse.py --mode pr --file results_change.csv + + # Check A/A test results (symmetric thresholds) + python benchmark_result_parse.py --mode aa --file results_change.csv + + # Check search-only benchmarks + python benchmark_result_parse.py --mode pr --file results_change.csv --data search + """ + ) + parser.add_argument( + '--mode', + type=str, + default='aa', + choices=['aa', 'pr', 'lkg'], + help='Benchmark mode: aa=A/A test (symmetric), pr=PR test (directional), lkg=last known good' + ) + parser.add_argument( + '--data', + type=str, + default='both', + choices=['both', 'search'], + help='Type of benchmark data: both=full benchmark, search=search-only' + ) + parser.add_argument( + '--file', + type=str, + default=None, + help='Path to CSV file (overrides FILE_PATH env var)' + ) + parser.add_argument( + '--no-comment', + action='store_true', + help='Skip posting PR comment even in pr mode' + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + + # Get file path + file_path = args.file or os.getenv('FILE_PATH') + if not file_path: + print("ERROR: No input file specified. Use --file or set FILE_PATH env var.") + return 1 + + if not os.path.exists(file_path): + print(f"ERROR: File not found: {file_path}") + return 1 + + print(f"Benchmark mode: {args.mode}") + print(f"Data type: {args.data}") + print(f"Input file: {file_path}") + + # Parse CSV + data_template = get_data_template(args.data) + data = parse_csv(file_path, data_template) + + # Debug output + print("\nParsed data:") + print(json.dumps({k: {sk: sv for sk, sv in v.items() if sv} for k, v in data.items() if any(v.values())}, indent=2)) + + # Check thresholds + run_id = os.getenv('GITHUB_RUN_ID') + has_failures, report = check_thresholds(data, DATA_THRESHOLDS, args.mode, run_id) + + if has_failures: + print("\n" + report) + + # Write to GitHub step summary + write_github_step_summary(report) + + # Post PR comment if in pr mode + if args.mode == 'pr' and not args.no_comment: + post_github_pr_comment(report) + + # Set output for downstream steps + write_github_output('benchmark_failed', 'true') + + return 1 + + print("\n✅ All benchmark values passed!") + write_github_step_summary("### ✅ Benchmark Check Passed\n\nAll metrics within acceptable thresholds.") + write_github_output('benchmark_failed', 'false') + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/compare_disk_index_json_output.py b/.github/scripts/compare_disk_index_json_output.py new file mode 100644 index 0000000000..e3fa5afce8 --- /dev/null +++ b/.github/scripts/compare_disk_index_json_output.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT license. + +""" +Compare two disk-index benchmark JSON files and emit a diff CSV. + +This script takes baseline and branch (target) JSON files from the benchmark crate's +disk-index benchmarks and produces a CSV file comparing the metrics with deviation percentages. + +The output format matches the CSV structure expected by benchmark_result_parse.py: + Parent Span Name, Span Name, Stat Key, Stat Value (Target), Stat Value (Baseline), Deviation (%) + +Migrated from ADO: .pipelines/templates/compare_disk_index_json_output.py + +Usage: + python compare_disk_index_json_output.py \\ + --baseline baseline/target/tmp/_benchmark_crate_baseline.json \\ + --branch diskann_rust/target/tmp/_benchmark_crate_target.json \\ + --out diskann_rust/target/tmp/_change.csv +""" + +import json +import csv +import argparse +from typing import List, Dict, Any, Optional + + +def load_json(path: str) -> List[Dict[str, Any]]: + """Load JSON file and return the parsed content.""" + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def calc_deviation(baseline: float, target: float) -> str: + """Calculate the percentage deviation from baseline to target.""" + try: + if baseline != 0: + dev = ((target - baseline) / baseline) * 100 + return f"{dev:.2f}" + return "" + except Exception: + return "" + + +def extract_build_metrics(results: Dict[str, Any]) -> Dict[str, Any]: + """Extract build metrics from the results structure.""" + if not results: + return {} + + build = results.get("build", {}) + if not build: + return {} + + metrics = {} + + # Total build time (in seconds) + build_time = build.get("build_time") + if build_time: + # build_time is in microseconds, convert to seconds + metrics["total_time"] = build_time / 1e6 + + # Extract span metrics + span_metrics = build.get("span_metrics", {}) + spans = span_metrics.get("spans", []) + + for span in spans: + span_name = span.get("span_name", "") + span_data = span.get("metrics", {}) + + if span_name == "DiskIndexBuild-PqConstruction": + metrics["pq_construction_time"] = span_data.get("duration_seconds", 0) + elif span_name == "DiskIndexBuild-InmemIndexBuild": + metrics["inmem_index_build_time"] = span_data.get("duration_seconds", 0) + elif span_name == "DiskIndexBuild-DiskLayout": + metrics["disk_layout_time"] = span_data.get("duration_seconds", 0) + elif span_name == "disk-index-build": + metrics["total_build_duration"] = span_data.get("duration_seconds", 0) + + return metrics + + +def extract_search_metrics(results: Dict[str, Any], search_l: int, beam_width: int) -> Dict[str, Any]: + """Extract search metrics for a specific search_l value.""" + if not results: + return {} + + search = results.get("search", {}) + if not search: + return {} + + metrics = {} + + # Find the search result for the specified search_l + search_results = search.get("search_results_per_l", []) + for sr in search_results: + if sr.get("search_l") == search_l: + metrics["qps"] = sr.get("qps", 0) + metrics["recall"] = sr.get("recall", 0) + metrics["mean_latency"] = sr.get("mean_latency", 0) + metrics["mean_ios"] = sr.get("mean_ios", 0) + metrics["mean_comps"] = sr.get("mean_comparisons", 0) + metrics["mean_hops"] = sr.get("mean_hops", 0) + metrics["mean_io_time"] = sr.get("mean_io_time", 0) + metrics["mean_cpus"] = sr.get("mean_cpu_time", 0) + metrics["latency_95"] = sr.get("p999_latency", 0) # Use p999 as proxy for 95th percentile + break + + # Also try span metrics + span_metrics = search.get("span_metrics", {}) + spans = span_metrics.get("spans", []) + + search_span_name = f"search-with-L={search_l}-bw={beam_width}" + for span in spans: + if span.get("span_name") == search_span_name: + span_data = span.get("metrics", {}) + # Override with span metrics if they exist + if "qps" in span_data: + metrics["qps"] = span_data["qps"] + if "recall" in span_data: + metrics["recall"] = span_data["recall"] + if "mean_latency" in span_data: + metrics["mean_latency"] = span_data["mean_latency"] + if "mean_ios" in span_data: + metrics["mean_ios"] = span_data["mean_ios"] + if "mean_comps" in span_data: + metrics["mean_comps"] = span_data["mean_comps"] + if "mean_hops" in span_data: + metrics["mean_hops"] = span_data["mean_hops"] + if "mean_io_time" in span_data: + metrics["mean_io_time"] = span_data["mean_io_time"] + if "mean_cpus" in span_data: + metrics["mean_cpus"] = span_data["mean_cpus"] + break + + return metrics + + +def make_rows(baseline_list: List[Dict], target_list: List[Dict]) -> List[List[str]]: + """Generate comparison rows for the CSV output.""" + rows = [] + + for baseline, target in zip(baseline_list, target_list): + baseline_results = baseline.get("results", {}) + target_results = target.get("results", {}) + + # Get input info for context + inp = target.get("input", {}) + content = inp.get("content", {}) + search_phase = content.get("search_phase", {}) + + # Determine search_l and beam_width for search metrics + search_list = search_phase.get("search_list", [2000]) + beam_width = search_phase.get("beam_width", 4) + + # Use the first (or primary) search_l value + primary_search_l = search_list[0] if search_list else 2000 + + # Extract build metrics + baseline_build = extract_build_metrics(baseline_results) + target_build = extract_build_metrics(target_results) + + # Build metrics rows + build_metrics = [ + ("total_time", "total build time (s)"), + ("pq_construction_time", "PQ construction (s)"), + ("inmem_index_build_time", "in-memory index build (s)"), + ("disk_layout_time", "disk layout (s)"), + ] + + for key, display_name in build_metrics: + if key in target_build or key in baseline_build: + target_val = target_build.get(key, 0) + baseline_val = baseline_build.get(key, 0) + rows.append([ + "index-build statistics", + display_name, + key, + str(target_val), + str(baseline_val), + calc_deviation(baseline_val, target_val) + ]) + + # Extract search metrics for the primary search_l + baseline_search = extract_search_metrics(baseline_results, primary_search_l, beam_width) + target_search = extract_search_metrics(target_results, primary_search_l, beam_width) + + search_span_name = f"search-with-L={primary_search_l}-bw={beam_width}" + + # Search metrics rows + search_metrics = [ + ("qps", "queries per second"), + ("recall", "recall (%)"), + ("mean_latency", "mean latency (μs)"), + ("latency_95", "p999 latency (μs)"), + ("mean_ios", "mean IOs"), + ("mean_comps", "mean comparisons"), + ("mean_hops", "mean hops"), + ("mean_io_time", "mean IO time (μs)"), + ("mean_cpus", "mean CPU time (μs)"), + ] + + for key, display_name in search_metrics: + if key in target_search or key in baseline_search: + target_val = target_search.get(key, 0) + baseline_val = baseline_search.get(key, 0) + rows.append([ + search_span_name, + display_name, + key, + str(target_val), + str(baseline_val), + calc_deviation(baseline_val, target_val) + ]) + + return rows + + +def write_csv(rows: List[List[str]], out_path: str): + """Write the comparison rows to a CSV file.""" + header = [ + "Parent Span Name", + "Span Name", + "Stat Key", + "Stat Value (Target)", + "Stat Value (Baseline)", + "Deviation (%)" + ] + with open(out_path, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow(header) + writer.writerows(rows) + + +def main(): + parser = argparse.ArgumentParser( + description="Compare two disk-index benchmark JSONs and emit a diff CSV." + ) + parser.add_argument("--baseline", "-b", required=True, help="Path to baseline JSON") + parser.add_argument("--branch", "-r", required=True, help="Path to branch/target JSON") + parser.add_argument("--out", "-o", required=True, help="Where to write output CSV") + args = parser.parse_args() + + baseline_list = load_json(args.baseline) + target_list = load_json(args.branch) + + if len(baseline_list) != len(target_list): + raise ValueError( + f"baseline/branch JSON arrays differ in length: {len(baseline_list)} vs {len(target_list)}" + ) + + rows = make_rows(baseline_list, target_list) + write_csv(rows, args.out) + print(f"✓ Written diff CSV to {args.out}") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 0000000000..daf181ac64 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,318 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT license. + +# DiskANN Benchmarks Workflow +# Migrated from ADO pipeline: .pipelines/DiskANN-Benchmarks.yml +# +# This workflow runs macro benchmarks comparing the current branch against a baseline. +# It is manually triggered and requires a baseline reference (branch, tag, or commit). + +name: Benchmarks + +on: + workflow_dispatch: + inputs: + baseline_ref: + description: 'A branch, commit SHA, or tag name to compare the current branch with' + required: true + default: 'main' + type: string + +# Cancel in-progress runs when a new run is triggered +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + +env: + RUST_BACKTRACE: 1 + # Use the Rust version specified in rust-toolchain.toml + rust_stable: "1.92" + +defaults: + run: + shell: bash + +permissions: + contents: read + pull-requests: write # Required for posting PR comments + +jobs: + # Macro benchmark: Mimir Enron dataset + macro-benchmark-mimir-enron: + name: Macro Benchmark - Mimir Enron + runs-on: ubuntu-latest + # TODO: For production benchmarks, consider using a self-hosted runner with: + # - NVMe storage for consistent I/O performance + # - CPU pinning (taskset) for reduced variance + # - Dedicated hardware to avoid noisy neighbor effects + timeout-minutes: 120 + + steps: + - name: Checkout current branch + uses: actions/checkout@v4 + with: + path: diskann_rust + lfs: true + + - name: Checkout baseline (${{ inputs.baseline_ref }}) + uses: actions/checkout@v4 + with: + ref: ${{ inputs.baseline_ref }} + path: baseline + lfs: true + + - name: Install Rust ${{ env.rust_stable }} + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ env.rust_stable }} + + - name: Cache Rust dependencies (current) + uses: Swatinem/rust-cache@v2 + with: + workspaces: diskann_rust -> target + key: benchmark-current + + - name: Cache Rust dependencies (baseline) + uses: Swatinem/rust-cache@v2 + with: + workspaces: baseline -> target + key: benchmark-baseline + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y openssl libssl-dev pkg-config python3-pip + pip install csvtomd numpy scipy + + # Download the public Wikipedia-100K dataset via big-ann-benchmarks + # Dataset: 100K Cohere Wikipedia embeddings (768-dim, float32, cosine distance) + # Source: https://github.com/harsha-simhadri/big-ann-benchmarks + - name: Clone big-ann-benchmarks + run: git clone --depth 1 https://github.com/harsha-simhadri/big-ann-benchmarks.git + + - name: Download wikipedia-100K dataset + working-directory: big-ann-benchmarks + run: python create_dataset.py --dataset wikipedia-100K + + - name: Copy dataset to benchmark directories + run: | + mkdir -p diskann_rust/target/tmp baseline/target/tmp + cp -r big-ann-benchmarks/data/wikipedia_cohere diskann_rust/target/tmp/ + cp -r big-ann-benchmarks/data/wikipedia_cohere baseline/target/tmp/ + + - name: Run baseline benchmark + working-directory: baseline + run: | + # Note: For accurate benchmarks, consider using CPU pinning on self-hosted runners: + # sudo taskset -c 0,2,4,6 ionice -c 1 -n 0 cargo run ... + cargo run -p diskann-benchmark --features disk-index --release -- \ + run --input-file diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json \ + --output-file target/tmp/wikipedia-100K_benchmark_crate_baseline.json + + - name: Run current branch benchmark + working-directory: diskann_rust + run: | + cargo run -p diskann-benchmark --features disk-index --release -- \ + run --input-file diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json \ + --output-file target/tmp/wikipedia-100K_benchmark_crate_target.json + + - name: Generate diff stats (baseline vs target) + continue-on-error: true + run: | + python diskann_rust/.github/scripts/compare_disk_index_json_output.py \ + --baseline baseline/target/tmp/wikipedia-100K_benchmark_crate_baseline.json \ + --branch diskann_rust/target/tmp/wikipedia-100K_benchmark_crate_target.json \ + --out diskann_rust/target/tmp/wikipedia-100K_change.csv + + - name: Convert results to Markdown + working-directory: diskann_rust + run: | + csvtomd target/tmp/wikipedia-100K_change.csv > target/tmp/wikipedia-100K_change.md + echo "### Benchmark Results: Wikipedia-100K Dataset" >> $GITHUB_STEP_SUMMARY + cat target/tmp/wikipedia-100K_change.md >> $GITHUB_STEP_SUMMARY + + - name: Validate benchmark results + working-directory: diskann_rust + run: | + python .github/scripts/benchmark_result_parse.py \ + --mode pr \ + --file target/tmp/wikipedia-100K_change.csv + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }} + GITHUB_RUN_ID: ${{ github.run_id }} + + - name: Upload benchmark results + uses: actions/upload-artifact@v4 + if: always() # Upload even if validation fails + with: + name: benchmark-results-wikipedia-100K + path: | + diskann_rust/target/tmp/wikipedia-100K_change.csv + diskann_rust/target/tmp/wikipedia-100K_change.md + diskann_rust/target/tmp/wikipedia-100K_benchmark_crate_target.json + baseline/target/tmp/wikipedia-100K_benchmark_crate_baseline.json + retention-days: 30 + + # Macro benchmark: OAI Large dataset + macro-benchmark-oai-large: + name: Macro Benchmark - OAI Large + runs-on: ubuntu-latest + # TODO: For production benchmarks, consider using a self-hosted runner + timeout-minutes: 120 + + steps: + - name: Checkout current branch + uses: actions/checkout@v4 + with: + path: diskann_rust + lfs: true + + - name: Checkout baseline (${{ inputs.baseline_ref }}) + uses: actions/checkout@v4 + with: + ref: ${{ inputs.baseline_ref }} + path: baseline + lfs: true + + - name: Install Rust ${{ env.rust_stable }} + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ env.rust_stable }} + + - name: Cache Rust dependencies (current) + uses: Swatinem/rust-cache@v2 + with: + workspaces: diskann_rust -> target + key: benchmark-current + + - name: Cache Rust dependencies (baseline) + uses: Swatinem/rust-cache@v2 + with: + workspaces: baseline -> target + key: benchmark-baseline + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y openssl libssl-dev pkg-config python3-pip + pip install csvtomd numpy scipy + + # Download the public Wikipedia-100K dataset via big-ann-benchmarks + # Dataset: 100K Cohere Wikipedia embeddings (768-dim, float32, cosine distance) + # Source: https://github.com/harsha-simhadri/big-ann-benchmarks + - name: Clone big-ann-benchmarks + run: git clone --depth 1 https://github.com/harsha-simhadri/big-ann-benchmarks.git + + - name: Download wikipedia-100K dataset + working-directory: big-ann-benchmarks + run: python create_dataset.py --dataset wikipedia-100K + + - name: Copy dataset to benchmark directories + run: | + mkdir -p diskann_rust/target/tmp baseline/target/tmp + cp -r big-ann-benchmarks/data/wikipedia_cohere diskann_rust/target/tmp/ + cp -r big-ann-benchmarks/data/wikipedia_cohere baseline/target/tmp/ + + - name: Run baseline benchmark + working-directory: baseline + run: | + cargo run -p diskann-benchmark --features disk-index --release -- \ + run --input-file diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json \ + --output-file target/tmp/wikipedia-100K_benchmark_crate_baseline.json + + - name: Run current branch benchmark + working-directory: diskann_rust + run: | + cargo run -p diskann-benchmark --features disk-index --release -- \ + run --input-file diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json \ + --output-file target/tmp/wikipedia-100K_benchmark_crate_target.json + + - name: Generate diff stats (baseline vs target) + continue-on-error: true + run: | + python diskann_rust/.github/scripts/compare_disk_index_json_output.py \ + --baseline baseline/target/tmp/wikipedia-100K_benchmark_crate_baseline.json \ + --branch diskann_rust/target/tmp/wikipedia-100K_benchmark_crate_target.json \ + --out diskann_rust/target/tmp/wikipedia-100K_change.csv + + - name: Convert results to Markdown + working-directory: diskann_rust + run: | + csvtomd target/tmp/wikipedia-100K_change.csv > target/tmp/wikipedia-100K_change.md + echo "### Benchmark Results: Wikipedia-100K Dataset" >> $GITHUB_STEP_SUMMARY + cat target/tmp/wikipedia-100K_change.md >> $GITHUB_STEP_SUMMARY + + - name: Validate benchmark results + working-directory: diskann_rust + run: | + python .github/scripts/benchmark_result_parse.py \ + --mode pr \ + --file target/tmp/wikipedia-100K_change.csv + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }} + GITHUB_RUN_ID: ${{ github.run_id }} + + - name: Upload benchmark results + uses: actions/upload-artifact@v4 + if: always() # Upload even if validation fails + with: + name: benchmark-results-oai-wikipedia-100K + path: | + diskann_rust/target/tmp/wikipedia-100K_change.csv + diskann_rust/target/tmp/wikipedia-100K_change.md + diskann_rust/target/tmp/wikipedia-100K_benchmark_crate_target.json + baseline/target/tmp/wikipedia-100K_benchmark_crate_baseline.json + retention-days: 30 + + # NOTE: IAI micro-benchmarks are temporarily disabled in the ADO pipeline + # due to callgrind not running with Rust version `ms-1.86.0`. + # Uncomment when ready to enable: + # + # micro-benchmark-iai: + # name: Micro Benchmark - IAI + # runs-on: ubuntu-latest + # timeout-minutes: 120 + # + # steps: + # - name: Checkout current branch + # uses: actions/checkout@v4 + # with: + # path: diskann_rust + # + # - name: Checkout baseline (${{ inputs.baseline_ref }}) + # uses: actions/checkout@v4 + # with: + # ref: ${{ inputs.baseline_ref }} + # path: baseline + # + # - name: Install Rust ${{ env.rust_stable }} + # uses: dtolnay/rust-toolchain@master + # with: + # toolchain: ${{ env.rust_stable }} + # + # - name: Install valgrind and iai-callgrind-runner + # run: | + # sudo apt-get update + # sudo apt-get install -y valgrind + # cargo install --version 0.14.0 iai-callgrind-runner + # + # - name: Run baseline IAI benchmarks + # working-directory: baseline + # run: | + # cargo bench --bench bench_main_iai + # cargo bench --bench bench_main_vector_iai + # + # - name: Copy IAI baseline files + # run: | + # mkdir -p diskann_rust/target + # cp -R baseline/target/iai diskann_rust/target/ + # + # - name: Run current branch IAI benchmarks + # working-directory: diskann_rust + # run: | + # cargo bench --bench bench_main_iai + # cargo bench --bench bench_main_vector_iai diff --git a/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json b/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json new file mode 100644 index 0000000000..1557a594a2 --- /dev/null +++ b/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json @@ -0,0 +1,40 @@ +{ + "search_directories": [ + "target/tmp" + ], + "jobs": [ + { + "type": "disk-index", + "content": { + "source": { + "disk-index-source": "Build", + "data_type": "float32", + "data": "wikipedia_cohere/wikipedia_base.bin.crop_nb_100000", + "distance": "cosine_normalized", + "dim": 768, + "max_degree": 32, + "l_build": 50, + "num_threads": 4, + "build_ram_limit_gb": 4.0, + "num_pq_chunks": 96, + "quantization_type": "FP", + "save_path": "wikipedia_100k_benchmark_index" + }, + "search_phase": { + "queries": "wikipedia_cohere/wikipedia_query.bin", + "groundtruth": "wikipedia_cohere/wikipedia-100K", + "search_list": [ + 100, + 200 + ], + "beam_width": 4, + "recall_at": 100, + "num_threads": 4, + "is_flat_search": false, + "distance": "cosine_normalized", + "vector_filters_file": null + } + } + } + ] +} From 8e3c7df8c88953496ac74f2174a1f632f0235362 Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Fri, 20 Mar 2026 13:37:35 +0800 Subject: [PATCH 02/41] Fix openai-100K distance metric and add gitignore patterns --- .github/scripts/benchmark_result_parse.py | 115 +++++++++++++----- .../scripts/compare_disk_index_json_output.py | 2 - .github/workflows/benchmarks.yml | 57 +++++---- .../openai-100K-disk-index.json | 40 ++++++ 4 files changed, 155 insertions(+), 59 deletions(-) create mode 100644 diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json diff --git a/.github/scripts/benchmark_result_parse.py b/.github/scripts/benchmark_result_parse.py index 0308b29900..0bbede446e 100644 --- a/.github/scripts/benchmark_result_parse.py +++ b/.github/scripts/benchmark_result_parse.py @@ -8,8 +8,6 @@ Parses benchmark CSV results and validates against thresholds. Posts comments to GitHub PRs when regressions are detected. -Migrated from ADO: .pipelines/templates/BenchmarkResultParse.py - Usage: python benchmark_result_parse.py --mode pr --file results.csv python benchmark_result_parse.py --mode aa --file results.csv --data search @@ -72,10 +70,30 @@ "mean_comps": [], "mean_hops": [], "recall": [] + }, + "search-with-L=100-bw=4": { + "latency_95": [], + "mean_latency": [], + "mean_io_time": [], + "mean_cpus": [], + "qps": [], + "mean_ios": [], + "mean_comps": [], + "mean_hops": [], + "recall": [] + }, + "search-with-L=200-bw=4": { + "latency_95": [], + "mean_latency": [], + "mean_io_time": [], + "mean_cpus": [], + "qps": [], + "mean_ios": [], + "mean_comps": [], + "mean_hops": [], + "recall": [] } } - -# Template for search-only benchmark data DATA_TEMPLATE_SEARCH = { "search_disk_index-search_completed": { "duration_seconds": [], @@ -94,6 +112,28 @@ "mean_comps": [], "mean_hops": [], "recall": [] + }, + "search-with-L=100-bw=4": { + "latency_95": [], + "mean_latency": [], + "mean_io_time": [], + "mean_cpus": [], + "qps": [], + "mean_ios": [], + "mean_comps": [], + "mean_hops": [], + "recall": [] + }, + "search-with-L=200-bw=4": { + "latency_95": [], + "mean_latency": [], + "mean_io_time": [], + "mean_cpus": [], + "qps": [], + "mean_ios": [], + "mean_comps": [], + "mean_hops": [], + "recall": [] } } @@ -136,6 +176,28 @@ "mean_comps": [1, 'LT', 50000], "mean_hops": [1, 'LT', ""], "recall": [1, 'GT', 95.1] + }, + "search-with-L=100-bw=4": { + "latency_95": [10, 'LT', ""], + "mean_latency": [10, 'LT', ""], + "mean_io_time": [10, 'LT', ""], + "mean_cpus": [10, 'LT', ""], + "qps": [10, 'GT', ""], + "mean_ios": [10, 'LT', ""], + "mean_comps": [10, 'LT', ""], + "mean_hops": [10, 'LT', ""], + "recall": [1, 'GT', ""] + }, + "search-with-L=200-bw=4": { + "latency_95": [10, 'LT', ""], + "mean_latency": [10, 'LT', ""], + "mean_io_time": [10, 'LT', ""], + "mean_cpus": [10, 'LT', ""], + "qps": [10, 'GT', ""], + "mean_ios": [10, 'LT', ""], + "mean_comps": [10, 'LT', ""], + "mean_hops": [10, 'LT', ""], + "recall": [1, 'GT', ""] } } @@ -147,35 +209,32 @@ def parse_csv(file_path: str, data: dict[str, dict[str, list]]) -> dict[str, dict[str, list]]: """ Parse benchmark CSV file and populate data structure. - - CSV format expected: - Column 0: (unused) - Column 1: Category name (e.g., "search-with-L=2000-bw=4") - Column 2: Metric name (e.g., "qps") - Column 3: Current value - Column 4: Baseline value - Column 5: Change percentage + + CSV format produced by compare_disk_index_json_output.py: + Column 0: Parent Span Name (category, e.g. "index-build statistics") + Column 1: Span Name (display name, unused for matching) + Column 2: Stat Key (metric key, e.g. "qps") + Column 3: Stat Value (Target) + Column 4: Stat Value (Baseline) + Column 5: Deviation (%) """ with open(file_path, 'r', encoding='utf-8') as f: reader = csv.reader(f) next(reader) # Skip header row - - current_key = None + for row in reader: if len(row) < 6: continue - - # Column 1 contains category name (only set on first row of category) - if row[1]: - current_key = row[1] - elif current_key and current_key in data: - metric_name = row[2] - if metric_name in data[current_key]: - # Append: [current_value, baseline_value, change_percentage] - data[current_key][metric_name].append(row[3]) # current - data[current_key][metric_name].append(row[4]) # baseline - data[current_key][metric_name].append(row[5]) # change % - + + category = row[0].strip() + metric_name = row[2].strip() + + if category in data and metric_name in data[category]: + # Append: [current_value, baseline_value, change_percentage] + data[category][metric_name].append(row[3]) # target (current) + data[category][metric_name].append(row[4]) # baseline + data[category][metric_name].append(row[5]) # deviation % + return data @@ -286,8 +345,8 @@ def check_thresholds( values = data[category][metric] if not values: - print(f"ERROR: {category}/{metric} has no data") - return True, f"Missing data for {category}/{metric}" + # No data for this metric in the CSV — skip silently + continue # Parse values: [current, baseline, change%] try: diff --git a/.github/scripts/compare_disk_index_json_output.py b/.github/scripts/compare_disk_index_json_output.py index e3fa5afce8..ca9c9d26b2 100644 --- a/.github/scripts/compare_disk_index_json_output.py +++ b/.github/scripts/compare_disk_index_json_output.py @@ -11,8 +11,6 @@ The output format matches the CSV structure expected by benchmark_result_parse.py: Parent Span Name, Span Name, Stat Key, Stat Value (Target), Stat Value (Baseline), Deviation (%) -Migrated from ADO: .pipelines/templates/compare_disk_index_json_output.py - Usage: python compare_disk_index_json_output.py \\ --baseline baseline/target/tmp/_benchmark_crate_baseline.json \\ diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index daf181ac64..875c008340 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -2,7 +2,6 @@ # Licensed under the MIT license. # DiskANN Benchmarks Workflow -# Migrated from ADO pipeline: .pipelines/DiskANN-Benchmarks.yml # # This workflow runs macro benchmarks comparing the current branch against a baseline. # It is manually triggered and requires a baseline reference (branch, tag, or commit). @@ -37,9 +36,9 @@ permissions: pull-requests: write # Required for posting PR comments jobs: - # Macro benchmark: Mimir Enron dataset - macro-benchmark-mimir-enron: - name: Macro Benchmark - Mimir Enron + # Macro benchmark: Wikipedia-100K dataset + macro-benchmark-wikipedia-100K: + name: Macro Benchmark - Wikipedia 100K runs-on: ubuntu-latest # TODO: For production benchmarks, consider using a self-hosted runner with: # - NVMe storage for consistent I/O performance @@ -155,9 +154,9 @@ jobs: baseline/target/tmp/wikipedia-100K_benchmark_crate_baseline.json retention-days: 30 - # Macro benchmark: OAI Large dataset + # Macro benchmark: OpenAI ArXiv dataset macro-benchmark-oai-large: - name: Macro Benchmark - OAI Large + name: Macro Benchmark - OAI ArXiv 100K runs-on: ubuntu-latest # TODO: For production benchmarks, consider using a self-hosted runner timeout-minutes: 120 @@ -199,57 +198,57 @@ jobs: sudo apt-get install -y openssl libssl-dev pkg-config python3-pip pip install csvtomd numpy scipy - # Download the public Wikipedia-100K dataset via big-ann-benchmarks - # Dataset: 100K Cohere Wikipedia embeddings (768-dim, float32, cosine distance) + # Download the public OpenAI ArXiv 100K dataset via big-ann-benchmarks + # Dataset: 100K OpenAI embeddings of ArXiv papers (1536-dim, float32, euclidean distance) # Source: https://github.com/harsha-simhadri/big-ann-benchmarks - name: Clone big-ann-benchmarks run: git clone --depth 1 https://github.com/harsha-simhadri/big-ann-benchmarks.git - - name: Download wikipedia-100K dataset + - name: Download openai-100K dataset working-directory: big-ann-benchmarks - run: python create_dataset.py --dataset wikipedia-100K + run: python create_dataset.py --dataset openai-100K - name: Copy dataset to benchmark directories run: | mkdir -p diskann_rust/target/tmp baseline/target/tmp - cp -r big-ann-benchmarks/data/wikipedia_cohere diskann_rust/target/tmp/ - cp -r big-ann-benchmarks/data/wikipedia_cohere baseline/target/tmp/ + cp -r big-ann-benchmarks/data/OpenAIArXiv diskann_rust/target/tmp/ + cp -r big-ann-benchmarks/data/OpenAIArXiv baseline/target/tmp/ - name: Run baseline benchmark working-directory: baseline run: | cargo run -p diskann-benchmark --features disk-index --release -- \ - run --input-file diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json \ - --output-file target/tmp/wikipedia-100K_benchmark_crate_baseline.json + run --input-file diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json \ + --output-file target/tmp/openai-100K_benchmark_crate_baseline.json - name: Run current branch benchmark working-directory: diskann_rust run: | cargo run -p diskann-benchmark --features disk-index --release -- \ - run --input-file diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json \ - --output-file target/tmp/wikipedia-100K_benchmark_crate_target.json + run --input-file diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json \ + --output-file target/tmp/openai-100K_benchmark_crate_target.json - name: Generate diff stats (baseline vs target) continue-on-error: true run: | python diskann_rust/.github/scripts/compare_disk_index_json_output.py \ - --baseline baseline/target/tmp/wikipedia-100K_benchmark_crate_baseline.json \ - --branch diskann_rust/target/tmp/wikipedia-100K_benchmark_crate_target.json \ - --out diskann_rust/target/tmp/wikipedia-100K_change.csv + --baseline baseline/target/tmp/openai-100K_benchmark_crate_baseline.json \ + --branch diskann_rust/target/tmp/openai-100K_benchmark_crate_target.json \ + --out diskann_rust/target/tmp/openai-100K_change.csv - name: Convert results to Markdown working-directory: diskann_rust run: | - csvtomd target/tmp/wikipedia-100K_change.csv > target/tmp/wikipedia-100K_change.md - echo "### Benchmark Results: Wikipedia-100K Dataset" >> $GITHUB_STEP_SUMMARY - cat target/tmp/wikipedia-100K_change.md >> $GITHUB_STEP_SUMMARY + csvtomd target/tmp/openai-100K_change.csv > target/tmp/openai-100K_change.md + echo "### Benchmark Results: OpenAI ArXiv 100K Dataset" >> $GITHUB_STEP_SUMMARY + cat target/tmp/openai-100K_change.md >> $GITHUB_STEP_SUMMARY - name: Validate benchmark results working-directory: diskann_rust run: | python .github/scripts/benchmark_result_parse.py \ --mode pr \ - --file target/tmp/wikipedia-100K_change.csv + --file target/tmp/openai-100K_change.csv env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_REPOSITORY: ${{ github.repository }} @@ -260,15 +259,15 @@ jobs: uses: actions/upload-artifact@v4 if: always() # Upload even if validation fails with: - name: benchmark-results-oai-wikipedia-100K + name: benchmark-results-openai-100K path: | - diskann_rust/target/tmp/wikipedia-100K_change.csv - diskann_rust/target/tmp/wikipedia-100K_change.md - diskann_rust/target/tmp/wikipedia-100K_benchmark_crate_target.json - baseline/target/tmp/wikipedia-100K_benchmark_crate_baseline.json + diskann_rust/target/tmp/openai-100K_change.csv + diskann_rust/target/tmp/openai-100K_change.md + diskann_rust/target/tmp/openai-100K_benchmark_crate_target.json + baseline/target/tmp/openai-100K_benchmark_crate_baseline.json retention-days: 30 - # NOTE: IAI micro-benchmarks are temporarily disabled in the ADO pipeline + # NOTE: IAI micro-benchmarks are temporarily disabled # due to callgrind not running with Rust version `ms-1.86.0`. # Uncomment when ready to enable: # diff --git a/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json b/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json new file mode 100644 index 0000000000..969724cae2 --- /dev/null +++ b/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json @@ -0,0 +1,40 @@ +{ + "search_directories": [ + "target/tmp" + ], + "jobs": [ + { + "type": "disk-index", + "content": { + "source": { + "disk-index-source": "Build", + "data_type": "float32", + "data": "OpenAIArXiv/openai_base.bin.crop_nb_100000", + "distance": "squared_l2", + "dim": 1536, + "max_degree": 32, + "l_build": 50, + "num_threads": 8, + "build_ram_limit_gb": 4.0, + "num_pq_chunks": 192, + "quantization_type": "FP", + "save_path": "openai_100k_benchmark_index" + }, + "search_phase": { + "queries": "OpenAIArXiv/openai_query.bin", + "groundtruth": "OpenAIArXiv/openai-100K", + "search_list": [ + 100, + 200 + ], + "beam_width": 4, + "recall_at": 100, + "num_threads": 4, + "is_flat_search": false, + "distance": "squared_l2", + "vector_filters_file": null + } + } + } + ] +} From 9101383e6c09cbb198838957ae077ba295c8dc1a Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Fri, 20 Mar 2026 13:47:49 +0800 Subject: [PATCH 03/41] Add push trigger to benchmarks workflow for pre-merge testing --- .github/workflows/benchmarks.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 875c008340..3375601164 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -16,6 +16,14 @@ on: required: true default: 'main' type: string + push: + branches: + - 'user/tianyuanyuan/add-benchmark-pipeline' + paths: + - 'diskann-benchmark/perf_test_inputs/**-disk-index.json' + - '.github/workflows/benchmarks.yml' + - '.github/scripts/compare_disk_index_json_output.py' + - '.github/scripts/benchmark_result_parse.py' # Cancel in-progress runs when a new run is triggered concurrency: @@ -53,10 +61,10 @@ jobs: path: diskann_rust lfs: true - - name: Checkout baseline (${{ inputs.baseline_ref }}) + - name: Checkout baseline (${{ inputs.baseline_ref || 'main' }}) uses: actions/checkout@v4 with: - ref: ${{ inputs.baseline_ref }} + ref: ${{ inputs.baseline_ref || 'main' }} path: baseline lfs: true @@ -168,10 +176,10 @@ jobs: path: diskann_rust lfs: true - - name: Checkout baseline (${{ inputs.baseline_ref }}) + - name: Checkout baseline (${{ inputs.baseline_ref || 'main' }}) uses: actions/checkout@v4 with: - ref: ${{ inputs.baseline_ref }} + ref: ${{ inputs.baseline_ref || 'main' }} path: baseline lfs: true From 5f43fb6b9b70b0cb9082070b5d8d28589b3d1c7c Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Fri, 20 Mar 2026 13:58:51 +0800 Subject: [PATCH 04/41] Fix baseline run: use input config from current branch checkout --- .github/workflows/benchmarks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 3375601164..a4df8213f4 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -113,7 +113,7 @@ jobs: # Note: For accurate benchmarks, consider using CPU pinning on self-hosted runners: # sudo taskset -c 0,2,4,6 ionice -c 1 -n 0 cargo run ... cargo run -p diskann-benchmark --features disk-index --release -- \ - run --input-file diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json \ + run --input-file ../diskann_rust/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json \ --output-file target/tmp/wikipedia-100K_benchmark_crate_baseline.json - name: Run current branch benchmark @@ -226,7 +226,7 @@ jobs: working-directory: baseline run: | cargo run -p diskann-benchmark --features disk-index --release -- \ - run --input-file diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json \ + run --input-file ../diskann_rust/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json \ --output-file target/tmp/openai-100K_benchmark_crate_baseline.json - name: Run current branch benchmark From 19003b9b1403eed2c266693d2f3d133951ba5f25 Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Fri, 20 Mar 2026 14:17:29 +0800 Subject: [PATCH 05/41] Fix markdown conversion: replace broken csvtomd with inline Python --- .github/workflows/benchmarks.yml | 34 ++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index a4df8213f4..0670af06e0 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -89,7 +89,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y openssl libssl-dev pkg-config python3-pip - pip install csvtomd numpy scipy + pip install numpy scipy # Download the public Wikipedia-100K dataset via big-ann-benchmarks # Dataset: 100K Cohere Wikipedia embeddings (768-dim, float32, cosine distance) @@ -134,9 +134,18 @@ jobs: - name: Convert results to Markdown working-directory: diskann_rust run: | - csvtomd target/tmp/wikipedia-100K_change.csv > target/tmp/wikipedia-100K_change.md - echo "### Benchmark Results: Wikipedia-100K Dataset" >> $GITHUB_STEP_SUMMARY - cat target/tmp/wikipedia-100K_change.md >> $GITHUB_STEP_SUMMARY + python3 -c " + import csv, os + rows = list(csv.reader(open('target/tmp/wikipedia-100K_change.csv'))) + if len(rows) < 2: + print('No data'); exit(0) + header = rows[0] + sep = ['---'] * len(header) + md = '\n'.join(' | '.join(r) for r in [header, sep] + rows[1:]) + open('target/tmp/wikipedia-100K_change.md', 'w').write(md + '\n') + " + echo '### Benchmark Results: Wikipedia-100K Dataset' >> "$GITHUB_STEP_SUMMARY" + cat target/tmp/wikipedia-100K_change.md >> "$GITHUB_STEP_SUMMARY" - name: Validate benchmark results working-directory: diskann_rust @@ -204,7 +213,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y openssl libssl-dev pkg-config python3-pip - pip install csvtomd numpy scipy + pip install numpy scipy # Download the public OpenAI ArXiv 100K dataset via big-ann-benchmarks # Dataset: 100K OpenAI embeddings of ArXiv papers (1536-dim, float32, euclidean distance) @@ -247,9 +256,18 @@ jobs: - name: Convert results to Markdown working-directory: diskann_rust run: | - csvtomd target/tmp/openai-100K_change.csv > target/tmp/openai-100K_change.md - echo "### Benchmark Results: OpenAI ArXiv 100K Dataset" >> $GITHUB_STEP_SUMMARY - cat target/tmp/openai-100K_change.md >> $GITHUB_STEP_SUMMARY + python3 -c " + import csv, os + rows = list(csv.reader(open('target/tmp/openai-100K_change.csv'))) + if len(rows) < 2: + print('No data'); exit(0) + header = rows[0] + sep = ['---'] * len(header) + md = '\n'.join(' | '.join(r) for r in [header, sep] + rows[1:]) + open('target/tmp/openai-100K_change.md', 'w').write(md + '\n') + " + echo '### Benchmark Results: OpenAI ArXiv 100K Dataset' >> "$GITHUB_STEP_SUMMARY" + cat target/tmp/openai-100K_change.md >> "$GITHUB_STEP_SUMMARY" - name: Validate benchmark results working-directory: diskann_rust From 262bf5da0c65ff49701bdda17399aedaa2641984 Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Fri, 20 Mar 2026 16:51:59 +0800 Subject: [PATCH 06/41] Update benchmark configs: align build/search params to fix low recall --- .../perf_test_inputs/openai-100K-disk-index.json | 13 ++++++------- .../perf_test_inputs/wikipedia-100K-disk-index.json | 13 ++++++------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json b/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json index 969724cae2..9ae7e148be 100644 --- a/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json +++ b/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json @@ -12,11 +12,11 @@ "data": "OpenAIArXiv/openai_base.bin.crop_nb_100000", "distance": "squared_l2", "dim": 1536, - "max_degree": 32, - "l_build": 50, + "max_degree": 59, + "l_build": 64, "num_threads": 8, - "build_ram_limit_gb": 4.0, - "num_pq_chunks": 192, + "build_ram_limit_gb": 10.0, + "num_pq_chunks": 384, "quantization_type": "FP", "save_path": "openai_100k_benchmark_index" }, @@ -24,11 +24,10 @@ "queries": "OpenAIArXiv/openai_query.bin", "groundtruth": "OpenAIArXiv/openai-100K", "search_list": [ - 100, - 200 + 2000 ], "beam_width": 4, - "recall_at": 100, + "recall_at": 1000, "num_threads": 4, "is_flat_search": false, "distance": "squared_l2", diff --git a/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json b/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json index 1557a594a2..7deaf788ea 100644 --- a/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json +++ b/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json @@ -12,11 +12,11 @@ "data": "wikipedia_cohere/wikipedia_base.bin.crop_nb_100000", "distance": "cosine_normalized", "dim": 768, - "max_degree": 32, - "l_build": 50, + "max_degree": 59, + "l_build": 72, "num_threads": 4, - "build_ram_limit_gb": 4.0, - "num_pq_chunks": 96, + "build_ram_limit_gb": 10.0, + "num_pq_chunks": 192, "quantization_type": "FP", "save_path": "wikipedia_100k_benchmark_index" }, @@ -24,11 +24,10 @@ "queries": "wikipedia_cohere/wikipedia_query.bin", "groundtruth": "wikipedia_cohere/wikipedia-100K", "search_list": [ - 100, - 200 + 2000 ], "beam_width": 4, - "recall_at": 100, + "recall_at": 1000, "num_threads": 4, "is_flat_search": false, "distance": "cosine_normalized", From 1699b47d97b75c82ab87e0a335a61f6b99470ffb Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Mon, 23 Mar 2026 13:13:34 +0800 Subject: [PATCH 07/41] Fix recall_at: set to 100 to match groundtruth file K=100 --- diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json | 2 +- .../perf_test_inputs/wikipedia-100K-disk-index.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json b/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json index 9ae7e148be..93c1358bab 100644 --- a/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json +++ b/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json @@ -27,7 +27,7 @@ 2000 ], "beam_width": 4, - "recall_at": 1000, + "recall_at": 100, "num_threads": 4, "is_flat_search": false, "distance": "squared_l2", diff --git a/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json b/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json index 7deaf788ea..1c0af41b73 100644 --- a/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json +++ b/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json @@ -27,7 +27,7 @@ 2000 ], "beam_width": 4, - "recall_at": 1000, + "recall_at": 100, "num_threads": 4, "is_flat_search": false, "distance": "cosine_normalized", From 635db8ba3f86e2c9c2cf9beceda513474ec41554 Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Mon, 23 Mar 2026 13:54:31 +0800 Subject: [PATCH 08/41] Remove stale absolute contracts (qps/recall/total_time): calibrated for ADO mimir-enron, not applicable to public datasets on GitHub runners. Threshold calibration tracked in PBI. --- .github/scripts/benchmark_result_parse.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/scripts/benchmark_result_parse.py b/.github/scripts/benchmark_result_parse.py index 0bbede446e..ab3549dc16 100644 --- a/.github/scripts/benchmark_result_parse.py +++ b/.github/scripts/benchmark_result_parse.py @@ -162,7 +162,8 @@ "total_duration_seconds": [10, 'LT', ""], }, "index-build statistics": { - "total_time": [10, 'LT', 1206], + # total_time contract TBD: requires baseline run on target hardware (see PBI: threshold calibration) + "total_time": [10, 'LT', ""], "total_comparisons": [1, 'LT', ""], "search_hops": [1, 'LT', ""] }, @@ -171,11 +172,14 @@ "mean_latency": [10, 'LT', ""], "mean_io_time": [10, 'LT', ""], "mean_cpus": [10, 'LT', ""], - "qps": [10, 'GT', 29], - "mean_ios": [1, 'LT', 2026], - "mean_comps": [1, 'LT', 50000], + # qps/recall/mean_ios/mean_comps contracts TBD: prior values were calibrated for + # internal mimir-enron 1M-vector dataset on production hardware, not applicable here. + # See PBI: define alert thresholds for public dataset benchmarks. + "qps": [10, 'GT', ""], + "mean_ios": [1, 'LT', ""], + "mean_comps": [1, 'LT', ""], "mean_hops": [1, 'LT', ""], - "recall": [1, 'GT', 95.1] + "recall": [1, 'GT', ""] }, "search-with-L=100-bw=4": { "latency_95": [10, 'LT', ""], From 4a0ac3fdb60723cc3f8d26c8a58549d1dbfbf6f5 Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Mon, 23 Mar 2026 13:55:32 +0800 Subject: [PATCH 09/41] remove comments --- .github/scripts/benchmark_result_parse.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/scripts/benchmark_result_parse.py b/.github/scripts/benchmark_result_parse.py index ab3549dc16..600a66f355 100644 --- a/.github/scripts/benchmark_result_parse.py +++ b/.github/scripts/benchmark_result_parse.py @@ -162,7 +162,6 @@ "total_duration_seconds": [10, 'LT', ""], }, "index-build statistics": { - # total_time contract TBD: requires baseline run on target hardware (see PBI: threshold calibration) "total_time": [10, 'LT', ""], "total_comparisons": [1, 'LT', ""], "search_hops": [1, 'LT', ""] @@ -172,9 +171,6 @@ "mean_latency": [10, 'LT', ""], "mean_io_time": [10, 'LT', ""], "mean_cpus": [10, 'LT', ""], - # qps/recall/mean_ios/mean_comps contracts TBD: prior values were calibrated for - # internal mimir-enron 1M-vector dataset on production hardware, not applicable here. - # See PBI: define alert thresholds for public dataset benchmarks. "qps": [10, 'GT', ""], "mean_ios": [1, 'LT', ""], "mean_comps": [1, 'LT', ""], From 8f40af3ec28af22245d0e348c1c079683156ec19 Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Mon, 23 Mar 2026 14:47:43 +0800 Subject: [PATCH 10/41] Fix build_ram_limit_gb: reduce 10->4 to fit GitHub runner RAM (7GB limit) --- diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json | 2 +- .../perf_test_inputs/wikipedia-100K-disk-index.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json b/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json index 93c1358bab..940269195b 100644 --- a/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json +++ b/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json @@ -15,7 +15,7 @@ "max_degree": 59, "l_build": 64, "num_threads": 8, - "build_ram_limit_gb": 10.0, + "build_ram_limit_gb": 4.0, "num_pq_chunks": 384, "quantization_type": "FP", "save_path": "openai_100k_benchmark_index" diff --git a/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json b/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json index 1c0af41b73..d15026e635 100644 --- a/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json +++ b/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json @@ -15,7 +15,7 @@ "max_degree": 59, "l_build": 72, "num_threads": 4, - "build_ram_limit_gb": 10.0, + "build_ram_limit_gb": 4.0, "num_pq_chunks": 192, "quantization_type": "FP", "save_path": "wikipedia_100k_benchmark_index" From 5d9e0f08b7d54f4b5f3eaecc5f8ee7536fe1c1e7 Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Mon, 23 Mar 2026 15:21:22 +0800 Subject: [PATCH 11/41] Fix wikipedia distance: cosine_normalized->cosine (vectors are not L2-normalized, metric is inner product) --- .../perf_test_inputs/wikipedia-100K-disk-index.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json b/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json index d15026e635..c4131e7208 100644 --- a/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json +++ b/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json @@ -10,7 +10,7 @@ "disk-index-source": "Build", "data_type": "float32", "data": "wikipedia_cohere/wikipedia_base.bin.crop_nb_100000", - "distance": "cosine_normalized", + "distance": "cosine", "dim": 768, "max_degree": 59, "l_build": 72, @@ -30,7 +30,7 @@ "recall_at": 100, "num_threads": 4, "is_flat_search": false, - "distance": "cosine_normalized", + "distance": "cosine", "vector_filters_file": null } } From 54ac011756f69f50ecab10aecb0793cb82540b70 Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Mon, 23 Mar 2026 15:51:15 +0800 Subject: [PATCH 12/41] Fix wikipedia distance: cosine->inner_product (groundtruth uses raw ip, not cosine similarity) --- .../perf_test_inputs/wikipedia-100K-disk-index.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json b/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json index c4131e7208..6a52b1e323 100644 --- a/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json +++ b/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json @@ -10,7 +10,7 @@ "disk-index-source": "Build", "data_type": "float32", "data": "wikipedia_cohere/wikipedia_base.bin.crop_nb_100000", - "distance": "cosine", + "distance": "inner_product", "dim": 768, "max_degree": 59, "l_build": 72, @@ -30,7 +30,7 @@ "recall_at": 100, "num_threads": 4, "is_flat_search": false, - "distance": "cosine", + "distance": "inner_product", "vector_filters_file": null } } From 8a56a912b67101b7e693a1665878297632c95725 Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Tue, 24 Mar 2026 15:09:03 +0800 Subject: [PATCH 13/41] Align build threads --- diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json | 2 +- .../perf_test_inputs/wikipedia-100K-disk-index.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json b/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json index 940269195b..b9f3e195d2 100644 --- a/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json +++ b/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json @@ -14,7 +14,7 @@ "dim": 1536, "max_degree": 59, "l_build": 64, - "num_threads": 8, + "num_threads": 1, "build_ram_limit_gb": 4.0, "num_pq_chunks": 384, "quantization_type": "FP", diff --git a/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json b/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json index 6a52b1e323..5093eaf4d8 100644 --- a/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json +++ b/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json @@ -14,7 +14,7 @@ "dim": 768, "max_degree": 59, "l_build": 72, - "num_threads": 4, + "num_threads": 1, "build_ram_limit_gb": 4.0, "num_pq_chunks": 192, "quantization_type": "FP", From 0e36b9d67672b9aaf2276593c44cb67e9dd51cbb Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Tue, 24 Mar 2026 15:51:14 +0800 Subject: [PATCH 14/41] Speed up benchmarks: build threads 1->4, openai pq_chunks 384->192 --- .../perf_test_inputs/openai-100K-disk-index.json | 4 ++-- .../perf_test_inputs/wikipedia-100K-disk-index.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json b/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json index b9f3e195d2..3723d66b6d 100644 --- a/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json +++ b/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json @@ -14,9 +14,9 @@ "dim": 1536, "max_degree": 59, "l_build": 64, - "num_threads": 1, + "num_threads": 4, "build_ram_limit_gb": 4.0, - "num_pq_chunks": 384, + "num_pq_chunks": 192, "quantization_type": "FP", "save_path": "openai_100k_benchmark_index" }, diff --git a/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json b/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json index 5093eaf4d8..6a52b1e323 100644 --- a/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json +++ b/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json @@ -14,7 +14,7 @@ "dim": 768, "max_degree": 59, "l_build": 72, - "num_threads": 1, + "num_threads": 4, "build_ram_limit_gb": 4.0, "num_pq_chunks": 192, "quantization_type": "FP", From f7c7f9db3612775d9fc4f0587a3b88634aeb9ba2 Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Tue, 24 Mar 2026 16:24:44 +0800 Subject: [PATCH 15/41] Temp: disable concurrency cancellation for A/A batch testing --- .github/workflows/benchmarks.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 0670af06e0..fc0de7c076 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -27,8 +27,10 @@ on: # Cancel in-progress runs when a new run is triggered concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} - cancel-in-progress: true + # Use unique group per run for A/A testing (allows parallel runs). + # For production, change back to: github.event.pull_request.number || github.sha + group: ${{ github.workflow }}-${{ github.run_id }} + cancel-in-progress: false env: RUST_BACKTRACE: 1 From 18c5fafbff87759020757056195264d2cbf1722a Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Thu, 26 Mar 2026 13:56:35 +0800 Subject: [PATCH 16/41] revert A/A test settings, update OpenAI config to SQ_1_2.0 --- .github/workflows/benchmarks.yml | 10 ++++------ .../perf_test_inputs/openai-100K-disk-index.json | 6 +++--- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index fc0de7c076..aa659dd098 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -27,10 +27,8 @@ on: # Cancel in-progress runs when a new run is triggered concurrency: - # Use unique group per run for A/A testing (allows parallel runs). - # For production, change back to: github.event.pull_request.number || github.sha - group: ${{ github.workflow }}-${{ github.run_id }} - cancel-in-progress: false + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true env: RUST_BACKTRACE: 1 @@ -91,7 +89,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y openssl libssl-dev pkg-config python3-pip - pip install numpy scipy + pip install csvtomd numpy scipy # Download the public Wikipedia-100K dataset via big-ann-benchmarks # Dataset: 100K Cohere Wikipedia embeddings (768-dim, float32, cosine distance) @@ -215,7 +213,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y openssl libssl-dev pkg-config python3-pip - pip install numpy scipy + pip install csvtomd numpy scipy # Download the public OpenAI ArXiv 100K dataset via big-ann-benchmarks # Dataset: 100K OpenAI embeddings of ArXiv papers (1536-dim, float32, euclidean distance) diff --git a/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json b/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json index 3723d66b6d..3a2a1d9e2f 100644 --- a/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json +++ b/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json @@ -13,11 +13,11 @@ "distance": "squared_l2", "dim": 1536, "max_degree": 59, - "l_build": 64, + "l_build": 80, "num_threads": 4, "build_ram_limit_gb": 4.0, - "num_pq_chunks": 192, - "quantization_type": "FP", + "num_pq_chunks": 384, + "quantization_type": "SQ_1_2.0", "save_path": "openai_100k_benchmark_index" }, "search_phase": { From 5ee4506f2967f2e34c16d736337a2d41fc068151 Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Fri, 27 Mar 2026 11:09:16 +0800 Subject: [PATCH 17/41] Remove micro-benchmark-iai comments --- .github/workflows/benchmarks.yml | 53 +------------------------------- 1 file changed, 1 insertion(+), 52 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index aa659dd098..324bb1cda8 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -124,7 +124,6 @@ jobs: --output-file target/tmp/wikipedia-100K_benchmark_crate_target.json - name: Generate diff stats (baseline vs target) - continue-on-error: true run: | python diskann_rust/.github/scripts/compare_disk_index_json_output.py \ --baseline baseline/target/tmp/wikipedia-100K_benchmark_crate_baseline.json \ @@ -246,7 +245,6 @@ jobs: --output-file target/tmp/openai-100K_benchmark_crate_target.json - name: Generate diff stats (baseline vs target) - continue-on-error: true run: | python diskann_rust/.github/scripts/compare_disk_index_json_output.py \ --baseline baseline/target/tmp/openai-100K_benchmark_crate_baseline.json \ @@ -291,53 +289,4 @@ jobs: diskann_rust/target/tmp/openai-100K_change.md diskann_rust/target/tmp/openai-100K_benchmark_crate_target.json baseline/target/tmp/openai-100K_benchmark_crate_baseline.json - retention-days: 30 - - # NOTE: IAI micro-benchmarks are temporarily disabled - # due to callgrind not running with Rust version `ms-1.86.0`. - # Uncomment when ready to enable: - # - # micro-benchmark-iai: - # name: Micro Benchmark - IAI - # runs-on: ubuntu-latest - # timeout-minutes: 120 - # - # steps: - # - name: Checkout current branch - # uses: actions/checkout@v4 - # with: - # path: diskann_rust - # - # - name: Checkout baseline (${{ inputs.baseline_ref }}) - # uses: actions/checkout@v4 - # with: - # ref: ${{ inputs.baseline_ref }} - # path: baseline - # - # - name: Install Rust ${{ env.rust_stable }} - # uses: dtolnay/rust-toolchain@master - # with: - # toolchain: ${{ env.rust_stable }} - # - # - name: Install valgrind and iai-callgrind-runner - # run: | - # sudo apt-get update - # sudo apt-get install -y valgrind - # cargo install --version 0.14.0 iai-callgrind-runner - # - # - name: Run baseline IAI benchmarks - # working-directory: baseline - # run: | - # cargo bench --bench bench_main_iai - # cargo bench --bench bench_main_vector_iai - # - # - name: Copy IAI baseline files - # run: | - # mkdir -p diskann_rust/target - # cp -R baseline/target/iai diskann_rust/target/ - # - # - name: Run current branch IAI benchmarks - # working-directory: diskann_rust - # run: | - # cargo bench --bench bench_main_iai - # cargo bench --bench bench_main_vector_iai + retention-days: 30 \ No newline at end of file From 9ffdefd718d4623c7f4a511fcd0e9bf7529bfaa0 Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Fri, 27 Mar 2026 11:47:59 +0800 Subject: [PATCH 18/41] use GitHub Release assets for benchmark datasets --- .github/workflows/benchmarks.yml | 34 +++++++++++--------------------- 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 324bb1cda8..a2fd6ad34d 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -91,21 +91,16 @@ jobs: sudo apt-get install -y openssl libssl-dev pkg-config python3-pip pip install csvtomd numpy scipy - # Download the public Wikipedia-100K dataset via big-ann-benchmarks + # Download pre-packaged Wikipedia-100K dataset from GitHub Release # Dataset: 100K Cohere Wikipedia embeddings (768-dim, float32, cosine distance) - # Source: https://github.com/harsha-simhadri/big-ann-benchmarks - - name: Clone big-ann-benchmarks - run: git clone --depth 1 https://github.com/harsha-simhadri/big-ann-benchmarks.git - - name: Download wikipedia-100K dataset - working-directory: big-ann-benchmarks - run: python create_dataset.py --dataset wikipedia-100K - - - name: Copy dataset to benchmark directories + env: + GH_TOKEN: ${{ github.token }} run: | mkdir -p diskann_rust/target/tmp baseline/target/tmp - cp -r big-ann-benchmarks/data/wikipedia_cohere diskann_rust/target/tmp/ - cp -r big-ann-benchmarks/data/wikipedia_cohere baseline/target/tmp/ + gh release download benchmark-data-v1 --repo ${{ github.repository }} --pattern 'wikipedia-100K.tar.gz' --dir . + tar xzf wikipedia-100K.tar.gz -C diskann_rust/target/tmp/ + cp -r diskann_rust/target/tmp/wikipedia_cohere baseline/target/tmp/ - name: Run baseline benchmark working-directory: baseline @@ -214,21 +209,16 @@ jobs: sudo apt-get install -y openssl libssl-dev pkg-config python3-pip pip install csvtomd numpy scipy - # Download the public OpenAI ArXiv 100K dataset via big-ann-benchmarks + # Download pre-packaged OpenAI ArXiv 100K dataset from GitHub Release # Dataset: 100K OpenAI embeddings of ArXiv papers (1536-dim, float32, euclidean distance) - # Source: https://github.com/harsha-simhadri/big-ann-benchmarks - - name: Clone big-ann-benchmarks - run: git clone --depth 1 https://github.com/harsha-simhadri/big-ann-benchmarks.git - - name: Download openai-100K dataset - working-directory: big-ann-benchmarks - run: python create_dataset.py --dataset openai-100K - - - name: Copy dataset to benchmark directories + env: + GH_TOKEN: ${{ github.token }} run: | mkdir -p diskann_rust/target/tmp baseline/target/tmp - cp -r big-ann-benchmarks/data/OpenAIArXiv diskann_rust/target/tmp/ - cp -r big-ann-benchmarks/data/OpenAIArXiv baseline/target/tmp/ + gh release download benchmark-data-v1 --repo ${{ github.repository }} --pattern 'openai-100K.tar.gz' --dir . + tar xzf openai-100K.tar.gz -C diskann_rust/target/tmp/ + cp -r diskann_rust/target/tmp/OpenAIArXiv baseline/target/tmp/ - name: Run baseline benchmark working-directory: baseline From 18310cbcaaf63ef9f7da3580f7c9908805e814e9 Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Fri, 27 Mar 2026 14:09:01 +0800 Subject: [PATCH 19/41] extract csv-to-markdown into reusable script --- .github/scripts/csv_to_markdown.py | 50 ++++++++++++++++++++++++++++++ .github/workflows/benchmarks.yml | 32 +++++-------------- 2 files changed, 58 insertions(+), 24 deletions(-) create mode 100644 .github/scripts/csv_to_markdown.py diff --git a/.github/scripts/csv_to_markdown.py b/.github/scripts/csv_to_markdown.py new file mode 100644 index 0000000000..885a202081 --- /dev/null +++ b/.github/scripts/csv_to_markdown.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT license. + +"""Convert a CSV file to a Markdown table and optionally append to GitHub Step Summary.""" + +import argparse +import csv +import os +import sys + + +def csv_to_markdown(csv_path: str) -> str: + """Convert a CSV file to a Markdown table string.""" + with open(csv_path) as f: + rows = list(csv.reader(f)) + if len(rows) < 2: + return "" + header = rows[0] + sep = ["---"] * len(header) + return "\n".join(" | ".join(r) for r in [header, sep] + rows[1:]) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--csv", required=True, help="Input CSV file path") + parser.add_argument("--md", required=True, help="Output Markdown file path") + parser.add_argument("--title", default="", help="Section title for GitHub Step Summary") + args = parser.parse_args() + + md = csv_to_markdown(args.csv) + if not md: + print("No data") + return 0 + + with open(args.md, "w") as f: + f.write(md + "\n") + + # Append to GitHub Step Summary if available + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_path and args.title: + with open(summary_path, "a") as f: + f.write(f"### {args.title}\n") + f.write(md + "\n") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index a2fd6ad34d..a5c5c3a03b 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -128,18 +128,10 @@ jobs: - name: Convert results to Markdown working-directory: diskann_rust run: | - python3 -c " - import csv, os - rows = list(csv.reader(open('target/tmp/wikipedia-100K_change.csv'))) - if len(rows) < 2: - print('No data'); exit(0) - header = rows[0] - sep = ['---'] * len(header) - md = '\n'.join(' | '.join(r) for r in [header, sep] + rows[1:]) - open('target/tmp/wikipedia-100K_change.md', 'w').write(md + '\n') - " - echo '### Benchmark Results: Wikipedia-100K Dataset' >> "$GITHUB_STEP_SUMMARY" - cat target/tmp/wikipedia-100K_change.md >> "$GITHUB_STEP_SUMMARY" + python .github/scripts/csv_to_markdown.py \ + --csv target/tmp/wikipedia-100K_change.csv \ + --md target/tmp/wikipedia-100K_change.md \ + --title 'Benchmark Results: Wikipedia-100K Dataset' - name: Validate benchmark results working-directory: diskann_rust @@ -244,18 +236,10 @@ jobs: - name: Convert results to Markdown working-directory: diskann_rust run: | - python3 -c " - import csv, os - rows = list(csv.reader(open('target/tmp/openai-100K_change.csv'))) - if len(rows) < 2: - print('No data'); exit(0) - header = rows[0] - sep = ['---'] * len(header) - md = '\n'.join(' | '.join(r) for r in [header, sep] + rows[1:]) - open('target/tmp/openai-100K_change.md', 'w').write(md + '\n') - " - echo '### Benchmark Results: OpenAI ArXiv 100K Dataset' >> "$GITHUB_STEP_SUMMARY" - cat target/tmp/openai-100K_change.md >> "$GITHUB_STEP_SUMMARY" + python .github/scripts/csv_to_markdown.py \ + --csv target/tmp/openai-100K_change.csv \ + --md target/tmp/openai-100K_change.md \ + --title 'Benchmark Results: OpenAI ArXiv 100K Dataset' - name: Validate benchmark results working-directory: diskann_rust From 6c70dfad26b65bc65d876315bc33e470aeda93d1 Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Fri, 27 Mar 2026 15:03:00 +0800 Subject: [PATCH 20/41] calibrate contract thresholds from GitHub runner data --- .github/scripts/benchmark_result_parse.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/scripts/benchmark_result_parse.py b/.github/scripts/benchmark_result_parse.py index 600a66f355..c510360568 100644 --- a/.github/scripts/benchmark_result_parse.py +++ b/.github/scripts/benchmark_result_parse.py @@ -162,20 +162,28 @@ "total_duration_seconds": [10, 'LT', ""], }, "index-build statistics": { - "total_time": [10, 'LT', ""], + # Calibrated from 5 GitHub runner runs (10 observations): + # Wikipedia: 35.9–37.2s, OpenAI: 23.0–76.4s (SQ_1_2.0 variance) + # Contract: worst × 1.5 to absorb shared-runner variance + "total_time": [10, 'LT', 115], "total_comparisons": [1, 'LT', ""], "search_hops": [1, 'LT', ""] }, "search-with-L=2000-bw=4": { + # Calibrated from 5 GitHub runner runs (10 observations): + # QPS: 9.56–9.75 (both datasets) + # Recall: wiki 99.87%, oai 99.67–99.91% + # mean_ios: ~2007 (deterministic) + # mean_comps: wiki ~27609, oai 21618–24733 "latency_95": [10, 'LT', ""], "mean_latency": [10, 'LT', ""], "mean_io_time": [10, 'LT', ""], "mean_cpus": [10, 'LT', ""], - "qps": [10, 'GT', ""], - "mean_ios": [1, 'LT', ""], - "mean_comps": [1, 'LT', ""], + "qps": [10, 'GT', 6.5], + "mean_ios": [1, 'LT', 2410], + "mean_comps": [1, 'LT', 33200], "mean_hops": [1, 'LT', ""], - "recall": [1, 'GT', ""] + "recall": [1, 'GT', 98.0] }, "search-with-L=100-bw=4": { "latency_95": [10, 'LT', ""], From 046db0413baae7a8b7bd9fd435c62469aa77e099 Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Fri, 27 Mar 2026 15:10:12 +0800 Subject: [PATCH 21/41] add daily A/A benchmark stability test with failure notification --- .github/workflows/benchmarks-aa.yml | 276 ++++++++++++++++++++++++++++ 1 file changed, 276 insertions(+) create mode 100644 .github/workflows/benchmarks-aa.yml diff --git a/.github/workflows/benchmarks-aa.yml b/.github/workflows/benchmarks-aa.yml new file mode 100644 index 0000000000..b272f7c698 --- /dev/null +++ b/.github/workflows/benchmarks-aa.yml @@ -0,0 +1,276 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT license. + +# DiskANN Daily A/A Benchmark Stability Test +# +# Runs main vs main at 9 AM UTC every day to detect environment noise. +# If any threshold is breached, a GitHub issue is created to notify @microsoft/diskann-admin. +# Can also be triggered manually for debugging. + +name: Benchmarks (A/A) + +on: + schedule: + # Daily at 9 AM UTC + - cron: '0 9 * * *' + workflow_dispatch: # Allow manual trigger for debugging + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: true + +env: + RUST_BACKTRACE: 1 + rust_stable: "1.92" + +defaults: + run: + shell: bash + +permissions: + contents: read + issues: write # Required for creating failure notification issues + +jobs: + # A/A benchmark: Wikipedia-100K dataset (main vs main) + aa-wikipedia-100K: + name: A/A - Wikipedia 100K + runs-on: ubuntu-latest + timeout-minutes: 120 + + steps: + - name: Checkout main (target) + uses: actions/checkout@v4 + with: + ref: main + path: diskann_rust + lfs: true + + - name: Checkout main (baseline) + uses: actions/checkout@v4 + with: + ref: main + path: baseline + lfs: true + + - name: Install Rust ${{ env.rust_stable }} + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ env.rust_stable }} + + - name: Cache Rust dependencies (target) + uses: Swatinem/rust-cache@v2 + with: + workspaces: diskann_rust -> target + key: aa-target + + - name: Cache Rust dependencies (baseline) + uses: Swatinem/rust-cache@v2 + with: + workspaces: baseline -> target + key: aa-baseline + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y openssl libssl-dev pkg-config + + # Download pre-packaged Wikipedia-100K dataset from GitHub Release + - name: Download wikipedia-100K dataset + env: + GH_TOKEN: ${{ github.token }} + run: | + mkdir -p diskann_rust/target/tmp baseline/target/tmp + gh release download benchmark-data-v1 --repo ${{ github.repository }} --pattern 'wikipedia-100K.tar.gz' --dir . + tar xzf wikipedia-100K.tar.gz -C diskann_rust/target/tmp/ + cp -r diskann_rust/target/tmp/wikipedia_cohere baseline/target/tmp/ + + - name: Run baseline benchmark + working-directory: baseline + run: | + cargo run -p diskann-benchmark --features disk-index --release -- \ + run --input-file ../diskann_rust/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json \ + --output-file target/tmp/wikipedia-100K_benchmark_crate_baseline.json + + - name: Run target benchmark + working-directory: diskann_rust + run: | + cargo run -p diskann-benchmark --features disk-index --release -- \ + run --input-file diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json \ + --output-file target/tmp/wikipedia-100K_benchmark_crate_target.json + + - name: Generate diff stats (baseline vs target) + run: | + python diskann_rust/.github/scripts/compare_disk_index_json_output.py \ + --baseline baseline/target/tmp/wikipedia-100K_benchmark_crate_baseline.json \ + --branch diskann_rust/target/tmp/wikipedia-100K_benchmark_crate_target.json \ + --out diskann_rust/target/tmp/wikipedia-100K_change.csv + + - name: Convert results to Markdown + working-directory: diskann_rust + run: | + python .github/scripts/csv_to_markdown.py \ + --csv target/tmp/wikipedia-100K_change.csv \ + --md target/tmp/wikipedia-100K_change.md \ + --title 'A/A Results: Wikipedia-100K Dataset' + + - name: Validate benchmark results + working-directory: diskann_rust + run: | + python .github/scripts/benchmark_result_parse.py \ + --mode aa \ + --file target/tmp/wikipedia-100K_change.csv + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + + - name: Upload benchmark results + uses: actions/upload-artifact@v4 + if: always() + with: + name: aa-results-wikipedia-100K + path: | + diskann_rust/target/tmp/wikipedia-100K_change.csv + diskann_rust/target/tmp/wikipedia-100K_change.md + diskann_rust/target/tmp/wikipedia-100K_benchmark_crate_target.json + baseline/target/tmp/wikipedia-100K_benchmark_crate_baseline.json + retention-days: 30 + + # A/A benchmark: OpenAI ArXiv 100K dataset (main vs main) + aa-openai-100K: + name: A/A - OAI ArXiv 100K + runs-on: ubuntu-latest + timeout-minutes: 120 + + steps: + - name: Checkout main (target) + uses: actions/checkout@v4 + with: + ref: main + path: diskann_rust + lfs: true + + - name: Checkout main (baseline) + uses: actions/checkout@v4 + with: + ref: main + path: baseline + lfs: true + + - name: Install Rust ${{ env.rust_stable }} + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ env.rust_stable }} + + - name: Cache Rust dependencies (target) + uses: Swatinem/rust-cache@v2 + with: + workspaces: diskann_rust -> target + key: aa-target + + - name: Cache Rust dependencies (baseline) + uses: Swatinem/rust-cache@v2 + with: + workspaces: baseline -> target + key: aa-baseline + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y openssl libssl-dev pkg-config + + # Download pre-packaged OpenAI ArXiv 100K dataset from GitHub Release + - name: Download openai-100K dataset + env: + GH_TOKEN: ${{ github.token }} + run: | + mkdir -p diskann_rust/target/tmp baseline/target/tmp + gh release download benchmark-data-v1 --repo ${{ github.repository }} --pattern 'openai-100K.tar.gz' --dir . + tar xzf openai-100K.tar.gz -C diskann_rust/target/tmp/ + cp -r diskann_rust/target/tmp/OpenAIArXiv baseline/target/tmp/ + + - name: Run baseline benchmark + working-directory: baseline + run: | + cargo run -p diskann-benchmark --features disk-index --release -- \ + run --input-file ../diskann_rust/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json \ + --output-file target/tmp/openai-100K_benchmark_crate_baseline.json + + - name: Run target benchmark + working-directory: diskann_rust + run: | + cargo run -p diskann-benchmark --features disk-index --release -- \ + run --input-file diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json \ + --output-file target/tmp/openai-100K_benchmark_crate_target.json + + - name: Generate diff stats (baseline vs target) + run: | + python diskann_rust/.github/scripts/compare_disk_index_json_output.py \ + --baseline baseline/target/tmp/openai-100K_benchmark_crate_baseline.json \ + --branch diskann_rust/target/tmp/openai-100K_benchmark_crate_target.json \ + --out diskann_rust/target/tmp/openai-100K_change.csv + + - name: Convert results to Markdown + working-directory: diskann_rust + run: | + python .github/scripts/csv_to_markdown.py \ + --csv target/tmp/openai-100K_change.csv \ + --md target/tmp/openai-100K_change.md \ + --title 'A/A Results: OpenAI ArXiv 100K Dataset' + + - name: Validate benchmark results + working-directory: diskann_rust + run: | + python .github/scripts/benchmark_result_parse.py \ + --mode aa \ + --file target/tmp/openai-100K_change.csv + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + + - name: Upload benchmark results + uses: actions/upload-artifact@v4 + if: always() + with: + name: aa-results-openai-100K + path: | + diskann_rust/target/tmp/openai-100K_change.csv + diskann_rust/target/tmp/openai-100K_change.md + diskann_rust/target/tmp/openai-100K_benchmark_crate_target.json + baseline/target/tmp/openai-100K_benchmark_crate_baseline.json + retention-days: 30 + + # Notify diskann-admin on A/A failure + notify-on-failure: + name: Notify on A/A Failure + needs: [aa-wikipedia-100K, aa-openai-100K] + runs-on: ubuntu-latest + if: failure() + steps: + - name: Create GitHub issue for A/A failure + uses: actions/github-script@v7 + with: + script: | + const date = new Date().toISOString().split('T')[0]; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: `[Benchmark A/A] Daily stability test failed – ${date}`, + body: [ + `## Daily A/A Benchmark Failure`, + ``, + `The scheduled A/A benchmark run (main vs main) **failed** on ${date}.`, + `This indicates environment noise exceeded the configured thresholds.`, + ``, + `**Run:** ${runUrl}`, + ``, + `Please review the benchmark artifacts and determine if thresholds need tuning`, + `or if there is a runner environment issue.`, + ``, + `/cc @microsoft/diskann-admin`, + ].join('\n'), + labels: ['benchmark', 'A/A-failure'], + }); From 128791fc92b8c351b10308012f59b06b8cdd6f17 Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Fri, 27 Mar 2026 15:30:03 +0800 Subject: [PATCH 22/41] widen mean_cpus threshold to 15% for shared-runner CPU noise --- .github/scripts/benchmark_result_parse.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/benchmark_result_parse.py b/.github/scripts/benchmark_result_parse.py index c510360568..27385406db 100644 --- a/.github/scripts/benchmark_result_parse.py +++ b/.github/scripts/benchmark_result_parse.py @@ -178,7 +178,7 @@ "latency_95": [10, 'LT', ""], "mean_latency": [10, 'LT', ""], "mean_io_time": [10, 'LT', ""], - "mean_cpus": [10, 'LT', ""], + "mean_cpus": [15, 'LT', ""], # wider threshold — CPU time is noisy on shared runners "qps": [10, 'GT', 6.5], "mean_ios": [1, 'LT', 2410], "mean_comps": [1, 'LT', 33200], @@ -189,7 +189,7 @@ "latency_95": [10, 'LT', ""], "mean_latency": [10, 'LT', ""], "mean_io_time": [10, 'LT', ""], - "mean_cpus": [10, 'LT', ""], + "mean_cpus": [15, 'LT', ""], # wider threshold — CPU time is noisy on shared runners "qps": [10, 'GT', ""], "mean_ios": [10, 'LT', ""], "mean_comps": [10, 'LT', ""], @@ -200,7 +200,7 @@ "latency_95": [10, 'LT', ""], "mean_latency": [10, 'LT', ""], "mean_io_time": [10, 'LT', ""], - "mean_cpus": [10, 'LT', ""], + "mean_cpus": [15, 'LT', ""], # wider threshold — CPU time is noisy on shared runners "qps": [10, 'GT', ""], "mean_ios": [10, 'LT', ""], "mean_comps": [10, 'LT', ""], From 0c10c8c4c34fd92fbd68c403fc8523e56676ae13 Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Mon, 30 Mar 2026 15:32:20 +0800 Subject: [PATCH 23/41] move benchmark datasets to separate repo (YuanyuanTian-hh/diskann-benchmark-data) --- .github/workflows/benchmarks-aa.yml | 6 ++++-- .github/workflows/benchmarks.yml | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmarks-aa.yml b/.github/workflows/benchmarks-aa.yml index b272f7c698..c1fd45ad80 100644 --- a/.github/workflows/benchmarks-aa.yml +++ b/.github/workflows/benchmarks-aa.yml @@ -76,12 +76,13 @@ jobs: sudo apt-get install -y openssl libssl-dev pkg-config # Download pre-packaged Wikipedia-100K dataset from GitHub Release + # Source: https://github.com/harsha-simhadri/big-ann-benchmarks - name: Download wikipedia-100K dataset env: GH_TOKEN: ${{ github.token }} run: | mkdir -p diskann_rust/target/tmp baseline/target/tmp - gh release download benchmark-data-v1 --repo ${{ github.repository }} --pattern 'wikipedia-100K.tar.gz' --dir . + gh release download v1 --repo YuanyuanTian-hh/diskann-benchmark-data --pattern 'wikipedia-100K.tar.gz' --dir . tar xzf wikipedia-100K.tar.gz -C diskann_rust/target/tmp/ cp -r diskann_rust/target/tmp/wikipedia_cohere baseline/target/tmp/ @@ -181,12 +182,13 @@ jobs: sudo apt-get install -y openssl libssl-dev pkg-config # Download pre-packaged OpenAI ArXiv 100K dataset from GitHub Release + # Source: https://github.com/harsha-simhadri/big-ann-benchmarks - name: Download openai-100K dataset env: GH_TOKEN: ${{ github.token }} run: | mkdir -p diskann_rust/target/tmp baseline/target/tmp - gh release download benchmark-data-v1 --repo ${{ github.repository }} --pattern 'openai-100K.tar.gz' --dir . + gh release download v1 --repo YuanyuanTian-hh/diskann-benchmark-data --pattern 'openai-100K.tar.gz' --dir . tar xzf openai-100K.tar.gz -C diskann_rust/target/tmp/ cp -r diskann_rust/target/tmp/OpenAIArXiv baseline/target/tmp/ diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index a5c5c3a03b..8cdf767a94 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -93,12 +93,13 @@ jobs: # Download pre-packaged Wikipedia-100K dataset from GitHub Release # Dataset: 100K Cohere Wikipedia embeddings (768-dim, float32, cosine distance) + # Source: https://github.com/harsha-simhadri/big-ann-benchmarks - name: Download wikipedia-100K dataset env: GH_TOKEN: ${{ github.token }} run: | mkdir -p diskann_rust/target/tmp baseline/target/tmp - gh release download benchmark-data-v1 --repo ${{ github.repository }} --pattern 'wikipedia-100K.tar.gz' --dir . + gh release download v1 --repo YuanyuanTian-hh/diskann-benchmark-data --pattern 'wikipedia-100K.tar.gz' --dir . tar xzf wikipedia-100K.tar.gz -C diskann_rust/target/tmp/ cp -r diskann_rust/target/tmp/wikipedia_cohere baseline/target/tmp/ @@ -203,12 +204,13 @@ jobs: # Download pre-packaged OpenAI ArXiv 100K dataset from GitHub Release # Dataset: 100K OpenAI embeddings of ArXiv papers (1536-dim, float32, euclidean distance) + # Source: https://github.com/harsha-simhadri/big-ann-benchmarks - name: Download openai-100K dataset env: GH_TOKEN: ${{ github.token }} run: | mkdir -p diskann_rust/target/tmp baseline/target/tmp - gh release download benchmark-data-v1 --repo ${{ github.repository }} --pattern 'openai-100K.tar.gz' --dir . + gh release download v1 --repo YuanyuanTian-hh/diskann-benchmark-data --pattern 'openai-100K.tar.gz' --dir . tar xzf openai-100K.tar.gz -C diskann_rust/target/tmp/ cp -r diskann_rust/target/tmp/OpenAIArXiv baseline/target/tmp/ From be08e1f972e6c4c3c9d31030702ad4caa590436e Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Mon, 30 Mar 2026 15:57:22 +0800 Subject: [PATCH 24/41] consolidate 3 benchmark scripts into single benchmark_validate.py Replaces the previous 3-step pipeline (JSONCSVMarkdownvalidate) with a single script that reads both JSONs directly, compares metrics, writes Markdown to step summary, checks thresholds, and posts PR comments. Removed: - compare_disk_index_json_output.py (JSON diff CSV) - csv_to_markdown.py (CSV Markdown) - benchmark_result_parse.py (CSV threshold check) Also removes pip install csvtomd/numpy/scipy all scripts now use stdlib only. --- .github/scripts/benchmark_result_parse.py | 574 ------------------ .github/scripts/benchmark_validate.py | 425 +++++++++++++ .../scripts/compare_disk_index_json_output.py | 256 -------- .github/scripts/csv_to_markdown.py | 50 -- .github/workflows/benchmarks-aa.yml | 48 +- .github/workflows/benchmarks.yml | 57 +- 6 files changed, 444 insertions(+), 966 deletions(-) delete mode 100644 .github/scripts/benchmark_result_parse.py create mode 100644 .github/scripts/benchmark_validate.py delete mode 100644 .github/scripts/compare_disk_index_json_output.py delete mode 100644 .github/scripts/csv_to_markdown.py diff --git a/.github/scripts/benchmark_result_parse.py b/.github/scripts/benchmark_result_parse.py deleted file mode 100644 index 27385406db..0000000000 --- a/.github/scripts/benchmark_result_parse.py +++ /dev/null @@ -1,574 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT license. - -""" -Benchmark Result Parser for GitHub Actions - -Parses benchmark CSV results and validates against thresholds. -Posts comments to GitHub PRs when regressions are detected. - -Usage: - python benchmark_result_parse.py --mode pr --file results.csv - python benchmark_result_parse.py --mode aa --file results.csv --data search - -Environment Variables (for PR comments): - GITHUB_TOKEN: GitHub token for API access - GITHUB_REPOSITORY: Owner/repo (e.g., "microsoft/DiskANN") - GITHUB_PR_NUMBER: Pull request number - GITHUB_RUN_ID: Workflow run ID for linking to logs -""" - -import csv -import os -import sys -import argparse -import json -from typing import Any - -# Optional: requests for posting PR comments -try: - import requests - HAS_REQUESTS = True -except ImportError: - HAS_REQUESTS = False - - -# ============================================================================= -# Data Structures -# ============================================================================= - -# Template for full benchmark data (build + search) -DATA_TEMPLATE_FULL = { - "DiskIndexBuild-PqConstruction": { - "duration_seconds": [], - "peak_memory_usage": [] - }, - "DiskIndexBuild-InmemIndexBuild": { - "duration_seconds": [], - "peak_memory_usage": [] - }, - "search_disk_index-search_completed": { - "duration_seconds": [], - "peak_memory_usage": [] - }, - "disk_index_perf_test": { - "total_duration_seconds": [], - }, - "index-build statistics": { - "total_time": [], - "total_comparisons": [], - "search_hops": [] - }, - "search-with-L=2000-bw=4": { - "latency_95": [], - "mean_latency": [], - "mean_io_time": [], - "mean_cpus": [], - "qps": [], - "mean_ios": [], - "mean_comps": [], - "mean_hops": [], - "recall": [] - }, - "search-with-L=100-bw=4": { - "latency_95": [], - "mean_latency": [], - "mean_io_time": [], - "mean_cpus": [], - "qps": [], - "mean_ios": [], - "mean_comps": [], - "mean_hops": [], - "recall": [] - }, - "search-with-L=200-bw=4": { - "latency_95": [], - "mean_latency": [], - "mean_io_time": [], - "mean_cpus": [], - "qps": [], - "mean_ios": [], - "mean_comps": [], - "mean_hops": [], - "recall": [] - } -} -DATA_TEMPLATE_SEARCH = { - "search_disk_index-search_completed": { - "duration_seconds": [], - "peak_memory_usage": [] - }, - "disk_index_perf_test": { - "total_duration_seconds": [], - }, - "search-with-L=2000-bw=4": { - "latency_95": [], - "mean_latency": [], - "mean_io_time": [], - "mean_cpus": [], - "qps": [], - "mean_ios": [], - "mean_comps": [], - "mean_hops": [], - "recall": [] - }, - "search-with-L=100-bw=4": { - "latency_95": [], - "mean_latency": [], - "mean_io_time": [], - "mean_cpus": [], - "qps": [], - "mean_ios": [], - "mean_comps": [], - "mean_hops": [], - "recall": [] - }, - "search-with-L=200-bw=4": { - "latency_95": [], - "mean_latency": [], - "mean_io_time": [], - "mean_cpus": [], - "qps": [], - "mean_ios": [], - "mean_comps": [], - "mean_hops": [], - "recall": [] - } -} - -# Thresholds for benchmark values -# Format: [threshold_percentage, direction, contract_value] -# - threshold_percentage: Maximum allowed deviation percentage -# - direction: 'GT' = higher is better, 'LT' = lower is better -# - contract_value: Promised performance value (empty string if none) -# -# For 'GT' metrics (like QPS, recall): regression if value decreases beyond threshold -# For 'LT' metrics (like latency, memory): regression if value increases beyond threshold -DATA_THRESHOLDS = { - "DiskIndexBuild-PqConstruction": { - "duration_seconds": [10, 'LT', ""], - "peak_memory_usage": [10, 'LT', ""] - }, - "DiskIndexBuild-InmemIndexBuild": { - "duration_seconds": [10, 'LT', ""], - "peak_memory_usage": [10, 'LT', ""] - }, - "search_disk_index-search_completed": { - "duration_seconds": [10, 'LT', ""], - "peak_memory_usage": [10, 'LT', 1.42] - }, - "disk_index_perf_test": { - "total_duration_seconds": [10, 'LT', ""], - }, - "index-build statistics": { - # Calibrated from 5 GitHub runner runs (10 observations): - # Wikipedia: 35.9–37.2s, OpenAI: 23.0–76.4s (SQ_1_2.0 variance) - # Contract: worst × 1.5 to absorb shared-runner variance - "total_time": [10, 'LT', 115], - "total_comparisons": [1, 'LT', ""], - "search_hops": [1, 'LT', ""] - }, - "search-with-L=2000-bw=4": { - # Calibrated from 5 GitHub runner runs (10 observations): - # QPS: 9.56–9.75 (both datasets) - # Recall: wiki 99.87%, oai 99.67–99.91% - # mean_ios: ~2007 (deterministic) - # mean_comps: wiki ~27609, oai 21618–24733 - "latency_95": [10, 'LT', ""], - "mean_latency": [10, 'LT', ""], - "mean_io_time": [10, 'LT', ""], - "mean_cpus": [15, 'LT', ""], # wider threshold — CPU time is noisy on shared runners - "qps": [10, 'GT', 6.5], - "mean_ios": [1, 'LT', 2410], - "mean_comps": [1, 'LT', 33200], - "mean_hops": [1, 'LT', ""], - "recall": [1, 'GT', 98.0] - }, - "search-with-L=100-bw=4": { - "latency_95": [10, 'LT', ""], - "mean_latency": [10, 'LT', ""], - "mean_io_time": [10, 'LT', ""], - "mean_cpus": [15, 'LT', ""], # wider threshold — CPU time is noisy on shared runners - "qps": [10, 'GT', ""], - "mean_ios": [10, 'LT', ""], - "mean_comps": [10, 'LT', ""], - "mean_hops": [10, 'LT', ""], - "recall": [1, 'GT', ""] - }, - "search-with-L=200-bw=4": { - "latency_95": [10, 'LT', ""], - "mean_latency": [10, 'LT', ""], - "mean_io_time": [10, 'LT', ""], - "mean_cpus": [15, 'LT', ""], # wider threshold — CPU time is noisy on shared runners - "qps": [10, 'GT', ""], - "mean_ios": [10, 'LT', ""], - "mean_comps": [10, 'LT', ""], - "mean_hops": [10, 'LT', ""], - "recall": [1, 'GT', ""] - } -} - - -# ============================================================================= -# CSV Parsing -# ============================================================================= - -def parse_csv(file_path: str, data: dict[str, dict[str, list]]) -> dict[str, dict[str, list]]: - """ - Parse benchmark CSV file and populate data structure. - - CSV format produced by compare_disk_index_json_output.py: - Column 0: Parent Span Name (category, e.g. "index-build statistics") - Column 1: Span Name (display name, unused for matching) - Column 2: Stat Key (metric key, e.g. "qps") - Column 3: Stat Value (Target) - Column 4: Stat Value (Baseline) - Column 5: Deviation (%) - """ - with open(file_path, 'r', encoding='utf-8') as f: - reader = csv.reader(f) - next(reader) # Skip header row - - for row in reader: - if len(row) < 6: - continue - - category = row[0].strip() - metric_name = row[2].strip() - - if category in data and metric_name in data[category]: - # Append: [current_value, baseline_value, change_percentage] - data[category][metric_name].append(row[3]) # target (current) - data[category][metric_name].append(row[4]) # baseline - data[category][metric_name].append(row[5]) # deviation % - - return data - - -def get_data_template(data_type: str) -> dict[str, dict[str, list]]: - """Get a fresh copy of the data template.""" - import copy - if data_type == 'search': - return copy.deepcopy(DATA_TEMPLATE_SEARCH) - return copy.deepcopy(DATA_TEMPLATE_FULL) - - -# ============================================================================= -# Threshold Checking -# ============================================================================= - -def get_target_change_range(threshold: float, direction: str, mode: str) -> tuple[float, float]: - """ - Calculate acceptable change range based on threshold and direction. - - Args: - threshold: Maximum allowed deviation percentage - direction: 'GT' (higher is better) or 'LT' (lower is better) - mode: 'aa' (A/A test, symmetric) or 'pr' (PR test, directional) - - Returns: - Tuple of (min_allowed, max_allowed) change percentages - """ - if mode == 'aa': - # A/A test: symmetric threshold - return (-threshold, threshold) - else: - # PR test: directional threshold - if direction == 'GT': - # Higher is better: allow any improvement, flag regressions - return (-threshold, float('inf')) - else: - # Lower is better: allow any improvement (negative change), flag increases - return (float('-inf'), threshold) - - -def format_interval(start: float, end: float) -> str: - """Format a numeric interval as a string.""" - start_str = '-inf' if start == float('-inf') else f"{start}%" - end_str = 'inf' if end == float('inf') else f"{end}%" - return f"({start_str} - {end_str})" - - -def is_change_threshold_failed(change: float, target_range: tuple[float, float]) -> bool: - """Check if the change exceeds the allowed threshold range.""" - return change < target_range[0] or change > target_range[1] - - -def is_promise_broken(current_value: float, target_value: Any, direction: str) -> tuple[bool, str]: - """ - Check if the current value violates a promised contract value. - - Returns: - Tuple of (is_broken, formatted_target_value) - """ - if target_value == "": - return False, "N/A" - - target_value = float(target_value) - - if direction == 'GT': - # Higher is better: current should be >= target - if current_value < target_value: - return True, f"> {target_value}" - else: - # Lower is better: current should be <= target - if current_value > target_value: - return True, f"< {target_value}" - - return False, str(target_value) - - -def get_outcome_message(threshold_failed: bool, promise_broken: bool) -> str: - """Generate human-readable outcome message.""" - if threshold_failed and promise_broken: - return 'Regression detected, Promise broken' - elif promise_broken: - return 'Promise broken' - elif threshold_failed: - return 'Regression detected' - return 'OK' - - -def check_thresholds( - data: dict[str, dict[str, list]], - thresholds: dict[str, dict[str, list]], - mode: str, - run_id: str | None = None -) -> tuple[bool, str]: - """ - Check all metrics against their thresholds. - - Returns: - Tuple of (has_failures, failure_report_markdown) - """ - failed_rows = [] - - for category in data: - for metric in data[category]: - # Skip metrics without thresholds defined - if category not in thresholds or metric not in thresholds[category]: - print(f"Skipping {category}/{metric} - no threshold defined") - continue - - values = data[category][metric] - if not values: - # No data for this metric in the CSV — skip silently - continue - - # Parse values: [current, baseline, change%] - try: - value_current = float(values[0]) - value_baseline = float(values[1]) - change = float(values[2]) if values[2] else 0.0 - except (ValueError, IndexError) as e: - print(f"ERROR: Failed to parse {category}/{metric}: {e}") - return True, f"Parse error for {category}/{metric}" - - # Get threshold config - threshold_config = thresholds[category][metric] - threshold_pct = threshold_config[0] - direction = threshold_config[1] - contract_value = threshold_config[2] - - # Check thresholds - target_range = get_target_change_range(threshold_pct, direction, mode) - threshold_failed = is_change_threshold_failed(change, target_range) - promise_broken, target_formatted = is_promise_broken(value_current, contract_value, direction) - - if threshold_failed: - print(f"THRESHOLD FAILED: {category}/{metric} change={change}% allowed={format_interval(*target_range)}") - if promise_broken: - print(f"PROMISE BROKEN: {category}/{metric} value={value_current} required={target_formatted}") - - if threshold_failed or promise_broken: - outcome = get_outcome_message(threshold_failed, promise_broken) - failed_rows.append( - f"| {category}/{metric} | {value_baseline} | {value_current} | " - f"{target_formatted} | {change}% | {format_interval(*target_range)} | {outcome} |" - ) - - if failed_rows: - # Build failure report - logs_link = "" - if run_id: - repo = os.getenv('GITHUB_REPOSITORY', 'microsoft/DiskANN') - logs_link = f"https://github.com/{repo}/actions/runs/{run_id}" - - report = "### ❌ Benchmark Check Failed\n\n" - if logs_link: - report += f"Please investigate the [workflow logs]({logs_link}) to determine if the failure is due to your changes.\n\n" - - report += "| Metric | Baseline | Current | Contract | Change | Allowed | Outcome |\n" - report += "|--------|----------|---------|----------|--------|---------|--------|\n" - report += "\n".join(failed_rows) - - return True, report - - return False, "" - - -# ============================================================================= -# GitHub Integration -# ============================================================================= - -def post_github_pr_comment(comment: str) -> bool: - """ - Post a comment to a GitHub pull request. - - Requires environment variables: - GITHUB_TOKEN: Personal access token or GitHub Actions token - GITHUB_REPOSITORY: Owner/repo format - GITHUB_PR_NUMBER: Pull request number - """ - if not HAS_REQUESTS: - print("WARNING: 'requests' module not available, cannot post PR comment") - return False - - token = os.getenv('GITHUB_TOKEN') - repo = os.getenv('GITHUB_REPOSITORY') - pr_number = os.getenv('GITHUB_PR_NUMBER') - - if not all([token, repo, pr_number]): - print("WARNING: Missing GitHub environment variables for PR comment") - print(f" GITHUB_TOKEN: {'set' if token else 'missing'}") - print(f" GITHUB_REPOSITORY: {repo or 'missing'}") - print(f" GITHUB_PR_NUMBER: {pr_number or 'missing'}") - return False - - url = f"https://api.github.com/repos/{repo}/issues/{pr_number}/comments" - headers = { - "Accept": "application/vnd.github+json", - "Authorization": f"Bearer {token}", - "X-GitHub-Api-Version": "2022-11-28" - } - body = {"body": comment} - - try: - response = requests.post(url, headers=headers, json=body, timeout=30) - response.raise_for_status() - print(f"Successfully posted comment to PR #{pr_number}") - return True - except requests.RequestException as e: - print(f"ERROR: Failed to post PR comment: {e}") - return False - - -def write_github_step_summary(content: str) -> None: - """Write content to GitHub Actions step summary.""" - summary_file = os.getenv('GITHUB_STEP_SUMMARY') - if summary_file: - with open(summary_file, 'a', encoding='utf-8') as f: - f.write(content) - f.write("\n") - - -def write_github_output(name: str, value: str) -> None: - """Write an output variable for GitHub Actions.""" - output_file = os.getenv('GITHUB_OUTPUT') - if output_file: - with open(output_file, 'a', encoding='utf-8') as f: - f.write(f"{name}={value}\n") - - -# ============================================================================= -# Main -# ============================================================================= - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description='Parse benchmark results and validate against thresholds.', - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=""" -Examples: - # Check PR benchmark results - python benchmark_result_parse.py --mode pr --file results_change.csv - - # Check A/A test results (symmetric thresholds) - python benchmark_result_parse.py --mode aa --file results_change.csv - - # Check search-only benchmarks - python benchmark_result_parse.py --mode pr --file results_change.csv --data search - """ - ) - parser.add_argument( - '--mode', - type=str, - default='aa', - choices=['aa', 'pr', 'lkg'], - help='Benchmark mode: aa=A/A test (symmetric), pr=PR test (directional), lkg=last known good' - ) - parser.add_argument( - '--data', - type=str, - default='both', - choices=['both', 'search'], - help='Type of benchmark data: both=full benchmark, search=search-only' - ) - parser.add_argument( - '--file', - type=str, - default=None, - help='Path to CSV file (overrides FILE_PATH env var)' - ) - parser.add_argument( - '--no-comment', - action='store_true', - help='Skip posting PR comment even in pr mode' - ) - return parser.parse_args() - - -def main() -> int: - args = parse_args() - - # Get file path - file_path = args.file or os.getenv('FILE_PATH') - if not file_path: - print("ERROR: No input file specified. Use --file or set FILE_PATH env var.") - return 1 - - if not os.path.exists(file_path): - print(f"ERROR: File not found: {file_path}") - return 1 - - print(f"Benchmark mode: {args.mode}") - print(f"Data type: {args.data}") - print(f"Input file: {file_path}") - - # Parse CSV - data_template = get_data_template(args.data) - data = parse_csv(file_path, data_template) - - # Debug output - print("\nParsed data:") - print(json.dumps({k: {sk: sv for sk, sv in v.items() if sv} for k, v in data.items() if any(v.values())}, indent=2)) - - # Check thresholds - run_id = os.getenv('GITHUB_RUN_ID') - has_failures, report = check_thresholds(data, DATA_THRESHOLDS, args.mode, run_id) - - if has_failures: - print("\n" + report) - - # Write to GitHub step summary - write_github_step_summary(report) - - # Post PR comment if in pr mode - if args.mode == 'pr' and not args.no_comment: - post_github_pr_comment(report) - - # Set output for downstream steps - write_github_output('benchmark_failed', 'true') - - return 1 - - print("\n✅ All benchmark values passed!") - write_github_step_summary("### ✅ Benchmark Check Passed\n\nAll metrics within acceptable thresholds.") - write_github_output('benchmark_failed', 'false') - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/.github/scripts/benchmark_validate.py b/.github/scripts/benchmark_validate.py new file mode 100644 index 0000000000..cb69f60547 --- /dev/null +++ b/.github/scripts/benchmark_validate.py @@ -0,0 +1,425 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT license. + +""" +Benchmark Validator for GitHub Actions + +Compares two benchmark JSON outputs (baseline vs target), checks thresholds, +writes a Markdown summary, and optionally posts a PR comment on failure. + +This single script replaces the previous three-step pipeline: + compare_disk_index_json_output.py → csv_to_markdown.py → benchmark_result_parse.py + +Usage: + # PR mode (directional thresholds, posts PR comment on failure) + python benchmark_validate.py --mode pr --baseline baseline.json --target target.json + + # A/A mode (symmetric thresholds) + python benchmark_validate.py --mode aa --baseline baseline.json --target target.json + +Environment Variables (for PR comments): + GITHUB_TOKEN: GitHub token for API access + GITHUB_REPOSITORY: Owner/repo (e.g., "microsoft/DiskANN") + GITHUB_PR_NUMBER: Pull request number + GITHUB_RUN_ID: Workflow run ID for linking to logs + GITHUB_STEP_SUMMARY: Path to step summary file +""" + +import json +import os +import sys +import argparse +from typing import Any +from urllib.request import urlopen, Request +from urllib.error import URLError + + +# ============================================================================= +# JSON Extraction +# ============================================================================= + +def load_json(path: str) -> list[dict[str, Any]]: + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def extract_build_metrics(results: dict) -> dict[str, float]: + build = results.get("build", {}) + if not build: + return {} + + metrics: dict[str, float] = {} + + build_time = build.get("build_time") + if build_time: + metrics["total_time"] = build_time / 1e6 # μs → s + + for span in build.get("span_metrics", {}).get("spans", []): + name = span.get("span_name", "") + data = span.get("metrics", {}) + if name == "DiskIndexBuild-PqConstruction": + metrics["pq_construction_time"] = data.get("duration_seconds", 0) + elif name == "DiskIndexBuild-InmemIndexBuild": + metrics["inmem_index_build_time"] = data.get("duration_seconds", 0) + elif name == "DiskIndexBuild-DiskLayout": + metrics["disk_layout_time"] = data.get("duration_seconds", 0) + + return metrics + + +def extract_search_metrics(results: dict, search_l: int, beam_width: int) -> dict[str, float]: + search = results.get("search", {}) + if not search: + return {} + + metrics: dict[str, float] = {} + + # From search_results_per_l + for sr in search.get("search_results_per_l", []): + if sr.get("search_l") == search_l: + metrics["qps"] = sr.get("qps", 0) + metrics["recall"] = sr.get("recall", 0) + metrics["mean_latency"] = sr.get("mean_latency", 0) + metrics["mean_ios"] = sr.get("mean_ios", 0) + metrics["mean_comps"] = sr.get("mean_comparisons", 0) + metrics["mean_hops"] = sr.get("mean_hops", 0) + metrics["mean_io_time"] = sr.get("mean_io_time", 0) + metrics["mean_cpus"] = sr.get("mean_cpu_time", 0) + metrics["latency_95"] = sr.get("p999_latency", 0) + break + + # Override with span metrics if available + span_name = f"search-with-L={search_l}-bw={beam_width}" + for span in search.get("span_metrics", {}).get("spans", []): + if span.get("span_name") == span_name: + data = span.get("metrics", {}) + for key in ("qps", "recall", "mean_latency", "mean_ios", "mean_comps", + "mean_hops", "mean_io_time", "mean_cpus"): + if key in data: + metrics[key] = data[key] + break + + return metrics + + +def compute_diff(baseline_json: list[dict], target_json: list[dict]) -> list[dict]: + """ + Compare baseline and target JSONs. + Returns a flat list of metric diffs: + [{category, metric, baseline, target, deviation}, ...] + """ + rows = [] + + for baseline, target in zip(baseline_json, target_json): + b_results = baseline.get("results", {}) + t_results = target.get("results", {}) + + inp = target.get("input", {}) + search_phase = inp.get("content", {}).get("search_phase", {}) + search_list = search_phase.get("search_list", [2000]) + beam_width = search_phase.get("beam_width", 4) + primary_l = search_list[0] if search_list else 2000 + + # Build metrics + b_build = extract_build_metrics(b_results) + t_build = extract_build_metrics(t_results) + + for key in ("total_time", "pq_construction_time", "inmem_index_build_time", "disk_layout_time"): + if key in t_build or key in b_build: + bv = b_build.get(key, 0) + tv = t_build.get(key, 0) + rows.append({ + "category": "index-build statistics", + "metric": key, + "baseline": bv, + "target": tv, + "deviation": ((tv - bv) / bv * 100) if bv else 0, + }) + + # Search metrics + b_search = extract_search_metrics(b_results, primary_l, beam_width) + t_search = extract_search_metrics(t_results, primary_l, beam_width) + span_cat = f"search-with-L={primary_l}-bw={beam_width}" + + for key in ("qps", "recall", "mean_latency", "latency_95", "mean_ios", + "mean_comps", "mean_hops", "mean_io_time", "mean_cpus"): + if key in t_search or key in b_search: + bv = b_search.get(key, 0) + tv = t_search.get(key, 0) + rows.append({ + "category": span_cat, + "metric": key, + "baseline": bv, + "target": tv, + "deviation": ((tv - bv) / bv * 100) if bv else 0, + }) + + return rows + + +# ============================================================================= +# Thresholds +# ============================================================================= + +# Format: [max_deviation_%, direction, contract_value] +# direction: 'GT' = higher is better, 'LT' = lower is better +# contract_value: absolute limit (empty string = none) +THRESHOLDS: dict[str, dict[str, list]] = { + "DiskIndexBuild-PqConstruction": { + "duration_seconds": [10, "LT", ""], + "peak_memory_usage": [10, "LT", ""], + }, + "DiskIndexBuild-InmemIndexBuild": { + "duration_seconds": [10, "LT", ""], + "peak_memory_usage": [10, "LT", ""], + }, + "search_disk_index-search_completed": { + "duration_seconds": [10, "LT", ""], + "peak_memory_usage": [10, "LT", 1.42], + }, + "disk_index_perf_test": { + "total_duration_seconds": [10, "LT", ""], + }, + "index-build statistics": { + # Calibrated from 5 GitHub runner runs (10 observations): + # Wikipedia: 35.9–37.2s, OpenAI: 23.0–76.4s (SQ_1_2.0 variance) + # Contract: worst × 1.5 to absorb shared-runner variance + "total_time": [10, "LT", 115], + "total_comparisons": [1, "LT", ""], + "search_hops": [1, "LT", ""], + }, + "search-with-L=2000-bw=4": { + # Calibrated from 5 GitHub runner runs (10 observations) + "latency_95": [10, "LT", ""], + "mean_latency": [10, "LT", ""], + "mean_io_time": [10, "LT", ""], + "mean_cpus": [15, "LT", ""], # wider — CPU time is noisy on shared runners + "qps": [10, "GT", 6.5], + "mean_ios": [1, "LT", 2410], + "mean_comps": [1, "LT", 33200], + "mean_hops": [1, "LT", ""], + "recall": [1, "GT", 98.0], + }, + "search-with-L=100-bw=4": { + "latency_95": [10, "LT", ""], + "mean_latency": [10, "LT", ""], + "mean_io_time": [10, "LT", ""], + "mean_cpus": [15, "LT", ""], + "qps": [10, "GT", ""], + "mean_ios": [10, "LT", ""], + "mean_comps": [10, "LT", ""], + "mean_hops": [10, "LT", ""], + "recall": [1, "GT", ""], + }, + "search-with-L=200-bw=4": { + "latency_95": [10, "LT", ""], + "mean_latency": [10, "LT", ""], + "mean_io_time": [10, "LT", ""], + "mean_cpus": [15, "LT", ""], + "qps": [10, "GT", ""], + "mean_ios": [10, "LT", ""], + "mean_comps": [10, "LT", ""], + "mean_hops": [10, "LT", ""], + "recall": [1, "GT", ""], + }, +} + + +def allowed_range(threshold: float, direction: str, mode: str) -> tuple[float, float]: + """Acceptable change range (in %).""" + if mode == "aa": + return (-threshold, threshold) + if direction == "GT": + return (-threshold, float("inf")) + return (float("-inf"), threshold) + + +def fmt_range(lo: float, hi: float) -> str: + lo_s = "-inf" if lo == float("-inf") else f"{lo}%" + hi_s = "inf" if hi == float("inf") else f"{hi}%" + return f"({lo_s} – {hi_s})" + + +def check_contract(value: float, contract: Any, direction: str) -> tuple[bool, str]: + """Check if value violates a hard contract. Returns (broken, formatted_contract).""" + if contract == "": + return False, "N/A" + contract = float(contract) + if direction == "GT" and value < contract: + return True, f"> {contract}" + if direction == "LT" and value > contract: + return True, f"< {contract}" + return False, str(contract) + + +# ============================================================================= +# Validation +# ============================================================================= + +def validate(diffs: list[dict], mode: str, run_id: str | None) -> tuple[bool, str]: + """ + Check all diffs against thresholds. + Returns (has_failures, markdown_report). + """ + failed_rows: list[str] = [] + + for d in diffs: + cat, metric = d["category"], d["metric"] + if cat not in THRESHOLDS or metric not in THRESHOLDS[cat]: + continue + + pct, direction, contract = THRESHOLDS[cat][metric] + rng = allowed_range(pct, direction, mode) + dev = d["deviation"] + + threshold_failed = dev < rng[0] or dev > rng[1] + contract_broken, contract_fmt = check_contract(d["target"], contract, direction) + + if threshold_failed: + print(f"THRESHOLD FAILED: {cat}/{metric} change={dev:.2f}% allowed={fmt_range(*rng)}") + if contract_broken: + print(f"CONTRACT BROKEN: {cat}/{metric} value={d['target']} required={contract_fmt}") + + if threshold_failed or contract_broken: + outcome = [] + if threshold_failed: + outcome.append("Regression detected") + if contract_broken: + outcome.append("Contract broken") + failed_rows.append( + f"| {cat}/{metric} | {d['baseline']:.4g} | {d['target']:.4g} | " + f"{contract_fmt} | {dev:.2f}% | {fmt_range(*rng)} | {', '.join(outcome)} |" + ) + + if not failed_rows: + return False, "" + + logs_link = "" + if run_id: + repo = os.getenv("GITHUB_REPOSITORY", "microsoft/DiskANN") + logs_link = f"https://github.com/{repo}/actions/runs/{run_id}" + + report = "### ❌ Benchmark Check Failed\n\n" + if logs_link: + report += f"Please investigate the [workflow logs]({logs_link}) to determine if the failure is due to your changes.\n\n" + report += "| Metric | Baseline | Current | Contract | Change | Allowed | Outcome |\n" + report += "|--------|----------|---------|----------|--------|---------|--------|\n" + report += "\n".join(failed_rows) + + return True, report + + +# ============================================================================= +# Markdown output +# ============================================================================= + +def diffs_to_markdown(diffs: list[dict], title: str) -> str: + """Render diffs as a Markdown table.""" + lines = [ + f"### {title}", + "", + "| Category | Metric | Baseline | Current | Change |", + "|----------|--------|----------|---------|--------|", + ] + for d in diffs: + lines.append( + f"| {d['category']} | {d['metric']} | {d['baseline']:.4g} | " + f"{d['target']:.4g} | {d['deviation']:+.2f}% |" + ) + return "\n".join(lines) + + +# ============================================================================= +# GitHub helpers (stdlib only — no requests dependency) +# ============================================================================= + +def post_pr_comment(body: str) -> bool: + token = os.getenv("GITHUB_TOKEN") + repo = os.getenv("GITHUB_REPOSITORY") + pr = os.getenv("GITHUB_PR_NUMBER") + if not all([token, repo, pr]): + print("WARNING: Missing GitHub env vars for PR comment " + f"(TOKEN={'set' if token else 'missing'}, REPO={repo or 'missing'}, PR={pr or 'missing'})") + return False + + url = f"https://api.github.com/repos/{repo}/issues/{pr}/comments" + data = json.dumps({"body": body}).encode() + req = Request(url, data=data, method="POST", headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "X-GitHub-Api-Version": "2022-11-28", + "Content-Type": "application/json", + }) + try: + with urlopen(req, timeout=30) as resp: + if resp.status < 300: + print(f"Posted comment to PR #{pr}") + return True + except URLError as e: + print(f"ERROR posting PR comment: {e}") + return False + + +def write_step_summary(content: str) -> None: + path = os.getenv("GITHUB_STEP_SUMMARY") + if path: + with open(path, "a", encoding="utf-8") as f: + f.write(content + "\n") + + +# ============================================================================= +# Main +# ============================================================================= + +def main() -> int: + parser = argparse.ArgumentParser( + description="Compare two benchmark JSONs, validate thresholds, output Markdown." + ) + parser.add_argument("--mode", choices=["aa", "pr"], default="aa", + help="aa = symmetric thresholds, pr = directional") + parser.add_argument("--baseline", required=True, help="Baseline JSON path") + parser.add_argument("--target", required=True, help="Target JSON path") + parser.add_argument("--title", default="Benchmark Results", + help="Title for the Markdown summary table") + parser.add_argument("--no-comment", action="store_true", + help="Skip posting PR comment on failure") + args = parser.parse_args() + + print(f"Mode: {args.mode}") + print(f"Baseline: {args.baseline}") + print(f"Target: {args.target}") + + baseline = load_json(args.baseline) + target = load_json(args.target) + + if len(baseline) != len(target): + print(f"ERROR: JSON arrays differ in length: {len(baseline)} vs {len(target)}") + return 1 + + # Compare + diffs = compute_diff(baseline, target) + print(f"\nCompared {len(diffs)} metrics") + + # Write Markdown summary + md = diffs_to_markdown(diffs, args.title) + write_step_summary(md) + + # Validate thresholds + run_id = os.getenv("GITHUB_RUN_ID") + has_failures, report = validate(diffs, args.mode, run_id) + + if has_failures: + print("\n" + report) + write_step_summary(report) + if args.mode == "pr" and not args.no_comment: + post_pr_comment(report) + return 1 + + print("\n✅ All metrics within thresholds") + write_step_summary("### ✅ Benchmark Check Passed\n\nAll metrics within acceptable thresholds.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/compare_disk_index_json_output.py b/.github/scripts/compare_disk_index_json_output.py deleted file mode 100644 index ca9c9d26b2..0000000000 --- a/.github/scripts/compare_disk_index_json_output.py +++ /dev/null @@ -1,256 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT license. - -""" -Compare two disk-index benchmark JSON files and emit a diff CSV. - -This script takes baseline and branch (target) JSON files from the benchmark crate's -disk-index benchmarks and produces a CSV file comparing the metrics with deviation percentages. - -The output format matches the CSV structure expected by benchmark_result_parse.py: - Parent Span Name, Span Name, Stat Key, Stat Value (Target), Stat Value (Baseline), Deviation (%) - -Usage: - python compare_disk_index_json_output.py \\ - --baseline baseline/target/tmp/_benchmark_crate_baseline.json \\ - --branch diskann_rust/target/tmp/_benchmark_crate_target.json \\ - --out diskann_rust/target/tmp/_change.csv -""" - -import json -import csv -import argparse -from typing import List, Dict, Any, Optional - - -def load_json(path: str) -> List[Dict[str, Any]]: - """Load JSON file and return the parsed content.""" - with open(path, "r", encoding="utf-8") as f: - return json.load(f) - - -def calc_deviation(baseline: float, target: float) -> str: - """Calculate the percentage deviation from baseline to target.""" - try: - if baseline != 0: - dev = ((target - baseline) / baseline) * 100 - return f"{dev:.2f}" - return "" - except Exception: - return "" - - -def extract_build_metrics(results: Dict[str, Any]) -> Dict[str, Any]: - """Extract build metrics from the results structure.""" - if not results: - return {} - - build = results.get("build", {}) - if not build: - return {} - - metrics = {} - - # Total build time (in seconds) - build_time = build.get("build_time") - if build_time: - # build_time is in microseconds, convert to seconds - metrics["total_time"] = build_time / 1e6 - - # Extract span metrics - span_metrics = build.get("span_metrics", {}) - spans = span_metrics.get("spans", []) - - for span in spans: - span_name = span.get("span_name", "") - span_data = span.get("metrics", {}) - - if span_name == "DiskIndexBuild-PqConstruction": - metrics["pq_construction_time"] = span_data.get("duration_seconds", 0) - elif span_name == "DiskIndexBuild-InmemIndexBuild": - metrics["inmem_index_build_time"] = span_data.get("duration_seconds", 0) - elif span_name == "DiskIndexBuild-DiskLayout": - metrics["disk_layout_time"] = span_data.get("duration_seconds", 0) - elif span_name == "disk-index-build": - metrics["total_build_duration"] = span_data.get("duration_seconds", 0) - - return metrics - - -def extract_search_metrics(results: Dict[str, Any], search_l: int, beam_width: int) -> Dict[str, Any]: - """Extract search metrics for a specific search_l value.""" - if not results: - return {} - - search = results.get("search", {}) - if not search: - return {} - - metrics = {} - - # Find the search result for the specified search_l - search_results = search.get("search_results_per_l", []) - for sr in search_results: - if sr.get("search_l") == search_l: - metrics["qps"] = sr.get("qps", 0) - metrics["recall"] = sr.get("recall", 0) - metrics["mean_latency"] = sr.get("mean_latency", 0) - metrics["mean_ios"] = sr.get("mean_ios", 0) - metrics["mean_comps"] = sr.get("mean_comparisons", 0) - metrics["mean_hops"] = sr.get("mean_hops", 0) - metrics["mean_io_time"] = sr.get("mean_io_time", 0) - metrics["mean_cpus"] = sr.get("mean_cpu_time", 0) - metrics["latency_95"] = sr.get("p999_latency", 0) # Use p999 as proxy for 95th percentile - break - - # Also try span metrics - span_metrics = search.get("span_metrics", {}) - spans = span_metrics.get("spans", []) - - search_span_name = f"search-with-L={search_l}-bw={beam_width}" - for span in spans: - if span.get("span_name") == search_span_name: - span_data = span.get("metrics", {}) - # Override with span metrics if they exist - if "qps" in span_data: - metrics["qps"] = span_data["qps"] - if "recall" in span_data: - metrics["recall"] = span_data["recall"] - if "mean_latency" in span_data: - metrics["mean_latency"] = span_data["mean_latency"] - if "mean_ios" in span_data: - metrics["mean_ios"] = span_data["mean_ios"] - if "mean_comps" in span_data: - metrics["mean_comps"] = span_data["mean_comps"] - if "mean_hops" in span_data: - metrics["mean_hops"] = span_data["mean_hops"] - if "mean_io_time" in span_data: - metrics["mean_io_time"] = span_data["mean_io_time"] - if "mean_cpus" in span_data: - metrics["mean_cpus"] = span_data["mean_cpus"] - break - - return metrics - - -def make_rows(baseline_list: List[Dict], target_list: List[Dict]) -> List[List[str]]: - """Generate comparison rows for the CSV output.""" - rows = [] - - for baseline, target in zip(baseline_list, target_list): - baseline_results = baseline.get("results", {}) - target_results = target.get("results", {}) - - # Get input info for context - inp = target.get("input", {}) - content = inp.get("content", {}) - search_phase = content.get("search_phase", {}) - - # Determine search_l and beam_width for search metrics - search_list = search_phase.get("search_list", [2000]) - beam_width = search_phase.get("beam_width", 4) - - # Use the first (or primary) search_l value - primary_search_l = search_list[0] if search_list else 2000 - - # Extract build metrics - baseline_build = extract_build_metrics(baseline_results) - target_build = extract_build_metrics(target_results) - - # Build metrics rows - build_metrics = [ - ("total_time", "total build time (s)"), - ("pq_construction_time", "PQ construction (s)"), - ("inmem_index_build_time", "in-memory index build (s)"), - ("disk_layout_time", "disk layout (s)"), - ] - - for key, display_name in build_metrics: - if key in target_build or key in baseline_build: - target_val = target_build.get(key, 0) - baseline_val = baseline_build.get(key, 0) - rows.append([ - "index-build statistics", - display_name, - key, - str(target_val), - str(baseline_val), - calc_deviation(baseline_val, target_val) - ]) - - # Extract search metrics for the primary search_l - baseline_search = extract_search_metrics(baseline_results, primary_search_l, beam_width) - target_search = extract_search_metrics(target_results, primary_search_l, beam_width) - - search_span_name = f"search-with-L={primary_search_l}-bw={beam_width}" - - # Search metrics rows - search_metrics = [ - ("qps", "queries per second"), - ("recall", "recall (%)"), - ("mean_latency", "mean latency (μs)"), - ("latency_95", "p999 latency (μs)"), - ("mean_ios", "mean IOs"), - ("mean_comps", "mean comparisons"), - ("mean_hops", "mean hops"), - ("mean_io_time", "mean IO time (μs)"), - ("mean_cpus", "mean CPU time (μs)"), - ] - - for key, display_name in search_metrics: - if key in target_search or key in baseline_search: - target_val = target_search.get(key, 0) - baseline_val = baseline_search.get(key, 0) - rows.append([ - search_span_name, - display_name, - key, - str(target_val), - str(baseline_val), - calc_deviation(baseline_val, target_val) - ]) - - return rows - - -def write_csv(rows: List[List[str]], out_path: str): - """Write the comparison rows to a CSV file.""" - header = [ - "Parent Span Name", - "Span Name", - "Stat Key", - "Stat Value (Target)", - "Stat Value (Baseline)", - "Deviation (%)" - ] - with open(out_path, "w", newline="", encoding="utf-8") as f: - writer = csv.writer(f) - writer.writerow(header) - writer.writerows(rows) - - -def main(): - parser = argparse.ArgumentParser( - description="Compare two disk-index benchmark JSONs and emit a diff CSV." - ) - parser.add_argument("--baseline", "-b", required=True, help="Path to baseline JSON") - parser.add_argument("--branch", "-r", required=True, help="Path to branch/target JSON") - parser.add_argument("--out", "-o", required=True, help="Where to write output CSV") - args = parser.parse_args() - - baseline_list = load_json(args.baseline) - target_list = load_json(args.branch) - - if len(baseline_list) != len(target_list): - raise ValueError( - f"baseline/branch JSON arrays differ in length: {len(baseline_list)} vs {len(target_list)}" - ) - - rows = make_rows(baseline_list, target_list) - write_csv(rows, args.out) - print(f"✓ Written diff CSV to {args.out}") - - -if __name__ == "__main__": - main() diff --git a/.github/scripts/csv_to_markdown.py b/.github/scripts/csv_to_markdown.py deleted file mode 100644 index 885a202081..0000000000 --- a/.github/scripts/csv_to_markdown.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT license. - -"""Convert a CSV file to a Markdown table and optionally append to GitHub Step Summary.""" - -import argparse -import csv -import os -import sys - - -def csv_to_markdown(csv_path: str) -> str: - """Convert a CSV file to a Markdown table string.""" - with open(csv_path) as f: - rows = list(csv.reader(f)) - if len(rows) < 2: - return "" - header = rows[0] - sep = ["---"] * len(header) - return "\n".join(" | ".join(r) for r in [header, sep] + rows[1:]) - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--csv", required=True, help="Input CSV file path") - parser.add_argument("--md", required=True, help="Output Markdown file path") - parser.add_argument("--title", default="", help="Section title for GitHub Step Summary") - args = parser.parse_args() - - md = csv_to_markdown(args.csv) - if not md: - print("No data") - return 0 - - with open(args.md, "w") as f: - f.write(md + "\n") - - # Append to GitHub Step Summary if available - summary_path = os.environ.get("GITHUB_STEP_SUMMARY") - if summary_path and args.title: - with open(summary_path, "a") as f: - f.write(f"### {args.title}\n") - f.write(md + "\n") - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/.github/workflows/benchmarks-aa.yml b/.github/workflows/benchmarks-aa.yml index c1fd45ad80..d03f123f8f 100644 --- a/.github/workflows/benchmarks-aa.yml +++ b/.github/workflows/benchmarks-aa.yml @@ -100,27 +100,13 @@ jobs: run --input-file diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json \ --output-file target/tmp/wikipedia-100K_benchmark_crate_target.json - - name: Generate diff stats (baseline vs target) - run: | - python diskann_rust/.github/scripts/compare_disk_index_json_output.py \ - --baseline baseline/target/tmp/wikipedia-100K_benchmark_crate_baseline.json \ - --branch diskann_rust/target/tmp/wikipedia-100K_benchmark_crate_target.json \ - --out diskann_rust/target/tmp/wikipedia-100K_change.csv - - - name: Convert results to Markdown - working-directory: diskann_rust - run: | - python .github/scripts/csv_to_markdown.py \ - --csv target/tmp/wikipedia-100K_change.csv \ - --md target/tmp/wikipedia-100K_change.md \ - --title 'A/A Results: Wikipedia-100K Dataset' - - name: Validate benchmark results - working-directory: diskann_rust run: | - python .github/scripts/benchmark_result_parse.py \ + python diskann_rust/.github/scripts/benchmark_validate.py \ --mode aa \ - --file target/tmp/wikipedia-100K_change.csv + --baseline baseline/target/tmp/wikipedia-100K_benchmark_crate_baseline.json \ + --target diskann_rust/target/tmp/wikipedia-100K_benchmark_crate_target.json \ + --title 'A/A Results: Wikipedia-100K Dataset' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_REPOSITORY: ${{ github.repository }} @@ -132,8 +118,6 @@ jobs: with: name: aa-results-wikipedia-100K path: | - diskann_rust/target/tmp/wikipedia-100K_change.csv - diskann_rust/target/tmp/wikipedia-100K_change.md diskann_rust/target/tmp/wikipedia-100K_benchmark_crate_target.json baseline/target/tmp/wikipedia-100K_benchmark_crate_baseline.json retention-days: 30 @@ -206,27 +190,13 @@ jobs: run --input-file diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json \ --output-file target/tmp/openai-100K_benchmark_crate_target.json - - name: Generate diff stats (baseline vs target) - run: | - python diskann_rust/.github/scripts/compare_disk_index_json_output.py \ - --baseline baseline/target/tmp/openai-100K_benchmark_crate_baseline.json \ - --branch diskann_rust/target/tmp/openai-100K_benchmark_crate_target.json \ - --out diskann_rust/target/tmp/openai-100K_change.csv - - - name: Convert results to Markdown - working-directory: diskann_rust - run: | - python .github/scripts/csv_to_markdown.py \ - --csv target/tmp/openai-100K_change.csv \ - --md target/tmp/openai-100K_change.md \ - --title 'A/A Results: OpenAI ArXiv 100K Dataset' - - name: Validate benchmark results - working-directory: diskann_rust run: | - python .github/scripts/benchmark_result_parse.py \ + python diskann_rust/.github/scripts/benchmark_validate.py \ --mode aa \ - --file target/tmp/openai-100K_change.csv + --baseline baseline/target/tmp/openai-100K_benchmark_crate_baseline.json \ + --target diskann_rust/target/tmp/openai-100K_benchmark_crate_target.json \ + --title 'A/A Results: OpenAI ArXiv 100K Dataset' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_REPOSITORY: ${{ github.repository }} @@ -238,8 +208,6 @@ jobs: with: name: aa-results-openai-100K path: | - diskann_rust/target/tmp/openai-100K_change.csv - diskann_rust/target/tmp/openai-100K_change.md diskann_rust/target/tmp/openai-100K_benchmark_crate_target.json baseline/target/tmp/openai-100K_benchmark_crate_baseline.json retention-days: 30 diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 8cdf767a94..a7cc477539 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -22,8 +22,7 @@ on: paths: - 'diskann-benchmark/perf_test_inputs/**-disk-index.json' - '.github/workflows/benchmarks.yml' - - '.github/scripts/compare_disk_index_json_output.py' - - '.github/scripts/benchmark_result_parse.py' + - '.github/scripts/benchmark_validate.py' # Cancel in-progress runs when a new run is triggered concurrency: @@ -88,8 +87,7 @@ jobs: - name: Install system dependencies run: | sudo apt-get update - sudo apt-get install -y openssl libssl-dev pkg-config python3-pip - pip install csvtomd numpy scipy + sudo apt-get install -y openssl libssl-dev pkg-config # Download pre-packaged Wikipedia-100K dataset from GitHub Release # Dataset: 100K Cohere Wikipedia embeddings (768-dim, float32, cosine distance) @@ -119,27 +117,13 @@ jobs: run --input-file diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json \ --output-file target/tmp/wikipedia-100K_benchmark_crate_target.json - - name: Generate diff stats (baseline vs target) - run: | - python diskann_rust/.github/scripts/compare_disk_index_json_output.py \ - --baseline baseline/target/tmp/wikipedia-100K_benchmark_crate_baseline.json \ - --branch diskann_rust/target/tmp/wikipedia-100K_benchmark_crate_target.json \ - --out diskann_rust/target/tmp/wikipedia-100K_change.csv - - - name: Convert results to Markdown - working-directory: diskann_rust - run: | - python .github/scripts/csv_to_markdown.py \ - --csv target/tmp/wikipedia-100K_change.csv \ - --md target/tmp/wikipedia-100K_change.md \ - --title 'Benchmark Results: Wikipedia-100K Dataset' - - name: Validate benchmark results - working-directory: diskann_rust run: | - python .github/scripts/benchmark_result_parse.py \ + python diskann_rust/.github/scripts/benchmark_validate.py \ --mode pr \ - --file target/tmp/wikipedia-100K_change.csv + --baseline baseline/target/tmp/wikipedia-100K_benchmark_crate_baseline.json \ + --target diskann_rust/target/tmp/wikipedia-100K_benchmark_crate_target.json \ + --title 'Benchmark Results: Wikipedia-100K Dataset' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_REPOSITORY: ${{ github.repository }} @@ -152,8 +136,6 @@ jobs: with: name: benchmark-results-wikipedia-100K path: | - diskann_rust/target/tmp/wikipedia-100K_change.csv - diskann_rust/target/tmp/wikipedia-100K_change.md diskann_rust/target/tmp/wikipedia-100K_benchmark_crate_target.json baseline/target/tmp/wikipedia-100K_benchmark_crate_baseline.json retention-days: 30 @@ -199,8 +181,7 @@ jobs: - name: Install system dependencies run: | sudo apt-get update - sudo apt-get install -y openssl libssl-dev pkg-config python3-pip - pip install csvtomd numpy scipy + sudo apt-get install -y openssl libssl-dev pkg-config # Download pre-packaged OpenAI ArXiv 100K dataset from GitHub Release # Dataset: 100K OpenAI embeddings of ArXiv papers (1536-dim, float32, euclidean distance) @@ -228,27 +209,13 @@ jobs: run --input-file diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json \ --output-file target/tmp/openai-100K_benchmark_crate_target.json - - name: Generate diff stats (baseline vs target) - run: | - python diskann_rust/.github/scripts/compare_disk_index_json_output.py \ - --baseline baseline/target/tmp/openai-100K_benchmark_crate_baseline.json \ - --branch diskann_rust/target/tmp/openai-100K_benchmark_crate_target.json \ - --out diskann_rust/target/tmp/openai-100K_change.csv - - - name: Convert results to Markdown - working-directory: diskann_rust - run: | - python .github/scripts/csv_to_markdown.py \ - --csv target/tmp/openai-100K_change.csv \ - --md target/tmp/openai-100K_change.md \ - --title 'Benchmark Results: OpenAI ArXiv 100K Dataset' - - name: Validate benchmark results - working-directory: diskann_rust run: | - python .github/scripts/benchmark_result_parse.py \ + python diskann_rust/.github/scripts/benchmark_validate.py \ --mode pr \ - --file target/tmp/openai-100K_change.csv + --baseline baseline/target/tmp/openai-100K_benchmark_crate_baseline.json \ + --target diskann_rust/target/tmp/openai-100K_benchmark_crate_target.json \ + --title 'Benchmark Results: OpenAI ArXiv 100K Dataset' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_REPOSITORY: ${{ github.repository }} @@ -261,8 +228,6 @@ jobs: with: name: benchmark-results-openai-100K path: | - diskann_rust/target/tmp/openai-100K_change.csv - diskann_rust/target/tmp/openai-100K_change.md diskann_rust/target/tmp/openai-100K_benchmark_crate_target.json baseline/target/tmp/openai-100K_benchmark_crate_baseline.json retention-days: 30 \ No newline at end of file From 938e0b49e5304284cdd6e961e9514c3334d9c266 Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Tue, 31 Mar 2026 09:57:01 +0800 Subject: [PATCH 25/41] switch benchmark jobs to self-hosted 1ES runner pool (diskann-github) --- .github/workflows/benchmarks-aa.yml | 4 ++-- .github/workflows/benchmarks.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmarks-aa.yml b/.github/workflows/benchmarks-aa.yml index d03f123f8f..efd2995643 100644 --- a/.github/workflows/benchmarks-aa.yml +++ b/.github/workflows/benchmarks-aa.yml @@ -35,7 +35,7 @@ jobs: # A/A benchmark: Wikipedia-100K dataset (main vs main) aa-wikipedia-100K: name: A/A - Wikipedia 100K - runs-on: ubuntu-latest + runs-on: [ self-hosted, 1ES.Pool=diskann-github, ubuntu-latest ] timeout-minutes: 120 steps: @@ -125,7 +125,7 @@ jobs: # A/A benchmark: OpenAI ArXiv 100K dataset (main vs main) aa-openai-100K: name: A/A - OAI ArXiv 100K - runs-on: ubuntu-latest + runs-on: [ self-hosted, 1ES.Pool=diskann-github, ubuntu-latest ] timeout-minutes: 120 steps: diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index a7cc477539..9f4cf2fc51 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -46,7 +46,7 @@ jobs: # Macro benchmark: Wikipedia-100K dataset macro-benchmark-wikipedia-100K: name: Macro Benchmark - Wikipedia 100K - runs-on: ubuntu-latest + runs-on: [ self-hosted, 1ES.Pool=diskann-github, ubuntu-latest ] # TODO: For production benchmarks, consider using a self-hosted runner with: # - NVMe storage for consistent I/O performance # - CPU pinning (taskset) for reduced variance @@ -143,7 +143,7 @@ jobs: # Macro benchmark: OpenAI ArXiv dataset macro-benchmark-oai-large: name: Macro Benchmark - OAI ArXiv 100K - runs-on: ubuntu-latest + runs-on: [ self-hosted, 1ES.Pool=diskann-github, ubuntu-latest ] # TODO: For production benchmarks, consider using a self-hosted runner timeout-minutes: 120 From 1a510b2cc7e9dfcff28d3faea43c0302e20cb222 Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Tue, 31 Mar 2026 10:05:07 +0800 Subject: [PATCH 26/41] replace gh CLI with curl for dataset downloads (gh not available on 1ES runners) --- .github/workflows/benchmarks-aa.yml | 8 ++------ .github/workflows/benchmarks.yml | 8 ++------ 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/benchmarks-aa.yml b/.github/workflows/benchmarks-aa.yml index efd2995643..ed77832333 100644 --- a/.github/workflows/benchmarks-aa.yml +++ b/.github/workflows/benchmarks-aa.yml @@ -78,11 +78,9 @@ jobs: # Download pre-packaged Wikipedia-100K dataset from GitHub Release # Source: https://github.com/harsha-simhadri/big-ann-benchmarks - name: Download wikipedia-100K dataset - env: - GH_TOKEN: ${{ github.token }} run: | mkdir -p diskann_rust/target/tmp baseline/target/tmp - gh release download v1 --repo YuanyuanTian-hh/diskann-benchmark-data --pattern 'wikipedia-100K.tar.gz' --dir . + curl -L -o wikipedia-100K.tar.gz https://github.com/YuanyuanTian-hh/diskann-benchmark-data/releases/download/v1/wikipedia-100K.tar.gz tar xzf wikipedia-100K.tar.gz -C diskann_rust/target/tmp/ cp -r diskann_rust/target/tmp/wikipedia_cohere baseline/target/tmp/ @@ -168,11 +166,9 @@ jobs: # Download pre-packaged OpenAI ArXiv 100K dataset from GitHub Release # Source: https://github.com/harsha-simhadri/big-ann-benchmarks - name: Download openai-100K dataset - env: - GH_TOKEN: ${{ github.token }} run: | mkdir -p diskann_rust/target/tmp baseline/target/tmp - gh release download v1 --repo YuanyuanTian-hh/diskann-benchmark-data --pattern 'openai-100K.tar.gz' --dir . + curl -L -o openai-100K.tar.gz https://github.com/YuanyuanTian-hh/diskann-benchmark-data/releases/download/v1/openai-100K.tar.gz tar xzf openai-100K.tar.gz -C diskann_rust/target/tmp/ cp -r diskann_rust/target/tmp/OpenAIArXiv baseline/target/tmp/ diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 9f4cf2fc51..d5015df882 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -93,11 +93,9 @@ jobs: # Dataset: 100K Cohere Wikipedia embeddings (768-dim, float32, cosine distance) # Source: https://github.com/harsha-simhadri/big-ann-benchmarks - name: Download wikipedia-100K dataset - env: - GH_TOKEN: ${{ github.token }} run: | mkdir -p diskann_rust/target/tmp baseline/target/tmp - gh release download v1 --repo YuanyuanTian-hh/diskann-benchmark-data --pattern 'wikipedia-100K.tar.gz' --dir . + curl -L -o wikipedia-100K.tar.gz https://github.com/YuanyuanTian-hh/diskann-benchmark-data/releases/download/v1/wikipedia-100K.tar.gz tar xzf wikipedia-100K.tar.gz -C diskann_rust/target/tmp/ cp -r diskann_rust/target/tmp/wikipedia_cohere baseline/target/tmp/ @@ -187,11 +185,9 @@ jobs: # Dataset: 100K OpenAI embeddings of ArXiv papers (1536-dim, float32, euclidean distance) # Source: https://github.com/harsha-simhadri/big-ann-benchmarks - name: Download openai-100K dataset - env: - GH_TOKEN: ${{ github.token }} run: | mkdir -p diskann_rust/target/tmp baseline/target/tmp - gh release download v1 --repo YuanyuanTian-hh/diskann-benchmark-data --pattern 'openai-100K.tar.gz' --dir . + curl -L -o openai-100K.tar.gz https://github.com/YuanyuanTian-hh/diskann-benchmark-data/releases/download/v1/openai-100K.tar.gz tar xzf openai-100K.tar.gz -C diskann_rust/target/tmp/ cp -r diskann_rust/target/tmp/OpenAIArXiv baseline/target/tmp/ From 3775fa91c473b50c877157c67538dc5d52525ea5 Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Tue, 31 Mar 2026 10:21:47 +0800 Subject: [PATCH 27/41] fix latency_95: read p95_latency instead of p999_latency from benchmark JSON --- .github/scripts/benchmark_validate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/benchmark_validate.py b/.github/scripts/benchmark_validate.py index cb69f60547..ebcb953328 100644 --- a/.github/scripts/benchmark_validate.py +++ b/.github/scripts/benchmark_validate.py @@ -86,7 +86,7 @@ def extract_search_metrics(results: dict, search_l: int, beam_width: int) -> dic metrics["mean_hops"] = sr.get("mean_hops", 0) metrics["mean_io_time"] = sr.get("mean_io_time", 0) metrics["mean_cpus"] = sr.get("mean_cpu_time", 0) - metrics["latency_95"] = sr.get("p999_latency", 0) + metrics["latency_95"] = sr.get("p95_latency", 0) break # Override with span metrics if available From 128886a8c401103c84cc5c22a6a98051b418f717 Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Wed, 1 Apr 2026 11:29:20 +0800 Subject: [PATCH 28/41] revert to ubuntu-latest runners, switch dataset source to BAB v0.4.0 --- .github/workflows/benchmarks-aa.yml | 8 ++++---- .github/workflows/benchmarks.yml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/benchmarks-aa.yml b/.github/workflows/benchmarks-aa.yml index ed77832333..9161400af5 100644 --- a/.github/workflows/benchmarks-aa.yml +++ b/.github/workflows/benchmarks-aa.yml @@ -35,7 +35,7 @@ jobs: # A/A benchmark: Wikipedia-100K dataset (main vs main) aa-wikipedia-100K: name: A/A - Wikipedia 100K - runs-on: [ self-hosted, 1ES.Pool=diskann-github, ubuntu-latest ] + runs-on: ubuntu-latest timeout-minutes: 120 steps: @@ -80,7 +80,7 @@ jobs: - name: Download wikipedia-100K dataset run: | mkdir -p diskann_rust/target/tmp baseline/target/tmp - curl -L -o wikipedia-100K.tar.gz https://github.com/YuanyuanTian-hh/diskann-benchmark-data/releases/download/v1/wikipedia-100K.tar.gz + curl -L -o wikipedia-100K.tar.gz https://github.com/harsha-simhadri/big-ann-benchmarks/releases/download/v0.4.0/wikipedia-100K.tar.gz tar xzf wikipedia-100K.tar.gz -C diskann_rust/target/tmp/ cp -r diskann_rust/target/tmp/wikipedia_cohere baseline/target/tmp/ @@ -123,7 +123,7 @@ jobs: # A/A benchmark: OpenAI ArXiv 100K dataset (main vs main) aa-openai-100K: name: A/A - OAI ArXiv 100K - runs-on: [ self-hosted, 1ES.Pool=diskann-github, ubuntu-latest ] + runs-on: ubuntu-latest timeout-minutes: 120 steps: @@ -168,7 +168,7 @@ jobs: - name: Download openai-100K dataset run: | mkdir -p diskann_rust/target/tmp baseline/target/tmp - curl -L -o openai-100K.tar.gz https://github.com/YuanyuanTian-hh/diskann-benchmark-data/releases/download/v1/openai-100K.tar.gz + curl -L -o openai-100K.tar.gz https://github.com/harsha-simhadri/big-ann-benchmarks/releases/download/v0.4.0/openai-100K.tar.gz tar xzf openai-100K.tar.gz -C diskann_rust/target/tmp/ cp -r diskann_rust/target/tmp/OpenAIArXiv baseline/target/tmp/ diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index d5015df882..aa832b0949 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -46,7 +46,7 @@ jobs: # Macro benchmark: Wikipedia-100K dataset macro-benchmark-wikipedia-100K: name: Macro Benchmark - Wikipedia 100K - runs-on: [ self-hosted, 1ES.Pool=diskann-github, ubuntu-latest ] + runs-on: ubuntu-latest # TODO: For production benchmarks, consider using a self-hosted runner with: # - NVMe storage for consistent I/O performance # - CPU pinning (taskset) for reduced variance @@ -95,7 +95,7 @@ jobs: - name: Download wikipedia-100K dataset run: | mkdir -p diskann_rust/target/tmp baseline/target/tmp - curl -L -o wikipedia-100K.tar.gz https://github.com/YuanyuanTian-hh/diskann-benchmark-data/releases/download/v1/wikipedia-100K.tar.gz + curl -L -o wikipedia-100K.tar.gz https://github.com/harsha-simhadri/big-ann-benchmarks/releases/download/v0.4.0/wikipedia-100K.tar.gz tar xzf wikipedia-100K.tar.gz -C diskann_rust/target/tmp/ cp -r diskann_rust/target/tmp/wikipedia_cohere baseline/target/tmp/ @@ -141,7 +141,7 @@ jobs: # Macro benchmark: OpenAI ArXiv dataset macro-benchmark-oai-large: name: Macro Benchmark - OAI ArXiv 100K - runs-on: [ self-hosted, 1ES.Pool=diskann-github, ubuntu-latest ] + runs-on: ubuntu-latest # TODO: For production benchmarks, consider using a self-hosted runner timeout-minutes: 120 @@ -187,7 +187,7 @@ jobs: - name: Download openai-100K dataset run: | mkdir -p diskann_rust/target/tmp baseline/target/tmp - curl -L -o openai-100K.tar.gz https://github.com/YuanyuanTian-hh/diskann-benchmark-data/releases/download/v1/openai-100K.tar.gz + curl -L -o openai-100K.tar.gz https://github.com/harsha-simhadri/big-ann-benchmarks/releases/download/v0.4.0/openai-100K.tar.gz tar xzf openai-100K.tar.gz -C diskann_rust/target/tmp/ cp -r diskann_rust/target/tmp/OpenAIArXiv baseline/target/tmp/ From 59a03433c99ccfe6bea1cc502f00beb794d6cc4d Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Thu, 2 Apr 2026 11:17:30 +0800 Subject: [PATCH 29/41] address PR review: reduce search_list to 200, remove hardcoded Rust version, fix missing-field handling, clean up orphaned thresholds, switch data source to BAB v0.4.0 --- .github/scripts/benchmark_validate.py | 93 +++++++------------ .github/workflows/benchmarks-aa.yml | 9 +- .github/workflows/benchmarks.yml | 10 +- .../openai-100K-disk-index.json | 2 +- .../wikipedia-100K-disk-index.json | 2 +- 5 files changed, 46 insertions(+), 70 deletions(-) diff --git a/.github/scripts/benchmark_validate.py b/.github/scripts/benchmark_validate.py index ebcb953328..9081de76c2 100644 --- a/.github/scripts/benchmark_validate.py +++ b/.github/scripts/benchmark_validate.py @@ -59,11 +59,11 @@ def extract_build_metrics(results: dict) -> dict[str, float]: name = span.get("span_name", "") data = span.get("metrics", {}) if name == "DiskIndexBuild-PqConstruction": - metrics["pq_construction_time"] = data.get("duration_seconds", 0) + metrics["pq_construction_time"] = data.get("duration_seconds") elif name == "DiskIndexBuild-InmemIndexBuild": - metrics["inmem_index_build_time"] = data.get("duration_seconds", 0) + metrics["inmem_index_build_time"] = data.get("duration_seconds") elif name == "DiskIndexBuild-DiskLayout": - metrics["disk_layout_time"] = data.get("duration_seconds", 0) + metrics["disk_layout_time"] = data.get("duration_seconds") return metrics @@ -78,15 +78,15 @@ def extract_search_metrics(results: dict, search_l: int, beam_width: int) -> dic # From search_results_per_l for sr in search.get("search_results_per_l", []): if sr.get("search_l") == search_l: - metrics["qps"] = sr.get("qps", 0) - metrics["recall"] = sr.get("recall", 0) - metrics["mean_latency"] = sr.get("mean_latency", 0) - metrics["mean_ios"] = sr.get("mean_ios", 0) - metrics["mean_comps"] = sr.get("mean_comparisons", 0) - metrics["mean_hops"] = sr.get("mean_hops", 0) - metrics["mean_io_time"] = sr.get("mean_io_time", 0) - metrics["mean_cpus"] = sr.get("mean_cpu_time", 0) - metrics["latency_95"] = sr.get("p95_latency", 0) + metrics["qps"] = sr.get("qps") + metrics["recall"] = sr.get("recall") + metrics["mean_latency"] = sr.get("mean_latency") + metrics["mean_ios"] = sr.get("mean_ios") + metrics["mean_comps"] = sr.get("mean_comparisons") + metrics["mean_hops"] = sr.get("mean_hops") + metrics["mean_io_time"] = sr.get("mean_io_time") + metrics["mean_cpus"] = sr.get("mean_cpu_time") + metrics["latency_95"] = sr.get("p95_latency") break # Override with span metrics if available @@ -117,25 +117,26 @@ def compute_diff(baseline_json: list[dict], target_json: list[dict]) -> list[dic inp = target.get("input", {}) search_phase = inp.get("content", {}).get("search_phase", {}) - search_list = search_phase.get("search_list", [2000]) + search_list = search_phase.get("search_list", [200]) beam_width = search_phase.get("beam_width", 4) - primary_l = search_list[0] if search_list else 2000 + primary_l = search_list[0] if search_list else 200 # Build metrics b_build = extract_build_metrics(b_results) t_build = extract_build_metrics(t_results) for key in ("total_time", "pq_construction_time", "inmem_index_build_time", "disk_layout_time"): - if key in t_build or key in b_build: - bv = b_build.get(key, 0) - tv = t_build.get(key, 0) - rows.append({ - "category": "index-build statistics", - "metric": key, - "baseline": bv, - "target": tv, - "deviation": ((tv - bv) / bv * 100) if bv else 0, - }) + bv = b_build.get(key) + tv = t_build.get(key) + if bv is None or tv is None: + continue # skip metrics missing from either side + rows.append({ + "category": "index-build statistics", + "metric": key, + "baseline": bv, + "target": tv, + "deviation": ((tv - bv) / bv * 100) if bv else 0, + }) # Search metrics b_search = extract_search_metrics(b_results, primary_l, beam_width) @@ -144,16 +145,17 @@ def compute_diff(baseline_json: list[dict], target_json: list[dict]) -> list[dic for key in ("qps", "recall", "mean_latency", "latency_95", "mean_ios", "mean_comps", "mean_hops", "mean_io_time", "mean_cpus"): - if key in t_search or key in b_search: - bv = b_search.get(key, 0) - tv = t_search.get(key, 0) - rows.append({ - "category": span_cat, - "metric": key, - "baseline": bv, - "target": tv, - "deviation": ((tv - bv) / bv * 100) if bv else 0, - }) + bv = b_search.get(key) + tv = t_search.get(key) + if bv is None or tv is None: + continue # skip metrics missing from either side + rows.append({ + "category": span_cat, + "metric": key, + "baseline": bv, + "target": tv, + "deviation": ((tv - bv) / bv * 100) if bv else 0, + }) return rows @@ -189,29 +191,6 @@ def compute_diff(baseline_json: list[dict], target_json: list[dict]) -> list[dic "total_comparisons": [1, "LT", ""], "search_hops": [1, "LT", ""], }, - "search-with-L=2000-bw=4": { - # Calibrated from 5 GitHub runner runs (10 observations) - "latency_95": [10, "LT", ""], - "mean_latency": [10, "LT", ""], - "mean_io_time": [10, "LT", ""], - "mean_cpus": [15, "LT", ""], # wider — CPU time is noisy on shared runners - "qps": [10, "GT", 6.5], - "mean_ios": [1, "LT", 2410], - "mean_comps": [1, "LT", 33200], - "mean_hops": [1, "LT", ""], - "recall": [1, "GT", 98.0], - }, - "search-with-L=100-bw=4": { - "latency_95": [10, "LT", ""], - "mean_latency": [10, "LT", ""], - "mean_io_time": [10, "LT", ""], - "mean_cpus": [15, "LT", ""], - "qps": [10, "GT", ""], - "mean_ios": [10, "LT", ""], - "mean_comps": [10, "LT", ""], - "mean_hops": [10, "LT", ""], - "recall": [1, "GT", ""], - }, "search-with-L=200-bw=4": { "latency_95": [10, "LT", ""], "mean_latency": [10, "LT", ""], diff --git a/.github/workflows/benchmarks-aa.yml b/.github/workflows/benchmarks-aa.yml index 9161400af5..c8256bd59d 100644 --- a/.github/workflows/benchmarks-aa.yml +++ b/.github/workflows/benchmarks-aa.yml @@ -21,7 +21,6 @@ concurrency: env: RUST_BACKTRACE: 1 - rust_stable: "1.92" defaults: run: @@ -53,10 +52,10 @@ jobs: path: baseline lfs: true - - name: Install Rust ${{ env.rust_stable }} + - name: Install Rust uses: dtolnay/rust-toolchain@master with: - toolchain: ${{ env.rust_stable }} + toolchain: stable - name: Cache Rust dependencies (target) uses: Swatinem/rust-cache@v2 @@ -141,10 +140,10 @@ jobs: path: baseline lfs: true - - name: Install Rust ${{ env.rust_stable }} + - name: Install Rust uses: dtolnay/rust-toolchain@master with: - toolchain: ${{ env.rust_stable }} + toolchain: stable - name: Cache Rust dependencies (target) uses: Swatinem/rust-cache@v2 diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index aa832b0949..1942673554 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -31,8 +31,6 @@ concurrency: env: RUST_BACKTRACE: 1 - # Use the Rust version specified in rust-toolchain.toml - rust_stable: "1.92" defaults: run: @@ -67,10 +65,10 @@ jobs: path: baseline lfs: true - - name: Install Rust ${{ env.rust_stable }} + - name: Install Rust uses: dtolnay/rust-toolchain@master with: - toolchain: ${{ env.rust_stable }} + toolchain: stable - name: Cache Rust dependencies (current) uses: Swatinem/rust-cache@v2 @@ -159,10 +157,10 @@ jobs: path: baseline lfs: true - - name: Install Rust ${{ env.rust_stable }} + - name: Install Rust uses: dtolnay/rust-toolchain@master with: - toolchain: ${{ env.rust_stable }} + toolchain: stable - name: Cache Rust dependencies (current) uses: Swatinem/rust-cache@v2 diff --git a/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json b/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json index 3a2a1d9e2f..d021640fc1 100644 --- a/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json +++ b/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json @@ -24,7 +24,7 @@ "queries": "OpenAIArXiv/openai_query.bin", "groundtruth": "OpenAIArXiv/openai-100K", "search_list": [ - 2000 + 200 ], "beam_width": 4, "recall_at": 100, diff --git a/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json b/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json index 6a52b1e323..e5f06aa1b7 100644 --- a/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json +++ b/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json @@ -24,7 +24,7 @@ "queries": "wikipedia_cohere/wikipedia_query.bin", "groundtruth": "wikipedia_cohere/wikipedia-100K", "search_list": [ - 2000 + 200 ], "beam_width": 4, "recall_at": 100, From 92e36f68d5b74e7b3f750138ee8291212b23eed6 Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Thu, 2 Apr 2026 11:30:38 +0800 Subject: [PATCH 30/41] widen latency_95 threshold to 15% for shared-runner noise --- .github/scripts/benchmark_validate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/benchmark_validate.py b/.github/scripts/benchmark_validate.py index 9081de76c2..9dd5de14b5 100644 --- a/.github/scripts/benchmark_validate.py +++ b/.github/scripts/benchmark_validate.py @@ -192,7 +192,7 @@ def compute_diff(baseline_json: list[dict], target_json: list[dict]) -> list[dic "search_hops": [1, "LT", ""], }, "search-with-L=200-bw=4": { - "latency_95": [10, "LT", ""], + "latency_95": [15, "LT", ""], # wider — p95 latency is noisy on shared runners "mean_latency": [10, "LT", ""], "mean_io_time": [10, "LT", ""], "mean_cpus": [15, "LT", ""], From 2d2a06d3a27de14117c6042449b4c12f5fac4df4 Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Thu, 2 Apr 2026 12:12:57 +0800 Subject: [PATCH 31/41] replace push trigger with pull_request trigger targeting main --- .github/workflows/benchmarks.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 1942673554..d75e0efe72 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -16,11 +16,21 @@ on: required: true default: 'main' type: string - push: + pull_request: branches: - - 'user/tianyuanyuan/add-benchmark-pipeline' + - main paths: - - 'diskann-benchmark/perf_test_inputs/**-disk-index.json' + - 'diskann/**' + - 'diskann-disk/**' + - 'diskann-linalg/**' + - 'diskann-providers/**' + - 'diskann-quantization/**' + - 'diskann-vector/**' + - 'diskann-wide/**' + - 'diskann-utils/**' + - 'diskann-platform/**' + - 'diskann-label-filter/**' + - 'diskann-benchmark/**' - '.github/workflows/benchmarks.yml' - '.github/scripts/benchmark_validate.py' From 6a70f36da0856c0499871fa33269a6be79b46529 Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Tue, 7 Apr 2026 16:01:32 +0800 Subject: [PATCH 32/41] implement Regression trait for disk-index benchmarks - Add Deserialize to DiskIndexStats, DiskSearchStats, DiskSearchResult, DiskBuildStats - Implement Regression trait for DiskIndex with typed before/after comparison - Add DiskIndexTolerance type with configurable thresholds for 7 metrics - Create disk-index-tolerances.json (10% build/QPS, 1% recall/IOs/comps, 15% latency) - Switch registration from register() to register_regression() - Replace Python benchmark_validate.py with Rust-native check run in both workflows - Delete benchmark_validate.py (no longer needed) --- .github/scripts/benchmark_validate.py | 404 ------------------ .github/workflows/benchmarks-aa.yml | 32 +- .github/workflows/benchmarks.yml | 36 +- .../disk-index-tolerances.json | 22 + .../src/backend/disk_index/benchmarks.rs | 329 +++++++++++++- .../src/backend/disk_index/build.rs | 8 +- .../src/backend/disk_index/search.rs | 6 +- 7 files changed, 380 insertions(+), 457 deletions(-) delete mode 100644 .github/scripts/benchmark_validate.py create mode 100644 diskann-benchmark/perf_test_inputs/disk-index-tolerances.json diff --git a/.github/scripts/benchmark_validate.py b/.github/scripts/benchmark_validate.py deleted file mode 100644 index 9dd5de14b5..0000000000 --- a/.github/scripts/benchmark_validate.py +++ /dev/null @@ -1,404 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT license. - -""" -Benchmark Validator for GitHub Actions - -Compares two benchmark JSON outputs (baseline vs target), checks thresholds, -writes a Markdown summary, and optionally posts a PR comment on failure. - -This single script replaces the previous three-step pipeline: - compare_disk_index_json_output.py → csv_to_markdown.py → benchmark_result_parse.py - -Usage: - # PR mode (directional thresholds, posts PR comment on failure) - python benchmark_validate.py --mode pr --baseline baseline.json --target target.json - - # A/A mode (symmetric thresholds) - python benchmark_validate.py --mode aa --baseline baseline.json --target target.json - -Environment Variables (for PR comments): - GITHUB_TOKEN: GitHub token for API access - GITHUB_REPOSITORY: Owner/repo (e.g., "microsoft/DiskANN") - GITHUB_PR_NUMBER: Pull request number - GITHUB_RUN_ID: Workflow run ID for linking to logs - GITHUB_STEP_SUMMARY: Path to step summary file -""" - -import json -import os -import sys -import argparse -from typing import Any -from urllib.request import urlopen, Request -from urllib.error import URLError - - -# ============================================================================= -# JSON Extraction -# ============================================================================= - -def load_json(path: str) -> list[dict[str, Any]]: - with open(path, "r", encoding="utf-8") as f: - return json.load(f) - - -def extract_build_metrics(results: dict) -> dict[str, float]: - build = results.get("build", {}) - if not build: - return {} - - metrics: dict[str, float] = {} - - build_time = build.get("build_time") - if build_time: - metrics["total_time"] = build_time / 1e6 # μs → s - - for span in build.get("span_metrics", {}).get("spans", []): - name = span.get("span_name", "") - data = span.get("metrics", {}) - if name == "DiskIndexBuild-PqConstruction": - metrics["pq_construction_time"] = data.get("duration_seconds") - elif name == "DiskIndexBuild-InmemIndexBuild": - metrics["inmem_index_build_time"] = data.get("duration_seconds") - elif name == "DiskIndexBuild-DiskLayout": - metrics["disk_layout_time"] = data.get("duration_seconds") - - return metrics - - -def extract_search_metrics(results: dict, search_l: int, beam_width: int) -> dict[str, float]: - search = results.get("search", {}) - if not search: - return {} - - metrics: dict[str, float] = {} - - # From search_results_per_l - for sr in search.get("search_results_per_l", []): - if sr.get("search_l") == search_l: - metrics["qps"] = sr.get("qps") - metrics["recall"] = sr.get("recall") - metrics["mean_latency"] = sr.get("mean_latency") - metrics["mean_ios"] = sr.get("mean_ios") - metrics["mean_comps"] = sr.get("mean_comparisons") - metrics["mean_hops"] = sr.get("mean_hops") - metrics["mean_io_time"] = sr.get("mean_io_time") - metrics["mean_cpus"] = sr.get("mean_cpu_time") - metrics["latency_95"] = sr.get("p95_latency") - break - - # Override with span metrics if available - span_name = f"search-with-L={search_l}-bw={beam_width}" - for span in search.get("span_metrics", {}).get("spans", []): - if span.get("span_name") == span_name: - data = span.get("metrics", {}) - for key in ("qps", "recall", "mean_latency", "mean_ios", "mean_comps", - "mean_hops", "mean_io_time", "mean_cpus"): - if key in data: - metrics[key] = data[key] - break - - return metrics - - -def compute_diff(baseline_json: list[dict], target_json: list[dict]) -> list[dict]: - """ - Compare baseline and target JSONs. - Returns a flat list of metric diffs: - [{category, metric, baseline, target, deviation}, ...] - """ - rows = [] - - for baseline, target in zip(baseline_json, target_json): - b_results = baseline.get("results", {}) - t_results = target.get("results", {}) - - inp = target.get("input", {}) - search_phase = inp.get("content", {}).get("search_phase", {}) - search_list = search_phase.get("search_list", [200]) - beam_width = search_phase.get("beam_width", 4) - primary_l = search_list[0] if search_list else 200 - - # Build metrics - b_build = extract_build_metrics(b_results) - t_build = extract_build_metrics(t_results) - - for key in ("total_time", "pq_construction_time", "inmem_index_build_time", "disk_layout_time"): - bv = b_build.get(key) - tv = t_build.get(key) - if bv is None or tv is None: - continue # skip metrics missing from either side - rows.append({ - "category": "index-build statistics", - "metric": key, - "baseline": bv, - "target": tv, - "deviation": ((tv - bv) / bv * 100) if bv else 0, - }) - - # Search metrics - b_search = extract_search_metrics(b_results, primary_l, beam_width) - t_search = extract_search_metrics(t_results, primary_l, beam_width) - span_cat = f"search-with-L={primary_l}-bw={beam_width}" - - for key in ("qps", "recall", "mean_latency", "latency_95", "mean_ios", - "mean_comps", "mean_hops", "mean_io_time", "mean_cpus"): - bv = b_search.get(key) - tv = t_search.get(key) - if bv is None or tv is None: - continue # skip metrics missing from either side - rows.append({ - "category": span_cat, - "metric": key, - "baseline": bv, - "target": tv, - "deviation": ((tv - bv) / bv * 100) if bv else 0, - }) - - return rows - - -# ============================================================================= -# Thresholds -# ============================================================================= - -# Format: [max_deviation_%, direction, contract_value] -# direction: 'GT' = higher is better, 'LT' = lower is better -# contract_value: absolute limit (empty string = none) -THRESHOLDS: dict[str, dict[str, list]] = { - "DiskIndexBuild-PqConstruction": { - "duration_seconds": [10, "LT", ""], - "peak_memory_usage": [10, "LT", ""], - }, - "DiskIndexBuild-InmemIndexBuild": { - "duration_seconds": [10, "LT", ""], - "peak_memory_usage": [10, "LT", ""], - }, - "search_disk_index-search_completed": { - "duration_seconds": [10, "LT", ""], - "peak_memory_usage": [10, "LT", 1.42], - }, - "disk_index_perf_test": { - "total_duration_seconds": [10, "LT", ""], - }, - "index-build statistics": { - # Calibrated from 5 GitHub runner runs (10 observations): - # Wikipedia: 35.9–37.2s, OpenAI: 23.0–76.4s (SQ_1_2.0 variance) - # Contract: worst × 1.5 to absorb shared-runner variance - "total_time": [10, "LT", 115], - "total_comparisons": [1, "LT", ""], - "search_hops": [1, "LT", ""], - }, - "search-with-L=200-bw=4": { - "latency_95": [15, "LT", ""], # wider — p95 latency is noisy on shared runners - "mean_latency": [10, "LT", ""], - "mean_io_time": [10, "LT", ""], - "mean_cpus": [15, "LT", ""], - "qps": [10, "GT", ""], - "mean_ios": [10, "LT", ""], - "mean_comps": [10, "LT", ""], - "mean_hops": [10, "LT", ""], - "recall": [1, "GT", ""], - }, -} - - -def allowed_range(threshold: float, direction: str, mode: str) -> tuple[float, float]: - """Acceptable change range (in %).""" - if mode == "aa": - return (-threshold, threshold) - if direction == "GT": - return (-threshold, float("inf")) - return (float("-inf"), threshold) - - -def fmt_range(lo: float, hi: float) -> str: - lo_s = "-inf" if lo == float("-inf") else f"{lo}%" - hi_s = "inf" if hi == float("inf") else f"{hi}%" - return f"({lo_s} – {hi_s})" - - -def check_contract(value: float, contract: Any, direction: str) -> tuple[bool, str]: - """Check if value violates a hard contract. Returns (broken, formatted_contract).""" - if contract == "": - return False, "N/A" - contract = float(contract) - if direction == "GT" and value < contract: - return True, f"> {contract}" - if direction == "LT" and value > contract: - return True, f"< {contract}" - return False, str(contract) - - -# ============================================================================= -# Validation -# ============================================================================= - -def validate(diffs: list[dict], mode: str, run_id: str | None) -> tuple[bool, str]: - """ - Check all diffs against thresholds. - Returns (has_failures, markdown_report). - """ - failed_rows: list[str] = [] - - for d in diffs: - cat, metric = d["category"], d["metric"] - if cat not in THRESHOLDS or metric not in THRESHOLDS[cat]: - continue - - pct, direction, contract = THRESHOLDS[cat][metric] - rng = allowed_range(pct, direction, mode) - dev = d["deviation"] - - threshold_failed = dev < rng[0] or dev > rng[1] - contract_broken, contract_fmt = check_contract(d["target"], contract, direction) - - if threshold_failed: - print(f"THRESHOLD FAILED: {cat}/{metric} change={dev:.2f}% allowed={fmt_range(*rng)}") - if contract_broken: - print(f"CONTRACT BROKEN: {cat}/{metric} value={d['target']} required={contract_fmt}") - - if threshold_failed or contract_broken: - outcome = [] - if threshold_failed: - outcome.append("Regression detected") - if contract_broken: - outcome.append("Contract broken") - failed_rows.append( - f"| {cat}/{metric} | {d['baseline']:.4g} | {d['target']:.4g} | " - f"{contract_fmt} | {dev:.2f}% | {fmt_range(*rng)} | {', '.join(outcome)} |" - ) - - if not failed_rows: - return False, "" - - logs_link = "" - if run_id: - repo = os.getenv("GITHUB_REPOSITORY", "microsoft/DiskANN") - logs_link = f"https://github.com/{repo}/actions/runs/{run_id}" - - report = "### ❌ Benchmark Check Failed\n\n" - if logs_link: - report += f"Please investigate the [workflow logs]({logs_link}) to determine if the failure is due to your changes.\n\n" - report += "| Metric | Baseline | Current | Contract | Change | Allowed | Outcome |\n" - report += "|--------|----------|---------|----------|--------|---------|--------|\n" - report += "\n".join(failed_rows) - - return True, report - - -# ============================================================================= -# Markdown output -# ============================================================================= - -def diffs_to_markdown(diffs: list[dict], title: str) -> str: - """Render diffs as a Markdown table.""" - lines = [ - f"### {title}", - "", - "| Category | Metric | Baseline | Current | Change |", - "|----------|--------|----------|---------|--------|", - ] - for d in diffs: - lines.append( - f"| {d['category']} | {d['metric']} | {d['baseline']:.4g} | " - f"{d['target']:.4g} | {d['deviation']:+.2f}% |" - ) - return "\n".join(lines) - - -# ============================================================================= -# GitHub helpers (stdlib only — no requests dependency) -# ============================================================================= - -def post_pr_comment(body: str) -> bool: - token = os.getenv("GITHUB_TOKEN") - repo = os.getenv("GITHUB_REPOSITORY") - pr = os.getenv("GITHUB_PR_NUMBER") - if not all([token, repo, pr]): - print("WARNING: Missing GitHub env vars for PR comment " - f"(TOKEN={'set' if token else 'missing'}, REPO={repo or 'missing'}, PR={pr or 'missing'})") - return False - - url = f"https://api.github.com/repos/{repo}/issues/{pr}/comments" - data = json.dumps({"body": body}).encode() - req = Request(url, data=data, method="POST", headers={ - "Accept": "application/vnd.github+json", - "Authorization": f"Bearer {token}", - "X-GitHub-Api-Version": "2022-11-28", - "Content-Type": "application/json", - }) - try: - with urlopen(req, timeout=30) as resp: - if resp.status < 300: - print(f"Posted comment to PR #{pr}") - return True - except URLError as e: - print(f"ERROR posting PR comment: {e}") - return False - - -def write_step_summary(content: str) -> None: - path = os.getenv("GITHUB_STEP_SUMMARY") - if path: - with open(path, "a", encoding="utf-8") as f: - f.write(content + "\n") - - -# ============================================================================= -# Main -# ============================================================================= - -def main() -> int: - parser = argparse.ArgumentParser( - description="Compare two benchmark JSONs, validate thresholds, output Markdown." - ) - parser.add_argument("--mode", choices=["aa", "pr"], default="aa", - help="aa = symmetric thresholds, pr = directional") - parser.add_argument("--baseline", required=True, help="Baseline JSON path") - parser.add_argument("--target", required=True, help="Target JSON path") - parser.add_argument("--title", default="Benchmark Results", - help="Title for the Markdown summary table") - parser.add_argument("--no-comment", action="store_true", - help="Skip posting PR comment on failure") - args = parser.parse_args() - - print(f"Mode: {args.mode}") - print(f"Baseline: {args.baseline}") - print(f"Target: {args.target}") - - baseline = load_json(args.baseline) - target = load_json(args.target) - - if len(baseline) != len(target): - print(f"ERROR: JSON arrays differ in length: {len(baseline)} vs {len(target)}") - return 1 - - # Compare - diffs = compute_diff(baseline, target) - print(f"\nCompared {len(diffs)} metrics") - - # Write Markdown summary - md = diffs_to_markdown(diffs, args.title) - write_step_summary(md) - - # Validate thresholds - run_id = os.getenv("GITHUB_RUN_ID") - has_failures, report = validate(diffs, args.mode, run_id) - - if has_failures: - print("\n" + report) - write_step_summary(report) - if args.mode == "pr" and not args.no_comment: - post_pr_comment(report) - return 1 - - print("\n✅ All metrics within thresholds") - write_step_summary("### ✅ Benchmark Check Passed\n\nAll metrics within acceptable thresholds.") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/.github/workflows/benchmarks-aa.yml b/.github/workflows/benchmarks-aa.yml index c8256bd59d..e02399e15e 100644 --- a/.github/workflows/benchmarks-aa.yml +++ b/.github/workflows/benchmarks-aa.yml @@ -98,16 +98,14 @@ jobs: --output-file target/tmp/wikipedia-100K_benchmark_crate_target.json - name: Validate benchmark results + working-directory: diskann_rust run: | - python diskann_rust/.github/scripts/benchmark_validate.py \ - --mode aa \ - --baseline baseline/target/tmp/wikipedia-100K_benchmark_crate_baseline.json \ - --target diskann_rust/target/tmp/wikipedia-100K_benchmark_crate_target.json \ - --title 'A/A Results: Wikipedia-100K Dataset' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_REPOSITORY: ${{ github.repository }} - GITHUB_RUN_ID: ${{ github.run_id }} + cargo run -p diskann-benchmark --features disk-index --release -- \ + check run \ + --tolerances diskann-benchmark/perf_test_inputs/disk-index-tolerances.json \ + --input-file diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json \ + --before ../baseline/target/tmp/wikipedia-100K_benchmark_crate_baseline.json \ + --after target/tmp/wikipedia-100K_benchmark_crate_target.json - name: Upload benchmark results uses: actions/upload-artifact@v4 @@ -186,16 +184,14 @@ jobs: --output-file target/tmp/openai-100K_benchmark_crate_target.json - name: Validate benchmark results + working-directory: diskann_rust run: | - python diskann_rust/.github/scripts/benchmark_validate.py \ - --mode aa \ - --baseline baseline/target/tmp/openai-100K_benchmark_crate_baseline.json \ - --target diskann_rust/target/tmp/openai-100K_benchmark_crate_target.json \ - --title 'A/A Results: OpenAI ArXiv 100K Dataset' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_REPOSITORY: ${{ github.repository }} - GITHUB_RUN_ID: ${{ github.run_id }} + cargo run -p diskann-benchmark --features disk-index --release -- \ + check run \ + --tolerances diskann-benchmark/perf_test_inputs/disk-index-tolerances.json \ + --input-file diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json \ + --before ../baseline/target/tmp/openai-100K_benchmark_crate_baseline.json \ + --after target/tmp/openai-100K_benchmark_crate_target.json - name: Upload benchmark results uses: actions/upload-artifact@v4 diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index d75e0efe72..89fb0cedc9 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -31,8 +31,8 @@ on: - 'diskann-platform/**' - 'diskann-label-filter/**' - 'diskann-benchmark/**' + - 'diskann-benchmark-runner/**' - '.github/workflows/benchmarks.yml' - - '.github/scripts/benchmark_validate.py' # Cancel in-progress runs when a new run is triggered concurrency: @@ -124,17 +124,14 @@ jobs: --output-file target/tmp/wikipedia-100K_benchmark_crate_target.json - name: Validate benchmark results + working-directory: diskann_rust run: | - python diskann_rust/.github/scripts/benchmark_validate.py \ - --mode pr \ - --baseline baseline/target/tmp/wikipedia-100K_benchmark_crate_baseline.json \ - --target diskann_rust/target/tmp/wikipedia-100K_benchmark_crate_target.json \ - --title 'Benchmark Results: Wikipedia-100K Dataset' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_REPOSITORY: ${{ github.repository }} - GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }} - GITHUB_RUN_ID: ${{ github.run_id }} + cargo run -p diskann-benchmark --features disk-index --release -- \ + check run \ + --tolerances diskann-benchmark/perf_test_inputs/disk-index-tolerances.json \ + --input-file diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json \ + --before ../baseline/target/tmp/wikipedia-100K_benchmark_crate_baseline.json \ + --after target/tmp/wikipedia-100K_benchmark_crate_target.json - name: Upload benchmark results uses: actions/upload-artifact@v4 @@ -214,17 +211,14 @@ jobs: --output-file target/tmp/openai-100K_benchmark_crate_target.json - name: Validate benchmark results + working-directory: diskann_rust run: | - python diskann_rust/.github/scripts/benchmark_validate.py \ - --mode pr \ - --baseline baseline/target/tmp/openai-100K_benchmark_crate_baseline.json \ - --target diskann_rust/target/tmp/openai-100K_benchmark_crate_target.json \ - --title 'Benchmark Results: OpenAI ArXiv 100K Dataset' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_REPOSITORY: ${{ github.repository }} - GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }} - GITHUB_RUN_ID: ${{ github.run_id }} + cargo run -p diskann-benchmark --features disk-index --release -- \ + check run \ + --tolerances diskann-benchmark/perf_test_inputs/disk-index-tolerances.json \ + --input-file diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json \ + --before ../baseline/target/tmp/openai-100K_benchmark_crate_baseline.json \ + --after target/tmp/openai-100K_benchmark_crate_target.json - name: Upload benchmark results uses: actions/upload-artifact@v4 diff --git a/diskann-benchmark/perf_test_inputs/disk-index-tolerances.json b/diskann-benchmark/perf_test_inputs/disk-index-tolerances.json new file mode 100644 index 0000000000..e6b8c47814 --- /dev/null +++ b/diskann-benchmark/perf_test_inputs/disk-index-tolerances.json @@ -0,0 +1,22 @@ +{ + "checks": [ + { + "input": { + "type": "disk-index", + "content": {} + }, + "tolerance": { + "type": "disk-index-tolerance", + "content": { + "build_time_regression": 0.10, + "qps_regression": 0.10, + "recall_regression": 0.01, + "mean_ios_regression": 0.01, + "mean_comps_regression": 0.01, + "mean_latency_regression": 0.15, + "p95_latency_regression": 0.15 + } + } + } + ] +} diff --git a/diskann-benchmark/src/backend/disk_index/benchmarks.rs b/diskann-benchmark/src/backend/disk_index/benchmarks.rs index 71c89f846b..8dddee375a 100644 --- a/diskann-benchmark/src/backend/disk_index/benchmarks.rs +++ b/diskann-benchmark/src/backend/disk_index/benchmarks.rs @@ -3,15 +3,19 @@ * Licensed under the MIT license. */ -use serde::Serialize; -use std::io::Write; +use serde::{Deserialize, Serialize}; +use std::{fmt, io::Write}; use diskann::utils::VectorRepr; use diskann_benchmark_runner::{ + benchmark::{PassFail, Regression}, dispatcher::{DispatchRule, FailureScore, MatchScore}, output::Output, - utils::datatype::{DataType, Type}, - Benchmark, Checkpoint, + utils::{ + datatype::{DataType, Type}, + num::{relative_change, NonNegativeFinite}, + }, + Any, Benchmark, CheckDeserialization, Checker, Checkpoint, Input, }; use diskann_providers::storage::FileStorageProvider; use half::f16; @@ -30,7 +34,7 @@ struct DiskIndex<'a, T> { _vector_type: std::marker::PhantomData, } -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, Deserialize)] pub(super) struct DiskIndexStats { pub(super) build: Option, pub(super) search: DiskSearchStats, @@ -124,8 +128,315 @@ where //////////////////////////// pub(super) fn register_benchmarks(benchmarks: &mut diskann_benchmark_runner::registry::Benchmarks) { - benchmarks.register::>("disk-index-f32"); - benchmarks.register::>("disk-index-f16"); - benchmarks.register::>("disk-index-u8"); - benchmarks.register::>("disk-index-i8"); + benchmarks.register_regression::>("disk-index-f32"); + benchmarks.register_regression::>("disk-index-f16"); + benchmarks.register_regression::>("disk-index-u8"); + benchmarks.register_regression::>("disk-index-i8"); +} + +///////////////////////// +// Regression Checking // +///////////////////////// + +/// Tolerance thresholds for disk-index regression checks. +/// +/// Each field specifies the maximum allowed relative increase (for "lower is better" metrics) +/// or decrease (for "higher is better" metrics) before a regression is flagged. +/// +/// For example, `recall_regression: 0.01` means recall must not drop by more than 1%. +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub(super) struct DiskIndexTolerance { + /// Max allowed relative increase in build time (e.g., 0.10 = 10%). + build_time_regression: NonNegativeFinite, + /// Max allowed relative decrease in QPS (e.g., 0.10 = 10%). + qps_regression: NonNegativeFinite, + /// Max allowed relative decrease in recall (e.g., 0.01 = 1%). + recall_regression: NonNegativeFinite, + /// Max allowed relative increase in mean I/Os. + mean_ios_regression: NonNegativeFinite, + /// Max allowed relative increase in mean comparisons. + mean_comps_regression: NonNegativeFinite, + /// Max allowed relative increase in mean latency. + mean_latency_regression: NonNegativeFinite, + /// Max allowed relative increase in p95 latency. + p95_latency_regression: NonNegativeFinite, +} + +impl DiskIndexTolerance { + const fn tag() -> &'static str { + "disk-index-tolerance" + } +} + +impl CheckDeserialization for DiskIndexTolerance { + fn check_deserialization(&mut self, _checker: &mut Checker) -> Result<(), anyhow::Error> { + Ok(()) + } +} + +impl Input for DiskIndexTolerance { + fn tag() -> &'static str { + Self::tag() + } + + fn try_deserialize( + serialized: &serde_json::Value, + checker: &mut Checker, + ) -> anyhow::Result { + checker.any(Self::deserialize(serialized)?) + } + + fn example() -> anyhow::Result { + const DEFAULT: NonNegativeFinite = match NonNegativeFinite::new(0.10) { + Ok(v) => v, + Err(_) => panic!("use a non-negative finite value"), + }; + const RECALL: NonNegativeFinite = match NonNegativeFinite::new(0.01) { + Ok(v) => v, + Err(_) => panic!("use a non-negative finite value"), + }; + + Ok(serde_json::to_value(DiskIndexTolerance { + build_time_regression: DEFAULT, + qps_regression: DEFAULT, + recall_regression: RECALL, + mean_ios_regression: DEFAULT, + mean_comps_regression: DEFAULT, + mean_latency_regression: DEFAULT, + p95_latency_regression: DEFAULT, + })?) + } +} + +/// A single metric comparison in the regression check. +#[derive(Debug, Serialize)] +struct MetricComparison { + metric: &'static str, + before: f64, + after: f64, + change_pct: String, + tolerance_pct: f64, + passed: bool, + remark: String, +} + +/// Aggregated result of a disk-index regression check. +#[derive(Debug, Serialize)] +struct DiskIndexCheckResult { + search_l: u32, + comparisons: Vec, +} + +impl fmt::Display for DiskIndexCheckResult { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!( + f, + " Search L={}: {:>15} {:>15} {:>12} {:>12} {}", + self.search_l, "Before", "After", "Change", "Tolerance", "Remark" + )?; + writeln!( + f, + " {}", + "=".repeat(90) + )?; + for c in &self.comparisons { + writeln!( + f, + " {:>20}, {:>14.3}, {:>14.3}, {:>11}, {:>11.1}%, {}", + c.metric, c.before, c.after, c.change_pct, c.tolerance_pct * 100.0, c.remark + )?; + } + Ok(()) + } +} + +/// Check a "lower is better" metric (latency, IOs, comparisons). +/// Regression = value increased beyond tolerance. +fn check_lower_is_better( + name: &'static str, + before: f64, + after: f64, + tolerance: NonNegativeFinite, + passed: &mut bool, +) -> MetricComparison { + let (change_pct, remark, metric_passed) = match relative_change(before, after) { + Ok(change) => { + let ok = change <= tolerance.get(); + if !ok { + *passed = false; + } + ( + format!("{:.3}%", change * 100.0), + if ok { String::new() } else { "REGRESSION".to_string() }, + ok, + ) + } + Err(e) => { + *passed = false; + ("invalid".to_string(), e.to_string(), false) + } + }; + MetricComparison { + metric: name, + before, + after, + change_pct, + tolerance_pct: tolerance.get(), + passed: metric_passed, + remark, + } +} + +/// Check a "higher is better" metric (QPS, recall). +/// Regression = value decreased beyond tolerance. +fn check_higher_is_better( + name: &'static str, + before: f64, + after: f64, + tolerance: NonNegativeFinite, + passed: &mut bool, +) -> MetricComparison { + // Flip before/after so that a decrease becomes a positive relative_change + let (change_pct, remark, metric_passed) = match relative_change(before, after) { + Ok(change) => { + // For higher-is-better, a negative change is a regression + let ok = -change <= tolerance.get(); + if !ok { + *passed = false; + } + ( + format!("{:.3}%", change * 100.0), + if ok { String::new() } else { "REGRESSION".to_string() }, + ok, + ) + } + Err(e) => { + *passed = false; + ("invalid".to_string(), e.to_string(), false) + } + }; + MetricComparison { + metric: name, + before, + after, + change_pct, + tolerance_pct: tolerance.get(), + passed: metric_passed, + remark, + } +} + +impl Regression for DiskIndex<'static, T> +where + T: VectorRepr + 'static, + Type: DispatchRule, +{ + type Tolerances = DiskIndexTolerance; + type Pass = DiskIndexCheckResult; + type Fail = DiskIndexCheckResult; + + fn check( + tolerances: &DiskIndexTolerance, + _input: &DiskIndexOperation, + before: &DiskIndexStats, + after: &DiskIndexStats, + ) -> anyhow::Result> { + let mut passed = true; + let mut comparisons = Vec::new(); + + // Check build time if both sides have it + if let (Some(b_build), Some(a_build)) = (&before.build, &after.build) { + let b_time = b_build.build_time_seconds(); + let a_time = a_build.build_time_seconds(); + comparisons.push(check_lower_is_better( + "build_time", + b_time, + a_time, + tolerances.build_time_regression, + &mut passed, + )); + } + + // Check search metrics for each matching search_l + anyhow::ensure!( + before.search.search_results_per_l.len() == after.search.search_results_per_l.len(), + "before has {} search_l entries but after has {}", + before.search.search_results_per_l.len(), + after.search.search_results_per_l.len(), + ); + + for (b_sr, a_sr) in before + .search + .search_results_per_l + .iter() + .zip(after.search.search_results_per_l.iter()) + { + anyhow::ensure!( + b_sr.search_l == a_sr.search_l, + "search_l mismatch: before={} after={}", + b_sr.search_l, + a_sr.search_l, + ); + + comparisons.push(check_higher_is_better( + "qps", + b_sr.qps as f64, + a_sr.qps as f64, + tolerances.qps_regression, + &mut passed, + )); + comparisons.push(check_higher_is_better( + "recall", + b_sr.recall as f64, + a_sr.recall as f64, + tolerances.recall_regression, + &mut passed, + )); + comparisons.push(check_lower_is_better( + "mean_latency", + b_sr.mean_latency, + a_sr.mean_latency, + tolerances.mean_latency_regression, + &mut passed, + )); + comparisons.push(check_lower_is_better( + "p95_latency", + b_sr.p95_latency.as_f64(), + a_sr.p95_latency.as_f64(), + tolerances.p95_latency_regression, + &mut passed, + )); + comparisons.push(check_lower_is_better( + "mean_ios", + b_sr.mean_ios, + a_sr.mean_ios, + tolerances.mean_ios_regression, + &mut passed, + )); + comparisons.push(check_lower_is_better( + "mean_comparisons", + b_sr.mean_comparisons, + a_sr.mean_comparisons, + tolerances.mean_comps_regression, + &mut passed, + )); + } + + let search_l = before + .search + .search_results_per_l + .first() + .map(|s| s.search_l) + .unwrap_or(0); + let result = DiskIndexCheckResult { + search_l, + comparisons, + }; + + if passed { + Ok(PassFail::Pass(result)) + } else { + Ok(PassFail::Fail(result)) + } + } } diff --git a/diskann-benchmark/src/backend/disk_index/build.rs b/diskann-benchmark/src/backend/disk_index/build.rs index b6ebf3b837..8229f859e4 100644 --- a/diskann-benchmark/src/backend/disk_index/build.rs +++ b/diskann-benchmark/src/backend/disk_index/build.rs @@ -3,7 +3,7 @@ * Licensed under the MIT license. */ -use serde::Serialize; +use serde::{Deserialize, Serialize}; use std::fmt; use diskann::{ @@ -31,7 +31,7 @@ use crate::{ inputs::disk::DiskIndexBuild, }; -#[derive(Serialize, Debug)] +#[derive(Serialize, Deserialize, Debug)] pub(super) struct DiskBuildStats { build_time: MicroSeconds, span_metrics: serde_json::Value, @@ -44,6 +44,10 @@ impl DiskBuildStats { span_metrics, } } + + pub(super) fn build_time_seconds(&self) -> f64 { + self.build_time.as_seconds() + } } impl fmt::Display for DiskBuildStats { diff --git a/diskann-benchmark/src/backend/disk_index/search.rs b/diskann-benchmark/src/backend/disk_index/search.rs index 65e5804a76..3f9637d5b7 100644 --- a/diskann-benchmark/src/backend/disk_index/search.rs +++ b/diskann-benchmark/src/backend/disk_index/search.rs @@ -28,7 +28,7 @@ use diskann_providers::{ }; use diskann_tools::utils::{search_index_utils, KRecallAtN}; use diskann_utils::views::Matrix; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use crate::{ backend::disk_index::{graph_data_type::GraphData, json_spancollector::JsonSpanCollector}, @@ -36,7 +36,7 @@ use crate::{ utils::{datafiles, SimilarityMeasure}, }; -#[derive(Serialize, Debug)] +#[derive(Serialize, Deserialize, Debug)] pub(super) struct DiskSearchStats { pub(super) num_threads: usize, pub(super) beam_width: usize, @@ -49,7 +49,7 @@ pub(super) struct DiskSearchStats { span_metrics: serde_json::Value, } -#[derive(Serialize, Debug)] +#[derive(Serialize, Deserialize, Debug)] pub(super) struct DiskSearchResult { pub(super) search_l: u32, pub(super) qps: f32, From 2b9e57bfd6b1adf92a0297a95ffc2e4ebedd1897 Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Tue, 7 Apr 2026 16:50:43 +0800 Subject: [PATCH 33/41] test: set strict tolerances (0.1%) to verify pipeline failure detection --- .../disk-index-tolerances.json | 14 ++++----- .../src/backend/disk_index/benchmarks.rs | 29 ++++++++++++------- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/diskann-benchmark/perf_test_inputs/disk-index-tolerances.json b/diskann-benchmark/perf_test_inputs/disk-index-tolerances.json index e6b8c47814..5bbdfe8c1b 100644 --- a/diskann-benchmark/perf_test_inputs/disk-index-tolerances.json +++ b/diskann-benchmark/perf_test_inputs/disk-index-tolerances.json @@ -8,13 +8,13 @@ "tolerance": { "type": "disk-index-tolerance", "content": { - "build_time_regression": 0.10, - "qps_regression": 0.10, - "recall_regression": 0.01, - "mean_ios_regression": 0.01, - "mean_comps_regression": 0.01, - "mean_latency_regression": 0.15, - "p95_latency_regression": 0.15 + "build_time_regression": 0.001, + "qps_regression": 0.001, + "recall_regression": 0.001, + "mean_ios_regression": 0.001, + "mean_comps_regression": 0.001, + "mean_latency_regression": 0.001, + "p95_latency_regression": 0.001 } } } diff --git a/diskann-benchmark/src/backend/disk_index/benchmarks.rs b/diskann-benchmark/src/backend/disk_index/benchmarks.rs index 8dddee375a..35fc4023c7 100644 --- a/diskann-benchmark/src/backend/disk_index/benchmarks.rs +++ b/diskann-benchmark/src/backend/disk_index/benchmarks.rs @@ -231,19 +231,20 @@ impl fmt::Display for DiskIndexCheckResult { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { writeln!( f, - " Search L={}: {:>15} {:>15} {:>12} {:>12} {}", - self.search_l, "Before", "After", "Change", "Tolerance", "Remark" - )?; - writeln!( - f, - " {}", - "=".repeat(90) + " Search L={}: {:>15} {:>15} {:>12} {:>12} Remark", + self.search_l, "Before", "After", "Change", "Tolerance" )?; + writeln!(f, " {}", "=".repeat(90))?; for c in &self.comparisons { writeln!( f, " {:>20}, {:>14.3}, {:>14.3}, {:>11}, {:>11.1}%, {}", - c.metric, c.before, c.after, c.change_pct, c.tolerance_pct * 100.0, c.remark + c.metric, + c.before, + c.after, + c.change_pct, + c.tolerance_pct * 100.0, + c.remark )?; } Ok(()) @@ -267,7 +268,11 @@ fn check_lower_is_better( } ( format!("{:.3}%", change * 100.0), - if ok { String::new() } else { "REGRESSION".to_string() }, + if ok { + String::new() + } else { + "REGRESSION".to_string() + }, ok, ) } @@ -306,7 +311,11 @@ fn check_higher_is_better( } ( format!("{:.3}%", change * 100.0), - if ok { String::new() } else { "REGRESSION".to_string() }, + if ok { + String::new() + } else { + "REGRESSION".to_string() + }, ok, ) } From e55a8faafdd328aafe1c721676403face1a5d40a Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Tue, 7 Apr 2026 16:52:37 +0800 Subject: [PATCH 34/41] revert tolerances to production values --- .../perf_test_inputs/disk-index-tolerances.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/diskann-benchmark/perf_test_inputs/disk-index-tolerances.json b/diskann-benchmark/perf_test_inputs/disk-index-tolerances.json index 5bbdfe8c1b..e6b8c47814 100644 --- a/diskann-benchmark/perf_test_inputs/disk-index-tolerances.json +++ b/diskann-benchmark/perf_test_inputs/disk-index-tolerances.json @@ -8,13 +8,13 @@ "tolerance": { "type": "disk-index-tolerance", "content": { - "build_time_regression": 0.001, - "qps_regression": 0.001, - "recall_regression": 0.001, - "mean_ios_regression": 0.001, - "mean_comps_regression": 0.001, - "mean_latency_regression": 0.001, - "p95_latency_regression": 0.001 + "build_time_regression": 0.10, + "qps_regression": 0.10, + "recall_regression": 0.01, + "mean_ios_regression": 0.01, + "mean_comps_regression": 0.01, + "mean_latency_regression": 0.15, + "p95_latency_regression": 0.15 } } } From a1d1e06de15c3186287ba6eb31bf373820d507f8 Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Wed, 8 Apr 2026 10:00:19 +0800 Subject: [PATCH 35/41] test for failure --- .../perf_test_inputs/disk-index-tolerances.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/diskann-benchmark/perf_test_inputs/disk-index-tolerances.json b/diskann-benchmark/perf_test_inputs/disk-index-tolerances.json index e6b8c47814..5bbdfe8c1b 100644 --- a/diskann-benchmark/perf_test_inputs/disk-index-tolerances.json +++ b/diskann-benchmark/perf_test_inputs/disk-index-tolerances.json @@ -8,13 +8,13 @@ "tolerance": { "type": "disk-index-tolerance", "content": { - "build_time_regression": 0.10, - "qps_regression": 0.10, - "recall_regression": 0.01, - "mean_ios_regression": 0.01, - "mean_comps_regression": 0.01, - "mean_latency_regression": 0.15, - "p95_latency_regression": 0.15 + "build_time_regression": 0.001, + "qps_regression": 0.001, + "recall_regression": 0.001, + "mean_ios_regression": 0.001, + "mean_comps_regression": 0.001, + "mean_latency_regression": 0.001, + "p95_latency_regression": 0.001 } } } From f2c14e82af6cebbb0d569932becfb9297c3b7b35 Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Wed, 8 Apr 2026 10:30:27 +0800 Subject: [PATCH 36/41] revert test code --- .../perf_test_inputs/disk-index-tolerances.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/diskann-benchmark/perf_test_inputs/disk-index-tolerances.json b/diskann-benchmark/perf_test_inputs/disk-index-tolerances.json index 5bbdfe8c1b..e6b8c47814 100644 --- a/diskann-benchmark/perf_test_inputs/disk-index-tolerances.json +++ b/diskann-benchmark/perf_test_inputs/disk-index-tolerances.json @@ -8,13 +8,13 @@ "tolerance": { "type": "disk-index-tolerance", "content": { - "build_time_regression": 0.001, - "qps_regression": 0.001, - "recall_regression": 0.001, - "mean_ios_regression": 0.001, - "mean_comps_regression": 0.001, - "mean_latency_regression": 0.001, - "p95_latency_regression": 0.001 + "build_time_regression": 0.10, + "qps_regression": 0.10, + "recall_regression": 0.01, + "mean_ios_regression": 0.01, + "mean_comps_regression": 0.01, + "mean_latency_regression": 0.15, + "p95_latency_regression": 0.15 } } } From 0aefcf7d640e814330d83fca68ce60e2386b1177 Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Thu, 9 Apr 2026 11:54:53 +0800 Subject: [PATCH 37/41] address review: refactor workflows to matrix strategy, update build params --- .github/workflows/benchmarks-aa.yml | 136 ++++------------- .github/workflows/benchmarks.yml | 138 ++++-------------- .../openai-100K-disk-index.json | 2 +- .../wikipedia-100K-disk-index.json | 6 +- 4 files changed, 65 insertions(+), 217 deletions(-) diff --git a/.github/workflows/benchmarks-aa.yml b/.github/workflows/benchmarks-aa.yml index e02399e15e..dc1fd7abba 100644 --- a/.github/workflows/benchmarks-aa.yml +++ b/.github/workflows/benchmarks-aa.yml @@ -31,11 +31,23 @@ permissions: issues: write # Required for creating failure notification issues jobs: - # A/A benchmark: Wikipedia-100K dataset (main vs main) - aa-wikipedia-100K: - name: A/A - Wikipedia 100K + # A/A benchmark: run main vs main to detect environment noise. + aa-benchmark: + name: A/A - ${{ matrix.dataset }} runs-on: ubuntu-latest timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + include: + - dataset: wikipedia-100K + config: wikipedia-100K-disk-index.json + archive: wikipedia-100K.tar.gz + data_dir: wikipedia_cohere + - dataset: openai-100K + config: openai-100K-disk-index.json + archive: openai-100K.tar.gz + data_dir: OpenAIArXiv steps: - name: Checkout main (target) @@ -74,28 +86,28 @@ jobs: sudo apt-get update sudo apt-get install -y openssl libssl-dev pkg-config - # Download pre-packaged Wikipedia-100K dataset from GitHub Release + # Download pre-packaged dataset from GitHub Release # Source: https://github.com/harsha-simhadri/big-ann-benchmarks - - name: Download wikipedia-100K dataset + - name: Download ${{ matrix.dataset }} dataset run: | mkdir -p diskann_rust/target/tmp baseline/target/tmp - curl -L -o wikipedia-100K.tar.gz https://github.com/harsha-simhadri/big-ann-benchmarks/releases/download/v0.4.0/wikipedia-100K.tar.gz - tar xzf wikipedia-100K.tar.gz -C diskann_rust/target/tmp/ - cp -r diskann_rust/target/tmp/wikipedia_cohere baseline/target/tmp/ + curl -L -o ${{ matrix.archive }} https://github.com/harsha-simhadri/big-ann-benchmarks/releases/download/v0.4.0/${{ matrix.archive }} + tar xzf ${{ matrix.archive }} -C diskann_rust/target/tmp/ + cp -r diskann_rust/target/tmp/${{ matrix.data_dir }} baseline/target/tmp/ - name: Run baseline benchmark working-directory: baseline run: | cargo run -p diskann-benchmark --features disk-index --release -- \ - run --input-file ../diskann_rust/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json \ - --output-file target/tmp/wikipedia-100K_benchmark_crate_baseline.json + run --input-file ../diskann_rust/diskann-benchmark/perf_test_inputs/${{ matrix.config }} \ + --output-file target/tmp/${{ matrix.dataset }}_benchmark_crate_baseline.json - name: Run target benchmark working-directory: diskann_rust run: | cargo run -p diskann-benchmark --features disk-index --release -- \ - run --input-file diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json \ - --output-file target/tmp/wikipedia-100K_benchmark_crate_target.json + run --input-file diskann-benchmark/perf_test_inputs/${{ matrix.config }} \ + --output-file target/tmp/${{ matrix.dataset }}_benchmark_crate_target.json - name: Validate benchmark results working-directory: diskann_rust @@ -103,110 +115,24 @@ jobs: cargo run -p diskann-benchmark --features disk-index --release -- \ check run \ --tolerances diskann-benchmark/perf_test_inputs/disk-index-tolerances.json \ - --input-file diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json \ - --before ../baseline/target/tmp/wikipedia-100K_benchmark_crate_baseline.json \ - --after target/tmp/wikipedia-100K_benchmark_crate_target.json + --input-file diskann-benchmark/perf_test_inputs/${{ matrix.config }} \ + --before ../baseline/target/tmp/${{ matrix.dataset }}_benchmark_crate_baseline.json \ + --after target/tmp/${{ matrix.dataset }}_benchmark_crate_target.json - name: Upload benchmark results uses: actions/upload-artifact@v4 if: always() with: - name: aa-results-wikipedia-100K + name: aa-results-${{ matrix.dataset }} path: | - diskann_rust/target/tmp/wikipedia-100K_benchmark_crate_target.json - baseline/target/tmp/wikipedia-100K_benchmark_crate_baseline.json - retention-days: 30 - - # A/A benchmark: OpenAI ArXiv 100K dataset (main vs main) - aa-openai-100K: - name: A/A - OAI ArXiv 100K - runs-on: ubuntu-latest - timeout-minutes: 120 - - steps: - - name: Checkout main (target) - uses: actions/checkout@v4 - with: - ref: main - path: diskann_rust - lfs: true - - - name: Checkout main (baseline) - uses: actions/checkout@v4 - with: - ref: main - path: baseline - lfs: true - - - name: Install Rust - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - - - name: Cache Rust dependencies (target) - uses: Swatinem/rust-cache@v2 - with: - workspaces: diskann_rust -> target - key: aa-target - - - name: Cache Rust dependencies (baseline) - uses: Swatinem/rust-cache@v2 - with: - workspaces: baseline -> target - key: aa-baseline - - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y openssl libssl-dev pkg-config - - # Download pre-packaged OpenAI ArXiv 100K dataset from GitHub Release - # Source: https://github.com/harsha-simhadri/big-ann-benchmarks - - name: Download openai-100K dataset - run: | - mkdir -p diskann_rust/target/tmp baseline/target/tmp - curl -L -o openai-100K.tar.gz https://github.com/harsha-simhadri/big-ann-benchmarks/releases/download/v0.4.0/openai-100K.tar.gz - tar xzf openai-100K.tar.gz -C diskann_rust/target/tmp/ - cp -r diskann_rust/target/tmp/OpenAIArXiv baseline/target/tmp/ - - - name: Run baseline benchmark - working-directory: baseline - run: | - cargo run -p diskann-benchmark --features disk-index --release -- \ - run --input-file ../diskann_rust/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json \ - --output-file target/tmp/openai-100K_benchmark_crate_baseline.json - - - name: Run target benchmark - working-directory: diskann_rust - run: | - cargo run -p diskann-benchmark --features disk-index --release -- \ - run --input-file diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json \ - --output-file target/tmp/openai-100K_benchmark_crate_target.json - - - name: Validate benchmark results - working-directory: diskann_rust - run: | - cargo run -p diskann-benchmark --features disk-index --release -- \ - check run \ - --tolerances diskann-benchmark/perf_test_inputs/disk-index-tolerances.json \ - --input-file diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json \ - --before ../baseline/target/tmp/openai-100K_benchmark_crate_baseline.json \ - --after target/tmp/openai-100K_benchmark_crate_target.json - - - name: Upload benchmark results - uses: actions/upload-artifact@v4 - if: always() - with: - name: aa-results-openai-100K - path: | - diskann_rust/target/tmp/openai-100K_benchmark_crate_target.json - baseline/target/tmp/openai-100K_benchmark_crate_baseline.json + diskann_rust/target/tmp/${{ matrix.dataset }}_benchmark_crate_target.json + baseline/target/tmp/${{ matrix.dataset }}_benchmark_crate_baseline.json retention-days: 30 # Notify diskann-admin on A/A failure notify-on-failure: name: Notify on A/A Failure - needs: [aa-wikipedia-100K, aa-openai-100K] + needs: [aa-benchmark] runs-on: ubuntu-latest if: failure() steps: diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 89fb0cedc9..a9c8ebb024 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -51,15 +51,27 @@ permissions: pull-requests: write # Required for posting PR comments jobs: - # Macro benchmark: Wikipedia-100K dataset - macro-benchmark-wikipedia-100K: - name: Macro Benchmark - Wikipedia 100K + # Macro benchmark: compare current branch against baseline + macro-benchmark: + name: Macro Benchmark - ${{ matrix.dataset }} runs-on: ubuntu-latest # TODO: For production benchmarks, consider using a self-hosted runner with: # - NVMe storage for consistent I/O performance # - CPU pinning (taskset) for reduced variance # - Dedicated hardware to avoid noisy neighbor effects timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + include: + - dataset: wikipedia-100K + config: wikipedia-100K-disk-index.json + archive: wikipedia-100K.tar.gz + data_dir: wikipedia_cohere + - dataset: openai-100K + config: openai-100K-disk-index.json + archive: openai-100K.tar.gz + data_dir: OpenAIArXiv steps: - name: Checkout current branch @@ -97,31 +109,28 @@ jobs: sudo apt-get update sudo apt-get install -y openssl libssl-dev pkg-config - # Download pre-packaged Wikipedia-100K dataset from GitHub Release - # Dataset: 100K Cohere Wikipedia embeddings (768-dim, float32, cosine distance) + # Download pre-packaged dataset from GitHub Release # Source: https://github.com/harsha-simhadri/big-ann-benchmarks - - name: Download wikipedia-100K dataset + - name: Download ${{ matrix.dataset }} dataset run: | mkdir -p diskann_rust/target/tmp baseline/target/tmp - curl -L -o wikipedia-100K.tar.gz https://github.com/harsha-simhadri/big-ann-benchmarks/releases/download/v0.4.0/wikipedia-100K.tar.gz - tar xzf wikipedia-100K.tar.gz -C diskann_rust/target/tmp/ - cp -r diskann_rust/target/tmp/wikipedia_cohere baseline/target/tmp/ + curl -L -o ${{ matrix.archive }} https://github.com/harsha-simhadri/big-ann-benchmarks/releases/download/v0.4.0/${{ matrix.archive }} + tar xzf ${{ matrix.archive }} -C diskann_rust/target/tmp/ + cp -r diskann_rust/target/tmp/${{ matrix.data_dir }} baseline/target/tmp/ - name: Run baseline benchmark working-directory: baseline run: | - # Note: For accurate benchmarks, consider using CPU pinning on self-hosted runners: - # sudo taskset -c 0,2,4,6 ionice -c 1 -n 0 cargo run ... cargo run -p diskann-benchmark --features disk-index --release -- \ - run --input-file ../diskann_rust/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json \ - --output-file target/tmp/wikipedia-100K_benchmark_crate_baseline.json + run --input-file ../diskann_rust/diskann-benchmark/perf_test_inputs/${{ matrix.config }} \ + --output-file target/tmp/${{ matrix.dataset }}_benchmark_crate_baseline.json - name: Run current branch benchmark working-directory: diskann_rust run: | cargo run -p diskann-benchmark --features disk-index --release -- \ - run --input-file diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json \ - --output-file target/tmp/wikipedia-100K_benchmark_crate_target.json + run --input-file diskann-benchmark/perf_test_inputs/${{ matrix.config }} \ + --output-file target/tmp/${{ matrix.dataset }}_benchmark_crate_target.json - name: Validate benchmark results working-directory: diskann_rust @@ -129,103 +138,16 @@ jobs: cargo run -p diskann-benchmark --features disk-index --release -- \ check run \ --tolerances diskann-benchmark/perf_test_inputs/disk-index-tolerances.json \ - --input-file diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json \ - --before ../baseline/target/tmp/wikipedia-100K_benchmark_crate_baseline.json \ - --after target/tmp/wikipedia-100K_benchmark_crate_target.json + --input-file diskann-benchmark/perf_test_inputs/${{ matrix.config }} \ + --before ../baseline/target/tmp/${{ matrix.dataset }}_benchmark_crate_baseline.json \ + --after target/tmp/${{ matrix.dataset }}_benchmark_crate_target.json - name: Upload benchmark results uses: actions/upload-artifact@v4 if: always() # Upload even if validation fails with: - name: benchmark-results-wikipedia-100K + name: benchmark-results-${{ matrix.dataset }} path: | - diskann_rust/target/tmp/wikipedia-100K_benchmark_crate_target.json - baseline/target/tmp/wikipedia-100K_benchmark_crate_baseline.json - retention-days: 30 - - # Macro benchmark: OpenAI ArXiv dataset - macro-benchmark-oai-large: - name: Macro Benchmark - OAI ArXiv 100K - runs-on: ubuntu-latest - # TODO: For production benchmarks, consider using a self-hosted runner - timeout-minutes: 120 - - steps: - - name: Checkout current branch - uses: actions/checkout@v4 - with: - path: diskann_rust - lfs: true - - - name: Checkout baseline (${{ inputs.baseline_ref || 'main' }}) - uses: actions/checkout@v4 - with: - ref: ${{ inputs.baseline_ref || 'main' }} - path: baseline - lfs: true - - - name: Install Rust - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - - - name: Cache Rust dependencies (current) - uses: Swatinem/rust-cache@v2 - with: - workspaces: diskann_rust -> target - key: benchmark-current - - - name: Cache Rust dependencies (baseline) - uses: Swatinem/rust-cache@v2 - with: - workspaces: baseline -> target - key: benchmark-baseline - - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y openssl libssl-dev pkg-config - - # Download pre-packaged OpenAI ArXiv 100K dataset from GitHub Release - # Dataset: 100K OpenAI embeddings of ArXiv papers (1536-dim, float32, euclidean distance) - # Source: https://github.com/harsha-simhadri/big-ann-benchmarks - - name: Download openai-100K dataset - run: | - mkdir -p diskann_rust/target/tmp baseline/target/tmp - curl -L -o openai-100K.tar.gz https://github.com/harsha-simhadri/big-ann-benchmarks/releases/download/v0.4.0/openai-100K.tar.gz - tar xzf openai-100K.tar.gz -C diskann_rust/target/tmp/ - cp -r diskann_rust/target/tmp/OpenAIArXiv baseline/target/tmp/ - - - name: Run baseline benchmark - working-directory: baseline - run: | - cargo run -p diskann-benchmark --features disk-index --release -- \ - run --input-file ../diskann_rust/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json \ - --output-file target/tmp/openai-100K_benchmark_crate_baseline.json - - - name: Run current branch benchmark - working-directory: diskann_rust - run: | - cargo run -p diskann-benchmark --features disk-index --release -- \ - run --input-file diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json \ - --output-file target/tmp/openai-100K_benchmark_crate_target.json - - - name: Validate benchmark results - working-directory: diskann_rust - run: | - cargo run -p diskann-benchmark --features disk-index --release -- \ - check run \ - --tolerances diskann-benchmark/perf_test_inputs/disk-index-tolerances.json \ - --input-file diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json \ - --before ../baseline/target/tmp/openai-100K_benchmark_crate_baseline.json \ - --after target/tmp/openai-100K_benchmark_crate_target.json - - - name: Upload benchmark results - uses: actions/upload-artifact@v4 - if: always() # Upload even if validation fails - with: - name: benchmark-results-openai-100K - path: | - diskann_rust/target/tmp/openai-100K_benchmark_crate_target.json - baseline/target/tmp/openai-100K_benchmark_crate_baseline.json + diskann_rust/target/tmp/${{ matrix.dataset }}_benchmark_crate_target.json + baseline/target/tmp/${{ matrix.dataset }}_benchmark_crate_baseline.json retention-days: 30 \ No newline at end of file diff --git a/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json b/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json index d021640fc1..e90f509555 100644 --- a/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json +++ b/diskann-benchmark/perf_test_inputs/openai-100K-disk-index.json @@ -14,7 +14,7 @@ "dim": 1536, "max_degree": 59, "l_build": 80, - "num_threads": 4, + "num_threads": 8, "build_ram_limit_gb": 4.0, "num_pq_chunks": 384, "quantization_type": "SQ_1_2.0", diff --git a/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json b/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json index e5f06aa1b7..cb9e63616c 100644 --- a/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json +++ b/diskann-benchmark/perf_test_inputs/wikipedia-100K-disk-index.json @@ -13,11 +13,11 @@ "distance": "inner_product", "dim": 768, "max_degree": 59, - "l_build": 72, - "num_threads": 4, + "l_build": 80, + "num_threads": 8, "build_ram_limit_gb": 4.0, "num_pq_chunks": 192, - "quantization_type": "FP", + "quantization_type": "SQ_1_2.0", "save_path": "wikipedia_100k_benchmark_index" }, "search_phase": { From 85199c2e742f005eb3c0c10a0a593019c543a6b0 Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Fri, 10 Apr 2026 11:14:16 +0800 Subject: [PATCH 38/41] Address hildebrandmw review: Direction enum, Table display, composite action, workflow renames - Fuse check_lower/check_higher into check_metric with Direction enum - Use Table for aligned regression output - Prefix metric names with L{value}: for multiple search_l entries - Rename benchmarks[-aa].yml to disk-benchmarks[-aa].yml - Factor shared setup into .github/actions/setup-disk-benchmark/action.yml - A/A: build once, run twice (no duplicate clone/compile) - Add PERF_INPUTS workflow-level env var --- .../actions/setup-disk-benchmark/action.yml | 51 ++++++ ...nchmarks-aa.yml => disk-benchmarks-aa.yml} | 74 +++------ .../{benchmarks.yml => disk-benchmarks.yml} | 53 +++--- .../src/backend/disk_index/benchmarks.rs | 156 +++++++----------- 4 files changed, 160 insertions(+), 174 deletions(-) create mode 100644 .github/actions/setup-disk-benchmark/action.yml rename .github/workflows/{benchmarks-aa.yml => disk-benchmarks-aa.yml} (59%) rename .github/workflows/{benchmarks.yml => disk-benchmarks.yml} (67%) diff --git a/.github/actions/setup-disk-benchmark/action.yml b/.github/actions/setup-disk-benchmark/action.yml new file mode 100644 index 0000000000..386584fcd7 --- /dev/null +++ b/.github/actions/setup-disk-benchmark/action.yml @@ -0,0 +1,51 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT license. + +# Shared setup for disk-index benchmark workflows. +# Installs system dependencies and downloads a dataset from big-ann-benchmarks. + +name: Setup Disk Benchmark +description: Install dependencies and download a benchmark dataset + +inputs: + dataset: + description: 'Dataset name (e.g. wikipedia-100K, openai-100K)' + required: true + archive: + description: 'Archive filename to download (e.g. wikipedia-100K.tar.gz)' + required: true + extract-to: + description: 'Directory to extract the dataset into' + required: true + rust-cache-key: + description: 'Suffix for the Rust dependency cache key' + required: true + +runs: + using: composite + steps: + - name: Install Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + + - name: Cache Rust dependencies + uses: Swatinem/rust-cache@v2 + with: + workspaces: diskann_rust -> target + key: ${{ inputs.rust-cache-key }} + + - name: Install system dependencies + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y openssl libssl-dev pkg-config + + - name: Download ${{ inputs.dataset }} dataset + shell: bash + env: + BAB_RELEASE_URL: https://github.com/harsha-simhadri/big-ann-benchmarks/releases/download/v0.4.0 + run: | + mkdir -p ${{ inputs.extract-to }} + curl -L -o ${{ inputs.archive }} ${{ env.BAB_RELEASE_URL }}/${{ inputs.archive }} + tar xzf ${{ inputs.archive }} -C ${{ inputs.extract-to }} diff --git a/.github/workflows/benchmarks-aa.yml b/.github/workflows/disk-benchmarks-aa.yml similarity index 59% rename from .github/workflows/benchmarks-aa.yml rename to .github/workflows/disk-benchmarks-aa.yml index dc1fd7abba..6e53cb5fc5 100644 --- a/.github/workflows/benchmarks-aa.yml +++ b/.github/workflows/disk-benchmarks-aa.yml @@ -7,7 +7,7 @@ # If any threshold is breached, a GitHub issue is created to notify @microsoft/diskann-admin. # Can also be triggered manually for debugging. -name: Benchmarks (A/A) +name: Disk Benchmarks (A/A) on: schedule: @@ -21,6 +21,7 @@ concurrency: env: RUST_BACKTRACE: 1 + PERF_INPUTS: diskann-benchmark/perf_test_inputs defaults: run: @@ -43,81 +44,54 @@ jobs: - dataset: wikipedia-100K config: wikipedia-100K-disk-index.json archive: wikipedia-100K.tar.gz - data_dir: wikipedia_cohere - dataset: openai-100K config: openai-100K-disk-index.json archive: openai-100K.tar.gz - data_dir: OpenAIArXiv steps: - - name: Checkout main (target) + - name: Checkout main uses: actions/checkout@v4 with: ref: main path: diskann_rust lfs: true - - name: Checkout main (baseline) - uses: actions/checkout@v4 - with: - ref: main - path: baseline - lfs: true - - - name: Install Rust - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - - - name: Cache Rust dependencies (target) - uses: Swatinem/rust-cache@v2 + - name: Setup benchmark environment + uses: ./diskann_rust/.github/actions/setup-disk-benchmark with: - workspaces: diskann_rust -> target - key: aa-target + dataset: ${{ matrix.dataset }} + archive: ${{ matrix.archive }} + extract-to: diskann_rust/target/tmp + rust-cache-key: aa - - name: Cache Rust dependencies (baseline) - uses: Swatinem/rust-cache@v2 - with: - workspaces: baseline -> target - key: aa-baseline - - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y openssl libssl-dev pkg-config - - # Download pre-packaged dataset from GitHub Release - # Source: https://github.com/harsha-simhadri/big-ann-benchmarks - - name: Download ${{ matrix.dataset }} dataset - run: | - mkdir -p diskann_rust/target/tmp baseline/target/tmp - curl -L -o ${{ matrix.archive }} https://github.com/harsha-simhadri/big-ann-benchmarks/releases/download/v0.4.0/${{ matrix.archive }} - tar xzf ${{ matrix.archive }} -C diskann_rust/target/tmp/ - cp -r diskann_rust/target/tmp/${{ matrix.data_dir }} baseline/target/tmp/ + # A/A: build once, run twice (identical code — only detecting environment noise) + - name: Build benchmark binary + working-directory: diskann_rust + run: cargo build -p diskann-benchmark --features disk-index --release - name: Run baseline benchmark - working-directory: baseline + working-directory: diskann_rust run: | cargo run -p diskann-benchmark --features disk-index --release -- \ - run --input-file ../diskann_rust/diskann-benchmark/perf_test_inputs/${{ matrix.config }} \ - --output-file target/tmp/${{ matrix.dataset }}_benchmark_crate_baseline.json + run --input-file ${{ env.PERF_INPUTS }}/${{ matrix.config }} \ + --output-file target/tmp/${{ matrix.dataset }}_baseline.json - name: Run target benchmark working-directory: diskann_rust run: | cargo run -p diskann-benchmark --features disk-index --release -- \ - run --input-file diskann-benchmark/perf_test_inputs/${{ matrix.config }} \ - --output-file target/tmp/${{ matrix.dataset }}_benchmark_crate_target.json + run --input-file ${{ env.PERF_INPUTS }}/${{ matrix.config }} \ + --output-file target/tmp/${{ matrix.dataset }}_target.json - name: Validate benchmark results working-directory: diskann_rust run: | cargo run -p diskann-benchmark --features disk-index --release -- \ check run \ - --tolerances diskann-benchmark/perf_test_inputs/disk-index-tolerances.json \ - --input-file diskann-benchmark/perf_test_inputs/${{ matrix.config }} \ - --before ../baseline/target/tmp/${{ matrix.dataset }}_benchmark_crate_baseline.json \ - --after target/tmp/${{ matrix.dataset }}_benchmark_crate_target.json + --tolerances ${{ env.PERF_INPUTS }}/disk-index-tolerances.json \ + --input-file ${{ env.PERF_INPUTS }}/${{ matrix.config }} \ + --before target/tmp/${{ matrix.dataset }}_baseline.json \ + --after target/tmp/${{ matrix.dataset }}_target.json - name: Upload benchmark results uses: actions/upload-artifact@v4 @@ -125,8 +99,8 @@ jobs: with: name: aa-results-${{ matrix.dataset }} path: | - diskann_rust/target/tmp/${{ matrix.dataset }}_benchmark_crate_target.json - baseline/target/tmp/${{ matrix.dataset }}_benchmark_crate_baseline.json + diskann_rust/target/tmp/${{ matrix.dataset }}_target.json + diskann_rust/target/tmp/${{ matrix.dataset }}_baseline.json retention-days: 30 # Notify diskann-admin on A/A failure diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/disk-benchmarks.yml similarity index 67% rename from .github/workflows/benchmarks.yml rename to .github/workflows/disk-benchmarks.yml index a9c8ebb024..3202f4eaf1 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/disk-benchmarks.yml @@ -6,7 +6,7 @@ # This workflow runs macro benchmarks comparing the current branch against a baseline. # It is manually triggered and requires a baseline reference (branch, tag, or commit). -name: Benchmarks +name: Disk Benchmarks on: workflow_dispatch: @@ -32,7 +32,7 @@ on: - 'diskann-label-filter/**' - 'diskann-benchmark/**' - 'diskann-benchmark-runner/**' - - '.github/workflows/benchmarks.yml' + - '.github/workflows/disk-benchmarks.yml' # Cancel in-progress runs when a new run is triggered concurrency: @@ -41,6 +41,7 @@ concurrency: env: RUST_BACKTRACE: 1 + PERF_INPUTS: diskann-benchmark/perf_test_inputs defaults: run: @@ -87,16 +88,13 @@ jobs: path: baseline lfs: true - - name: Install Rust - uses: dtolnay/rust-toolchain@master + - name: Setup benchmark environment + uses: ./diskann_rust/.github/actions/setup-disk-benchmark with: - toolchain: stable - - - name: Cache Rust dependencies (current) - uses: Swatinem/rust-cache@v2 - with: - workspaces: diskann_rust -> target - key: benchmark-current + dataset: ${{ matrix.dataset }} + archive: ${{ matrix.archive }} + extract-to: diskann_rust/target/tmp + rust-cache-key: benchmark-current - name: Cache Rust dependencies (baseline) uses: Swatinem/rust-cache@v2 @@ -104,43 +102,34 @@ jobs: workspaces: baseline -> target key: benchmark-baseline - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y openssl libssl-dev pkg-config - - # Download pre-packaged dataset from GitHub Release - # Source: https://github.com/harsha-simhadri/big-ann-benchmarks - - name: Download ${{ matrix.dataset }} dataset + - name: Copy dataset to baseline run: | - mkdir -p diskann_rust/target/tmp baseline/target/tmp - curl -L -o ${{ matrix.archive }} https://github.com/harsha-simhadri/big-ann-benchmarks/releases/download/v0.4.0/${{ matrix.archive }} - tar xzf ${{ matrix.archive }} -C diskann_rust/target/tmp/ + mkdir -p baseline/target/tmp cp -r diskann_rust/target/tmp/${{ matrix.data_dir }} baseline/target/tmp/ - name: Run baseline benchmark working-directory: baseline run: | cargo run -p diskann-benchmark --features disk-index --release -- \ - run --input-file ../diskann_rust/diskann-benchmark/perf_test_inputs/${{ matrix.config }} \ - --output-file target/tmp/${{ matrix.dataset }}_benchmark_crate_baseline.json + run --input-file ../diskann_rust/${{ env.PERF_INPUTS }}/${{ matrix.config }} \ + --output-file target/tmp/${{ matrix.dataset }}_baseline.json - name: Run current branch benchmark working-directory: diskann_rust run: | cargo run -p diskann-benchmark --features disk-index --release -- \ - run --input-file diskann-benchmark/perf_test_inputs/${{ matrix.config }} \ - --output-file target/tmp/${{ matrix.dataset }}_benchmark_crate_target.json + run --input-file ${{ env.PERF_INPUTS }}/${{ matrix.config }} \ + --output-file target/tmp/${{ matrix.dataset }}_target.json - name: Validate benchmark results working-directory: diskann_rust run: | cargo run -p diskann-benchmark --features disk-index --release -- \ check run \ - --tolerances diskann-benchmark/perf_test_inputs/disk-index-tolerances.json \ - --input-file diskann-benchmark/perf_test_inputs/${{ matrix.config }} \ - --before ../baseline/target/tmp/${{ matrix.dataset }}_benchmark_crate_baseline.json \ - --after target/tmp/${{ matrix.dataset }}_benchmark_crate_target.json + --tolerances ${{ env.PERF_INPUTS }}/disk-index-tolerances.json \ + --input-file ${{ env.PERF_INPUTS }}/${{ matrix.config }} \ + --before ../baseline/target/tmp/${{ matrix.dataset }}_baseline.json \ + --after target/tmp/${{ matrix.dataset }}_target.json - name: Upload benchmark results uses: actions/upload-artifact@v4 @@ -148,6 +137,6 @@ jobs: with: name: benchmark-results-${{ matrix.dataset }} path: | - diskann_rust/target/tmp/${{ matrix.dataset }}_benchmark_crate_target.json - baseline/target/tmp/${{ matrix.dataset }}_benchmark_crate_baseline.json + diskann_rust/target/tmp/${{ matrix.dataset }}_target.json + baseline/target/tmp/${{ matrix.dataset }}_baseline.json retention-days: 30 \ No newline at end of file diff --git a/diskann-benchmark/src/backend/disk_index/benchmarks.rs b/diskann-benchmark/src/backend/disk_index/benchmarks.rs index 35fc4023c7..1950eb309c 100644 --- a/diskann-benchmark/src/backend/disk_index/benchmarks.rs +++ b/diskann-benchmark/src/backend/disk_index/benchmarks.rs @@ -13,6 +13,7 @@ use diskann_benchmark_runner::{ output::Output, utils::{ datatype::{DataType, Type}, + fmt::Table, num::{relative_change, NonNegativeFinite}, }, Any, Benchmark, CheckDeserialization, Checker, Checkpoint, Input, @@ -208,6 +209,13 @@ impl Input for DiskIndexTolerance { } } +/// Whether a metric improves when its value goes down or up. +#[derive(Clone, Copy)] +enum Direction { + LowerIsBetter, + HigherIsBetter, +} + /// A single metric comparison in the regression check. #[derive(Debug, Serialize)] struct MetricComparison { @@ -223,89 +231,48 @@ struct MetricComparison { /// Aggregated result of a disk-index regression check. #[derive(Debug, Serialize)] struct DiskIndexCheckResult { - search_l: u32, comparisons: Vec, } impl fmt::Display for DiskIndexCheckResult { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - writeln!( - f, - " Search L={}: {:>15} {:>15} {:>12} {:>12} Remark", - self.search_l, "Before", "After", "Change", "Tolerance" - )?; - writeln!(f, " {}", "=".repeat(90))?; - for c in &self.comparisons { - writeln!( - f, - " {:>20}, {:>14.3}, {:>14.3}, {:>11}, {:>11.1}%, {}", - c.metric, - c.before, - c.after, - c.change_pct, - c.tolerance_pct * 100.0, - c.remark - )?; - } - Ok(()) - } -} - -/// Check a "lower is better" metric (latency, IOs, comparisons). -/// Regression = value increased beyond tolerance. -fn check_lower_is_better( - name: &'static str, - before: f64, - after: f64, - tolerance: NonNegativeFinite, - passed: &mut bool, -) -> MetricComparison { - let (change_pct, remark, metric_passed) = match relative_change(before, after) { - Ok(change) => { - let ok = change <= tolerance.get(); - if !ok { - *passed = false; + let header = ["Metric", "Before", "After", "Change", "Tolerance", "Remark"]; + let mut table = Table::new(header, self.comparisons.len()); + + for (i, c) in self.comparisons.iter().enumerate() { + let mut row = table.row(i); + row.insert(c.metric, 0); + row.insert(format!("{:.3}", c.before), 1); + row.insert(format!("{:.3}", c.after), 2); + row.insert(&c.change_pct, 3); + row.insert(format!("{:.1}%", c.tolerance_pct * 100.0), 4); + if !c.remark.is_empty() { + row.insert(&c.remark, 5); } - ( - format!("{:.3}%", change * 100.0), - if ok { - String::new() - } else { - "REGRESSION".to_string() - }, - ok, - ) - } - Err(e) => { - *passed = false; - ("invalid".to_string(), e.to_string(), false) } - }; - MetricComparison { - metric: name, - before, - after, - change_pct, - tolerance_pct: tolerance.get(), - passed: metric_passed, - remark, + + table.fmt(f) } } -/// Check a "higher is better" metric (QPS, recall). -/// Regression = value decreased beyond tolerance. -fn check_higher_is_better( +/// Check a metric for regression. +/// +/// For `LowerIsBetter` metrics (latency, IOs), regression = value increased beyond tolerance. +/// For `HigherIsBetter` metrics (QPS, recall), regression = value decreased beyond tolerance. +fn check_metric( name: &'static str, + direction: Direction, before: f64, after: f64, tolerance: NonNegativeFinite, passed: &mut bool, ) -> MetricComparison { - // Flip before/after so that a decrease becomes a positive relative_change let (change_pct, remark, metric_passed) = match relative_change(before, after) { Ok(change) => { - // For higher-is-better, a negative change is a regression - let ok = -change <= tolerance.get(); + let ok = match direction { + Direction::LowerIsBetter => change <= tolerance.get(), + Direction::HigherIsBetter => -change <= tolerance.get(), + }; if !ok { *passed = false; } @@ -350,17 +317,18 @@ where before: &DiskIndexStats, after: &DiskIndexStats, ) -> anyhow::Result> { + use Direction::{HigherIsBetter, LowerIsBetter}; + let mut passed = true; let mut comparisons = Vec::new(); // Check build time if both sides have it if let (Some(b_build), Some(a_build)) = (&before.build, &after.build) { - let b_time = b_build.build_time_seconds(); - let a_time = a_build.build_time_seconds(); - comparisons.push(check_lower_is_better( + comparisons.push(check_metric( "build_time", - b_time, - a_time, + LowerIsBetter, + b_build.build_time_seconds(), + a_build.build_time_seconds(), tolerances.build_time_regression, &mut passed, )); @@ -387,43 +355,56 @@ where a_sr.search_l, ); - comparisons.push(check_higher_is_better( - "qps", + // Prefix metric names with L value when multiple search_l entries exist. + let prefix = if before.search.search_results_per_l.len() > 1 { + format!("L{}:", b_sr.search_l) + } else { + String::new() + }; + + comparisons.push(check_metric( + Box::leak(format!("{}qps", prefix).into_boxed_str()), + HigherIsBetter, b_sr.qps as f64, a_sr.qps as f64, tolerances.qps_regression, &mut passed, )); - comparisons.push(check_higher_is_better( - "recall", + comparisons.push(check_metric( + Box::leak(format!("{}recall", prefix).into_boxed_str()), + HigherIsBetter, b_sr.recall as f64, a_sr.recall as f64, tolerances.recall_regression, &mut passed, )); - comparisons.push(check_lower_is_better( - "mean_latency", + comparisons.push(check_metric( + Box::leak(format!("{}mean_latency", prefix).into_boxed_str()), + LowerIsBetter, b_sr.mean_latency, a_sr.mean_latency, tolerances.mean_latency_regression, &mut passed, )); - comparisons.push(check_lower_is_better( - "p95_latency", + comparisons.push(check_metric( + Box::leak(format!("{}p95_latency", prefix).into_boxed_str()), + LowerIsBetter, b_sr.p95_latency.as_f64(), a_sr.p95_latency.as_f64(), tolerances.p95_latency_regression, &mut passed, )); - comparisons.push(check_lower_is_better( - "mean_ios", + comparisons.push(check_metric( + Box::leak(format!("{}mean_ios", prefix).into_boxed_str()), + LowerIsBetter, b_sr.mean_ios, a_sr.mean_ios, tolerances.mean_ios_regression, &mut passed, )); - comparisons.push(check_lower_is_better( - "mean_comparisons", + comparisons.push(check_metric( + Box::leak(format!("{}mean_comparisons", prefix).into_boxed_str()), + LowerIsBetter, b_sr.mean_comparisons, a_sr.mean_comparisons, tolerances.mean_comps_regression, @@ -431,16 +412,7 @@ where )); } - let search_l = before - .search - .search_results_per_l - .first() - .map(|s| s.search_l) - .unwrap_or(0); - let result = DiskIndexCheckResult { - search_l, - comparisons, - }; + let result = DiskIndexCheckResult { comparisons }; if passed { Ok(PassFail::Pass(result)) From 9d9666879ae6adc95b56b75f5a1cf439e4b41c6b Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Fri, 10 Apr 2026 11:25:12 +0800 Subject: [PATCH 39/41] Fix E0521: clone strings for Table::Row::insert ('static bound) --- diskann-benchmark/src/backend/disk_index/benchmarks.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/diskann-benchmark/src/backend/disk_index/benchmarks.rs b/diskann-benchmark/src/backend/disk_index/benchmarks.rs index 1950eb309c..89756f8656 100644 --- a/diskann-benchmark/src/backend/disk_index/benchmarks.rs +++ b/diskann-benchmark/src/backend/disk_index/benchmarks.rs @@ -244,10 +244,10 @@ impl fmt::Display for DiskIndexCheckResult { row.insert(c.metric, 0); row.insert(format!("{:.3}", c.before), 1); row.insert(format!("{:.3}", c.after), 2); - row.insert(&c.change_pct, 3); + row.insert(c.change_pct.clone(), 3); row.insert(format!("{:.1}%", c.tolerance_pct * 100.0), 4); if !c.remark.is_empty() { - row.insert(&c.remark, 5); + row.insert(c.remark.clone(), 5); } } From 12d6db827bd7f17da553f94f4cbb9c393e464e3e Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Tue, 14 Apr 2026 12:30:59 +0800 Subject: [PATCH 40/41] Address review: String metric, remove write perm, rustup show, drop cache --- .../actions/setup-disk-benchmark/action.yml | 14 ++----------- .github/workflows/disk-benchmarks-aa.yml | 1 - .github/workflows/disk-benchmarks.yml | 8 -------- .../src/backend/disk_index/benchmarks.rs | 20 +++++++++---------- 4 files changed, 12 insertions(+), 31 deletions(-) diff --git a/.github/actions/setup-disk-benchmark/action.yml b/.github/actions/setup-disk-benchmark/action.yml index 386584fcd7..19c6fcbaac 100644 --- a/.github/actions/setup-disk-benchmark/action.yml +++ b/.github/actions/setup-disk-benchmark/action.yml @@ -17,23 +17,13 @@ inputs: extract-to: description: 'Directory to extract the dataset into' required: true - rust-cache-key: - description: 'Suffix for the Rust dependency cache key' - required: true runs: using: composite steps: - name: Install Rust - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@v2 - with: - workspaces: diskann_rust -> target - key: ${{ inputs.rust-cache-key }} + shell: bash + run: rustup show - name: Install system dependencies shell: bash diff --git a/.github/workflows/disk-benchmarks-aa.yml b/.github/workflows/disk-benchmarks-aa.yml index 6e53cb5fc5..33455302c0 100644 --- a/.github/workflows/disk-benchmarks-aa.yml +++ b/.github/workflows/disk-benchmarks-aa.yml @@ -62,7 +62,6 @@ jobs: dataset: ${{ matrix.dataset }} archive: ${{ matrix.archive }} extract-to: diskann_rust/target/tmp - rust-cache-key: aa # A/A: build once, run twice (identical code — only detecting environment noise) - name: Build benchmark binary diff --git a/.github/workflows/disk-benchmarks.yml b/.github/workflows/disk-benchmarks.yml index 3202f4eaf1..919e74c156 100644 --- a/.github/workflows/disk-benchmarks.yml +++ b/.github/workflows/disk-benchmarks.yml @@ -49,7 +49,6 @@ defaults: permissions: contents: read - pull-requests: write # Required for posting PR comments jobs: # Macro benchmark: compare current branch against baseline @@ -94,13 +93,6 @@ jobs: dataset: ${{ matrix.dataset }} archive: ${{ matrix.archive }} extract-to: diskann_rust/target/tmp - rust-cache-key: benchmark-current - - - name: Cache Rust dependencies (baseline) - uses: Swatinem/rust-cache@v2 - with: - workspaces: baseline -> target - key: benchmark-baseline - name: Copy dataset to baseline run: | diff --git a/diskann-benchmark/src/backend/disk_index/benchmarks.rs b/diskann-benchmark/src/backend/disk_index/benchmarks.rs index 89756f8656..fa9b036ad0 100644 --- a/diskann-benchmark/src/backend/disk_index/benchmarks.rs +++ b/diskann-benchmark/src/backend/disk_index/benchmarks.rs @@ -219,7 +219,7 @@ enum Direction { /// A single metric comparison in the regression check. #[derive(Debug, Serialize)] struct MetricComparison { - metric: &'static str, + metric: String, before: f64, after: f64, change_pct: String, @@ -241,7 +241,7 @@ impl fmt::Display for DiskIndexCheckResult { for (i, c) in self.comparisons.iter().enumerate() { let mut row = table.row(i); - row.insert(c.metric, 0); + row.insert(c.metric.clone(), 0); row.insert(format!("{:.3}", c.before), 1); row.insert(format!("{:.3}", c.after), 2); row.insert(c.change_pct.clone(), 3); @@ -260,7 +260,7 @@ impl fmt::Display for DiskIndexCheckResult { /// For `LowerIsBetter` metrics (latency, IOs), regression = value increased beyond tolerance. /// For `HigherIsBetter` metrics (QPS, recall), regression = value decreased beyond tolerance. fn check_metric( - name: &'static str, + name: String, direction: Direction, before: f64, after: f64, @@ -325,7 +325,7 @@ where // Check build time if both sides have it if let (Some(b_build), Some(a_build)) = (&before.build, &after.build) { comparisons.push(check_metric( - "build_time", + "build_time".to_string(), LowerIsBetter, b_build.build_time_seconds(), a_build.build_time_seconds(), @@ -363,7 +363,7 @@ where }; comparisons.push(check_metric( - Box::leak(format!("{}qps", prefix).into_boxed_str()), + format!("{prefix}qps"), HigherIsBetter, b_sr.qps as f64, a_sr.qps as f64, @@ -371,7 +371,7 @@ where &mut passed, )); comparisons.push(check_metric( - Box::leak(format!("{}recall", prefix).into_boxed_str()), + format!("{prefix}recall"), HigherIsBetter, b_sr.recall as f64, a_sr.recall as f64, @@ -379,7 +379,7 @@ where &mut passed, )); comparisons.push(check_metric( - Box::leak(format!("{}mean_latency", prefix).into_boxed_str()), + format!("{prefix}mean_latency"), LowerIsBetter, b_sr.mean_latency, a_sr.mean_latency, @@ -387,7 +387,7 @@ where &mut passed, )); comparisons.push(check_metric( - Box::leak(format!("{}p95_latency", prefix).into_boxed_str()), + format!("{prefix}p95_latency"), LowerIsBetter, b_sr.p95_latency.as_f64(), a_sr.p95_latency.as_f64(), @@ -395,7 +395,7 @@ where &mut passed, )); comparisons.push(check_metric( - Box::leak(format!("{}mean_ios", prefix).into_boxed_str()), + format!("{prefix}mean_ios"), LowerIsBetter, b_sr.mean_ios, a_sr.mean_ios, @@ -403,7 +403,7 @@ where &mut passed, )); comparisons.push(check_metric( - Box::leak(format!("{}mean_comparisons", prefix).into_boxed_str()), + format!("{prefix}mean_comparisons"), LowerIsBetter, b_sr.mean_comparisons, a_sr.mean_comparisons, From 68673a9f032fdd1cbe135e6479141e5accd08d59 Mon Sep 17 00:00:00 2001 From: "Yuanyuan Tian (from Dev Box)" Date: Wed, 15 Apr 2026 11:19:44 +0800 Subject: [PATCH 41/41] Change A/A failure notification to @microsoft/diskann-disk-maintainers --- .github/workflows/disk-benchmarks-aa.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/disk-benchmarks-aa.yml b/.github/workflows/disk-benchmarks-aa.yml index 33455302c0..7057cade85 100644 --- a/.github/workflows/disk-benchmarks-aa.yml +++ b/.github/workflows/disk-benchmarks-aa.yml @@ -130,7 +130,7 @@ jobs: `Please review the benchmark artifacts and determine if thresholds need tuning`, `or if there is a runner environment issue.`, ``, - `/cc @microsoft/diskann-admin`, + `/cc @microsoft/diskann-disk-maintainers`, ].join('\n'), labels: ['benchmark', 'A/A-failure'], });