Unified: Drop the separate swift-syntax binary and make tests runnable both with Bazel and cargo

unified/BUILD.bazel

@@ -43,19 +43,9 @@ codeql_pkg_files(
  name = "extractor-arch",
  exes = [
  "//unified/extractor",
- ],
- prefix = "tools/{CODEQL_PLATFORM}",
-)
-
-# The Swift front-end parser (wrapper + real binary + bundled Swift runtime),
-# shipped next to the extractor. Only on platforms where swift-syntax builds
-# (Linux/macOS); elsewhere the group is empty so the pack still builds (Swift
-# extraction is simply unavailable there).
-pkg_filegroup(
- name = "swift-syntax-parse-arch",
- srcs = select_os(
+ ] + select_os(
+ linux = ["//unified/swift-syntax-rs:swift_runtime_libs"],
  otherwise = [],
- posix = ["//unified/swift-syntax-rs:swift-syntax-parse-pkg"],
  ),
  prefix = "tools/{CODEQL_PLATFORM}",
 )
@@ -66,7 +56,6 @@ codeql_pack(
  ":codeql-extractor-yml",
  ":dbscheme-group",
  ":extractor-arch",
- ":swift-syntax-parse-arch",
  "//unified/tools",
  ],
 )

unified/extractor/BUILD.bazel

@@ -1,5 +1,7 @@
+load("@rules_rust//rust:defs.bzl", "rust_test")
 load("//misc/bazel:rust.bzl", "codeql_rust_binary")
 load("//misc/bazel/3rdparty/tree_sitter_extractors_deps:defs.bzl", "aliases", "all_crate_deps")
+load("//unified:platforms.bzl", "UNIFIED_SUPPORTED_PLATFORMS")
 
 exports_files(["Cargo.toml"])
 
@@ -11,14 +13,71 @@ codeql_rust_binary(
  "ast_types.yml",
  "swift_node_types.yml",
  ],
+ data = select({
+ "@platforms//os:linux": ["//unified/swift-syntax-rs:swift_runtime_libs"],
+ "//conditions:default": [],
+ }),
  proc_macro_deps = all_crate_deps(
  proc_macro = True,
  ),
+ target_compatible_with = UNIFIED_SUPPORTED_PLATFORMS,
  visibility = ["//visibility:public"],
  deps = all_crate_deps(
  normal = True,
  ) + [
  "//shared/tree-sitter-extractor",
  "//shared/yeast",
+ "//unified/swift-syntax-rs:swift_syntax_rs",
  ],
 )
+
+_TESTS = {
+ "corpus_tests": {
+ "data": glob(["tests/corpus/**"]),
+ "compile_data": [],
+ "size": "medium",
+ },
+ # `include_str!`s a checked-in `parse_to_json` dump.
+ "swift_syntax_pipeline": {
+ "data": [],
+ "compile_data": glob(["tests/fixtures/**"]),
+ "size": "small",
+ },
+}
+
+[
+ rust_test(
+ name = test_name,
+ size = spec["size"],
+ srcs = ["tests/%s.rs" % test_name] + glob(["src/**/*.rs"]),
+ aliases = aliases(),
+ compile_data = [
+ "ast_types.yml",
+ "swift_node_types.yml",
+ ] + spec["compile_data"],
+ crate_root = "tests/%s.rs" % test_name,
+ data = spec["data"] + select({
+ "@platforms//os:linux": ["//unified/swift-syntax-rs:swift_runtime_libs"],
+ "//conditions:default": [],
+ }),
+ edition = "2024",
+ proc_macro_deps = all_crate_deps(
+ proc_macro = True,
+ ),
+ rustc_flags = ["--cfg=bazel"],
+ target_compatible_with = UNIFIED_SUPPORTED_PLATFORMS,
+ deps = all_crate_deps(
+ normal = True,
+ ) + [
+ "//shared/tree-sitter-extractor",
+ "//shared/yeast",
+ "//unified/swift-syntax-rs:swift_syntax_rs",
+ ],
+ )
+ for test_name, spec in _TESTS.items()
+]
+
+test_suite(
+ name = "all_tests",
+ tests = [":%s" % test_name for test_name in _TESTS],
+)

unified/extractor/Cargo.toml

@@ -18,3 +18,4 @@ serde_json = "1.0.145"
 
 codeql-extractor = { path = "../../shared/tree-sitter-extractor" }
 yeast = { path = "../../shared/yeast" }
+swift-syntax-rs = { path = "../swift-syntax-rs" }

unified/extractor/build.rs

@@ -0,0 +1,12 @@
+fn main() {
+ println!("cargo:rustc-check-cfg=cfg(bazel)");
+
+ if let Ok(dir) = std::env::var("DEP_SWIFTSYNTAXFFI_LIBDIR") {
+ println!("cargo:rustc-link-search=native={dir}");
+ println!("cargo:rustc-link-lib=dylib=SwiftSyntaxFFI");
+ println!("cargo:rustc-link-arg=-Wl,-rpath,{dir}");
+ }
+ if let Ok(dir) = std::env::var("DEP_SWIFTSYNTAXFFI_RUNTIMEDIR") {
+ println!("cargo:rustc-link-arg=-Wl,-rpath,{dir}");
+ }
+}

unified/extractor/src/languages/swift/parse.rs

@@ -1,121 +1,22 @@
-//! Swift front-end parser: shells out to the separate `swift-syntax-parse`
-//! binary (which links swift-syntax) to obtain a JSON syntax tree, then adapts
-//! that JSON into a `yeast::Ast` via the pure-Rust [`swift_adapter`] module.
-//!
-//! Running the parser in a separate process keeps the Swift toolchain out of
-//! the extractor's own build: the extractor never links Swift, so working on
-//! other (e.g. tree-sitter based) languages needs no Swift toolchain. Each call
-//! spawns the parser afresh; a longer-lived parser process could be swapped in
-//! behind this same seam later without touching the extraction pipeline.
-
-use std::io::Write;
-use std::process::{Command, Stdio};
+//! Swift front-end parser: calls into the `swift-syntax-rs` crate (which links
+//! swift-syntax) to obtain a JSON syntax tree, then adapts that JSON into a
+//! `yeast::Ast` via the pure-Rust [`swift_adapter`] module.
 
 use codeql_extractor::extractor::ParsedTree;
 
 use super::swift_adapter;
 
-/// Environment variable naming the `swift-syntax-parse` executable. When unset,
-/// the parser is resolved next to the extractor executable, then on `PATH`.
-const PARSE_BIN_ENV: &str = "CODEQL_EXTRACTOR_UNIFIED_SWIFT_SYNTAX_PARSE";
-
-/// Base name of the `swift-syntax-parse` executable as shipped / looked up.
-const PARSE_BIN_NAME: &str = "swift-syntax-parse";
-
 /// Parse Swift `source` into a [`ParsedTree`] (a raw `yeast::Ast` plus
 /// side-channel `extra` tokens), ready to be desugared via `run_from_ast`.
 pub fn parse(source: &[u8]) -> Result<ParsedTree, String> {
  let source =
  std::str::from_utf8(source).map_err(|e| format!("Swift source is not valid UTF-8: {e}"))?;
- let json = run_parser(source)?;
+ let json =
+ swift_syntax_rs::parse_to_json(source).map_err(|e| format!("Swift parser failed: {e}"))?;
  let mut adapted = swift_adapter::json_to_ast(&json)?;
  adapted.ast.set_source(source.as_bytes().to_vec());
  Ok(ParsedTree {
  ast: adapted.ast,
  extras: adapted.extras,
  })
 }
-
-/// The `swift-syntax-parse` executable to invoke, resolved in priority order:
-///
-/// 1. the `CODEQL_EXTRACTOR_UNIFIED_SWIFT_SYNTAX_PARSE` override, if set;
-/// 2. a copy shipped next to the extractor executable — this is how the CodeQL
-/// extractor pack lays it out (`tools/<platform>/{extractor,
-/// swift-syntax-parse}`), so a packaged extractor is self-contained with no
-/// environment setup;
-/// 3. a bare `swift-syntax-parse`, looked up on `PATH`.
-fn parse_bin() -> String {
- if let Ok(bin) = std::env::var(PARSE_BIN_ENV) {
- if !bin.is_empty() {
- return bin;
- }
- }
- if let Ok(exe) = std::env::current_exe() {
- if let Some(sibling) = exe.parent().map(|dir| dir.join(PARSE_BIN_NAME)) {
- if sibling.is_file() {
- return sibling.to_string_lossy().into_owned();
- }
- }
- }
- PARSE_BIN_NAME.to_string()
-}
-
-/// Whether the `swift-syntax-parse` executable can be launched at all.
-///
-/// This reports availability of the *executable*, deliberately not whether
-/// parsing succeeds: a binary that launches but then crashes or emits invalid
-/// JSON is still "available", so callers run and surface the failure rather
-/// than silently skipping. Only a genuinely missing/unlaunchable binary (e.g.
-/// no Swift toolchain is installed) reports `false`.
-pub fn binary_available() -> bool {
- match Command::new(parse_bin())
- .stdin(Stdio::null())
- .stdout(Stdio::null())
- .stderr(Stdio::null())
- .spawn()
- {
- Ok(mut child) => {
- let _ = child.wait();
- true
- }
- Err(e) if e.kind() == std::io::ErrorKind::NotFound => false,
- // Any other spawn failure (e.g. a permissions problem) is a genuine
- // issue worth surfacing, so treat the parser as available and let the
- // caller fail rather than masking it as "unavailable".
- Err(_) => true,
- }
-}
-
-/// Run the external parser, feeding `source` on stdin and returning its JSON
-/// stdout.
-fn run_parser(source: &str) -> Result<String, String> {
- let bin = parse_bin();
- let mut child = Command::new(&bin)
- .stdin(Stdio::piped())
- .stdout(Stdio::piped())
- .stderr(Stdio::piped())
- .spawn()
- .map_err(|e| format!("failed to spawn Swift parser `{bin}`: {e}"))?;
-
- // The parser reads all of stdin before writing any stdout, so writing the
- // whole source and then closing stdin (by dropping it) cannot deadlock.
- child
- .stdin
- .take()
- .expect("child stdin was piped")
- .write_all(source.as_bytes())
- .map_err(|e| format!("failed to write source to Swift parser `{bin}`: {e}"))?;
-
- let output = child
- .wait_with_output()
- .map_err(|e| format!("failed to run Swift parser `{bin}`: {e}"))?;
- if !output.status.success() {
- return Err(format!(
- "Swift parser `{bin}` failed ({}): {}",
- output.status,
- String::from_utf8_lossy(&output.stderr).trim()
- ));
- }
- String::from_utf8(output.stdout)
- .map_err(|e| format!("Swift parser produced non-UTF-8 output: {e}"))
-}

unified/extractor/tests/corpus_tests.rs

@@ -20,20 +20,6 @@ fn update_mode_enabled() -> bool {
  .unwrap_or(false)
 }
 
-/// Whether the external swift-syntax parser is available. When the parser
-/// binary genuinely cannot be found/launched (e.g. no Swift toolchain, and
-/// neither `CODEQL_EXTRACTOR_UNIFIED_SWIFT_SYNTAX_PARSE` nor a `swift-syntax-parse`
-/// on `PATH`), the corpus test is skipped rather than failed — it cannot run
-/// without the Swift-backed parser.
-///
-/// Crucially this checks only that the executable *launches*: a parser that is
-/// present but crashes, emits invalid JSON, or otherwise regresses is
-/// considered available, so the suite runs and fails (rather than silently
-/// skipping the very failures CI needs to catch).
-fn parser_available() -> bool {
- languages::swift_parse::binary_available()
-}
-
 /// Parse a corpus `.output` file. The file holds a single test case made of
 /// three sections separated by `---` delimiter lines:
 ///
@@ -110,19 +96,29 @@ fn collect_corpus_stems(dir: &Path, out: &mut Vec<std::path::PathBuf>) {
  }
 }
 
+#[cfg(bazel)]
+fn corpus_dir() -> std::path::PathBuf {
+ let base = std::path::PathBuf::from(
+ std::env::var("RUNFILES_DIR").expect("RUNFILES_DIR not set"),
+ );
+ std::fs::read_dir(&base)
+ .expect("failed to read RUNFILES_DIR")
+ .filter_map(Result::ok)
+ .map(|entry| entry.path().join("unified/extractor/tests/corpus"))
+ .find(|path| path.exists())
+ .expect("corpus not found under any runfiles repo root")
+}
+
+#[cfg(not(bazel))]
+fn corpus_dir() -> std::path::PathBuf {
+ std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/corpus")
+}
+
 #[test]
 fn test_corpus() {
- if !parser_available() {
- eprintln!(
- "skipping test_corpus: the swift-syntax parser is unavailable \
- (set CODEQL_EXTRACTOR_UNIFIED_SWIFT_SYNTAX_PARSE or put \
- `swift-syntax-parse` on PATH)"
- );
- return;
- }
  let update_mode = update_mode_enabled();
  let all_languages = languages::all_language_specs();
- let corpus_dir = Path::new("tests/corpus");
+ let corpus_dir = corpus_dir();
 
  for lang in all_languages {
  let output_schema = yeast::node_types_yaml::schema_from_yaml(languages::OUTPUT_AST_SCHEMA)

unified/platforms.bzl

@@ -0,0 +1,9 @@
+"""Shared platform constraint for the unified extractor."""
+
+# swift-syntax requires a Swift toolchain, which is only available
+# through rules_swift.
+UNIFIED_SUPPORTED_PLATFORMS = select({
+ "@platforms//os:linux": [],
+ "@platforms//os:macos": [],
+ "//conditions:default": ["@platforms//:incompatible"],
+})