Introduce NightMonkey: an ahead-of-time JS-to-Wasm compiler.

.cargo/config.toml.in

@@ -2,8 +2,29 @@
 # It was generated by `mach vendor rust`.
 # Please do not edit.
 
-[source.crates-io]
-replace-with = "vendored-sources"
+# DEVELOPMENT OVERRIDE for this fork: crates.io is not replaced by
+# third_party/rust, so the AOT compiler's dependencies (waffle and the
+# wasm-tools crates) are fetched from the network and third_party/ carries no
+# NightMonkey diff.
+#
+# This is a deliberate choice for this tree, not a temporary state. The fork
+# has no offline-build constraint; mozilla-central does, which is why the
+# stanza exists upstream. Integrating with the vendoring scheme is a question
+# for an upstreaming effort, if there is one -- not a prerequisite for working
+# here.
+#
+# It has to be tree-wide rather than NightMonkey-scoped: `night-compiler` is a
+# workspace member (the in-process lane links it into the jsrust staticlib),
+# so `cargo metadata` resolves waffle for every build in the tree, including
+# builds that never compile it.
+#
+# Locally patched crates are unaffected -- those go through [patch.crates-io]
+# onto build/rust/* paths -- and the git-sourced crates below keep their
+# vendored replacement. To restore upstream behavior, vendor the compiler's
+# dependencies and re-enable:
+#
+#   [source.crates-io]
+#   replace-with = "vendored-sources"
 
 [source."git+https://github.com/FirefoxGraphics/aa-stroke?rev=5776bdfc8ad664a1503db668fab397d818a5f98a"]
 git = "https://github.com/FirefoxGraphics/aa-stroke"

.github/workflows/main.yml

@@ -8,18 +8,25 @@ defaults:
   run:
     shell: bash
 
-jobs:    
+jobs:
   test:
     strategy:
       fail-fast: false
       matrix:
         test: [jstests, jit-test]
-        mozconfig: [ debug, pbl-debug, pbl-release, release, aot-ics-release ]
+        mozconfig: [ debug, pbl-debug, pbl-release, release, aot-ics-release, nightmonkey ]
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
       with:
         submodules: true
     - run: ./mach --no-interactive bootstrap --application-choice=js
+    - if: matrix.mozconfig == 'nightmonkey'
+      # mach bootstrap only fetches host-target toolchains; the wasm32-wasi
+      # sysroot (crt1.o, libc) must be fetched explicitly into ~/.mozbuild.
+      run: |
+        rustup component add rust-src
+        rustup target add wasm32-wasip1
+        cd ~/.mozbuild && "$GITHUB_WORKSPACE/mach" --log-no-times artifact toolchain --from-build sysroot-wasm32-wasi
     - run: MOZCONFIG=.github/workflows/mozconfig-${{ matrix.mozconfig }} ./mach build
-    - run: MOZCONFIG=.github/workflows/mozconfig-${{ matrix.mozconfig }} ./mach ${{ matrix.test }} ${{ matrix.test == 'jit-test' && '--exclude wasm/atomicity.js' || '' }}
+    - run: MOZCONFIG=.github/workflows/mozconfig-${{ matrix.mozconfig }} ./mach ${{ matrix.test }} ${{ matrix.mozconfig == 'nightmonkey' && '--shell obj-nightmonkey-inprocess/dist/bin/inproc-shell.sh' || '' }} ${{ matrix.test == 'jit-test' && '--exclude wasm/atomicity.js' || '' }}

.github/workflows/mozconfig-nightmonkey

@@ -0,0 +1,4 @@
+# The NightMonkey in-process test shell (wasm32-wasi) plus the wasm-jit-runner
+# host that runs it. The sourced config sets MOZ_OBJDIR=obj-nightmonkey-inprocess,
+# where the test step finds dist/bin/inproc-shell.sh to use as the shell.
+. "$topsrcdir/js/src/night/configs/mozconfig-nightmonkey-inprocess"

.prettierignore

@@ -1140,6 +1140,7 @@ js/examples/
 js/public/
 js/src/devtools/
 js/src/jit-test/
+js/src/night/tests/
 js/src/tests/
 js/src/Y.js
 

Cargo.toml

@@ -7,6 +7,7 @@
 members = [
   "browser/app/nmhproxy/",
   "js/src/rust",
+  "js/src/night/compiler",
   "netwerk/base/idna_glue",
   "netwerk/test/http3server",
   "security/manager/ssl/abridged_certs",

build/moz.configure/rust.configure

@@ -417,6 +417,13 @@ def detect_rustc_target(
         if len(narrowed) == 1:
             return narrowed[0].rust_target
 
+        # Failing that, prefer the generic "unknown" vendor over a
+        # vendor-specific target (rustc 1.98 added x86_64-oe-linux-gnu beside
+        # x86_64-unknown-linux-gnu, and the host vendor is "pc").
+        narrowed = [c for c in candidates if c.target.vendor == "unknown"]
+        if len(narrowed) == 1:
+            return narrowed[0].rust_target
+
         return None
 
     rustc_target = find_candidate(candidates)

build/workspace-hack/Cargo.toml

@@ -273,6 +273,7 @@ gkrust = ["dep:allocator-api2", "dep:arrayvec", "dep:bindgen", "dep:bitflags", "
 gkrust-gtest = ["gkrust"]
 http3server = ["dep:allocator-api2", "dep:arrayvec", "dep:bindgen", "dep:bitflags", "dep:byteorder", "dep:bytes", "dep:chrono", "dep:clap", "dep:dist-bin", "dep:env_logger", "dep:fnv", "dep:form_urlencoded", "dep:futures", "dep:futures-channel", "dep:futures-core", "dep:futures-executor", "dep:futures-sink", "dep:futures-util", "dep:getrandom", "dep:hashbrown", "dep:hex", "dep:hyper", "dep:icu_locale_core", "dep:icu_properties", "dep:idna", "dep:indexmap", "dep:itertools", "dep:log", "dep:memchr", "dep:mio", "dep:nom", "dep:num-integer", "dep:num-traits", "dep:once_cell", "dep:percent-encoding", "dep:regex", "dep:semver", "dep:serde_json", "dep:smallvec", "dep:stable_deref_trait", "dep:strsim", "dep:time", "dep:time-macros", "dep:tinystr", "dep:tokio", "dep:tokio-util", "dep:toml", "dep:tracing", "dep:unicode-bidi", "dep:url", "dep:windows", "dep:windows-sys", "dep:yoke", "dep:zerocopy", "dep:zerofrom", "dep:zerovec"]
 jsrust = ["dep:allocator-api2", "dep:arrayvec", "dep:byteorder", "dep:cc", "dep:env_logger", "dep:form_urlencoded", "dep:getrandom", "dep:hashbrown", "dep:icu_locale_core", "dep:icu_properties", "dep:idna", "dep:indexmap", "dep:log", "dep:memchr", "dep:num-traits", "dep:once_cell", "dep:percent-encoding", "dep:semver", "dep:smallvec", "dep:stable_deref_trait", "dep:tinystr", "dep:unicode-bidi", "dep:url", "dep:yoke", "dep:zerocopy", "dep:zerofrom", "dep:zerovec"]
+night-compiler = []
 minidump-analyzer-export = ["dep:allocator-api2", "dep:arrayvec", "dep:bitflags", "dep:byteorder", "dep:clap", "dep:env_logger", "dep:futures-channel", "dep:futures-core", "dep:futures-executor", "dep:futures-sink", "dep:futures-util", "dep:getrandom", "dep:hashbrown", "dep:hex", "dep:indexmap", "dep:log", "dep:memchr", "dep:nom", "dep:num-traits", "dep:object", "dep:once_cell", "dep:scroll", "dep:serde_json", "dep:time", "dep:time-macros", "dep:tracing", "dep:uuid", "dep:windows-sys", "dep:zerocopy"]
 mozwer_s = ["dep:allocator-api2", "dep:byteorder", "dep:getrandom", "dep:hashbrown", "dep:indexmap", "dep:log", "dep:once_cell", "dep:scroll", "dep:serde_json", "dep:uuid", "dep:windows-sys", "dep:zerocopy"]
 nmhproxy = ["dep:allocator-api2", "dep:bitflags", "dep:byteorder", "dep:form_urlencoded", "dep:hashbrown", "dep:icu_locale_core", "dep:icu_properties", "dep:idna", "dep:indexmap", "dep:once_cell", "dep:percent-encoding", "dep:serde_json", "dep:smallvec", "dep:stable_deref_trait", "dep:tinystr", "dep:unicode-bidi", "dep:url", "dep:windows-sys", "dep:yoke", "dep:zerocopy", "dep:zerofrom", "dep:zerovec"]

config/check_spidermonkey_style.py

@@ -48,6 +48,10 @@
     "js/src/devtools/",  # auxiliary stuff
     "js/src/editline/",  # imported code
     "js/src/gdb/",  # auxiliary stuff
+    "js/src/night/nightmonkey/",  # snapshot transform tool (own crate)
+    "js/src/night/snapshot/",  # own crate (build outputs)
+    "js/src/night/snapshot-dump/",  # host dump tool (own crate)
+    "js/src/night/wasm-jit-runner/",  # vendored tool (own crate)
     "js/src/vtune/",  # imported code
     "js/src/zydis/",  # imported code
     "js/src/xsum/",  # imported code
@@ -104,6 +108,7 @@
         "unicode/uniset.h",  # ICU
         "unicode/unistr.h",  # ICU
         "unicode/utypes.h",  # ICU
+        "night/snapshot/night-snapshot.h",  # lives in an ignored crate dir
         "vtune/VTuneWrapper.h",  # VTune
         "wasm/WasmBuiltinModuleGenerated.h",  # generated in $OBJDIR"
         "zydis/ZydisAPI.h",  # Zydis

eslint-ignores.config.mjs

@@ -189,6 +189,7 @@ export default [
   "js/public/",
   "js/src/devtools/",
   "js/src/jit-test/",
+  "js/src/night/tests/",
   "js/src/tests/",
   "js/src/Y.js",
 

js/moz.configure

@@ -263,6 +263,75 @@ set_config(
     depends_if("--enable-aot-ics-enforce")(lambda _: True),
 )
 
+# NightMonkey, the AOT compilation tier (wasm32 shell targets only): links
+# the night runtime (js/src/night/runtime/) into the shell, supports wizer
+# snapshot registration, and builds the `nightmonkey` host binary (the
+# snapshot AOT compiler).
+option(
+    "--enable-nightmonkey",
+    default=False,
+    help="{Enable|Disable} the NightMonkey AOT tier (wasm32 targets only)",
+)
+
+
+@depends("--enable-nightmonkey", target)
+def nightmonkey(value, target):
+    if value:
+        if target.cpu != "wasm32":
+            die(
+                "--enable-nightmonkey requires a wasm32 target (e.g. --target=wasm32-unknown-wasi)"
+            )
+        return True
+
+
+set_config("ENABLE_JS_NIGHTMONKEY", nightmonkey)
+set_define("ENABLE_JS_NIGHTMONKEY", nightmonkey)
+
+# NightMonkey in-process compilation (test-only): compiles scripts inside
+# the running wasm shell (--night-inprocess) and injects the compiled
+# bodies through the wasm-jit-runner hostcalls, whose imports it adds to
+# the shell module; also builds the wasm-jit-runner host binary.
+option(
+    "--enable-nightmonkey-inprocess",
+    default=False,
+    help="{Enable|Disable} NightMonkey in-process compilation "
+    "(requires --enable-nightmonkey)",
+)
+
+
+@depends("--enable-nightmonkey-inprocess", nightmonkey)
+def nightmonkey_inprocess(value, nightmonkey):
+    if value:
+        if not nightmonkey:
+            die("--enable-nightmonkey-inprocess requires --enable-nightmonkey")
+        return True
+
+
+set_config("ENABLE_JS_NIGHTMONKEY_INPROCESS", nightmonkey_inprocess)
+set_define("ENABLE_JS_NIGHTMONKEY_INPROCESS", nightmonkey_inprocess)
+
+# NightMonkey runtime diagnostics: opt-in summaries and crash-on-failure in
+# the in-process test lane. Off even in debug builds, because crashing on a
+# failed in-process batch is right for a test lane and wrong for a developer
+# who merely ran the shell without the runner.
+option(
+    "--enable-nightmonkey-debug",
+    default=False,
+    help="{Enable|Disable} NightMonkey runtime diagnostics "
+    "(requires --enable-nightmonkey)",
+)
+
+
+@depends("--enable-nightmonkey-debug", nightmonkey)
+def nightmonkey_debug(value, nightmonkey):
+    if value:
+        if not nightmonkey:
+            die("--enable-nightmonkey-debug requires --enable-nightmonkey")
+        return True
+
+
+set_config("NIGHTMONKEY_DEBUG", nightmonkey_debug)
+set_define("NIGHTMONKEY_DEBUG", nightmonkey_debug)
 
 # Enable JS Streams
 # ===================================================