esm: add experimental support for addon modules

PR-URL: #55844Fixes: #40541Fixes: #55821
Reviewed-By: Guy Bedford <guybedford@gmail.com>
Reviewed-By: Geoffrey Booth <webadmin@geoffreybooth.com>
Reviewed-By: Joyee Cheung <joyeec9h3@gmail.com>
Reviewed-By: Jacob Smith <jacob@frende.me>
Reviewed-By: Antoine du Hamel <duhamelantoine1995@gmail.com>

Author: legendecas

doc/api/cli.md

@@ -45,8 +45,11 @@ Otherwise, the file is loaded using the CommonJS module loader. See
 
 When loading, the [ES module loader][Modules loaders] loads the program
 entry point, the `node` command will accept as input only files with `.js`,
-`.mjs`, or `.cjs` extensions; and with `.wasm` extensions when
-[`--experimental-wasm-modules`][] is enabled.
+`.mjs`, or `.cjs` extensions. With the following flags, additional file
+extensions are enabled:
+
+*[`--experimental-wasm-modules`][] for files with `.wasm` extension.
+*[`--experimental-addon-modules`][] for files with `.node` extension.
 
 ## Options
 
@@ -879,6 +882,16 @@ and `"` are usable.
 It is possible to run code containing inline types by passing
 [`--experimental-strip-types`][].
 
+### `--experimental-addon-modules`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+> Stability: 1.0 - Early development
+
+Enable experimental import support for `.node` addons.
+
 ### `--experimental-async-context-frame`
 
 <!-- YAML
@@ -3046,6 +3059,7 @@ one is included in the list below.
 *`--enable-source-maps`
 *`--entry-url`
 *`--experimental-abortcontroller`
+*`--experimental-addon-modules`
 *`--experimental-async-context-frame`
 *`--experimental-detect-module`
 *`--experimental-eventsource`
@@ -3617,6 +3631,7 @@ node --stack-trace-limit=12 -p -e "Error.stackTraceLimit" # prints 12
 [`--diagnostic-dir`]: #--diagnostic-dirdirectory
 [`--env-file-if-exists`]: #--env-file-if-existsconfig
 [`--env-file`]: #--env-fileconfig
+[`--experimental-addon-modules`]: #--experimental-addon-modules
 [`--experimental-sea-config`]: single-executable-applications.md#generating-single-executable-preparation-blobs
 [`--experimental-strip-types`]: #--experimental-strip-types
 [`--experimental-wasm-modules`]: #--experimental-wasm-modules

doc/api/esm.md

@@ -1039,18 +1039,21 @@ _isImports_, _conditions_)
 > 5. If `--experimental-wasm-modules` is enabled and _url_ ends in
 > _".wasm"_, then
 > 1. Return _"wasm"_.
-> 6. Let _packageURL_ be the result of **LOOKUP\_PACKAGE\_SCOPE**(_url_).
-> 7. Let _pjson_ be the result of **READ\_PACKAGE\_JSON**(_packageURL_).
-> 8. Let _packageType_ be **null**.
-> 9. If _pjson?.type_ is _"module"_ or _"commonjs"_, then
-> 1. Set _packageType_ to _pjson.type_.
-> 10. If _url_ ends in _".js"_, then
+> 6. If `--experimental-addon-modules` is enabled and _url_ ends in
+> _".node"_, then
+> 1. Return _"addon"_.
+> 7. Let _packageURL_ be the result of **LOOKUP\_PACKAGE\_SCOPE**(_url_).
+> 8. Let _pjson_ be the result of **READ\_PACKAGE\_JSON**(_packageURL_).
+> 9. Let _packageType_ be **null**.
+> 10. If _pjson?.type_ is _"module"_ or _"commonjs"_, then
+> 1. Set _packageType_ to _pjson.type_.
+> 11. If _url_ ends in _".js"_, then
 > 1. If _packageType_ is not **null**, then
 > 1. Return _packageType_.
 > 2. If the result of **DETECT\_MODULE\_SYNTAX**(_source_) is true, then
 > 1. Return _"module"_.
 > 3. Return _"commonjs"_.
-> 11. If _url_ does not have any extension, then
+> 12. If _url_ does not have any extension, then
 > 1. If _packageType_ is _"module"_ and `--experimental-wasm-modules` is
 > enabled and the file at _url_ contains the header for a WebAssembly
 > module, then
@@ -1060,7 +1063,7 @@ _isImports_, _conditions_)
 > 3. If the result of **DETECT\_MODULE\_SYNTAX**(_source_) is true, then
 > 1. Return _"module"_.
 > 4. Return _"commonjs"_.
-> 12. Return **undefined** (will throw during load phase).
+> 13. Return **undefined** (will throw during load phase).
 
 **LOOKUP\_PACKAGE\_SCOPE**(_url_)
 

doc/api/module.md

@@ -1160,6 +1160,7 @@ The final value of `format` must be one of the following:
 
 | `format` | Description | Acceptable types for `source` returned by `load` |
 | ------------ | ------------------------------ | -------------------------------------------------- |
+| `'addon'` | Load a Node.js addon | {null} |
 | `'builtin'` | Load a Node.js builtin module | {null} |
 | `'commonjs'` | Load a Node.js CommonJS module | {string\|ArrayBuffer\|TypedArray\|null\|undefined} |
 | `'json'` | Load a JSON file | {string\|ArrayBuffer\|TypedArray} |

doc/node.1

@@ -163,6 +163,9 @@ Enable Source Map V3 support for stack traces.
 .ItFl-entry-url
 Interpret the entry point as a URL.
 .
+.ItFl-experimental-addon-modules
+Enable experimental addon module support.
+.
 .ItFl-experimental-import-meta-resolve
 Enable experimental ES modules support for import.meta.resolve().
 .

lib/internal/modules/esm/formats.js

@@ -10,6 +10,7 @@ const fsBindings = internalBinding('fs');
 const{fs: fsConstants}=internalBinding('constants');
 
 constexperimentalWasmModules=getOptionValue('--experimental-wasm-modules');
+constexperimentalAddonModules=getOptionValue('--experimental-addon-modules');
 
 constextensionFormatMap={
 '__proto__': null,
@@ -23,6 +24,10 @@ if (experimentalWasmModules) {
 extensionFormatMap['.wasm']='wasm';
 }
 
+if(experimentalAddonModules){
+extensionFormatMap['.node']='addon';
+}
+
 if(getOptionValue('--experimental-strip-types')){
 extensionFormatMap['.ts']='module-typescript';
 extensionFormatMap['.mts']='module-typescript';

lib/internal/modules/esm/load.js

@@ -105,6 +105,9 @@ async function defaultLoad(url, context = kEmptyObject) {
 if(urlInstance.protocol==='node:'){
 source=null;
 format??='builtin';
+}elseif(format==='addon'){
+// Skip loading addon file content. It must be loaded with dlopen from file system.
+source=null;
 }elseif(format!=='commonjs'){
 if(source==null){
 ({ responseURL, source }=awaitgetSource(urlInstance,context));

lib/internal/modules/esm/translators.js

@@ -3,7 +3,6 @@
 const{
  ArrayPrototypeMap,
  ArrayPrototypePush,
- Boolean,
  FunctionPrototypeCall,
  JSONParse,
  ObjectKeys,
@@ -52,6 +51,7 @@ let debug = require('internal/util/debuglog').debuglog('esm', (fn) => {
 });
 const{ emitExperimentalWarning, kEmptyObject, setOwnProperty, isWindows }=require('internal/util');
 const{
+ERR_INVALID_RETURN_PROPERTY_VALUE,
 ERR_UNKNOWN_BUILTIN_MODULE,
 }=require('internal/errors').codes;
 const{ maybeCacheSourceMap }=require('internal/source_map/source_map_cache');
@@ -185,7 +185,7 @@ function createCJSModuleWrap(url, source, isMain, format, loadCJS = loadCJSModul
 // In case the source was not provided by the `load` step, we need fetch it now.
 source=stringify(source??getSource(newURL(url)).source);
 
-const{ exportNames, module }=cjsPreparseModuleExports(filename,source,isMain,format);
+const{ exportNames, module }=cjsPreparseModuleExports(filename,source,format);
 cjsCache.set(url,module);
 
 constwrapperNames=[...exportNames,'module.exports'];
@@ -229,6 +229,47 @@ function createCJSModuleWrap(url, source, isMain, format, loadCJS = loadCJSModul
 },module);
 }
 
+/**
+ * Creates a ModuleWrap object for a CommonJS module without source texts.
+ * @param {string} url - The URL of the module.
+ * @param {boolean} isMain - Whether the module is the main module.
+ * @returns {ModuleWrap} The ModuleWrap object for the CommonJS module.
+ */
+functioncreateCJSNoSourceModuleWrap(url,isMain){
+debug(`Translating CJSModule without source ${url}`);
+
+constfilename=urlToFilename(url);
+
+constmodule=cjsEmplaceModuleCacheEntry(filename);
+cjsCache.set(url,module);
+
+if(isMain){
+setOwnProperty(process,'mainModule',module);
+}
+
+// Addon export names are not known until the addon is loaded.
+constexportNames=['default','module.exports'];
+returnnewModuleWrap(url,undefined,exportNames,functionevaluationCallback(){
+debug(`Loading CJSModule ${url}`);
+
+if(!module.loaded){
+wrapModuleLoad(filename,null,isMain);
+}
+
+/** @type {import('./loader').ModuleExports} */
+letexports;
+if(module[kModuleExport]!==undefined){
+exports=module[kModuleExport];
+module[kModuleExport]=undefined;
+}else{
+({ exports }=module);
+}
+
+this.setExport('default',exports);
+this.setExport('module.exports',exports);
+},module);
+}
+
 translators.set('commonjs-sync',functionrequireCommonJS(url,source,isMain){
 initCJSParseSync();
 
@@ -280,26 +321,38 @@ translators.set('commonjs', function commonjsStrategy(url, source, isMain) {
 returncreateCJSModuleWrap(url,source,isMain,'commonjs',cjsLoader);
 });
 
+/**
+ * Get or create an entry in the CJS module cache for the given filename.
+ * @param {string} filename CJS module filename
+ * @returns {CJSModule} the cached CJS module entry
+ */
+functioncjsEmplaceModuleCacheEntry(filename,exportNames){
+// TODO: Do we want to keep hitting the user mutable CJS loader here?
+letcjsMod=CJSModule._cache[filename];
+if(cjsMod){
+returncjsMod;
+}
+
+cjsMod=newCJSModule(filename);
+cjsMod.filename=filename;
+cjsMod.paths=CJSModule._nodeModulePaths(cjsMod.path);
+cjsMod[kIsCachedByESMLoader]=true;
+CJSModule._cache[filename]=cjsMod;
+
+returncjsMod;
+}
+
 /**
  * Pre-parses a CommonJS module's exports and re-exports.
  * @param {string} filename - The filename of the module.
  * @param {string} [source] - The source code of the module.
- * @param {boolean} isMain - Whether it is pre-parsing for the entry point.
- * @param {string} format
+ * @param {string} [format]
  */
-functioncjsPreparseModuleExports(filename,source,isMain,format){
-letmodule=CJSModule._cache[filename];
-if(module&&module[kModuleExportNames]!==undefined){
+functioncjsPreparseModuleExports(filename,source,format){
+constmodule=cjsEmplaceModuleCacheEntry(filename);
+if(module[kModuleExportNames]!==undefined){
 return{ module,exportNames: module[kModuleExportNames]};
 }
-constloaded=Boolean(module);
-if(!loaded){
-module=newCJSModule(filename);
-module.filename=filename;
-module.paths=CJSModule._nodeModulePaths(module.path);
-module[kIsCachedByESMLoader]=true;
-CJSModule._cache[filename]=module;
-}
 
 if(source===undefined){
 ({ source }=loadSourceForCJSWithHooks(module,filename,format));
@@ -340,7 +393,7 @@ function cjsPreparseModuleExports(filename, source, isMain, format) {
 
 if(format==='commonjs'||
 (!BuiltinModule.normalizeRequirableId(resolved)&&findLongestRegisteredExtension(resolved)==='.js')){
-const{exportNames: reexportNames}=cjsPreparseModuleExports(resolved,undefined,false,format);
+const{exportNames: reexportNames}=cjsPreparseModuleExports(resolved,undefined,format);
 for(constnameofreexportNames){
 exportNames.add(name);
 }
@@ -462,6 +515,25 @@ translators.set('wasm', async function(url, source) {
 }).module;
 });
 
+// Strategy for loading a addon
+translators.set('addon',functiontranslateAddon(url,source,isMain){
+emitExperimentalWarning('Importing addons');
+
+// The addon must be loaded from file system with dlopen. Assert
+// the source is null.
+if(source!==null){
+thrownewERR_INVALID_RETURN_PROPERTY_VALUE(
+'null',
+'load',
+'source',
+source);
+}
+
+debug(`Translating addon ${url}`);
+
+returncreateCJSNoSourceModuleWrap(url,isMain);
+});
+
 // Strategy for loading a commonjs TypeScript module
 translators.set('commonjs-typescript',function(url,source){
 emitExperimentalWarning('Type Stripping');

src/node_options.cc

@@ -409,6 +409,10 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() {
 "Treat the entrypoint as a URL",
  &EnvironmentOptions::entry_is_url,
 kAllowedInEnvvar);
+AddOption("--experimental-addon-modules",
+"experimental import support for addons",
+ &EnvironmentOptions::experimental_addon_modules,
+kAllowedInEnvvar);
 AddOption("--experimental-abortcontroller", "", NoOp{}, kAllowedInEnvvar);
 AddOption("--experimental-eventsource",
 "experimental EventSource API",

src/node_options.h

@@ -120,6 +120,7 @@ class EnvironmentOptions : public Options {
 bool require_module = true;
  std::string dns_result_order;
 bool enable_source_maps = false;
+bool experimental_addon_modules = false;
 bool experimental_eventsource = false;
 bool experimental_fetch = true;
 bool experimental_websocket = true;

test/addons/.gitignore

@@ -5,3 +5,4 @@ Makefile
 *.mk
 gyp-mac-tool
 /*/build
+/esm/node_modules/*/build