esm: add experimental support for addon modules

PR-URL: #55844
Backport-PR-URL: #59961Fixes: #40541Fixes: #55821
Reviewed-By: Guy Bedford <guybedford@gmail.com>
Reviewed-By: Geoffrey Booth <webadmin@geoffreybooth.com>
Reviewed-By: Joyee Cheung <joyeec9h3@gmail.com>
Reviewed-By: Jacob Smith <jacob@frende.me>
Reviewed-By: Antoine du Hamel <duhamelantoine1995@gmail.com>

Author: legendecas

doc/api/cli.md

@@ -49,7 +49,10 @@ Otherwise, the file is loaded using the CommonJS module loader. See
 When loading, the [ES module loader][Modules loaders] loads the program
 entry point, the `node` command will accept as input only files with `.js`,
 `.mjs`, `.cjs` or `.wasm` extensions; and with no extension when
-[`--experimental-default-type=module`][] is passed.
+[`--experimental-default-type=module`][] is passed. With the following flags,
+additional file extensions are enabled:
+
+*[`--experimental-addon-modules`][] for files with `.node` extension.
 
 ## Options
 
@@ -895,6 +898,16 @@ and `"` are usable.
 It is possible to run code containing inline types unless the
 [`--no-experimental-strip-types`][] flag is provided.
 
+### `--experimental-addon-modules`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+> Stability: 1.0 - Early development
+
+Enable experimental import support for `.node` addons.
+
 ### `--experimental-async-context-frame`
 
 <!-- YAML
@@ -3330,6 +3343,7 @@ one is included in the list below.
 *`--enable-source-maps`
 *`--entry-url`
 *`--experimental-abortcontroller`
+*`--experimental-addon-modules`
 *`--experimental-async-context-frame`
 *`--experimental-default-type`
 *`--experimental-detect-module`
@@ -3926,6 +3940,7 @@ node --stack-trace-limit=12 -p -e "Error.stackTraceLimit" # prints 12
 [`--disable-sigusr1`]: #--disable-sigusr1
 [`--env-file-if-exists`]: #--env-file-if-existsfile
 [`--env-file`]: #--env-filefile
+[`--experimental-addon-modules`]: #--experimental-addon-modules
 [`--experimental-default-type=module`]: #--experimental-default-typetype
 [`--experimental-sea-config`]: single-executable-applications.md#generating-single-executable-preparation-blobs
 [`--heap-prof-dir`]: #--heap-prof-dir

doc/api/esm.md

@@ -1129,18 +1129,21 @@ _isImports_, _conditions_)
 > 5. If _url_ ends in
 > _".wasm"_, then
 > 1. Return _"wasm"_.
-> 6. Let _packageURL_ be the result of **LOOKUP\_PACKAGE\_SCOPE**(_url_).
-> 7. Let _pjson_ be the result of **READ\_PACKAGE\_JSON**(_packageURL_).
-> 8. Let _packageType_ be **null**.
-> 9. If _pjson?.type_ is _"module"_ or _"commonjs"_, then
-> 1. Set _packageType_ to _pjson.type_.
-> 10. If _url_ ends in _".js"_, then
+> 6. If `--experimental-addon-modules` is enabled and _url_ ends in
+> _".node"_, then
+> 1. Return _"addon"_.
+> 7. Let _packageURL_ be the result of **LOOKUP\_PACKAGE\_SCOPE**(_url_).
+> 8. Let _pjson_ be the result of **READ\_PACKAGE\_JSON**(_packageURL_).
+> 9. Let _packageType_ be **null**.
+> 10. If _pjson?.type_ is _"module"_ or _"commonjs"_, then
+> 1. Set _packageType_ to _pjson.type_.
+> 11. If _url_ ends in _".js"_, then
 > 1. If _packageType_ is not **null**, then
 > 1. Return _packageType_.
 > 2. If the result of **DETECT\_MODULE\_SYNTAX**(_source_) is true, then
 > 1. Return _"module"_.
 > 3. Return _"commonjs"_.
-> 11. If _url_ does not have any extension, then
+> 12. If _url_ does not have any extension, then
 > 1. If _packageType_ is _"module"_ and the file at _url_ contains the
 > header for a WebAssembly module, then
 > 1. Return _"wasm"_.
@@ -1149,7 +1152,7 @@ _isImports_, _conditions_)
 > 3. If the result of **DETECT\_MODULE\_SYNTAX**(_source_) is true, then
 > 1. Return _"module"_.
 > 4. Return _"commonjs"_.
-> 12. Return **undefined** (will throw during load phase).
+> 13. Return **undefined** (will throw during load phase).
 
 **LOOKUP\_PACKAGE\_SCOPE**(_url_)
 

doc/api/module.md

@@ -1326,6 +1326,7 @@ The final value of `format` must be one of the following:
 
 | `format` | Description | Acceptable types for `source` returned by `load` |
 | ----------------------- | ----------------------------------------------------- | -------------------------------------------------- |
+| `'addon'` | Load a Node.js addon | {null} |
 | `'builtin'` | Load a Node.js builtin module | {null} |
 | `'commonjs-typescript'` | Load a Node.js CommonJS module with TypeScript syntax | {string\|ArrayBuffer\|TypedArray\|null\|undefined} |
 | `'commonjs'` | Load a Node.js CommonJS module | {string\|ArrayBuffer\|TypedArray\|null\|undefined} |

doc/node-config-schema.json

@@ -121,6 +121,9 @@
 "entry-url": {
 "type": "boolean"
  },
+"experimental-addon-modules": {
+"type": "boolean"
+ },
 "experimental-async-context-frame": {
 "type": "boolean"
  },

doc/node.1

@@ -163,6 +163,9 @@ Enable Source Map V3 support for stack traces.
 .ItFl-entry-url
 Interpret the entry point as a URL.
 .
+.ItFl-experimental-addon-modules
+Enable experimental addon module support.
+.
 .ItFl-experimental-default-typeNs=NsArtype
 Interpret as either ES modules or CommonJS modules input via --eval or STDIN, when --input-type is unspecified;
 .jsorextensionlessfileswithnosiblingorparentpackage.json;

lib/internal/modules/esm/formats.js

@@ -18,6 +18,9 @@ const extensionFormatMap = {
 '.wasm': 'wasm',
 };
 
+if(getOptionValue('--experimental-addon-modules')){
+extensionFormatMap['.node']='addon';
+}
 if(getOptionValue('--experimental-strip-types')){
 extensionFormatMap['.ts']='module-typescript';
 extensionFormatMap['.mts']='module-typescript';

lib/internal/modules/esm/load.js

@@ -84,6 +84,9 @@ function defaultLoad(url, context = kEmptyObject) {
 if(urlInstance.protocol==='node:'){
 source=null;
 format??='builtin';
+}elseif(format==='addon'){
+// Skip loading addon file content. It must be loaded with dlopen from file system.
+source=null;
 }elseif(format!=='commonjs'||defaultType==='module'){
 if(source==null){
 ({ responseURL, source }=getSourceSync(urlInstance,context));

lib/internal/modules/esm/translators.js

@@ -2,7 +2,6 @@
 
 const{
  ArrayPrototypePush,
- Boolean,
  FunctionPrototypeCall,
  JSONParse,
  ObjectAssign,
@@ -52,6 +51,7 @@ let debug = require('internal/util/debuglog').debuglog('esm', (fn) => {
 });
 const{ emitExperimentalWarning, kEmptyObject, setOwnProperty, isWindows }=require('internal/util');
 const{
+ERR_INVALID_RETURN_PROPERTY_VALUE,
 ERR_UNKNOWN_BUILTIN_MODULE,
 }=require('internal/errors').codes;
 const{ maybeCacheSourceMap }=require('internal/source_map/source_map_cache');
@@ -186,7 +186,7 @@ function createCJSModuleWrap(url, source, isMain, format, loadCJS = loadCJSModul
 // In case the source was not provided by the `load` step, we need fetch it now.
 source=stringify(source??getSource(newURL(url)).source);
 
-const{ exportNames, module }=cjsPreparseModuleExports(filename,source,isMain,format);
+const{ exportNames, module }=cjsPreparseModuleExports(filename,source,format);
 cjsCache.set(url,module);
 constnamesWithDefault=exportNames.has('default') ?
 [...exportNames] : ['default', ...exportNames];
@@ -227,6 +227,47 @@ function createCJSModuleWrap(url, source, isMain, format, loadCJS = loadCJSModul
 },module);
 }
 
+/**
+ * Creates a ModuleWrap object for a CommonJS module without source texts.
+ * @param {string} url - The URL of the module.
+ * @param {boolean} isMain - Whether the module is the main module.
+ * @returns {ModuleWrap} The ModuleWrap object for the CommonJS module.
+ */
+functioncreateCJSNoSourceModuleWrap(url,isMain){
+debug(`Translating CJSModule without source ${url}`);
+
+constfilename=urlToFilename(url);
+
+constmodule=cjsEmplaceModuleCacheEntry(filename);
+cjsCache.set(url,module);
+
+if(isMain){
+setOwnProperty(process,'mainModule',module);
+}
+
+// Addon export names are not known until the addon is loaded.
+constexportNames=['default','module.exports'];
+returnnewModuleWrap(url,undefined,exportNames,functionevaluationCallback(){
+debug(`Loading CJSModule ${url}`);
+
+if(!module.loaded){
+wrapModuleLoad(filename,null,isMain);
+}
+
+/** @type {import('./loader').ModuleExports} */
+letexports;
+if(module[kModuleExport]!==undefined){
+exports=module[kModuleExport];
+module[kModuleExport]=undefined;
+}else{
+({ exports }=module);
+}
+
+this.setExport('default',exports);
+this.setExport('module.exports',exports);
+},module);
+}
+
 translators.set('commonjs-sync',functionrequireCommonJS(url,source,isMain){
 initCJSParseSync();
 
@@ -277,26 +318,38 @@ translators.set('commonjs', function commonjsStrategy(url, source, isMain) {
 returncreateCJSModuleWrap(url,source,isMain,'commonjs',cjsLoader);
 });
 
+/**
+ * Get or create an entry in the CJS module cache for the given filename.
+ * @param {string} filename CJS module filename
+ * @returns {CJSModule} the cached CJS module entry
+ */
+functioncjsEmplaceModuleCacheEntry(filename,exportNames){
+// TODO: Do we want to keep hitting the user mutable CJS loader here?
+letcjsMod=CJSModule._cache[filename];
+if(cjsMod){
+returncjsMod;
+}
+
+cjsMod=newCJSModule(filename);
+cjsMod.filename=filename;
+cjsMod.paths=CJSModule._nodeModulePaths(cjsMod.path);
+cjsMod[kIsCachedByESMLoader]=true;
+CJSModule._cache[filename]=cjsMod;
+
+returncjsMod;
+}
+
 /**
  * Pre-parses a CommonJS module's exports and re-exports.
  * @param {string} filename - The filename of the module.
  * @param {string} [source] - The source code of the module.
- * @param {boolean} isMain - Whether it is pre-parsing for the entry point.
- * @param {string} format
+ * @param {string} [format]
  */
-functioncjsPreparseModuleExports(filename,source,isMain,format){
-letmodule=CJSModule._cache[filename];
-if(module&&module[kModuleExportNames]!==undefined){
+functioncjsPreparseModuleExports(filename,source,format){
+constmodule=cjsEmplaceModuleCacheEntry(filename);
+if(module[kModuleExportNames]!==undefined){
 return{ module,exportNames: module[kModuleExportNames]};
 }
-constloaded=Boolean(module);
-if(!loaded){
-module=newCJSModule(filename);
-module.filename=filename;
-module.paths=CJSModule._nodeModulePaths(module.path);
-module[kIsCachedByESMLoader]=true;
-CJSModule._cache[filename]=module;
-}
 
 if(source===undefined){
 ({ source }=loadSourceForCJSWithHooks(module,filename,format));
@@ -337,7 +390,7 @@ function cjsPreparseModuleExports(filename, source, isMain, format) {
 
 if(format==='commonjs'||
 (!BuiltinModule.normalizeRequirableId(resolved)&&findLongestRegisteredExtension(resolved)==='.js')){
-const{exportNames: reexportNames}=cjsPreparseModuleExports(resolved,undefined,false,format);
+const{exportNames: reexportNames}=cjsPreparseModuleExports(resolved,undefined,format);
 for(constnameofreexportNames){
 exportNames.add(name);
 }
@@ -519,6 +572,25 @@ translators.set('wasm', function(url, source) {
 returnmodule;
 });
 
+// Strategy for loading a addon
+translators.set('addon',functiontranslateAddon(url,source,isMain){
+emitExperimentalWarning('Importing addons');
+
+// The addon must be loaded from file system with dlopen. Assert
+// the source is null.
+if(source!==null){
+thrownewERR_INVALID_RETURN_PROPERTY_VALUE(
+'null',
+'load',
+'source',
+source);
+}
+
+debug(`Translating addon ${url}`);
+
+returncreateCJSNoSourceModuleWrap(url,isMain);
+});
+
 // Strategy for loading a commonjs TypeScript module
 translators.set('commonjs-typescript',function(url,source,isMain){
 assertBufferSource(source,true,'load');

src/node_options.cc

@@ -424,6 +424,10 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() {
 "Treat the entrypoint as a URL",
  &EnvironmentOptions::entry_is_url,
 kAllowedInEnvvar);
+AddOption("--experimental-addon-modules",
+"experimental import support for addons",
+ &EnvironmentOptions::experimental_addon_modules,
+kAllowedInEnvvar);
 AddOption("--experimental-abortcontroller", "", NoOp{}, kAllowedInEnvvar);
 AddOption("--experimental-eventsource",
 "experimental EventSource API",

src/node_options.h

@@ -121,6 +121,7 @@ class EnvironmentOptions : public Options {
 bool require_module = true;
  std::string dns_result_order;
 bool enable_source_maps = false;
+bool experimental_addon_modules = false;
 bool experimental_eventsource = false;
 bool experimental_fetch = true;
 bool experimental_websocket = true;