module: support main w/o extension, pjson cache

This adds support for ensuring that the top-level main into Node is
supported loading when it has no extension for backwards-compat with
NodeJS bin workflows.
In addition package.json caching is implemented in the module lookup
process.
Backport-PR-URL: #18923
PR-URL: #18728
Reviewed-By: Benjamin Gruenbaum <benjamingr@gmail.com>

Author: guybedford

doc/api/esm.md

@@ -113,17 +113,22 @@ The resolve hook returns the resolved file URL and module format for a
 given module specifier and parent file URL:
 
 ```js
-importurlfrom'url';
+constbaseURL=newURL('file://');
+baseURL.pathname=process.cwd() +'/';
 
-exportasyncfunctionresolve(specifier, parentModuleURL, defaultResolver) {
+exportasyncfunctionresolve(specifier,
+parentModuleURL=baseURL,
+defaultResolver) {
 return {
  url:newURL(specifier, parentModuleURL).href,
  format:'esm'
  };
 }
 ```
 
-The default NodeJS ES module resolution function is provided as a third
+The parentURL is provided as `undefined` when performing main Node.js load itself.
+
+The default Node.js ES module resolution function is provided as a third
 argument to the resolver for easy compatibility workflows.
 
 In addition to returning the resolved file URL value, the resolve hook also
@@ -152,7 +157,10 @@ import Module from 'module';
 constbuiltins=Module.builtinModules;
 constJS_EXTENSIONS=newSet(['.js', '.mjs']);
 
-exportfunctionresolve(specifier, parentModuleURL/*, defaultResolve */) {
+constbaseURL=newURL('file://');
+baseURL.pathname=process.cwd() +'/';
+
+exportfunctionresolve(specifier, parentModuleURL=baseURL, defaultResolve) {
 if (builtins.includes(specifier)) {
 return {
  url: specifier,

lib/internal/bootstrap_node.js

@@ -105,6 +105,7 @@
 process.emitWarning(
 'The ESM module loader is experimental.',
 'ExperimentalWarning',undefined);
+NativeModule.require('internal/process/modules').setup();
 }
 
 

lib/internal/loader/DefaultResolve.js

@@ -2,7 +2,6 @@
 
 const{URL}=require('url');
 constCJSmodule=require('module');
-constinternalURLModule=require('internal/url');
 constinternalFS=require('internal/fs');
 constNativeModule=require('native_module');
 const{ extname }=require('path');
@@ -11,6 +10,7 @@ const preserveSymlinks = !!process.binding('config').preserveSymlinks;
 consterrors=require('internal/errors');
 const{resolve: moduleWrapResolve}=internalBinding('module_wrap');
 constStringStartsWith=Function.call.bind(String.prototype.startsWith);
+const{ getURLFromFilePath, getPathFromURL }=require('internal/url');
 
 constrealpathCache=newMap();
 
@@ -57,7 +57,8 @@ function resolve(specifier, parentURL) {
 
 leturl;
 try{
-url=search(specifier,parentURL);
+url=search(specifier,
+parentURL||getURLFromFilePath(`${process.cwd()}/`).href);
 }catch(e){
 if(typeofe.message==='string'&&
 StringStartsWith(e.message,'Cannot find module'))
@@ -66,17 +67,27 @@ function resolve(specifier, parentURL) {
 }
 
 if(!preserveSymlinks){
-constreal=realpathSync(internalURLModule.getPathFromURL(url),{
+constreal=realpathSync(getPathFromURL(url),{
 [internalFS.realpathCacheKey]: realpathCache
 });
 constold=url;
-url=internalURLModule.getURLFromFilePath(real);
+url=getURLFromFilePath(real);
 url.search=old.search;
 url.hash=old.hash;
 }
 
 constext=extname(url.pathname);
-return{url: `${url}`,format: extensionFormatMap[ext]||ext};
+
+letformat=extensionFormatMap[ext];
+if(!format){
+constisMain=parentURL===undefined;
+if(isMain)
+format='cjs';
+else
+thrownewerrors.Error('ERR_UNKNOWN_FILE_EXTENSION',url.pathname);
+}
+
+return{url: `${url}`, format };
 }
 
 module.exports=resolve;

lib/internal/loader/Loader.js

@@ -1,51 +1,21 @@
 'use strict';
 
-constpath=require('path');
-const{ getURLFromFilePath,URL}=require('internal/url');
 consterrors=require('internal/errors');
-
 constModuleMap=require('internal/loader/ModuleMap');
 constModuleJob=require('internal/loader/ModuleJob');
 constdefaultResolve=require('internal/loader/DefaultResolve');
 constcreateDynamicModule=require('internal/loader/CreateDynamicModule');
 consttranslators=require('internal/loader/Translators');
-const{ setImportModuleDynamicallyCallback }=internalBinding('module_wrap');
+
 constFunctionBind=Function.call.bind(Function.prototype.bind);
 
 constdebug=require('util').debuglog('esm');
 
-// Returns a file URL for the current working directory.
-functiongetURLStringForCwd(){
-try{
-returngetURLFromFilePath(`${process.cwd()}/`).href;
-}catch(e){
-e.stack;
-// If the current working directory no longer exists.
-if(e.code==='ENOENT'){
-returnundefined;
-}
-throwe;
-}
-}
-
-functionnormalizeReferrerURL(referrer){
-if(typeofreferrer==='string'&&path.isAbsolute(referrer)){
-returngetURLFromFilePath(referrer).href;
-}
-returnnewURL(referrer).href;
-}
-
 /* A Loader instance is used as the main entry point for loading ES modules.
  * Currently, this is a singleton -- there is only one used for loading
  * the main module and everything in its dependency graph. */
 classLoader{
-constructor(base=getURLStringForCwd()){
-if(typeofbase!=='string')
-thrownewerrors.TypeError('ERR_INVALID_ARG_TYPE','base','string');
-
-this.base=base;
-this.isMain=true;
-
+constructor(){
 // methods which translate input code or other information
 // into es modules
 this.translators=translators;
@@ -71,8 +41,9 @@ class Loader {
 this._dynamicInstantiate=undefined;
 }
 
-asyncresolve(specifier,parentURL=this.base){
-if(typeofparentURL!=='string')
+asyncresolve(specifier,parentURL){
+constisMain=parentURL===undefined;
+if(!isMain&&typeofparentURL!=='string')
 thrownewerrors.TypeError('ERR_INVALID_ARG_TYPE','parentURL','string');
 
 const{ url, format }=
@@ -93,7 +64,7 @@ class Loader {
 return{ url, format };
 }
 
-asyncimport(specifier,parent=this.base){
+asyncimport(specifier,parent){
 constjob=awaitthis.getModuleJob(specifier,parent);
 constmodule=awaitjob.run();
 returnmodule.namespace();
@@ -107,7 +78,7 @@ class Loader {
 this._dynamicInstantiate=FunctionBind(dynamicInstantiate,null);
 }
 
-asyncgetModuleJob(specifier,parentURL=this.base){
+asyncgetModuleJob(specifier,parentURL){
 const{ url, format }=awaitthis.resolve(specifier,parentURL);
 letjob=this.moduleMap.get(url);
 if(job!==undefined)
@@ -134,24 +105,16 @@ class Loader {
 }
 
 letinspectBrk=false;
-if(this.isMain){
-if(process._breakFirstLine){
-deleteprocess._breakFirstLine;
-inspectBrk=true;
-}
-this.isMain=false;
+if(process._breakFirstLine){
+deleteprocess._breakFirstLine;
+inspectBrk=true;
 }
 job=newModuleJob(this,url,loaderInstance,inspectBrk);
 this.moduleMap.set(url,job);
 returnjob;
 }
-
-staticregisterImportDynamicallyCallback(loader){
-setImportModuleDynamicallyCallback(async(referrer,specifier)=>{
-returnloader.import(specifier,normalizeReferrerURL(referrer));
-});
-}
 }
 
 Object.setPrototypeOf(Loader.prototype,null);
+
 module.exports=Loader;

lib/internal/loader/Translators.js

@@ -19,7 +19,7 @@ const JsonParse = JSON.parse;
 consttranslators=newSafeMap();
 module.exports=translators;
 
-// Stragety for loading a standard JavaScript module
+// Strategy for loading a standard JavaScript module
 translators.set('esm',async(url)=>{
 constsource=`${awaitreadFileAsync(newURL(url))}`;
 debug(`Translating StandardModule ${url}`);
@@ -62,7 +62,7 @@ translators.set('builtin', async (url) => {
 });
 });
 
-// Stragety for loading a node native module
+// Strategy for loading a node native module
 translators.set('addon',async(url)=>{
 debug(`Translating NativeModule ${url}`);
 returncreateDynamicModule(['default'],url,(reflect)=>{
@@ -74,7 +74,7 @@ translators.set('addon', async (url) => {
 });
 });
 
-// Stragety for loading a JSON file
+// Strategy for loading a JSON file
 translators.set('json',async(url)=>{
 debug(`Translating JSONModule ${url}`);
 returncreateDynamicModule(['default'],url,(reflect)=>{

lib/internal/process/modules.js

@@ -0,0 +1,47 @@
+'use strict';
+
+const{
+ setImportModuleDynamicallyCallback
+}=internalBinding('module_wrap');
+
+const{ getURLFromFilePath }=require('internal/url');
+constLoader=require('internal/loader/Loader');
+constpath=require('path');
+const{URL}=require('url');
+
+functionnormalizeReferrerURL(referrer){
+if(typeofreferrer==='string'&&path.isAbsolute(referrer)){
+returngetURLFromFilePath(referrer).href;
+}
+returnnewURL(referrer).href;
+}
+
+letloaderResolve;
+exports.loaderPromise=newPromise((resolve,reject)=>{
+loaderResolve=resolve;
+});
+
+exports.ESMLoader=undefined;
+
+exports.setup=function(){
+letESMLoader=newLoader();
+constloaderPromise=(async()=>{
+constuserLoader=process.binding('config').userLoader;
+if(userLoader){
+consthooks=awaitESMLoader.import(
+userLoader,getURLFromFilePath(`${process.cwd()}/`).href);
+ESMLoader=newLoader();
+ESMLoader.hook(hooks);
+exports.ESMLoader=ESMLoader;
+}
+returnESMLoader;
+})();
+loaderResolve(loaderPromise);
+
+setImportModuleDynamicallyCallback(async(referrer,specifier)=>{
+constloader=awaitloaderPromise;
+returnloader.import(specifier,normalizeReferrerURL(referrer));
+});
+
+exports.ESMLoader=ESMLoader;
+};

lib/module.js

@@ -24,7 +24,6 @@
 constNativeModule=require('native_module');
 constutil=require('util');
 const{ decorateErrorStack }=require('internal/util');
-constinternalModule=require('internal/module');
 const{ getURLFromFilePath }=require('internal/url');
 constvm=require('vm');
 constassert=require('assert').ok;
@@ -35,6 +34,7 @@ const {
  internalModuleReadFile,
  internalModuleStat
 }=process.binding('fs');
+constinternalModule=require('internal/module');
 constpreserveSymlinks=!!process.binding('config').preserveSymlinks;
 constexperimentalModules=!!process.binding('config').experimentalModules;
 
@@ -43,10 +43,9 @@ const errors = require('internal/errors');
 module.exports=Module;
 
 // these are below module.exports for the circular reference
-constLoader=require('internal/loader/Loader');
+constinternalESModule=require('internal/process/modules');
 constModuleJob=require('internal/loader/ModuleJob');
 constcreateDynamicModule=require('internal/loader/CreateDynamicModule');
-letESMLoader;
 
 functionstat(filename){
 filename=path.toNamespacedPath(filename);
@@ -444,7 +443,6 @@ Module._resolveLookupPaths = function(request, parent, newReturn) {
 return(newReturn ? parentDir : [id,parentDir]);
 };
 
-
 // Check the cache for the requested file.
 // 1. If a module already exists in the cache: return its exports object.
 // 2. If the module is native: call `NativeModule.require()` with the
@@ -457,22 +455,10 @@ Module._load = function(request, parent, isMain) {
 debug('Module._load REQUEST %s parent: %s',request,parent.id);
 }
 
-if(isMain&&experimentalModules){
-(async()=>{
-// loader setup
-if(!ESMLoader){
-ESMLoader=newLoader();
-constuserLoader=process.binding('config').userLoader;
-if(userLoader){
-ESMLoader.isMain=false;
-consthooks=awaitESMLoader.import(userLoader);
-ESMLoader=newLoader();
-ESMLoader.hook(hooks);
-}
-}
-Loader.registerImportDynamicallyCallback(ESMLoader);
-awaitESMLoader.import(getURLFromFilePath(request).pathname);
-})()
+if(experimentalModules&&isMain){
+internalESModule.loaderPromise.then((loader)=>{
+returnloader.import(getURLFromFilePath(request).pathname);
+})
 .catch((e)=>{
 decorateErrorStack(e);
 console.error(e);
@@ -575,7 +561,8 @@ Module.prototype.load = function(filename) {
 Module._extensions[extension](this,filename);
 this.loaded=true;
 
-if(ESMLoader){
+if(experimentalModules){
+constESMLoader=internalESModule.ESMLoader;
 consturl=getURLFromFilePath(filename);
 consturlString=`${url}`;
 constexports=this.exports;

node.gyp

@@ -115,6 +115,7 @@
 'lib/internal/net.js',
 'lib/internal/module.js',
 'lib/internal/os.js',
+'lib/internal/process/modules.js',
 'lib/internal/process/next_tick.js',
 'lib/internal/process/promises.js',
 'lib/internal/process/stdio.js',