build: pass directory instead of list of files to js2c.py

On Windows there is a limit to the length of commands, so there
will be an error once the lengths of the JS file names combined
exceed that limit. This patch modifies js2c.py so that
it now takes a --directory argument to glob for .js and
.mjs files in addition to the list of files passed directly.
We still pass the additional files we include from deps/
directly through the command line, as we only includes some of
them so we cannot simply glob, but those are limited so listing
them out should be fine.
Refs: https://docs.microsoft.com/en-us/troubleshoot/windows-client/shell-experience/command-line-string-limitation
PR-URL: #39069
Refs: #38971
Reviewed-By: Chengzhong Wu <legendecas@gmail.com>
Reviewed-By: James M Snell <jasnell@gmail.com>

Author: joyeecheung

node.gyp

@@ -27,251 +27,15 @@
 'node_lib_target_name%': 'libnode',
 'node_intermediate_lib_type%': 'static_library',
 'node_builtin_modules_path%': '',
+# We list the deps/ files out instead of globbing them in js2c.py since we
+# only include a subset of all the files under these directories.
+# The lengths of their file names combined should not exceed the
+# Windows command length limit or there would be an error.
+# See https://docs.microsoft.com/en-us/troubleshoot/windows-client/shell-experience/command-line-string-limitation
 'library_files': [
-'lib/internal/bootstrap/environment.js',
-'lib/internal/bootstrap/loaders.js',
-'lib/internal/bootstrap/node.js',
-'lib/internal/bootstrap/pre_execution.js',
-'lib/internal/bootstrap/switches/does_own_process_state.js',
-'lib/internal/bootstrap/switches/does_not_own_process_state.js',
-'lib/internal/bootstrap/switches/is_main_thread.js',
-'lib/internal/bootstrap/switches/is_not_main_thread.js',
-'lib/internal/per_context/primordials.js',
-'lib/internal/per_context/domexception.js',
-'lib/internal/per_context/messageport.js',
-'lib/async_hooks.js',
-'lib/assert.js',
-'lib/assert/strict.js',
-'lib/buffer.js',
-'lib/child_process.js',
-'lib/console.js',
-'lib/constants.js',
-'lib/crypto.js',
-'lib/cluster.js',
-'lib/diagnostics_channel.js',
-'lib/dgram.js',
-'lib/dns.js',
-'lib/dns/promises.js',
-'lib/domain.js',
-'lib/events.js',
-'lib/fs.js',
-'lib/fs/promises.js',
-'lib/http.js',
-'lib/http2.js',
-'lib/_http_agent.js',
-'lib/_http_client.js',
-'lib/_http_common.js',
-'lib/_http_incoming.js',
-'lib/_http_outgoing.js',
-'lib/_http_server.js',
-'lib/https.js',
-'lib/inspector.js',
-'lib/module.js',
-'lib/net.js',
-'lib/os.js',
-'lib/path.js',
-'lib/path/posix.js',
-'lib/path/win32.js',
-'lib/perf_hooks.js',
-'lib/process.js',
-'lib/punycode.js',
-'lib/querystring.js',
-'lib/readline.js',
-'lib/repl.js',
-'lib/stream.js',
-'lib/stream/promises.js',
-'lib/_stream_readable.js',
-'lib/_stream_writable.js',
-'lib/_stream_duplex.js',
-'lib/_stream_transform.js',
-'lib/_stream_passthrough.js',
-'lib/_stream_wrap.js',
-'lib/string_decoder.js',
-'lib/sys.js',
-'lib/timers/promises.js',
-'lib/timers.js',
-'lib/tls.js',
-'lib/_tls_common.js',
-'lib/_tls_wrap.js',
-'lib/trace_events.js',
-'lib/tty.js',
-'lib/url.js',
-'lib/util.js',
-'lib/util/types.js',
-'lib/v8.js',
-'lib/vm.js',
-'lib/wasi.js',
-'lib/worker_threads.js',
-'lib/zlib.js',
-'lib/internal/abort_controller.js',
-'lib/internal/assert.js',
-'lib/internal/assert/assertion_error.js',
-'lib/internal/assert/calltracker.js',
-'lib/internal/async_hooks.js',
-'lib/internal/blob.js',
-'lib/internal/blocklist.js',
-'lib/internal/buffer.js',
-'lib/internal/cli_table.js',
-'lib/internal/child_process.js',
-'lib/internal/child_process/serialization.js',
-'lib/internal/cluster/child.js',
-'lib/internal/cluster/primary.js',
-'lib/internal/cluster/round_robin_handle.js',
-'lib/internal/cluster/shared_handle.js',
-'lib/internal/cluster/utils.js',
-'lib/internal/cluster/worker.js',
-'lib/internal/console/constructor.js',
-'lib/internal/console/global.js',
-'lib/internal/crypto/aes.js',
-'lib/internal/crypto/certificate.js',
-'lib/internal/crypto/cipher.js',
-'lib/internal/crypto/diffiehellman.js',
-'lib/internal/crypto/dsa.js',
-'lib/internal/crypto/ec.js',
-'lib/internal/crypto/hash.js',
-'lib/internal/crypto/hashnames.js',
-'lib/internal/crypto/hkdf.js',
-'lib/internal/crypto/keygen.js',
-'lib/internal/crypto/keys.js',
-'lib/internal/crypto/mac.js',
-'lib/internal/crypto/pbkdf2.js',
-'lib/internal/crypto/random.js',
-'lib/internal/crypto/rsa.js',
-'lib/internal/crypto/scrypt.js',
-'lib/internal/crypto/sig.js',
-'lib/internal/crypto/util.js',
-'lib/internal/crypto/webcrypto.js',
-'lib/internal/crypto/x509.js',
-'lib/internal/constants.js',
-'lib/internal/debugger/_inspect.js',
-'lib/internal/debugger/inspect_client.js',
-'lib/internal/debugger/inspect_repl.js',
-'lib/internal/dgram.js',
-'lib/internal/dns/promises.js',
-'lib/internal/dns/utils.js',
-'lib/internal/dtrace.js',
-'lib/internal/encoding.js',
-'lib/internal/errors.js',
-'lib/internal/error_serdes.js',
-'lib/internal/event_target.js',
-'lib/internal/fixed_queue.js',
-'lib/internal/freelist.js',
-'lib/internal/freeze_intrinsics.js',
-'lib/internal/fs/dir.js',
-'lib/internal/fs/promises.js',
-'lib/internal/fs/read_file_context.js',
-'lib/internal/fs/rimraf.js',
-'lib/internal/fs/streams.js',
-'lib/internal/fs/sync_write_stream.js',
-'lib/internal/fs/utils.js',
-'lib/internal/fs/watchers.js',
-'lib/internal/http.js',
-'lib/internal/heap_utils.js',
-'lib/internal/histogram.js',
-'lib/internal/idna.js',
-'lib/internal/inspector_async_hook.js',
-'lib/internal/js_stream_socket.js',
-'lib/internal/legacy/processbinding.js',
-'lib/internal/linkedlist.js',
-'lib/internal/main/check_syntax.js',
-'lib/internal/main/eval_string.js',
-'lib/internal/main/eval_stdin.js',
-'lib/internal/main/inspect.js',
-'lib/internal/main/print_help.js',
-'lib/internal/main/prof_process.js',
-'lib/internal/main/repl.js',
-'lib/internal/main/run_main_module.js',
-'lib/internal/main/worker_thread.js',
-'lib/internal/modules/run_main.js',
-'lib/internal/modules/package_json_reader.js',
-'lib/internal/modules/cjs/helpers.js',
-'lib/internal/modules/cjs/loader.js',
-'lib/internal/modules/esm/loader.js',
-'lib/internal/modules/esm/create_dynamic_module.js',
-'lib/internal/modules/esm/get_format.js',
-'lib/internal/modules/esm/get_source.js',
-'lib/internal/modules/esm/module_job.js',
-'lib/internal/modules/esm/module_map.js',
-'lib/internal/modules/esm/resolve.js',
-'lib/internal/modules/esm/transform_source.js',
-'lib/internal/modules/esm/translators.js',
-'lib/internal/net.js',
-'lib/internal/options.js',
-'lib/internal/perf/perf.js',
-'lib/internal/perf/nodetiming.js',
-'lib/internal/perf/usertiming.js',
-'lib/internal/perf/observe.js',
-'lib/internal/perf/event_loop_delay.js',
-'lib/internal/perf/event_loop_utilization.js',
-'lib/internal/perf/timerify.js',
-'lib/internal/policy/manifest.js',
-'lib/internal/policy/sri.js',
-'lib/internal/priority_queue.js',
-'lib/internal/process/esm_loader.js',
-'lib/internal/process/execution.js',
-'lib/internal/process/per_thread.js',
-'lib/internal/process/policy.js',
-'lib/internal/process/promises.js',
-'lib/internal/process/warning.js',
-'lib/internal/process/worker_thread_only.js',
-'lib/internal/process/report.js',
-'lib/internal/process/signal.js',
-'lib/internal/process/task_queues.js',
-'lib/internal/querystring.js',
-'lib/internal/readline/callbacks.js',
-'lib/internal/readline/emitKeypressEvents.js',
-'lib/internal/readline/utils.js',
-'lib/internal/repl.js',
-'lib/internal/repl/await.js',
-'lib/internal/repl/history.js',
-'lib/internal/repl/utils.js',
-'lib/internal/socketaddress.js',
-'lib/internal/socket_list.js',
-'lib/internal/source_map/prepare_stack_trace.js',
-'lib/internal/source_map/source_map.js',
-'lib/internal/source_map/source_map_cache.js',
-'lib/internal/test/binding.js',
-'lib/internal/test/transfer.js',
-'lib/internal/timers.js',
-'lib/internal/tls.js',
-'lib/internal/trace_events_async_hooks.js',
-'lib/internal/tty.js',
-'lib/internal/url.js',
-'lib/internal/util.js',
-'lib/internal/util/comparisons.js',
-'lib/internal/util/debuglog.js',
-'lib/internal/util/inspect.js',
-'lib/internal/util/inspector.js',
-'lib/internal/util/iterable_weak_map.js',
-'lib/internal/util/types.js',
-'lib/internal/http2/core.js',
-'lib/internal/http2/compat.js',
-'lib/internal/http2/util.js',
-'lib/internal/v8_prof_polyfill.js',
-'lib/internal/v8_prof_processor.js',
-'lib/internal/validators.js',
-'lib/internal/stream_base_commons.js',
-'lib/internal/vm/module.js',
-'lib/internal/worker.js',
-'lib/internal/worker/io.js',
-'lib/internal/worker/js_transferable.js',
-'lib/internal/watchdog.js',
-'lib/internal/streams/lazy_transform.js',
-'lib/internal/streams/add-abort-signal.js',
-'lib/internal/streams/buffer_list.js',
-'lib/internal/streams/duplexpair.js',
-'lib/internal/streams/from.js',
-'lib/internal/streams/legacy.js',
-'lib/internal/streams/readable.js',
-'lib/internal/streams/writable.js',
-'lib/internal/streams/duplex.js',
-'lib/internal/streams/passthrough.js',
-'lib/internal/streams/transform.js',
-'lib/internal/streams/destroy.js',
-'lib/internal/streams/state.js',
-'lib/internal/streams/pipeline.js',
-'lib/internal/streams/end-of-stream.js',
-'lib/internal/streams/utils.js',
+'<!@(python tools/search_files.py --ext js lib)',
+ ],
+'deps_files': [
 'deps/v8/tools/splaytree.mjs',
 'deps/v8/tools/codemap.mjs',
 'deps/v8/tools/consarray.mjs',
@@ -794,6 +558,7 @@
 'deps/v8/include/v8.h',
 # javascript files to make for an even more pleasant IDE experience
 '<@(library_files)',
+'<@(deps_files)',
 # node.gyp is added by default, common.gypi is added for change detection
 'common.gypi',
  ],
@@ -1047,14 +812,21 @@
 # Put the code first so it's a dependency and can be used for invocation.
 'tools/js2c.py',
 '<@(library_files)',
+'<@(deps_files)',
 'config.gypi'
  ],
 'outputs': [
 '<(SHARED_INTERMEDIATE_DIR)/node_javascript.cc',
  ],
 'action': [
-'python', '<@(_inputs)',
-'--target', '<@(_outputs)',
+'python',
+'tools/js2c.py',
+'--directory',
+'lib',
+'--target',
+'<@(_outputs)',
+'config.gypi',
+'<@(deps_files)',
  ],
  },
  ],
@@ -1598,6 +1370,7 @@
  ],
 'sources': [
 '<@(library_files)',
+'<@(deps_files)',
 'common.gypi',
  ],
 'direct_dependent_settings': {

tools/js2c.py

@@ -36,6 +36,7 @@
 importre
 importfunctools
 importcodecs
+importutils
 
 defReadFile(filename):
 ifis_verbose:
@@ -204,12 +205,23 @@ def main():
 fromfile_prefix_chars='@'
  )
 parser.add_argument('--target', help='output file')
+parser.add_argument(
+'--directory',
+default=None,
+help='input file directory')
 parser.add_argument('--verbose', action='store_true', help='output file')
 parser.add_argument('sources', nargs='*', help='input files')
 options=parser.parse_args()
 globalis_verbose
 is_verbose=options.verbose
-source_files=functools.reduce(SourceFileByExt, options.sources, {})
+sources=options.sources
+ifoptions.directoryisnotNone:
+js_files=utils.SearchFiles(options.directory, 'js')
+mjs_files=utils.SearchFiles(options.directory, 'mjs')
+sources=js_files+mjs_files+options.sources
+
+source_files=functools.reduce(SourceFileByExt, sources, {})
+
 # Should have exactly 3 types: `.js`, `.mjs` and `.gypi`
 assertlen(source_files) ==3
 # Currently config.gypi is the only `.gypi` file allowed

tools/search_files.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+
+"""
+This is a utility for recursively searching files under
+a specified directory
+"""
+
+importargparse
+importutils
+
+defmain():
+parser=argparse.ArgumentParser(
+description='Search files with a specific extension under a directory',
+fromfile_prefix_chars='@'
+ )
+parser.add_argument('--ext', required=True, help='extension to search for')
+parser.add_argument('directory', help='input directory')
+options=parser.parse_args()
+print('\n'.join(utils.SearchFiles(options.directory, options.ext)))
+
+if__name__=="__main__":
+main()

tools/utils.py

@@ -26,8 +26,10 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
+importglob
 importplatform
 importre
+importsys
 
 
 # Reads a .list file into an array of strings
@@ -106,3 +108,10 @@ def GuessArchitecture():
 
 defIsWindows():
 returnGuessOS() =='win32'
+
+
+defSearchFiles(dir, ext):
+list=glob.glob(dir+'/**/*.'+ext, recursive=True)
+ifsys.platform=='win32':
+list= [ x.replace('\\', '/')forxinlist]
+returnlist