fix: limit concurrency when generating BUILD files in npm_install and yarn_install (#3509)

* chore: add .gitattributes to ignore changes in generated docs files on PR reviews
* fix: limit concurrency when generating BUILD files in npm_install and yarn_install

Author: gregmagolan

.gitattributes

@@ -0,0 +1,12 @@
+docs/Built-ins.mdlinguist-generated=true
+docs/Concatjs.mdlinguist-generated=true
+docs/Cypress.mdlinguist-generated=true
+docs/esbuild.mdlinguist-generated=true
+docs/Jasmine.mdlinguist-generated=true
+docs/Karma.mdlinguist-generated=true
+docs/Protractor.mdlinguist-generated=true
+docs/Providers.mdlinguist-generated=true
+docs/Rollup.mdlinguist-generated=true
+docs/Terser.mdlinguist-generated=true
+docs/Toolchains.mdlinguist-generated=true
+docs/TypeScript.mdlinguist-generated=true

docs/Built-ins.md

@@ -55,6 +55,7 @@ let LEGACY_NODE_MODULES_PACKAGE_NAME = '$node_modules$';
 letconfig: any={
 exports_directories_only: false,
 generate_local_modules_build_files: false,
+generate_build_files_concurrency_limit: 64,
 included_files: [],
 links: {},
 package_json: 'package.json',
@@ -688,7 +689,7 @@ async function findPackagesAndPush(pkgs: Dep[], p: string, dependencies: Set<str
 
 constlisting=awaitfs.readdir(p);
 
-awaitPromise.all(listing.map(asyncf=>{
+letpromises=listing.map(asyncf=>{
 // filter out folders such as `.bin` which can create
 // issues on Windows since these are "hidden" by default
 if(f.startsWith('.'))return[];
@@ -702,7 +703,17 @@ async function findPackagesAndPush(pkgs: Dep[], p: string, dependencies: Set<str
 awaitfindPackagesAndPush(pkgs,path.posix.join(pf,'node_modules'),dependencies);
 }
 }
-}));
+});
+
+if(config.generate_build_files_concurrency_limit<1){
+// unlimited concurrency
+awaitPromise.all(promises);
+}else{
+while(promises.length){
+// run batches of generate_build_files_concurrency_limit at a time
+awaitPromise.all(promises.splice(0,config.generate_build_files_concurrency_limit))
+}
+}
 }
 
 /**

internal/npm_install/generate_build_file.ts

@@ -13,6 +13,7 @@ let LEGACY_NODE_MODULES_PACKAGE_NAME = '$node_modules$';
 letconfig={
 exports_directories_only: false,
 generate_local_modules_build_files: false,
+generate_build_files_concurrency_limit: 64,
 included_files: [],
 links: {},
 package_json: 'package.json',
@@ -443,7 +444,7 @@ async function findPackagesAndPush(pkgs, p, dependencies) {
 return;
 }
 constlisting=awaitfs_1.promises.readdir(p);
-awaitPromise.all(listing.map(async(f)=>{
+letpromises=listing.map(async(f)=>{
 if(f.startsWith('.'))
 return[];
 constpf=path.posix.join(p,f);
@@ -456,7 +457,15 @@ async function findPackagesAndPush(pkgs, p, dependencies) {
 awaitfindPackagesAndPush(pkgs,path.posix.join(pf,'node_modules'),dependencies);
 }
 }
-}));
+});
+if(config.generate_build_files_concurrency_limit<1){
+awaitPromise.all(promises);
+}
+else{
+while(promises.length){
+awaitPromise.all(promises.splice(0,config.generate_build_files_concurrency_limit));
+}
+}
 }
 asyncfunctionfindScopes(){
 constp=nodeModulesFolder();

internal/npm_install/index.js

@@ -359,6 +359,14 @@ Using managed_directories will mean that
 default=3600,
 doc="""Maximum duration of the package manager execution in seconds.""",
  ),
+"generate_build_files_concurrency_limit": attr.int(
+default=64,
+doc="""Limit the maximum concurrency of npm package processing when generating
+ BUILD files from the node_modules tree. Unlimited concurrency can lead to too many
+ open files errors (https://github.com/bazelbuild/rules_nodejs/issues/3507).
+
+ Set to 0 or negative for unlimited concurrency.""",
+ ),
 })
 
 def_apply_pre_install_patches(repository_ctx):
@@ -466,6 +474,7 @@ def _create_build_files(repository_ctx, rule_type, node, lock_file, generate_loc
 generate_config_json=json.encode(
 struct(
 exports_directories_only=repository_ctx.attr.exports_directories_only,
+generate_build_files_concurrency_limit=repository_ctx.attr.generate_build_files_concurrency_limit,
 generate_local_modules_build_files=generate_local_modules_build_files,
 included_files=repository_ctx.attr.included_files,
 links=validated_links,
@@ -475,8 +484,8 @@ def _create_build_files(repository_ctx, rule_type, node, lock_file, generate_loc
 rule_type=rule_type,
 strict_visibility=repository_ctx.attr.strict_visibility,
 workspace=repository_ctx.attr.name,
-workspace_rerooted_path=_WORKSPACE_REROOTED_PATH,
 workspace_rerooted_package_json_dir=paths.normalize(paths.join(_WORKSPACE_REROOTED_PATH, package_json_dir)),
+workspace_rerooted_path=_WORKSPACE_REROOTED_PATH,
  ),
  )
 repository_ctx.file("generate_config.json", generate_config_json)

internal/npm_install/npm_install.bzl

@@ -163,24 +163,28 @@ js_library(
  },
 package_json="//internal/linker/test/multi_linker/test_a:package.json",
 yarn_lock="//internal/linker/test/multi_linker/test_a:yarn.lock",
+generate_build_files_concurrency_limit=0,
  )
 
 yarn_install(
 name="internal_test_multi_linker_test_b_deps",
 package_json="//internal/linker/test/multi_linker/test_b:package.json",
 yarn_lock="//internal/linker/test/multi_linker/test_b:yarn.lock",
+generate_build_files_concurrency_limit=0,
  )
 
 yarn_install(
 name="internal_test_multi_linker_test_c_deps",
 package_json="//internal/linker/test/multi_linker/test_c:package.json",
 yarn_lock="//internal/linker/test/multi_linker/test_c:yarn.lock",
+generate_build_files_concurrency_limit=0,
  )
 
 yarn_install(
 name="internal_test_multi_linker_test_d_deps",
 package_json="//internal/linker/test/multi_linker/test_d:package.json",
 yarn_lock="//internal/linker/test/multi_linker/test_d:yarn.lock",
+generate_build_files_concurrency_limit=0,
  )
 
 yarn_install(
@@ -189,6 +193,7 @@ js_library(
 args= ["--production"],
 package_json="//internal/linker/test/multi_linker/lib_b:package.json",
 yarn_lock="//internal/linker/test/multi_linker/lib_b:yarn.lock",
+generate_build_files_concurrency_limit=0,
  )
 
 yarn_install(
@@ -197,6 +202,7 @@ js_library(
 args= ["--production"],
 package_json="//internal/linker/test/multi_linker/lib_c:lib/package.json",
 yarn_lock="//internal/linker/test/multi_linker/lib_c:lib/yarn.lock",
+generate_build_files_concurrency_limit=0,
  )
 
 yarn_install(