fs: add recursive option to readdir and opendir

Adds a naive, linear recursive algorithm for the following methods:
readdir, readdirSync, opendir, opendirSync, and the promise based
equivalents.
Fixes: #34992
PR-URL: #41439
Refs: nodejs/tooling#130
Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com>
Reviewed-By: Matteo Collina <matteo.collina@gmail.com>
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Moshe Atlow <moshe@atlow.co.il>

Author: Ethan Arrowood

doc/api/fs.md

@@ -1214,6 +1214,9 @@ a colon, Node.js will open a file system stream, as described by
 <!-- YAML
 added: v12.12.0
 changes:
+ - version: REPLACEME
+ pr-url: https://github.com/nodejs/node/pull/41439
+ description: Added `recursive` option.
  - version:
  - v13.1.0
  - v12.16.0
@@ -1227,6 +1230,8 @@ changes:
  * `bufferSize` {number} Number of directory entries that are buffered
  internally when reading from the directory. Higher values lead to better
  performance but higher memory usage. **Default:** `32`
+ * `recursive` {boolean} Resolved `Dir` will be an {AsyncIterable}
+ containing all sub files and directories. **Default:** `false`
 * Returns: {Promise} Fulfills with an {fs.Dir}.
 
 Asynchronously open a directory for iterative scanning. See the POSIX
@@ -1260,6 +1265,9 @@ closed after the iterator exits.
 <!-- YAML
 added: v10.0.0
 changes:
+ - version: REPLACEME
+ pr-url: https://github.com/nodejs/node/pull/41439
+ description: Added `recursive` option.
  - version: v10.11.0
  pr-url: https://github.com/nodejs/node/pull/22020
  description: New option `withFileTypes` was added.
@@ -1269,6 +1277,7 @@ changes:
 * `options` {string|Object}
  * `encoding` {string} **Default:** `'utf8'`
  * `withFileTypes` {boolean} **Default:** `false`
+ * `recursive` {boolean} **Default:** `false`
 * Returns: {Promise} Fulfills with an array of the names of the files in
  the directory excluding `'.'` and `'..'`.
 
@@ -3344,6 +3353,9 @@ Functions based on `fs.open()` exhibit this behavior as well:
 <!-- YAML
 added: v12.12.0
 changes:
+ - version: REPLACEME
+ pr-url: https://github.com/nodejs/node/pull/41439
+ description: Added `recursive` option.
  - version: v18.0.0
  pr-url: https://github.com/nodejs/node/pull/41678
  description: Passing an invalid callback to the `callback` argument
@@ -3362,6 +3374,7 @@ changes:
  * `bufferSize` {number} Number of directory entries that are buffered
  internally when reading from the directory. Higher values lead to better
  performance but higher memory usage. **Default:** `32`
+ * `recursive` {boolean} **Default:** `false`
 * `callback` {Function}
  * `err` {Error}
  * `dir` {fs.Dir}
@@ -3478,6 +3491,9 @@ above values.
 <!-- YAML
 added: v0.1.8
 changes:
+ - version: REPLACEME
+ pr-url: https://github.com/nodejs/node/pull/41439
+ description: Added `recursive` option.
  - version: v18.0.0
  pr-url: https://github.com/nodejs/node/pull/41678
  description: Passing an invalid callback to the `callback` argument
@@ -3507,6 +3523,7 @@ changes:
 * `options` {string|Object}
  * `encoding` {string} **Default:** `'utf8'`
  * `withFileTypes` {boolean} **Default:** `false`
+ * `recursive` {boolean} **Default:** `false`
 * `callback` {Function}
  * `err` {Error}
  * `files` {string\[]|Buffer\[]|fs.Dirent\[]}
@@ -5470,6 +5487,9 @@ object with an `encoding` property specifying the character encoding to use.
 <!-- YAML
 added: v12.12.0
 changes:
+ - version: REPLACEME
+ pr-url: https://github.com/nodejs/node/pull/41439
+ description: Added `recursive` option.
  - version:
  - v13.1.0
  - v12.16.0
@@ -5483,6 +5503,7 @@ changes:
  * `bufferSize` {number} Number of directory entries that are buffered
  internally when reading from the directory. Higher values lead to better
  performance but higher memory usage. **Default:** `32`
+ * `recursive` {boolean} **Default:** `false`
 * Returns: {fs.Dir}
 
 Synchronously open a directory. See opendir(3).
@@ -5526,6 +5547,9 @@ this API: [`fs.open()`][].
 <!-- YAML
 added: v0.1.21
 changes:
+ - version: REPLACEME
+ pr-url: https://github.com/nodejs/node/pull/41439
+ description: Added `recursive` option.
  - version: v10.10.0
  pr-url: https://github.com/nodejs/node/pull/22020
  description: New option `withFileTypes` was added.
@@ -5539,6 +5563,7 @@ changes:
 * `options` {string|Object}
  * `encoding` {string} **Default:** `'utf8'`
  * `withFileTypes` {boolean} **Default:** `false`
+ * `recursive` {boolean} **Default:** `false`
 * Returns: {string\[]|Buffer\[]|fs.Dirent\[]}
 
 Reads the contents of the directory.
@@ -6384,6 +6409,16 @@ The file name that this {fs.Dirent} object refers to. The type of this
 value is determined by the `options.encoding` passed to [`fs.readdir()`][] or
 [`fs.readdirSync()`][].
 
+#### `dirent.path`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* {string}
+
+The base path that this {fs.Dirent} object refers to.
+
 ### Class: `fs.FSWatcher`
 
 <!-- YAML

lib/fs.js

@@ -1399,6 +1399,36 @@ function mkdirSync(path, options) {
 }
 }
 
+// TODO(Ethan-Arrowood): Make this iterative too
+functionreaddirSyncRecursive(path,origPath,options){
+nullCheck(path,'path',true);
+constctx={ path };
+constresult=binding.readdir(pathModule.toNamespacedPath(path),
+options.encoding,!!options.withFileTypes,undefined,ctx);
+handleErrorFromBinding(ctx);
+returnoptions.withFileTypes ?
+getDirents(path,result).flatMap((dirent)=>{
+return[
+dirent,
+ ...(dirent.isDirectory() ?
+readdirSyncRecursive(
+pathModule.join(path,dirent.name),
+origPath,
+options,
+) : []),
+];
+}) :
+result.flatMap((ent)=>{
+constinnerPath=pathModule.join(path,ent);
+constrelativePath=pathModule.relative(origPath,innerPath);
+conststat=binding.internalModuleStat(innerPath);
+return[
+relativePath,
+ ...(stat===1 ? readdirSyncRecursive(innerPath,origPath,options) : []),
+];
+});
+}
+
 /**
  * Reads the contents of a directory.
  * @param {string | Buffer | URL} path
@@ -1416,6 +1446,14 @@ function readdir(path, options, callback) {
 callback=makeCallback(typeofoptions==='function' ? options : callback);
 options=getOptions(options);
 path=getValidatedPath(path);
+if(options.recursive!=null){
+validateBoolean(options.recursive,'options.recursive');
+}
+
+if(options.recursive){
+callback(null,readdirSyncRecursive(path,path,options));
+return;
+}
 
 constreq=newFSReqCallback();
 if(!options.withFileTypes){
@@ -1439,12 +1477,21 @@ function readdir(path, options, callback) {
  * @param {string | {
  * encoding?: string;
  * withFileTypes?: boolean;
+ * recursive?: boolean;
  * }} [options]
  * @returns {string | Buffer[] | Dirent[]}
  */
 functionreaddirSync(path,options){
 options=getOptions(options);
 path=getValidatedPath(path);
+if(options.recursive!=null){
+validateBoolean(options.recursive,'options.recursive');
+}
+
+if(options.recursive){
+returnreaddirSyncRecursive(path,path,options);
+}
+
 constctx={ path };
 constresult=binding.readdir(pathModule.toNamespacedPath(path),
 options.encoding,!!options.withFileTypes,

lib/internal/fs/dir.js

@@ -2,8 +2,7 @@
 
 const{
  ArrayPrototypePush,
- ArrayPrototypeSlice,
- ArrayPrototypeSplice,
+ ArrayPrototypeShift,
  FunctionPrototypeBind,
  ObjectDefineProperty,
  PromiseReject,
@@ -99,13 +98,21 @@ class Dir {
 }
 
 if(this[kDirBufferedEntries].length>0){
-const{0: name,1: type}=
-ArrayPrototypeSplice(this[kDirBufferedEntries],0,2);
-if(maybeSync)
-process.nextTick(getDirent,this[kDirPath],name,type,callback);
-else
-getDirent(this[kDirPath],name,type,callback);
-return;
+try{
+constdirent=ArrayPrototypeShift(this[kDirBufferedEntries]);
+
+if(this[kDirOptions].recursive&&dirent.isDirectory()){
+this.readSyncRecursive(dirent);
+}
+
+if(maybeSync)
+process.nextTick(callback,null,dirent);
+else
+callback(null,dirent);
+return;
+}catch(error){
+returncallback(error);
+}
 }
 
 constreq=newFSReqCallback();
@@ -120,8 +127,16 @@ class Dir {
 returncallback(err,result);
 }
 
-this[kDirBufferedEntries]=ArrayPrototypeSlice(result,2);
-getDirent(this[kDirPath],result[0],result[1],callback);
+try{
+this.processReadResult(this[kDirPath],result);
+constdirent=ArrayPrototypeShift(this[kDirBufferedEntries]);
+if(this[kDirOptions].recursive&&dirent.isDirectory()){
+this.readSyncRecursive(dirent);
+}
+callback(null,dirent);
+}catch(error){
+callback(error);
+}
 };
 
 this[kDirOperationQueue]=[];
@@ -132,6 +147,45 @@ class Dir {
 );
 }
 
+processReadResult(path,result){
+for(leti=0;i<result.length;i+=2){
+ArrayPrototypePush(
+this[kDirBufferedEntries],
+getDirent(
+pathModule.join(path,result[i]),
+result[i],
+result[i+1],
+),
+);
+}
+}
+
+// TODO(Ethan-Arrowood): Review this implementation. Make it iterative.
+// Can we better leverage the `kDirOperationQueue`?
+readSyncRecursive(dirent){
+constctx={path: dirent.path};
+consthandle=dirBinding.opendir(
+pathModule.toNamespacedPath(dirent.path),
+this[kDirOptions].encoding,
+undefined,
+ctx,
+);
+handleErrorFromBinding(ctx);
+constresult=handle.read(
+this[kDirOptions].encoding,
+this[kDirOptions].bufferSize,
+undefined,
+ctx,
+);
+
+if(result){
+this.processReadResult(dirent.path,result);
+}
+
+handle.close(undefined,ctx);
+handleErrorFromBinding(ctx);
+}
+
 readSync(){
 if(this[kDirClosed]===true){
 thrownewERR_DIR_CLOSED();
@@ -142,9 +196,11 @@ class Dir {
 }
 
 if(this[kDirBufferedEntries].length>0){
-const{0: name,1: type}=
-ArrayPrototypeSplice(this[kDirBufferedEntries],0,2);
-returngetDirent(this[kDirPath],name,type);
+constdirent=ArrayPrototypeShift(this[kDirBufferedEntries]);
+if(this[kDirOptions].recursive&&dirent.isDirectory()){
+this.readSyncRecursive(dirent);
+}
+returndirent;
 }
 
 constctx={path: this[kDirPath]};
@@ -160,8 +216,13 @@ class Dir {
 returnresult;
 }
 
-this[kDirBufferedEntries]=ArrayPrototypeSlice(result,2);
-returngetDirent(this[kDirPath],result[0],result[1]);
+this.processReadResult(this[kDirPath],result);
+
+constdirent=ArrayPrototypeShift(this[kDirBufferedEntries]);
+if(this[kDirOptions].recursive&&dirent.isDirectory()){
+this.readSyncRecursive(dirent);
+}
+returndirent;
 }
 
 close(callback){