fs: improve fsPromises readFile performance

Linkgoron · MylesBorins · commit b277776845e1 · 2021-03-10T11:40:42.000-05:00
Improve the fsPromises readFile performance by allocating only one buffer, when size is known, increase the size of the readbuffer chunks, and dont read more data if size bytes have been read Refs: #37583 PR-URL: #37608 Reviewed-By: Benjamin Gruenbaum <benjamingr@gmail.com> Reviewed-By: James M Snell <jasnell@gmail.com> Reviewed-By: Anna Henningsen <anna@addaleax.net>
diff --git a/lib/internal/fs/promises.js b/lib/internal/fs/promises.js
@@ -4,9 +4,8 @@
 // See https://github.com/libuv/libuv/pull/1501.
 const kIoMaxLength = 2 ** 31 - 1;
 
-// Note: This is different from kReadFileBufferLength used for non-promisified
-// fs.readFile.
-const kReadFileMaxChunkSize = 2 ** 14;
+const kReadFileBufferLength = 512 * 1024;
+const kReadFileUnknownBufferLength = 64 * 1024;
 const kWriteFileMaxChunkSize = 2 ** 14;
 
 const {
@@ -316,25 +315,46 @@ async function readFileHandle(filehandle, options) {
 if (size > kIoMaxLength)
 throw new ERR_FS_FILE_TOO_LARGE(size);
 
- const chunks = [];
- let isFirstChunk = true;
- const firstChunkSize = size === 0 ? kReadFileMaxChunkSize : size;
- const chunkSize = MathMin(firstChunkSize, kReadFileMaxChunkSize);
 let endOfFile = false;
+ let totalRead = 0;
+ const noSize = size === 0;
+ const buffers = [];
+ const fullBuffer = noSize ? undefined : Buffer.allocUnsafeSlow(size);
 do {
 if (signal?.aborted) {
 throw lazyDOMException('The operation was aborted', 'AbortError');
 }
- const buf = Buffer.alloc(isFirstChunk ? firstChunkSize : chunkSize);
- const { bytesRead, buffer } =
- await read(filehandle, buf, 0, buf.length, -1);
- endOfFile = bytesRead === 0;
- if (bytesRead > 0)
- ArrayPrototypePush(chunks, buffer.slice(0, bytesRead));
- isFirstChunk = false;
+ let buffer;
+ let offset;
+ let length;
+ if (noSize) {
+ buffer = Buffer.allocUnsafeSlow(kReadFileUnknownBufferLength);
+ offset = 0;
+ length = kReadFileUnknownBufferLength;
+ } else {
+ buffer = fullBuffer;
+ offset = totalRead;
+ length = MathMin(size - totalRead, kReadFileBufferLength);
+ }
+
+ const bytesRead = (await binding.read(filehandle.fd, buffer, offset,
+ length, -1, kUsePromises)) || 0;
+ totalRead += bytesRead;
+ endOfFile = bytesRead === 0 || totalRead === size;
+ if (noSize && bytesRead > 0) {
+ const isBufferFull = bytesRead === kReadFileUnknownBufferLength;
+ const chunkBuffer = isBufferFull ? buffer : buffer.slice(0, bytesRead);
+ ArrayPrototypePush(buffers, chunkBuffer);
+ }
 } while (!endOfFile);
 
- const result = chunks.length === 1 ? chunks[0] : Buffer.concat(chunks);
+ let result;
+ if (size > 0) {
+ result = totalRead === size ? fullBuffer : fullBuffer.slice(0, totalRead);
+ } else {
+ result = buffers.length === 1 ? buffers[0] : Buffer.concat(buffers,
+ totalRead);
+ }
 
 return options.encoding ? result.toString(options.encoding) : result;
 }
diff --git a/test/parallel/test-fs-promises-file-handle-readFile.js b/test/parallel/test-fs-promises-file-handle-readFile.js
@@ -10,7 +10,7 @@ const {
 open,
 readFile,
 writeFile,
- truncate
+ truncate,
 } = fs.promises;
 const path = require('path');
 const tmpdir = require('../common/tmpdir');
@@ -64,6 +64,7 @@ async function doReadAndCancel() {
 await assert.rejects(readFile(fileHandle, { signal }), {
 name: 'AbortError'
 });
+ await fileHandle.close();
 }
 
 // Signal aborted on first tick
@@ -74,10 +75,11 @@ async function doReadAndCancel() {
 fs.writeFileSync(filePathForHandle, buffer);
 const controller = new AbortController();
 const { signal } = controller;
- tick(1, () => controller.abort());
+ process.nextTick(() => controller.abort());
 await assert.rejects(readFile(fileHandle, { signal }), {
 name: 'AbortError'
- });
+ }, 'tick-0');
+ await fileHandle.close();
 }
 
 // Signal aborted right before buffer read
@@ -90,10 +92,12 @@ async function doReadAndCancel() {
 
 const controller = new AbortController();
 const { signal } = controller;
- tick(2, () => controller.abort());
+ tick(1, () => controller.abort());
 await assert.rejects(fileHandle.readFile({ signal, encoding: 'utf8' }), {
 name: 'AbortError'
- });
+ }, 'tick-1');
+
+ await fileHandle.close();
 }
 
 // Validate file size is within range for reading
@@ -111,6 +115,7 @@ async function doReadAndCancel() {
 name: 'RangeError',
 code: 'ERR_FS_FILE_TOO_LARGE'
 });
+ await fileHandle.close();
 }
 }
 

-Original file line number
+Diff line change
 // See https://github.com/libuv/libuv/pull/1501.
 constkIoMaxLength=2**31-1;
 -// Note: This is different from kReadFileBufferLength used for non-promisified
 -// fs.readFile.
 -constkReadFileMaxChunkSize=2**14;
 +constkReadFileBufferLength=512*1024;
 +constkReadFileUnknownBufferLength=64*1024;
 constkWriteFileMaxChunkSize=2**14;
 const{
 if(size>kIoMaxLength)
 thrownewERR_FS_FILE_TOO_LARGE(size);
 -constchunks=[];
 -letisFirstChunk=true;
 -constfirstChunkSize=size===0 ? kReadFileMaxChunkSize : size;
 -constchunkSize=MathMin(firstChunkSize,kReadFileMaxChunkSize);
 letendOfFile=false;
 +lettotalRead=0;
 +constnoSize=size===0;
 +constbuffers=[];
 +constfullBuffer=noSize ? undefined : Buffer.allocUnsafeSlow(size);
 do{
 if(signal?.aborted){
 throwlazyDOMException('The operation was aborted','AbortError');
+}
 -constbuf=Buffer.alloc(isFirstChunk ? firstChunkSize : chunkSize);
 -const{ bytesRead, buffer }=
 -awaitread(filehandle,buf,0,buf.length,-1);
 -endOfFile=bytesRead===0;
 -if(bytesRead>0)
 -ArrayPrototypePush(chunks,buffer.slice(0,bytesRead));
 -isFirstChunk=false;
 +letbuffer;
 +letoffset;
 +letlength;
 +if(noSize){
 +buffer=Buffer.allocUnsafeSlow(kReadFileUnknownBufferLength);
 +offset=0;
 +length=kReadFileUnknownBufferLength;
 +}else{
 +buffer=fullBuffer;
 +offset=totalRead;
 +length=MathMin(size-totalRead,kReadFileBufferLength);
 +}
++
 +constbytesRead=(awaitbinding.read(filehandle.fd,buffer,offset,
 +length,-1,kUsePromises))||0;
 +totalRead+=bytesRead;
 +endOfFile=bytesRead===0||totalRead===size;
 +if(noSize&&bytesRead>0){
 +constisBufferFull=bytesRead===kReadFileUnknownBufferLength;
 +constchunkBuffer=isBufferFull ? buffer : buffer.slice(0,bytesRead);
 +ArrayPrototypePush(buffers,chunkBuffer);
 +}
 }while(!endOfFile);
 -constresult=chunks.length===1 ? chunks[0] : Buffer.concat(chunks);
 +letresult;
 +if(size>0){
 +result=totalRead===size ? fullBuffer : fullBuffer.slice(0,totalRead);
 +}else{
 +result=buffers.length===1 ? buffers[0] : Buffer.concat(buffers,
 +totalRead);
 +}
 returnoptions.encoding ? result.toString(options.encoding) : result;
+}
-Original file line number
+Diff line change
  open,
  readFile,
  writeFile,
 - truncate
 + truncate,
 }=fs.promises;
 constpath=require('path');
 consttmpdir=require('../common/tmpdir');
 awaitassert.rejects(readFile(fileHandle,{ signal }),{
 name: 'AbortError'
 });
 +awaitfileHandle.close();
+}
 // Signal aborted on first tick
 fs.writeFileSync(filePathForHandle,buffer);
 constcontroller=newAbortController();
 const{ signal }=controller;
 -tick(1,()=>controller.abort());
 +process.nextTick(()=>controller.abort());
 awaitassert.rejects(readFile(fileHandle,{ signal }),{
 name: 'AbortError'
 -});
 +},'tick-0');
 +awaitfileHandle.close();
+}
 // Signal aborted right before buffer read
 constcontroller=newAbortController();
 const{ signal }=controller;
 -tick(2,()=>controller.abort());
 +tick(1,()=>controller.abort());
 awaitassert.rejects(fileHandle.readFile({ signal,encoding: 'utf8'}),{
 name: 'AbortError'
 -});
 +},'tick-1');
++
 +awaitfileHandle.close();
+}
 // Validate file size is within range for reading
 name: 'RangeError',
 code: 'ERR_FS_FILE_TOO_LARGE'
 });
 +awaitfileHandle.close();
+}
+}