Commit 77fedbd

Browse files
codebytereaduh95
authored andcommitted
fs: read small files in one thread pool round trip
fs.readFile(path) took four libuv thread pool round trips for a typical small file -- open, fstat, read and close, each its own uv_fs request with its own queue wait, completion callback and JS/C++ crossing -- and fs.promises.readFile(path) did the same through a FileHandle. For the small files applications read most, the round trips are the cost, and each occupies a slot in the pool that concurrent dns.lookup(), fs and crypto work is also queueing for. Add ReadFileJob (an AsyncWrap + ThreadPoolWork) that performs open + fstat + read-to-EOF + close as one pool task and reports the whole content, or, when the file turns out to be larger than one chunk (kReadFileBufferLength, 512 KiB), stops after fstat() and hands the fd and size back so that the existing chunked reader continues unchanged (large reads stay interleaved and abortable between chunks, and still save the fstat round trip). fs.readFile() and fs.promises.readFile() use it for path arguments without a user buffer; file descriptors, FileHandles, options.buffer and an active VFS keep their paths. Behavior is otherwise kept: same bytes for every size and encoding; open failures report syscall 'open' with the path, read failures 'read'; permission errors are delivered through the callback/promise as before; an abort that arrives while the read is in flight still wins; the job is an FSREQCALLBACK resource for async_hooks; a handed back fd is tracked exactly like one from a plain open(). Tests that asserted the internal open/fstat/read/close request chain, used readFile() as a proxy for an fstat trace event, or injected faults through FileHandle.prototype for path-based reads are adjusted to keep testing what they test (a file just over one chunk where the chain shape matters, fs.fstat() for the fstat trace, a larger file so the FileHandle path is taken). fs.readFile() of 4 KiB files at concurrency 64 goes from ~51k to ~306k files per second, and a mixed stat/readFile/dns.lookup burst from ~66k to ~312k operations per second. Signed-off-by: Shelley Vohr <shelley.vohr@gmail.com> PR-URL: #65327 Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com> Reviewed-By: James M Snell <jasnell@gmail.com>
1 parent e2f952f commit 77fedbd

12 files changed

Lines changed: 613 additions & 30 deletions

β€Žlib/fs.jsβ€Ž

Lines changed: 63 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ const {
8181

8282
const{
8383
FSReqCallback,
84+
ReadFileJob,
8485
}=binding;
8586
const{ toPathIfFileURL }=require('internal/url');
8687
const{
@@ -98,6 +99,7 @@ const {
9899
const{
99100
constants: {
100101
kIoMaxLength,
102+
kReadFileBufferLength,
101103
kMaxUserId,
102104
},
103105
copyObject,
@@ -428,10 +430,70 @@ function readFile(path, options, callback) {
428430
return;
429431

430432
constflagsNumber=stringToFlags(options.flag,'options.flag');
433+
path=getValidatedPath(path);
434+
if(options.buffer===undefined){
435+
// Open + fstat + read + close in one thread pool round trip for files of
436+
// up to one chunk; larger files come back as an open fd + size and take
437+
// the chunked reader below (readFileAfterOneShot). `true`: a handed-back
438+
// fd will be closed through fs.close(), so track it as unmanaged.
439+
constjob=newReadFileJob(path,flagsNumber,kReadFileBufferLength,true);
440+
job.context=context;
441+
job.ondone=readFileAfterOneShot;
442+
constaccessError=job.run(path);
443+
if(accessError!==undefined){
444+
// Not scheduled: report it the way the request-based open() did.
445+
callback(accessError);
446+
}
447+
return;
448+
}
431449
constreq=newFSReqCallback();
432450
req.context=context;
433451
req.oncomplete=readFileAfterOpen;
434-
binding.open(getValidatedPath(path),flagsNumber,0o666,req);
452+
binding.open(path,flagsNumber,0o666,req);
453+
}
454+
455+
functionreadFileAfterOneShot(err,buffer,fd,size,closeErr){
456+
constcontext=this.context;
457+
if(err){
458+
context.callback(err);
459+
return;
460+
}
461+
if(fd!==-1){
462+
// (context.read() below performs the abort check for this case.)
463+
// Larger than one chunk: continue exactly like after open + fstat.
464+
context.fd=fd;
465+
context.size=size;
466+
if(size>kIoMaxLength){
467+
returncontext.close(newERR_FS_FILE_TOO_LARGE(size));
468+
}
469+
try{
470+
context.prepare();
471+
}catch(err){
472+
returncontext.close(err);
473+
}
474+
context.read();
475+
return;
476+
}
477+
if(closeErr){
478+
context.callback(closeErr);
479+
return;
480+
}
481+
if(context.signal?.aborted){
482+
// An abort that arrived while the read was in flight wins, as it did when
483+
// it was noticed between the open/fstat/read steps.
484+
context.callback(newAbortError(undefined,{cause: context.signal.reason}));
485+
return;
486+
}
487+
letresult=buffer;
488+
if(context.encoding){
489+
try{
490+
result=buffer.toString(context.encoding);
491+
}catch(err){
492+
context.callback(err);
493+
return;
494+
}
495+
}
496+
context.callback(null,result);
435497
}
436498

437499
functiontryStatSync(fd,isUserFd){

β€Žlib/internal/fs/promises.jsβ€Ž

Lines changed: 56 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1211,26 +1211,32 @@ async function readFileHandleWithUserBuffer(filehandle, options, size) {
12111211
returnencoding ? buffer.toString(encoding) : buffer.subarray(0,totalRead);
12121212
}
12131213

1214-
asyncfunctionreadFileHandle(filehandle,options){
1214+
asyncfunctionreadFileHandle(filehandle,options,knownRegularFileSize){
12151215
constsignal=options?.signal;
12161216
constencoding=options?.encoding;
12171217
constdecoder=encoding&&newStringDecoder(encoding);
12181218

12191219
checkAborted(signal);
12201220

1221-
conststatFields=awaitPromisePrototypeThen(
1222-
binding.fstat(filehandle.fd,false,kUsePromises),
1223-
undefined,
1224-
handleErrorFromBinding,
1225-
);
1226-
1227-
checkAborted(signal);
1228-
12291221
letsize=0;
12301222
letlength=0;
1231-
if((statFields[1/* mode */]&S_IFMT)===S_IFREG){
1232-
size=statFields[8/* size */];
1223+
if(knownRegularFileSize!==undefined){
1224+
// Handed over by readFile() together with an already open fd.
1225+
size=knownRegularFileSize;
12331226
length=encoding ? MathMin(size,kReadFileBufferLength) : size;
1227+
}else{
1228+
conststatFields=awaitPromisePrototypeThen(
1229+
binding.fstat(filehandle.fd,false,kUsePromises),
1230+
undefined,
1231+
handleErrorFromBinding,
1232+
);
1233+
1234+
checkAborted(signal);
1235+
1236+
if((statFields[1/* mode */]&S_IFMT)===S_IFREG){
1237+
size=statFields[8/* size */];
1238+
length=encoding ? MathMin(size,kReadFileBufferLength) : size;
1239+
}
12341240
}
12351241
if(length===0){
12361242
length=kReadFileUnknownBufferLength;
@@ -2146,10 +2152,49 @@ async function readFile(path, options) {
21462152

21472153
checkAborted(options.signal);
21482154

2155+
if(options.buffer===undefined&&vfsState.handlers===null){
2156+
// Open + fstat + read + close in one thread pool round trip for files of
2157+
// up to one chunk; larger files come back as an open fd + size and are
2158+
// read by readFileHandle() as before.
2159+
path=getValidatedPath(path);
2160+
const{0: buffer,1: fd,2: size}=awaitreadFileInOneRoundTrip(path,stringToFlags(flag));
2161+
if(fd===-1){
2162+
checkAborted(options.signal);// An abort during the read still wins.
2163+
returnoptions.encoding ? buffer.toString(options.encoding) : buffer;
2164+
}
2165+
constfilehandle=newFileHandle(newbinding.FileHandle(fd));
2166+
returnhandleFdClose(readFileHandle(filehandle,options,size),filehandle.close);
2167+
}
2168+
21492169
constfd=awaitopen(path,flag,0o666);
21502170
returnhandleFdClose(readFileHandle(fd,options),fd.close);
21512171
}
21522172

2173+
/**
2174+
* @param {string|Buffer} path Validated path
2175+
* @param {number} flagsNumber
2176+
* @returns {Promise<[Buffer|undefined, number, number|undefined]>} [buffer, -1] or [undefined, fd, size]
2177+
*/
2178+
functionreadFileInOneRoundTrip(path,flagsNumber){
2179+
returnnewPromise((resolve,reject)=>{
2180+
constjob=newbinding.ReadFileJob(path,flagsNumber,kReadFileBufferLength);
2181+
job.ondone=(err,buffer,fd,size,closeErr)=>{
2182+
consterror=err??closeErr;
2183+
if(error!=null){
2184+
ErrorCaptureStackTrace(error,readFileInOneRoundTrip);
2185+
reject(error);
2186+
}else{
2187+
resolve([buffer,fd,size]);
2188+
}
2189+
};
2190+
constaccessError=job.run(path);
2191+
if(accessError!==undefined){
2192+
ErrorCaptureStackTrace(accessError,readFileInOneRoundTrip);
2193+
reject(accessError);
2194+
}
2195+
});
2196+
}
2197+
21532198
asyncfunction*_watch(filename,options=kEmptyObject){
21542199
consth=vfsState.handlers;
21552200
if(h!==null){

0 commit comments

Comments
Β (0)
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '
Skip to content

Commit 77fedbd

Browse files
codebytereaduh95
authored andcommitted
fs: read small files in one thread pool round trip
fs.readFile(path) took four libuv thread pool round trips for a typical small file -- open, fstat, read and close, each its own uv_fs request with its own queue wait, completion callback and JS/C++ crossing -- and fs.promises.readFile(path) did the same through a FileHandle. For the small files applications read most, the round trips are the cost, and each occupies a slot in the pool that concurrent dns.lookup(), fs and crypto work is also queueing for. Add ReadFileJob (an AsyncWrap + ThreadPoolWork) that performs open + fstat + read-to-EOF + close as one pool task and reports the whole content, or, when the file turns out to be larger than one chunk (kReadFileBufferLength, 512 KiB), stops after fstat() and hands the fd and size back so that the existing chunked reader continues unchanged (large reads stay interleaved and abortable between chunks, and still save the fstat round trip). fs.readFile() and fs.promises.readFile() use it for path arguments without a user buffer; file descriptors, FileHandles, options.buffer and an active VFS keep their paths. Behavior is otherwise kept: same bytes for every size and encoding; open failures report syscall 'open' with the path, read failures 'read'; permission errors are delivered through the callback/promise as before; an abort that arrives while the read is in flight still wins; the job is an FSREQCALLBACK resource for async_hooks; a handed back fd is tracked exactly like one from a plain open(). Tests that asserted the internal open/fstat/read/close request chain, used readFile() as a proxy for an fstat trace event, or injected faults through FileHandle.prototype for path-based reads are adjusted to keep testing what they test (a file just over one chunk where the chain shape matters, fs.fstat() for the fstat trace, a larger file so the FileHandle path is taken). fs.readFile() of 4 KiB files at concurrency 64 goes from ~51k to ~306k files per second, and a mixed stat/readFile/dns.lookup burst from ~66k to ~312k operations per second. Signed-off-by: Shelley Vohr <shelley.vohr@gmail.com> PR-URL: #65327 Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com> Reviewed-By: James M Snell <jasnell@gmail.com>
1 parent e2f952f commit 77fedbd

12 files changed

Lines changed: 613 additions & 30 deletions

β€Žlib/fs.jsβ€Ž

Lines changed: 63 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ const {
8181

8282
const{
8383
FSReqCallback,
84+
ReadFileJob,
8485
}=binding;
8586
const{ toPathIfFileURL }=require('internal/url');
8687
const{
@@ -98,6 +99,7 @@ const {
9899
const{
99100
constants: {
100101
kIoMaxLength,
102+
kReadFileBufferLength,
101103
kMaxUserId,
102104
},
103105
copyObject,
@@ -428,10 +430,70 @@ function readFile(path, options, callback) {
428430
return;
429431

430432
constflagsNumber=stringToFlags(options.flag,'options.flag');
433+
path=getValidatedPath(path);
434+
if(options.buffer===undefined){
435+
// Open + fstat + read + close in one thread pool round trip for files of
436+
// up to one chunk; larger files come back as an open fd + size and take
437+
// the chunked reader below (readFileAfterOneShot). `true`: a handed-back
438+
// fd will be closed through fs.close(), so track it as unmanaged.
439+
constjob=newReadFileJob(path,flagsNumber,kReadFileBufferLength,true);
440+
job.context=context;
441+
job.ondone=readFileAfterOneShot;
442+
constaccessError=job.run(path);
443+
if(accessError!==undefined){
444+
// Not scheduled: report it the way the request-based open() did.
445+
callback(accessError);
446+
}
447+
return;
448+
}
431449
constreq=newFSReqCallback();
432450
req.context=context;
433451
req.oncomplete=readFileAfterOpen;
434-
binding.open(getValidatedPath(path),flagsNumber,0o666,req);
452+
binding.open(path,flagsNumber,0o666,req);
453+
}
454+
455+
functionreadFileAfterOneShot(err,buffer,fd,size,closeErr){
456+
constcontext=this.context;
457+
if(err){
458+
context.callback(err);
459+
return;
460+
}
461+
if(fd!==-1){
462+
// (context.read() below performs the abort check for this case.)
463+
// Larger than one chunk: continue exactly like after open + fstat.
464+
context.fd=fd;
465+
context.size=size;
466+
if(size>kIoMaxLength){
467+
returncontext.close(newERR_FS_FILE_TOO_LARGE(size));
468+
}
469+
try{
470+
context.prepare();
471+
}catch(err){
472+
returncontext.close(err);
473+
}
474+
context.read();
475+
return;
476+
}
477+
if(closeErr){
478+
context.callback(closeErr);
479+
return;
480+
}
481+
if(context.signal?.aborted){
482+
// An abort that arrived while the read was in flight wins, as it did when
483+
// it was noticed between the open/fstat/read steps.
484+
context.callback(newAbortError(undefined,{cause: context.signal.reason}));
485+
return;
486+
}
487+
letresult=buffer;
488+
if(context.encoding){
489+
try{
490+
result=buffer.toString(context.encoding);
491+
}catch(err){
492+
context.callback(err);
493+
return;
494+
}
495+
}
496+
context.callback(null,result);
435497
}
436498

437499
functiontryStatSync(fd,isUserFd){

β€Žlib/internal/fs/promises.jsβ€Ž

Lines changed: 56 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1211,26 +1211,32 @@ async function readFileHandleWithUserBuffer(filehandle, options, size) {
12111211
returnencoding ? buffer.toString(encoding) : buffer.subarray(0,totalRead);
12121212
}
12131213

1214-
asyncfunctionreadFileHandle(filehandle,options){
1214+
asyncfunctionreadFileHandle(filehandle,options,knownRegularFileSize){
12151215
constsignal=options?.signal;
12161216
constencoding=options?.encoding;
12171217
constdecoder=encoding&&newStringDecoder(encoding);
12181218

12191219
checkAborted(signal);
12201220

1221-
conststatFields=awaitPromisePrototypeThen(
1222-
binding.fstat(filehandle.fd,false,kUsePromises),
1223-
undefined,
1224-
handleErrorFromBinding,
1225-
);
1226-
1227-
checkAborted(signal);
1228-
12291221
letsize=0;
12301222
letlength=0;
1231-
if((statFields[1/* mode */]&S_IFMT)===S_IFREG){
1232-
size=statFields[8/* size */];
1223+
if(knownRegularFileSize!==undefined){
1224+
// Handed over by readFile() together with an already open fd.
1225+
size=knownRegularFileSize;
12331226
length=encoding ? MathMin(size,kReadFileBufferLength) : size;
1227+
}else{
1228+
conststatFields=awaitPromisePrototypeThen(
1229+
binding.fstat(filehandle.fd,false,kUsePromises),
1230+
undefined,
1231+
handleErrorFromBinding,
1232+
);
1233+
1234+
checkAborted(signal);
1235+
1236+
if((statFields[1/* mode */]&S_IFMT)===S_IFREG){
1237+
size=statFields[8/* size */];
1238+
length=encoding ? MathMin(size,kReadFileBufferLength) : size;
1239+
}
12341240
}
12351241
if(length===0){
12361242
length=kReadFileUnknownBufferLength;
@@ -2146,10 +2152,49 @@ async function readFile(path, options) {
21462152

21472153
checkAborted(options.signal);
21482154

2155+
if(options.buffer===undefined&&vfsState.handlers===null){
2156+
// Open + fstat + read + close in one thread pool round trip for files of
2157+
// up to one chunk; larger files come back as an open fd + size and are
2158+
// read by readFileHandle() as before.
2159+
path=getValidatedPath(path);
2160+
const{0: buffer,1: fd,2: size}=awaitreadFileInOneRoundTrip(path,stringToFlags(flag));
2161+
if(fd===-1){
2162+
checkAborted(options.signal);// An abort during the read still wins.
2163+
returnoptions.encoding ? buffer.toString(options.encoding) : buffer;
2164+
}
2165+
constfilehandle=newFileHandle(newbinding.FileHandle(fd));
2166+
returnhandleFdClose(readFileHandle(filehandle,options,size),filehandle.close);
2167+
}
2168+
21492169
constfd=awaitopen(path,flag,0o666);
21502170
returnhandleFdClose(readFileHandle(fd,options),fd.close);
21512171
}
21522172

2173+
/**
2174+
* @param {string|Buffer} path Validated path
2175+
* @param {number} flagsNumber
2176+
* @returns {Promise<[Buffer|undefined, number, number|undefined]>} [buffer, -1] or [undefined, fd, size]
2177+
*/
2178+
functionreadFileInOneRoundTrip(path,flagsNumber){
2179+
returnnewPromise((resolve,reject)=>{
2180+
constjob=newbinding.ReadFileJob(path,flagsNumber,kReadFileBufferLength);
2181+
job.ondone=(err,buffer,fd,size,closeErr)=>{
2182+
consterror=err??closeErr;
2183+
if(error!=null){
2184+
ErrorCaptureStackTrace(error,readFileInOneRoundTrip);
2185+
reject(error);
2186+
}else{
2187+
resolve([buffer,fd,size]);
2188+
}
2189+
};
2190+
constaccessError=job.run(path);
2191+
if(accessError!==undefined){
2192+
ErrorCaptureStackTrace(accessError,readFileInOneRoundTrip);
2193+
reject(accessError);
2194+
}
2195+
});
2196+
}
2197+
21532198
asyncfunction*_watch(filename,options=kEmptyObject){
21542199
consth=vfsState.handlers;
21552200
if(h!==null){

0 commit comments

Comments
Β (0)
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix"); } } catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content

Commit 77fedbd

Browse files
codebytereaduh95
authored andcommitted
fs: read small files in one thread pool round trip
fs.readFile(path) took four libuv thread pool round trips for a typical small file -- open, fstat, read and close, each its own uv_fs request with its own queue wait, completion callback and JS/C++ crossing -- and fs.promises.readFile(path) did the same through a FileHandle. For the small files applications read most, the round trips are the cost, and each occupies a slot in the pool that concurrent dns.lookup(), fs and crypto work is also queueing for. Add ReadFileJob (an AsyncWrap + ThreadPoolWork) that performs open + fstat + read-to-EOF + close as one pool task and reports the whole content, or, when the file turns out to be larger than one chunk (kReadFileBufferLength, 512 KiB), stops after fstat() and hands the fd and size back so that the existing chunked reader continues unchanged (large reads stay interleaved and abortable between chunks, and still save the fstat round trip). fs.readFile() and fs.promises.readFile() use it for path arguments without a user buffer; file descriptors, FileHandles, options.buffer and an active VFS keep their paths. Behavior is otherwise kept: same bytes for every size and encoding; open failures report syscall 'open' with the path, read failures 'read'; permission errors are delivered through the callback/promise as before; an abort that arrives while the read is in flight still wins; the job is an FSREQCALLBACK resource for async_hooks; a handed back fd is tracked exactly like one from a plain open(). Tests that asserted the internal open/fstat/read/close request chain, used readFile() as a proxy for an fstat trace event, or injected faults through FileHandle.prototype for path-based reads are adjusted to keep testing what they test (a file just over one chunk where the chain shape matters, fs.fstat() for the fstat trace, a larger file so the FileHandle path is taken). fs.readFile() of 4 KiB files at concurrency 64 goes from ~51k to ~306k files per second, and a mixed stat/readFile/dns.lookup burst from ~66k to ~312k operations per second. Signed-off-by: Shelley Vohr <shelley.vohr@gmail.com> PR-URL: #65327 Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com> Reviewed-By: James M Snell <jasnell@gmail.com>
1 parent e2f952f commit 77fedbd

12 files changed

Lines changed: 613 additions & 30 deletions

β€Žlib/fs.jsβ€Ž

Lines changed: 63 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ const {
8181

8282
const{
8383
FSReqCallback,
84+
ReadFileJob,
8485
}=binding;
8586
const{ toPathIfFileURL }=require('internal/url');
8687
const{
@@ -98,6 +99,7 @@ const {
9899
const{
99100
constants: {
100101
kIoMaxLength,
102+
kReadFileBufferLength,
101103
kMaxUserId,
102104
},
103105
copyObject,
@@ -428,10 +430,70 @@ function readFile(path, options, callback) {
428430
return;
429431

430432
constflagsNumber=stringToFlags(options.flag,'options.flag');
433+
path=getValidatedPath(path);
434+
if(options.buffer===undefined){
435+
// Open + fstat + read + close in one thread pool round trip for files of
436+
// up to one chunk; larger files come back as an open fd + size and take
437+
// the chunked reader below (readFileAfterOneShot). `true`: a handed-back
438+
// fd will be closed through fs.close(), so track it as unmanaged.
439+
constjob=newReadFileJob(path,flagsNumber,kReadFileBufferLength,true);
440+
job.context=context;
441+
job.ondone=readFileAfterOneShot;
442+
constaccessError=job.run(path);
443+
if(accessError!==undefined){
444+
// Not scheduled: report it the way the request-based open() did.
445+
callback(accessError);
446+
}
447+
return;
448+
}
431449
constreq=newFSReqCallback();
432450
req.context=context;
433451
req.oncomplete=readFileAfterOpen;
434-
binding.open(getValidatedPath(path),flagsNumber,0o666,req);
452+
binding.open(path,flagsNumber,0o666,req);
453+
}
454+
455+
functionreadFileAfterOneShot(err,buffer,fd,size,closeErr){
456+
constcontext=this.context;
457+
if(err){
458+
context.callback(err);
459+
return;
460+
}
461+
if(fd!==-1){
462+
// (context.read() below performs the abort check for this case.)
463+
// Larger than one chunk: continue exactly like after open + fstat.
464+
context.fd=fd;
465+
context.size=size;
466+
if(size>kIoMaxLength){
467+
returncontext.close(newERR_FS_FILE_TOO_LARGE(size));
468+
}
469+
try{
470+
context.prepare();
471+
}catch(err){
472+
returncontext.close(err);
473+
}
474+
context.read();
475+
return;
476+
}
477+
if(closeErr){
478+
context.callback(closeErr);
479+
return;
480+
}
481+
if(context.signal?.aborted){
482+
// An abort that arrived while the read was in flight wins, as it did when
483+
// it was noticed between the open/fstat/read steps.
484+
context.callback(newAbortError(undefined,{cause: context.signal.reason}));
485+
return;
486+
}
487+
letresult=buffer;
488+
if(context.encoding){
489+
try{
490+
result=buffer.toString(context.encoding);
491+
}catch(err){
492+
context.callback(err);
493+
return;
494+
}
495+
}
496+
context.callback(null,result);
435497
}
436498

437499
functiontryStatSync(fd,isUserFd){

β€Žlib/internal/fs/promises.jsβ€Ž

Lines changed: 56 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1211,26 +1211,32 @@ async function readFileHandleWithUserBuffer(filehandle, options, size) {
12111211
returnencoding ? buffer.toString(encoding) : buffer.subarray(0,totalRead);
12121212
}
12131213

1214-
asyncfunctionreadFileHandle(filehandle,options){
1214+
asyncfunctionreadFileHandle(filehandle,options,knownRegularFileSize){
12151215
constsignal=options?.signal;
12161216
constencoding=options?.encoding;
12171217
constdecoder=encoding&&newStringDecoder(encoding);
12181218

12191219
checkAborted(signal);
12201220

1221-
conststatFields=awaitPromisePrototypeThen(
1222-
binding.fstat(filehandle.fd,false,kUsePromises),
1223-
undefined,
1224-
handleErrorFromBinding,
1225-
);
1226-
1227-
checkAborted(signal);
1228-
12291221
letsize=0;
12301222
letlength=0;
1231-
if((statFields[1/* mode */]&S_IFMT)===S_IFREG){
1232-
size=statFields[8/* size */];
1223+
if(knownRegularFileSize!==undefined){
1224+
// Handed over by readFile() together with an already open fd.
1225+
size=knownRegularFileSize;
12331226
length=encoding ? MathMin(size,kReadFileBufferLength) : size;
1227+
}else{
1228+
conststatFields=awaitPromisePrototypeThen(
1229+
binding.fstat(filehandle.fd,false,kUsePromises),
1230+
undefined,
1231+
handleErrorFromBinding,
1232+
);
1233+
1234+
checkAborted(signal);
1235+
1236+
if((statFields[1/* mode */]&S_IFMT)===S_IFREG){
1237+
size=statFields[8/* size */];
1238+
length=encoding ? MathMin(size,kReadFileBufferLength) : size;
1239+
}
12341240
}
12351241
if(length===0){
12361242
length=kReadFileUnknownBufferLength;
@@ -2146,10 +2152,49 @@ async function readFile(path, options) {
21462152

21472153
checkAborted(options.signal);
21482154

2155+
if(options.buffer===undefined&&vfsState.handlers===null){
2156+
// Open + fstat + read + close in one thread pool round trip for files of
2157+
// up to one chunk; larger files come back as an open fd + size and are
2158+
// read by readFileHandle() as before.
2159+
path=getValidatedPath(path);
2160+
const{0: buffer,1: fd,2: size}=awaitreadFileInOneRoundTrip(path,stringToFlags(flag));
2161+
if(fd===-1){
2162+
checkAborted(options.signal);// An abort during the read still wins.
2163+
returnoptions.encoding ? buffer.toString(options.encoding) : buffer;
2164+
}
2165+
constfilehandle=newFileHandle(newbinding.FileHandle(fd));
2166+
returnhandleFdClose(readFileHandle(filehandle,options,size),filehandle.close);
2167+
}
2168+
21492169
constfd=awaitopen(path,flag,0o666);
21502170
returnhandleFdClose(readFileHandle(fd,options),fd.close);
21512171
}
21522172

2173+
/**
2174+
* @param {string|Buffer} path Validated path
2175+
* @param {number} flagsNumber
2176+
* @returns {Promise<[Buffer|undefined, number, number|undefined]>} [buffer, -1] or [undefined, fd, size]
2177+
*/
2178+
functionreadFileInOneRoundTrip(path,flagsNumber){
2179+
returnnewPromise((resolve,reject)=>{
2180+
constjob=newbinding.ReadFileJob(path,flagsNumber,kReadFileBufferLength);
2181+
job.ondone=(err,buffer,fd,size,closeErr)=>{
2182+
consterror=err??closeErr;
2183+
if(error!=null){
2184+
ErrorCaptureStackTrace(error,readFileInOneRoundTrip);
2185+
reject(error);
2186+
}else{
2187+
resolve([buffer,fd,size]);
2188+
}
2189+
};
2190+
constaccessError=job.run(path);
2191+
if(accessError!==undefined){
2192+
ErrorCaptureStackTrace(accessError,readFileInOneRoundTrip);
2193+
reject(accessError);
2194+
}
2195+
});
2196+
}
2197+
21532198
asyncfunction*_watch(filename,options=kEmptyObject){
21542199
consth=vfsState.handlers;
21552200
if(h!==null){

0 commit comments

Comments
Β (0)
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms"); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content

Commit 77fedbd

Browse files
codebytereaduh95
authored andcommitted
fs: read small files in one thread pool round trip
fs.readFile(path) took four libuv thread pool round trips for a typical small file -- open, fstat, read and close, each its own uv_fs request with its own queue wait, completion callback and JS/C++ crossing -- and fs.promises.readFile(path) did the same through a FileHandle. For the small files applications read most, the round trips are the cost, and each occupies a slot in the pool that concurrent dns.lookup(), fs and crypto work is also queueing for. Add ReadFileJob (an AsyncWrap + ThreadPoolWork) that performs open + fstat + read-to-EOF + close as one pool task and reports the whole content, or, when the file turns out to be larger than one chunk (kReadFileBufferLength, 512 KiB), stops after fstat() and hands the fd and size back so that the existing chunked reader continues unchanged (large reads stay interleaved and abortable between chunks, and still save the fstat round trip). fs.readFile() and fs.promises.readFile() use it for path arguments without a user buffer; file descriptors, FileHandles, options.buffer and an active VFS keep their paths. Behavior is otherwise kept: same bytes for every size and encoding; open failures report syscall 'open' with the path, read failures 'read'; permission errors are delivered through the callback/promise as before; an abort that arrives while the read is in flight still wins; the job is an FSREQCALLBACK resource for async_hooks; a handed back fd is tracked exactly like one from a plain open(). Tests that asserted the internal open/fstat/read/close request chain, used readFile() as a proxy for an fstat trace event, or injected faults through FileHandle.prototype for path-based reads are adjusted to keep testing what they test (a file just over one chunk where the chain shape matters, fs.fstat() for the fstat trace, a larger file so the FileHandle path is taken). fs.readFile() of 4 KiB files at concurrency 64 goes from ~51k to ~306k files per second, and a mixed stat/readFile/dns.lookup burst from ~66k to ~312k operations per second. Signed-off-by: Shelley Vohr <shelley.vohr@gmail.com> PR-URL: #65327 Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com> Reviewed-By: James M Snell <jasnell@gmail.com>
1 parent e2f952f commit 77fedbd

12 files changed

Lines changed: 613 additions & 30 deletions

β€Žlib/fs.jsβ€Ž

Lines changed: 63 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ const {
8181

8282
const{
8383
FSReqCallback,
84+
ReadFileJob,
8485
}=binding;
8586
const{ toPathIfFileURL }=require('internal/url');
8687
const{
@@ -98,6 +99,7 @@ const {
9899
const{
99100
constants: {
100101
kIoMaxLength,
102+
kReadFileBufferLength,
101103
kMaxUserId,
102104
},
103105
copyObject,
@@ -428,10 +430,70 @@ function readFile(path, options, callback) {
428430
return;
429431

430432
constflagsNumber=stringToFlags(options.flag,'options.flag');
433+
path=getValidatedPath(path);
434+
if(options.buffer===undefined){
435+
// Open + fstat + read + close in one thread pool round trip for files of
436+
// up to one chunk; larger files come back as an open fd + size and take
437+
// the chunked reader below (readFileAfterOneShot). `true`: a handed-back
438+
// fd will be closed through fs.close(), so track it as unmanaged.
439+
constjob=newReadFileJob(path,flagsNumber,kReadFileBufferLength,true);
440+
job.context=context;
441+
job.ondone=readFileAfterOneShot;
442+
constaccessError=job.run(path);
443+
if(accessError!==undefined){
444+
// Not scheduled: report it the way the request-based open() did.
445+
callback(accessError);
446+
}
447+
return;
448+
}
431449
constreq=newFSReqCallback();
432450
req.context=context;
433451
req.oncomplete=readFileAfterOpen;
434-
binding.open(getValidatedPath(path),flagsNumber,0o666,req);
452+
binding.open(path,flagsNumber,0o666,req);
453+
}
454+
455+
functionreadFileAfterOneShot(err,buffer,fd,size,closeErr){
456+
constcontext=this.context;
457+
if(err){
458+
context.callback(err);
459+
return;
460+
}
461+
if(fd!==-1){
462+
// (context.read() below performs the abort check for this case.)
463+
// Larger than one chunk: continue exactly like after open + fstat.
464+
context.fd=fd;
465+
context.size=size;
466+
if(size>kIoMaxLength){
467+
returncontext.close(newERR_FS_FILE_TOO_LARGE(size));
468+
}
469+
try{
470+
context.prepare();
471+
}catch(err){
472+
returncontext.close(err);
473+
}
474+
context.read();
475+
return;
476+
}
477+
if(closeErr){
478+
context.callback(closeErr);
479+
return;
480+
}
481+
if(context.signal?.aborted){
482+
// An abort that arrived while the read was in flight wins, as it did when
483+
// it was noticed between the open/fstat/read steps.
484+
context.callback(newAbortError(undefined,{cause: context.signal.reason}));
485+
return;
486+
}
487+
letresult=buffer;
488+
if(context.encoding){
489+
try{
490+
result=buffer.toString(context.encoding);
491+
}catch(err){
492+
context.callback(err);
493+
return;
494+
}
495+
}
496+
context.callback(null,result);
435497
}
436498

437499
functiontryStatSync(fd,isUserFd){

β€Žlib/internal/fs/promises.jsβ€Ž

Lines changed: 56 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1211,26 +1211,32 @@ async function readFileHandleWithUserBuffer(filehandle, options, size) {
12111211
returnencoding ? buffer.toString(encoding) : buffer.subarray(0,totalRead);
12121212
}
12131213

1214-
asyncfunctionreadFileHandle(filehandle,options){
1214+
asyncfunctionreadFileHandle(filehandle,options,knownRegularFileSize){
12151215
constsignal=options?.signal;
12161216
constencoding=options?.encoding;
12171217
constdecoder=encoding&&newStringDecoder(encoding);
12181218

12191219
checkAborted(signal);
12201220

1221-
conststatFields=awaitPromisePrototypeThen(
1222-
binding.fstat(filehandle.fd,false,kUsePromises),
1223-
undefined,
1224-
handleErrorFromBinding,
1225-
);
1226-
1227-
checkAborted(signal);
1228-
12291221
letsize=0;
12301222
letlength=0;
1231-
if((statFields[1/* mode */]&S_IFMT)===S_IFREG){
1232-
size=statFields[8/* size */];
1223+
if(knownRegularFileSize!==undefined){
1224+
// Handed over by readFile() together with an already open fd.
1225+
size=knownRegularFileSize;
12331226
length=encoding ? MathMin(size,kReadFileBufferLength) : size;
1227+
}else{
1228+
conststatFields=awaitPromisePrototypeThen(
1229+
binding.fstat(filehandle.fd,false,kUsePromises),
1230+
undefined,
1231+
handleErrorFromBinding,
1232+
);
1233+
1234+
checkAborted(signal);
1235+
1236+
if((statFields[1/* mode */]&S_IFMT)===S_IFREG){
1237+
size=statFields[8/* size */];
1238+
length=encoding ? MathMin(size,kReadFileBufferLength) : size;
1239+
}
12341240
}
12351241
if(length===0){
12361242
length=kReadFileUnknownBufferLength;
@@ -2146,10 +2152,49 @@ async function readFile(path, options) {
21462152

21472153
checkAborted(options.signal);
21482154

2155+
if(options.buffer===undefined&&vfsState.handlers===null){
2156+
// Open + fstat + read + close in one thread pool round trip for files of
2157+
// up to one chunk; larger files come back as an open fd + size and are
2158+
// read by readFileHandle() as before.
2159+
path=getValidatedPath(path);
2160+
const{0: buffer,1: fd,2: size}=awaitreadFileInOneRoundTrip(path,stringToFlags(flag));
2161+
if(fd===-1){
2162+
checkAborted(options.signal);// An abort during the read still wins.
2163+
returnoptions.encoding ? buffer.toString(options.encoding) : buffer;
2164+
}
2165+
constfilehandle=newFileHandle(newbinding.FileHandle(fd));
2166+
returnhandleFdClose(readFileHandle(filehandle,options,size),filehandle.close);
2167+
}
2168+
21492169
constfd=awaitopen(path,flag,0o666);
21502170
returnhandleFdClose(readFileHandle(fd,options),fd.close);
21512171
}
21522172

2173+
/**
2174+
* @param {string|Buffer} path Validated path
2175+
* @param {number} flagsNumber
2176+
* @returns {Promise<[Buffer|undefined, number, number|undefined]>} [buffer, -1] or [undefined, fd, size]
2177+
*/
2178+
functionreadFileInOneRoundTrip(path,flagsNumber){
2179+
returnnewPromise((resolve,reject)=>{
2180+
constjob=newbinding.ReadFileJob(path,flagsNumber,kReadFileBufferLength);
2181+
job.ondone=(err,buffer,fd,size,closeErr)=>{
2182+
consterror=err??closeErr;
2183+
if(error!=null){
2184+
ErrorCaptureStackTrace(error,readFileInOneRoundTrip);
2185+
reject(error);
2186+
}else{
2187+
resolve([buffer,fd,size]);
2188+
}
2189+
};
2190+
constaccessError=job.run(path);
2191+
if(accessError!==undefined){
2192+
ErrorCaptureStackTrace(accessError,readFileInOneRoundTrip);
2193+
reject(accessError);
2194+
}
2195+
});
2196+
}
2197+
21532198
asyncfunction*_watch(filename,options=kEmptyObject){
21542199
consth=vfsState.handlers;
21552200
if(h!==null){

0 commit comments

Comments
Β (0)
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links"); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + '
Skip to content

Commit 77fedbd

Browse files
codebytereaduh95
authored andcommitted
fs: read small files in one thread pool round trip
fs.readFile(path) took four libuv thread pool round trips for a typical small file -- open, fstat, read and close, each its own uv_fs request with its own queue wait, completion callback and JS/C++ crossing -- and fs.promises.readFile(path) did the same through a FileHandle. For the small files applications read most, the round trips are the cost, and each occupies a slot in the pool that concurrent dns.lookup(), fs and crypto work is also queueing for. Add ReadFileJob (an AsyncWrap + ThreadPoolWork) that performs open + fstat + read-to-EOF + close as one pool task and reports the whole content, or, when the file turns out to be larger than one chunk (kReadFileBufferLength, 512 KiB), stops after fstat() and hands the fd and size back so that the existing chunked reader continues unchanged (large reads stay interleaved and abortable between chunks, and still save the fstat round trip). fs.readFile() and fs.promises.readFile() use it for path arguments without a user buffer; file descriptors, FileHandles, options.buffer and an active VFS keep their paths. Behavior is otherwise kept: same bytes for every size and encoding; open failures report syscall 'open' with the path, read failures 'read'; permission errors are delivered through the callback/promise as before; an abort that arrives while the read is in flight still wins; the job is an FSREQCALLBACK resource for async_hooks; a handed back fd is tracked exactly like one from a plain open(). Tests that asserted the internal open/fstat/read/close request chain, used readFile() as a proxy for an fstat trace event, or injected faults through FileHandle.prototype for path-based reads are adjusted to keep testing what they test (a file just over one chunk where the chain shape matters, fs.fstat() for the fstat trace, a larger file so the FileHandle path is taken). fs.readFile() of 4 KiB files at concurrency 64 goes from ~51k to ~306k files per second, and a mixed stat/readFile/dns.lookup burst from ~66k to ~312k operations per second. Signed-off-by: Shelley Vohr <shelley.vohr@gmail.com> PR-URL: #65327 Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com> Reviewed-By: James M Snell <jasnell@gmail.com>
1 parent e2f952f commit 77fedbd

12 files changed

Lines changed: 613 additions & 30 deletions

β€Žlib/fs.jsβ€Ž

Lines changed: 63 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ const {
8181

8282
const{
8383
FSReqCallback,
84+
ReadFileJob,
8485
}=binding;
8586
const{ toPathIfFileURL }=require('internal/url');
8687
const{
@@ -98,6 +99,7 @@ const {
9899
const{
99100
constants: {
100101
kIoMaxLength,
102+
kReadFileBufferLength,
101103
kMaxUserId,
102104
},
103105
copyObject,
@@ -428,10 +430,70 @@ function readFile(path, options, callback) {
428430
return;
429431

430432
constflagsNumber=stringToFlags(options.flag,'options.flag');
433+
path=getValidatedPath(path);
434+
if(options.buffer===undefined){
435+
// Open + fstat + read + close in one thread pool round trip for files of
436+
// up to one chunk; larger files come back as an open fd + size and take
437+
// the chunked reader below (readFileAfterOneShot). `true`: a handed-back
438+
// fd will be closed through fs.close(), so track it as unmanaged.
439+
constjob=newReadFileJob(path,flagsNumber,kReadFileBufferLength,true);
440+
job.context=context;
441+
job.ondone=readFileAfterOneShot;
442+
constaccessError=job.run(path);
443+
if(accessError!==undefined){
444+
// Not scheduled: report it the way the request-based open() did.
445+
callback(accessError);
446+
}
447+
return;
448+
}
431449
constreq=newFSReqCallback();
432450
req.context=context;
433451
req.oncomplete=readFileAfterOpen;
434-
binding.open(getValidatedPath(path),flagsNumber,0o666,req);
452+
binding.open(path,flagsNumber,0o666,req);
453+
}
454+
455+
functionreadFileAfterOneShot(err,buffer,fd,size,closeErr){
456+
constcontext=this.context;
457+
if(err){
458+
context.callback(err);
459+
return;
460+
}
461+
if(fd!==-1){
462+
// (context.read() below performs the abort check for this case.)
463+
// Larger than one chunk: continue exactly like after open + fstat.
464+
context.fd=fd;
465+
context.size=size;
466+
if(size>kIoMaxLength){
467+
returncontext.close(newERR_FS_FILE_TOO_LARGE(size));
468+
}
469+
try{
470+
context.prepare();
471+
}catch(err){
472+
returncontext.close(err);
473+
}
474+
context.read();
475+
return;
476+
}
477+
if(closeErr){
478+
context.callback(closeErr);
479+
return;
480+
}
481+
if(context.signal?.aborted){
482+
// An abort that arrived while the read was in flight wins, as it did when
483+
// it was noticed between the open/fstat/read steps.
484+
context.callback(newAbortError(undefined,{cause: context.signal.reason}));
485+
return;
486+
}
487+
letresult=buffer;
488+
if(context.encoding){
489+
try{
490+
result=buffer.toString(context.encoding);
491+
}catch(err){
492+
context.callback(err);
493+
return;
494+
}
495+
}
496+
context.callback(null,result);
435497
}
436498

437499
functiontryStatSync(fd,isUserFd){

β€Žlib/internal/fs/promises.jsβ€Ž

Lines changed: 56 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1211,26 +1211,32 @@ async function readFileHandleWithUserBuffer(filehandle, options, size) {
12111211
returnencoding ? buffer.toString(encoding) : buffer.subarray(0,totalRead);
12121212
}
12131213

1214-
asyncfunctionreadFileHandle(filehandle,options){
1214+
asyncfunctionreadFileHandle(filehandle,options,knownRegularFileSize){
12151215
constsignal=options?.signal;
12161216
constencoding=options?.encoding;
12171217
constdecoder=encoding&&newStringDecoder(encoding);
12181218

12191219
checkAborted(signal);
12201220

1221-
conststatFields=awaitPromisePrototypeThen(
1222-
binding.fstat(filehandle.fd,false,kUsePromises),
1223-
undefined,
1224-
handleErrorFromBinding,
1225-
);
1226-
1227-
checkAborted(signal);
1228-
12291221
letsize=0;
12301222
letlength=0;
1231-
if((statFields[1/* mode */]&S_IFMT)===S_IFREG){
1232-
size=statFields[8/* size */];
1223+
if(knownRegularFileSize!==undefined){
1224+
// Handed over by readFile() together with an already open fd.
1225+
size=knownRegularFileSize;
12331226
length=encoding ? MathMin(size,kReadFileBufferLength) : size;
1227+
}else{
1228+
conststatFields=awaitPromisePrototypeThen(
1229+
binding.fstat(filehandle.fd,false,kUsePromises),
1230+
undefined,
1231+
handleErrorFromBinding,
1232+
);
1233+
1234+
checkAborted(signal);
1235+
1236+
if((statFields[1/* mode */]&S_IFMT)===S_IFREG){
1237+
size=statFields[8/* size */];
1238+
length=encoding ? MathMin(size,kReadFileBufferLength) : size;
1239+
}
12341240
}
12351241
if(length===0){
12361242
length=kReadFileUnknownBufferLength;
@@ -2146,10 +2152,49 @@ async function readFile(path, options) {
21462152

21472153
checkAborted(options.signal);
21482154

2155+
if(options.buffer===undefined&&vfsState.handlers===null){
2156+
// Open + fstat + read + close in one thread pool round trip for files of
2157+
// up to one chunk; larger files come back as an open fd + size and are
2158+
// read by readFileHandle() as before.
2159+
path=getValidatedPath(path);
2160+
const{0: buffer,1: fd,2: size}=awaitreadFileInOneRoundTrip(path,stringToFlags(flag));
2161+
if(fd===-1){
2162+
checkAborted(options.signal);// An abort during the read still wins.
2163+
returnoptions.encoding ? buffer.toString(options.encoding) : buffer;
2164+
}
2165+
constfilehandle=newFileHandle(newbinding.FileHandle(fd));
2166+
returnhandleFdClose(readFileHandle(filehandle,options,size),filehandle.close);
2167+
}
2168+
21492169
constfd=awaitopen(path,flag,0o666);
21502170
returnhandleFdClose(readFileHandle(fd,options),fd.close);
21512171
}
21522172

2173+
/**
2174+
* @param {string|Buffer} path Validated path
2175+
* @param {number} flagsNumber
2176+
* @returns {Promise<[Buffer|undefined, number, number|undefined]>} [buffer, -1] or [undefined, fd, size]
2177+
*/
2178+
functionreadFileInOneRoundTrip(path,flagsNumber){
2179+
returnnewPromise((resolve,reject)=>{
2180+
constjob=newbinding.ReadFileJob(path,flagsNumber,kReadFileBufferLength);
2181+
job.ondone=(err,buffer,fd,size,closeErr)=>{
2182+
consterror=err??closeErr;
2183+
if(error!=null){
2184+
ErrorCaptureStackTrace(error,readFileInOneRoundTrip);
2185+
reject(error);
2186+
}else{
2187+
resolve([buffer,fd,size]);
2188+
}
2189+
};
2190+
constaccessError=job.run(path);
2191+
if(accessError!==undefined){
2192+
ErrorCaptureStackTrace(accessError,readFileInOneRoundTrip);
2193+
reject(accessError);
2194+
}
2195+
});
2196+
}
2197+
21532198
asyncfunction*_watch(filename,options=kEmptyObject){
21542199
consth=vfsState.handlers;
21552200
if(h!==null){

0 commit comments

Comments
Β (0)
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default"); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content

Commit 77fedbd

Browse files
codebytereaduh95
authored andcommitted
fs: read small files in one thread pool round trip
fs.readFile(path) took four libuv thread pool round trips for a typical small file -- open, fstat, read and close, each its own uv_fs request with its own queue wait, completion callback and JS/C++ crossing -- and fs.promises.readFile(path) did the same through a FileHandle. For the small files applications read most, the round trips are the cost, and each occupies a slot in the pool that concurrent dns.lookup(), fs and crypto work is also queueing for. Add ReadFileJob (an AsyncWrap + ThreadPoolWork) that performs open + fstat + read-to-EOF + close as one pool task and reports the whole content, or, when the file turns out to be larger than one chunk (kReadFileBufferLength, 512 KiB), stops after fstat() and hands the fd and size back so that the existing chunked reader continues unchanged (large reads stay interleaved and abortable between chunks, and still save the fstat round trip). fs.readFile() and fs.promises.readFile() use it for path arguments without a user buffer; file descriptors, FileHandles, options.buffer and an active VFS keep their paths. Behavior is otherwise kept: same bytes for every size and encoding; open failures report syscall 'open' with the path, read failures 'read'; permission errors are delivered through the callback/promise as before; an abort that arrives while the read is in flight still wins; the job is an FSREQCALLBACK resource for async_hooks; a handed back fd is tracked exactly like one from a plain open(). Tests that asserted the internal open/fstat/read/close request chain, used readFile() as a proxy for an fstat trace event, or injected faults through FileHandle.prototype for path-based reads are adjusted to keep testing what they test (a file just over one chunk where the chain shape matters, fs.fstat() for the fstat trace, a larger file so the FileHandle path is taken). fs.readFile() of 4 KiB files at concurrency 64 goes from ~51k to ~306k files per second, and a mixed stat/readFile/dns.lookup burst from ~66k to ~312k operations per second. Signed-off-by: Shelley Vohr <shelley.vohr@gmail.com> PR-URL: #65327 Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com> Reviewed-By: James M Snell <jasnell@gmail.com>
1 parent e2f952f commit 77fedbd

12 files changed

Lines changed: 613 additions & 30 deletions

β€Žlib/fs.jsβ€Ž

Lines changed: 63 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ const {
8181

8282
const{
8383
FSReqCallback,
84+
ReadFileJob,
8485
}=binding;
8586
const{ toPathIfFileURL }=require('internal/url');
8687
const{
@@ -98,6 +99,7 @@ const {
9899
const{
99100
constants: {
100101
kIoMaxLength,
102+
kReadFileBufferLength,
101103
kMaxUserId,
102104
},
103105
copyObject,
@@ -428,10 +430,70 @@ function readFile(path, options, callback) {
428430
return;
429431

430432
constflagsNumber=stringToFlags(options.flag,'options.flag');
433+
path=getValidatedPath(path);
434+
if(options.buffer===undefined){
435+
// Open + fstat + read + close in one thread pool round trip for files of
436+
// up to one chunk; larger files come back as an open fd + size and take
437+
// the chunked reader below (readFileAfterOneShot). `true`: a handed-back
438+
// fd will be closed through fs.close(), so track it as unmanaged.
439+
constjob=newReadFileJob(path,flagsNumber,kReadFileBufferLength,true);
440+
job.context=context;
441+
job.ondone=readFileAfterOneShot;
442+
constaccessError=job.run(path);
443+
if(accessError!==undefined){
444+
// Not scheduled: report it the way the request-based open() did.
445+
callback(accessError);
446+
}
447+
return;
448+
}
431449
constreq=newFSReqCallback();
432450
req.context=context;
433451
req.oncomplete=readFileAfterOpen;
434-
binding.open(getValidatedPath(path),flagsNumber,0o666,req);
452+
binding.open(path,flagsNumber,0o666,req);
453+
}
454+
455+
functionreadFileAfterOneShot(err,buffer,fd,size,closeErr){
456+
constcontext=this.context;
457+
if(err){
458+
context.callback(err);
459+
return;
460+
}
461+
if(fd!==-1){
462+
// (context.read() below performs the abort check for this case.)
463+
// Larger than one chunk: continue exactly like after open + fstat.
464+
context.fd=fd;
465+
context.size=size;
466+
if(size>kIoMaxLength){
467+
returncontext.close(newERR_FS_FILE_TOO_LARGE(size));
468+
}
469+
try{
470+
context.prepare();
471+
}catch(err){
472+
returncontext.close(err);
473+
}
474+
context.read();
475+
return;
476+
}
477+
if(closeErr){
478+
context.callback(closeErr);
479+
return;
480+
}
481+
if(context.signal?.aborted){
482+
// An abort that arrived while the read was in flight wins, as it did when
483+
// it was noticed between the open/fstat/read steps.
484+
context.callback(newAbortError(undefined,{cause: context.signal.reason}));
485+
return;
486+
}
487+
letresult=buffer;
488+
if(context.encoding){
489+
try{
490+
result=buffer.toString(context.encoding);
491+
}catch(err){
492+
context.callback(err);
493+
return;
494+
}
495+
}
496+
context.callback(null,result);
435497
}
436498

437499
functiontryStatSync(fd,isUserFd){

β€Žlib/internal/fs/promises.jsβ€Ž

Lines changed: 56 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1211,26 +1211,32 @@ async function readFileHandleWithUserBuffer(filehandle, options, size) {
12111211
returnencoding ? buffer.toString(encoding) : buffer.subarray(0,totalRead);
12121212
}
12131213

1214-
asyncfunctionreadFileHandle(filehandle,options){
1214+
asyncfunctionreadFileHandle(filehandle,options,knownRegularFileSize){
12151215
constsignal=options?.signal;
12161216
constencoding=options?.encoding;
12171217
constdecoder=encoding&&newStringDecoder(encoding);
12181218

12191219
checkAborted(signal);
12201220

1221-
conststatFields=awaitPromisePrototypeThen(
1222-
binding.fstat(filehandle.fd,false,kUsePromises),
1223-
undefined,
1224-
handleErrorFromBinding,
1225-
);
1226-
1227-
checkAborted(signal);
1228-
12291221
letsize=0;
12301222
letlength=0;
1231-
if((statFields[1/* mode */]&S_IFMT)===S_IFREG){
1232-
size=statFields[8/* size */];
1223+
if(knownRegularFileSize!==undefined){
1224+
// Handed over by readFile() together with an already open fd.
1225+
size=knownRegularFileSize;
12331226
length=encoding ? MathMin(size,kReadFileBufferLength) : size;
1227+
}else{
1228+
conststatFields=awaitPromisePrototypeThen(
1229+
binding.fstat(filehandle.fd,false,kUsePromises),
1230+
undefined,
1231+
handleErrorFromBinding,
1232+
);
1233+
1234+
checkAborted(signal);
1235+
1236+
if((statFields[1/* mode */]&S_IFMT)===S_IFREG){
1237+
size=statFields[8/* size */];
1238+
length=encoding ? MathMin(size,kReadFileBufferLength) : size;
1239+
}
12341240
}
12351241
if(length===0){
12361242
length=kReadFileUnknownBufferLength;
@@ -2146,10 +2152,49 @@ async function readFile(path, options) {
21462152

21472153
checkAborted(options.signal);
21482154

2155+
if(options.buffer===undefined&&vfsState.handlers===null){
2156+
// Open + fstat + read + close in one thread pool round trip for files of
2157+
// up to one chunk; larger files come back as an open fd + size and are
2158+
// read by readFileHandle() as before.
2159+
path=getValidatedPath(path);
2160+
const{0: buffer,1: fd,2: size}=awaitreadFileInOneRoundTrip(path,stringToFlags(flag));
2161+
if(fd===-1){
2162+
checkAborted(options.signal);// An abort during the read still wins.
2163+
returnoptions.encoding ? buffer.toString(options.encoding) : buffer;
2164+
}
2165+
constfilehandle=newFileHandle(newbinding.FileHandle(fd));
2166+
returnhandleFdClose(readFileHandle(filehandle,options,size),filehandle.close);
2167+
}
2168+
21492169
constfd=awaitopen(path,flag,0o666);
21502170
returnhandleFdClose(readFileHandle(fd,options),fd.close);
21512171
}
21522172

2173+
/**
2174+
* @param {string|Buffer} path Validated path
2175+
* @param {number} flagsNumber
2176+
* @returns {Promise<[Buffer|undefined, number, number|undefined]>} [buffer, -1] or [undefined, fd, size]
2177+
*/
2178+
functionreadFileInOneRoundTrip(path,flagsNumber){
2179+
returnnewPromise((resolve,reject)=>{
2180+
constjob=newbinding.ReadFileJob(path,flagsNumber,kReadFileBufferLength);
2181+
job.ondone=(err,buffer,fd,size,closeErr)=>{
2182+
consterror=err??closeErr;
2183+
if(error!=null){
2184+
ErrorCaptureStackTrace(error,readFileInOneRoundTrip);
2185+
reject(error);
2186+
}else{
2187+
resolve([buffer,fd,size]);
2188+
}
2189+
};
2190+
constaccessError=job.run(path);
2191+
if(accessError!==undefined){
2192+
ErrorCaptureStackTrace(accessError,readFileInOneRoundTrip);
2193+
reject(accessError);
2194+
}
2195+
});
2196+
}
2197+
21532198
asyncfunction*_watch(filename,options=kEmptyObject){
21542199
consth=vfsState.handlers;
21552200
if(h!==null){

0 commit comments

Comments
Β (0)
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers"); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content

Commit 77fedbd

Browse files
codebytereaduh95
authored andcommitted
fs: read small files in one thread pool round trip
fs.readFile(path) took four libuv thread pool round trips for a typical small file -- open, fstat, read and close, each its own uv_fs request with its own queue wait, completion callback and JS/C++ crossing -- and fs.promises.readFile(path) did the same through a FileHandle. For the small files applications read most, the round trips are the cost, and each occupies a slot in the pool that concurrent dns.lookup(), fs and crypto work is also queueing for. Add ReadFileJob (an AsyncWrap + ThreadPoolWork) that performs open + fstat + read-to-EOF + close as one pool task and reports the whole content, or, when the file turns out to be larger than one chunk (kReadFileBufferLength, 512 KiB), stops after fstat() and hands the fd and size back so that the existing chunked reader continues unchanged (large reads stay interleaved and abortable between chunks, and still save the fstat round trip). fs.readFile() and fs.promises.readFile() use it for path arguments without a user buffer; file descriptors, FileHandles, options.buffer and an active VFS keep their paths. Behavior is otherwise kept: same bytes for every size and encoding; open failures report syscall 'open' with the path, read failures 'read'; permission errors are delivered through the callback/promise as before; an abort that arrives while the read is in flight still wins; the job is an FSREQCALLBACK resource for async_hooks; a handed back fd is tracked exactly like one from a plain open(). Tests that asserted the internal open/fstat/read/close request chain, used readFile() as a proxy for an fstat trace event, or injected faults through FileHandle.prototype for path-based reads are adjusted to keep testing what they test (a file just over one chunk where the chain shape matters, fs.fstat() for the fstat trace, a larger file so the FileHandle path is taken). fs.readFile() of 4 KiB files at concurrency 64 goes from ~51k to ~306k files per second, and a mixed stat/readFile/dns.lookup burst from ~66k to ~312k operations per second. Signed-off-by: Shelley Vohr <shelley.vohr@gmail.com> PR-URL: #65327 Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com> Reviewed-By: James M Snell <jasnell@gmail.com>
1 parent e2f952f commit 77fedbd

12 files changed

Lines changed: 613 additions & 30 deletions

β€Žlib/fs.jsβ€Ž

Lines changed: 63 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ const {
8181

8282
const{
8383
FSReqCallback,
84+
ReadFileJob,
8485
}=binding;
8586
const{ toPathIfFileURL }=require('internal/url');
8687
const{
@@ -98,6 +99,7 @@ const {
9899
const{
99100
constants: {
100101
kIoMaxLength,
102+
kReadFileBufferLength,
101103
kMaxUserId,
102104
},
103105
copyObject,
@@ -428,10 +430,70 @@ function readFile(path, options, callback) {
428430
return;
429431

430432
constflagsNumber=stringToFlags(options.flag,'options.flag');
433+
path=getValidatedPath(path);
434+
if(options.buffer===undefined){
435+
// Open + fstat + read + close in one thread pool round trip for files of
436+
// up to one chunk; larger files come back as an open fd + size and take
437+
// the chunked reader below (readFileAfterOneShot). `true`: a handed-back
438+
// fd will be closed through fs.close(), so track it as unmanaged.
439+
constjob=newReadFileJob(path,flagsNumber,kReadFileBufferLength,true);
440+
job.context=context;
441+
job.ondone=readFileAfterOneShot;
442+
constaccessError=job.run(path);
443+
if(accessError!==undefined){
444+
// Not scheduled: report it the way the request-based open() did.
445+
callback(accessError);
446+
}
447+
return;
448+
}
431449
constreq=newFSReqCallback();
432450
req.context=context;
433451
req.oncomplete=readFileAfterOpen;
434-
binding.open(getValidatedPath(path),flagsNumber,0o666,req);
452+
binding.open(path,flagsNumber,0o666,req);
453+
}
454+
455+
functionreadFileAfterOneShot(err,buffer,fd,size,closeErr){
456+
constcontext=this.context;
457+
if(err){
458+
context.callback(err);
459+
return;
460+
}
461+
if(fd!==-1){
462+
// (context.read() below performs the abort check for this case.)
463+
// Larger than one chunk: continue exactly like after open + fstat.
464+
context.fd=fd;
465+
context.size=size;
466+
if(size>kIoMaxLength){
467+
returncontext.close(newERR_FS_FILE_TOO_LARGE(size));
468+
}
469+
try{
470+
context.prepare();
471+
}catch(err){
472+
returncontext.close(err);
473+
}
474+
context.read();
475+
return;
476+
}
477+
if(closeErr){
478+
context.callback(closeErr);
479+
return;
480+
}
481+
if(context.signal?.aborted){
482+
// An abort that arrived while the read was in flight wins, as it did when
483+
// it was noticed between the open/fstat/read steps.
484+
context.callback(newAbortError(undefined,{cause: context.signal.reason}));
485+
return;
486+
}
487+
letresult=buffer;
488+
if(context.encoding){
489+
try{
490+
result=buffer.toString(context.encoding);
491+
}catch(err){
492+
context.callback(err);
493+
return;
494+
}
495+
}
496+
context.callback(null,result);
435497
}
436498

437499
functiontryStatSync(fd,isUserFd){

β€Žlib/internal/fs/promises.jsβ€Ž

Lines changed: 56 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1211,26 +1211,32 @@ async function readFileHandleWithUserBuffer(filehandle, options, size) {
12111211
returnencoding ? buffer.toString(encoding) : buffer.subarray(0,totalRead);
12121212
}
12131213

1214-
asyncfunctionreadFileHandle(filehandle,options){
1214+
asyncfunctionreadFileHandle(filehandle,options,knownRegularFileSize){
12151215
constsignal=options?.signal;
12161216
constencoding=options?.encoding;
12171217
constdecoder=encoding&&newStringDecoder(encoding);
12181218

12191219
checkAborted(signal);
12201220

1221-
conststatFields=awaitPromisePrototypeThen(
1222-
binding.fstat(filehandle.fd,false,kUsePromises),
1223-
undefined,
1224-
handleErrorFromBinding,
1225-
);
1226-
1227-
checkAborted(signal);
1228-
12291221
letsize=0;
12301222
letlength=0;
1231-
if((statFields[1/* mode */]&S_IFMT)===S_IFREG){
1232-
size=statFields[8/* size */];
1223+
if(knownRegularFileSize!==undefined){
1224+
// Handed over by readFile() together with an already open fd.
1225+
size=knownRegularFileSize;
12331226
length=encoding ? MathMin(size,kReadFileBufferLength) : size;
1227+
}else{
1228+
conststatFields=awaitPromisePrototypeThen(
1229+
binding.fstat(filehandle.fd,false,kUsePromises),
1230+
undefined,
1231+
handleErrorFromBinding,
1232+
);
1233+
1234+
checkAborted(signal);
1235+
1236+
if((statFields[1/* mode */]&S_IFMT)===S_IFREG){
1237+
size=statFields[8/* size */];
1238+
length=encoding ? MathMin(size,kReadFileBufferLength) : size;
1239+
}
12341240
}
12351241
if(length===0){
12361242
length=kReadFileUnknownBufferLength;
@@ -2146,10 +2152,49 @@ async function readFile(path, options) {
21462152

21472153
checkAborted(options.signal);
21482154

2155+
if(options.buffer===undefined&&vfsState.handlers===null){
2156+
// Open + fstat + read + close in one thread pool round trip for files of
2157+
// up to one chunk; larger files come back as an open fd + size and are
2158+
// read by readFileHandle() as before.
2159+
path=getValidatedPath(path);
2160+
const{0: buffer,1: fd,2: size}=awaitreadFileInOneRoundTrip(path,stringToFlags(flag));
2161+
if(fd===-1){
2162+
checkAborted(options.signal);// An abort during the read still wins.
2163+
returnoptions.encoding ? buffer.toString(options.encoding) : buffer;
2164+
}
2165+
constfilehandle=newFileHandle(newbinding.FileHandle(fd));
2166+
returnhandleFdClose(readFileHandle(filehandle,options,size),filehandle.close);
2167+
}
2168+
21492169
constfd=awaitopen(path,flag,0o666);
21502170
returnhandleFdClose(readFileHandle(fd,options),fd.close);
21512171
}
21522172

2173+
/**
2174+
* @param {string|Buffer} path Validated path
2175+
* @param {number} flagsNumber
2176+
* @returns {Promise<[Buffer|undefined, number, number|undefined]>} [buffer, -1] or [undefined, fd, size]
2177+
*/
2178+
functionreadFileInOneRoundTrip(path,flagsNumber){
2179+
returnnewPromise((resolve,reject)=>{
2180+
constjob=newbinding.ReadFileJob(path,flagsNumber,kReadFileBufferLength);
2181+
job.ondone=(err,buffer,fd,size,closeErr)=>{
2182+
consterror=err??closeErr;
2183+
if(error!=null){
2184+
ErrorCaptureStackTrace(error,readFileInOneRoundTrip);
2185+
reject(error);
2186+
}else{
2187+
resolve([buffer,fd,size]);
2188+
}
2189+
};
2190+
constaccessError=job.run(path);
2191+
if(accessError!==undefined){
2192+
ErrorCaptureStackTrace(accessError,readFileInOneRoundTrip);
2193+
reject(accessError);
2194+
}
2195+
});
2196+
}
2197+
21532198
asyncfunction*_watch(filename,options=kEmptyObject){
21542199
consth=vfsState.handlers;
21552200
if(h!==null){

0 commit comments

Comments
Β (0)
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode"); } } catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); } })(); })();
Skip to content

Commit 77fedbd

Browse files
codebytereaduh95
authored andcommitted
fs: read small files in one thread pool round trip
fs.readFile(path) took four libuv thread pool round trips for a typical small file -- open, fstat, read and close, each its own uv_fs request with its own queue wait, completion callback and JS/C++ crossing -- and fs.promises.readFile(path) did the same through a FileHandle. For the small files applications read most, the round trips are the cost, and each occupies a slot in the pool that concurrent dns.lookup(), fs and crypto work is also queueing for. Add ReadFileJob (an AsyncWrap + ThreadPoolWork) that performs open + fstat + read-to-EOF + close as one pool task and reports the whole content, or, when the file turns out to be larger than one chunk (kReadFileBufferLength, 512 KiB), stops after fstat() and hands the fd and size back so that the existing chunked reader continues unchanged (large reads stay interleaved and abortable between chunks, and still save the fstat round trip). fs.readFile() and fs.promises.readFile() use it for path arguments without a user buffer; file descriptors, FileHandles, options.buffer and an active VFS keep their paths. Behavior is otherwise kept: same bytes for every size and encoding; open failures report syscall 'open' with the path, read failures 'read'; permission errors are delivered through the callback/promise as before; an abort that arrives while the read is in flight still wins; the job is an FSREQCALLBACK resource for async_hooks; a handed back fd is tracked exactly like one from a plain open(). Tests that asserted the internal open/fstat/read/close request chain, used readFile() as a proxy for an fstat trace event, or injected faults through FileHandle.prototype for path-based reads are adjusted to keep testing what they test (a file just over one chunk where the chain shape matters, fs.fstat() for the fstat trace, a larger file so the FileHandle path is taken). fs.readFile() of 4 KiB files at concurrency 64 goes from ~51k to ~306k files per second, and a mixed stat/readFile/dns.lookup burst from ~66k to ~312k operations per second. Signed-off-by: Shelley Vohr <shelley.vohr@gmail.com> PR-URL: #65327 Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com> Reviewed-By: James M Snell <jasnell@gmail.com>
1 parent e2f952f commit 77fedbd

12 files changed

Lines changed: 613 additions & 30 deletions

β€Žlib/fs.jsβ€Ž

Lines changed: 63 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ const {
8181

8282
const{
8383
FSReqCallback,
84+
ReadFileJob,
8485
}=binding;
8586
const{ toPathIfFileURL }=require('internal/url');
8687
const{
@@ -98,6 +99,7 @@ const {
9899
const{
99100
constants: {
100101
kIoMaxLength,
102+
kReadFileBufferLength,
101103
kMaxUserId,
102104
},
103105
copyObject,
@@ -428,10 +430,70 @@ function readFile(path, options, callback) {
428430
return;
429431

430432
constflagsNumber=stringToFlags(options.flag,'options.flag');
433+
path=getValidatedPath(path);
434+
if(options.buffer===undefined){
435+
// Open + fstat + read + close in one thread pool round trip for files of
436+
// up to one chunk; larger files come back as an open fd + size and take
437+
// the chunked reader below (readFileAfterOneShot). `true`: a handed-back
438+
// fd will be closed through fs.close(), so track it as unmanaged.
439+
constjob=newReadFileJob(path,flagsNumber,kReadFileBufferLength,true);
440+
job.context=context;
441+
job.ondone=readFileAfterOneShot;
442+
constaccessError=job.run(path);
443+
if(accessError!==undefined){
444+
// Not scheduled: report it the way the request-based open() did.
445+
callback(accessError);
446+
}
447+
return;
448+
}
431449
constreq=newFSReqCallback();
432450
req.context=context;
433451
req.oncomplete=readFileAfterOpen;
434-
binding.open(getValidatedPath(path),flagsNumber,0o666,req);
452+
binding.open(path,flagsNumber,0o666,req);
453+
}
454+
455+
functionreadFileAfterOneShot(err,buffer,fd,size,closeErr){
456+
constcontext=this.context;
457+
if(err){
458+
context.callback(err);
459+
return;
460+
}
461+
if(fd!==-1){
462+
// (context.read() below performs the abort check for this case.)
463+
// Larger than one chunk: continue exactly like after open + fstat.
464+
context.fd=fd;
465+
context.size=size;
466+
if(size>kIoMaxLength){
467+
returncontext.close(newERR_FS_FILE_TOO_LARGE(size));
468+
}
469+
try{
470+
context.prepare();
471+
}catch(err){
472+
returncontext.close(err);
473+
}
474+
context.read();
475+
return;
476+
}
477+
if(closeErr){
478+
context.callback(closeErr);
479+
return;
480+
}
481+
if(context.signal?.aborted){
482+
// An abort that arrived while the read was in flight wins, as it did when
483+
// it was noticed between the open/fstat/read steps.
484+
context.callback(newAbortError(undefined,{cause: context.signal.reason}));
485+
return;
486+
}
487+
letresult=buffer;
488+
if(context.encoding){
489+
try{
490+
result=buffer.toString(context.encoding);
491+
}catch(err){
492+
context.callback(err);
493+
return;
494+
}
495+
}
496+
context.callback(null,result);
435497
}
436498

437499
functiontryStatSync(fd,isUserFd){

β€Žlib/internal/fs/promises.jsβ€Ž

Lines changed: 56 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1211,26 +1211,32 @@ async function readFileHandleWithUserBuffer(filehandle, options, size) {
12111211
returnencoding ? buffer.toString(encoding) : buffer.subarray(0,totalRead);
12121212
}
12131213

1214-
asyncfunctionreadFileHandle(filehandle,options){
1214+
asyncfunctionreadFileHandle(filehandle,options,knownRegularFileSize){
12151215
constsignal=options?.signal;
12161216
constencoding=options?.encoding;
12171217
constdecoder=encoding&&newStringDecoder(encoding);
12181218

12191219
checkAborted(signal);
12201220

1221-
conststatFields=awaitPromisePrototypeThen(
1222-
binding.fstat(filehandle.fd,false,kUsePromises),
1223-
undefined,
1224-
handleErrorFromBinding,
1225-
);
1226-
1227-
checkAborted(signal);
1228-
12291221
letsize=0;
12301222
letlength=0;
1231-
if((statFields[1/* mode */]&S_IFMT)===S_IFREG){
1232-
size=statFields[8/* size */];
1223+
if(knownRegularFileSize!==undefined){
1224+
// Handed over by readFile() together with an already open fd.
1225+
size=knownRegularFileSize;
12331226
length=encoding ? MathMin(size,kReadFileBufferLength) : size;
1227+
}else{
1228+
conststatFields=awaitPromisePrototypeThen(
1229+
binding.fstat(filehandle.fd,false,kUsePromises),
1230+
undefined,
1231+
handleErrorFromBinding,
1232+
);
1233+
1234+
checkAborted(signal);
1235+
1236+
if((statFields[1/* mode */]&S_IFMT)===S_IFREG){
1237+
size=statFields[8/* size */];
1238+
length=encoding ? MathMin(size,kReadFileBufferLength) : size;
1239+
}
12341240
}
12351241
if(length===0){
12361242
length=kReadFileUnknownBufferLength;
@@ -2146,10 +2152,49 @@ async function readFile(path, options) {
21462152

21472153
checkAborted(options.signal);
21482154

2155+
if(options.buffer===undefined&&vfsState.handlers===null){
2156+
// Open + fstat + read + close in one thread pool round trip for files of
2157+
// up to one chunk; larger files come back as an open fd + size and are
2158+
// read by readFileHandle() as before.
2159+
path=getValidatedPath(path);
2160+
const{0: buffer,1: fd,2: size}=awaitreadFileInOneRoundTrip(path,stringToFlags(flag));
2161+
if(fd===-1){
2162+
checkAborted(options.signal);// An abort during the read still wins.
2163+
returnoptions.encoding ? buffer.toString(options.encoding) : buffer;
2164+
}
2165+
constfilehandle=newFileHandle(newbinding.FileHandle(fd));
2166+
returnhandleFdClose(readFileHandle(filehandle,options,size),filehandle.close);
2167+
}
2168+
21492169
constfd=awaitopen(path,flag,0o666);
21502170
returnhandleFdClose(readFileHandle(fd,options),fd.close);
21512171
}
21522172

2173+
/**
2174+
* @param {string|Buffer} path Validated path
2175+
* @param {number} flagsNumber
2176+
* @returns {Promise<[Buffer|undefined, number, number|undefined]>} [buffer, -1] or [undefined, fd, size]
2177+
*/
2178+
functionreadFileInOneRoundTrip(path,flagsNumber){
2179+
returnnewPromise((resolve,reject)=>{
2180+
constjob=newbinding.ReadFileJob(path,flagsNumber,kReadFileBufferLength);
2181+
job.ondone=(err,buffer,fd,size,closeErr)=>{
2182+
consterror=err??closeErr;
2183+
if(error!=null){
2184+
ErrorCaptureStackTrace(error,readFileInOneRoundTrip);
2185+
reject(error);
2186+
}else{
2187+
resolve([buffer,fd,size]);
2188+
}
2189+
};
2190+
constaccessError=job.run(path);
2191+
if(accessError!==undefined){
2192+
ErrorCaptureStackTrace(accessError,readFileInOneRoundTrip);
2193+
reject(accessError);
2194+
}
2195+
});
2196+
}
2197+
21532198
asyncfunction*_watch(filename,options=kEmptyObject){
21542199
consth=vfsState.handlers;
21552200
if(h!==null){

0 commit comments

Comments
Β (0)