fs: use sized reads for large files in readFileUtf8

fs.readFileSync(path, 'utf8') read the whole file in 8 KiB read() calls
appended to a std::string, i.e. one syscall and a potential
reallocation per 8 KiB (an 8 MiB file took ~1400 read() calls).
Keep the exact old sequence for small files (one read into the 8 KiB
stack buffer, one read reporting EOF). Once a read fills the stack
buffer, read the rest directly into one heap buffer sized from fstat()
(plus one byte so that the EOF read does not force growth), growing
geometrically only when the size is unavailable or wrong. The size is
only an allocation hint: reading continues until read() reports EOF,
so procfs/sysfs files, FIFOs, files that change while being read and
file descriptors positioned mid-file behave as before, and the bytes
handed to StringBytes::Encode() are exactly the ones read.
Signed-off-by: Shelley Vohr <shelley.vohr@gmail.com>
PR-URL: #65328
Reviewed-By: Matteo Collina <matteo.collina@gmail.com>
Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com>

Author: codebytere

src/node_file.cc

@@ -2931,10 +2931,20 @@ static void ReadFileUtf8(const FunctionCallbackInfo<Value>& args) {
 uv_fs_req_cleanup(&req);
  });
 
+// Past the first 8 KiB, read into one heap buffer sized from fstat(); the
+// size is only a hint, reading continues until read() reports EOF.
  std::string result{};
 char buffer[8192];
 uv_buf_t buf = uv_buf_init(buffer, sizeof(buffer));
 
+char* big = nullptr;
+size_t big_len = 0;
+size_t big_cap = 0;
+bool sized = false;
+auto free_big = OnScopeLeave([&big]() { free(big); });
+constexprsize_tkMinChunk = 64 * 1024;
+constexprsize_tkMaxChunk = 8 * 1024 * 1024;
+
 FS_SYNC_TRACE_BEGIN(read);
 while (true) {
 auto r = uv_fs_read(nullptr, &req, file, &buf, 1, -1, nullptr);
@@ -2947,12 +2957,62 @@ static void ReadFileUtf8(const FunctionCallbackInfo<Value>& args) {
 if (r <= 0) {
 break;
  }
- result.append(buf.base, r);
+if (big == nullptr) {
+ result.append(buf.base, r);
+if (static_cast<size_t>(r) < sizeof(buffer)) {
+continue;
+ }
+// Switch to the heap buffer.
+uv_fs_req_cleanup(&req);
+ big_cap = kMinChunk;
+ big = UncheckedMalloc<char>(big_cap);
+if (big == nullptr) {
+FS_SYNC_TRACE_END(read);
+returnTHROW_ERR_MEMORY_ALLOCATION_FAILED(env);
+ }
+memcpy(big, result.data(), result.size());
+ big_len = result.size();
+ result = std::string();
+ } else {
+ big_len += static_cast<size_t>(r);
+ }
+if (big_len == big_cap) {
+// +1 leaves room for the read() that reports EOF.
+size_t new_cap =
+ big_cap + std::min(kMaxChunk, std::max(kMinChunk, big_cap));
+if (!sized) {
+ sized = true;
+uv_fs_req_cleanup(&req);
+uv_fs_t stat_req;
+if (uv_fs_fstat(nullptr, &stat_req, file, nullptr) == 0) {
+constuv_stat_t* const st =
+static_cast<constuv_stat_t*>(stat_req.ptr);
+if ((st->st_mode & S_IFMT) == S_IFREG &&
+static_cast<uint64_t>(st->st_size) > big_len &&
+static_cast<uint64_t>(st->st_size) <
+static_cast<uint64_t>(v8::String::kMaxLength)) {
+ new_cap = static_cast<size_t>(st->st_size) + 1;
+ }
+ }
+uv_fs_req_cleanup(&stat_req);
+ }
+char* const grown = UncheckedRealloc<char>(big, new_cap);
+if (grown == nullptr) {
+FS_SYNC_TRACE_END(read);
+returnTHROW_ERR_MEMORY_ALLOCATION_FAILED(env);
+ }
+ big = grown;
+ big_cap = new_cap;
+ }
+ buf = uv_buf_init(big + big_len, std::min(kMaxChunk, big_cap - big_len));
  }
 FS_SYNC_TRACE_END(read);
 
  Local<Value> val;
-if (!ToV8Value(env->context(), result, isolate).ToLocal(&val)) {
+const std::string_view content = big != nullptr
+ ? std::string_view(big, big_len)
+ : std::string_view(result);
+if (!ToV8Value(env->context(), content, isolate).ToLocal(&val)) {
 return;
  }
 

test/parallel/test-fs-readfilesync-utf8-sizes.js

@@ -0,0 +1,98 @@
+'use strict';
+// fs.readFileSync(path, 'utf8') takes a dedicated native path. Its result must
+// equal fs.readFileSync(path).toString('utf8') for every file size (in
+// particular around its internal 8 KiB stack buffer and for multi-megabyte
+// files), for file descriptors positioned mid-file, and for files whose
+// reported size is wrong (procfs reports 0, sysfs reports a page).
+constcommon=require('../common');
+consttmpdir=require('../common/tmpdir');
+constassert=require('assert');
+constfs=require('fs');
+
+tmpdir.refresh();
+
+functioncontent(size){
+// Multi-byte characters straddling every possible chunk boundary.
+constunit='abcdé€\u{1F600}\n';
+lets=unit.repeat(Math.ceil(size/unit.length));
+s=s.slice(0,size);
+// Avoid ending on a lone surrogate produced by slice().
+if(/[\ud800-\udbff]$/.test(s))s=s.slice(0,-1)+'x';
+returns;
+}
+
+constsizes=[0,1,8190,8191,8192,8193,8194,16383,16384,16385,
+65535,65536,65537,100000,(1<<20)-1,1<<20,(1<<20)+1,
+(8<<20)+5];
+for(constsizeofsizes){
+constfile=tmpdir.resolve(`f-${size}.txt`);
+conststr=content(size);
+fs.writeFileSync(file,str);
+constexpected=fs.readFileSync(file).toString('utf8');
+assert.strictEqual(fs.readFileSync(file,'utf8'),expected,`size ${size} by path`);
+assert.strictEqual(fs.readFileSync(file,{encoding: 'utf-8'}),expected,`size ${size} utf-8 alias`);
+// By fd: from the start (leaves the fd at EOF), then at EOF, then from a
+// mid-file position on a fresh fd.
+letfd=fs.openSync(file,'r');
+try{
+assert.strictEqual(fs.readFileSync(fd,'utf8'),expected,`size ${size} by fd`);
+assert.strictEqual(fs.readFileSync(fd,'utf8'),'',`size ${size} by fd at EOF`);
+}finally{
+fs.closeSync(fd);
+}
+if(size>10){
+fd=fs.openSync(file,'r');
+try{
+// Advance the fd 3 bytes (inside the ASCII prefix, so still valid UTF-8).
+assert.strictEqual(fs.readSync(fd,Buffer.alloc(3),0,3,null),3);
+assert.strictEqual(fs.readFileSync(fd,'utf8'),Buffer.from(expected).subarray(3).toString('utf8'),
+`size ${size} by fd at offset 3`);
+}finally{
+fs.closeSync(fd);
+}
+}
+}
+
+// Binary garbage is decoded with replacement characters identically.
+{
+constfile=tmpdir.resolve('binary.bin');
+constbuf=Buffer.alloc(20000);
+for(leti=0;i<buf.length;i++)buf[i]=(i*7919)&0xff;
+fs.writeFileSync(file,buf);
+assert.strictEqual(fs.readFileSync(file,'utf8'),buf.toString('utf8'));
+}
+
+// Files whose st_size does not describe their content.
+if(common.isLinux){
+for(constfileof['/proc/self/status','/proc/self/smaps','/proc/cpuinfo',
+'/proc/version','/sys/kernel/mm/transparent_hugepage/enabled']){
+letviaBuffer;
+try{
+viaBuffer=fs.readFileSync(file);
+}catch{
+continue;// Not available in this environment.
+}
+constviaUtf8=fs.readFileSync(file,'utf8');
+if(file!=='/proc/version'&&file.startsWith('/proc/')){
+// Content legitimately differs between two reads; compare shape instead.
+assert.ok(viaUtf8.length>0);
+assert.strictEqual(viaUtf8.split('\n').length>5,true,file);
+// Of these, smaps reliably exceeds the 8 KiB stack buffer.
+if(file==='/proc/self/smaps')assert.ok(viaUtf8.length>8192,`smaps is only ${viaUtf8.length} chars`);
+}else{
+assert.strictEqual(viaUtf8,viaBuffer.toString('utf8'),file);
+}
+}
+}
+
+// Directory: same outcome either way (EISDIR, except on platforms where
+// read() accepts directories, e.g. AIX).
+functionoutcome(read){
+try{
+returnread();
+}catch(err){
+returnerr.code;
+}
+}
+assert.strictEqual(outcome(()=>fs.readFileSync(tmpdir.path,'utf8')),
+outcome(()=>fs.readFileSync(tmpdir.path).toString('utf8')));

src/node_file.cc

@@ -2931,10 +2931,20 @@ static void ReadFileUtf8(const FunctionCallbackInfo<Value>& args) {
 uv_fs_req_cleanup(&req);
  });
 
+// Past the first 8 KiB, read into one heap buffer sized from fstat(); the
+// size is only a hint, reading continues until read() reports EOF.
  std::string result{};
 char buffer[8192];
 uv_buf_t buf = uv_buf_init(buffer, sizeof(buffer));
 
+char* big = nullptr;
+size_t big_len = 0;
+size_t big_cap = 0;
+bool sized = false;
+auto free_big = OnScopeLeave([&big]() { free(big); });
+constexprsize_tkMinChunk = 64 * 1024;
+constexprsize_tkMaxChunk = 8 * 1024 * 1024;
+
 FS_SYNC_TRACE_BEGIN(read);
 while (true) {
 auto r = uv_fs_read(nullptr, &req, file, &buf, 1, -1, nullptr);
@@ -2947,12 +2957,62 @@ static void ReadFileUtf8(const FunctionCallbackInfo<Value>& args) {
 if (r <= 0) {
 break;
  }
- result.append(buf.base, r);
+if (big == nullptr) {
+ result.append(buf.base, r);
+if (static_cast<size_t>(r) < sizeof(buffer)) {
+continue;
+ }
+// Switch to the heap buffer.
+uv_fs_req_cleanup(&req);
+ big_cap = kMinChunk;
+ big = UncheckedMalloc<char>(big_cap);
+if (big == nullptr) {
+FS_SYNC_TRACE_END(read);
+returnTHROW_ERR_MEMORY_ALLOCATION_FAILED(env);
+ }
+memcpy(big, result.data(), result.size());
+ big_len = result.size();
+ result = std::string();
+ } else {
+ big_len += static_cast<size_t>(r);
+ }
+if (big_len == big_cap) {
+// +1 leaves room for the read() that reports EOF.
+size_t new_cap =
+ big_cap + std::min(kMaxChunk, std::max(kMinChunk, big_cap));
+if (!sized) {
+ sized = true;
+uv_fs_req_cleanup(&req);
+uv_fs_t stat_req;
+if (uv_fs_fstat(nullptr, &stat_req, file, nullptr) == 0) {
+constuv_stat_t* const st =
+static_cast<constuv_stat_t*>(stat_req.ptr);
+if ((st->st_mode & S_IFMT) == S_IFREG &&
+static_cast<uint64_t>(st->st_size) > big_len &&
+static_cast<uint64_t>(st->st_size) <
+static_cast<uint64_t>(v8::String::kMaxLength)) {
+ new_cap = static_cast<size_t>(st->st_size) + 1;
+ }
+ }
+uv_fs_req_cleanup(&stat_req);
+ }
+char* const grown = UncheckedRealloc<char>(big, new_cap);
+if (grown == nullptr) {
+FS_SYNC_TRACE_END(read);
+returnTHROW_ERR_MEMORY_ALLOCATION_FAILED(env);
+ }
+ big = grown;
+ big_cap = new_cap;
+ }
+ buf = uv_buf_init(big + big_len, std::min(kMaxChunk, big_cap - big_len));
  }
 FS_SYNC_TRACE_END(read);
 
  Local<Value> val;
-if (!ToV8Value(env->context(), result, isolate).ToLocal(&val)) {
+const std::string_view content = big != nullptr
+ ? std::string_view(big, big_len)
+ : std::string_view(result);
+if (!ToV8Value(env->context(), content, isolate).ToLocal(&val)) {
 return;
  }
 

test/parallel/test-fs-readfilesync-utf8-sizes.js

@@ -0,0 +1,98 @@
+'use strict';
+// fs.readFileSync(path, 'utf8') takes a dedicated native path. Its result must
+// equal fs.readFileSync(path).toString('utf8') for every file size (in
+// particular around its internal 8 KiB stack buffer and for multi-megabyte
+// files), for file descriptors positioned mid-file, and for files whose
+// reported size is wrong (procfs reports 0, sysfs reports a page).
+constcommon=require('../common');
+consttmpdir=require('../common/tmpdir');
+constassert=require('assert');
+constfs=require('fs');
+
+tmpdir.refresh();
+
+functioncontent(size){
+// Multi-byte characters straddling every possible chunk boundary.
+constunit='abcdé€\u{1F600}\n';
+lets=unit.repeat(Math.ceil(size/unit.length));
+s=s.slice(0,size);
+// Avoid ending on a lone surrogate produced by slice().
+if(/[\ud800-\udbff]$/.test(s))s=s.slice(0,-1)+'x';
+returns;
+}
+
+constsizes=[0,1,8190,8191,8192,8193,8194,16383,16384,16385,
+65535,65536,65537,100000,(1<<20)-1,1<<20,(1<<20)+1,
+(8<<20)+5];
+for(constsizeofsizes){
+constfile=tmpdir.resolve(`f-${size}.txt`);
+conststr=content(size);
+fs.writeFileSync(file,str);
+constexpected=fs.readFileSync(file).toString('utf8');
+assert.strictEqual(fs.readFileSync(file,'utf8'),expected,`size ${size} by path`);
+assert.strictEqual(fs.readFileSync(file,{encoding: 'utf-8'}),expected,`size ${size} utf-8 alias`);
+// By fd: from the start (leaves the fd at EOF), then at EOF, then from a
+// mid-file position on a fresh fd.
+letfd=fs.openSync(file,'r');
+try{
+assert.strictEqual(fs.readFileSync(fd,'utf8'),expected,`size ${size} by fd`);
+assert.strictEqual(fs.readFileSync(fd,'utf8'),'',`size ${size} by fd at EOF`);
+}finally{
+fs.closeSync(fd);
+}
+if(size>10){
+fd=fs.openSync(file,'r');
+try{
+// Advance the fd 3 bytes (inside the ASCII prefix, so still valid UTF-8).
+assert.strictEqual(fs.readSync(fd,Buffer.alloc(3),0,3,null),3);
+assert.strictEqual(fs.readFileSync(fd,'utf8'),Buffer.from(expected).subarray(3).toString('utf8'),
+`size ${size} by fd at offset 3`);
+}finally{
+fs.closeSync(fd);
+}
+}
+}
+
+// Binary garbage is decoded with replacement characters identically.
+{
+constfile=tmpdir.resolve('binary.bin');
+constbuf=Buffer.alloc(20000);
+for(leti=0;i<buf.length;i++)buf[i]=(i*7919)&0xff;
+fs.writeFileSync(file,buf);
+assert.strictEqual(fs.readFileSync(file,'utf8'),buf.toString('utf8'));
+}
+
+// Files whose st_size does not describe their content.
+if(common.isLinux){
+for(constfileof['/proc/self/status','/proc/self/smaps','/proc/cpuinfo',
+'/proc/version','/sys/kernel/mm/transparent_hugepage/enabled']){
+letviaBuffer;
+try{
+viaBuffer=fs.readFileSync(file);
+}catch{
+continue;// Not available in this environment.
+}
+constviaUtf8=fs.readFileSync(file,'utf8');
+if(file!=='/proc/version'&&file.startsWith('/proc/')){
+// Content legitimately differs between two reads; compare shape instead.
+assert.ok(viaUtf8.length>0);
+assert.strictEqual(viaUtf8.split('\n').length>5,true,file);
+// Of these, smaps reliably exceeds the 8 KiB stack buffer.
+if(file==='/proc/self/smaps')assert.ok(viaUtf8.length>8192,`smaps is only ${viaUtf8.length} chars`);
+}else{
+assert.strictEqual(viaUtf8,viaBuffer.toString('utf8'),file);
+}
+}
+}
+
+// Directory: same outcome either way (EISDIR, except on platforms where
+// read() accepts directories, e.g. AIX).
+functionoutcome(read){
+try{
+returnread();
+}catch(err){
+returnerr.code;
+}
+}
+assert.strictEqual(outcome(()=>fs.readFileSync(tmpdir.path,'utf8')),
+outcome(()=>fs.readFileSync(tmpdir.path).toString('utf8')));

src/node_file.cc

@@ -2931,10 +2931,20 @@ static void ReadFileUtf8(const FunctionCallbackInfo<Value>& args) {
 uv_fs_req_cleanup(&req);
  });
 
+// Past the first 8 KiB, read into one heap buffer sized from fstat(); the
+// size is only a hint, reading continues until read() reports EOF.
  std::string result{};
 char buffer[8192];
 uv_buf_t buf = uv_buf_init(buffer, sizeof(buffer));
 
+char* big = nullptr;
+size_t big_len = 0;
+size_t big_cap = 0;
+bool sized = false;
+auto free_big = OnScopeLeave([&big]() { free(big); });
+constexprsize_tkMinChunk = 64 * 1024;
+constexprsize_tkMaxChunk = 8 * 1024 * 1024;
+
 FS_SYNC_TRACE_BEGIN(read);
 while (true) {
 auto r = uv_fs_read(nullptr, &req, file, &buf, 1, -1, nullptr);
@@ -2947,12 +2957,62 @@ static void ReadFileUtf8(const FunctionCallbackInfo<Value>& args) {
 if (r <= 0) {
 break;
  }
- result.append(buf.base, r);
+if (big == nullptr) {
+ result.append(buf.base, r);
+if (static_cast<size_t>(r) < sizeof(buffer)) {
+continue;
+ }
+// Switch to the heap buffer.
+uv_fs_req_cleanup(&req);
+ big_cap = kMinChunk;
+ big = UncheckedMalloc<char>(big_cap);
+if (big == nullptr) {
+FS_SYNC_TRACE_END(read);
+returnTHROW_ERR_MEMORY_ALLOCATION_FAILED(env);
+ }
+memcpy(big, result.data(), result.size());
+ big_len = result.size();
+ result = std::string();
+ } else {
+ big_len += static_cast<size_t>(r);
+ }
+if (big_len == big_cap) {
+// +1 leaves room for the read() that reports EOF.
+size_t new_cap =
+ big_cap + std::min(kMaxChunk, std::max(kMinChunk, big_cap));
+if (!sized) {
+ sized = true;
+uv_fs_req_cleanup(&req);
+uv_fs_t stat_req;
+if (uv_fs_fstat(nullptr, &stat_req, file, nullptr) == 0) {
+constuv_stat_t* const st =
+static_cast<constuv_stat_t*>(stat_req.ptr);
+if ((st->st_mode & S_IFMT) == S_IFREG &&
+static_cast<uint64_t>(st->st_size) > big_len &&
+static_cast<uint64_t>(st->st_size) <
+static_cast<uint64_t>(v8::String::kMaxLength)) {
+ new_cap = static_cast<size_t>(st->st_size) + 1;
+ }
+ }
+uv_fs_req_cleanup(&stat_req);
+ }
+char* const grown = UncheckedRealloc<char>(big, new_cap);
+if (grown == nullptr) {
+FS_SYNC_TRACE_END(read);
+returnTHROW_ERR_MEMORY_ALLOCATION_FAILED(env);
+ }
+ big = grown;
+ big_cap = new_cap;
+ }
+ buf = uv_buf_init(big + big_len, std::min(kMaxChunk, big_cap - big_len));
  }
 FS_SYNC_TRACE_END(read);
 
  Local<Value> val;
-if (!ToV8Value(env->context(), result, isolate).ToLocal(&val)) {
+const std::string_view content = big != nullptr
+ ? std::string_view(big, big_len)
+ : std::string_view(result);
+if (!ToV8Value(env->context(), content, isolate).ToLocal(&val)) {
 return;
  }
 

test/parallel/test-fs-readfilesync-utf8-sizes.js

@@ -0,0 +1,98 @@
+'use strict';
+// fs.readFileSync(path, 'utf8') takes a dedicated native path. Its result must
+// equal fs.readFileSync(path).toString('utf8') for every file size (in
+// particular around its internal 8 KiB stack buffer and for multi-megabyte
+// files), for file descriptors positioned mid-file, and for files whose
+// reported size is wrong (procfs reports 0, sysfs reports a page).
+constcommon=require('../common');
+consttmpdir=require('../common/tmpdir');
+constassert=require('assert');
+constfs=require('fs');
+
+tmpdir.refresh();
+
+functioncontent(size){
+// Multi-byte characters straddling every possible chunk boundary.
+constunit='abcdé€\u{1F600}\n';
+lets=unit.repeat(Math.ceil(size/unit.length));
+s=s.slice(0,size);
+// Avoid ending on a lone surrogate produced by slice().
+if(/[\ud800-\udbff]$/.test(s))s=s.slice(0,-1)+'x';
+returns;
+}
+
+constsizes=[0,1,8190,8191,8192,8193,8194,16383,16384,16385,
+65535,65536,65537,100000,(1<<20)-1,1<<20,(1<<20)+1,
+(8<<20)+5];
+for(constsizeofsizes){
+constfile=tmpdir.resolve(`f-${size}.txt`);
+conststr=content(size);
+fs.writeFileSync(file,str);
+constexpected=fs.readFileSync(file).toString('utf8');
+assert.strictEqual(fs.readFileSync(file,'utf8'),expected,`size ${size} by path`);
+assert.strictEqual(fs.readFileSync(file,{encoding: 'utf-8'}),expected,`size ${size} utf-8 alias`);
+// By fd: from the start (leaves the fd at EOF), then at EOF, then from a
+// mid-file position on a fresh fd.
+letfd=fs.openSync(file,'r');
+try{
+assert.strictEqual(fs.readFileSync(fd,'utf8'),expected,`size ${size} by fd`);
+assert.strictEqual(fs.readFileSync(fd,'utf8'),'',`size ${size} by fd at EOF`);
+}finally{
+fs.closeSync(fd);
+}
+if(size>10){
+fd=fs.openSync(file,'r');
+try{
+// Advance the fd 3 bytes (inside the ASCII prefix, so still valid UTF-8).
+assert.strictEqual(fs.readSync(fd,Buffer.alloc(3),0,3,null),3);
+assert.strictEqual(fs.readFileSync(fd,'utf8'),Buffer.from(expected).subarray(3).toString('utf8'),
+`size ${size} by fd at offset 3`);
+}finally{
+fs.closeSync(fd);
+}
+}
+}
+
+// Binary garbage is decoded with replacement characters identically.
+{
+constfile=tmpdir.resolve('binary.bin');
+constbuf=Buffer.alloc(20000);
+for(leti=0;i<buf.length;i++)buf[i]=(i*7919)&0xff;
+fs.writeFileSync(file,buf);
+assert.strictEqual(fs.readFileSync(file,'utf8'),buf.toString('utf8'));
+}
+
+// Files whose st_size does not describe their content.
+if(common.isLinux){
+for(constfileof['/proc/self/status','/proc/self/smaps','/proc/cpuinfo',
+'/proc/version','/sys/kernel/mm/transparent_hugepage/enabled']){
+letviaBuffer;
+try{
+viaBuffer=fs.readFileSync(file);
+}catch{
+continue;// Not available in this environment.
+}
+constviaUtf8=fs.readFileSync(file,'utf8');
+if(file!=='/proc/version'&&file.startsWith('/proc/')){
+// Content legitimately differs between two reads; compare shape instead.
+assert.ok(viaUtf8.length>0);
+assert.strictEqual(viaUtf8.split('\n').length>5,true,file);
+// Of these, smaps reliably exceeds the 8 KiB stack buffer.
+if(file==='/proc/self/smaps')assert.ok(viaUtf8.length>8192,`smaps is only ${viaUtf8.length} chars`);
+}else{
+assert.strictEqual(viaUtf8,viaBuffer.toString('utf8'),file);
+}
+}
+}
+
+// Directory: same outcome either way (EISDIR, except on platforms where
+// read() accepts directories, e.g. AIX).
+functionoutcome(read){
+try{
+returnread();
+}catch(err){
+returnerr.code;
+}
+}
+assert.strictEqual(outcome(()=>fs.readFileSync(tmpdir.path,'utf8')),
+outcome(()=>fs.readFileSync(tmpdir.path).toString('utf8')));

src/node_file.cc

@@ -2931,10 +2931,20 @@ static void ReadFileUtf8(const FunctionCallbackInfo<Value>& args) {
 uv_fs_req_cleanup(&req);
  });
 
+// Past the first 8 KiB, read into one heap buffer sized from fstat(); the
+// size is only a hint, reading continues until read() reports EOF.
  std::string result{};
 char buffer[8192];
 uv_buf_t buf = uv_buf_init(buffer, sizeof(buffer));
 
+char* big = nullptr;
+size_t big_len = 0;
+size_t big_cap = 0;
+bool sized = false;
+auto free_big = OnScopeLeave([&big]() { free(big); });
+constexprsize_tkMinChunk = 64 * 1024;
+constexprsize_tkMaxChunk = 8 * 1024 * 1024;
+
 FS_SYNC_TRACE_BEGIN(read);
 while (true) {
 auto r = uv_fs_read(nullptr, &req, file, &buf, 1, -1, nullptr);
@@ -2947,12 +2957,62 @@ static void ReadFileUtf8(const FunctionCallbackInfo<Value>& args) {
 if (r <= 0) {
 break;
  }
- result.append(buf.base, r);
+if (big == nullptr) {
+ result.append(buf.base, r);
+if (static_cast<size_t>(r) < sizeof(buffer)) {
+continue;
+ }
+// Switch to the heap buffer.
+uv_fs_req_cleanup(&req);
+ big_cap = kMinChunk;
+ big = UncheckedMalloc<char>(big_cap);
+if (big == nullptr) {
+FS_SYNC_TRACE_END(read);
+returnTHROW_ERR_MEMORY_ALLOCATION_FAILED(env);
+ }
+memcpy(big, result.data(), result.size());
+ big_len = result.size();
+ result = std::string();
+ } else {
+ big_len += static_cast<size_t>(r);
+ }
+if (big_len == big_cap) {
+// +1 leaves room for the read() that reports EOF.
+size_t new_cap =
+ big_cap + std::min(kMaxChunk, std::max(kMinChunk, big_cap));
+if (!sized) {
+ sized = true;
+uv_fs_req_cleanup(&req);
+uv_fs_t stat_req;
+if (uv_fs_fstat(nullptr, &stat_req, file, nullptr) == 0) {
+constuv_stat_t* const st =
+static_cast<constuv_stat_t*>(stat_req.ptr);
+if ((st->st_mode & S_IFMT) == S_IFREG &&
+static_cast<uint64_t>(st->st_size) > big_len &&
+static_cast<uint64_t>(st->st_size) <
+static_cast<uint64_t>(v8::String::kMaxLength)) {
+ new_cap = static_cast<size_t>(st->st_size) + 1;
+ }
+ }
+uv_fs_req_cleanup(&stat_req);
+ }
+char* const grown = UncheckedRealloc<char>(big, new_cap);
+if (grown == nullptr) {
+FS_SYNC_TRACE_END(read);
+returnTHROW_ERR_MEMORY_ALLOCATION_FAILED(env);
+ }
+ big = grown;
+ big_cap = new_cap;
+ }
+ buf = uv_buf_init(big + big_len, std::min(kMaxChunk, big_cap - big_len));
  }
 FS_SYNC_TRACE_END(read);
 
  Local<Value> val;
-if (!ToV8Value(env->context(), result, isolate).ToLocal(&val)) {
+const std::string_view content = big != nullptr
+ ? std::string_view(big, big_len)
+ : std::string_view(result);
+if (!ToV8Value(env->context(), content, isolate).ToLocal(&val)) {
 return;
  }
 

test/parallel/test-fs-readfilesync-utf8-sizes.js

@@ -0,0 +1,98 @@
+'use strict';
+// fs.readFileSync(path, 'utf8') takes a dedicated native path. Its result must
+// equal fs.readFileSync(path).toString('utf8') for every file size (in
+// particular around its internal 8 KiB stack buffer and for multi-megabyte
+// files), for file descriptors positioned mid-file, and for files whose
+// reported size is wrong (procfs reports 0, sysfs reports a page).
+constcommon=require('../common');
+consttmpdir=require('../common/tmpdir');
+constassert=require('assert');
+constfs=require('fs');
+
+tmpdir.refresh();
+
+functioncontent(size){
+// Multi-byte characters straddling every possible chunk boundary.
+constunit='abcdé€\u{1F600}\n';
+lets=unit.repeat(Math.ceil(size/unit.length));
+s=s.slice(0,size);
+// Avoid ending on a lone surrogate produced by slice().
+if(/[\ud800-\udbff]$/.test(s))s=s.slice(0,-1)+'x';
+returns;
+}
+
+constsizes=[0,1,8190,8191,8192,8193,8194,16383,16384,16385,
+65535,65536,65537,100000,(1<<20)-1,1<<20,(1<<20)+1,
+(8<<20)+5];
+for(constsizeofsizes){
+constfile=tmpdir.resolve(`f-${size}.txt`);
+conststr=content(size);
+fs.writeFileSync(file,str);
+constexpected=fs.readFileSync(file).toString('utf8');
+assert.strictEqual(fs.readFileSync(file,'utf8'),expected,`size ${size} by path`);
+assert.strictEqual(fs.readFileSync(file,{encoding: 'utf-8'}),expected,`size ${size} utf-8 alias`);
+// By fd: from the start (leaves the fd at EOF), then at EOF, then from a
+// mid-file position on a fresh fd.
+letfd=fs.openSync(file,'r');
+try{
+assert.strictEqual(fs.readFileSync(fd,'utf8'),expected,`size ${size} by fd`);
+assert.strictEqual(fs.readFileSync(fd,'utf8'),'',`size ${size} by fd at EOF`);
+}finally{
+fs.closeSync(fd);
+}
+if(size>10){
+fd=fs.openSync(file,'r');
+try{
+// Advance the fd 3 bytes (inside the ASCII prefix, so still valid UTF-8).
+assert.strictEqual(fs.readSync(fd,Buffer.alloc(3),0,3,null),3);
+assert.strictEqual(fs.readFileSync(fd,'utf8'),Buffer.from(expected).subarray(3).toString('utf8'),
+`size ${size} by fd at offset 3`);
+}finally{
+fs.closeSync(fd);
+}
+}
+}
+
+// Binary garbage is decoded with replacement characters identically.
+{
+constfile=tmpdir.resolve('binary.bin');
+constbuf=Buffer.alloc(20000);
+for(leti=0;i<buf.length;i++)buf[i]=(i*7919)&0xff;
+fs.writeFileSync(file,buf);
+assert.strictEqual(fs.readFileSync(file,'utf8'),buf.toString('utf8'));
+}
+
+// Files whose st_size does not describe their content.
+if(common.isLinux){
+for(constfileof['/proc/self/status','/proc/self/smaps','/proc/cpuinfo',
+'/proc/version','/sys/kernel/mm/transparent_hugepage/enabled']){
+letviaBuffer;
+try{
+viaBuffer=fs.readFileSync(file);
+}catch{
+continue;// Not available in this environment.
+}
+constviaUtf8=fs.readFileSync(file,'utf8');
+if(file!=='/proc/version'&&file.startsWith('/proc/')){
+// Content legitimately differs between two reads; compare shape instead.
+assert.ok(viaUtf8.length>0);
+assert.strictEqual(viaUtf8.split('\n').length>5,true,file);
+// Of these, smaps reliably exceeds the 8 KiB stack buffer.
+if(file==='/proc/self/smaps')assert.ok(viaUtf8.length>8192,`smaps is only ${viaUtf8.length} chars`);
+}else{
+assert.strictEqual(viaUtf8,viaBuffer.toString('utf8'),file);
+}
+}
+}
+
+// Directory: same outcome either way (EISDIR, except on platforms where
+// read() accepts directories, e.g. AIX).
+functionoutcome(read){
+try{
+returnread();
+}catch(err){
+returnerr.code;
+}
+}
+assert.strictEqual(outcome(()=>fs.readFileSync(tmpdir.path,'utf8')),
+outcome(()=>fs.readFileSync(tmpdir.path).toString('utf8')));

src/node_file.cc

@@ -2931,10 +2931,20 @@ static void ReadFileUtf8(const FunctionCallbackInfo<Value>& args) {
 uv_fs_req_cleanup(&req);
  });
 
+// Past the first 8 KiB, read into one heap buffer sized from fstat(); the
+// size is only a hint, reading continues until read() reports EOF.
  std::string result{};
 char buffer[8192];
 uv_buf_t buf = uv_buf_init(buffer, sizeof(buffer));
 
+char* big = nullptr;
+size_t big_len = 0;
+size_t big_cap = 0;
+bool sized = false;
+auto free_big = OnScopeLeave([&big]() { free(big); });
+constexprsize_tkMinChunk = 64 * 1024;
+constexprsize_tkMaxChunk = 8 * 1024 * 1024;
+
 FS_SYNC_TRACE_BEGIN(read);
 while (true) {
 auto r = uv_fs_read(nullptr, &req, file, &buf, 1, -1, nullptr);
@@ -2947,12 +2957,62 @@ static void ReadFileUtf8(const FunctionCallbackInfo<Value>& args) {
 if (r <= 0) {
 break;
  }
- result.append(buf.base, r);
+if (big == nullptr) {
+ result.append(buf.base, r);
+if (static_cast<size_t>(r) < sizeof(buffer)) {
+continue;
+ }
+// Switch to the heap buffer.
+uv_fs_req_cleanup(&req);
+ big_cap = kMinChunk;
+ big = UncheckedMalloc<char>(big_cap);
+if (big == nullptr) {
+FS_SYNC_TRACE_END(read);
+returnTHROW_ERR_MEMORY_ALLOCATION_FAILED(env);
+ }
+memcpy(big, result.data(), result.size());
+ big_len = result.size();
+ result = std::string();
+ } else {
+ big_len += static_cast<size_t>(r);
+ }
+if (big_len == big_cap) {
+// +1 leaves room for the read() that reports EOF.
+size_t new_cap =
+ big_cap + std::min(kMaxChunk, std::max(kMinChunk, big_cap));
+if (!sized) {
+ sized = true;
+uv_fs_req_cleanup(&req);
+uv_fs_t stat_req;
+if (uv_fs_fstat(nullptr, &stat_req, file, nullptr) == 0) {
+constuv_stat_t* const st =
+static_cast<constuv_stat_t*>(stat_req.ptr);
+if ((st->st_mode & S_IFMT) == S_IFREG &&
+static_cast<uint64_t>(st->st_size) > big_len &&
+static_cast<uint64_t>(st->st_size) <
+static_cast<uint64_t>(v8::String::kMaxLength)) {
+ new_cap = static_cast<size_t>(st->st_size) + 1;
+ }
+ }
+uv_fs_req_cleanup(&stat_req);
+ }
+char* const grown = UncheckedRealloc<char>(big, new_cap);
+if (grown == nullptr) {
+FS_SYNC_TRACE_END(read);
+returnTHROW_ERR_MEMORY_ALLOCATION_FAILED(env);
+ }
+ big = grown;
+ big_cap = new_cap;
+ }
+ buf = uv_buf_init(big + big_len, std::min(kMaxChunk, big_cap - big_len));
  }
 FS_SYNC_TRACE_END(read);
 
  Local<Value> val;
-if (!ToV8Value(env->context(), result, isolate).ToLocal(&val)) {
+const std::string_view content = big != nullptr
+ ? std::string_view(big, big_len)
+ : std::string_view(result);
+if (!ToV8Value(env->context(), content, isolate).ToLocal(&val)) {
 return;
  }
 

test/parallel/test-fs-readfilesync-utf8-sizes.js

@@ -0,0 +1,98 @@
+'use strict';
+// fs.readFileSync(path, 'utf8') takes a dedicated native path. Its result must
+// equal fs.readFileSync(path).toString('utf8') for every file size (in
+// particular around its internal 8 KiB stack buffer and for multi-megabyte
+// files), for file descriptors positioned mid-file, and for files whose
+// reported size is wrong (procfs reports 0, sysfs reports a page).
+constcommon=require('../common');
+consttmpdir=require('../common/tmpdir');
+constassert=require('assert');
+constfs=require('fs');
+
+tmpdir.refresh();
+
+functioncontent(size){
+// Multi-byte characters straddling every possible chunk boundary.
+constunit='abcdé€\u{1F600}\n';
+lets=unit.repeat(Math.ceil(size/unit.length));
+s=s.slice(0,size);
+// Avoid ending on a lone surrogate produced by slice().
+if(/[\ud800-\udbff]$/.test(s))s=s.slice(0,-1)+'x';
+returns;
+}
+
+constsizes=[0,1,8190,8191,8192,8193,8194,16383,16384,16385,
+65535,65536,65537,100000,(1<<20)-1,1<<20,(1<<20)+1,
+(8<<20)+5];
+for(constsizeofsizes){
+constfile=tmpdir.resolve(`f-${size}.txt`);
+conststr=content(size);
+fs.writeFileSync(file,str);
+constexpected=fs.readFileSync(file).toString('utf8');
+assert.strictEqual(fs.readFileSync(file,'utf8'),expected,`size ${size} by path`);
+assert.strictEqual(fs.readFileSync(file,{encoding: 'utf-8'}),expected,`size ${size} utf-8 alias`);
+// By fd: from the start (leaves the fd at EOF), then at EOF, then from a
+// mid-file position on a fresh fd.
+letfd=fs.openSync(file,'r');
+try{
+assert.strictEqual(fs.readFileSync(fd,'utf8'),expected,`size ${size} by fd`);
+assert.strictEqual(fs.readFileSync(fd,'utf8'),'',`size ${size} by fd at EOF`);
+}finally{
+fs.closeSync(fd);
+}
+if(size>10){
+fd=fs.openSync(file,'r');
+try{
+// Advance the fd 3 bytes (inside the ASCII prefix, so still valid UTF-8).
+assert.strictEqual(fs.readSync(fd,Buffer.alloc(3),0,3,null),3);
+assert.strictEqual(fs.readFileSync(fd,'utf8'),Buffer.from(expected).subarray(3).toString('utf8'),
+`size ${size} by fd at offset 3`);
+}finally{
+fs.closeSync(fd);
+}
+}
+}
+
+// Binary garbage is decoded with replacement characters identically.
+{
+constfile=tmpdir.resolve('binary.bin');
+constbuf=Buffer.alloc(20000);
+for(leti=0;i<buf.length;i++)buf[i]=(i*7919)&0xff;
+fs.writeFileSync(file,buf);
+assert.strictEqual(fs.readFileSync(file,'utf8'),buf.toString('utf8'));
+}
+
+// Files whose st_size does not describe their content.
+if(common.isLinux){
+for(constfileof['/proc/self/status','/proc/self/smaps','/proc/cpuinfo',
+'/proc/version','/sys/kernel/mm/transparent_hugepage/enabled']){
+letviaBuffer;
+try{
+viaBuffer=fs.readFileSync(file);
+}catch{
+continue;// Not available in this environment.
+}
+constviaUtf8=fs.readFileSync(file,'utf8');
+if(file!=='/proc/version'&&file.startsWith('/proc/')){
+// Content legitimately differs between two reads; compare shape instead.
+assert.ok(viaUtf8.length>0);
+assert.strictEqual(viaUtf8.split('\n').length>5,true,file);
+// Of these, smaps reliably exceeds the 8 KiB stack buffer.
+if(file==='/proc/self/smaps')assert.ok(viaUtf8.length>8192,`smaps is only ${viaUtf8.length} chars`);
+}else{
+assert.strictEqual(viaUtf8,viaBuffer.toString('utf8'),file);
+}
+}
+}
+
+// Directory: same outcome either way (EISDIR, except on platforms where
+// read() accepts directories, e.g. AIX).
+functionoutcome(read){
+try{
+returnread();
+}catch(err){
+returnerr.code;
+}
+}
+assert.strictEqual(outcome(()=>fs.readFileSync(tmpdir.path,'utf8')),
+outcome(()=>fs.readFileSync(tmpdir.path).toString('utf8')));

src/node_file.cc

@@ -2931,10 +2931,20 @@ static void ReadFileUtf8(const FunctionCallbackInfo<Value>& args) {
 uv_fs_req_cleanup(&req);
  });
 
+// Past the first 8 KiB, read into one heap buffer sized from fstat(); the
+// size is only a hint, reading continues until read() reports EOF.
  std::string result{};
 char buffer[8192];
 uv_buf_t buf = uv_buf_init(buffer, sizeof(buffer));
 
+char* big = nullptr;
+size_t big_len = 0;
+size_t big_cap = 0;
+bool sized = false;
+auto free_big = OnScopeLeave([&big]() { free(big); });
+constexprsize_tkMinChunk = 64 * 1024;
+constexprsize_tkMaxChunk = 8 * 1024 * 1024;
+
 FS_SYNC_TRACE_BEGIN(read);
 while (true) {
 auto r = uv_fs_read(nullptr, &req, file, &buf, 1, -1, nullptr);
@@ -2947,12 +2957,62 @@ static void ReadFileUtf8(const FunctionCallbackInfo<Value>& args) {
 if (r <= 0) {
 break;
  }
- result.append(buf.base, r);
+if (big == nullptr) {
+ result.append(buf.base, r);
+if (static_cast<size_t>(r) < sizeof(buffer)) {
+continue;
+ }
+// Switch to the heap buffer.
+uv_fs_req_cleanup(&req);
+ big_cap = kMinChunk;
+ big = UncheckedMalloc<char>(big_cap);
+if (big == nullptr) {
+FS_SYNC_TRACE_END(read);
+returnTHROW_ERR_MEMORY_ALLOCATION_FAILED(env);
+ }
+memcpy(big, result.data(), result.size());
+ big_len = result.size();
+ result = std::string();
+ } else {
+ big_len += static_cast<size_t>(r);
+ }
+if (big_len == big_cap) {
+// +1 leaves room for the read() that reports EOF.
+size_t new_cap =
+ big_cap + std::min(kMaxChunk, std::max(kMinChunk, big_cap));
+if (!sized) {
+ sized = true;
+uv_fs_req_cleanup(&req);
+uv_fs_t stat_req;
+if (uv_fs_fstat(nullptr, &stat_req, file, nullptr) == 0) {
+constuv_stat_t* const st =
+static_cast<constuv_stat_t*>(stat_req.ptr);
+if ((st->st_mode & S_IFMT) == S_IFREG &&
+static_cast<uint64_t>(st->st_size) > big_len &&
+static_cast<uint64_t>(st->st_size) <
+static_cast<uint64_t>(v8::String::kMaxLength)) {
+ new_cap = static_cast<size_t>(st->st_size) + 1;
+ }
+ }
+uv_fs_req_cleanup(&stat_req);
+ }
+char* const grown = UncheckedRealloc<char>(big, new_cap);
+if (grown == nullptr) {
+FS_SYNC_TRACE_END(read);
+returnTHROW_ERR_MEMORY_ALLOCATION_FAILED(env);
+ }
+ big = grown;
+ big_cap = new_cap;
+ }
+ buf = uv_buf_init(big + big_len, std::min(kMaxChunk, big_cap - big_len));
  }
 FS_SYNC_TRACE_END(read);
 
  Local<Value> val;
-if (!ToV8Value(env->context(), result, isolate).ToLocal(&val)) {
+const std::string_view content = big != nullptr
+ ? std::string_view(big, big_len)
+ : std::string_view(result);
+if (!ToV8Value(env->context(), content, isolate).ToLocal(&val)) {
 return;
  }
 

test/parallel/test-fs-readfilesync-utf8-sizes.js

@@ -0,0 +1,98 @@
+'use strict';
+// fs.readFileSync(path, 'utf8') takes a dedicated native path. Its result must
+// equal fs.readFileSync(path).toString('utf8') for every file size (in
+// particular around its internal 8 KiB stack buffer and for multi-megabyte
+// files), for file descriptors positioned mid-file, and for files whose
+// reported size is wrong (procfs reports 0, sysfs reports a page).
+constcommon=require('../common');
+consttmpdir=require('../common/tmpdir');
+constassert=require('assert');
+constfs=require('fs');
+
+tmpdir.refresh();
+
+functioncontent(size){
+// Multi-byte characters straddling every possible chunk boundary.
+constunit='abcdé€\u{1F600}\n';
+lets=unit.repeat(Math.ceil(size/unit.length));
+s=s.slice(0,size);
+// Avoid ending on a lone surrogate produced by slice().
+if(/[\ud800-\udbff]$/.test(s))s=s.slice(0,-1)+'x';
+returns;
+}
+
+constsizes=[0,1,8190,8191,8192,8193,8194,16383,16384,16385,
+65535,65536,65537,100000,(1<<20)-1,1<<20,(1<<20)+1,
+(8<<20)+5];
+for(constsizeofsizes){
+constfile=tmpdir.resolve(`f-${size}.txt`);
+conststr=content(size);
+fs.writeFileSync(file,str);
+constexpected=fs.readFileSync(file).toString('utf8');
+assert.strictEqual(fs.readFileSync(file,'utf8'),expected,`size ${size} by path`);
+assert.strictEqual(fs.readFileSync(file,{encoding: 'utf-8'}),expected,`size ${size} utf-8 alias`);
+// By fd: from the start (leaves the fd at EOF), then at EOF, then from a
+// mid-file position on a fresh fd.
+letfd=fs.openSync(file,'r');
+try{
+assert.strictEqual(fs.readFileSync(fd,'utf8'),expected,`size ${size} by fd`);
+assert.strictEqual(fs.readFileSync(fd,'utf8'),'',`size ${size} by fd at EOF`);
+}finally{
+fs.closeSync(fd);
+}
+if(size>10){
+fd=fs.openSync(file,'r');
+try{
+// Advance the fd 3 bytes (inside the ASCII prefix, so still valid UTF-8).
+assert.strictEqual(fs.readSync(fd,Buffer.alloc(3),0,3,null),3);
+assert.strictEqual(fs.readFileSync(fd,'utf8'),Buffer.from(expected).subarray(3).toString('utf8'),
+`size ${size} by fd at offset 3`);
+}finally{
+fs.closeSync(fd);
+}
+}
+}
+
+// Binary garbage is decoded with replacement characters identically.
+{
+constfile=tmpdir.resolve('binary.bin');
+constbuf=Buffer.alloc(20000);
+for(leti=0;i<buf.length;i++)buf[i]=(i*7919)&0xff;
+fs.writeFileSync(file,buf);
+assert.strictEqual(fs.readFileSync(file,'utf8'),buf.toString('utf8'));
+}
+
+// Files whose st_size does not describe their content.
+if(common.isLinux){
+for(constfileof['/proc/self/status','/proc/self/smaps','/proc/cpuinfo',
+'/proc/version','/sys/kernel/mm/transparent_hugepage/enabled']){
+letviaBuffer;
+try{
+viaBuffer=fs.readFileSync(file);
+}catch{
+continue;// Not available in this environment.
+}
+constviaUtf8=fs.readFileSync(file,'utf8');
+if(file!=='/proc/version'&&file.startsWith('/proc/')){
+// Content legitimately differs between two reads; compare shape instead.
+assert.ok(viaUtf8.length>0);
+assert.strictEqual(viaUtf8.split('\n').length>5,true,file);
+// Of these, smaps reliably exceeds the 8 KiB stack buffer.
+if(file==='/proc/self/smaps')assert.ok(viaUtf8.length>8192,`smaps is only ${viaUtf8.length} chars`);
+}else{
+assert.strictEqual(viaUtf8,viaBuffer.toString('utf8'),file);
+}
+}
+}
+
+// Directory: same outcome either way (EISDIR, except on platforms where
+// read() accepts directories, e.g. AIX).
+functionoutcome(read){
+try{
+returnread();
+}catch(err){
+returnerr.code;
+}
+}
+assert.strictEqual(outcome(()=>fs.readFileSync(tmpdir.path,'utf8')),
+outcome(()=>fs.readFileSync(tmpdir.path).toString('utf8')));

src/node_file.cc

@@ -2931,10 +2931,20 @@ static void ReadFileUtf8(const FunctionCallbackInfo<Value>& args) {
 uv_fs_req_cleanup(&req);
  });
 
+// Past the first 8 KiB, read into one heap buffer sized from fstat(); the
+// size is only a hint, reading continues until read() reports EOF.
  std::string result{};
 char buffer[8192];
 uv_buf_t buf = uv_buf_init(buffer, sizeof(buffer));
 
+char* big = nullptr;
+size_t big_len = 0;
+size_t big_cap = 0;
+bool sized = false;
+auto free_big = OnScopeLeave([&big]() { free(big); });
+constexprsize_tkMinChunk = 64 * 1024;
+constexprsize_tkMaxChunk = 8 * 1024 * 1024;
+
 FS_SYNC_TRACE_BEGIN(read);
 while (true) {
 auto r = uv_fs_read(nullptr, &req, file, &buf, 1, -1, nullptr);
@@ -2947,12 +2957,62 @@ static void ReadFileUtf8(const FunctionCallbackInfo<Value>& args) {
 if (r <= 0) {
 break;
  }
- result.append(buf.base, r);
+if (big == nullptr) {
+ result.append(buf.base, r);
+if (static_cast<size_t>(r) < sizeof(buffer)) {
+continue;
+ }
+// Switch to the heap buffer.
+uv_fs_req_cleanup(&req);
+ big_cap = kMinChunk;
+ big = UncheckedMalloc<char>(big_cap);
+if (big == nullptr) {
+FS_SYNC_TRACE_END(read);
+returnTHROW_ERR_MEMORY_ALLOCATION_FAILED(env);
+ }
+memcpy(big, result.data(), result.size());
+ big_len = result.size();
+ result = std::string();
+ } else {
+ big_len += static_cast<size_t>(r);
+ }
+if (big_len == big_cap) {
+// +1 leaves room for the read() that reports EOF.
+size_t new_cap =
+ big_cap + std::min(kMaxChunk, std::max(kMinChunk, big_cap));
+if (!sized) {
+ sized = true;
+uv_fs_req_cleanup(&req);
+uv_fs_t stat_req;
+if (uv_fs_fstat(nullptr, &stat_req, file, nullptr) == 0) {
+constuv_stat_t* const st =
+static_cast<constuv_stat_t*>(stat_req.ptr);
+if ((st->st_mode & S_IFMT) == S_IFREG &&
+static_cast<uint64_t>(st->st_size) > big_len &&
+static_cast<uint64_t>(st->st_size) <
+static_cast<uint64_t>(v8::String::kMaxLength)) {
+ new_cap = static_cast<size_t>(st->st_size) + 1;
+ }
+ }
+uv_fs_req_cleanup(&stat_req);
+ }
+char* const grown = UncheckedRealloc<char>(big, new_cap);
+if (grown == nullptr) {
+FS_SYNC_TRACE_END(read);
+returnTHROW_ERR_MEMORY_ALLOCATION_FAILED(env);
+ }
+ big = grown;
+ big_cap = new_cap;
+ }
+ buf = uv_buf_init(big + big_len, std::min(kMaxChunk, big_cap - big_len));
  }
 FS_SYNC_TRACE_END(read);
 
  Local<Value> val;
-if (!ToV8Value(env->context(), result, isolate).ToLocal(&val)) {
+const std::string_view content = big != nullptr
+ ? std::string_view(big, big_len)
+ : std::string_view(result);
+if (!ToV8Value(env->context(), content, isolate).ToLocal(&val)) {
 return;
  }
 

test/parallel/test-fs-readfilesync-utf8-sizes.js

@@ -0,0 +1,98 @@
+'use strict';
+// fs.readFileSync(path, 'utf8') takes a dedicated native path. Its result must
+// equal fs.readFileSync(path).toString('utf8') for every file size (in
+// particular around its internal 8 KiB stack buffer and for multi-megabyte
+// files), for file descriptors positioned mid-file, and for files whose
+// reported size is wrong (procfs reports 0, sysfs reports a page).
+constcommon=require('../common');
+consttmpdir=require('../common/tmpdir');
+constassert=require('assert');
+constfs=require('fs');
+
+tmpdir.refresh();
+
+functioncontent(size){
+// Multi-byte characters straddling every possible chunk boundary.
+constunit='abcdé€\u{1F600}\n';
+lets=unit.repeat(Math.ceil(size/unit.length));
+s=s.slice(0,size);
+// Avoid ending on a lone surrogate produced by slice().
+if(/[\ud800-\udbff]$/.test(s))s=s.slice(0,-1)+'x';
+returns;
+}
+
+constsizes=[0,1,8190,8191,8192,8193,8194,16383,16384,16385,
+65535,65536,65537,100000,(1<<20)-1,1<<20,(1<<20)+1,
+(8<<20)+5];
+for(constsizeofsizes){
+constfile=tmpdir.resolve(`f-${size}.txt`);
+conststr=content(size);
+fs.writeFileSync(file,str);
+constexpected=fs.readFileSync(file).toString('utf8');
+assert.strictEqual(fs.readFileSync(file,'utf8'),expected,`size ${size} by path`);
+assert.strictEqual(fs.readFileSync(file,{encoding: 'utf-8'}),expected,`size ${size} utf-8 alias`);
+// By fd: from the start (leaves the fd at EOF), then at EOF, then from a
+// mid-file position on a fresh fd.
+letfd=fs.openSync(file,'r');
+try{
+assert.strictEqual(fs.readFileSync(fd,'utf8'),expected,`size ${size} by fd`);
+assert.strictEqual(fs.readFileSync(fd,'utf8'),'',`size ${size} by fd at EOF`);
+}finally{
+fs.closeSync(fd);
+}
+if(size>10){
+fd=fs.openSync(file,'r');
+try{
+// Advance the fd 3 bytes (inside the ASCII prefix, so still valid UTF-8).
+assert.strictEqual(fs.readSync(fd,Buffer.alloc(3),0,3,null),3);
+assert.strictEqual(fs.readFileSync(fd,'utf8'),Buffer.from(expected).subarray(3).toString('utf8'),
+`size ${size} by fd at offset 3`);
+}finally{
+fs.closeSync(fd);
+}
+}
+}
+
+// Binary garbage is decoded with replacement characters identically.
+{
+constfile=tmpdir.resolve('binary.bin');
+constbuf=Buffer.alloc(20000);
+for(leti=0;i<buf.length;i++)buf[i]=(i*7919)&0xff;
+fs.writeFileSync(file,buf);
+assert.strictEqual(fs.readFileSync(file,'utf8'),buf.toString('utf8'));
+}
+
+// Files whose st_size does not describe their content.
+if(common.isLinux){
+for(constfileof['/proc/self/status','/proc/self/smaps','/proc/cpuinfo',
+'/proc/version','/sys/kernel/mm/transparent_hugepage/enabled']){
+letviaBuffer;
+try{
+viaBuffer=fs.readFileSync(file);
+}catch{
+continue;// Not available in this environment.
+}
+constviaUtf8=fs.readFileSync(file,'utf8');
+if(file!=='/proc/version'&&file.startsWith('/proc/')){
+// Content legitimately differs between two reads; compare shape instead.
+assert.ok(viaUtf8.length>0);
+assert.strictEqual(viaUtf8.split('\n').length>5,true,file);
+// Of these, smaps reliably exceeds the 8 KiB stack buffer.
+if(file==='/proc/self/smaps')assert.ok(viaUtf8.length>8192,`smaps is only ${viaUtf8.length} chars`);
+}else{
+assert.strictEqual(viaUtf8,viaBuffer.toString('utf8'),file);
+}
+}
+}
+
+// Directory: same outcome either way (EISDIR, except on platforms where
+// read() accepts directories, e.g. AIX).
+functionoutcome(read){
+try{
+returnread();
+}catch(err){
+returnerr.code;
+}
+}
+assert.strictEqual(outcome(()=>fs.readFileSync(tmpdir.path,'utf8')),
+outcome(()=>fs.readFileSync(tmpdir.path).toString('utf8')));

src/node_file.cc

@@ -2931,10 +2931,20 @@ static void ReadFileUtf8(const FunctionCallbackInfo<Value>& args) {
 uv_fs_req_cleanup(&req);
  });
 
+// Past the first 8 KiB, read into one heap buffer sized from fstat(); the
+// size is only a hint, reading continues until read() reports EOF.
  std::string result{};
 char buffer[8192];
 uv_buf_t buf = uv_buf_init(buffer, sizeof(buffer));
 
+char* big = nullptr;
+size_t big_len = 0;
+size_t big_cap = 0;
+bool sized = false;
+auto free_big = OnScopeLeave([&big]() { free(big); });
+constexprsize_tkMinChunk = 64 * 1024;
+constexprsize_tkMaxChunk = 8 * 1024 * 1024;
+
 FS_SYNC_TRACE_BEGIN(read);
 while (true) {
 auto r = uv_fs_read(nullptr, &req, file, &buf, 1, -1, nullptr);
@@ -2947,12 +2957,62 @@ static void ReadFileUtf8(const FunctionCallbackInfo<Value>& args) {
 if (r <= 0) {
 break;
  }
- result.append(buf.base, r);
+if (big == nullptr) {
+ result.append(buf.base, r);
+if (static_cast<size_t>(r) < sizeof(buffer)) {
+continue;
+ }
+// Switch to the heap buffer.
+uv_fs_req_cleanup(&req);
+ big_cap = kMinChunk;
+ big = UncheckedMalloc<char>(big_cap);
+if (big == nullptr) {
+FS_SYNC_TRACE_END(read);
+returnTHROW_ERR_MEMORY_ALLOCATION_FAILED(env);
+ }
+memcpy(big, result.data(), result.size());
+ big_len = result.size();
+ result = std::string();
+ } else {
+ big_len += static_cast<size_t>(r);
+ }
+if (big_len == big_cap) {
+// +1 leaves room for the read() that reports EOF.
+size_t new_cap =
+ big_cap + std::min(kMaxChunk, std::max(kMinChunk, big_cap));
+if (!sized) {
+ sized = true;
+uv_fs_req_cleanup(&req);
+uv_fs_t stat_req;
+if (uv_fs_fstat(nullptr, &stat_req, file, nullptr) == 0) {
+constuv_stat_t* const st =
+static_cast<constuv_stat_t*>(stat_req.ptr);
+if ((st->st_mode & S_IFMT) == S_IFREG &&
+static_cast<uint64_t>(st->st_size) > big_len &&
+static_cast<uint64_t>(st->st_size) <
+static_cast<uint64_t>(v8::String::kMaxLength)) {
+ new_cap = static_cast<size_t>(st->st_size) + 1;
+ }
+ }
+uv_fs_req_cleanup(&stat_req);
+ }
+char* const grown = UncheckedRealloc<char>(big, new_cap);
+if (grown == nullptr) {
+FS_SYNC_TRACE_END(read);
+returnTHROW_ERR_MEMORY_ALLOCATION_FAILED(env);
+ }
+ big = grown;
+ big_cap = new_cap;
+ }
+ buf = uv_buf_init(big + big_len, std::min(kMaxChunk, big_cap - big_len));
  }
 FS_SYNC_TRACE_END(read);
 
  Local<Value> val;
-if (!ToV8Value(env->context(), result, isolate).ToLocal(&val)) {
+const std::string_view content = big != nullptr
+ ? std::string_view(big, big_len)
+ : std::string_view(result);
+if (!ToV8Value(env->context(), content, isolate).ToLocal(&val)) {
 return;
  }
 

test/parallel/test-fs-readfilesync-utf8-sizes.js

@@ -0,0 +1,98 @@
+'use strict';
+// fs.readFileSync(path, 'utf8') takes a dedicated native path. Its result must
+// equal fs.readFileSync(path).toString('utf8') for every file size (in
+// particular around its internal 8 KiB stack buffer and for multi-megabyte
+// files), for file descriptors positioned mid-file, and for files whose
+// reported size is wrong (procfs reports 0, sysfs reports a page).
+constcommon=require('../common');
+consttmpdir=require('../common/tmpdir');
+constassert=require('assert');
+constfs=require('fs');
+
+tmpdir.refresh();
+
+functioncontent(size){
+// Multi-byte characters straddling every possible chunk boundary.
+constunit='abcdé€\u{1F600}\n';
+lets=unit.repeat(Math.ceil(size/unit.length));
+s=s.slice(0,size);
+// Avoid ending on a lone surrogate produced by slice().
+if(/[\ud800-\udbff]$/.test(s))s=s.slice(0,-1)+'x';
+returns;
+}
+
+constsizes=[0,1,8190,8191,8192,8193,8194,16383,16384,16385,
+65535,65536,65537,100000,(1<<20)-1,1<<20,(1<<20)+1,
+(8<<20)+5];
+for(constsizeofsizes){
+constfile=tmpdir.resolve(`f-${size}.txt`);
+conststr=content(size);
+fs.writeFileSync(file,str);
+constexpected=fs.readFileSync(file).toString('utf8');
+assert.strictEqual(fs.readFileSync(file,'utf8'),expected,`size ${size} by path`);
+assert.strictEqual(fs.readFileSync(file,{encoding: 'utf-8'}),expected,`size ${size} utf-8 alias`);
+// By fd: from the start (leaves the fd at EOF), then at EOF, then from a
+// mid-file position on a fresh fd.
+letfd=fs.openSync(file,'r');
+try{
+assert.strictEqual(fs.readFileSync(fd,'utf8'),expected,`size ${size} by fd`);
+assert.strictEqual(fs.readFileSync(fd,'utf8'),'',`size ${size} by fd at EOF`);
+}finally{
+fs.closeSync(fd);
+}
+if(size>10){
+fd=fs.openSync(file,'r');
+try{
+// Advance the fd 3 bytes (inside the ASCII prefix, so still valid UTF-8).
+assert.strictEqual(fs.readSync(fd,Buffer.alloc(3),0,3,null),3);
+assert.strictEqual(fs.readFileSync(fd,'utf8'),Buffer.from(expected).subarray(3).toString('utf8'),
+`size ${size} by fd at offset 3`);
+}finally{
+fs.closeSync(fd);
+}
+}
+}
+
+// Binary garbage is decoded with replacement characters identically.
+{
+constfile=tmpdir.resolve('binary.bin');
+constbuf=Buffer.alloc(20000);
+for(leti=0;i<buf.length;i++)buf[i]=(i*7919)&0xff;
+fs.writeFileSync(file,buf);
+assert.strictEqual(fs.readFileSync(file,'utf8'),buf.toString('utf8'));
+}
+
+// Files whose st_size does not describe their content.
+if(common.isLinux){
+for(constfileof['/proc/self/status','/proc/self/smaps','/proc/cpuinfo',
+'/proc/version','/sys/kernel/mm/transparent_hugepage/enabled']){
+letviaBuffer;
+try{
+viaBuffer=fs.readFileSync(file);
+}catch{
+continue;// Not available in this environment.
+}
+constviaUtf8=fs.readFileSync(file,'utf8');
+if(file!=='/proc/version'&&file.startsWith('/proc/')){
+// Content legitimately differs between two reads; compare shape instead.
+assert.ok(viaUtf8.length>0);
+assert.strictEqual(viaUtf8.split('\n').length>5,true,file);
+// Of these, smaps reliably exceeds the 8 KiB stack buffer.
+if(file==='/proc/self/smaps')assert.ok(viaUtf8.length>8192,`smaps is only ${viaUtf8.length} chars`);
+}else{
+assert.strictEqual(viaUtf8,viaBuffer.toString('utf8'),file);
+}
+}
+}
+
+// Directory: same outcome either way (EISDIR, except on platforms where
+// read() accepts directories, e.g. AIX).
+functionoutcome(read){
+try{
+returnread();
+}catch(err){
+returnerr.code;
+}
+}
+assert.strictEqual(outcome(()=>fs.readFileSync(tmpdir.path,'utf8')),
+outcome(()=>fs.readFileSync(tmpdir.path).toString('utf8')));