src: use simdutf for two-byte strings in UTF-8 writes

StringBytes::Write() already used simdutf to encode one-byte strings
as UTF-8 but sent every two-byte (UTF-16) string through
v8::String::WriteUtf8V2(), which is several times slower. That path
is behind Buffer.from(string), buf.write(), fs.write*() with string
data and every string written to a libuv stream, and JSON.stringify()
output is a two-byte string as soon as any value in the payload is
outside Latin-1.
Encode two-byte strings with simdutf as well whenever their UTF-8 form
is guaranteed to fit in the target: well-formed input is converted
directly, and input with unpaired surrogates is converted from a copy
passed through simdutf::to_well_formed_utf16(), which replaces each
unpaired surrogate with U+FFFD exactly like kReplaceInvalidUtf8 (this
mirrors what TextEncoder already does). Writes that have to truncate
at a character boundary keep using WriteUtf8V2(), so their output is
byte-for-byte unchanged, and so do strings of up to 32 code units, for
which V8 is already as fast (the same threshold TextEncoder uses).
buf.write() of a 2 KiB two-byte string improves ~5x (astral-heavy and
lone-surrogate strings ~3.5x and ~5x), Buffer.from() of a 64 KiB JSON
string ~2.7x; one-byte strings are unaffected.
Signed-off-by: Shelley Vohr <shelley.vohr@gmail.com>
PR-URL: #65324
Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com>
Reviewed-By: Gürgün Dayıoğlu <hey@gurgun.day>
Reviewed-By: Daniel Lemire <daniel@lemire.me>
Reviewed-By: James M Snell <jasnell@gmail.com>

Author: codebytere

benchmark/buffers/buffer-write-string-utf8.js

@@ -0,0 +1,36 @@
+'use strict';
+
+// buf.write(string, 'utf8') for strings whose in-memory representation is
+// one-byte (Latin-1) or two-byte (UTF-16), which take different encoder paths.
+constcommon=require('../common.js');
+constbench=common.createBenchmark(main,{
+chars: ['one-byte','two-byte','two-byte-astral','two-byte-lone-surrogate'],
+len: [16,256,2048,65536],
+n: [5e5],
+});
+
+functionmakeString(chars,len){
+switch(chars){
+case'one-byte':
+return'aé'.repeat(len/2);
+case'two-byte':
+return'aé€日'.repeat(len/4);
+case'two-byte-astral':
+return'aé€日\u{1F600}'.repeat(len/6).padEnd(len,'a');
+case'two-byte-lone-surrogate':
+return'aé€日'.repeat(len/4-1)+'ab\ud800c';
+default:
+thrownewError(chars);
+}
+}
+
+functionmain({ chars, len, n }){
+conststring=makeString(chars,len);
+constbuf=Buffer.allocUnsafe(Buffer.byteLength(string));
+if(len>=65536)n=Math.floor(n/32);
+bench.start();
+for(leti=0;i<n;++i){
+buf.write(string,0,'utf8');
+}
+bench.end(n);
+}

src/string_bytes.cc

@@ -310,9 +310,37 @@ size_t StringBytes::Write(Isolate* isolate,
  input_view.length(),
  buf,
  buflen);
- } else {
+ } elseif (input_view.length() <= 32) {
+// V8 is as fast for tiny strings (same threshold TextEncoder uses).
  nbytes = str->WriteUtf8V2(
  isolate, buf, buflen, String::WriteFlags::kReplaceInvalidUtf8);
+ } else {
+// Use simdutf for two-byte strings as well whenever the UTF-8 form
+// is guaranteed to fit; truncating writes (which must stop at a
+// character boundary) keep going through V8 so that their output
+// stays byte-for-byte identical.
+constchar16_t* data =
+reinterpret_cast<constchar16_t*>(input_view.data16());
+constsize_t length = input_view.length();
+ MaybeStackBuffer<char16_t, 1024> well_formed;
+if (!simdutf::validate_utf16(data, length)) {
+// Unpaired surrogates: encode a copy in which each of them has been
+// replaced with U+FFFD, which is what kReplaceInvalidUtf8 produces.
+ well_formed.AllocateSufficientStorage(length);
+simdutf::to_well_formed_utf16(data, length, well_formed.out());
+ data = well_formed.out();
+ }
+// A UTF-16 code unit never expands to more than 3 UTF-8 bytes, so
+// 3 * length is what StorageSize() hands most callers; only compute
+// the exact length when the buffer is smaller than that.
+if (buflen >= 3 * length ||
+ buflen >= simdutf::utf8_length_from_utf16(data, length)) {
+ nbytes = simdutf::convert_utf16_to_utf8(data, length, buf);
+ } else {
+// Does not fit: let V8 truncate at a character boundary.
+ nbytes = str->WriteUtf8V2(
+ isolate, buf, buflen, String::WriteFlags::kReplaceInvalidUtf8);
+ }
  }
 break;
 

test/parallel/test-buffer-write-utf8-two-byte.js

@@ -0,0 +1,128 @@
+'use strict';
+// UTF-8 encoding of two-byte (UTF-16) JS strings through the Buffer write
+// paths (Buffer.from, Buffer#write, Buffer.byteLength) must:
+// - produce standard UTF-8 for well-formed input of any size,
+// - replace lone surrogates with U+FFFD (EF BF BD),
+// - never write a partial character when the target is too small,
+// independent of which internal fast path handles the string.
+require('../common');
+constassert=require('assert');
+
+// Reference encoder written out longhand so the test does not depend on the
+// implementation under test (TextEncoder shares code with it).
+functionutf8Reference(str){
+constout=[];
+for(leti=0;i<str.length;i++){
+letcp=str.charCodeAt(i);
+if(cp>=0xd800&&cp<=0xdbff){
+constnext=i+1<str.length ? str.charCodeAt(i+1) : 0;
+if(next>=0xdc00&&next<=0xdfff){
+cp=0x10000+((cp-0xd800)<<10)+(next-0xdc00);
+i++;
+}else{
+cp=0xfffd;
+}
+}elseif(cp>=0xdc00&&cp<=0xdfff){
+cp=0xfffd;
+}
+if(cp<0x80){
+out.push(cp);
+}elseif(cp<0x800){
+out.push(0xc0|(cp>>6),0x80|(cp&0x3f));
+}elseif(cp<0x10000){
+out.push(0xe0|(cp>>12),0x80|((cp>>6)&0x3f),0x80|(cp&0x3f));
+}else{
+out.push(0xf0|(cp>>18),0x80|((cp>>12)&0x3f),
+0x80|((cp>>6)&0x3f),0x80|(cp&0x3f));
+}
+}
+returnBuffer.from(out);
+}
+
+functioncheckFull(str,label){
+constexpected=utf8Reference(str);
+assert.deepStrictEqual(Buffer.from(str,'utf8'),expected,`${label}: Buffer.from`);
+assert.strictEqual(Buffer.byteLength(str,'utf8'),expected.length,`${label}: byteLength`);
+// Exact-size target.
+constexact=Buffer.alloc(expected.length);
+assert.strictEqual(exact.write(str,'utf8'),expected.length,`${label}: write exact`);
+assert.deepStrictEqual(exact,expected,`${label}: write exact bytes`);
+// Oversized target (3 bytes per code unit is what most internal callers allocate).
+constbig=Buffer.alloc(str.length*3+7,0xaa);
+assert.strictEqual(big.write(str,2,'utf8'),expected.length,`${label}: write big`);
+assert.deepStrictEqual(big.subarray(2,2+expected.length),expected,`${label}: write big bytes`);
+assert.strictEqual(big[0],0xaa);
+assert.strictEqual(big[2+expected.length],0xaa,`${label}: no overrun`);
+}
+
+// Truncating writes must stop before the first character that does not fit.
+functioncheckTruncation(str,label){
+constexpected=utf8Reference(str);
+for(letsize=0;size<=Math.min(expected.length,70);size++){
+consttarget=Buffer.alloc(size+1,0xaa);
+constn=target.write(str,0,size,'utf8');
+assert.ok(n<=size,`${label}: size=${size} wrote ${n}`);
+assert.deepStrictEqual(target.subarray(0,n),expected.subarray(0,n),`${label}: prefix size=${size}`);
+assert.strictEqual(target[size],0xaa,`${label}: overrun size=${size}`);
+// What was written must be a whole number of characters: the next byte in
+// the reference (if any) has to be a lead byte, not a continuation byte.
+if(n<expected.length){
+assert.notStrictEqual(expected[n]&0xc0,0x80,`${label}: split char at size=${size}`);
+// And it stopped only because the next character really did not fit.
+letnext=n+1;
+while(next<expected.length&&(expected[next]&0xc0)===0x80)next++;
+assert.ok(next>size,`${label}: stopped early at size=${size} (n=${n}, next=${next})`);
+}
+}
+}
+
+// Force a two-byte representation even for ASCII/Latin-1 content by building
+// the string from a two-byte seed and slicing (V8 keeps the representation).
+functiontwoByte(str){
+consts=('\u{1F600}'+str).slice(2);
+assert.strictEqual(s,str);
+returns;
+}
+
+constsamples={
+ascii: 'The quick brown fox jumps over the lazy dog 0123456789',
+latin1: 'français élan über naïve façade ÿ',
+bmp: '日本語テキストとハングル한국어',
+astral: 'emoji \u{1F600}\u{1F4A9} math \u{1D49C} han \u{20BB7}',
+mixed: 'a é 日 \u{1F600} b ü 本 \u{1F4A9}',
+};
+
+for(const[name,base]ofObject.entries(samples)){
+for(constrepeatof[1,3,40,700,12000]){
+conststr=twoByte(base.repeat(repeat));
+checkFull(str,`${name} x${repeat}`);
+}
+checkTruncation(twoByte(base.repeat(3)),`${name} truncation`);
+}
+
+// Lone surrogates in various positions and sizes -> U+FFFD, rest intact.
+consthigh='\ud83d';
+constlow='\ude00';
+constsurrogateCases={
+'lone high': `ab${high}cd`,
+'lone low': `ab${low}cd`,
+'reversed pair': `ab${low}${high}cd`,
+'high at end': `abcd${high}`,
+'low at start': `${low}abcd`,
+'high high low': `${high}${high}${low}x`,
+'pair then lone': `${high}${low}${high}`,
+'only lone': high,
+};
+for(const[name,base]ofObject.entries(surrogateCases)){
+for(constpadof['','x'.repeat(50),'é'.repeat(300),'日'.repeat(30000)]){
+conststr=pad+base+pad;
+checkFull(str,`${name} pad=${pad.length}`);
+}
+checkTruncation(base+'zz',`${name} truncation`);
+}
+
+// Buffer.from of a large two-byte string equals TextEncoder output.
+{
+conststr=twoByte(samples.mixed.repeat(50000));
+assert.deepStrictEqual(Buffer.from(str),Buffer.from(newTextEncoder().encode(str)));
+}

benchmark/buffers/buffer-write-string-utf8.js

@@ -0,0 +1,36 @@
+'use strict';
+
+// buf.write(string, 'utf8') for strings whose in-memory representation is
+// one-byte (Latin-1) or two-byte (UTF-16), which take different encoder paths.
+constcommon=require('../common.js');
+constbench=common.createBenchmark(main,{
+chars: ['one-byte','two-byte','two-byte-astral','two-byte-lone-surrogate'],
+len: [16,256,2048,65536],
+n: [5e5],
+});
+
+functionmakeString(chars,len){
+switch(chars){
+case'one-byte':
+return'aé'.repeat(len/2);
+case'two-byte':
+return'aé€日'.repeat(len/4);
+case'two-byte-astral':
+return'aé€日\u{1F600}'.repeat(len/6).padEnd(len,'a');
+case'two-byte-lone-surrogate':
+return'aé€日'.repeat(len/4-1)+'ab\ud800c';
+default:
+thrownewError(chars);
+}
+}
+
+functionmain({ chars, len, n }){
+conststring=makeString(chars,len);
+constbuf=Buffer.allocUnsafe(Buffer.byteLength(string));
+if(len>=65536)n=Math.floor(n/32);
+bench.start();
+for(leti=0;i<n;++i){
+buf.write(string,0,'utf8');
+}
+bench.end(n);
+}

src/string_bytes.cc

@@ -310,9 +310,37 @@ size_t StringBytes::Write(Isolate* isolate,
  input_view.length(),
  buf,
  buflen);
- } else {
+ } elseif (input_view.length() <= 32) {
+// V8 is as fast for tiny strings (same threshold TextEncoder uses).
  nbytes = str->WriteUtf8V2(
  isolate, buf, buflen, String::WriteFlags::kReplaceInvalidUtf8);
+ } else {
+// Use simdutf for two-byte strings as well whenever the UTF-8 form
+// is guaranteed to fit; truncating writes (which must stop at a
+// character boundary) keep going through V8 so that their output
+// stays byte-for-byte identical.
+constchar16_t* data =
+reinterpret_cast<constchar16_t*>(input_view.data16());
+constsize_t length = input_view.length();
+ MaybeStackBuffer<char16_t, 1024> well_formed;
+if (!simdutf::validate_utf16(data, length)) {
+// Unpaired surrogates: encode a copy in which each of them has been
+// replaced with U+FFFD, which is what kReplaceInvalidUtf8 produces.
+ well_formed.AllocateSufficientStorage(length);
+simdutf::to_well_formed_utf16(data, length, well_formed.out());
+ data = well_formed.out();
+ }
+// A UTF-16 code unit never expands to more than 3 UTF-8 bytes, so
+// 3 * length is what StorageSize() hands most callers; only compute
+// the exact length when the buffer is smaller than that.
+if (buflen >= 3 * length ||
+ buflen >= simdutf::utf8_length_from_utf16(data, length)) {
+ nbytes = simdutf::convert_utf16_to_utf8(data, length, buf);
+ } else {
+// Does not fit: let V8 truncate at a character boundary.
+ nbytes = str->WriteUtf8V2(
+ isolate, buf, buflen, String::WriteFlags::kReplaceInvalidUtf8);
+ }
  }
 break;
 

test/parallel/test-buffer-write-utf8-two-byte.js

@@ -0,0 +1,128 @@
+'use strict';
+// UTF-8 encoding of two-byte (UTF-16) JS strings through the Buffer write
+// paths (Buffer.from, Buffer#write, Buffer.byteLength) must:
+// - produce standard UTF-8 for well-formed input of any size,
+// - replace lone surrogates with U+FFFD (EF BF BD),
+// - never write a partial character when the target is too small,
+// independent of which internal fast path handles the string.
+require('../common');
+constassert=require('assert');
+
+// Reference encoder written out longhand so the test does not depend on the
+// implementation under test (TextEncoder shares code with it).
+functionutf8Reference(str){
+constout=[];
+for(leti=0;i<str.length;i++){
+letcp=str.charCodeAt(i);
+if(cp>=0xd800&&cp<=0xdbff){
+constnext=i+1<str.length ? str.charCodeAt(i+1) : 0;
+if(next>=0xdc00&&next<=0xdfff){
+cp=0x10000+((cp-0xd800)<<10)+(next-0xdc00);
+i++;
+}else{
+cp=0xfffd;
+}
+}elseif(cp>=0xdc00&&cp<=0xdfff){
+cp=0xfffd;
+}
+if(cp<0x80){
+out.push(cp);
+}elseif(cp<0x800){
+out.push(0xc0|(cp>>6),0x80|(cp&0x3f));
+}elseif(cp<0x10000){
+out.push(0xe0|(cp>>12),0x80|((cp>>6)&0x3f),0x80|(cp&0x3f));
+}else{
+out.push(0xf0|(cp>>18),0x80|((cp>>12)&0x3f),
+0x80|((cp>>6)&0x3f),0x80|(cp&0x3f));
+}
+}
+returnBuffer.from(out);
+}
+
+functioncheckFull(str,label){
+constexpected=utf8Reference(str);
+assert.deepStrictEqual(Buffer.from(str,'utf8'),expected,`${label}: Buffer.from`);
+assert.strictEqual(Buffer.byteLength(str,'utf8'),expected.length,`${label}: byteLength`);
+// Exact-size target.
+constexact=Buffer.alloc(expected.length);
+assert.strictEqual(exact.write(str,'utf8'),expected.length,`${label}: write exact`);
+assert.deepStrictEqual(exact,expected,`${label}: write exact bytes`);
+// Oversized target (3 bytes per code unit is what most internal callers allocate).
+constbig=Buffer.alloc(str.length*3+7,0xaa);
+assert.strictEqual(big.write(str,2,'utf8'),expected.length,`${label}: write big`);
+assert.deepStrictEqual(big.subarray(2,2+expected.length),expected,`${label}: write big bytes`);
+assert.strictEqual(big[0],0xaa);
+assert.strictEqual(big[2+expected.length],0xaa,`${label}: no overrun`);
+}
+
+// Truncating writes must stop before the first character that does not fit.
+functioncheckTruncation(str,label){
+constexpected=utf8Reference(str);
+for(letsize=0;size<=Math.min(expected.length,70);size++){
+consttarget=Buffer.alloc(size+1,0xaa);
+constn=target.write(str,0,size,'utf8');
+assert.ok(n<=size,`${label}: size=${size} wrote ${n}`);
+assert.deepStrictEqual(target.subarray(0,n),expected.subarray(0,n),`${label}: prefix size=${size}`);
+assert.strictEqual(target[size],0xaa,`${label}: overrun size=${size}`);
+// What was written must be a whole number of characters: the next byte in
+// the reference (if any) has to be a lead byte, not a continuation byte.
+if(n<expected.length){
+assert.notStrictEqual(expected[n]&0xc0,0x80,`${label}: split char at size=${size}`);
+// And it stopped only because the next character really did not fit.
+letnext=n+1;
+while(next<expected.length&&(expected[next]&0xc0)===0x80)next++;
+assert.ok(next>size,`${label}: stopped early at size=${size} (n=${n}, next=${next})`);
+}
+}
+}
+
+// Force a two-byte representation even for ASCII/Latin-1 content by building
+// the string from a two-byte seed and slicing (V8 keeps the representation).
+functiontwoByte(str){
+consts=('\u{1F600}'+str).slice(2);
+assert.strictEqual(s,str);
+returns;
+}
+
+constsamples={
+ascii: 'The quick brown fox jumps over the lazy dog 0123456789',
+latin1: 'français élan über naïve façade ÿ',
+bmp: '日本語テキストとハングル한국어',
+astral: 'emoji \u{1F600}\u{1F4A9} math \u{1D49C} han \u{20BB7}',
+mixed: 'a é 日 \u{1F600} b ü 本 \u{1F4A9}',
+};
+
+for(const[name,base]ofObject.entries(samples)){
+for(constrepeatof[1,3,40,700,12000]){
+conststr=twoByte(base.repeat(repeat));
+checkFull(str,`${name} x${repeat}`);
+}
+checkTruncation(twoByte(base.repeat(3)),`${name} truncation`);
+}
+
+// Lone surrogates in various positions and sizes -> U+FFFD, rest intact.
+consthigh='\ud83d';
+constlow='\ude00';
+constsurrogateCases={
+'lone high': `ab${high}cd`,
+'lone low': `ab${low}cd`,
+'reversed pair': `ab${low}${high}cd`,
+'high at end': `abcd${high}`,
+'low at start': `${low}abcd`,
+'high high low': `${high}${high}${low}x`,
+'pair then lone': `${high}${low}${high}`,
+'only lone': high,
+};
+for(const[name,base]ofObject.entries(surrogateCases)){
+for(constpadof['','x'.repeat(50),'é'.repeat(300),'日'.repeat(30000)]){
+conststr=pad+base+pad;
+checkFull(str,`${name} pad=${pad.length}`);
+}
+checkTruncation(base+'zz',`${name} truncation`);
+}
+
+// Buffer.from of a large two-byte string equals TextEncoder output.
+{
+conststr=twoByte(samples.mixed.repeat(50000));
+assert.deepStrictEqual(Buffer.from(str),Buffer.from(newTextEncoder().encode(str)));
+}

benchmark/buffers/buffer-write-string-utf8.js

@@ -0,0 +1,36 @@
+'use strict';
+
+// buf.write(string, 'utf8') for strings whose in-memory representation is
+// one-byte (Latin-1) or two-byte (UTF-16), which take different encoder paths.
+constcommon=require('../common.js');
+constbench=common.createBenchmark(main,{
+chars: ['one-byte','two-byte','two-byte-astral','two-byte-lone-surrogate'],
+len: [16,256,2048,65536],
+n: [5e5],
+});
+
+functionmakeString(chars,len){
+switch(chars){
+case'one-byte':
+return'aé'.repeat(len/2);
+case'two-byte':
+return'aé€日'.repeat(len/4);
+case'two-byte-astral':
+return'aé€日\u{1F600}'.repeat(len/6).padEnd(len,'a');
+case'two-byte-lone-surrogate':
+return'aé€日'.repeat(len/4-1)+'ab\ud800c';
+default:
+thrownewError(chars);
+}
+}
+
+functionmain({ chars, len, n }){
+conststring=makeString(chars,len);
+constbuf=Buffer.allocUnsafe(Buffer.byteLength(string));
+if(len>=65536)n=Math.floor(n/32);
+bench.start();
+for(leti=0;i<n;++i){
+buf.write(string,0,'utf8');
+}
+bench.end(n);
+}

src/string_bytes.cc

@@ -310,9 +310,37 @@ size_t StringBytes::Write(Isolate* isolate,
  input_view.length(),
  buf,
  buflen);
- } else {
+ } elseif (input_view.length() <= 32) {
+// V8 is as fast for tiny strings (same threshold TextEncoder uses).
  nbytes = str->WriteUtf8V2(
  isolate, buf, buflen, String::WriteFlags::kReplaceInvalidUtf8);
+ } else {
+// Use simdutf for two-byte strings as well whenever the UTF-8 form
+// is guaranteed to fit; truncating writes (which must stop at a
+// character boundary) keep going through V8 so that their output
+// stays byte-for-byte identical.
+constchar16_t* data =
+reinterpret_cast<constchar16_t*>(input_view.data16());
+constsize_t length = input_view.length();
+ MaybeStackBuffer<char16_t, 1024> well_formed;
+if (!simdutf::validate_utf16(data, length)) {
+// Unpaired surrogates: encode a copy in which each of them has been
+// replaced with U+FFFD, which is what kReplaceInvalidUtf8 produces.
+ well_formed.AllocateSufficientStorage(length);
+simdutf::to_well_formed_utf16(data, length, well_formed.out());
+ data = well_formed.out();
+ }
+// A UTF-16 code unit never expands to more than 3 UTF-8 bytes, so
+// 3 * length is what StorageSize() hands most callers; only compute
+// the exact length when the buffer is smaller than that.
+if (buflen >= 3 * length ||
+ buflen >= simdutf::utf8_length_from_utf16(data, length)) {
+ nbytes = simdutf::convert_utf16_to_utf8(data, length, buf);
+ } else {
+// Does not fit: let V8 truncate at a character boundary.
+ nbytes = str->WriteUtf8V2(
+ isolate, buf, buflen, String::WriteFlags::kReplaceInvalidUtf8);
+ }
  }
 break;
 

test/parallel/test-buffer-write-utf8-two-byte.js

@@ -0,0 +1,128 @@
+'use strict';
+// UTF-8 encoding of two-byte (UTF-16) JS strings through the Buffer write
+// paths (Buffer.from, Buffer#write, Buffer.byteLength) must:
+// - produce standard UTF-8 for well-formed input of any size,
+// - replace lone surrogates with U+FFFD (EF BF BD),
+// - never write a partial character when the target is too small,
+// independent of which internal fast path handles the string.
+require('../common');
+constassert=require('assert');
+
+// Reference encoder written out longhand so the test does not depend on the
+// implementation under test (TextEncoder shares code with it).
+functionutf8Reference(str){
+constout=[];
+for(leti=0;i<str.length;i++){
+letcp=str.charCodeAt(i);
+if(cp>=0xd800&&cp<=0xdbff){
+constnext=i+1<str.length ? str.charCodeAt(i+1) : 0;
+if(next>=0xdc00&&next<=0xdfff){
+cp=0x10000+((cp-0xd800)<<10)+(next-0xdc00);
+i++;
+}else{
+cp=0xfffd;
+}
+}elseif(cp>=0xdc00&&cp<=0xdfff){
+cp=0xfffd;
+}
+if(cp<0x80){
+out.push(cp);
+}elseif(cp<0x800){
+out.push(0xc0|(cp>>6),0x80|(cp&0x3f));
+}elseif(cp<0x10000){
+out.push(0xe0|(cp>>12),0x80|((cp>>6)&0x3f),0x80|(cp&0x3f));
+}else{
+out.push(0xf0|(cp>>18),0x80|((cp>>12)&0x3f),
+0x80|((cp>>6)&0x3f),0x80|(cp&0x3f));
+}
+}
+returnBuffer.from(out);
+}
+
+functioncheckFull(str,label){
+constexpected=utf8Reference(str);
+assert.deepStrictEqual(Buffer.from(str,'utf8'),expected,`${label}: Buffer.from`);
+assert.strictEqual(Buffer.byteLength(str,'utf8'),expected.length,`${label}: byteLength`);
+// Exact-size target.
+constexact=Buffer.alloc(expected.length);
+assert.strictEqual(exact.write(str,'utf8'),expected.length,`${label}: write exact`);
+assert.deepStrictEqual(exact,expected,`${label}: write exact bytes`);
+// Oversized target (3 bytes per code unit is what most internal callers allocate).
+constbig=Buffer.alloc(str.length*3+7,0xaa);
+assert.strictEqual(big.write(str,2,'utf8'),expected.length,`${label}: write big`);
+assert.deepStrictEqual(big.subarray(2,2+expected.length),expected,`${label}: write big bytes`);
+assert.strictEqual(big[0],0xaa);
+assert.strictEqual(big[2+expected.length],0xaa,`${label}: no overrun`);
+}
+
+// Truncating writes must stop before the first character that does not fit.
+functioncheckTruncation(str,label){
+constexpected=utf8Reference(str);
+for(letsize=0;size<=Math.min(expected.length,70);size++){
+consttarget=Buffer.alloc(size+1,0xaa);
+constn=target.write(str,0,size,'utf8');
+assert.ok(n<=size,`${label}: size=${size} wrote ${n}`);
+assert.deepStrictEqual(target.subarray(0,n),expected.subarray(0,n),`${label}: prefix size=${size}`);
+assert.strictEqual(target[size],0xaa,`${label}: overrun size=${size}`);
+// What was written must be a whole number of characters: the next byte in
+// the reference (if any) has to be a lead byte, not a continuation byte.
+if(n<expected.length){
+assert.notStrictEqual(expected[n]&0xc0,0x80,`${label}: split char at size=${size}`);
+// And it stopped only because the next character really did not fit.
+letnext=n+1;
+while(next<expected.length&&(expected[next]&0xc0)===0x80)next++;
+assert.ok(next>size,`${label}: stopped early at size=${size} (n=${n}, next=${next})`);
+}
+}
+}
+
+// Force a two-byte representation even for ASCII/Latin-1 content by building
+// the string from a two-byte seed and slicing (V8 keeps the representation).
+functiontwoByte(str){
+consts=('\u{1F600}'+str).slice(2);
+assert.strictEqual(s,str);
+returns;
+}
+
+constsamples={
+ascii: 'The quick brown fox jumps over the lazy dog 0123456789',
+latin1: 'français élan über naïve façade ÿ',
+bmp: '日本語テキストとハングル한국어',
+astral: 'emoji \u{1F600}\u{1F4A9} math \u{1D49C} han \u{20BB7}',
+mixed: 'a é 日 \u{1F600} b ü 本 \u{1F4A9}',
+};
+
+for(const[name,base]ofObject.entries(samples)){
+for(constrepeatof[1,3,40,700,12000]){
+conststr=twoByte(base.repeat(repeat));
+checkFull(str,`${name} x${repeat}`);
+}
+checkTruncation(twoByte(base.repeat(3)),`${name} truncation`);
+}
+
+// Lone surrogates in various positions and sizes -> U+FFFD, rest intact.
+consthigh='\ud83d';
+constlow='\ude00';
+constsurrogateCases={
+'lone high': `ab${high}cd`,
+'lone low': `ab${low}cd`,
+'reversed pair': `ab${low}${high}cd`,
+'high at end': `abcd${high}`,
+'low at start': `${low}abcd`,
+'high high low': `${high}${high}${low}x`,
+'pair then lone': `${high}${low}${high}`,
+'only lone': high,
+};
+for(const[name,base]ofObject.entries(surrogateCases)){
+for(constpadof['','x'.repeat(50),'é'.repeat(300),'日'.repeat(30000)]){
+conststr=pad+base+pad;
+checkFull(str,`${name} pad=${pad.length}`);
+}
+checkTruncation(base+'zz',`${name} truncation`);
+}
+
+// Buffer.from of a large two-byte string equals TextEncoder output.
+{
+conststr=twoByte(samples.mixed.repeat(50000));
+assert.deepStrictEqual(Buffer.from(str),Buffer.from(newTextEncoder().encode(str)));
+}

benchmark/buffers/buffer-write-string-utf8.js

@@ -0,0 +1,36 @@
+'use strict';
+
+// buf.write(string, 'utf8') for strings whose in-memory representation is
+// one-byte (Latin-1) or two-byte (UTF-16), which take different encoder paths.
+constcommon=require('../common.js');
+constbench=common.createBenchmark(main,{
+chars: ['one-byte','two-byte','two-byte-astral','two-byte-lone-surrogate'],
+len: [16,256,2048,65536],
+n: [5e5],
+});
+
+functionmakeString(chars,len){
+switch(chars){
+case'one-byte':
+return'aé'.repeat(len/2);
+case'two-byte':
+return'aé€日'.repeat(len/4);
+case'two-byte-astral':
+return'aé€日\u{1F600}'.repeat(len/6).padEnd(len,'a');
+case'two-byte-lone-surrogate':
+return'aé€日'.repeat(len/4-1)+'ab\ud800c';
+default:
+thrownewError(chars);
+}
+}
+
+functionmain({ chars, len, n }){
+conststring=makeString(chars,len);
+constbuf=Buffer.allocUnsafe(Buffer.byteLength(string));
+if(len>=65536)n=Math.floor(n/32);
+bench.start();
+for(leti=0;i<n;++i){
+buf.write(string,0,'utf8');
+}
+bench.end(n);
+}

src/string_bytes.cc

@@ -310,9 +310,37 @@ size_t StringBytes::Write(Isolate* isolate,
  input_view.length(),
  buf,
  buflen);
- } else {
+ } elseif (input_view.length() <= 32) {
+// V8 is as fast for tiny strings (same threshold TextEncoder uses).
  nbytes = str->WriteUtf8V2(
  isolate, buf, buflen, String::WriteFlags::kReplaceInvalidUtf8);
+ } else {
+// Use simdutf for two-byte strings as well whenever the UTF-8 form
+// is guaranteed to fit; truncating writes (which must stop at a
+// character boundary) keep going through V8 so that their output
+// stays byte-for-byte identical.
+constchar16_t* data =
+reinterpret_cast<constchar16_t*>(input_view.data16());
+constsize_t length = input_view.length();
+ MaybeStackBuffer<char16_t, 1024> well_formed;
+if (!simdutf::validate_utf16(data, length)) {
+// Unpaired surrogates: encode a copy in which each of them has been
+// replaced with U+FFFD, which is what kReplaceInvalidUtf8 produces.
+ well_formed.AllocateSufficientStorage(length);
+simdutf::to_well_formed_utf16(data, length, well_formed.out());
+ data = well_formed.out();
+ }
+// A UTF-16 code unit never expands to more than 3 UTF-8 bytes, so
+// 3 * length is what StorageSize() hands most callers; only compute
+// the exact length when the buffer is smaller than that.
+if (buflen >= 3 * length ||
+ buflen >= simdutf::utf8_length_from_utf16(data, length)) {
+ nbytes = simdutf::convert_utf16_to_utf8(data, length, buf);
+ } else {
+// Does not fit: let V8 truncate at a character boundary.
+ nbytes = str->WriteUtf8V2(
+ isolate, buf, buflen, String::WriteFlags::kReplaceInvalidUtf8);
+ }
  }
 break;
 

test/parallel/test-buffer-write-utf8-two-byte.js

@@ -0,0 +1,128 @@
+'use strict';
+// UTF-8 encoding of two-byte (UTF-16) JS strings through the Buffer write
+// paths (Buffer.from, Buffer#write, Buffer.byteLength) must:
+// - produce standard UTF-8 for well-formed input of any size,
+// - replace lone surrogates with U+FFFD (EF BF BD),
+// - never write a partial character when the target is too small,
+// independent of which internal fast path handles the string.
+require('../common');
+constassert=require('assert');
+
+// Reference encoder written out longhand so the test does not depend on the
+// implementation under test (TextEncoder shares code with it).
+functionutf8Reference(str){
+constout=[];
+for(leti=0;i<str.length;i++){
+letcp=str.charCodeAt(i);
+if(cp>=0xd800&&cp<=0xdbff){
+constnext=i+1<str.length ? str.charCodeAt(i+1) : 0;
+if(next>=0xdc00&&next<=0xdfff){
+cp=0x10000+((cp-0xd800)<<10)+(next-0xdc00);
+i++;
+}else{
+cp=0xfffd;
+}
+}elseif(cp>=0xdc00&&cp<=0xdfff){
+cp=0xfffd;
+}
+if(cp<0x80){
+out.push(cp);
+}elseif(cp<0x800){
+out.push(0xc0|(cp>>6),0x80|(cp&0x3f));
+}elseif(cp<0x10000){
+out.push(0xe0|(cp>>12),0x80|((cp>>6)&0x3f),0x80|(cp&0x3f));
+}else{
+out.push(0xf0|(cp>>18),0x80|((cp>>12)&0x3f),
+0x80|((cp>>6)&0x3f),0x80|(cp&0x3f));
+}
+}
+returnBuffer.from(out);
+}
+
+functioncheckFull(str,label){
+constexpected=utf8Reference(str);
+assert.deepStrictEqual(Buffer.from(str,'utf8'),expected,`${label}: Buffer.from`);
+assert.strictEqual(Buffer.byteLength(str,'utf8'),expected.length,`${label}: byteLength`);
+// Exact-size target.
+constexact=Buffer.alloc(expected.length);
+assert.strictEqual(exact.write(str,'utf8'),expected.length,`${label}: write exact`);
+assert.deepStrictEqual(exact,expected,`${label}: write exact bytes`);
+// Oversized target (3 bytes per code unit is what most internal callers allocate).
+constbig=Buffer.alloc(str.length*3+7,0xaa);
+assert.strictEqual(big.write(str,2,'utf8'),expected.length,`${label}: write big`);
+assert.deepStrictEqual(big.subarray(2,2+expected.length),expected,`${label}: write big bytes`);
+assert.strictEqual(big[0],0xaa);
+assert.strictEqual(big[2+expected.length],0xaa,`${label}: no overrun`);
+}
+
+// Truncating writes must stop before the first character that does not fit.
+functioncheckTruncation(str,label){
+constexpected=utf8Reference(str);
+for(letsize=0;size<=Math.min(expected.length,70);size++){
+consttarget=Buffer.alloc(size+1,0xaa);
+constn=target.write(str,0,size,'utf8');
+assert.ok(n<=size,`${label}: size=${size} wrote ${n}`);
+assert.deepStrictEqual(target.subarray(0,n),expected.subarray(0,n),`${label}: prefix size=${size}`);
+assert.strictEqual(target[size],0xaa,`${label}: overrun size=${size}`);
+// What was written must be a whole number of characters: the next byte in
+// the reference (if any) has to be a lead byte, not a continuation byte.
+if(n<expected.length){
+assert.notStrictEqual(expected[n]&0xc0,0x80,`${label}: split char at size=${size}`);
+// And it stopped only because the next character really did not fit.
+letnext=n+1;
+while(next<expected.length&&(expected[next]&0xc0)===0x80)next++;
+assert.ok(next>size,`${label}: stopped early at size=${size} (n=${n}, next=${next})`);
+}
+}
+}
+
+// Force a two-byte representation even for ASCII/Latin-1 content by building
+// the string from a two-byte seed and slicing (V8 keeps the representation).
+functiontwoByte(str){
+consts=('\u{1F600}'+str).slice(2);
+assert.strictEqual(s,str);
+returns;
+}
+
+constsamples={
+ascii: 'The quick brown fox jumps over the lazy dog 0123456789',
+latin1: 'français élan über naïve façade ÿ',
+bmp: '日本語テキストとハングル한국어',
+astral: 'emoji \u{1F600}\u{1F4A9} math \u{1D49C} han \u{20BB7}',
+mixed: 'a é 日 \u{1F600} b ü 本 \u{1F4A9}',
+};
+
+for(const[name,base]ofObject.entries(samples)){
+for(constrepeatof[1,3,40,700,12000]){
+conststr=twoByte(base.repeat(repeat));
+checkFull(str,`${name} x${repeat}`);
+}
+checkTruncation(twoByte(base.repeat(3)),`${name} truncation`);
+}
+
+// Lone surrogates in various positions and sizes -> U+FFFD, rest intact.
+consthigh='\ud83d';
+constlow='\ude00';
+constsurrogateCases={
+'lone high': `ab${high}cd`,
+'lone low': `ab${low}cd`,
+'reversed pair': `ab${low}${high}cd`,
+'high at end': `abcd${high}`,
+'low at start': `${low}abcd`,
+'high high low': `${high}${high}${low}x`,
+'pair then lone': `${high}${low}${high}`,
+'only lone': high,
+};
+for(const[name,base]ofObject.entries(surrogateCases)){
+for(constpadof['','x'.repeat(50),'é'.repeat(300),'日'.repeat(30000)]){
+conststr=pad+base+pad;
+checkFull(str,`${name} pad=${pad.length}`);
+}
+checkTruncation(base+'zz',`${name} truncation`);
+}
+
+// Buffer.from of a large two-byte string equals TextEncoder output.
+{
+conststr=twoByte(samples.mixed.repeat(50000));
+assert.deepStrictEqual(Buffer.from(str),Buffer.from(newTextEncoder().encode(str)));
+}

benchmark/buffers/buffer-write-string-utf8.js

@@ -0,0 +1,36 @@
+'use strict';
+
+// buf.write(string, 'utf8') for strings whose in-memory representation is
+// one-byte (Latin-1) or two-byte (UTF-16), which take different encoder paths.
+constcommon=require('../common.js');
+constbench=common.createBenchmark(main,{
+chars: ['one-byte','two-byte','two-byte-astral','two-byte-lone-surrogate'],
+len: [16,256,2048,65536],
+n: [5e5],
+});
+
+functionmakeString(chars,len){
+switch(chars){
+case'one-byte':
+return'aé'.repeat(len/2);
+case'two-byte':
+return'aé€日'.repeat(len/4);
+case'two-byte-astral':
+return'aé€日\u{1F600}'.repeat(len/6).padEnd(len,'a');
+case'two-byte-lone-surrogate':
+return'aé€日'.repeat(len/4-1)+'ab\ud800c';
+default:
+thrownewError(chars);
+}
+}
+
+functionmain({ chars, len, n }){
+conststring=makeString(chars,len);
+constbuf=Buffer.allocUnsafe(Buffer.byteLength(string));
+if(len>=65536)n=Math.floor(n/32);
+bench.start();
+for(leti=0;i<n;++i){
+buf.write(string,0,'utf8');
+}
+bench.end(n);
+}

src/string_bytes.cc

@@ -310,9 +310,37 @@ size_t StringBytes::Write(Isolate* isolate,
  input_view.length(),
  buf,
  buflen);
- } else {
+ } elseif (input_view.length() <= 32) {
+// V8 is as fast for tiny strings (same threshold TextEncoder uses).
  nbytes = str->WriteUtf8V2(
  isolate, buf, buflen, String::WriteFlags::kReplaceInvalidUtf8);
+ } else {
+// Use simdutf for two-byte strings as well whenever the UTF-8 form
+// is guaranteed to fit; truncating writes (which must stop at a
+// character boundary) keep going through V8 so that their output
+// stays byte-for-byte identical.
+constchar16_t* data =
+reinterpret_cast<constchar16_t*>(input_view.data16());
+constsize_t length = input_view.length();
+ MaybeStackBuffer<char16_t, 1024> well_formed;
+if (!simdutf::validate_utf16(data, length)) {
+// Unpaired surrogates: encode a copy in which each of them has been
+// replaced with U+FFFD, which is what kReplaceInvalidUtf8 produces.
+ well_formed.AllocateSufficientStorage(length);
+simdutf::to_well_formed_utf16(data, length, well_formed.out());
+ data = well_formed.out();
+ }
+// A UTF-16 code unit never expands to more than 3 UTF-8 bytes, so
+// 3 * length is what StorageSize() hands most callers; only compute
+// the exact length when the buffer is smaller than that.
+if (buflen >= 3 * length ||
+ buflen >= simdutf::utf8_length_from_utf16(data, length)) {
+ nbytes = simdutf::convert_utf16_to_utf8(data, length, buf);
+ } else {
+// Does not fit: let V8 truncate at a character boundary.
+ nbytes = str->WriteUtf8V2(
+ isolate, buf, buflen, String::WriteFlags::kReplaceInvalidUtf8);
+ }
  }
 break;
 

test/parallel/test-buffer-write-utf8-two-byte.js

@@ -0,0 +1,128 @@
+'use strict';
+// UTF-8 encoding of two-byte (UTF-16) JS strings through the Buffer write
+// paths (Buffer.from, Buffer#write, Buffer.byteLength) must:
+// - produce standard UTF-8 for well-formed input of any size,
+// - replace lone surrogates with U+FFFD (EF BF BD),
+// - never write a partial character when the target is too small,
+// independent of which internal fast path handles the string.
+require('../common');
+constassert=require('assert');
+
+// Reference encoder written out longhand so the test does not depend on the
+// implementation under test (TextEncoder shares code with it).
+functionutf8Reference(str){
+constout=[];
+for(leti=0;i<str.length;i++){
+letcp=str.charCodeAt(i);
+if(cp>=0xd800&&cp<=0xdbff){
+constnext=i+1<str.length ? str.charCodeAt(i+1) : 0;
+if(next>=0xdc00&&next<=0xdfff){
+cp=0x10000+((cp-0xd800)<<10)+(next-0xdc00);
+i++;
+}else{
+cp=0xfffd;
+}
+}elseif(cp>=0xdc00&&cp<=0xdfff){
+cp=0xfffd;
+}
+if(cp<0x80){
+out.push(cp);
+}elseif(cp<0x800){
+out.push(0xc0|(cp>>6),0x80|(cp&0x3f));
+}elseif(cp<0x10000){
+out.push(0xe0|(cp>>12),0x80|((cp>>6)&0x3f),0x80|(cp&0x3f));
+}else{
+out.push(0xf0|(cp>>18),0x80|((cp>>12)&0x3f),
+0x80|((cp>>6)&0x3f),0x80|(cp&0x3f));
+}
+}
+returnBuffer.from(out);
+}
+
+functioncheckFull(str,label){
+constexpected=utf8Reference(str);
+assert.deepStrictEqual(Buffer.from(str,'utf8'),expected,`${label}: Buffer.from`);
+assert.strictEqual(Buffer.byteLength(str,'utf8'),expected.length,`${label}: byteLength`);
+// Exact-size target.
+constexact=Buffer.alloc(expected.length);
+assert.strictEqual(exact.write(str,'utf8'),expected.length,`${label}: write exact`);
+assert.deepStrictEqual(exact,expected,`${label}: write exact bytes`);
+// Oversized target (3 bytes per code unit is what most internal callers allocate).
+constbig=Buffer.alloc(str.length*3+7,0xaa);
+assert.strictEqual(big.write(str,2,'utf8'),expected.length,`${label}: write big`);
+assert.deepStrictEqual(big.subarray(2,2+expected.length),expected,`${label}: write big bytes`);
+assert.strictEqual(big[0],0xaa);
+assert.strictEqual(big[2+expected.length],0xaa,`${label}: no overrun`);
+}
+
+// Truncating writes must stop before the first character that does not fit.
+functioncheckTruncation(str,label){
+constexpected=utf8Reference(str);
+for(letsize=0;size<=Math.min(expected.length,70);size++){
+consttarget=Buffer.alloc(size+1,0xaa);
+constn=target.write(str,0,size,'utf8');
+assert.ok(n<=size,`${label}: size=${size} wrote ${n}`);
+assert.deepStrictEqual(target.subarray(0,n),expected.subarray(0,n),`${label}: prefix size=${size}`);
+assert.strictEqual(target[size],0xaa,`${label}: overrun size=${size}`);
+// What was written must be a whole number of characters: the next byte in
+// the reference (if any) has to be a lead byte, not a continuation byte.
+if(n<expected.length){
+assert.notStrictEqual(expected[n]&0xc0,0x80,`${label}: split char at size=${size}`);
+// And it stopped only because the next character really did not fit.
+letnext=n+1;
+while(next<expected.length&&(expected[next]&0xc0)===0x80)next++;
+assert.ok(next>size,`${label}: stopped early at size=${size} (n=${n}, next=${next})`);
+}
+}
+}
+
+// Force a two-byte representation even for ASCII/Latin-1 content by building
+// the string from a two-byte seed and slicing (V8 keeps the representation).
+functiontwoByte(str){
+consts=('\u{1F600}'+str).slice(2);
+assert.strictEqual(s,str);
+returns;
+}
+
+constsamples={
+ascii: 'The quick brown fox jumps over the lazy dog 0123456789',
+latin1: 'français élan über naïve façade ÿ',
+bmp: '日本語テキストとハングル한국어',
+astral: 'emoji \u{1F600}\u{1F4A9} math \u{1D49C} han \u{20BB7}',
+mixed: 'a é 日 \u{1F600} b ü 本 \u{1F4A9}',
+};
+
+for(const[name,base]ofObject.entries(samples)){
+for(constrepeatof[1,3,40,700,12000]){
+conststr=twoByte(base.repeat(repeat));
+checkFull(str,`${name} x${repeat}`);
+}
+checkTruncation(twoByte(base.repeat(3)),`${name} truncation`);
+}
+
+// Lone surrogates in various positions and sizes -> U+FFFD, rest intact.
+consthigh='\ud83d';
+constlow='\ude00';
+constsurrogateCases={
+'lone high': `ab${high}cd`,
+'lone low': `ab${low}cd`,
+'reversed pair': `ab${low}${high}cd`,
+'high at end': `abcd${high}`,
+'low at start': `${low}abcd`,
+'high high low': `${high}${high}${low}x`,
+'pair then lone': `${high}${low}${high}`,
+'only lone': high,
+};
+for(const[name,base]ofObject.entries(surrogateCases)){
+for(constpadof['','x'.repeat(50),'é'.repeat(300),'日'.repeat(30000)]){
+conststr=pad+base+pad;
+checkFull(str,`${name} pad=${pad.length}`);
+}
+checkTruncation(base+'zz',`${name} truncation`);
+}
+
+// Buffer.from of a large two-byte string equals TextEncoder output.
+{
+conststr=twoByte(samples.mixed.repeat(50000));
+assert.deepStrictEqual(Buffer.from(str),Buffer.from(newTextEncoder().encode(str)));
+}

benchmark/buffers/buffer-write-string-utf8.js

@@ -0,0 +1,36 @@
+'use strict';
+
+// buf.write(string, 'utf8') for strings whose in-memory representation is
+// one-byte (Latin-1) or two-byte (UTF-16), which take different encoder paths.
+constcommon=require('../common.js');
+constbench=common.createBenchmark(main,{
+chars: ['one-byte','two-byte','two-byte-astral','two-byte-lone-surrogate'],
+len: [16,256,2048,65536],
+n: [5e5],
+});
+
+functionmakeString(chars,len){
+switch(chars){
+case'one-byte':
+return'aé'.repeat(len/2);
+case'two-byte':
+return'aé€日'.repeat(len/4);
+case'two-byte-astral':
+return'aé€日\u{1F600}'.repeat(len/6).padEnd(len,'a');
+case'two-byte-lone-surrogate':
+return'aé€日'.repeat(len/4-1)+'ab\ud800c';
+default:
+thrownewError(chars);
+}
+}
+
+functionmain({ chars, len, n }){
+conststring=makeString(chars,len);
+constbuf=Buffer.allocUnsafe(Buffer.byteLength(string));
+if(len>=65536)n=Math.floor(n/32);
+bench.start();
+for(leti=0;i<n;++i){
+buf.write(string,0,'utf8');
+}
+bench.end(n);
+}

src/string_bytes.cc

@@ -310,9 +310,37 @@ size_t StringBytes::Write(Isolate* isolate,
  input_view.length(),
  buf,
  buflen);
- } else {
+ } elseif (input_view.length() <= 32) {
+// V8 is as fast for tiny strings (same threshold TextEncoder uses).
  nbytes = str->WriteUtf8V2(
  isolate, buf, buflen, String::WriteFlags::kReplaceInvalidUtf8);
+ } else {
+// Use simdutf for two-byte strings as well whenever the UTF-8 form
+// is guaranteed to fit; truncating writes (which must stop at a
+// character boundary) keep going through V8 so that their output
+// stays byte-for-byte identical.
+constchar16_t* data =
+reinterpret_cast<constchar16_t*>(input_view.data16());
+constsize_t length = input_view.length();
+ MaybeStackBuffer<char16_t, 1024> well_formed;
+if (!simdutf::validate_utf16(data, length)) {
+// Unpaired surrogates: encode a copy in which each of them has been
+// replaced with U+FFFD, which is what kReplaceInvalidUtf8 produces.
+ well_formed.AllocateSufficientStorage(length);
+simdutf::to_well_formed_utf16(data, length, well_formed.out());
+ data = well_formed.out();
+ }
+// A UTF-16 code unit never expands to more than 3 UTF-8 bytes, so
+// 3 * length is what StorageSize() hands most callers; only compute
+// the exact length when the buffer is smaller than that.
+if (buflen >= 3 * length ||
+ buflen >= simdutf::utf8_length_from_utf16(data, length)) {
+ nbytes = simdutf::convert_utf16_to_utf8(data, length, buf);
+ } else {
+// Does not fit: let V8 truncate at a character boundary.
+ nbytes = str->WriteUtf8V2(
+ isolate, buf, buflen, String::WriteFlags::kReplaceInvalidUtf8);
+ }
  }
 break;
 

test/parallel/test-buffer-write-utf8-two-byte.js

@@ -0,0 +1,128 @@
+'use strict';
+// UTF-8 encoding of two-byte (UTF-16) JS strings through the Buffer write
+// paths (Buffer.from, Buffer#write, Buffer.byteLength) must:
+// - produce standard UTF-8 for well-formed input of any size,
+// - replace lone surrogates with U+FFFD (EF BF BD),
+// - never write a partial character when the target is too small,
+// independent of which internal fast path handles the string.
+require('../common');
+constassert=require('assert');
+
+// Reference encoder written out longhand so the test does not depend on the
+// implementation under test (TextEncoder shares code with it).
+functionutf8Reference(str){
+constout=[];
+for(leti=0;i<str.length;i++){
+letcp=str.charCodeAt(i);
+if(cp>=0xd800&&cp<=0xdbff){
+constnext=i+1<str.length ? str.charCodeAt(i+1) : 0;
+if(next>=0xdc00&&next<=0xdfff){
+cp=0x10000+((cp-0xd800)<<10)+(next-0xdc00);
+i++;
+}else{
+cp=0xfffd;
+}
+}elseif(cp>=0xdc00&&cp<=0xdfff){
+cp=0xfffd;
+}
+if(cp<0x80){
+out.push(cp);
+}elseif(cp<0x800){
+out.push(0xc0|(cp>>6),0x80|(cp&0x3f));
+}elseif(cp<0x10000){
+out.push(0xe0|(cp>>12),0x80|((cp>>6)&0x3f),0x80|(cp&0x3f));
+}else{
+out.push(0xf0|(cp>>18),0x80|((cp>>12)&0x3f),
+0x80|((cp>>6)&0x3f),0x80|(cp&0x3f));
+}
+}
+returnBuffer.from(out);
+}
+
+functioncheckFull(str,label){
+constexpected=utf8Reference(str);
+assert.deepStrictEqual(Buffer.from(str,'utf8'),expected,`${label}: Buffer.from`);
+assert.strictEqual(Buffer.byteLength(str,'utf8'),expected.length,`${label}: byteLength`);
+// Exact-size target.
+constexact=Buffer.alloc(expected.length);
+assert.strictEqual(exact.write(str,'utf8'),expected.length,`${label}: write exact`);
+assert.deepStrictEqual(exact,expected,`${label}: write exact bytes`);
+// Oversized target (3 bytes per code unit is what most internal callers allocate).
+constbig=Buffer.alloc(str.length*3+7,0xaa);
+assert.strictEqual(big.write(str,2,'utf8'),expected.length,`${label}: write big`);
+assert.deepStrictEqual(big.subarray(2,2+expected.length),expected,`${label}: write big bytes`);
+assert.strictEqual(big[0],0xaa);
+assert.strictEqual(big[2+expected.length],0xaa,`${label}: no overrun`);
+}
+
+// Truncating writes must stop before the first character that does not fit.
+functioncheckTruncation(str,label){
+constexpected=utf8Reference(str);
+for(letsize=0;size<=Math.min(expected.length,70);size++){
+consttarget=Buffer.alloc(size+1,0xaa);
+constn=target.write(str,0,size,'utf8');
+assert.ok(n<=size,`${label}: size=${size} wrote ${n}`);
+assert.deepStrictEqual(target.subarray(0,n),expected.subarray(0,n),`${label}: prefix size=${size}`);
+assert.strictEqual(target[size],0xaa,`${label}: overrun size=${size}`);
+// What was written must be a whole number of characters: the next byte in
+// the reference (if any) has to be a lead byte, not a continuation byte.
+if(n<expected.length){
+assert.notStrictEqual(expected[n]&0xc0,0x80,`${label}: split char at size=${size}`);
+// And it stopped only because the next character really did not fit.
+letnext=n+1;
+while(next<expected.length&&(expected[next]&0xc0)===0x80)next++;
+assert.ok(next>size,`${label}: stopped early at size=${size} (n=${n}, next=${next})`);
+}
+}
+}
+
+// Force a two-byte representation even for ASCII/Latin-1 content by building
+// the string from a two-byte seed and slicing (V8 keeps the representation).
+functiontwoByte(str){
+consts=('\u{1F600}'+str).slice(2);
+assert.strictEqual(s,str);
+returns;
+}
+
+constsamples={
+ascii: 'The quick brown fox jumps over the lazy dog 0123456789',
+latin1: 'français élan über naïve façade ÿ',
+bmp: '日本語テキストとハングル한국어',
+astral: 'emoji \u{1F600}\u{1F4A9} math \u{1D49C} han \u{20BB7}',
+mixed: 'a é 日 \u{1F600} b ü 本 \u{1F4A9}',
+};
+
+for(const[name,base]ofObject.entries(samples)){
+for(constrepeatof[1,3,40,700,12000]){
+conststr=twoByte(base.repeat(repeat));
+checkFull(str,`${name} x${repeat}`);
+}
+checkTruncation(twoByte(base.repeat(3)),`${name} truncation`);
+}
+
+// Lone surrogates in various positions and sizes -> U+FFFD, rest intact.
+consthigh='\ud83d';
+constlow='\ude00';
+constsurrogateCases={
+'lone high': `ab${high}cd`,
+'lone low': `ab${low}cd`,
+'reversed pair': `ab${low}${high}cd`,
+'high at end': `abcd${high}`,
+'low at start': `${low}abcd`,
+'high high low': `${high}${high}${low}x`,
+'pair then lone': `${high}${low}${high}`,
+'only lone': high,
+};
+for(const[name,base]ofObject.entries(surrogateCases)){
+for(constpadof['','x'.repeat(50),'é'.repeat(300),'日'.repeat(30000)]){
+conststr=pad+base+pad;
+checkFull(str,`${name} pad=${pad.length}`);
+}
+checkTruncation(base+'zz',`${name} truncation`);
+}
+
+// Buffer.from of a large two-byte string equals TextEncoder output.
+{
+conststr=twoByte(samples.mixed.repeat(50000));
+assert.deepStrictEqual(Buffer.from(str),Buffer.from(newTextEncoder().encode(str)));
+}

benchmark/buffers/buffer-write-string-utf8.js

@@ -0,0 +1,36 @@
+'use strict';
+
+// buf.write(string, 'utf8') for strings whose in-memory representation is
+// one-byte (Latin-1) or two-byte (UTF-16), which take different encoder paths.
+constcommon=require('../common.js');
+constbench=common.createBenchmark(main,{
+chars: ['one-byte','two-byte','two-byte-astral','two-byte-lone-surrogate'],
+len: [16,256,2048,65536],
+n: [5e5],
+});
+
+functionmakeString(chars,len){
+switch(chars){
+case'one-byte':
+return'aé'.repeat(len/2);
+case'two-byte':
+return'aé€日'.repeat(len/4);
+case'two-byte-astral':
+return'aé€日\u{1F600}'.repeat(len/6).padEnd(len,'a');
+case'two-byte-lone-surrogate':
+return'aé€日'.repeat(len/4-1)+'ab\ud800c';
+default:
+thrownewError(chars);
+}
+}
+
+functionmain({ chars, len, n }){
+conststring=makeString(chars,len);
+constbuf=Buffer.allocUnsafe(Buffer.byteLength(string));
+if(len>=65536)n=Math.floor(n/32);
+bench.start();
+for(leti=0;i<n;++i){
+buf.write(string,0,'utf8');
+}
+bench.end(n);
+}

src/string_bytes.cc

@@ -310,9 +310,37 @@ size_t StringBytes::Write(Isolate* isolate,
  input_view.length(),
  buf,
  buflen);
- } else {
+ } elseif (input_view.length() <= 32) {
+// V8 is as fast for tiny strings (same threshold TextEncoder uses).
  nbytes = str->WriteUtf8V2(
  isolate, buf, buflen, String::WriteFlags::kReplaceInvalidUtf8);
+ } else {
+// Use simdutf for two-byte strings as well whenever the UTF-8 form
+// is guaranteed to fit; truncating writes (which must stop at a
+// character boundary) keep going through V8 so that their output
+// stays byte-for-byte identical.
+constchar16_t* data =
+reinterpret_cast<constchar16_t*>(input_view.data16());
+constsize_t length = input_view.length();
+ MaybeStackBuffer<char16_t, 1024> well_formed;
+if (!simdutf::validate_utf16(data, length)) {
+// Unpaired surrogates: encode a copy in which each of them has been
+// replaced with U+FFFD, which is what kReplaceInvalidUtf8 produces.
+ well_formed.AllocateSufficientStorage(length);
+simdutf::to_well_formed_utf16(data, length, well_formed.out());
+ data = well_formed.out();
+ }
+// A UTF-16 code unit never expands to more than 3 UTF-8 bytes, so
+// 3 * length is what StorageSize() hands most callers; only compute
+// the exact length when the buffer is smaller than that.
+if (buflen >= 3 * length ||
+ buflen >= simdutf::utf8_length_from_utf16(data, length)) {
+ nbytes = simdutf::convert_utf16_to_utf8(data, length, buf);
+ } else {
+// Does not fit: let V8 truncate at a character boundary.
+ nbytes = str->WriteUtf8V2(
+ isolate, buf, buflen, String::WriteFlags::kReplaceInvalidUtf8);
+ }
  }
 break;
 

test/parallel/test-buffer-write-utf8-two-byte.js

@@ -0,0 +1,128 @@
+'use strict';
+// UTF-8 encoding of two-byte (UTF-16) JS strings through the Buffer write
+// paths (Buffer.from, Buffer#write, Buffer.byteLength) must:
+// - produce standard UTF-8 for well-formed input of any size,
+// - replace lone surrogates with U+FFFD (EF BF BD),
+// - never write a partial character when the target is too small,
+// independent of which internal fast path handles the string.
+require('../common');
+constassert=require('assert');
+
+// Reference encoder written out longhand so the test does not depend on the
+// implementation under test (TextEncoder shares code with it).
+functionutf8Reference(str){
+constout=[];
+for(leti=0;i<str.length;i++){
+letcp=str.charCodeAt(i);
+if(cp>=0xd800&&cp<=0xdbff){
+constnext=i+1<str.length ? str.charCodeAt(i+1) : 0;
+if(next>=0xdc00&&next<=0xdfff){
+cp=0x10000+((cp-0xd800)<<10)+(next-0xdc00);
+i++;
+}else{
+cp=0xfffd;
+}
+}elseif(cp>=0xdc00&&cp<=0xdfff){
+cp=0xfffd;
+}
+if(cp<0x80){
+out.push(cp);
+}elseif(cp<0x800){
+out.push(0xc0|(cp>>6),0x80|(cp&0x3f));
+}elseif(cp<0x10000){
+out.push(0xe0|(cp>>12),0x80|((cp>>6)&0x3f),0x80|(cp&0x3f));
+}else{
+out.push(0xf0|(cp>>18),0x80|((cp>>12)&0x3f),
+0x80|((cp>>6)&0x3f),0x80|(cp&0x3f));
+}
+}
+returnBuffer.from(out);
+}
+
+functioncheckFull(str,label){
+constexpected=utf8Reference(str);
+assert.deepStrictEqual(Buffer.from(str,'utf8'),expected,`${label}: Buffer.from`);
+assert.strictEqual(Buffer.byteLength(str,'utf8'),expected.length,`${label}: byteLength`);
+// Exact-size target.
+constexact=Buffer.alloc(expected.length);
+assert.strictEqual(exact.write(str,'utf8'),expected.length,`${label}: write exact`);
+assert.deepStrictEqual(exact,expected,`${label}: write exact bytes`);
+// Oversized target (3 bytes per code unit is what most internal callers allocate).
+constbig=Buffer.alloc(str.length*3+7,0xaa);
+assert.strictEqual(big.write(str,2,'utf8'),expected.length,`${label}: write big`);
+assert.deepStrictEqual(big.subarray(2,2+expected.length),expected,`${label}: write big bytes`);
+assert.strictEqual(big[0],0xaa);
+assert.strictEqual(big[2+expected.length],0xaa,`${label}: no overrun`);
+}
+
+// Truncating writes must stop before the first character that does not fit.
+functioncheckTruncation(str,label){
+constexpected=utf8Reference(str);
+for(letsize=0;size<=Math.min(expected.length,70);size++){
+consttarget=Buffer.alloc(size+1,0xaa);
+constn=target.write(str,0,size,'utf8');
+assert.ok(n<=size,`${label}: size=${size} wrote ${n}`);
+assert.deepStrictEqual(target.subarray(0,n),expected.subarray(0,n),`${label}: prefix size=${size}`);
+assert.strictEqual(target[size],0xaa,`${label}: overrun size=${size}`);
+// What was written must be a whole number of characters: the next byte in
+// the reference (if any) has to be a lead byte, not a continuation byte.
+if(n<expected.length){
+assert.notStrictEqual(expected[n]&0xc0,0x80,`${label}: split char at size=${size}`);
+// And it stopped only because the next character really did not fit.
+letnext=n+1;
+while(next<expected.length&&(expected[next]&0xc0)===0x80)next++;
+assert.ok(next>size,`${label}: stopped early at size=${size} (n=${n}, next=${next})`);
+}
+}
+}
+
+// Force a two-byte representation even for ASCII/Latin-1 content by building
+// the string from a two-byte seed and slicing (V8 keeps the representation).
+functiontwoByte(str){
+consts=('\u{1F600}'+str).slice(2);
+assert.strictEqual(s,str);
+returns;
+}
+
+constsamples={
+ascii: 'The quick brown fox jumps over the lazy dog 0123456789',
+latin1: 'français élan über naïve façade ÿ',
+bmp: '日本語テキストとハングル한국어',
+astral: 'emoji \u{1F600}\u{1F4A9} math \u{1D49C} han \u{20BB7}',
+mixed: 'a é 日 \u{1F600} b ü 本 \u{1F4A9}',
+};
+
+for(const[name,base]ofObject.entries(samples)){
+for(constrepeatof[1,3,40,700,12000]){
+conststr=twoByte(base.repeat(repeat));
+checkFull(str,`${name} x${repeat}`);
+}
+checkTruncation(twoByte(base.repeat(3)),`${name} truncation`);
+}
+
+// Lone surrogates in various positions and sizes -> U+FFFD, rest intact.
+consthigh='\ud83d';
+constlow='\ude00';
+constsurrogateCases={
+'lone high': `ab${high}cd`,
+'lone low': `ab${low}cd`,
+'reversed pair': `ab${low}${high}cd`,
+'high at end': `abcd${high}`,
+'low at start': `${low}abcd`,
+'high high low': `${high}${high}${low}x`,
+'pair then lone': `${high}${low}${high}`,
+'only lone': high,
+};
+for(const[name,base]ofObject.entries(surrogateCases)){
+for(constpadof['','x'.repeat(50),'é'.repeat(300),'日'.repeat(30000)]){
+conststr=pad+base+pad;
+checkFull(str,`${name} pad=${pad.length}`);
+}
+checkTruncation(base+'zz',`${name} truncation`);
+}
+
+// Buffer.from of a large two-byte string equals TextEncoder output.
+{
+conststr=twoByte(samples.mixed.repeat(50000));
+assert.deepStrictEqual(Buffer.from(str),Buffer.from(newTextEncoder().encode(str)));
+}

benchmark/buffers/buffer-write-string-utf8.js

@@ -0,0 +1,36 @@
+'use strict';
+
+// buf.write(string, 'utf8') for strings whose in-memory representation is
+// one-byte (Latin-1) or two-byte (UTF-16), which take different encoder paths.
+constcommon=require('../common.js');
+constbench=common.createBenchmark(main,{
+chars: ['one-byte','two-byte','two-byte-astral','two-byte-lone-surrogate'],
+len: [16,256,2048,65536],
+n: [5e5],
+});
+
+functionmakeString(chars,len){
+switch(chars){
+case'one-byte':
+return'aé'.repeat(len/2);
+case'two-byte':
+return'aé€日'.repeat(len/4);
+case'two-byte-astral':
+return'aé€日\u{1F600}'.repeat(len/6).padEnd(len,'a');
+case'two-byte-lone-surrogate':
+return'aé€日'.repeat(len/4-1)+'ab\ud800c';
+default:
+thrownewError(chars);
+}
+}
+
+functionmain({ chars, len, n }){
+conststring=makeString(chars,len);
+constbuf=Buffer.allocUnsafe(Buffer.byteLength(string));
+if(len>=65536)n=Math.floor(n/32);
+bench.start();
+for(leti=0;i<n;++i){
+buf.write(string,0,'utf8');
+}
+bench.end(n);
+}

src/string_bytes.cc

@@ -310,9 +310,37 @@ size_t StringBytes::Write(Isolate* isolate,
  input_view.length(),
  buf,
  buflen);
- } else {
+ } elseif (input_view.length() <= 32) {
+// V8 is as fast for tiny strings (same threshold TextEncoder uses).
  nbytes = str->WriteUtf8V2(
  isolate, buf, buflen, String::WriteFlags::kReplaceInvalidUtf8);
+ } else {
+// Use simdutf for two-byte strings as well whenever the UTF-8 form
+// is guaranteed to fit; truncating writes (which must stop at a
+// character boundary) keep going through V8 so that their output
+// stays byte-for-byte identical.
+constchar16_t* data =
+reinterpret_cast<constchar16_t*>(input_view.data16());
+constsize_t length = input_view.length();
+ MaybeStackBuffer<char16_t, 1024> well_formed;
+if (!simdutf::validate_utf16(data, length)) {
+// Unpaired surrogates: encode a copy in which each of them has been
+// replaced with U+FFFD, which is what kReplaceInvalidUtf8 produces.
+ well_formed.AllocateSufficientStorage(length);
+simdutf::to_well_formed_utf16(data, length, well_formed.out());
+ data = well_formed.out();
+ }
+// A UTF-16 code unit never expands to more than 3 UTF-8 bytes, so
+// 3 * length is what StorageSize() hands most callers; only compute
+// the exact length when the buffer is smaller than that.
+if (buflen >= 3 * length ||
+ buflen >= simdutf::utf8_length_from_utf16(data, length)) {
+ nbytes = simdutf::convert_utf16_to_utf8(data, length, buf);
+ } else {
+// Does not fit: let V8 truncate at a character boundary.
+ nbytes = str->WriteUtf8V2(
+ isolate, buf, buflen, String::WriteFlags::kReplaceInvalidUtf8);
+ }
  }
 break;
 

test/parallel/test-buffer-write-utf8-two-byte.js

@@ -0,0 +1,128 @@
+'use strict';
+// UTF-8 encoding of two-byte (UTF-16) JS strings through the Buffer write
+// paths (Buffer.from, Buffer#write, Buffer.byteLength) must:
+// - produce standard UTF-8 for well-formed input of any size,
+// - replace lone surrogates with U+FFFD (EF BF BD),
+// - never write a partial character when the target is too small,
+// independent of which internal fast path handles the string.
+require('../common');
+constassert=require('assert');
+
+// Reference encoder written out longhand so the test does not depend on the
+// implementation under test (TextEncoder shares code with it).
+functionutf8Reference(str){
+constout=[];
+for(leti=0;i<str.length;i++){
+letcp=str.charCodeAt(i);
+if(cp>=0xd800&&cp<=0xdbff){
+constnext=i+1<str.length ? str.charCodeAt(i+1) : 0;
+if(next>=0xdc00&&next<=0xdfff){
+cp=0x10000+((cp-0xd800)<<10)+(next-0xdc00);
+i++;
+}else{
+cp=0xfffd;
+}
+}elseif(cp>=0xdc00&&cp<=0xdfff){
+cp=0xfffd;
+}
+if(cp<0x80){
+out.push(cp);
+}elseif(cp<0x800){
+out.push(0xc0|(cp>>6),0x80|(cp&0x3f));
+}elseif(cp<0x10000){
+out.push(0xe0|(cp>>12),0x80|((cp>>6)&0x3f),0x80|(cp&0x3f));
+}else{
+out.push(0xf0|(cp>>18),0x80|((cp>>12)&0x3f),
+0x80|((cp>>6)&0x3f),0x80|(cp&0x3f));
+}
+}
+returnBuffer.from(out);
+}
+
+functioncheckFull(str,label){
+constexpected=utf8Reference(str);
+assert.deepStrictEqual(Buffer.from(str,'utf8'),expected,`${label}: Buffer.from`);
+assert.strictEqual(Buffer.byteLength(str,'utf8'),expected.length,`${label}: byteLength`);
+// Exact-size target.
+constexact=Buffer.alloc(expected.length);
+assert.strictEqual(exact.write(str,'utf8'),expected.length,`${label}: write exact`);
+assert.deepStrictEqual(exact,expected,`${label}: write exact bytes`);
+// Oversized target (3 bytes per code unit is what most internal callers allocate).
+constbig=Buffer.alloc(str.length*3+7,0xaa);
+assert.strictEqual(big.write(str,2,'utf8'),expected.length,`${label}: write big`);
+assert.deepStrictEqual(big.subarray(2,2+expected.length),expected,`${label}: write big bytes`);
+assert.strictEqual(big[0],0xaa);
+assert.strictEqual(big[2+expected.length],0xaa,`${label}: no overrun`);
+}
+
+// Truncating writes must stop before the first character that does not fit.
+functioncheckTruncation(str,label){
+constexpected=utf8Reference(str);
+for(letsize=0;size<=Math.min(expected.length,70);size++){
+consttarget=Buffer.alloc(size+1,0xaa);
+constn=target.write(str,0,size,'utf8');
+assert.ok(n<=size,`${label}: size=${size} wrote ${n}`);
+assert.deepStrictEqual(target.subarray(0,n),expected.subarray(0,n),`${label}: prefix size=${size}`);
+assert.strictEqual(target[size],0xaa,`${label}: overrun size=${size}`);
+// What was written must be a whole number of characters: the next byte in
+// the reference (if any) has to be a lead byte, not a continuation byte.
+if(n<expected.length){
+assert.notStrictEqual(expected[n]&0xc0,0x80,`${label}: split char at size=${size}`);
+// And it stopped only because the next character really did not fit.
+letnext=n+1;
+while(next<expected.length&&(expected[next]&0xc0)===0x80)next++;
+assert.ok(next>size,`${label}: stopped early at size=${size} (n=${n}, next=${next})`);
+}
+}
+}
+
+// Force a two-byte representation even for ASCII/Latin-1 content by building
+// the string from a two-byte seed and slicing (V8 keeps the representation).
+functiontwoByte(str){
+consts=('\u{1F600}'+str).slice(2);
+assert.strictEqual(s,str);
+returns;
+}
+
+constsamples={
+ascii: 'The quick brown fox jumps over the lazy dog 0123456789',
+latin1: 'français élan über naïve façade ÿ',
+bmp: '日本語テキストとハングル한국어',
+astral: 'emoji \u{1F600}\u{1F4A9} math \u{1D49C} han \u{20BB7}',
+mixed: 'a é 日 \u{1F600} b ü 本 \u{1F4A9}',
+};
+
+for(const[name,base]ofObject.entries(samples)){
+for(constrepeatof[1,3,40,700,12000]){
+conststr=twoByte(base.repeat(repeat));
+checkFull(str,`${name} x${repeat}`);
+}
+checkTruncation(twoByte(base.repeat(3)),`${name} truncation`);
+}
+
+// Lone surrogates in various positions and sizes -> U+FFFD, rest intact.
+consthigh='\ud83d';
+constlow='\ude00';
+constsurrogateCases={
+'lone high': `ab${high}cd`,
+'lone low': `ab${low}cd`,
+'reversed pair': `ab${low}${high}cd`,
+'high at end': `abcd${high}`,
+'low at start': `${low}abcd`,
+'high high low': `${high}${high}${low}x`,
+'pair then lone': `${high}${low}${high}`,
+'only lone': high,
+};
+for(const[name,base]ofObject.entries(surrogateCases)){
+for(constpadof['','x'.repeat(50),'é'.repeat(300),'日'.repeat(30000)]){
+conststr=pad+base+pad;
+checkFull(str,`${name} pad=${pad.length}`);
+}
+checkTruncation(base+'zz',`${name} truncation`);
+}
+
+// Buffer.from of a large two-byte string equals TextEncoder output.
+{
+conststr=twoByte(samples.mixed.repeat(50000));
+assert.deepStrictEqual(Buffer.from(str),Buffer.from(newTextEncoder().encode(str)));
+}