Fastbase64#10

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
lemire wants to merge 11 commits into from
Closed

Fastbase64 #10

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
7d37d8c
fix description of simdutf
lemire Apr 1, 2024
c157e23
deps: update simdutf to 5.1.0
Apr 1, 2024
d3cc29b
fix: use simdutf for base64 encoding
Apr 1, 2024
0571b0a
updating simdutf
Apr 1, 2024
e00ea55
implemented decoding using accelerated functions
Apr 2, 2024
be45574
implemented fast atob
Apr 2, 2024
ab17447
various fixes
Apr 2, 2024
f69f215
deps: update simdutf to 5.2.1
Apr 2, 2024
cef15cd
deps: update simdutf to 5.2.2
Apr 2, 2024
4c12582
simplify Base64ToBinary
Apr 2, 2024
4391837
identifying performance bottleneck
Apr 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5,754 changes: 5,505 additions & 249 deletions deps/simdutf/simdutf.cpp
View file
Open in desktop

Large diffs are not rendered by default.

316 changes: 309 additions & 7 deletions deps/simdutf/simdutf.h
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion doc/contributing/maintaining/maintaining-dependencies.md
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -295,7 +295,7 @@ a C++ library for fast JSON parsing.
### simdutf

The [simdutf](https://github.com/simdutf/simdutf) dependency is
a C++ library for fast UTF-8 decoding and encoding.
a C++ library for fast character and base64 decoding and encoding.

### undici

Expand Down
83 changes: 15 additions & 68 deletions lib/buffer.js
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -23,10 +23,8 @@

const {
Array,
ArrayFrom,
ArrayIsArray,
ArrayPrototypeForEach,
ArrayPrototypeIndexOf,
MathFloor,
MathMin,
MathTrunc,
Expand DownExpand Up@@ -1263,28 +1261,6 @@ function btoa(input) {
return buf.toString('base64');
}

// Refs: https://infra.spec.whatwg.org/#forgiving-base64-decode
const kForgivingBase64AllowedChars = [
// ASCII whitespace
// Refs: https://infra.spec.whatwg.org/#ascii-whitespace
0x09, 0x0A, 0x0C, 0x0D, 0x20,

// Uppercase letters
...ArrayFrom({ length: 26 }, (_, i) => StringPrototypeCharCodeAt('A') + i),

// Lowercase letters
...ArrayFrom({ length: 26 }, (_, i) => StringPrototypeCharCodeAt('a') + i),

// Decimal digits
...ArrayFrom({ length: 10 }, (_, i) => StringPrototypeCharCodeAt('0') + i),

0x2B, // +
0x2F, // /
0x3D, // =
];
const kEqualSignIndex = ArrayPrototypeIndexOf(kForgivingBase64AllowedChars,
0x3D);

function atob(input) {
// The implementation here has not been performance optimized in any way and
// should not be.
Expand All@@ -1294,54 +1270,25 @@ function atob(input) {
}

input = `${input}`;
let nonAsciiWhitespaceCharCount = 0;
let equalCharCount = 0;

for (let n = 0; n < input.length; n++) {
const index = ArrayPrototypeIndexOf(
kForgivingBase64AllowedChars,
StringPrototypeCharCodeAt(input, n));

if (index > 4) {
// The first 5 elements of `kForgivingBase64AllowedChars` are
// ASCII whitespace char codes.
nonAsciiWhitespaceCharCount++;

if (index === kEqualSignIndex) {
equalCharCount++;
} else if (equalCharCount) {
// The `=` char is only allowed at the end.
throw lazyDOMException('Invalid character', 'InvalidCharacterError');
}

if (equalCharCount > 2) {
// Only one more `=` is permitted after the first equal sign.
throw lazyDOMException('Invalid character', 'InvalidCharacterError');
}
} else if (index === -1) {
const len = base64ByteLength(input, input.length);
const b = Buffer.allocUnsafe(len);
// returns 0 on error
const binary_length = b.base64ToBinary(input, 0, input.length);
if (binary_length < 0) {
if (binary_length === -1)
throw lazyDOMException(
'The string to be decoded is not correctly encoded.',
'InvalidCharacterError');
else if (binary_length === -2)
throw lazyDOMException('Invalid character', 'InvalidCharacterError');
}
}

let reminder = nonAsciiWhitespaceCharCount % 4;

// See #2, #3, #4 - https://infra.spec.whatwg.org/#forgiving-base64
if (!reminder) {
// Remove all trailing `=` characters and get the new reminder.
reminder = (nonAsciiWhitespaceCharCount - equalCharCount) % 4;
} else if (equalCharCount) {
// `=` should not in the input if there's a reminder.
throw lazyDOMException('Invalid character', 'InvalidCharacterError');
}

// See #3 - https://infra.spec.whatwg.org/#forgiving-base64
if (reminder === 1) {
throw lazyDOMException(
'The string to be decoded is not correctly encoded.',
'InvalidCharacterError');
else
throw lazyDOMException(
'Potential buffer overflow. The binary output is too large.',
'InvalidCharacterError');
}

return Buffer.from(input, 'base64').toString('latin1');
return b.toString('latin1', 0, binary_length);
}

function isUtf8(input) {
Expand Down
2 changes: 2 additions & 0 deletions lib/internal/buffer.js
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@ const {
utf8Slice,
asciiWrite,
base64Write,
base64ToBinary,
base64urlWrite,
latin1Write,
hexWrite,
Expand DownExpand Up@@ -1038,6 +1039,7 @@ function addBufferPrototypeMethods(proto) {
proto.utf8Slice = utf8Slice;
proto.asciiWrite = asciiWrite;
proto.base64Write = base64Write;
proto.base64ToBinary = base64ToBinary;
proto.base64urlWrite = base64urlWrite;
proto.latin1Write = latin1Write;
proto.hexWrite = hexWrite;
Expand Down
68 changes: 68 additions & 0 deletions src/node_buffer.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -699,6 +699,72 @@ void Fill(const FunctionCallbackInfo<Value>& args) {
}
}

// on success: returns a non-negative integer indicating the size of the
// binary produced, it most be no larger than 2147483647 bytes.
// In case of error, a negativ value is returned:
// * -2 indicates an invalid character,
// * -1 indicates a single character remained,
// * -3 indicates a possible overflow (i.e., more than 2 GB output).
void Base64ToBinary(const FunctionCallbackInfo<Value>& args) {
Environment* env = Environment::GetCurrent(args);

THROW_AND_RETURN_UNLESS_BUFFER(env, args.This());
SPREAD_BUFFER_ARG(args.This(), ts_obj);

THROW_AND_RETURN_IF_NOT_STRING(env, args[0], "argument");

Local<String> str = args[0]->ToString(env->context()).ToLocalChecked();

size_t offset = 0;
size_t max_length = 0;

THROW_AND_RETURN_IF_OOB(ParseArrayIndex(env, args[1], 0, &offset));
if (offset > ts_obj_length) {
return node::THROW_ERR_BUFFER_OUT_OF_BOUNDS(
env, "\"offset\" is outside of buffer bounds");
}

THROW_AND_RETURN_IF_OOB(ParseArrayIndex(env, args[2], ts_obj_length - offset,
&max_length));
if (max_length == 0)
return args.GetReturnValue().Set(0);

char* buf = ts_obj_data + offset;
size_t buflen = max_length;
if(buflen > INT32_MAX) {
return args.GetReturnValue().Set(-3);
}

int32_t written{0};

if (str->IsExternalOneByte()) { // 8-bit case
auto ext = str->GetExternalOneByteStringResource();
simdutf::result r = simdutf::base64_to_binary_safe(ext->data(), ext->length(), buf, buflen, simdutf::base64_default);
if(r.error == simdutf::error_code::SUCCESS) {
written = buflen;
} else if(r.error == simdutf::error_code::INVALID_BASE64_CHARACTER) {
written = -2;
} else if(r.error == simdutf::error_code::BASE64_INPUT_REMAINDER) {
written = -1;
} else {
written = -3;
}
} else { // 16-bit case
String::Value value(env->isolate(), str);
simdutf::result r = simdutf::base64_to_binary_safe(reinterpret_cast<const char16_t*>(*value), value.length(), buf, buflen, simdutf::base64_default);
if(r.error == simdutf::error_code::SUCCESS) {
written = buflen;
} else if(r.error == simdutf::error_code::INVALID_BASE64_CHARACTER) {
written = -2;
} else if(r.error == simdutf::error_code::BASE64_INPUT_REMAINDER) {
written = -1;
} else {
written = -3;
}
}

args.GetReturnValue().Set(written);
}

template <encoding encoding>
void StringWrite(const FunctionCallbackInfo<Value>& args) {
Expand DownExpand Up@@ -1316,6 +1382,7 @@ void Initialize(Local<Object> target,

SetMethod(context, target, "asciiWrite", StringWrite<ASCII>);
SetMethod(context, target, "base64Write", StringWrite<BASE64>);
SetMethod(context, target, "base64ToBinary", Base64ToBinary);
SetMethod(context, target, "base64urlWrite", StringWrite<BASE64URL>);
SetMethod(context, target, "latin1Write", StringWrite<LATIN1>);
SetMethod(context, target, "hexWrite", StringWrite<HEX>);
Expand DownExpand Up@@ -1358,6 +1425,7 @@ void RegisterExternalReferences(ExternalReferenceRegistry* registry) {
registry->Register(StringSlice<UTF8>);

registry->Register(StringWrite<ASCII>);
registry->Register(Base64ToBinary);
registry->Register(StringWrite<BASE64>);
registry->Register(StringWrite<BASE64URL>);
registry->Register(StringWrite<LATIN1>);
Expand Down
70 changes: 60 additions & 10 deletions src/string_bytes.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -305,7 +305,7 @@ size_t StringBytes::Write(Isolate* isolate,
Local<Value> val,
enum encoding encoding) {
HandleScope scope(isolate);
size_t nbytes;
size_t nbytes{0};

CHECK(val->IsString() == true);
Local<String> str = val.As<String>();
Expand DownExpand Up@@ -346,14 +346,62 @@ size_t StringBytes::Write(Isolate* isolate,
}

case BASE64URL:
// Fall through
if (str->IsExternalOneByte()) { // 8-bit case
auto ext = str->GetExternalOneByteStringResource();
// Try with WHATWG base64 standard first, adapted for base64url
simdutf::result r = simdutf::base64_to_binary_safe(ext->data(), ext->length(), buf, buflen, simdutf::base64_url);
if(r.error == simdutf::error_code::SUCCESS) {
nbytes = buflen;
} else {
// The input does not follow the WHATWG forgiving-base64 specification adapted for base64url
// https://infra.spec.whatwg.org/#forgiving-base64-decode
nbytes = base64_decode(buf, buflen, ext->data(), ext->length());
}
} else { // 16-bit case
// Typically, a base64url string is stored as an 8-bit string within v8.
// Thus str->IsOneByte() is typically true. The next line thus often allocates
// a temporary 16-bit buffer to store a 16-bit copy of the 8-bit v8 string.
// Hence the creation of the String::Value value is likely a performance bottleneck.
String::Value value(isolate, str);
// Try with WHATWG base64 standard first
simdutf::result r = simdutf::base64_to_binary_safe(reinterpret_cast<const char16_t*>(*value), value.length(), buf, buflen, simdutf::base64_url);
if(r.error == simdutf::error_code::SUCCESS) {
nbytes = buflen;
} else {
// The input does not follow the WHATWG forgiving-base64 specification
// (adapted for base64url with + and / replaced by - and _).
// https://infra.spec.whatwg.org/#forgiving-base64-decode
nbytes = base64_decode(buf, buflen, *value, value.length());
}
}
break;
case BASE64:
if (str->IsExternalOneByte()) {
if (str->IsExternalOneByte()) { // 8-bit case
auto ext = str->GetExternalOneByteStringResource();
nbytes = base64_decode(buf, buflen, ext->data(), ext->length());
} else {
// Try with WHATWG base64 standard first
simdutf::result r = simdutf::base64_to_binary_safe(ext->data(), ext->length(), buf, buflen, simdutf::base64_default);
if(r.error == simdutf::error_code::SUCCESS) {
nbytes = buflen;
} else {
// The input does not follow the WHATWG forgiving-base64 specification
// https://infra.spec.whatwg.org/#forgiving-base64-decode
nbytes = base64_decode(buf, buflen, ext->data(), ext->length());
}
} else { // 16-bit case
// Typically, a base64 string is stored as an 8-bit string within v8.
// Thus str->IsOneByte() is typically true. The next line thus often allocates
// a temporary 16-bit buffer to store a 16-bit copy of the 8-bit v8 string.
// Hence the creation of the String::Value value is likely a performance bottleneck.
String::Value value(isolate, str);
nbytes = base64_decode(buf, buflen, *value, value.length());
// Try with WHATWG base64 standard first
simdutf::result r = simdutf::base64_to_binary_safe(reinterpret_cast<const char16_t*>(*value), value.length(), buf, buflen, simdutf::base64_default);
if(r.error == simdutf::error_code::SUCCESS) {
nbytes = buflen;
} else {
// The input does not follow the WHATWG base64 specification
// https://infra.spec.whatwg.org/#forgiving-base64-decode
nbytes = base64_decode(buf, buflen, *value, value.length());
}
}
break;

Expand DownExpand Up@@ -615,22 +663,24 @@ MaybeLocal<Value> StringBytes::Encode(Isolate* isolate,
*error = node::ERR_MEMORY_ALLOCATION_FAILED(isolate);
return MaybeLocal<Value>();
}

// by default, base64 uses libbase64's accelerated routine so
// it is not necessary to use simdutf.
size_t written = base64_encode(buf, buflen, dst, dlen);
CHECK_EQ(written, dlen);

return ExternOneByteString::New(isolate, dst, dlen, error);
}

case BASE64URL: {
size_t dlen = base64_encoded_size(buflen, Base64Mode::URL);
// When in URL mode, base64_encode uses a non-accelerated routine, so we
// adopt simdutf.
size_t dlen = simdutf::base64_length_from_binary(buflen, simdutf::base64_url);
char* dst = node::UncheckedMalloc(dlen);
if (dst == nullptr) {
*error = node::ERR_MEMORY_ALLOCATION_FAILED(isolate);
return MaybeLocal<Value>();
}

size_t written = base64_encode(buf, buflen, dst, dlen, Base64Mode::URL);
size_t written = simdutf::binary_to_base64(buf, buflen, dst, simdutf::base64_url);
CHECK_EQ(written, dlen);

return ExternOneByteString::New(isolate, dst, dlen, error);
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
Fastbase64#10

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

lemire
wants to merge
11
commits into

from

 Closed


Fastbase64
#10
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


11 commits


Select commit
Hold shift + click to select a range
7d37d8c

fix description of simdutf


lemire Apr 1, 2024
c157e23

deps: update simdutf to 5.1.0

Apr 1, 2024
d3cc29b

fix: use simdutf for base64 encoding

Apr 1, 2024
0571b0a

updating simdutf

Apr 1, 2024
e00ea55

implemented decoding using accelerated functions

Apr 2, 2024
be45574

implemented fast atob

Apr 2, 2024
ab17447

various fixes

Apr 2, 2024
f69f215

deps: update simdutf to 5.2.1

Apr 2, 2024
cef15cd

deps: update simdutf to 5.2.2

Apr 2, 2024
4c12582

simplify Base64ToBinary

Apr 2, 2024
4391837

identifying performance bottleneck

Apr 3, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


5,754 changes: 5,505 additions & 249 deletions
deps/simdutf/simdutf.cpp

View file


Open in desktop


Large diffs are not rendered by default.


316 changes: 309 additions & 7 deletions
deps/simdutf/simdutf.h

View file


Open in desktop


Large diffs are not rendered by default.


2 changes: 1 addition & 1 deletion
doc/contributing/maintaining/maintaining-dependencies.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -295,7 +295,7 @@ a C++ library for fast JSON parsing.
### simdutf

The [simdutf](https://github.com/simdutf/simdutf) dependency is
a C++ library for fast UTF-8 decoding and encoding.
a C++ library for fast character and base64 decoding and encoding.

### undici

Expand Down

83 changes: 15 additions & 68 deletions
lib/buffer.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,10 +23,8 @@

const {
 Array,
 ArrayFrom,
 ArrayIsArray,
 ArrayPrototypeForEach,
 ArrayPrototypeIndexOf,
 MathFloor,
 MathMin,
 MathTrunc,
Expand DownExpand Up@@ -1263,28 +1261,6 @@ function btoa(input) {
 return buf.toString('base64');
}

// Refs: https://infra.spec.whatwg.org/#forgiving-base64-decode
const kForgivingBase64AllowedChars = [
 // ASCII whitespace
 // Refs: https://infra.spec.whatwg.org/#ascii-whitespace
 0x09, 0x0A, 0x0C, 0x0D, 0x20,

 // Uppercase letters
 ...ArrayFrom({ length: 26 }, (_, i) => StringPrototypeCharCodeAt('A') + i),

 // Lowercase letters
 ...ArrayFrom({ length: 26 }, (_, i) => StringPrototypeCharCodeAt('a') + i),

 // Decimal digits
 ...ArrayFrom({ length: 10 }, (_, i) => StringPrototypeCharCodeAt('0') + i),

 0x2B, // +
 0x2F, // /
 0x3D, // =
];
const kEqualSignIndex = ArrayPrototypeIndexOf(kForgivingBase64AllowedChars,
 0x3D);

function atob(input) {
 // The implementation here has not been performance optimized in any way and
 // should not be.
Expand All@@ -1294,54 +1270,25 @@ function atob(input) {
 }

 input = `${input}`;
 let nonAsciiWhitespaceCharCount = 0;
 let equalCharCount = 0;

 for (let n = 0; n < input.length; n++) {
 const index = ArrayPrototypeIndexOf(
 kForgivingBase64AllowedChars,
 StringPrototypeCharCodeAt(input, n));

 if (index > 4) {
 // The first 5 elements of `kForgivingBase64AllowedChars` are
 // ASCII whitespace char codes.
 nonAsciiWhitespaceCharCount++;

 if (index === kEqualSignIndex) {
 equalCharCount++;
 } else if (equalCharCount) {
 // The `=` char is only allowed at the end.
 throw lazyDOMException('Invalid character', 'InvalidCharacterError');
 }

 if (equalCharCount > 2) {
 // Only one more `=` is permitted after the first equal sign.
 throw lazyDOMException('Invalid character', 'InvalidCharacterError');
 }
 } else if (index === -1) {
 const len = base64ByteLength(input, input.length);
 const b = Buffer.allocUnsafe(len);
 // returns 0 on error
 const binary_length = b.base64ToBinary(input, 0, input.length);
 if (binary_length < 0) {
 if (binary_length === -1)
 throw lazyDOMException(
 'The string to be decoded is not correctly encoded.',
 'InvalidCharacterError');
 else if (binary_length === -2)
 throw lazyDOMException('Invalid character', 'InvalidCharacterError');
 }
 }

 let reminder = nonAsciiWhitespaceCharCount % 4;

 // See #2, #3, #4 - https://infra.spec.whatwg.org/#forgiving-base64
 if (!reminder) {
 // Remove all trailing `=` characters and get the new reminder.
 reminder = (nonAsciiWhitespaceCharCount - equalCharCount) % 4;
 } else if (equalCharCount) {
 // `=` should not in the input if there's a reminder.
 throw lazyDOMException('Invalid character', 'InvalidCharacterError');
 }

 // See #3 - https://infra.spec.whatwg.org/#forgiving-base64
 if (reminder === 1) {
 throw lazyDOMException(
 'The string to be decoded is not correctly encoded.',
 'InvalidCharacterError');
 else
 throw lazyDOMException(
 'Potential buffer overflow. The binary output is too large.',
 'InvalidCharacterError');
 }

 return Buffer.from(input, 'base64').toString('latin1');
 return b.toString('latin1', 0, binary_length);
}

function isUtf8(input) {
Expand Down

2 changes: 2 additions & 0 deletions
lib/internal/buffer.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@ const {
 utf8Slice,
 asciiWrite,
 base64Write,
 base64ToBinary,
 base64urlWrite,
 latin1Write,
 hexWrite,
Expand DownExpand Up@@ -1038,6 +1039,7 @@ function addBufferPrototypeMethods(proto) {
 proto.utf8Slice = utf8Slice;
 proto.asciiWrite = asciiWrite;
 proto.base64Write = base64Write;
 proto.base64ToBinary = base64ToBinary;
 proto.base64urlWrite = base64urlWrite;
 proto.latin1Write = latin1Write;
 proto.hexWrite = hexWrite;
Expand Down

68 changes: 68 additions & 0 deletions
src/node_buffer.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -699,6 +699,72 @@ void Fill(const FunctionCallbackInfo<Value>& args) {
 }
}

// on success: returns a non-negative integer indicating the size of the
// binary produced, it most be no larger than 2147483647 bytes.
// In case of error, a negativ value is returned:
// * -2 indicates an invalid character,
// * -1 indicates a single character remained,
// * -3 indicates a possible overflow (i.e., more than 2 GB output).
void Base64ToBinary(const FunctionCallbackInfo<Value>& args) {
 Environment* env = Environment::GetCurrent(args);

 THROW_AND_RETURN_UNLESS_BUFFER(env, args.This());
 SPREAD_BUFFER_ARG(args.This(), ts_obj);

 THROW_AND_RETURN_IF_NOT_STRING(env, args[0], "argument");

 Local<String> str = args[0]->ToString(env->context()).ToLocalChecked();

 size_t offset = 0;
 size_t max_length = 0;

 THROW_AND_RETURN_IF_OOB(ParseArrayIndex(env, args[1], 0, &offset));
 if (offset > ts_obj_length) {
 return node::THROW_ERR_BUFFER_OUT_OF_BOUNDS(
 env, "\"offset\" is outside of buffer bounds");
 }

 THROW_AND_RETURN_IF_OOB(ParseArrayIndex(env, args[2], ts_obj_length - offset,
 &max_length));
 if (max_length == 0)
 return args.GetReturnValue().Set(0);

 char* buf = ts_obj_data + offset;
 size_t buflen = max_length;
 if(buflen > INT32_MAX) {
 return args.GetReturnValue().Set(-3);
 }

 int32_t written{0};

 if (str->IsExternalOneByte()) { // 8-bit case
 auto ext = str->GetExternalOneByteStringResource();
 simdutf::result r = simdutf::base64_to_binary_safe(ext->data(), ext->length(), buf, buflen, simdutf::base64_default);
 if(r.error == simdutf::error_code::SUCCESS) {
 written = buflen;
 } else if(r.error == simdutf::error_code::INVALID_BASE64_CHARACTER) {
 written = -2;
 } else if(r.error == simdutf::error_code::BASE64_INPUT_REMAINDER) {
 written = -1;
 } else {
 written = -3;
 }
 } else { // 16-bit case
 String::Value value(env->isolate(), str);
 simdutf::result r = simdutf::base64_to_binary_safe(reinterpret_cast<const char16_t*>(*value), value.length(), buf, buflen, simdutf::base64_default);
 if(r.error == simdutf::error_code::SUCCESS) {
 written = buflen;
 } else if(r.error == simdutf::error_code::INVALID_BASE64_CHARACTER) {
 written = -2;
 } else if(r.error == simdutf::error_code::BASE64_INPUT_REMAINDER) {
 written = -1;
 } else {
 written = -3;
 }
 }

 args.GetReturnValue().Set(written);
}

template <encoding encoding>
void StringWrite(const FunctionCallbackInfo<Value>& args) {
Expand DownExpand Up@@ -1316,6 +1382,7 @@ void Initialize(Local<Object> target,

 SetMethod(context, target, "asciiWrite", StringWrite<ASCII>);
 SetMethod(context, target, "base64Write", StringWrite<BASE64>);
 SetMethod(context, target, "base64ToBinary", Base64ToBinary);
 SetMethod(context, target, "base64urlWrite", StringWrite<BASE64URL>);
 SetMethod(context, target, "latin1Write", StringWrite<LATIN1>);
 SetMethod(context, target, "hexWrite", StringWrite<HEX>);
Expand DownExpand Up@@ -1358,6 +1425,7 @@ void RegisterExternalReferences(ExternalReferenceRegistry* registry) {
 registry->Register(StringSlice<UTF8>);

 registry->Register(StringWrite<ASCII>);
 registry->Register(Base64ToBinary);
 registry->Register(StringWrite<BASE64>);
 registry->Register(StringWrite<BASE64URL>);
 registry->Register(StringWrite<LATIN1>);
Expand Down

70 changes: 60 additions & 10 deletions
src/string_bytes.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -305,7 +305,7 @@ size_t StringBytes::Write(Isolate* isolate,
 Local<Value> val,
 enum encoding encoding) {
 HandleScope scope(isolate);
 size_t nbytes;
 size_t nbytes{0};

 CHECK(val->IsString() == true);
 Local<String> str = val.As<String>();
Expand DownExpand Up@@ -346,14 +346,62 @@ size_t StringBytes::Write(Isolate* isolate,
 }

 case BASE64URL:
 // Fall through
 if (str->IsExternalOneByte()) { // 8-bit case
 auto ext = str->GetExternalOneByteStringResource();
 // Try with WHATWG base64 standard first, adapted for base64url
 simdutf::result r = simdutf::base64_to_binary_safe(ext->data(), ext->length(), buf, buflen, simdutf::base64_url);
 if(r.error == simdutf::error_code::SUCCESS) {
 nbytes = buflen;
 } else {
 // The input does not follow the WHATWG forgiving-base64 specification adapted for base64url
 // https://infra.spec.whatwg.org/#forgiving-base64-decode
 nbytes = base64_decode(buf, buflen, ext->data(), ext->length());
 }
 } else { // 16-bit case
 // Typically, a base64url string is stored as an 8-bit string within v8.
 // Thus str->IsOneByte() is typically true. The next line thus often allocates
 // a temporary 16-bit buffer to store a 16-bit copy of the 8-bit v8 string.
 // Hence the creation of the String::Value value is likely a performance bottleneck.
 String::Value value(isolate, str);
 // Try with WHATWG base64 standard first
 simdutf::result r = simdutf::base64_to_binary_safe(reinterpret_cast<const char16_t*>(*value), value.length(), buf, buflen, simdutf::base64_url);
 if(r.error == simdutf::error_code::SUCCESS) {
 nbytes = buflen;
 } else {
 // The input does not follow the WHATWG forgiving-base64 specification
 // (adapted for base64url with + and / replaced by - and _).
 // https://infra.spec.whatwg.org/#forgiving-base64-decode
 nbytes = base64_decode(buf, buflen, *value, value.length());
 }
 }
 break;
 case BASE64:
 if (str->IsExternalOneByte()) {
 if (str->IsExternalOneByte()) { // 8-bit case
 auto ext = str->GetExternalOneByteStringResource();
 nbytes = base64_decode(buf, buflen, ext->data(), ext->length());
 } else {
 // Try with WHATWG base64 standard first
 simdutf::result r = simdutf::base64_to_binary_safe(ext->data(), ext->length(), buf, buflen, simdutf::base64_default);
 if(r.error == simdutf::error_code::SUCCESS) {
 nbytes = buflen;
 } else {
 // The input does not follow the WHATWG forgiving-base64 specification
 // https://infra.spec.whatwg.org/#forgiving-base64-decode
 nbytes = base64_decode(buf, buflen, ext->data(), ext->length());
 }
 } else { // 16-bit case
 // Typically, a base64 string is stored as an 8-bit string within v8.
 // Thus str->IsOneByte() is typically true. The next line thus often allocates
 // a temporary 16-bit buffer to store a 16-bit copy of the 8-bit v8 string.
 // Hence the creation of the String::Value value is likely a performance bottleneck.
 String::Value value(isolate, str);
 nbytes = base64_decode(buf, buflen, *value, value.length());
 // Try with WHATWG base64 standard first
 simdutf::result r = simdutf::base64_to_binary_safe(reinterpret_cast<const char16_t*>(*value), value.length(), buf, buflen, simdutf::base64_default);
 if(r.error == simdutf::error_code::SUCCESS) {
 nbytes = buflen;
 } else {
 // The input does not follow the WHATWG base64 specification
 // https://infra.spec.whatwg.org/#forgiving-base64-decode
 nbytes = base64_decode(buf, buflen, *value, value.length());
 }
 }
 break;

Expand DownExpand Up@@ -615,22 +663,24 @@ MaybeLocal<Value> StringBytes::Encode(Isolate* isolate,
 *error = node::ERR_MEMORY_ALLOCATION_FAILED(isolate);
 return MaybeLocal<Value>();
 }

 // by default, base64 uses libbase64's accelerated routine so
 // it is not necessary to use simdutf.
 size_t written = base64_encode(buf, buflen, dst, dlen);
 CHECK_EQ(written, dlen);

 return ExternOneByteString::New(isolate, dst, dlen, error);
 }

 case BASE64URL: {
 size_t dlen = base64_encoded_size(buflen, Base64Mode::URL);
 // When in URL mode, base64_encode uses a non-accelerated routine, so we
 // adopt simdutf.
 size_t dlen = simdutf::base64_length_from_binary(buflen, simdutf::base64_url);
 char* dst = node::UncheckedMalloc(dlen);
 if (dst == nullptr) {
 *error = node::ERR_MEMORY_ALLOCATION_FAILED(isolate);
 return MaybeLocal<Value>();
 }

 size_t written = base64_encode(buf, buflen, dst, dlen, Base64Mode::URL);
 size_t written = simdutf::binary_to_base64(buf, buflen, dst, simdutf::base64_url);
 CHECK_EQ(written, dlen);

 return ExternOneByteString::New(isolate, dst, dlen, error);
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Fastbase64#10

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

lemire
wants to merge
11
commits into

from

 Closed


Fastbase64
#10
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


11 commits


Select commit
Hold shift + click to select a range
7d37d8c

fix description of simdutf


lemire Apr 1, 2024
c157e23

deps: update simdutf to 5.1.0

Apr 1, 2024
d3cc29b

fix: use simdutf for base64 encoding

Apr 1, 2024
0571b0a

updating simdutf

Apr 1, 2024
e00ea55

implemented decoding using accelerated functions

Apr 2, 2024
be45574

implemented fast atob

Apr 2, 2024
ab17447

various fixes

Apr 2, 2024
f69f215

deps: update simdutf to 5.2.1

Apr 2, 2024
cef15cd

deps: update simdutf to 5.2.2

Apr 2, 2024
4c12582

simplify Base64ToBinary

Apr 2, 2024
4391837

identifying performance bottleneck

Apr 3, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


5,754 changes: 5,505 additions & 249 deletions
deps/simdutf/simdutf.cpp

View file


Open in desktop


Large diffs are not rendered by default.


316 changes: 309 additions & 7 deletions
deps/simdutf/simdutf.h

View file


Open in desktop


Large diffs are not rendered by default.


2 changes: 1 addition & 1 deletion
doc/contributing/maintaining/maintaining-dependencies.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -295,7 +295,7 @@ a C++ library for fast JSON parsing.
### simdutf

The [simdutf](https://github.com/simdutf/simdutf) dependency is
a C++ library for fast UTF-8 decoding and encoding.
a C++ library for fast character and base64 decoding and encoding.

### undici

Expand Down

83 changes: 15 additions & 68 deletions
lib/buffer.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,10 +23,8 @@

const {
 Array,
 ArrayFrom,
 ArrayIsArray,
 ArrayPrototypeForEach,
 ArrayPrototypeIndexOf,
 MathFloor,
 MathMin,
 MathTrunc,
Expand DownExpand Up@@ -1263,28 +1261,6 @@ function btoa(input) {
 return buf.toString('base64');
}

// Refs: https://infra.spec.whatwg.org/#forgiving-base64-decode
const kForgivingBase64AllowedChars = [
 // ASCII whitespace
 // Refs: https://infra.spec.whatwg.org/#ascii-whitespace
 0x09, 0x0A, 0x0C, 0x0D, 0x20,

 // Uppercase letters
 ...ArrayFrom({ length: 26 }, (_, i) => StringPrototypeCharCodeAt('A') + i),

 // Lowercase letters
 ...ArrayFrom({ length: 26 }, (_, i) => StringPrototypeCharCodeAt('a') + i),

 // Decimal digits
 ...ArrayFrom({ length: 10 }, (_, i) => StringPrototypeCharCodeAt('0') + i),

 0x2B, // +
 0x2F, // /
 0x3D, // =
];
const kEqualSignIndex = ArrayPrototypeIndexOf(kForgivingBase64AllowedChars,
 0x3D);

function atob(input) {
 // The implementation here has not been performance optimized in any way and
 // should not be.
Expand All@@ -1294,54 +1270,25 @@ function atob(input) {
 }

 input = `${input}`;
 let nonAsciiWhitespaceCharCount = 0;
 let equalCharCount = 0;

 for (let n = 0; n < input.length; n++) {
 const index = ArrayPrototypeIndexOf(
 kForgivingBase64AllowedChars,
 StringPrototypeCharCodeAt(input, n));

 if (index > 4) {
 // The first 5 elements of `kForgivingBase64AllowedChars` are
 // ASCII whitespace char codes.
 nonAsciiWhitespaceCharCount++;

 if (index === kEqualSignIndex) {
 equalCharCount++;
 } else if (equalCharCount) {
 // The `=` char is only allowed at the end.
 throw lazyDOMException('Invalid character', 'InvalidCharacterError');
 }

 if (equalCharCount > 2) {
 // Only one more `=` is permitted after the first equal sign.
 throw lazyDOMException('Invalid character', 'InvalidCharacterError');
 }
 } else if (index === -1) {
 const len = base64ByteLength(input, input.length);
 const b = Buffer.allocUnsafe(len);
 // returns 0 on error
 const binary_length = b.base64ToBinary(input, 0, input.length);
 if (binary_length < 0) {
 if (binary_length === -1)
 throw lazyDOMException(
 'The string to be decoded is not correctly encoded.',
 'InvalidCharacterError');
 else if (binary_length === -2)
 throw lazyDOMException('Invalid character', 'InvalidCharacterError');
 }
 }

 let reminder = nonAsciiWhitespaceCharCount % 4;

 // See #2, #3, #4 - https://infra.spec.whatwg.org/#forgiving-base64
 if (!reminder) {
 // Remove all trailing `=` characters and get the new reminder.
 reminder = (nonAsciiWhitespaceCharCount - equalCharCount) % 4;
 } else if (equalCharCount) {
 // `=` should not in the input if there's a reminder.
 throw lazyDOMException('Invalid character', 'InvalidCharacterError');
 }

 // See #3 - https://infra.spec.whatwg.org/#forgiving-base64
 if (reminder === 1) {
 throw lazyDOMException(
 'The string to be decoded is not correctly encoded.',
 'InvalidCharacterError');
 else
 throw lazyDOMException(
 'Potential buffer overflow. The binary output is too large.',
 'InvalidCharacterError');
 }

 return Buffer.from(input, 'base64').toString('latin1');
 return b.toString('latin1', 0, binary_length);
}

function isUtf8(input) {
Expand Down

2 changes: 2 additions & 0 deletions
lib/internal/buffer.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@ const {
 utf8Slice,
 asciiWrite,
 base64Write,
 base64ToBinary,
 base64urlWrite,
 latin1Write,
 hexWrite,
Expand DownExpand Up@@ -1038,6 +1039,7 @@ function addBufferPrototypeMethods(proto) {
 proto.utf8Slice = utf8Slice;
 proto.asciiWrite = asciiWrite;
 proto.base64Write = base64Write;
 proto.base64ToBinary = base64ToBinary;
 proto.base64urlWrite = base64urlWrite;
 proto.latin1Write = latin1Write;
 proto.hexWrite = hexWrite;
Expand Down

68 changes: 68 additions & 0 deletions
src/node_buffer.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -699,6 +699,72 @@ void Fill(const FunctionCallbackInfo<Value>& args) {
 }
}

// on success: returns a non-negative integer indicating the size of the
// binary produced, it most be no larger than 2147483647 bytes.
// In case of error, a negativ value is returned:
// * -2 indicates an invalid character,
// * -1 indicates a single character remained,
// * -3 indicates a possible overflow (i.e., more than 2 GB output).
void Base64ToBinary(const FunctionCallbackInfo<Value>& args) {
 Environment* env = Environment::GetCurrent(args);

 THROW_AND_RETURN_UNLESS_BUFFER(env, args.This());
 SPREAD_BUFFER_ARG(args.This(), ts_obj);

 THROW_AND_RETURN_IF_NOT_STRING(env, args[0], "argument");

 Local<String> str = args[0]->ToString(env->context()).ToLocalChecked();

 size_t offset = 0;
 size_t max_length = 0;

 THROW_AND_RETURN_IF_OOB(ParseArrayIndex(env, args[1], 0, &offset));
 if (offset > ts_obj_length) {
 return node::THROW_ERR_BUFFER_OUT_OF_BOUNDS(
 env, "\"offset\" is outside of buffer bounds");
 }

 THROW_AND_RETURN_IF_OOB(ParseArrayIndex(env, args[2], ts_obj_length - offset,
 &max_length));
 if (max_length == 0)
 return args.GetReturnValue().Set(0);

 char* buf = ts_obj_data + offset;
 size_t buflen = max_length;
 if(buflen > INT32_MAX) {
 return args.GetReturnValue().Set(-3);
 }

 int32_t written{0};

 if (str->IsExternalOneByte()) { // 8-bit case
 auto ext = str->GetExternalOneByteStringResource();
 simdutf::result r = simdutf::base64_to_binary_safe(ext->data(), ext->length(), buf, buflen, simdutf::base64_default);
 if(r.error == simdutf::error_code::SUCCESS) {
 written = buflen;
 } else if(r.error == simdutf::error_code::INVALID_BASE64_CHARACTER) {
 written = -2;
 } else if(r.error == simdutf::error_code::BASE64_INPUT_REMAINDER) {
 written = -1;
 } else {
 written = -3;
 }
 } else { // 16-bit case
 String::Value value(env->isolate(), str);
 simdutf::result r = simdutf::base64_to_binary_safe(reinterpret_cast<const char16_t*>(*value), value.length(), buf, buflen, simdutf::base64_default);
 if(r.error == simdutf::error_code::SUCCESS) {
 written = buflen;
 } else if(r.error == simdutf::error_code::INVALID_BASE64_CHARACTER) {
 written = -2;
 } else if(r.error == simdutf::error_code::BASE64_INPUT_REMAINDER) {
 written = -1;
 } else {
 written = -3;
 }
 }

 args.GetReturnValue().Set(written);
}

template <encoding encoding>
void StringWrite(const FunctionCallbackInfo<Value>& args) {
Expand DownExpand Up@@ -1316,6 +1382,7 @@ void Initialize(Local<Object> target,

 SetMethod(context, target, "asciiWrite", StringWrite<ASCII>);
 SetMethod(context, target, "base64Write", StringWrite<BASE64>);
 SetMethod(context, target, "base64ToBinary", Base64ToBinary);
 SetMethod(context, target, "base64urlWrite", StringWrite<BASE64URL>);
 SetMethod(context, target, "latin1Write", StringWrite<LATIN1>);
 SetMethod(context, target, "hexWrite", StringWrite<HEX>);
Expand DownExpand Up@@ -1358,6 +1425,7 @@ void RegisterExternalReferences(ExternalReferenceRegistry* registry) {
 registry->Register(StringSlice<UTF8>);

 registry->Register(StringWrite<ASCII>);
 registry->Register(Base64ToBinary);
 registry->Register(StringWrite<BASE64>);
 registry->Register(StringWrite<BASE64URL>);
 registry->Register(StringWrite<LATIN1>);
Expand Down

70 changes: 60 additions & 10 deletions
src/string_bytes.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -305,7 +305,7 @@ size_t StringBytes::Write(Isolate* isolate,
 Local<Value> val,
 enum encoding encoding) {
 HandleScope scope(isolate);
 size_t nbytes;
 size_t nbytes{0};

 CHECK(val->IsString() == true);
 Local<String> str = val.As<String>();
Expand DownExpand Up@@ -346,14 +346,62 @@ size_t StringBytes::Write(Isolate* isolate,
 }

 case BASE64URL:
 // Fall through
 if (str->IsExternalOneByte()) { // 8-bit case
 auto ext = str->GetExternalOneByteStringResource();
 // Try with WHATWG base64 standard first, adapted for base64url
 simdutf::result r = simdutf::base64_to_binary_safe(ext->data(), ext->length(), buf, buflen, simdutf::base64_url);
 if(r.error == simdutf::error_code::SUCCESS) {
 nbytes = buflen;
 } else {
 // The input does not follow the WHATWG forgiving-base64 specification adapted for base64url
 // https://infra.spec.whatwg.org/#forgiving-base64-decode
 nbytes = base64_decode(buf, buflen, ext->data(), ext->length());
 }
 } else { // 16-bit case
 // Typically, a base64url string is stored as an 8-bit string within v8.
 // Thus str->IsOneByte() is typically true. The next line thus often allocates
 // a temporary 16-bit buffer to store a 16-bit copy of the 8-bit v8 string.
 // Hence the creation of the String::Value value is likely a performance bottleneck.
 String::Value value(isolate, str);
 // Try with WHATWG base64 standard first
 simdutf::result r = simdutf::base64_to_binary_safe(reinterpret_cast<const char16_t*>(*value), value.length(), buf, buflen, simdutf::base64_url);
 if(r.error == simdutf::error_code::SUCCESS) {
 nbytes = buflen;
 } else {
 // The input does not follow the WHATWG forgiving-base64 specification
 // (adapted for base64url with + and / replaced by - and _).
 // https://infra.spec.whatwg.org/#forgiving-base64-decode
 nbytes = base64_decode(buf, buflen, *value, value.length());
 }
 }
 break;
 case BASE64:
 if (str->IsExternalOneByte()) {
 if (str->IsExternalOneByte()) { // 8-bit case
 auto ext = str->GetExternalOneByteStringResource();
 nbytes = base64_decode(buf, buflen, ext->data(), ext->length());
 } else {
 // Try with WHATWG base64 standard first
 simdutf::result r = simdutf::base64_to_binary_safe(ext->data(), ext->length(), buf, buflen, simdutf::base64_default);
 if(r.error == simdutf::error_code::SUCCESS) {
 nbytes = buflen;
 } else {
 // The input does not follow the WHATWG forgiving-base64 specification
 // https://infra.spec.whatwg.org/#forgiving-base64-decode
 nbytes = base64_decode(buf, buflen, ext->data(), ext->length());
 }
 } else { // 16-bit case
 // Typically, a base64 string is stored as an 8-bit string within v8.
 // Thus str->IsOneByte() is typically true. The next line thus often allocates
 // a temporary 16-bit buffer to store a 16-bit copy of the 8-bit v8 string.
 // Hence the creation of the String::Value value is likely a performance bottleneck.
 String::Value value(isolate, str);
 nbytes = base64_decode(buf, buflen, *value, value.length());
 // Try with WHATWG base64 standard first
 simdutf::result r = simdutf::base64_to_binary_safe(reinterpret_cast<const char16_t*>(*value), value.length(), buf, buflen, simdutf::base64_default);
 if(r.error == simdutf::error_code::SUCCESS) {
 nbytes = buflen;
 } else {
 // The input does not follow the WHATWG base64 specification
 // https://infra.spec.whatwg.org/#forgiving-base64-decode
 nbytes = base64_decode(buf, buflen, *value, value.length());
 }
 }
 break;

Expand DownExpand Up@@ -615,22 +663,24 @@ MaybeLocal<Value> StringBytes::Encode(Isolate* isolate,
 *error = node::ERR_MEMORY_ALLOCATION_FAILED(isolate);
 return MaybeLocal<Value>();
 }

 // by default, base64 uses libbase64's accelerated routine so
 // it is not necessary to use simdutf.
 size_t written = base64_encode(buf, buflen, dst, dlen);
 CHECK_EQ(written, dlen);

 return ExternOneByteString::New(isolate, dst, dlen, error);
 }

 case BASE64URL: {
 size_t dlen = base64_encoded_size(buflen, Base64Mode::URL);
 // When in URL mode, base64_encode uses a non-accelerated routine, so we
 // adopt simdutf.
 size_t dlen = simdutf::base64_length_from_binary(buflen, simdutf::base64_url);
 char* dst = node::UncheckedMalloc(dlen);
 if (dst == nullptr) {
 *error = node::ERR_MEMORY_ALLOCATION_FAILED(isolate);
 return MaybeLocal<Value>();
 }

 size_t written = base64_encode(buf, buflen, dst, dlen, Base64Mode::URL);
 size_t written = simdutf::binary_to_base64(buf, buflen, dst, simdutf::base64_url);
 CHECK_EQ(written, dlen);

 return ExternOneByteString::New(isolate, dst, dlen, error);
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Fastbase64#10

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

lemire
wants to merge
11
commits into

from

 Closed


Fastbase64
#10
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


11 commits


Select commit
Hold shift + click to select a range
7d37d8c

fix description of simdutf


lemire Apr 1, 2024
c157e23

deps: update simdutf to 5.1.0

Apr 1, 2024
d3cc29b

fix: use simdutf for base64 encoding

Apr 1, 2024
0571b0a

updating simdutf

Apr 1, 2024
e00ea55

implemented decoding using accelerated functions

Apr 2, 2024
be45574

implemented fast atob

Apr 2, 2024
ab17447

various fixes

Apr 2, 2024
f69f215

deps: update simdutf to 5.2.1

Apr 2, 2024
cef15cd

deps: update simdutf to 5.2.2

Apr 2, 2024
4c12582

simplify Base64ToBinary

Apr 2, 2024
4391837

identifying performance bottleneck

Apr 3, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


5,754 changes: 5,505 additions & 249 deletions
deps/simdutf/simdutf.cpp

View file


Open in desktop


Large diffs are not rendered by default.


316 changes: 309 additions & 7 deletions
deps/simdutf/simdutf.h

View file


Open in desktop


Large diffs are not rendered by default.


2 changes: 1 addition & 1 deletion
doc/contributing/maintaining/maintaining-dependencies.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -295,7 +295,7 @@ a C++ library for fast JSON parsing.
### simdutf

The [simdutf](https://github.com/simdutf/simdutf) dependency is
a C++ library for fast UTF-8 decoding and encoding.
a C++ library for fast character and base64 decoding and encoding.

### undici

Expand Down

83 changes: 15 additions & 68 deletions
lib/buffer.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,10 +23,8 @@

const {
 Array,
 ArrayFrom,
 ArrayIsArray,
 ArrayPrototypeForEach,
 ArrayPrototypeIndexOf,
 MathFloor,
 MathMin,
 MathTrunc,
Expand DownExpand Up@@ -1263,28 +1261,6 @@ function btoa(input) {
 return buf.toString('base64');
}

// Refs: https://infra.spec.whatwg.org/#forgiving-base64-decode
const kForgivingBase64AllowedChars = [
 // ASCII whitespace
 // Refs: https://infra.spec.whatwg.org/#ascii-whitespace
 0x09, 0x0A, 0x0C, 0x0D, 0x20,

 // Uppercase letters
 ...ArrayFrom({ length: 26 }, (_, i) => StringPrototypeCharCodeAt('A') + i),

 // Lowercase letters
 ...ArrayFrom({ length: 26 }, (_, i) => StringPrototypeCharCodeAt('a') + i),

 // Decimal digits
 ...ArrayFrom({ length: 10 }, (_, i) => StringPrototypeCharCodeAt('0') + i),

 0x2B, // +
 0x2F, // /
 0x3D, // =
];
const kEqualSignIndex = ArrayPrototypeIndexOf(kForgivingBase64AllowedChars,
 0x3D);

function atob(input) {
 // The implementation here has not been performance optimized in any way and
 // should not be.
Expand All@@ -1294,54 +1270,25 @@ function atob(input) {
 }

 input = `${input}`;
 let nonAsciiWhitespaceCharCount = 0;
 let equalCharCount = 0;

 for (let n = 0; n < input.length; n++) {
 const index = ArrayPrototypeIndexOf(
 kForgivingBase64AllowedChars,
 StringPrototypeCharCodeAt(input, n));

 if (index > 4) {
 // The first 5 elements of `kForgivingBase64AllowedChars` are
 // ASCII whitespace char codes.
 nonAsciiWhitespaceCharCount++;

 if (index === kEqualSignIndex) {
 equalCharCount++;
 } else if (equalCharCount) {
 // The `=` char is only allowed at the end.
 throw lazyDOMException('Invalid character', 'InvalidCharacterError');
 }

 if (equalCharCount > 2) {
 // Only one more `=` is permitted after the first equal sign.
 throw lazyDOMException('Invalid character', 'InvalidCharacterError');
 }
 } else if (index === -1) {
 const len = base64ByteLength(input, input.length);
 const b = Buffer.allocUnsafe(len);
 // returns 0 on error
 const binary_length = b.base64ToBinary(input, 0, input.length);
 if (binary_length < 0) {
 if (binary_length === -1)
 throw lazyDOMException(
 'The string to be decoded is not correctly encoded.',
 'InvalidCharacterError');
 else if (binary_length === -2)
 throw lazyDOMException('Invalid character', 'InvalidCharacterError');
 }
 }

 let reminder = nonAsciiWhitespaceCharCount % 4;

 // See #2, #3, #4 - https://infra.spec.whatwg.org/#forgiving-base64
 if (!reminder) {
 // Remove all trailing `=` characters and get the new reminder.
 reminder = (nonAsciiWhitespaceCharCount - equalCharCount) % 4;
 } else if (equalCharCount) {
 // `=` should not in the input if there's a reminder.
 throw lazyDOMException('Invalid character', 'InvalidCharacterError');
 }

 // See #3 - https://infra.spec.whatwg.org/#forgiving-base64
 if (reminder === 1) {
 throw lazyDOMException(
 'The string to be decoded is not correctly encoded.',
 'InvalidCharacterError');
 else
 throw lazyDOMException(
 'Potential buffer overflow. The binary output is too large.',
 'InvalidCharacterError');
 }

 return Buffer.from(input, 'base64').toString('latin1');
 return b.toString('latin1', 0, binary_length);
}

function isUtf8(input) {
Expand Down

2 changes: 2 additions & 0 deletions
lib/internal/buffer.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@ const {
 utf8Slice,
 asciiWrite,
 base64Write,
 base64ToBinary,
 base64urlWrite,
 latin1Write,
 hexWrite,
Expand DownExpand Up@@ -1038,6 +1039,7 @@ function addBufferPrototypeMethods(proto) {
 proto.utf8Slice = utf8Slice;
 proto.asciiWrite = asciiWrite;
 proto.base64Write = base64Write;
 proto.base64ToBinary = base64ToBinary;
 proto.base64urlWrite = base64urlWrite;
 proto.latin1Write = latin1Write;
 proto.hexWrite = hexWrite;
Expand Down

68 changes: 68 additions & 0 deletions
src/node_buffer.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -699,6 +699,72 @@ void Fill(const FunctionCallbackInfo<Value>& args) {
 }
}

// on success: returns a non-negative integer indicating the size of the
// binary produced, it most be no larger than 2147483647 bytes.
// In case of error, a negativ value is returned:
// * -2 indicates an invalid character,
// * -1 indicates a single character remained,
// * -3 indicates a possible overflow (i.e., more than 2 GB output).
void Base64ToBinary(const FunctionCallbackInfo<Value>& args) {
 Environment* env = Environment::GetCurrent(args);

 THROW_AND_RETURN_UNLESS_BUFFER(env, args.This());
 SPREAD_BUFFER_ARG(args.This(), ts_obj);

 THROW_AND_RETURN_IF_NOT_STRING(env, args[0], "argument");

 Local<String> str = args[0]->ToString(env->context()).ToLocalChecked();

 size_t offset = 0;
 size_t max_length = 0;

 THROW_AND_RETURN_IF_OOB(ParseArrayIndex(env, args[1], 0, &offset));
 if (offset > ts_obj_length) {
 return node::THROW_ERR_BUFFER_OUT_OF_BOUNDS(
 env, "\"offset\" is outside of buffer bounds");
 }

 THROW_AND_RETURN_IF_OOB(ParseArrayIndex(env, args[2], ts_obj_length - offset,
 &max_length));
 if (max_length == 0)
 return args.GetReturnValue().Set(0);

 char* buf = ts_obj_data + offset;
 size_t buflen = max_length;
 if(buflen > INT32_MAX) {
 return args.GetReturnValue().Set(-3);
 }

 int32_t written{0};

 if (str->IsExternalOneByte()) { // 8-bit case
 auto ext = str->GetExternalOneByteStringResource();
 simdutf::result r = simdutf::base64_to_binary_safe(ext->data(), ext->length(), buf, buflen, simdutf::base64_default);
 if(r.error == simdutf::error_code::SUCCESS) {
 written = buflen;
 } else if(r.error == simdutf::error_code::INVALID_BASE64_CHARACTER) {
 written = -2;
 } else if(r.error == simdutf::error_code::BASE64_INPUT_REMAINDER) {
 written = -1;
 } else {
 written = -3;
 }
 } else { // 16-bit case
 String::Value value(env->isolate(), str);
 simdutf::result r = simdutf::base64_to_binary_safe(reinterpret_cast<const char16_t*>(*value), value.length(), buf, buflen, simdutf::base64_default);
 if(r.error == simdutf::error_code::SUCCESS) {
 written = buflen;
 } else if(r.error == simdutf::error_code::INVALID_BASE64_CHARACTER) {
 written = -2;
 } else if(r.error == simdutf::error_code::BASE64_INPUT_REMAINDER) {
 written = -1;
 } else {
 written = -3;
 }
 }

 args.GetReturnValue().Set(written);
}

template <encoding encoding>
void StringWrite(const FunctionCallbackInfo<Value>& args) {
Expand DownExpand Up@@ -1316,6 +1382,7 @@ void Initialize(Local<Object> target,

 SetMethod(context, target, "asciiWrite", StringWrite<ASCII>);
 SetMethod(context, target, "base64Write", StringWrite<BASE64>);
 SetMethod(context, target, "base64ToBinary", Base64ToBinary);
 SetMethod(context, target, "base64urlWrite", StringWrite<BASE64URL>);
 SetMethod(context, target, "latin1Write", StringWrite<LATIN1>);
 SetMethod(context, target, "hexWrite", StringWrite<HEX>);
Expand DownExpand Up@@ -1358,6 +1425,7 @@ void RegisterExternalReferences(ExternalReferenceRegistry* registry) {
 registry->Register(StringSlice<UTF8>);

 registry->Register(StringWrite<ASCII>);
 registry->Register(Base64ToBinary);
 registry->Register(StringWrite<BASE64>);
 registry->Register(StringWrite<BASE64URL>);
 registry->Register(StringWrite<LATIN1>);
Expand Down

70 changes: 60 additions & 10 deletions
src/string_bytes.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -305,7 +305,7 @@ size_t StringBytes::Write(Isolate* isolate,
 Local<Value> val,
 enum encoding encoding) {
 HandleScope scope(isolate);
 size_t nbytes;
 size_t nbytes{0};

 CHECK(val->IsString() == true);
 Local<String> str = val.As<String>();
Expand DownExpand Up@@ -346,14 +346,62 @@ size_t StringBytes::Write(Isolate* isolate,
 }

 case BASE64URL:
 // Fall through
 if (str->IsExternalOneByte()) { // 8-bit case
 auto ext = str->GetExternalOneByteStringResource();
 // Try with WHATWG base64 standard first, adapted for base64url
 simdutf::result r = simdutf::base64_to_binary_safe(ext->data(), ext->length(), buf, buflen, simdutf::base64_url);
 if(r.error == simdutf::error_code::SUCCESS) {
 nbytes = buflen;
 } else {
 // The input does not follow the WHATWG forgiving-base64 specification adapted for base64url
 // https://infra.spec.whatwg.org/#forgiving-base64-decode
 nbytes = base64_decode(buf, buflen, ext->data(), ext->length());
 }
 } else { // 16-bit case
 // Typically, a base64url string is stored as an 8-bit string within v8.
 // Thus str->IsOneByte() is typically true. The next line thus often allocates
 // a temporary 16-bit buffer to store a 16-bit copy of the 8-bit v8 string.
 // Hence the creation of the String::Value value is likely a performance bottleneck.
 String::Value value(isolate, str);
 // Try with WHATWG base64 standard first
 simdutf::result r = simdutf::base64_to_binary_safe(reinterpret_cast<const char16_t*>(*value), value.length(), buf, buflen, simdutf::base64_url);
 if(r.error == simdutf::error_code::SUCCESS) {
 nbytes = buflen;
 } else {
 // The input does not follow the WHATWG forgiving-base64 specification
 // (adapted for base64url with + and / replaced by - and _).
 // https://infra.spec.whatwg.org/#forgiving-base64-decode
 nbytes = base64_decode(buf, buflen, *value, value.length());
 }
 }
 break;
 case BASE64:
 if (str->IsExternalOneByte()) {
 if (str->IsExternalOneByte()) { // 8-bit case
 auto ext = str->GetExternalOneByteStringResource();
 nbytes = base64_decode(buf, buflen, ext->data(), ext->length());
 } else {
 // Try with WHATWG base64 standard first
 simdutf::result r = simdutf::base64_to_binary_safe(ext->data(), ext->length(), buf, buflen, simdutf::base64_default);
 if(r.error == simdutf::error_code::SUCCESS) {
 nbytes = buflen;
 } else {
 // The input does not follow the WHATWG forgiving-base64 specification
 // https://infra.spec.whatwg.org/#forgiving-base64-decode
 nbytes = base64_decode(buf, buflen, ext->data(), ext->length());
 }
 } else { // 16-bit case
 // Typically, a base64 string is stored as an 8-bit string within v8.
 // Thus str->IsOneByte() is typically true. The next line thus often allocates
 // a temporary 16-bit buffer to store a 16-bit copy of the 8-bit v8 string.
 // Hence the creation of the String::Value value is likely a performance bottleneck.
 String::Value value(isolate, str);
 nbytes = base64_decode(buf, buflen, *value, value.length());
 // Try with WHATWG base64 standard first
 simdutf::result r = simdutf::base64_to_binary_safe(reinterpret_cast<const char16_t*>(*value), value.length(), buf, buflen, simdutf::base64_default);
 if(r.error == simdutf::error_code::SUCCESS) {
 nbytes = buflen;
 } else {
 // The input does not follow the WHATWG base64 specification
 // https://infra.spec.whatwg.org/#forgiving-base64-decode
 nbytes = base64_decode(buf, buflen, *value, value.length());
 }
 }
 break;

Expand DownExpand Up@@ -615,22 +663,24 @@ MaybeLocal<Value> StringBytes::Encode(Isolate* isolate,
 *error = node::ERR_MEMORY_ALLOCATION_FAILED(isolate);
 return MaybeLocal<Value>();
 }

 // by default, base64 uses libbase64's accelerated routine so
 // it is not necessary to use simdutf.
 size_t written = base64_encode(buf, buflen, dst, dlen);
 CHECK_EQ(written, dlen);

 return ExternOneByteString::New(isolate, dst, dlen, error);
 }

 case BASE64URL: {
 size_t dlen = base64_encoded_size(buflen, Base64Mode::URL);
 // When in URL mode, base64_encode uses a non-accelerated routine, so we
 // adopt simdutf.
 size_t dlen = simdutf::base64_length_from_binary(buflen, simdutf::base64_url);
 char* dst = node::UncheckedMalloc(dlen);
 if (dst == nullptr) {
 *error = node::ERR_MEMORY_ALLOCATION_FAILED(isolate);
 return MaybeLocal<Value>();
 }

 size_t written = base64_encode(buf, buflen, dst, dlen, Base64Mode::URL);
 size_t written = simdutf::binary_to_base64(buf, buflen, dst, simdutf::base64_url);
 CHECK_EQ(written, dlen);

 return ExternOneByteString::New(isolate, dst, dlen, error);
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
Fastbase64#10

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

lemire
wants to merge
11
commits into

from

 Closed


Fastbase64
#10
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


11 commits


Select commit
Hold shift + click to select a range
7d37d8c

fix description of simdutf


lemire Apr 1, 2024
c157e23

deps: update simdutf to 5.1.0

Apr 1, 2024
d3cc29b

fix: use simdutf for base64 encoding

Apr 1, 2024
0571b0a

updating simdutf

Apr 1, 2024
e00ea55

implemented decoding using accelerated functions

Apr 2, 2024
be45574

implemented fast atob

Apr 2, 2024
ab17447

various fixes

Apr 2, 2024
f69f215

deps: update simdutf to 5.2.1

Apr 2, 2024
cef15cd

deps: update simdutf to 5.2.2

Apr 2, 2024
4c12582

simplify Base64ToBinary

Apr 2, 2024
4391837

identifying performance bottleneck

Apr 3, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


5,754 changes: 5,505 additions & 249 deletions
deps/simdutf/simdutf.cpp

View file


Open in desktop


Large diffs are not rendered by default.


316 changes: 309 additions & 7 deletions
deps/simdutf/simdutf.h

View file


Open in desktop


Large diffs are not rendered by default.


2 changes: 1 addition & 1 deletion
doc/contributing/maintaining/maintaining-dependencies.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -295,7 +295,7 @@ a C++ library for fast JSON parsing.
### simdutf

The [simdutf](https://github.com/simdutf/simdutf) dependency is
a C++ library for fast UTF-8 decoding and encoding.
a C++ library for fast character and base64 decoding and encoding.

### undici

Expand Down

83 changes: 15 additions & 68 deletions
lib/buffer.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,10 +23,8 @@

const {
 Array,
 ArrayFrom,
 ArrayIsArray,
 ArrayPrototypeForEach,
 ArrayPrototypeIndexOf,
 MathFloor,
 MathMin,
 MathTrunc,
Expand DownExpand Up@@ -1263,28 +1261,6 @@ function btoa(input) {
 return buf.toString('base64');
}

// Refs: https://infra.spec.whatwg.org/#forgiving-base64-decode
const kForgivingBase64AllowedChars = [
 // ASCII whitespace
 // Refs: https://infra.spec.whatwg.org/#ascii-whitespace
 0x09, 0x0A, 0x0C, 0x0D, 0x20,

 // Uppercase letters
 ...ArrayFrom({ length: 26 }, (_, i) => StringPrototypeCharCodeAt('A') + i),

 // Lowercase letters
 ...ArrayFrom({ length: 26 }, (_, i) => StringPrototypeCharCodeAt('a') + i),

 // Decimal digits
 ...ArrayFrom({ length: 10 }, (_, i) => StringPrototypeCharCodeAt('0') + i),

 0x2B, // +
 0x2F, // /
 0x3D, // =
];
const kEqualSignIndex = ArrayPrototypeIndexOf(kForgivingBase64AllowedChars,
 0x3D);

function atob(input) {
 // The implementation here has not been performance optimized in any way and
 // should not be.
Expand All@@ -1294,54 +1270,25 @@ function atob(input) {
 }

 input = `${input}`;
 let nonAsciiWhitespaceCharCount = 0;
 let equalCharCount = 0;

 for (let n = 0; n < input.length; n++) {
 const index = ArrayPrototypeIndexOf(
 kForgivingBase64AllowedChars,
 StringPrototypeCharCodeAt(input, n));

 if (index > 4) {
 // The first 5 elements of `kForgivingBase64AllowedChars` are
 // ASCII whitespace char codes.
 nonAsciiWhitespaceCharCount++;

 if (index === kEqualSignIndex) {
 equalCharCount++;
 } else if (equalCharCount) {
 // The `=` char is only allowed at the end.
 throw lazyDOMException('Invalid character', 'InvalidCharacterError');
 }

 if (equalCharCount > 2) {
 // Only one more `=` is permitted after the first equal sign.
 throw lazyDOMException('Invalid character', 'InvalidCharacterError');
 }
 } else if (index === -1) {
 const len = base64ByteLength(input, input.length);
 const b = Buffer.allocUnsafe(len);
 // returns 0 on error
 const binary_length = b.base64ToBinary(input, 0, input.length);
 if (binary_length < 0) {
 if (binary_length === -1)
 throw lazyDOMException(
 'The string to be decoded is not correctly encoded.',
 'InvalidCharacterError');
 else if (binary_length === -2)
 throw lazyDOMException('Invalid character', 'InvalidCharacterError');
 }
 }

 let reminder = nonAsciiWhitespaceCharCount % 4;

 // See #2, #3, #4 - https://infra.spec.whatwg.org/#forgiving-base64
 if (!reminder) {
 // Remove all trailing `=` characters and get the new reminder.
 reminder = (nonAsciiWhitespaceCharCount - equalCharCount) % 4;
 } else if (equalCharCount) {
 // `=` should not in the input if there's a reminder.
 throw lazyDOMException('Invalid character', 'InvalidCharacterError');
 }

 // See #3 - https://infra.spec.whatwg.org/#forgiving-base64
 if (reminder === 1) {
 throw lazyDOMException(
 'The string to be decoded is not correctly encoded.',
 'InvalidCharacterError');
 else
 throw lazyDOMException(
 'Potential buffer overflow. The binary output is too large.',
 'InvalidCharacterError');
 }

 return Buffer.from(input, 'base64').toString('latin1');
 return b.toString('latin1', 0, binary_length);
}

function isUtf8(input) {
Expand Down

2 changes: 2 additions & 0 deletions
lib/internal/buffer.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@ const {
 utf8Slice,
 asciiWrite,
 base64Write,
 base64ToBinary,
 base64urlWrite,
 latin1Write,
 hexWrite,
Expand DownExpand Up@@ -1038,6 +1039,7 @@ function addBufferPrototypeMethods(proto) {
 proto.utf8Slice = utf8Slice;
 proto.asciiWrite = asciiWrite;
 proto.base64Write = base64Write;
 proto.base64ToBinary = base64ToBinary;
 proto.base64urlWrite = base64urlWrite;
 proto.latin1Write = latin1Write;
 proto.hexWrite = hexWrite;
Expand Down

68 changes: 68 additions & 0 deletions
src/node_buffer.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -699,6 +699,72 @@ void Fill(const FunctionCallbackInfo<Value>& args) {
 }
}

// on success: returns a non-negative integer indicating the size of the
// binary produced, it most be no larger than 2147483647 bytes.
// In case of error, a negativ value is returned:
// * -2 indicates an invalid character,
// * -1 indicates a single character remained,
// * -3 indicates a possible overflow (i.e., more than 2 GB output).
void Base64ToBinary(const FunctionCallbackInfo<Value>& args) {
 Environment* env = Environment::GetCurrent(args);

 THROW_AND_RETURN_UNLESS_BUFFER(env, args.This());
 SPREAD_BUFFER_ARG(args.This(), ts_obj);

 THROW_AND_RETURN_IF_NOT_STRING(env, args[0], "argument");

 Local<String> str = args[0]->ToString(env->context()).ToLocalChecked();

 size_t offset = 0;
 size_t max_length = 0;

 THROW_AND_RETURN_IF_OOB(ParseArrayIndex(env, args[1], 0, &offset));
 if (offset > ts_obj_length) {
 return node::THROW_ERR_BUFFER_OUT_OF_BOUNDS(
 env, "\"offset\" is outside of buffer bounds");
 }

 THROW_AND_RETURN_IF_OOB(ParseArrayIndex(env, args[2], ts_obj_length - offset,
 &max_length));
 if (max_length == 0)
 return args.GetReturnValue().Set(0);

 char* buf = ts_obj_data + offset;
 size_t buflen = max_length;
 if(buflen > INT32_MAX) {
 return args.GetReturnValue().Set(-3);
 }

 int32_t written{0};

 if (str->IsExternalOneByte()) { // 8-bit case
 auto ext = str->GetExternalOneByteStringResource();
 simdutf::result r = simdutf::base64_to_binary_safe(ext->data(), ext->length(), buf, buflen, simdutf::base64_default);
 if(r.error == simdutf::error_code::SUCCESS) {
 written = buflen;
 } else if(r.error == simdutf::error_code::INVALID_BASE64_CHARACTER) {
 written = -2;
 } else if(r.error == simdutf::error_code::BASE64_INPUT_REMAINDER) {
 written = -1;
 } else {
 written = -3;
 }
 } else { // 16-bit case
 String::Value value(env->isolate(), str);
 simdutf::result r = simdutf::base64_to_binary_safe(reinterpret_cast<const char16_t*>(*value), value.length(), buf, buflen, simdutf::base64_default);
 if(r.error == simdutf::error_code::SUCCESS) {
 written = buflen;
 } else if(r.error == simdutf::error_code::INVALID_BASE64_CHARACTER) {
 written = -2;
 } else if(r.error == simdutf::error_code::BASE64_INPUT_REMAINDER) {
 written = -1;
 } else {
 written = -3;
 }
 }

 args.GetReturnValue().Set(written);
}

template <encoding encoding>
void StringWrite(const FunctionCallbackInfo<Value>& args) {
Expand DownExpand Up@@ -1316,6 +1382,7 @@ void Initialize(Local<Object> target,

 SetMethod(context, target, "asciiWrite", StringWrite<ASCII>);
 SetMethod(context, target, "base64Write", StringWrite<BASE64>);
 SetMethod(context, target, "base64ToBinary", Base64ToBinary);
 SetMethod(context, target, "base64urlWrite", StringWrite<BASE64URL>);
 SetMethod(context, target, "latin1Write", StringWrite<LATIN1>);
 SetMethod(context, target, "hexWrite", StringWrite<HEX>);
Expand DownExpand Up@@ -1358,6 +1425,7 @@ void RegisterExternalReferences(ExternalReferenceRegistry* registry) {
 registry->Register(StringSlice<UTF8>);

 registry->Register(StringWrite<ASCII>);
 registry->Register(Base64ToBinary);
 registry->Register(StringWrite<BASE64>);
 registry->Register(StringWrite<BASE64URL>);
 registry->Register(StringWrite<LATIN1>);
Expand Down

70 changes: 60 additions & 10 deletions
src/string_bytes.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -305,7 +305,7 @@ size_t StringBytes::Write(Isolate* isolate,
 Local<Value> val,
 enum encoding encoding) {
 HandleScope scope(isolate);
 size_t nbytes;
 size_t nbytes{0};

 CHECK(val->IsString() == true);
 Local<String> str = val.As<String>();
Expand DownExpand Up@@ -346,14 +346,62 @@ size_t StringBytes::Write(Isolate* isolate,
 }

 case BASE64URL:
 // Fall through
 if (str->IsExternalOneByte()) { // 8-bit case
 auto ext = str->GetExternalOneByteStringResource();
 // Try with WHATWG base64 standard first, adapted for base64url
 simdutf::result r = simdutf::base64_to_binary_safe(ext->data(), ext->length(), buf, buflen, simdutf::base64_url);
 if(r.error == simdutf::error_code::SUCCESS) {
 nbytes = buflen;
 } else {
 // The input does not follow the WHATWG forgiving-base64 specification adapted for base64url
 // https://infra.spec.whatwg.org/#forgiving-base64-decode
 nbytes = base64_decode(buf, buflen, ext->data(), ext->length());
 }
 } else { // 16-bit case
 // Typically, a base64url string is stored as an 8-bit string within v8.
 // Thus str->IsOneByte() is typically true. The next line thus often allocates
 // a temporary 16-bit buffer to store a 16-bit copy of the 8-bit v8 string.
 // Hence the creation of the String::Value value is likely a performance bottleneck.
 String::Value value(isolate, str);
 // Try with WHATWG base64 standard first
 simdutf::result r = simdutf::base64_to_binary_safe(reinterpret_cast<const char16_t*>(*value), value.length(), buf, buflen, simdutf::base64_url);
 if(r.error == simdutf::error_code::SUCCESS) {
 nbytes = buflen;
 } else {
 // The input does not follow the WHATWG forgiving-base64 specification
 // (adapted for base64url with + and / replaced by - and _).
 // https://infra.spec.whatwg.org/#forgiving-base64-decode
 nbytes = base64_decode(buf, buflen, *value, value.length());
 }
 }
 break;
 case BASE64:
 if (str->IsExternalOneByte()) {
 if (str->IsExternalOneByte()) { // 8-bit case
 auto ext = str->GetExternalOneByteStringResource();
 nbytes = base64_decode(buf, buflen, ext->data(), ext->length());
 } else {
 // Try with WHATWG base64 standard first
 simdutf::result r = simdutf::base64_to_binary_safe(ext->data(), ext->length(), buf, buflen, simdutf::base64_default);
 if(r.error == simdutf::error_code::SUCCESS) {
 nbytes = buflen;
 } else {
 // The input does not follow the WHATWG forgiving-base64 specification
 // https://infra.spec.whatwg.org/#forgiving-base64-decode
 nbytes = base64_decode(buf, buflen, ext->data(), ext->length());
 }
 } else { // 16-bit case
 // Typically, a base64 string is stored as an 8-bit string within v8.
 // Thus str->IsOneByte() is typically true. The next line thus often allocates
 // a temporary 16-bit buffer to store a 16-bit copy of the 8-bit v8 string.
 // Hence the creation of the String::Value value is likely a performance bottleneck.
 String::Value value(isolate, str);
 nbytes = base64_decode(buf, buflen, *value, value.length());
 // Try with WHATWG base64 standard first
 simdutf::result r = simdutf::base64_to_binary_safe(reinterpret_cast<const char16_t*>(*value), value.length(), buf, buflen, simdutf::base64_default);
 if(r.error == simdutf::error_code::SUCCESS) {
 nbytes = buflen;
 } else {
 // The input does not follow the WHATWG base64 specification
 // https://infra.spec.whatwg.org/#forgiving-base64-decode
 nbytes = base64_decode(buf, buflen, *value, value.length());
 }
 }
 break;

Expand DownExpand Up@@ -615,22 +663,24 @@ MaybeLocal<Value> StringBytes::Encode(Isolate* isolate,
 *error = node::ERR_MEMORY_ALLOCATION_FAILED(isolate);
 return MaybeLocal<Value>();
 }

 // by default, base64 uses libbase64's accelerated routine so
 // it is not necessary to use simdutf.
 size_t written = base64_encode(buf, buflen, dst, dlen);
 CHECK_EQ(written, dlen);

 return ExternOneByteString::New(isolate, dst, dlen, error);
 }

 case BASE64URL: {
 size_t dlen = base64_encoded_size(buflen, Base64Mode::URL);
 // When in URL mode, base64_encode uses a non-accelerated routine, so we
 // adopt simdutf.
 size_t dlen = simdutf::base64_length_from_binary(buflen, simdutf::base64_url);
 char* dst = node::UncheckedMalloc(dlen);
 if (dst == nullptr) {
 *error = node::ERR_MEMORY_ALLOCATION_FAILED(isolate);
 return MaybeLocal<Value>();
 }

 size_t written = base64_encode(buf, buflen, dst, dlen, Base64Mode::URL);
 size_t written = simdutf::binary_to_base64(buf, buflen, dst, simdutf::base64_url);
 CHECK_EQ(written, dlen);

 return ExternOneByteString::New(isolate, dst, dlen, error);
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Fastbase64#10

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

lemire
wants to merge
11
commits into

from

 Closed


Fastbase64
#10
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


11 commits


Select commit
Hold shift + click to select a range
7d37d8c

fix description of simdutf


lemire Apr 1, 2024
c157e23

deps: update simdutf to 5.1.0

Apr 1, 2024
d3cc29b

fix: use simdutf for base64 encoding

Apr 1, 2024
0571b0a

updating simdutf

Apr 1, 2024
e00ea55

implemented decoding using accelerated functions

Apr 2, 2024
be45574

implemented fast atob

Apr 2, 2024
ab17447

various fixes

Apr 2, 2024
f69f215

deps: update simdutf to 5.2.1

Apr 2, 2024
cef15cd

deps: update simdutf to 5.2.2

Apr 2, 2024
4c12582

simplify Base64ToBinary

Apr 2, 2024
4391837

identifying performance bottleneck

Apr 3, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


5,754 changes: 5,505 additions & 249 deletions
deps/simdutf/simdutf.cpp

View file


Open in desktop


Large diffs are not rendered by default.


316 changes: 309 additions & 7 deletions
deps/simdutf/simdutf.h

View file


Open in desktop


Large diffs are not rendered by default.


2 changes: 1 addition & 1 deletion
doc/contributing/maintaining/maintaining-dependencies.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -295,7 +295,7 @@ a C++ library for fast JSON parsing.
### simdutf

The [simdutf](https://github.com/simdutf/simdutf) dependency is
a C++ library for fast UTF-8 decoding and encoding.
a C++ library for fast character and base64 decoding and encoding.

### undici

Expand Down

83 changes: 15 additions & 68 deletions
lib/buffer.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,10 +23,8 @@

const {
 Array,
 ArrayFrom,
 ArrayIsArray,
 ArrayPrototypeForEach,
 ArrayPrototypeIndexOf,
 MathFloor,
 MathMin,
 MathTrunc,
Expand DownExpand Up@@ -1263,28 +1261,6 @@ function btoa(input) {
 return buf.toString('base64');
}

// Refs: https://infra.spec.whatwg.org/#forgiving-base64-decode
const kForgivingBase64AllowedChars = [
 // ASCII whitespace
 // Refs: https://infra.spec.whatwg.org/#ascii-whitespace
 0x09, 0x0A, 0x0C, 0x0D, 0x20,

 // Uppercase letters
 ...ArrayFrom({ length: 26 }, (_, i) => StringPrototypeCharCodeAt('A') + i),

 // Lowercase letters
 ...ArrayFrom({ length: 26 }, (_, i) => StringPrototypeCharCodeAt('a') + i),

 // Decimal digits
 ...ArrayFrom({ length: 10 }, (_, i) => StringPrototypeCharCodeAt('0') + i),

 0x2B, // +
 0x2F, // /
 0x3D, // =
];
const kEqualSignIndex = ArrayPrototypeIndexOf(kForgivingBase64AllowedChars,
 0x3D);

function atob(input) {
 // The implementation here has not been performance optimized in any way and
 // should not be.
Expand All@@ -1294,54 +1270,25 @@ function atob(input) {
 }

 input = `${input}`;
 let nonAsciiWhitespaceCharCount = 0;
 let equalCharCount = 0;

 for (let n = 0; n < input.length; n++) {
 const index = ArrayPrototypeIndexOf(
 kForgivingBase64AllowedChars,
 StringPrototypeCharCodeAt(input, n));

 if (index > 4) {
 // The first 5 elements of `kForgivingBase64AllowedChars` are
 // ASCII whitespace char codes.
 nonAsciiWhitespaceCharCount++;

 if (index === kEqualSignIndex) {
 equalCharCount++;
 } else if (equalCharCount) {
 // The `=` char is only allowed at the end.
 throw lazyDOMException('Invalid character', 'InvalidCharacterError');
 }

 if (equalCharCount > 2) {
 // Only one more `=` is permitted after the first equal sign.
 throw lazyDOMException('Invalid character', 'InvalidCharacterError');
 }
 } else if (index === -1) {
 const len = base64ByteLength(input, input.length);
 const b = Buffer.allocUnsafe(len);
 // returns 0 on error
 const binary_length = b.base64ToBinary(input, 0, input.length);
 if (binary_length < 0) {
 if (binary_length === -1)
 throw lazyDOMException(
 'The string to be decoded is not correctly encoded.',
 'InvalidCharacterError');
 else if (binary_length === -2)
 throw lazyDOMException('Invalid character', 'InvalidCharacterError');
 }
 }

 let reminder = nonAsciiWhitespaceCharCount % 4;

 // See #2, #3, #4 - https://infra.spec.whatwg.org/#forgiving-base64
 if (!reminder) {
 // Remove all trailing `=` characters and get the new reminder.
 reminder = (nonAsciiWhitespaceCharCount - equalCharCount) % 4;
 } else if (equalCharCount) {
 // `=` should not in the input if there's a reminder.
 throw lazyDOMException('Invalid character', 'InvalidCharacterError');
 }

 // See #3 - https://infra.spec.whatwg.org/#forgiving-base64
 if (reminder === 1) {
 throw lazyDOMException(
 'The string to be decoded is not correctly encoded.',
 'InvalidCharacterError');
 else
 throw lazyDOMException(
 'Potential buffer overflow. The binary output is too large.',
 'InvalidCharacterError');
 }

 return Buffer.from(input, 'base64').toString('latin1');
 return b.toString('latin1', 0, binary_length);
}

function isUtf8(input) {
Expand Down

2 changes: 2 additions & 0 deletions
lib/internal/buffer.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@ const {
 utf8Slice,
 asciiWrite,
 base64Write,
 base64ToBinary,
 base64urlWrite,
 latin1Write,
 hexWrite,
Expand DownExpand Up@@ -1038,6 +1039,7 @@ function addBufferPrototypeMethods(proto) {
 proto.utf8Slice = utf8Slice;
 proto.asciiWrite = asciiWrite;
 proto.base64Write = base64Write;
 proto.base64ToBinary = base64ToBinary;
 proto.base64urlWrite = base64urlWrite;
 proto.latin1Write = latin1Write;
 proto.hexWrite = hexWrite;
Expand Down

68 changes: 68 additions & 0 deletions
src/node_buffer.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -699,6 +699,72 @@ void Fill(const FunctionCallbackInfo<Value>& args) {
 }
}

// on success: returns a non-negative integer indicating the size of the
// binary produced, it most be no larger than 2147483647 bytes.
// In case of error, a negativ value is returned:
// * -2 indicates an invalid character,
// * -1 indicates a single character remained,
// * -3 indicates a possible overflow (i.e., more than 2 GB output).
void Base64ToBinary(const FunctionCallbackInfo<Value>& args) {
 Environment* env = Environment::GetCurrent(args);

 THROW_AND_RETURN_UNLESS_BUFFER(env, args.This());
 SPREAD_BUFFER_ARG(args.This(), ts_obj);

 THROW_AND_RETURN_IF_NOT_STRING(env, args[0], "argument");

 Local<String> str = args[0]->ToString(env->context()).ToLocalChecked();

 size_t offset = 0;
 size_t max_length = 0;

 THROW_AND_RETURN_IF_OOB(ParseArrayIndex(env, args[1], 0, &offset));
 if (offset > ts_obj_length) {
 return node::THROW_ERR_BUFFER_OUT_OF_BOUNDS(
 env, "\"offset\" is outside of buffer bounds");
 }

 THROW_AND_RETURN_IF_OOB(ParseArrayIndex(env, args[2], ts_obj_length - offset,
 &max_length));
 if (max_length == 0)
 return args.GetReturnValue().Set(0);

 char* buf = ts_obj_data + offset;
 size_t buflen = max_length;
 if(buflen > INT32_MAX) {
 return args.GetReturnValue().Set(-3);
 }

 int32_t written{0};

 if (str->IsExternalOneByte()) { // 8-bit case
 auto ext = str->GetExternalOneByteStringResource();
 simdutf::result r = simdutf::base64_to_binary_safe(ext->data(), ext->length(), buf, buflen, simdutf::base64_default);
 if(r.error == simdutf::error_code::SUCCESS) {
 written = buflen;
 } else if(r.error == simdutf::error_code::INVALID_BASE64_CHARACTER) {
 written = -2;
 } else if(r.error == simdutf::error_code::BASE64_INPUT_REMAINDER) {
 written = -1;
 } else {
 written = -3;
 }
 } else { // 16-bit case
 String::Value value(env->isolate(), str);
 simdutf::result r = simdutf::base64_to_binary_safe(reinterpret_cast<const char16_t*>(*value), value.length(), buf, buflen, simdutf::base64_default);
 if(r.error == simdutf::error_code::SUCCESS) {
 written = buflen;
 } else if(r.error == simdutf::error_code::INVALID_BASE64_CHARACTER) {
 written = -2;
 } else if(r.error == simdutf::error_code::BASE64_INPUT_REMAINDER) {
 written = -1;
 } else {
 written = -3;
 }
 }

 args.GetReturnValue().Set(written);
}

template <encoding encoding>
void StringWrite(const FunctionCallbackInfo<Value>& args) {
Expand DownExpand Up@@ -1316,6 +1382,7 @@ void Initialize(Local<Object> target,

 SetMethod(context, target, "asciiWrite", StringWrite<ASCII>);
 SetMethod(context, target, "base64Write", StringWrite<BASE64>);
 SetMethod(context, target, "base64ToBinary", Base64ToBinary);
 SetMethod(context, target, "base64urlWrite", StringWrite<BASE64URL>);
 SetMethod(context, target, "latin1Write", StringWrite<LATIN1>);
 SetMethod(context, target, "hexWrite", StringWrite<HEX>);
Expand DownExpand Up@@ -1358,6 +1425,7 @@ void RegisterExternalReferences(ExternalReferenceRegistry* registry) {
 registry->Register(StringSlice<UTF8>);

 registry->Register(StringWrite<ASCII>);
 registry->Register(Base64ToBinary);
 registry->Register(StringWrite<BASE64>);
 registry->Register(StringWrite<BASE64URL>);
 registry->Register(StringWrite<LATIN1>);
Expand Down

70 changes: 60 additions & 10 deletions
src/string_bytes.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -305,7 +305,7 @@ size_t StringBytes::Write(Isolate* isolate,
 Local<Value> val,
 enum encoding encoding) {
 HandleScope scope(isolate);
 size_t nbytes;
 size_t nbytes{0};

 CHECK(val->IsString() == true);
 Local<String> str = val.As<String>();
Expand DownExpand Up@@ -346,14 +346,62 @@ size_t StringBytes::Write(Isolate* isolate,
 }

 case BASE64URL:
 // Fall through
 if (str->IsExternalOneByte()) { // 8-bit case
 auto ext = str->GetExternalOneByteStringResource();
 // Try with WHATWG base64 standard first, adapted for base64url
 simdutf::result r = simdutf::base64_to_binary_safe(ext->data(), ext->length(), buf, buflen, simdutf::base64_url);
 if(r.error == simdutf::error_code::SUCCESS) {
 nbytes = buflen;
 } else {
 // The input does not follow the WHATWG forgiving-base64 specification adapted for base64url
 // https://infra.spec.whatwg.org/#forgiving-base64-decode
 nbytes = base64_decode(buf, buflen, ext->data(), ext->length());
 }
 } else { // 16-bit case
 // Typically, a base64url string is stored as an 8-bit string within v8.
 // Thus str->IsOneByte() is typically true. The next line thus often allocates
 // a temporary 16-bit buffer to store a 16-bit copy of the 8-bit v8 string.
 // Hence the creation of the String::Value value is likely a performance bottleneck.
 String::Value value(isolate, str);
 // Try with WHATWG base64 standard first
 simdutf::result r = simdutf::base64_to_binary_safe(reinterpret_cast<const char16_t*>(*value), value.length(), buf, buflen, simdutf::base64_url);
 if(r.error == simdutf::error_code::SUCCESS) {
 nbytes = buflen;
 } else {
 // The input does not follow the WHATWG forgiving-base64 specification
 // (adapted for base64url with + and / replaced by - and _).
 // https://infra.spec.whatwg.org/#forgiving-base64-decode
 nbytes = base64_decode(buf, buflen, *value, value.length());
 }
 }
 break;
 case BASE64:
 if (str->IsExternalOneByte()) {
 if (str->IsExternalOneByte()) { // 8-bit case
 auto ext = str->GetExternalOneByteStringResource();
 nbytes = base64_decode(buf, buflen, ext->data(), ext->length());
 } else {
 // Try with WHATWG base64 standard first
 simdutf::result r = simdutf::base64_to_binary_safe(ext->data(), ext->length(), buf, buflen, simdutf::base64_default);
 if(r.error == simdutf::error_code::SUCCESS) {
 nbytes = buflen;
 } else {
 // The input does not follow the WHATWG forgiving-base64 specification
 // https://infra.spec.whatwg.org/#forgiving-base64-decode
 nbytes = base64_decode(buf, buflen, ext->data(), ext->length());
 }
 } else { // 16-bit case
 // Typically, a base64 string is stored as an 8-bit string within v8.
 // Thus str->IsOneByte() is typically true. The next line thus often allocates
 // a temporary 16-bit buffer to store a 16-bit copy of the 8-bit v8 string.
 // Hence the creation of the String::Value value is likely a performance bottleneck.
 String::Value value(isolate, str);
 nbytes = base64_decode(buf, buflen, *value, value.length());
 // Try with WHATWG base64 standard first
 simdutf::result r = simdutf::base64_to_binary_safe(reinterpret_cast<const char16_t*>(*value), value.length(), buf, buflen, simdutf::base64_default);
 if(r.error == simdutf::error_code::SUCCESS) {
 nbytes = buflen;
 } else {
 // The input does not follow the WHATWG base64 specification
 // https://infra.spec.whatwg.org/#forgiving-base64-decode
 nbytes = base64_decode(buf, buflen, *value, value.length());
 }
 }
 break;

Expand DownExpand Up@@ -615,22 +663,24 @@ MaybeLocal<Value> StringBytes::Encode(Isolate* isolate,
 *error = node::ERR_MEMORY_ALLOCATION_FAILED(isolate);
 return MaybeLocal<Value>();
 }

 // by default, base64 uses libbase64's accelerated routine so
 // it is not necessary to use simdutf.
 size_t written = base64_encode(buf, buflen, dst, dlen);
 CHECK_EQ(written, dlen);

 return ExternOneByteString::New(isolate, dst, dlen, error);
 }

 case BASE64URL: {
 size_t dlen = base64_encoded_size(buflen, Base64Mode::URL);
 // When in URL mode, base64_encode uses a non-accelerated routine, so we
 // adopt simdutf.
 size_t dlen = simdutf::base64_length_from_binary(buflen, simdutf::base64_url);
 char* dst = node::UncheckedMalloc(dlen);
 if (dst == nullptr) {
 *error = node::ERR_MEMORY_ALLOCATION_FAILED(isolate);
 return MaybeLocal<Value>();
 }

 size_t written = base64_encode(buf, buflen, dst, dlen, Base64Mode::URL);
 size_t written = simdutf::binary_to_base64(buf, buflen, dst, simdutf::base64_url);
 CHECK_EQ(written, dlen);

 return ExternOneByteString::New(isolate, dst, dlen, error);
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Fastbase64#10

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

lemire
wants to merge
11
commits into

from

 Closed


Fastbase64
#10
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


11 commits


Select commit
Hold shift + click to select a range
7d37d8c

fix description of simdutf


lemire Apr 1, 2024
c157e23

deps: update simdutf to 5.1.0

Apr 1, 2024
d3cc29b

fix: use simdutf for base64 encoding

Apr 1, 2024
0571b0a

updating simdutf

Apr 1, 2024
e00ea55

implemented decoding using accelerated functions

Apr 2, 2024
be45574

implemented fast atob

Apr 2, 2024
ab17447

various fixes

Apr 2, 2024
f69f215

deps: update simdutf to 5.2.1

Apr 2, 2024
cef15cd

deps: update simdutf to 5.2.2

Apr 2, 2024
4c12582

simplify Base64ToBinary

Apr 2, 2024
4391837

identifying performance bottleneck

Apr 3, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


5,754 changes: 5,505 additions & 249 deletions
deps/simdutf/simdutf.cpp

View file


Open in desktop


Large diffs are not rendered by default.


316 changes: 309 additions & 7 deletions
deps/simdutf/simdutf.h

View file


Open in desktop


Large diffs are not rendered by default.


2 changes: 1 addition & 1 deletion
doc/contributing/maintaining/maintaining-dependencies.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -295,7 +295,7 @@ a C++ library for fast JSON parsing.
### simdutf

The [simdutf](https://github.com/simdutf/simdutf) dependency is
a C++ library for fast UTF-8 decoding and encoding.
a C++ library for fast character and base64 decoding and encoding.

### undici

Expand Down

83 changes: 15 additions & 68 deletions
lib/buffer.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,10 +23,8 @@

const {
 Array,
 ArrayFrom,
 ArrayIsArray,
 ArrayPrototypeForEach,
 ArrayPrototypeIndexOf,
 MathFloor,
 MathMin,
 MathTrunc,
Expand DownExpand Up@@ -1263,28 +1261,6 @@ function btoa(input) {
 return buf.toString('base64');
}

// Refs: https://infra.spec.whatwg.org/#forgiving-base64-decode
const kForgivingBase64AllowedChars = [
 // ASCII whitespace
 // Refs: https://infra.spec.whatwg.org/#ascii-whitespace
 0x09, 0x0A, 0x0C, 0x0D, 0x20,

 // Uppercase letters
 ...ArrayFrom({ length: 26 }, (_, i) => StringPrototypeCharCodeAt('A') + i),

 // Lowercase letters
 ...ArrayFrom({ length: 26 }, (_, i) => StringPrototypeCharCodeAt('a') + i),

 // Decimal digits
 ...ArrayFrom({ length: 10 }, (_, i) => StringPrototypeCharCodeAt('0') + i),

 0x2B, // +
 0x2F, // /
 0x3D, // =
];
const kEqualSignIndex = ArrayPrototypeIndexOf(kForgivingBase64AllowedChars,
 0x3D);

function atob(input) {
 // The implementation here has not been performance optimized in any way and
 // should not be.
Expand All@@ -1294,54 +1270,25 @@ function atob(input) {
 }

 input = `${input}`;
 let nonAsciiWhitespaceCharCount = 0;
 let equalCharCount = 0;

 for (let n = 0; n < input.length; n++) {
 const index = ArrayPrototypeIndexOf(
 kForgivingBase64AllowedChars,
 StringPrototypeCharCodeAt(input, n));

 if (index > 4) {
 // The first 5 elements of `kForgivingBase64AllowedChars` are
 // ASCII whitespace char codes.
 nonAsciiWhitespaceCharCount++;

 if (index === kEqualSignIndex) {
 equalCharCount++;
 } else if (equalCharCount) {
 // The `=` char is only allowed at the end.
 throw lazyDOMException('Invalid character', 'InvalidCharacterError');
 }

 if (equalCharCount > 2) {
 // Only one more `=` is permitted after the first equal sign.
 throw lazyDOMException('Invalid character', 'InvalidCharacterError');
 }
 } else if (index === -1) {
 const len = base64ByteLength(input, input.length);
 const b = Buffer.allocUnsafe(len);
 // returns 0 on error
 const binary_length = b.base64ToBinary(input, 0, input.length);
 if (binary_length < 0) {
 if (binary_length === -1)
 throw lazyDOMException(
 'The string to be decoded is not correctly encoded.',
 'InvalidCharacterError');
 else if (binary_length === -2)
 throw lazyDOMException('Invalid character', 'InvalidCharacterError');
 }
 }

 let reminder = nonAsciiWhitespaceCharCount % 4;

 // See #2, #3, #4 - https://infra.spec.whatwg.org/#forgiving-base64
 if (!reminder) {
 // Remove all trailing `=` characters and get the new reminder.
 reminder = (nonAsciiWhitespaceCharCount - equalCharCount) % 4;
 } else if (equalCharCount) {
 // `=` should not in the input if there's a reminder.
 throw lazyDOMException('Invalid character', 'InvalidCharacterError');
 }

 // See #3 - https://infra.spec.whatwg.org/#forgiving-base64
 if (reminder === 1) {
 throw lazyDOMException(
 'The string to be decoded is not correctly encoded.',
 'InvalidCharacterError');
 else
 throw lazyDOMException(
 'Potential buffer overflow. The binary output is too large.',
 'InvalidCharacterError');
 }

 return Buffer.from(input, 'base64').toString('latin1');
 return b.toString('latin1', 0, binary_length);
}

function isUtf8(input) {
Expand Down

2 changes: 2 additions & 0 deletions
lib/internal/buffer.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@ const {
 utf8Slice,
 asciiWrite,
 base64Write,
 base64ToBinary,
 base64urlWrite,
 latin1Write,
 hexWrite,
Expand DownExpand Up@@ -1038,6 +1039,7 @@ function addBufferPrototypeMethods(proto) {
 proto.utf8Slice = utf8Slice;
 proto.asciiWrite = asciiWrite;
 proto.base64Write = base64Write;
 proto.base64ToBinary = base64ToBinary;
 proto.base64urlWrite = base64urlWrite;
 proto.latin1Write = latin1Write;
 proto.hexWrite = hexWrite;
Expand Down

68 changes: 68 additions & 0 deletions
src/node_buffer.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -699,6 +699,72 @@ void Fill(const FunctionCallbackInfo<Value>& args) {
 }
}

// on success: returns a non-negative integer indicating the size of the
// binary produced, it most be no larger than 2147483647 bytes.
// In case of error, a negativ value is returned:
// * -2 indicates an invalid character,
// * -1 indicates a single character remained,
// * -3 indicates a possible overflow (i.e., more than 2 GB output).
void Base64ToBinary(const FunctionCallbackInfo<Value>& args) {
 Environment* env = Environment::GetCurrent(args);

 THROW_AND_RETURN_UNLESS_BUFFER(env, args.This());
 SPREAD_BUFFER_ARG(args.This(), ts_obj);

 THROW_AND_RETURN_IF_NOT_STRING(env, args[0], "argument");

 Local<String> str = args[0]->ToString(env->context()).ToLocalChecked();

 size_t offset = 0;
 size_t max_length = 0;

 THROW_AND_RETURN_IF_OOB(ParseArrayIndex(env, args[1], 0, &offset));
 if (offset > ts_obj_length) {
 return node::THROW_ERR_BUFFER_OUT_OF_BOUNDS(
 env, "\"offset\" is outside of buffer bounds");
 }

 THROW_AND_RETURN_IF_OOB(ParseArrayIndex(env, args[2], ts_obj_length - offset,
 &max_length));
 if (max_length == 0)
 return args.GetReturnValue().Set(0);

 char* buf = ts_obj_data + offset;
 size_t buflen = max_length;
 if(buflen > INT32_MAX) {
 return args.GetReturnValue().Set(-3);
 }

 int32_t written{0};

 if (str->IsExternalOneByte()) { // 8-bit case
 auto ext = str->GetExternalOneByteStringResource();
 simdutf::result r = simdutf::base64_to_binary_safe(ext->data(), ext->length(), buf, buflen, simdutf::base64_default);
 if(r.error == simdutf::error_code::SUCCESS) {
 written = buflen;
 } else if(r.error == simdutf::error_code::INVALID_BASE64_CHARACTER) {
 written = -2;
 } else if(r.error == simdutf::error_code::BASE64_INPUT_REMAINDER) {
 written = -1;
 } else {
 written = -3;
 }
 } else { // 16-bit case
 String::Value value(env->isolate(), str);
 simdutf::result r = simdutf::base64_to_binary_safe(reinterpret_cast<const char16_t*>(*value), value.length(), buf, buflen, simdutf::base64_default);
 if(r.error == simdutf::error_code::SUCCESS) {
 written = buflen;
 } else if(r.error == simdutf::error_code::INVALID_BASE64_CHARACTER) {
 written = -2;
 } else if(r.error == simdutf::error_code::BASE64_INPUT_REMAINDER) {
 written = -1;
 } else {
 written = -3;
 }
 }

 args.GetReturnValue().Set(written);
}

template <encoding encoding>
void StringWrite(const FunctionCallbackInfo<Value>& args) {
Expand DownExpand Up@@ -1316,6 +1382,7 @@ void Initialize(Local<Object> target,

 SetMethod(context, target, "asciiWrite", StringWrite<ASCII>);
 SetMethod(context, target, "base64Write", StringWrite<BASE64>);
 SetMethod(context, target, "base64ToBinary", Base64ToBinary);
 SetMethod(context, target, "base64urlWrite", StringWrite<BASE64URL>);
 SetMethod(context, target, "latin1Write", StringWrite<LATIN1>);
 SetMethod(context, target, "hexWrite", StringWrite<HEX>);
Expand DownExpand Up@@ -1358,6 +1425,7 @@ void RegisterExternalReferences(ExternalReferenceRegistry* registry) {
 registry->Register(StringSlice<UTF8>);

 registry->Register(StringWrite<ASCII>);
 registry->Register(Base64ToBinary);
 registry->Register(StringWrite<BASE64>);
 registry->Register(StringWrite<BASE64URL>);
 registry->Register(StringWrite<LATIN1>);
Expand Down

70 changes: 60 additions & 10 deletions
src/string_bytes.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -305,7 +305,7 @@ size_t StringBytes::Write(Isolate* isolate,
 Local<Value> val,
 enum encoding encoding) {
 HandleScope scope(isolate);
 size_t nbytes;
 size_t nbytes{0};

 CHECK(val->IsString() == true);
 Local<String> str = val.As<String>();
Expand DownExpand Up@@ -346,14 +346,62 @@ size_t StringBytes::Write(Isolate* isolate,
 }

 case BASE64URL:
 // Fall through
 if (str->IsExternalOneByte()) { // 8-bit case
 auto ext = str->GetExternalOneByteStringResource();
 // Try with WHATWG base64 standard first, adapted for base64url
 simdutf::result r = simdutf::base64_to_binary_safe(ext->data(), ext->length(), buf, buflen, simdutf::base64_url);
 if(r.error == simdutf::error_code::SUCCESS) {
 nbytes = buflen;
 } else {
 // The input does not follow the WHATWG forgiving-base64 specification adapted for base64url
 // https://infra.spec.whatwg.org/#forgiving-base64-decode
 nbytes = base64_decode(buf, buflen, ext->data(), ext->length());
 }
 } else { // 16-bit case
 // Typically, a base64url string is stored as an 8-bit string within v8.
 // Thus str->IsOneByte() is typically true. The next line thus often allocates
 // a temporary 16-bit buffer to store a 16-bit copy of the 8-bit v8 string.
 // Hence the creation of the String::Value value is likely a performance bottleneck.
 String::Value value(isolate, str);
 // Try with WHATWG base64 standard first
 simdutf::result r = simdutf::base64_to_binary_safe(reinterpret_cast<const char16_t*>(*value), value.length(), buf, buflen, simdutf::base64_url);
 if(r.error == simdutf::error_code::SUCCESS) {
 nbytes = buflen;
 } else {
 // The input does not follow the WHATWG forgiving-base64 specification
 // (adapted for base64url with + and / replaced by - and _).
 // https://infra.spec.whatwg.org/#forgiving-base64-decode
 nbytes = base64_decode(buf, buflen, *value, value.length());
 }
 }
 break;
 case BASE64:
 if (str->IsExternalOneByte()) {
 if (str->IsExternalOneByte()) { // 8-bit case
 auto ext = str->GetExternalOneByteStringResource();
 nbytes = base64_decode(buf, buflen, ext->data(), ext->length());
 } else {
 // Try with WHATWG base64 standard first
 simdutf::result r = simdutf::base64_to_binary_safe(ext->data(), ext->length(), buf, buflen, simdutf::base64_default);
 if(r.error == simdutf::error_code::SUCCESS) {
 nbytes = buflen;
 } else {
 // The input does not follow the WHATWG forgiving-base64 specification
 // https://infra.spec.whatwg.org/#forgiving-base64-decode
 nbytes = base64_decode(buf, buflen, ext->data(), ext->length());
 }
 } else { // 16-bit case
 // Typically, a base64 string is stored as an 8-bit string within v8.
 // Thus str->IsOneByte() is typically true. The next line thus often allocates
 // a temporary 16-bit buffer to store a 16-bit copy of the 8-bit v8 string.
 // Hence the creation of the String::Value value is likely a performance bottleneck.
 String::Value value(isolate, str);
 nbytes = base64_decode(buf, buflen, *value, value.length());
 // Try with WHATWG base64 standard first
 simdutf::result r = simdutf::base64_to_binary_safe(reinterpret_cast<const char16_t*>(*value), value.length(), buf, buflen, simdutf::base64_default);
 if(r.error == simdutf::error_code::SUCCESS) {
 nbytes = buflen;
 } else {
 // The input does not follow the WHATWG base64 specification
 // https://infra.spec.whatwg.org/#forgiving-base64-decode
 nbytes = base64_decode(buf, buflen, *value, value.length());
 }
 }
 break;

Expand DownExpand Up@@ -615,22 +663,24 @@ MaybeLocal<Value> StringBytes::Encode(Isolate* isolate,
 *error = node::ERR_MEMORY_ALLOCATION_FAILED(isolate);
 return MaybeLocal<Value>();
 }

 // by default, base64 uses libbase64's accelerated routine so
 // it is not necessary to use simdutf.
 size_t written = base64_encode(buf, buflen, dst, dlen);
 CHECK_EQ(written, dlen);

 return ExternOneByteString::New(isolate, dst, dlen, error);
 }

 case BASE64URL: {
 size_t dlen = base64_encoded_size(buflen, Base64Mode::URL);
 // When in URL mode, base64_encode uses a non-accelerated routine, so we
 // adopt simdutf.
 size_t dlen = simdutf::base64_length_from_binary(buflen, simdutf::base64_url);
 char* dst = node::UncheckedMalloc(dlen);
 if (dst == nullptr) {
 *error = node::ERR_MEMORY_ALLOCATION_FAILED(isolate);
 return MaybeLocal<Value>();
 }

 size_t written = base64_encode(buf, buflen, dst, dlen, Base64Mode::URL);
 size_t written = simdutf::binary_to_base64(buf, buflen, dst, simdutf::base64_url);
 CHECK_EQ(written, dlen);

 return ExternOneByteString::New(isolate, dst, dlen, error);
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
Fastbase64#10

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

lemire
wants to merge
11
commits into

from

 Closed


Fastbase64
#10
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


11 commits


Select commit
Hold shift + click to select a range
7d37d8c

fix description of simdutf


lemire Apr 1, 2024
c157e23

deps: update simdutf to 5.1.0

Apr 1, 2024
d3cc29b

fix: use simdutf for base64 encoding

Apr 1, 2024
0571b0a

updating simdutf

Apr 1, 2024
e00ea55

implemented decoding using accelerated functions

Apr 2, 2024
be45574

implemented fast atob

Apr 2, 2024
ab17447

various fixes

Apr 2, 2024
f69f215

deps: update simdutf to 5.2.1

Apr 2, 2024
cef15cd

deps: update simdutf to 5.2.2

Apr 2, 2024
4c12582

simplify Base64ToBinary

Apr 2, 2024
4391837

identifying performance bottleneck

Apr 3, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


5,754 changes: 5,505 additions & 249 deletions
deps/simdutf/simdutf.cpp

View file


Open in desktop


Large diffs are not rendered by default.


316 changes: 309 additions & 7 deletions
deps/simdutf/simdutf.h

View file


Open in desktop


Large diffs are not rendered by default.


2 changes: 1 addition & 1 deletion
doc/contributing/maintaining/maintaining-dependencies.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -295,7 +295,7 @@ a C++ library for fast JSON parsing.
### simdutf

The [simdutf](https://github.com/simdutf/simdutf) dependency is
a C++ library for fast UTF-8 decoding and encoding.
a C++ library for fast character and base64 decoding and encoding.

### undici

Expand Down

83 changes: 15 additions & 68 deletions
lib/buffer.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,10 +23,8 @@

const {
 Array,
 ArrayFrom,
 ArrayIsArray,
 ArrayPrototypeForEach,
 ArrayPrototypeIndexOf,
 MathFloor,
 MathMin,
 MathTrunc,
Expand DownExpand Up@@ -1263,28 +1261,6 @@ function btoa(input) {
 return buf.toString('base64');
}

// Refs: https://infra.spec.whatwg.org/#forgiving-base64-decode
const kForgivingBase64AllowedChars = [
 // ASCII whitespace
 // Refs: https://infra.spec.whatwg.org/#ascii-whitespace
 0x09, 0x0A, 0x0C, 0x0D, 0x20,

 // Uppercase letters
 ...ArrayFrom({ length: 26 }, (_, i) => StringPrototypeCharCodeAt('A') + i),

 // Lowercase letters
 ...ArrayFrom({ length: 26 }, (_, i) => StringPrototypeCharCodeAt('a') + i),

 // Decimal digits
 ...ArrayFrom({ length: 10 }, (_, i) => StringPrototypeCharCodeAt('0') + i),

 0x2B, // +
 0x2F, // /
 0x3D, // =
];
const kEqualSignIndex = ArrayPrototypeIndexOf(kForgivingBase64AllowedChars,
 0x3D);

function atob(input) {
 // The implementation here has not been performance optimized in any way and
 // should not be.
Expand All@@ -1294,54 +1270,25 @@ function atob(input) {
 }

 input = `${input}`;
 let nonAsciiWhitespaceCharCount = 0;
 let equalCharCount = 0;

 for (let n = 0; n < input.length; n++) {
 const index = ArrayPrototypeIndexOf(
 kForgivingBase64AllowedChars,
 StringPrototypeCharCodeAt(input, n));

 if (index > 4) {
 // The first 5 elements of `kForgivingBase64AllowedChars` are
 // ASCII whitespace char codes.
 nonAsciiWhitespaceCharCount++;

 if (index === kEqualSignIndex) {
 equalCharCount++;
 } else if (equalCharCount) {
 // The `=` char is only allowed at the end.
 throw lazyDOMException('Invalid character', 'InvalidCharacterError');
 }

 if (equalCharCount > 2) {
 // Only one more `=` is permitted after the first equal sign.
 throw lazyDOMException('Invalid character', 'InvalidCharacterError');
 }
 } else if (index === -1) {
 const len = base64ByteLength(input, input.length);
 const b = Buffer.allocUnsafe(len);
 // returns 0 on error
 const binary_length = b.base64ToBinary(input, 0, input.length);
 if (binary_length < 0) {
 if (binary_length === -1)
 throw lazyDOMException(
 'The string to be decoded is not correctly encoded.',
 'InvalidCharacterError');
 else if (binary_length === -2)
 throw lazyDOMException('Invalid character', 'InvalidCharacterError');
 }
 }

 let reminder = nonAsciiWhitespaceCharCount % 4;

 // See #2, #3, #4 - https://infra.spec.whatwg.org/#forgiving-base64
 if (!reminder) {
 // Remove all trailing `=` characters and get the new reminder.
 reminder = (nonAsciiWhitespaceCharCount - equalCharCount) % 4;
 } else if (equalCharCount) {
 // `=` should not in the input if there's a reminder.
 throw lazyDOMException('Invalid character', 'InvalidCharacterError');
 }

 // See #3 - https://infra.spec.whatwg.org/#forgiving-base64
 if (reminder === 1) {
 throw lazyDOMException(
 'The string to be decoded is not correctly encoded.',
 'InvalidCharacterError');
 else
 throw lazyDOMException(
 'Potential buffer overflow. The binary output is too large.',
 'InvalidCharacterError');
 }

 return Buffer.from(input, 'base64').toString('latin1');
 return b.toString('latin1', 0, binary_length);
}

function isUtf8(input) {
Expand Down

2 changes: 2 additions & 0 deletions
lib/internal/buffer.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@ const {
 utf8Slice,
 asciiWrite,
 base64Write,
 base64ToBinary,
 base64urlWrite,
 latin1Write,
 hexWrite,
Expand DownExpand Up@@ -1038,6 +1039,7 @@ function addBufferPrototypeMethods(proto) {
 proto.utf8Slice = utf8Slice;
 proto.asciiWrite = asciiWrite;
 proto.base64Write = base64Write;
 proto.base64ToBinary = base64ToBinary;
 proto.base64urlWrite = base64urlWrite;
 proto.latin1Write = latin1Write;
 proto.hexWrite = hexWrite;
Expand Down

68 changes: 68 additions & 0 deletions
src/node_buffer.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -699,6 +699,72 @@ void Fill(const FunctionCallbackInfo<Value>& args) {
 }
}

// on success: returns a non-negative integer indicating the size of the
// binary produced, it most be no larger than 2147483647 bytes.
// In case of error, a negativ value is returned:
// * -2 indicates an invalid character,
// * -1 indicates a single character remained,
// * -3 indicates a possible overflow (i.e., more than 2 GB output).
void Base64ToBinary(const FunctionCallbackInfo<Value>& args) {
 Environment* env = Environment::GetCurrent(args);

 THROW_AND_RETURN_UNLESS_BUFFER(env, args.This());
 SPREAD_BUFFER_ARG(args.This(), ts_obj);

 THROW_AND_RETURN_IF_NOT_STRING(env, args[0], "argument");

 Local<String> str = args[0]->ToString(env->context()).ToLocalChecked();

 size_t offset = 0;
 size_t max_length = 0;

 THROW_AND_RETURN_IF_OOB(ParseArrayIndex(env, args[1], 0, &offset));
 if (offset > ts_obj_length) {
 return node::THROW_ERR_BUFFER_OUT_OF_BOUNDS(
 env, "\"offset\" is outside of buffer bounds");
 }

 THROW_AND_RETURN_IF_OOB(ParseArrayIndex(env, args[2], ts_obj_length - offset,
 &max_length));
 if (max_length == 0)
 return args.GetReturnValue().Set(0);

 char* buf = ts_obj_data + offset;
 size_t buflen = max_length;
 if(buflen > INT32_MAX) {
 return args.GetReturnValue().Set(-3);
 }

 int32_t written{0};

 if (str->IsExternalOneByte()) { // 8-bit case
 auto ext = str->GetExternalOneByteStringResource();
 simdutf::result r = simdutf::base64_to_binary_safe(ext->data(), ext->length(), buf, buflen, simdutf::base64_default);
 if(r.error == simdutf::error_code::SUCCESS) {
 written = buflen;
 } else if(r.error == simdutf::error_code::INVALID_BASE64_CHARACTER) {
 written = -2;
 } else if(r.error == simdutf::error_code::BASE64_INPUT_REMAINDER) {
 written = -1;
 } else {
 written = -3;
 }
 } else { // 16-bit case
 String::Value value(env->isolate(), str);
 simdutf::result r = simdutf::base64_to_binary_safe(reinterpret_cast<const char16_t*>(*value), value.length(), buf, buflen, simdutf::base64_default);
 if(r.error == simdutf::error_code::SUCCESS) {
 written = buflen;
 } else if(r.error == simdutf::error_code::INVALID_BASE64_CHARACTER) {
 written = -2;
 } else if(r.error == simdutf::error_code::BASE64_INPUT_REMAINDER) {
 written = -1;
 } else {
 written = -3;
 }
 }

 args.GetReturnValue().Set(written);
}

template <encoding encoding>
void StringWrite(const FunctionCallbackInfo<Value>& args) {
Expand DownExpand Up@@ -1316,6 +1382,7 @@ void Initialize(Local<Object> target,

 SetMethod(context, target, "asciiWrite", StringWrite<ASCII>);
 SetMethod(context, target, "base64Write", StringWrite<BASE64>);
 SetMethod(context, target, "base64ToBinary", Base64ToBinary);
 SetMethod(context, target, "base64urlWrite", StringWrite<BASE64URL>);
 SetMethod(context, target, "latin1Write", StringWrite<LATIN1>);
 SetMethod(context, target, "hexWrite", StringWrite<HEX>);
Expand DownExpand Up@@ -1358,6 +1425,7 @@ void RegisterExternalReferences(ExternalReferenceRegistry* registry) {
 registry->Register(StringSlice<UTF8>);

 registry->Register(StringWrite<ASCII>);
 registry->Register(Base64ToBinary);
 registry->Register(StringWrite<BASE64>);
 registry->Register(StringWrite<BASE64URL>);
 registry->Register(StringWrite<LATIN1>);
Expand Down

70 changes: 60 additions & 10 deletions
src/string_bytes.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -305,7 +305,7 @@ size_t StringBytes::Write(Isolate* isolate,
 Local<Value> val,
 enum encoding encoding) {
 HandleScope scope(isolate);
 size_t nbytes;
 size_t nbytes{0};

 CHECK(val->IsString() == true);
 Local<String> str = val.As<String>();
Expand DownExpand Up@@ -346,14 +346,62 @@ size_t StringBytes::Write(Isolate* isolate,
 }

 case BASE64URL:
 // Fall through
 if (str->IsExternalOneByte()) { // 8-bit case
 auto ext = str->GetExternalOneByteStringResource();
 // Try with WHATWG base64 standard first, adapted for base64url
 simdutf::result r = simdutf::base64_to_binary_safe(ext->data(), ext->length(), buf, buflen, simdutf::base64_url);
 if(r.error == simdutf::error_code::SUCCESS) {
 nbytes = buflen;
 } else {
 // The input does not follow the WHATWG forgiving-base64 specification adapted for base64url
 // https://infra.spec.whatwg.org/#forgiving-base64-decode
 nbytes = base64_decode(buf, buflen, ext->data(), ext->length());
 }
 } else { // 16-bit case
 // Typically, a base64url string is stored as an 8-bit string within v8.
 // Thus str->IsOneByte() is typically true. The next line thus often allocates
 // a temporary 16-bit buffer to store a 16-bit copy of the 8-bit v8 string.
 // Hence the creation of the String::Value value is likely a performance bottleneck.
 String::Value value(isolate, str);
 // Try with WHATWG base64 standard first
 simdutf::result r = simdutf::base64_to_binary_safe(reinterpret_cast<const char16_t*>(*value), value.length(), buf, buflen, simdutf::base64_url);
 if(r.error == simdutf::error_code::SUCCESS) {
 nbytes = buflen;
 } else {
 // The input does not follow the WHATWG forgiving-base64 specification
 // (adapted for base64url with + and / replaced by - and _).
 // https://infra.spec.whatwg.org/#forgiving-base64-decode
 nbytes = base64_decode(buf, buflen, *value, value.length());
 }
 }
 break;
 case BASE64:
 if (str->IsExternalOneByte()) {
 if (str->IsExternalOneByte()) { // 8-bit case
 auto ext = str->GetExternalOneByteStringResource();
 nbytes = base64_decode(buf, buflen, ext->data(), ext->length());
 } else {
 // Try with WHATWG base64 standard first
 simdutf::result r = simdutf::base64_to_binary_safe(ext->data(), ext->length(), buf, buflen, simdutf::base64_default);
 if(r.error == simdutf::error_code::SUCCESS) {
 nbytes = buflen;
 } else {
 // The input does not follow the WHATWG forgiving-base64 specification
 // https://infra.spec.whatwg.org/#forgiving-base64-decode
 nbytes = base64_decode(buf, buflen, ext->data(), ext->length());
 }
 } else { // 16-bit case
 // Typically, a base64 string is stored as an 8-bit string within v8.
 // Thus str->IsOneByte() is typically true. The next line thus often allocates
 // a temporary 16-bit buffer to store a 16-bit copy of the 8-bit v8 string.
 // Hence the creation of the String::Value value is likely a performance bottleneck.
 String::Value value(isolate, str);
 nbytes = base64_decode(buf, buflen, *value, value.length());
 // Try with WHATWG base64 standard first
 simdutf::result r = simdutf::base64_to_binary_safe(reinterpret_cast<const char16_t*>(*value), value.length(), buf, buflen, simdutf::base64_default);
 if(r.error == simdutf::error_code::SUCCESS) {
 nbytes = buflen;
 } else {
 // The input does not follow the WHATWG base64 specification
 // https://infra.spec.whatwg.org/#forgiving-base64-decode
 nbytes = base64_decode(buf, buflen, *value, value.length());
 }
 }
 break;

Expand DownExpand Up@@ -615,22 +663,24 @@ MaybeLocal<Value> StringBytes::Encode(Isolate* isolate,
 *error = node::ERR_MEMORY_ALLOCATION_FAILED(isolate);
 return MaybeLocal<Value>();
 }

 // by default, base64 uses libbase64's accelerated routine so
 // it is not necessary to use simdutf.
 size_t written = base64_encode(buf, buflen, dst, dlen);
 CHECK_EQ(written, dlen);

 return ExternOneByteString::New(isolate, dst, dlen, error);
 }

 case BASE64URL: {
 size_t dlen = base64_encoded_size(buflen, Base64Mode::URL);
 // When in URL mode, base64_encode uses a non-accelerated routine, so we
 // adopt simdutf.
 size_t dlen = simdutf::base64_length_from_binary(buflen, simdutf::base64_url);
 char* dst = node::UncheckedMalloc(dlen);
 if (dst == nullptr) {
 *error = node::ERR_MEMORY_ALLOCATION_FAILED(isolate);
 return MaybeLocal<Value>();
 }

 size_t written = base64_encode(buf, buflen, dst, dlen, Base64Mode::URL);
 size_t written = simdutf::binary_to_base64(buf, buflen, dst, simdutf::base64_url);
 CHECK_EQ(written, dlen);

 return ExternOneByteString::New(isolate, dst, dlen, error);
Expand Down