zlib: add support for brotli compression dictionary

This change adds JS API support for custom compression dictionaries
with Brotli in the zlib library. The underlying Brotli dependency
already supports this and zstd exposes something similar.
This follows the zstd approach for using a custom dictionary but
for Brotli.
Fixes: #52250
PR-URL: #61763
Reviewed-By: Anna Henningsen <anna@addaleax.net>
Reviewed-By: Colin Ihrig <cjihrig@gmail.com>
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Gürgün Dayıoğlu <hey@gurgun.day>

Author: rokob

lib/zlib.js

@@ -830,11 +830,29 @@ function Brotli(opts, mode) {
 });
 }
 
+letdictionary=opts?.dictionary;
+if(dictionary!==undefined&&!isArrayBufferView(dictionary)){
+if(isAnyArrayBuffer(dictionary)){
+dictionary=Buffer.from(dictionary);
+}else{
+thrownewERR_INVALID_ARG_TYPE(
+'options.dictionary',
+['Buffer','TypedArray','DataView','ArrayBuffer'],
+dictionary,
+);
+}
+}
+
 consthandle=mode===BROTLI_DECODE ?
 newbinding.BrotliDecoder(mode) : newbinding.BrotliEncoder(mode);
 
 this._writeState=newUint32Array(2);
-handle.init(brotliInitParamsArray,this._writeState,processCallback);
+handle.init(
+brotliInitParamsArray,
+this._writeState,
+processCallback,
+dictionary,
+);
 
 ZlibBase.call(this,opts,mode,handle,brotliDefaultOpts);
 }

src/node_zlib.cc

@@ -40,6 +40,7 @@
 
 #include"brotli/decode.h"
 #include"brotli/encode.h"
+#include"brotli/shared_dictionary.h"
 #include"zlib.h"
 #include"zstd.h"
 #include"zstd_errors.h"
@@ -256,7 +257,7 @@ class BrotliEncoderContext final : public BrotliContext {
 public:
 voidClose();
 voidDoThreadPoolWork();
- CompressionError Init();
+ CompressionError Init(std::vector<uint8_t>&& dictionary = {});
  CompressionError ResetStream();
  CompressionError SetParams(int key, uint32_t value);
  CompressionError GetErrorInfo() const;
@@ -268,13 +269,18 @@ class BrotliEncoderContext final : public BrotliContext {
 private:
 bool last_result_ = false;
  DeleteFnPtr<BrotliEncoderState, BrotliEncoderDestroyInstance> state_;
+ DeleteFnPtr<BrotliEncoderPreparedDictionary,
+ BrotliEncoderDestroyPreparedDictionary>
+ prepared_dictionary_;
+// Dictionary data must remain valid while the prepared dictionary is alive.
+ std::vector<uint8_t> dictionary_;
 };
 
 classBrotliDecoderContextfinal : public BrotliContext {
 public:
 voidClose();
 voidDoThreadPoolWork();
- CompressionError Init();
+ CompressionError Init(std::vector<uint8_t>&& dictionary = {});
  CompressionError ResetStream();
  CompressionError SetParams(int key, uint32_t value);
  CompressionError GetErrorInfo() const;
@@ -288,6 +294,8 @@ class BrotliDecoderContext final : public BrotliContext {
  BrotliDecoderErrorCode error_ = BROTLI_DECODER_NO_ERROR;
  std::string error_string_;
  DeleteFnPtr<BrotliDecoderState, BrotliDecoderDestroyInstance> state_;
+// Dictionary data must remain valid for the lifetime of the decoder.
+ std::vector<uint8_t> dictionary_;
 };
 
 classZstdContext : publicMemoryRetainer {
@@ -830,7 +838,8 @@ class BrotliCompressionStream final :
 staticvoidInit(const FunctionCallbackInfo<Value>& args) {
  BrotliCompressionStream* wrap;
 ASSIGN_OR_RETURN_UNWRAP(&wrap, args.This());
-CHECK(args.Length() == 3 && "init(params, writeResult, writeCallback)");
+CHECK((args.Length() == 3 || args.Length() == 4) &&
+"init(params, writeResult, writeCallback[, dictionary])");
 
 CHECK(args[1]->IsUint32Array());
 CHECK_GE(args[1].As<Uint32Array>()->Length(), 2);
@@ -841,7 +850,18 @@ class BrotliCompressionStream final :
  wrap->InitStream(write_result, write_js_callback);
 
  AllocScope alloc_scope(wrap);
- CompressionError err = wrap->context()->Init();
+ std::vector<uint8_t> dictionary;
+if (args.Length() == 4 && !args[3]->IsUndefined()) {
+if (!args[3]->IsArrayBufferView()) {
+THROW_ERR_INVALID_ARG_TYPE(
+ wrap->env(), "dictionary must be an ArrayBufferView if provided");
+return;
+ }
+ ArrayBufferViewContents<uint8_t> contents(args[3]);
+ dictionary.assign(contents.data(), contents.data() + contents.length());
+ }
+
+ CompressionError err = wrap->context()->Init(std::move(dictionary));
 if (err.IsError()) {
  wrap->EmitError(err);
 // TODO(addaleax): Sometimes we generate better error codes in C++ land,
@@ -1387,23 +1407,57 @@ void BrotliEncoderContext::DoThreadPoolWork() {
 
 voidBrotliEncoderContext::Close() {
  state_.reset();
+ prepared_dictionary_.reset();
+ dictionary_.clear();
  mode_ = NONE;
 }
 
-CompressionError BrotliEncoderContext::Init() {
+CompressionError BrotliEncoderContext::Init(std::vector<uint8_t>&& dictionary) {
  brotli_alloc_func alloc = CompressionStreamMemoryOwner::AllocForBrotli;
  brotli_free_func free = CompressionStreamMemoryOwner::FreeForZlib;
 void* opaque =
  CompressionStream<BrotliEncoderContext>::AllocatorOpaquePointerForContext(
 this);
+
+// Clean up any previous dictionary state before re-initializing.
+ prepared_dictionary_.reset();
+ dictionary_.clear();
+
  state_.reset(BrotliEncoderCreateInstance(alloc, free, opaque));
 if (!state_) {
 returnCompressionError("Could not initialize Brotli instance",
 "ERR_ZLIB_INITIALIZATION_FAILED",
  -1);
- } else {
-return CompressionError {};
  }
+
+if (!dictionary.empty()) {
+// The dictionary data must remain valid for the lifetime of the prepared
+// dictionary, so take ownership via move.
+ dictionary_ = std::move(dictionary);
+
+ prepared_dictionary_.reset(
+BrotliEncoderPrepareDictionary(BROTLI_SHARED_DICTIONARY_RAW,
+ dictionary_.size(),
+ dictionary_.data(),
+BROTLI_MAX_QUALITY,
+ alloc,
+ free,
+ opaque));
+if (!prepared_dictionary_) {
+returnCompressionError("Failed to prepare brotli dictionary",
+"ERR_ZLIB_DICTIONARY_LOAD_FAILED",
+ -1);
+ }
+
+if (!BrotliEncoderAttachPreparedDictionary(state_.get(),
+ prepared_dictionary_.get())) {
+returnCompressionError("Failed to attach brotli dictionary",
+"ERR_ZLIB_DICTIONARY_LOAD_FAILED",
+ -1);
+ }
+ }
+
+return CompressionError{};
 }
 
 CompressionError BrotliEncoderContext::ResetStream() {
@@ -1435,6 +1489,7 @@ CompressionError BrotliEncoderContext::GetErrorInfo() const {
 
 voidBrotliDecoderContext::Close() {
  state_.reset();
+ dictionary_.clear();
  mode_ = NONE;
 }
 
@@ -1455,20 +1510,39 @@ void BrotliDecoderContext::DoThreadPoolWork() {
  }
 }
 
-CompressionError BrotliDecoderContext::Init() {
+CompressionError BrotliDecoderContext::Init(std::vector<uint8_t>&& dictionary) {
  brotli_alloc_func alloc = CompressionStreamMemoryOwner::AllocForBrotli;
  brotli_free_func free = CompressionStreamMemoryOwner::FreeForZlib;
 void* opaque =
  CompressionStream<BrotliDecoderContext>::AllocatorOpaquePointerForContext(
 this);
+
+// Clean up any previous dictionary state before re-initializing.
+ dictionary_.clear();
+
  state_.reset(BrotliDecoderCreateInstance(alloc, free, opaque));
 if (!state_) {
 returnCompressionError("Could not initialize Brotli instance",
 "ERR_ZLIB_INITIALIZATION_FAILED",
  -1);
- } else {
-return CompressionError {};
  }
+
+if (!dictionary.empty()) {
+// The dictionary data must remain valid for the lifetime of the decoder,
+// so take ownership via move.
+ dictionary_ = std::move(dictionary);
+
+if (!BrotliDecoderAttachDictionary(state_.get(),
+BROTLI_SHARED_DICTIONARY_RAW,
+ dictionary_.size(),
+ dictionary_.data())) {
+returnCompressionError("Failed to attach brotli dictionary",
+"ERR_ZLIB_DICTIONARY_LOAD_FAILED",
+ -1);
+ }
+ }
+
+return CompressionError{};
 }
 
 CompressionError BrotliDecoderContext::ResetStream() {

test/parallel/test-zlib-brotli-dictionary.js

@@ -0,0 +1,126 @@
+'use strict';
+
+constcommon=require('../common');
+constassert=require('assert');
+constzlib=require('zlib');
+
+constdictionary=Buffer.from(
+`Lorem ipsum dolor sit amet, consectetur adipiscing elit. 
+ Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. 
+ Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.`
+);
+
+constinput=Buffer.from(
+`Lorem ipsum dolor sit amet, consectetur adipiscing elit. 
+ Lorem ipsum dolor sit amet, consectetur adipiscing elit. 
+ Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. 
+ Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. 
+ Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.`
+);
+
+// Test with convenience methods (async).
+zlib.brotliCompress(input,{ dictionary },common.mustSucceed((compressed)=>{
+assert(compressed.length<input.length,
+'compressed data should be smaller with dictionary');
+zlib.brotliDecompress(compressed,{ dictionary },common.mustSucceed((decompressed)=>{
+assert.strictEqual(decompressed.toString(),input.toString());
+}));
+}));
+
+// Test with streaming API.
+{
+constencoder=zlib.createBrotliCompress({ dictionary });
+constdecoder=zlib.createBrotliDecompress({ dictionary });
+
+constchunks=[];
+decoder.on('data',(chunk)=>chunks.push(chunk));
+decoder.on('end',common.mustCall(()=>{
+constresult=Buffer.concat(chunks);
+assert.strictEqual(result.toString(),input.toString());
+}));
+
+encoder.pipe(decoder);
+encoder.end(input);
+}
+
+// Test that dictionary improves compression ratio.
+{
+constwithDict=zlib.brotliCompressSync(input,{ dictionary });
+constwithoutDict=zlib.brotliCompressSync(input);
+
+// Dictionary-based compression should be at least as good as without.
+assert(withDict.length<=withoutDict.length,
+`Dictionary compression (${withDict.length}) should not be `+
+`larger than non-dictionary compression (${withoutDict.length})`);
+
+// Verify decompression with dictionary works.
+constdecompressed=zlib.brotliDecompressSync(withDict,{ dictionary });
+assert.strictEqual(decompressed.toString(),input.toString());
+}
+
+// Test that decompression without matching dictionary fails.
+{
+constcompressed=zlib.brotliCompressSync(input,{ dictionary });
+assert.throws(()=>{
+zlib.brotliDecompressSync(compressed);
+},(err)=>{
+assert.match(err.code,/ERR_/);
+returntrue;
+});
+}
+
+// Test that decompression with wrong dictionary fails.
+{
+constcompressed=zlib.brotliCompressSync(input,{ dictionary });
+constwrongDictionary=Buffer.from('this is the wrong dictionary');
+assert.throws(()=>{
+zlib.brotliDecompressSync(compressed,{dictionary: wrongDictionary});
+},(err)=>{
+assert.match(err.code,/ERR_/);
+returntrue;
+});
+}
+
+// Test that dictionary works with ArrayBuffer (converted to Buffer).
+{
+constarrayBufferDict=dictionary.buffer.slice(
+dictionary.byteOffset,
+dictionary.byteOffset+dictionary.byteLength,
+);
+constcompressed=zlib.brotliCompressSync(input,{dictionary: arrayBufferDict});
+constdecompressed=zlib.brotliDecompressSync(compressed,{dictionary: arrayBufferDict});
+assert.strictEqual(decompressed.toString(),input.toString());
+}
+
+// Test that dictionary works with TypedArray (Uint8Array).
+{
+constuint8Dict=newUint8Array(dictionary);
+constcompressed=zlib.brotliCompressSync(input,{dictionary: uint8Dict});
+constdecompressed=zlib.brotliDecompressSync(compressed,{dictionary: uint8Dict});
+assert.strictEqual(decompressed.toString(),input.toString());
+}
+
+// Test that invalid dictionary type throws ERR_INVALID_ARG_TYPE.
+for(constinvalidDictof['string',123,true,{object: true},[1,2,3]]){
+assert.throws(()=>{
+zlib.createBrotliCompress({dictionary: invalidDict});
+},{code: 'ERR_INVALID_ARG_TYPE'});
+
+assert.throws(()=>{
+zlib.createBrotliDecompress({dictionary: invalidDict});
+},{code: 'ERR_INVALID_ARG_TYPE'});
+}
+
+// Test with streaming API and wrong dictionary emits error event.
+{
+constcompressed=zlib.brotliCompressSync(input,{ dictionary });
+constwrongDict=Buffer.from('wrong dictionary data');
+constdecoder=zlib.createBrotliDecompress({dictionary: wrongDict});
+
+decoder.on('error',common.mustCall((err)=>{
+assert.match(err.code,/ERR_/);
+}));
+
+decoder.write(compressed);
+decoder.end();
+}