Skip to content

Commit e3258fd

Browse files
jasnellcodebytere
authored andcommitted
doc: update zlib doc
Just some general improvements to zlib docs and examples Signed-off-by: James M Snell <jasnell@gmail.com> PR-URL: #31665 Reviewed-By: Anna Henningsen <anna@addaleax.net> Reviewed-By: Luigi Pinca <luigipinca@gmail.com>
1 parent 8516602 commit e3258fd

1 file changed

Lines changed: 144 additions & 46 deletions

File tree

‎doc/api/zlib.md‎

Lines changed: 144 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -4,60 +4,121 @@
44

55
> Stability: 2 - Stable
66
7-
The `zlib` module provides compression functionality implemented using Gzip and
8-
Deflate/Inflate, as well as Brotli. It can be accessed using:
7+
The `zlib` module provides compression functionality implemented using Gzip,
8+
Deflate/Inflate, and Brotli.
9+
10+
To access it:
911

1012
```js
1113
constzlib=require('zlib');
1214
```
1315

16+
Compression and decompression are built around the Node.js [Streams API][].
17+
1418
Compressing or decompressing a stream (such as a file) can be accomplished by
15-
piping the source stream data through a `zlib` stream into a destination stream:
19+
piping the source stream through a `zlib``Transform` stream into a destination
20+
stream:
1621

1722
```js
18-
constgzip=zlib.createGzip();
19-
constfs=require('fs');
20-
constinp=fs.createReadStream('input.txt');
21-
constout=fs.createWriteStream('input.txt.gz');
22-
23-
inp.pipe(gzip)
24-
.on('error', () => {
25-
// handle error
26-
})
27-
.pipe(out)
28-
.on('error', () => {
29-
// handle error
23+
const { createGzip } =require('zlib');
24+
const { pipeline } =require('stream');
25+
const {
26+
createReadStream,
27+
createWriteStream
28+
} =require('fs');
29+
30+
constgzip=createGzip();
31+
constsource=createReadStream('input.txt');
32+
constdestination=createWriteStream('input.txt.gz');
33+
34+
pipeline(source, gzip, destination, (err) => {
35+
if (err) {
36+
console.error('An error occurred:', err);
37+
process.exitCode=1;
38+
}
39+
});
40+
41+
// Or, Promisified
42+
43+
const { promisify } =require('util');
44+
constpipe=promisify(pipeline);
45+
46+
asyncfunctiondo_gzip(input, output) {
47+
constgzip=createGzip();
48+
constsource=createReadStream(input);
49+
constdestination=createWriteStream(output);
50+
awaitpipe(source, gzip, destination);
51+
}
52+
53+
do_gzip('input.txt', 'input.txt.gz')
54+
.catch((err) => {
55+
console.error('An error occurred:', err);
56+
process.exitCode=1;
3057
});
3158
```
3259

3360
It is also possible to compress or decompress data in a single step:
3461

3562
```js
63+
const { deflate, unzip } =require('zlib');
64+
3665
constinput='.................................';
37-
zlib.deflate(input, (err, buffer) => {
38-
if (!err) {
39-
console.log(buffer.toString('base64'));
40-
} else {
41-
// handle error
66+
deflate(input, (err, buffer) => {
67+
if (err) {
68+
console.error('An error occurred:', err);
69+
process.exitCode=1;
4270
}
71+
console.log(buffer.toString('base64'));
4372
});
4473

4574
constbuffer=Buffer.from('eJzT0yMAAGTvBe8=', 'base64');
46-
zlib.unzip(buffer, (err, buffer) => {
47-
if (!err) {
48-
console.log(buffer.toString());
49-
} else {
50-
// handle error
75+
unzip(buffer, (err, buffer) => {
76+
if (err) {
77+
console.error('An error occurred:', err);
78+
process.exitCode=1;
5179
}
80+
console.log(buffer.toString());
5281
});
82+
83+
// Or, Promisified
84+
85+
const { promisify } =require('util');
86+
constdo_unzip=promisify(unzip);
87+
88+
do_unzip(buffer)
89+
.then((buf) =>console.log(buf.toString()))
90+
.catch((err) => {
91+
console.error('An error occurred:', err);
92+
process.exitCode=1;
93+
});
5394
```
5495

55-
## Threadpool Usage
96+
## Threadpool Usage and Performance Considerations
97+
98+
All `zlib` APIs, except those that are explicitly synchronous, use the Node.js
99+
internal threadpool. This can lead to surprising effects and performance
100+
limitations in some applications.
56101

57-
All zlib APIs, except those that are explicitly synchronous, use libuv's
58-
threadpool. This can lead to surprising effects in some applications, such as
59-
subpar performance (which can be mitigated by adjusting the [pool size][])
60-
and/or unrecoverable and catastrophic memory fragmentation.
102+
Creating and using a large number of zlib objects simultaneously can cause
103+
significant memory fragmentation.
104+
105+
```js
106+
constzlib=require('zlib');
107+
108+
constpayload=Buffer.from('This is some data');
109+
110+
// WARNING: DO NOT DO THIS!
111+
for (let i =0; i <30000; ++i) {
112+
zlib.deflate(payload, (err, buffer) => {});
113+
}
114+
```
115+
116+
In the preceding example, 30,000 deflate instances are created concurrently.
117+
Because of how some operating systems handle memory allocation and
118+
deallocation, this may lead to to significant memory fragmentation.
119+
120+
It is strongly recommended that the results of compression
121+
operations be cached to avoid duplication of effort.
61122

62123
## Compressing HTTP requests and responses
63124

@@ -80,26 +141,35 @@ tradeoffs involved in `zlib` usage.
80141
constzlib=require('zlib');
81142
consthttp=require('http');
82143
constfs=require('fs');
144+
const { pipeline } =require('stream');
145+
83146
constrequest=http.get({ host:'example.com',
84147
path:'/',
85148
port:80,
86149
headers: { 'Accept-Encoding':'br,gzip,deflate' } });
87150
request.on('response', (response) => {
88151
constoutput=fs.createWriteStream('example.com_index.html');
89152

153+
constonError= (err) => {
154+
if (err) {
155+
console.error('An error occurred:', err);
156+
process.exitCode=1;
157+
}
158+
};
159+
90160
switch (response.headers['content-encoding']) {
91161
case'br':
92-
response.pipe(zlib.createBrotliDecompress()).pipe(output);
162+
pipeline(response, zlib.createBrotliDecompress(), output, onError);
93163
break;
94164
// Or, just use zlib.createUnzip() to handle both of the following cases:
95165
case'gzip':
96-
response.pipe(zlib.createGunzip()).pipe(output);
166+
pipeline(response, zlib.createGunzip(), output, onError);
97167
break;
98168
case'deflate':
99-
response.pipe(zlib.createInflate()).pipe(output);
169+
pipeline(response, zlib.createInflate(), outout, onError);
100170
break;
101171
default:
102-
response.pipe(output);
172+
pipeline(response, output, onError);
103173
break;
104174
}
105175
});
@@ -112,6 +182,8 @@ request.on('response', (response) => {
112182
constzlib=require('zlib');
113183
consthttp=require('http');
114184
constfs=require('fs');
185+
const { pipeline } =require('stream');
186+
115187
http.createServer((request, response) => {
116188
constraw=fs.createReadStream('index.html');
117189
// Store both a compressed and an uncompressed version of the resource.
@@ -121,20 +193,32 @@ http.createServer((request, response) => {
121193
acceptEncoding ='';
122194
}
123195

196+
constonError= (err) => {
197+
if (err) {
198+
// If an error occurs, there's not much we can do because
199+
// the server has already sent the 200 response code and
200+
// some amount of data has already been sent to the client.
201+
// The best we can do is terminate the response immediately
202+
// and log the error.
203+
response.end();
204+
console.error('An error occurred:', err);
205+
}
206+
};
207+
124208
// Note: This is not a conformant accept-encoding parser.
125209
// See https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.3
126210
if (/\bdeflate\b/.test(acceptEncoding)) {
127211
response.writeHead(200, { 'Content-Encoding':'deflate' });
128-
raw.pipe(zlib.createDeflate()).pipe(response);
212+
pipeline(raw, zlib.createDeflate(), response, onError);
129213
} elseif (/\bgzip\b/.test(acceptEncoding)) {
130214
response.writeHead(200, { 'Content-Encoding':'gzip' });
131-
raw.pipe(zlib.createGzip()).pipe(response);
215+
pipeline(raw, zlib.createGzip(), response, onError);
132216
} elseif (/\bbr\b/.test(acceptEncoding)) {
133217
response.writeHead(200, { 'Content-Encoding':'br' });
134-
raw.pipe(zlib.createBrotliCompress()).pipe(response);
218+
pipeline(raw, zlib.createBrotliCompress(), response, onError);
135219
} else {
136220
response.writeHead(200, {});
137-
raw.pipe(response);
221+
pipeline(raw, response, onError);
138222
}
139223
}).listen(1337);
140224
```
@@ -154,11 +238,11 @@ zlib.unzip(
154238
// For Brotli, the equivalent is zlib.constants.BROTLI_OPERATION_FLUSH.
155239
{ finishFlush:zlib.constants.Z_SYNC_FLUSH },
156240
(err, buffer) => {
157-
if (!err) {
158-
console.log(buffer.toString());
159-
} else {
160-
// handle error
241+
if (err) {
242+
console.error('An error occurred:', err);
243+
process.exitCode=1;
161244
}
245+
console.log(buffer.toString());
162246
});
163247
```
164248

@@ -234,14 +318,28 @@ HTTP response to the client:
234318
```js
235319
constzlib=require('zlib');
236320
consthttp=require('http');
321+
const { pipeline } =require('stream');
237322

238323
http.createServer((request, response) => {
239324
// For the sake of simplicity, the Accept-Encoding checks are omitted.
240325
response.writeHead(200, { 'content-encoding':'gzip' });
241326
constoutput=zlib.createGzip();
242-
output.pipe(response);
327+
let i;
328+
329+
pipeline(output, response, (err) => {
330+
if (err) {
331+
// If an error occurs, there's not much we can do because
332+
// the server has already sent the 200 response code and
333+
// some amount of data has already been sent to the client.
334+
// The best we can do is terminate the response immediately
335+
// and log the error.
336+
clearInterval(i);
337+
response.end();
338+
console.error('An error occurred:', err);
339+
}
340+
});
243341

244-
setInterval(() => {
342+
i =setInterval(() => {
245343
output.write(`The current time is ${Date()}\n`, () => {
246344
// The data has been passed to zlib, but the compression algorithm may
247345
// have decided to buffer the data for more efficient compression.
@@ -399,7 +497,7 @@ changes:
399497

400498
<!--type=misc-->
401499

402-
Each zlib-based class takes an `options` object. All options are optional.
500+
Each zlib-based class takes an `options` object. No options are required.
403501

404502
Some options are only relevant when compressing and are
405503
ignored by the decompression classes.
@@ -1058,6 +1156,6 @@ Decompress a chunk of data with [`Unzip`][].
10581156
[Brotli parameters]: #zlib_brotli_constants
10591157
[Memory Usage Tuning]: #zlib_memory_usage_tuning
10601158
[RFC 7932]: https://www.rfc-editor.org/rfc/rfc7932.txt
1061-
[pool size]: cli.html#cli_uv_threadpool_size_size
1159+
[Streams API]: stream.md
10621160
[zlib documentation]: https://zlib.net/manual.html#Constants
10631161
[zlib.createGzip example]: #zlib_zlib

0 commit comments

Comments
 (0)