Threadpool usage and performance considerations#

Allzlib APIs, except those that are explicitly synchronous, use the Node.jsinternal threadpool. This can lead to surprising effects and performancelimitations in some applications.

Creating and using a large number of zlib objects simultaneously can causesignificant memory fragmentation.

import zlibfrom'node:zlib';import {Buffer }from'node:buffer';const payload =Buffer.from('This is some data');// WARNING: DO NOT DO THIS!for (let i =0; i <30000; ++i) {  zlib.deflate(payload,(err, buffer) => {});}const zlib =require('node:zlib');const payload =Buffer.from('This is some data');// WARNING: DO NOT DO THIS!for (let i =0; i <30000; ++i) {  zlib.deflate(payload,(err, buffer) => {});}copy

In the preceding example, 30,000 deflate instances are created concurrently.Because of how some operating systems handle memory allocation anddeallocation, this may lead to significant memory fragmentation.

It is strongly recommended that the results of compressionoperations be cached to avoid duplication of effort.

Compressing HTTP requests and responses#

Thenode:zlib module can be used to implement support for thegzip,deflate,br, andzstd content-encoding mechanisms defined byHTTP.

The HTTPAccept-Encoding header is used within an HTTP request to identifythe compression encodings accepted by the client. TheContent-Encodingheader is used to identify the compression encodings actually applied to amessage.

The examples given below are drastically simplified to show the basic concept.Usingzlib encoding can be expensive, and the results ought to be cached.SeeMemory usage tuning for more information on the speed/memory/compressiontradeoffs involved inzlib usage.

// Client request exampleimport fsfrom'node:fs';import zlibfrom'node:zlib';import httpfrom'node:http';import processfrom'node:process';import { pipeline }from'node:stream';const request = http.get({host:'example.com',path:'/',port:80,headers: {'Accept-Encoding':'br,gzip,deflate,zstd' } });request.on('response',(response) => {const output = fs.createWriteStream('example.com_index.html');constonError = (err) => {if (err) {console.error('An error occurred:', err);      process.exitCode =1;    }  };switch (response.headers['content-encoding']) {case'br':pipeline(response, zlib.createBrotliDecompress(), output, onError);break;// Or, just use zlib.createUnzip() to handle both of the following cases:case'gzip':pipeline(response, zlib.createGunzip(), output, onError);break;case'deflate':pipeline(response, zlib.createInflate(), output, onError);break;case'zstd':pipeline(response, zlib.createZstdDecompress(), output, onError);break;default:pipeline(response, output, onError);break;  }});// Client request exampleconst zlib =require('node:zlib');const http =require('node:http');const fs =require('node:fs');const { pipeline } =require('node:stream');const request = http.get({host:'example.com',path:'/',port:80,headers: {'Accept-Encoding':'br,gzip,deflate,zstd' } });request.on('response',(response) => {const output = fs.createWriteStream('example.com_index.html');constonError = (err) => {if (err) {console.error('An error occurred:', err);      process.exitCode =1;    }  };switch (response.headers['content-encoding']) {case'br':pipeline(response, zlib.createBrotliDecompress(), output, onError);break;// Or, just use zlib.createUnzip() to handle both of the following cases:case'gzip':pipeline(response, zlib.createGunzip(), output, onError);break;case'deflate':pipeline(response, zlib.createInflate(), output, onError);break;case'zstd':pipeline(response, zlib.createZstdDecompress(), output, onError);break;default:pipeline(response, output, onError);break;  }});copy

// server example// Running a gzip operation on every request is quite expensive.// It would be much more efficient to cache the compressed buffer.import zlibfrom'node:zlib';import httpfrom'node:http';import fsfrom'node:fs';import { pipeline }from'node:stream';http.createServer((request, response) => {const raw = fs.createReadStream('index.html');// Store both a compressed and an uncompressed version of the resource.  response.setHeader('Vary','Accept-Encoding');const acceptEncoding = request.headers['accept-encoding'] ||'';constonError = (err) => {if (err) {// If an error occurs, there's not much we can do because// the server has already sent the 200 response code and// some amount of data has already been sent to the client.// The best we can do is terminate the response immediately// and log the error.      response.end();console.error('An error occurred:', err);    }  };// Note: This is not a conformant accept-encoding parser.// See https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.3if (/\bdeflate\b/.test(acceptEncoding)) {    response.writeHead(200, {'Content-Encoding':'deflate' });pipeline(raw, zlib.createDeflate(), response, onError);  }elseif (/\bgzip\b/.test(acceptEncoding)) {    response.writeHead(200, {'Content-Encoding':'gzip' });pipeline(raw, zlib.createGzip(), response, onError);  }elseif (/\bbr\b/.test(acceptEncoding)) {    response.writeHead(200, {'Content-Encoding':'br' });pipeline(raw, zlib.createBrotliCompress(), response, onError);  }elseif (/\bzstd\b/.test(acceptEncoding)) {    response.writeHead(200, {'Content-Encoding':'zstd' });pipeline(raw, zlib.createZstdCompress(), response, onError);  }else {    response.writeHead(200, {});pipeline(raw, response, onError);  }}).listen(1337);// server example// Running a gzip operation on every request is quite expensive.// It would be much more efficient to cache the compressed buffer.const zlib =require('node:zlib');const http =require('node:http');const fs =require('node:fs');const { pipeline } =require('node:stream');http.createServer((request, response) => {const raw = fs.createReadStream('index.html');// Store both a compressed and an uncompressed version of the resource.  response.setHeader('Vary','Accept-Encoding');const acceptEncoding = request.headers['accept-encoding'] ||'';constonError = (err) => {if (err) {// If an error occurs, there's not much we can do because// the server has already sent the 200 response code and// some amount of data has already been sent to the client.// The best we can do is terminate the response immediately// and log the error.      response.end();console.error('An error occurred:', err);    }  };// Note: This is not a conformant accept-encoding parser.// See https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.3if (/\bdeflate\b/.test(acceptEncoding)) {    response.writeHead(200, {'Content-Encoding':'deflate' });pipeline(raw, zlib.createDeflate(), response, onError);  }elseif (/\bgzip\b/.test(acceptEncoding)) {    response.writeHead(200, {'Content-Encoding':'gzip' });pipeline(raw, zlib.createGzip(), response, onError);  }elseif (/\bbr\b/.test(acceptEncoding)) {    response.writeHead(200, {'Content-Encoding':'br' });pipeline(raw, zlib.createBrotliCompress(), response, onError);  }elseif (/\bzstd\b/.test(acceptEncoding)) {    response.writeHead(200, {'Content-Encoding':'zstd' });pipeline(raw, zlib.createZstdCompress(), response, onError);  }else {    response.writeHead(200, {});pipeline(raw, response, onError);  }}).listen(1337);copy

By default, thezlib methods will throw an error when decompressingtruncated data. However, if it is known that the data is incomplete, orthe desire is to inspect only the beginning of a compressed file, it ispossible to suppress the default error handling by changing the flushingmethod that is used to decompress the last chunk of input data:

// This is a truncated version of the buffer from the above examplesconst buffer =Buffer.from('eJzT0yMA','base64');zlib.unzip(  buffer,// For Brotli, the equivalent is zlib.constants.BROTLI_OPERATION_FLUSH.// For Zstd, the equivalent is zlib.constants.ZSTD_e_flush.  {finishFlush: zlib.constants.Z_SYNC_FLUSH },(err, buffer) => {if (err) {console.error('An error occurred:', err);      process.exitCode =1;    }console.log(buffer.toString());  });copy

This will not change the behavior in other error-throwing situations, e.g.when the input data has an invalid format. Using this method, it will not bepossible to determine whether the input ended prematurely or lacks theintegrity checks, making it necessary to manually check that thedecompressed result is valid.

Memory usage tuning#

(1 << (windowBits +2)) + (1 << (memLevel +9))copy

const options = {windowBits:14,memLevel:7 };copy

Flushing#

Calling.flush() on a compression stream will makezlib return as muchoutput as currently possible. This may come at the cost of degraded compressionquality, but can be useful when data needs to be available as soon as possible.

In the following example,flush() is used to write a compressed partialHTTP response to the client:

import zlibfrom'node:zlib';import httpfrom'node:http';import { pipeline }from'node:stream';http.createServer((request, response) => {// For the sake of simplicity, the Accept-Encoding checks are omitted.  response.writeHead(200, {'content-encoding':'gzip' });const output = zlib.createGzip();let i;pipeline(output, response,(err) => {if (err) {// If an error occurs, there's not much we can do because// the server has already sent the 200 response code and// some amount of data has already been sent to the client.// The best we can do is terminate the response immediately// and log the error.clearInterval(i);      response.end();console.error('An error occurred:', err);    }  });  i =setInterval(() => {    output.write(`The current time is${Date()}\n`,() => {// The data has been passed to zlib, but the compression algorithm may// have decided to buffer the data for more efficient compression.// Calling .flush() will make the data available as soon as the client// is ready to receive it.      output.flush();    });  },1000);}).listen(1337);const zlib =require('node:zlib');const http =require('node:http');const { pipeline } =require('node:stream');http.createServer((request, response) => {// For the sake of simplicity, the Accept-Encoding checks are omitted.  response.writeHead(200, {'content-encoding':'gzip' });const output = zlib.createGzip();let i;pipeline(output, response,(err) => {if (err) {// If an error occurs, there's not much we can do because// the server has already sent the 200 response code and// some amount of data has already been sent to the client.// The best we can do is terminate the response immediately// and log the error.clearInterval(i);      response.end();console.error('An error occurred:', err);    }  });  i =setInterval(() => {    output.write(`The current time is${Date()}\n`,() => {// The data has been passed to zlib, but the compression algorithm may// have decided to buffer the data for more efficient compression.// Calling .flush() will make the data available as soon as the client// is ready to receive it.      output.flush();    });  },1000);}).listen(1337);copy

Constants#

const stream = zlib.createZstdCompress({params: {    [zlib.constants.ZSTD_c_strategy]: zlib.constants.ZSTD_btultra,  },});copy

Class:Options#

Each zlib-based class takes anoptions object. No options are required.

Some options are only relevant when compressing and areignored by the decompression classes.

• flush<integer>Default:zlib.constants.Z_NO_FLUSH
• finishFlush<integer>Default:zlib.constants.Z_FINISH
• chunkSize<integer>Default:16 * 1024
• windowBits<integer>
• level<integer> (compression only)
• memLevel<integer> (compression only)
• strategy<integer> (compression only)
• dictionary<Buffer> |<TypedArray> |<DataView> |<ArrayBuffer> (deflate/inflate only,empty dictionary by default)
• info<boolean> (Iftrue, returns an object withbuffer andengine.)
• maxOutputLength<integer> Limits output size when usingconvenience methods.Default:buffer.kMaxLength

See thedeflateInit2 andinflateInit2 documentation for moreinformation.

Class:BrotliOptions#

Each Brotli-based class takes anoptions object. All options are optional.

• flush<integer>Default:zlib.constants.BROTLI_OPERATION_PROCESS
• finishFlush<integer>Default:zlib.constants.BROTLI_OPERATION_FINISH
• chunkSize<integer>Default:16 * 1024
• params<Object> Key-value object containing indexedBrotli parameters.
• maxOutputLength<integer> Limits output size when usingconvenience methods.Default:buffer.kMaxLength
• info<boolean> Iftrue, returns an object withbuffer andengine.Default:false

For example:

const stream = zlib.createBrotliCompress({chunkSize:32 *1024,params: {    [zlib.constants.BROTLI_PARAM_MODE]: zlib.constants.BROTLI_MODE_TEXT,    [zlib.constants.BROTLI_PARAM_QUALITY]:4,    [zlib.constants.BROTLI_PARAM_SIZE_HINT]: fs.statSync(inputFile).size,  },});copy

Class:zlib.BrotliCompress#

• Extends:ZlibBase

Compress data using the Brotli algorithm.

Class:zlib.BrotliDecompress#

• Extends:ZlibBase

Decompress data using the Brotli algorithm.

Class:zlib.Deflate#

• Extends:ZlibBase

Compress data using deflate.

Class:zlib.DeflateRaw#

• Extends:ZlibBase

Compress data using deflate, and do not append azlib header.

Class:zlib.Gunzip#

• Extends:ZlibBase

Decompress a gzip stream.

Class:zlib.Gzip#

• Extends:ZlibBase

Compress data using gzip.

Class:zlib.Inflate#

• Extends:ZlibBase

Decompress a deflate stream.

Class:zlib.InflateRaw#

• Extends:ZlibBase

Decompress a raw deflate stream.

Class:zlib.Unzip#

• Extends:ZlibBase

Decompress either a Gzip- or Deflate-compressed stream by auto-detectingthe header.

Class:zlib.ZlibBase#

• Extends:stream.Transform

Not exported by thenode:zlib module. It is documented here because it is thebase class of the compressor/decompressor classes.

This class inherits fromstream.Transform, allowingnode:zlib objects tobe used in pipes and similar stream operations.

Class:ZstdOptions#

Each Zstd-based class takes anoptions object. All options are optional.

• flush<integer>Default:zlib.constants.ZSTD_e_continue
• finishFlush<integer>Default:zlib.constants.ZSTD_e_end
• chunkSize<integer>Default:16 * 1024
• params<Object> Key-value object containing indexedZstd parameters.
• maxOutputLength<integer> Limits output size when usingconvenience methods.Default:buffer.kMaxLength
• info<boolean> Iftrue, returns an object withbuffer andengine.Default:false
• dictionary<Buffer> Optional dictionary used toimprove compression efficiency when compressing or decompressing data thatshares common patterns with the dictionary.

For example:

const stream = zlib.createZstdCompress({chunkSize:32 *1024,params: {    [zlib.constants.ZSTD_c_compressionLevel]:10,    [zlib.constants.ZSTD_c_checksumFlag]:1,  },});copy

Class:zlib.ZstdCompress#

Compress data using the Zstd algorithm.

Class:zlib.ZstdDecompress#

Decompress data using the Zstd algorithm.

zlib.constants#

Provides an object enumerating Zlib-related constants.

zlib.crc32(data[, value])#

• data<string> |<Buffer> |<TypedArray> |<DataView> Whendata is a string,it will be encoded as UTF-8 before being used for computation.
• value<integer> An optional starting value. It must be a 32-bit unsignedinteger.Default:0
• Returns:<integer> A 32-bit unsigned integer containing the checksum.

Computes a 32-bitCyclic Redundancy Check checksum ofdata. Ifvalue is specified, it is used as the starting value of the checksum,otherwise, 0 is used as the starting value.

The CRC algorithm is designed to compute checksums and to detect errorin data transmission. It's not suitable for cryptographic authentication.

To be consistent with other APIs, if thedata is a string, it willbe encoded with UTF-8 before being used for computation. If users onlyuse Node.js to compute and match the checksums, this works well withother APIs that uses the UTF-8 encoding by default.

Some third-party JavaScript libraries compute the checksum on astring based onstr.charCodeAt() so that it can be run in browsers.If users want to match the checksum computed with this kind of libraryin the browser, it's better to use the same library in Node.jsif it also runs in Node.js. If users have to usezlib.crc32() tomatch the checksum produced by such a third-party library:

1. If the library acceptsUint8Array as input, useTextEncoderin the browser to encode the string into aUint8Array with UTF-8encoding, and compute the checksum based on the UTF-8 encoded stringin the browser.
2. If the library only takes a string and compute the data based onstr.charCodeAt(), on the Node.js side, convert the string intoa buffer usingBuffer.from(str, 'utf16le').

import zlibfrom'node:zlib';import {Buffer }from'node:buffer';let crc = zlib.crc32('hello');// 907060870crc = zlib.crc32('world', crc);// 4192936109crc = zlib.crc32(Buffer.from('hello','utf16le'));// 1427272415crc = zlib.crc32(Buffer.from('world','utf16le'), crc);// 4150509955const zlib =require('node:zlib');const {Buffer } =require('node:buffer');let crc = zlib.crc32('hello');// 907060870crc = zlib.crc32('world', crc);// 4192936109crc = zlib.crc32(Buffer.from('hello','utf16le'));// 1427272415crc = zlib.crc32(Buffer.from('world','utf16le'), crc);// 4150509955copy

zlib.createBrotliCompress([options])#

• options<brotli options>

Creates and returns a newBrotliCompress object.

zlib.createBrotliDecompress([options])#

• options<brotli options>

Creates and returns a newBrotliDecompress object.

zlib.createDeflate([options])#

• options<zlib options>

Creates and returns a newDeflate object.

zlib.createDeflateRaw([options])#

• options<zlib options>

Creates and returns a newDeflateRaw object.

An upgrade of zlib from 1.2.8 to 1.2.11 changed behavior whenwindowBitsis set to 8 for raw deflate streams. zlib would automatically setwindowBitsto 9 if was initially set to 8. Newer versions of zlib will throw an exception,so Node.js restored the original behavior of upgrading a value of 8 to 9,since passingwindowBits = 9 to zlib actually results in a compressed streamthat effectively uses an 8-bit window only.

zlib.createGunzip([options])#

• options<zlib options>

Creates and returns a newGunzip object.

zlib.createGzip([options])#

• options<zlib options>

Creates and returns a newGzip object.Seeexample.

zlib.createInflate([options])#

• options<zlib options>

Creates and returns a newInflate object.

zlib.createInflateRaw([options])#

• options<zlib options>

Creates and returns a newInflateRaw object.

zlib.createUnzip([options])#

• options<zlib options>

Creates and returns a newUnzip object.

zlib.createZstdCompress([options])#

• options<zstd options>

Creates and returns a newZstdCompress object.

zlib.createZstdDecompress([options])#

• options<zstd options>

Creates and returns a newZstdDecompress object.

Convenience methods#

All of these take a<Buffer>,<TypedArray>,<DataView>,<ArrayBuffer>, or stringas the first argument, an optional second argumentto supply options to thezlib classes and will call the supplied callbackwithcallback(error, result).

Every method has a*Sync counterpart, which accept the same arguments, butwithout a callback.