An ESP32/ESP8266/RP2040 Arduino library to handle .tar, .gz and .tar.gz files

🆕 ESP32-targz now supports compression!

ESP32-targz is based on those two great libraries:

Scope

Tradeoffs

Limitations

Support Matrix

Usage

// Set **destination** filesystem by uncommenting one of these://#define DEST_FS_USES_SPIFFS//#define DEST_FS_USES_FFAT//#define DEST_FS_USES_SD//#define DEST_FS_USES_SD_MMC#defineDEST_FS_USES_LITTLEFS#include<ESP32-targz.h>// filesystem object will be available as "tarGzFS"

Extract content from.gz file

// mount spiffs (or any other filesystem)tarGzFS.begin();GzUnpacker*GZUnpacker=newGzUnpacker();GZUnpacker->haltOnError( true );// stop on fail (manual restart/reset required)GZUnpacker->setupFSCallbacks(targzTotalBytesFn,targzFreeBytesFn );// prevent the partition from exploding, recommendedGZUnpacker->setGzProgressCallback(BaseUnpacker::defaultProgressCallback );// targzNullProgressCallback or defaultProgressCallbackGZUnpacker->setLoggerCallback(BaseUnpacker::targzPrintLoggerCallback  );// gz log verbosity// expand one fileif( !GZUnpacker->gzExpander(tarGzFS,"/gz_example.gz",tarGzFS,"/gz_example.jpg") ) {Serial.printf("gzExpander failed with return code #%d",GZUnpacker->tarGzGetError() );    }// expand another fileif( !gzExpander(tarGzFS,"/blah.gz",tarGzFS,"/blah.jpg") ) {Serial.printf("operation failed with return code #%d",GZUnpacker->tarGzGetError() );    }

Expand contents from.tar file to/tmp folder

// mount spiffs (or any other filesystem)tarGzFS.begin();TarUnpacker*TARUnpacker=newTarUnpacker();TARUnpacker->haltOnError( true );// stop on fail (manual restart/reset required)TARUnpacker->setTarVerify( true );// true = enables health checks but slows down the overall processTARUnpacker->setupFSCallbacks(targzTotalBytesFn,targzFreeBytesFn );// prevent the partition from exploding, recommendedTARUnpacker->setTarProgressCallback(BaseUnpacker::defaultProgressCallback );// prints the untarring progress for each individual fileTARUnpacker->setTarStatusProgressCallback(BaseUnpacker::defaultTarStatusProgressCallback );// print the filenames as they're expandedTARUnpacker->setTarMessageCallback(BaseUnpacker::targzPrintLoggerCallback );// tar log verbosityif( !TARUnpacker->tarExpander(tarGzFS,"/tar_example.tar",tarGzFS,"/") ) {Serial.printf("tarExpander failed with return code #%d\n",TARUnpacker->tarGzGetError() );    }

Expand contents from.tar.gz to/tmp folder

// mount spiffs (or any other filesystem)tarGzFS.begin();TarGzUnpacker*TARGZUnpacker=newTarGzUnpacker();TARGZUnpacker->haltOnError( true );// stop on fail (manual restart/reset required)TARGZUnpacker->setTarVerify( true );// true = enables health checks but slows down the overall processTARGZUnpacker->setupFSCallbacks(targzTotalBytesFn,targzFreeBytesFn );// prevent the partition from exploding, recommendedTARGZUnpacker->setGzProgressCallback(BaseUnpacker::defaultProgressCallback );// targzNullProgressCallback or defaultProgressCallbackTARGZUnpacker->setLoggerCallback(BaseUnpacker::targzPrintLoggerCallback  );// gz log verbosityTARGZUnpacker->setTarProgressCallback(BaseUnpacker::defaultProgressCallback );// prints the untarring progress for each individual fileTARGZUnpacker->setTarStatusProgressCallback(BaseUnpacker::defaultTarStatusProgressCallback );// print the filenames as they're expandedTARGZUnpacker->setTarMessageCallback(BaseUnpacker::targzPrintLoggerCallback );// tar log verbosity// using an intermediate file (default is /tmp/tmp.tar)if( !TARGZUnpacker->tarGzExpander(tarGzFS,"/targz_example.tar.gz",tarGzFS,"/tmp") ) {Serial.printf("tarGzExpander+intermediate file failed with return code #%d\n",TARGZUnpacker->tarGzGetError() );    }// or without intermediate fileif( !TARGZUnpacker->tarGzExpander(tarGzFS,"/targz_example.tar.gz",tarGzFS,"/tmp",nullptr ) ) {Serial.printf("tarGzExpander+intermediate file failed with return code #%d\n",TARGZUnpacker->tarGzGetError() );    }

Flash the ESP with contents from.gz file

// mount spiffs (or any other filesystem)tarGzFS.begin();GzUnpacker*GZUnpacker=newGzUnpacker();GZUnpacker->haltOnError( true );// stop on fail (manual restart/reset required)GZUnpacker->setupFSCallbacks(targzTotalBytesFn,targzFreeBytesFn );// prevent the partition from exploding, recommendedGZUnpacker->setGzProgressCallback(BaseUnpacker::defaultProgressCallback );// targzNullProgressCallback or defaultProgressCallbackGZUnpacker->setLoggerCallback(BaseUnpacker::targzPrintLoggerCallback  );// gz log verbosityif( !GZUnpacker->gzUpdater(tarGzFS,firmwareFile,U_FLASH,/*don't restart after update*/false ) ) {Serial.printf("gzUpdater failed with return code #%d\n",GZUnpacker->tarGzGetError() );    }

ESP32 Only: Flash the ESP with contents from.gz stream

// mount spiffs (or any other filesystem)tarGzFS.begin();fs::Filefile=tarGzFS.open("/example_firmware.gz","r" );if (!file) {Serial.println("Can't open file");return;    }GzUnpacker*GZUnpacker=newGzUnpacker();GZUnpacker->haltOnError( true );// stop on fail (manual restart/reset required)GZUnpacker->setupFSCallbacks(targzTotalBytesFn,targzFreeBytesFn );// prevent the partition from exploding, recommendedGZUnpacker->setGzProgressCallback(BaseUnpacker::defaultProgressCallback );// targzNullProgressCallback or defaultProgressCallbackGZUnpacker->setLoggerCallback(BaseUnpacker::targzPrintLoggerCallback  );// gz log verbosityif( !GZUnpacker->gzStreamUpdater( (Stream*)&file,UPDATE_SIZE_UNKNOWN ) ) {Serial.printf("gzStreamUpdater failed with return code #%d\n",GZUnpacker->tarGzGetError() );    }

ESP32 Only: Direct expansion (no intermediate file) from.tar.gz. stream

// mount spiffs (or any other filesystem)tarGzFS.begin();fs::Filefile=tarGzFS.open("/example_archive.tgz","r" );if (!file) {Serial.println("Can't open file");return;    }TarGzUnpacker*TARGZUnpacker=newTarGzUnpacker();TARGZUnpacker->haltOnError( true );// stop on fail (manual restart/reset required)TARGZUnpacker->setTarVerify( true );// true = enables health checks but slows down the overall processTARGZUnpacker->setupFSCallbacks(targzTotalBytesFn,targzFreeBytesFn );// prevent the partition from exploding, recommendedTARGZUnpacker->setGzProgressCallback(BaseUnpacker::defaultProgressCallback );// targzNullProgressCallback or defaultProgressCallbackTARGZUnpacker->setLoggerCallback(BaseUnpacker::targzPrintLoggerCallback  );// gz log verbosityTARGZUnpacker->setTarProgressCallback(BaseUnpacker::defaultProgressCallback );// prints the untarring progress for each individual fileTARGZUnpacker->setTarStatusProgressCallback(BaseUnpacker::defaultTarStatusProgressCallback );// print the filenames as they're expandedTARGZUnpacker->setTarMessageCallback(BaseUnpacker::targzPrintLoggerCallback );// tar log verbosityif( !TARGZUnpacker->tarGzStreamExpander( (Stream*)&file,tarGzFS ) ) {Serial.printf("tarGzStreamExpander failed with return code #%d\n",TARGZUnpacker->tarGzGetError() );    }

ESP32 Only: Direct Update (no intermediate file) from.tar.gz. stream

TarGzUnpacker*TARGZUnpacker=newTarGzUnpacker();TARGZUnpacker->haltOnError( true );// stop on fail (manual restart/reset required)TARGZUnpacker->setTarVerify( false );// nothing to verify as we're writing a partitionTARGZUnpacker->setGzProgressCallback(BaseUnpacker::targzNullProgressCallback );// don't care about gz progressTARGZUnpacker->setTarProgressCallback(BaseUnpacker::defaultProgressCallback );// prints the untarring progress for each individual partitionTARGZUnpacker->setTarStatusProgressCallback(BaseUnpacker::defaultTarStatusProgressCallback );// print the filenames as they're expandedTARGZUnpacker->setTarMessageCallback(myTarMessageCallback/*BaseUnpacker::targzPrintLoggerCallback*/ );// tar log verbosity// mount SDSD.begin();// this .tar.gz file has both the "app.ino.bin" and "app.spiffs.bin" partitionsfs::Filefile=SD.open("/bundle_firmware.tar.gz","r" );if (!file) {Serial.println("Can't open file");return;    }// this could also be a HTTP/HTTPS/UDP/Ethernet StreamStream*streamptr=&file;if( !TARGZUnpacker->tarGzStreamUpdater(streamptr ) ) {Serial.printf("tarGzStreamUpdater failed with return code #%d\n",TARGZUnpacker->tarGzGetError() );    }else {Serial.println("Flashing successful, now restarting" );ESP.restart();    }

LZPacker::compress() signatures:

// buffer to stream (best compression)size_tcompress(uint8_t* srcBuf,size_t srcBufLen, Stream* dstStream );// buffer to buffer (best compression)size_tcompress(uint8_t* srcBuf,size_t srcBufLen,uint8_t** dstBufPtr );// stream to buffersize_tcompress( Stream* srcStream,size_t srcLen,uint8_t** dstBufPtr );// stream to streamsize_tcompress( Stream* srcStream,size_t srcLen, Stream* dstStream );// stream to filesize_tcompress( Stream* srcStream,size_t srcLen, fs::FS*dstFS,constchar* dstFilename );// file to filesize_tcompress( fs::FS *srcFS,constchar* srcFilename, fs::FS*dstFS,constchar* dstFilename );// file to streamsize_tcompress( fs::FS *srcFS,constchar* srcFilename, Stream* dstStream );

Compress to.gz (buffer to stream)

constchar*json="{\"hello\":\"world\"}";// input bufferFileout=LittleFS.open("/out.gz","w");// output streamsize_tcompressedSize=LZPacker::compress( (uint8_t*)json,strlen(json),&out );out.close();

Compress to.gz (buffer to buffer)

constchar*json="{\"hello\":\"world\"}";// input bufferuint8_t*compressedBytes;// output buffersize_tcompressedSize=LZPacker::compress( (uint8_t*)json,strlen(json),&compressedBytes);// do something with compressedBytesfree(compressedBytes);

Compress to.gz (stream to buffer)

Filein=LittleFS.open("/my.uncompressed.file.txt");// input streamuint8_t*compressedBytes;// output buffersize_tcompressedSize=LZPacker::compress(&in,in.size(),&compressedBytes );// do something with compressedBytesfree(compressedBytes);in.close();

Compress to.gz (stream to stream)

Filein=LittleFS.open("/my.uncompressed.file.txt");// input streamFileout=LittleFS.open("/out.gz","w");// output streamsize_tcompressedSize=LZPacker::compress(&in,in.size(),&out );out.close();in.close();

TarPacker::pack_files() signatures:

intpack_files(fs::FS *srcFS, std::vector<dir_entity_t> dirEntities, Stream* dstStream,constchar* tar_prefix=nullptr);intpack_files(fs::FS *srcFS, std::vector<dir_entity_t> dirEntities, fs::FS *dstFS,constchar*tar_output_file_path,constchar* tar_prefix=nullptr);

Pack to.tar (entities to File)

std::vector<TAR::dir_entity_t>dirEntities;TarPacker::collectDirEntities(&dirEntities,&LittleFS,"/folder/to/pack");  autopackedSize=TarPacker::pack_files(&LittleFS,dirEntities,&LittleFS, "/my.archive.tar");

Pack to.tar (entities to Stream)

std::vector<TAR::dir_entity_t>dirEntities;TarPacker::collectDirEntities(&dirEntities,&LittleFS,"/folder/to/pack");FiletarOutfile=LittleFS.open("/my.archive.tar","w");size_tpackedSize=TarPacker::pack_files(&LittleFS,dirEntities,&tarOutfile);tarOutfile.close();

TarGzPacker::compress() signatures:

intcompress(fs::FS *srcFS,constchar* srcDir, Stream* dstStream,constchar* tar_prefix=nullptr);intcompress(fs::FS *srcFS,constchar* srcDir, fs::FS *dstFS,constchar* tgz_name,constchar* tar_prefix=nullptr);intcompress(fs::FS *srcFS, std::vector<dir_entity_t> dirEntities, Stream* dstStream,constchar* tar_prefix=nullptr);intcompress(fs::FS *srcFS, std::vector<dir_entity_t> dirEntities, fs::FS *dstFS,constchar* tgz_name,constchar* tar_prefix=nullptr);

Pack & compress to.tar.gz file/stream (no filtering on source files/folders list, recursion applies)

FileTarGzOutFile=LittleFS.open("/my.archive.tar.gz","w");size_tcompressedSize=TarGzPacker::compress(&LittleFS/*source*/,"/folder/to/compress",&TarGzOutFile);TarGzOutFile.close();

Pack & compress to.tar.gz file/stream

std::vector<TAR::dir_entity_t>dirEntities;TarPacker::collectDirEntities(&dirEntities,&LittleFS/*source*/,"/folder/to/compress");// eventually filter content from dirEntitiesFileTarGzOutFile=LittleFS.open("/my.archive.tar.gz","w");size_tcompressedSize=TarGzPacker::compress(&LittleFS/*source*/,dirEntities,&TarGzOutFile);TarGzOutFile.close();

Pack & compress to.tar.gz file (no filtering on source files/folders list, recursion applies)

FileTarGzOutFile=LittleFS.open("/my.archive.tar.gz","w");size_tcompressedSize=TarGzPacker::compress(&LittleFS/*source*/,"/folder/to/compress",&LittleFS/*destination*/,"/my.archive.tar.gz");TarGzOutFile.close();

Pack & compress to.tar.gz file

std::vector<TAR::dir_entity_t>dirEntities;TarPacker::collectDirEntities(&dirEntities,&LittleFS/*source*/,"/folder/to/compress");// eventually filter content from dirEntitiesFileTarGzOutFile=LittleFS.open("/my.archive.tar.gz","w");size_tcompressedSize=TarGzPacker::compress(&LittleFS/*source*/,dirEntities,&LittleFS/*destination*/,"/my.archive.tar.gz");TarGzOutFile.close();

TarGzUnpacker/GzUnpacker/TarUnpacker Callbacks

// basic progress callback (valid for tar or gzip)voidmyBasicProgressCallback(uint8_tprogress )    {Serial.printf("Progress: %d\n",progress );    }// complex progress callback (valid for tar or gzip)voidmyProgressCallback(uint8_tprogress )    {staticint8_tmyLastProgress=-1;if(myLastProgress!=progress ) {if(myLastProgress==-1 ) {Serial.print("Progress: ");        }myLastProgress=progress;switch(progress ) {case0:Serial.print("0% ▓");break;case25:Serial.print(" 25% ");break;case50:Serial.print(" 50% ");break;case75:Serial.print(" 75% ");break;case100:Serial.print("▓ 100%\n");myLastProgress=-1;break;default:if(progress<100)Serial.print("▓" );break;        }      }    }// General Error/Warning/Info loggervoidmyLogger(constchar*format, ...)    {va_listargs;va_start(args,format);vprintf(format,args);va_end(args);    }// status callback for TAR (fired at file creation)voidmyTarStatusProgressCallback(constchar*name,size_tsize,size_ttotal_unpacked )    {Serial.printf("[TAR] %-64s %8d bytes - %8d Total bytes\n",name,size,total_unpacked );    }

TarGzUnpacker/GzUnpacker/TarUnpacker Return Codes

Test Suite

Known bugs

Debugging:

Resources

Alternate links

Credits: