@@ -168,11 +168,11 @@ def download_data(self, repo_id: str = None, local_dir: str = None):
168168token = self .HF_TOKEN ,
169169 )
170170# Unpack forecast parquet files from tar
171- # forecast_dir = f"{data_dir}/forecast/"
172- # forecast_tar = f"{data_dir}/forecast.tar.gz"
173- # with tarfile.open(forecast_tar, "r:gz") as tar:
174- # logger.info(f"Extracting {forecast_tar} to folder {forecast_dir}")
175- # tar.extractall(path=forecast_dir, filter="data")
171+ forecast_dir = f"{ data_dir }
172+ forecast_tar = f"{ data_dir }
173+ with tarfile .open (forecast_tar ,"r:gz" )as tar :
174+ logger .info (f"Extracting{ forecast_tar } { forecast_dir } )
175+ tar .extractall (path = forecast_dir ,filter = "data" )
176176
177177def upload_data (
178178self ,repo_id :str = None ,private :bool = True ,local_dir :str = None
@@ -204,10 +204,10 @@ def upload_data(
204204 )
205205# Compress forecast parquet files to pass hfhub limitation of 25k LFS files
206206forecast_dir = f"{ data_dir }
207- # forecast_tar = f"{data_dir}/forecast.tar.gz"
208- # with tarfile.open(forecast_tar, "w:gz") as tar:
209- # logger.info(f"Creating {forecast_tar} from folder {forecast_dir}")
210- # tar.add(forecast_dir, arcname=os.path.basename(forecast_dir))
207+ forecast_tar = f"{ data_dir }
208+ with tarfile .open (forecast_tar ,"w:gz" )as tar :
209+ logger .info (f"Creating{ forecast_tar } { forecast_dir } )
210+ tar .add (forecast_dir ,arcname = os .path .basename (forecast_dir ))
211211# upload select files to hfhub
212212logger .info (f"uploading folder{ data_dir } )
213213upload_folder (