Expand Up @@ -34,6 +34,9 @@ typedef struct DataPage static bool get_page_header(FILE *in, const char *fullpath, BackupPageHeader *bph, pg_crc32 *crc, bool use_crc32c); static BackupPageHeader2_v1* get_data_file_headers_v1(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, bool strict); #ifdef HAVE_LIBZ /* Implementation of zlib compression method */ static int32 Expand Down Expand Up @@ -300,7 +303,7 @@ prepare_page(pgFile *file, XLogRecPtr prev_backup_start_lsn, while (!page_is_valid && try_again--) { /* read the block */ int read_len = fio_pread(in, page, blknum * BLCKSZ); int read_len = fio_pread(in, page,((int64) blknum) * BLCKSZ); /* The block could have been truncated. It is fine. */ if (read_len == 0) Expand Down Expand Up @@ -489,7 +492,7 @@ backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpat CompressAlg calg, int clevel, uint32 checksum_version, HeaderMap *hdr_map, bool is_merge) { int rc;int64 rc;bool use_pagemap; char *errmsg = NULL; BlockNumber err_blknum = 0; Expand Down Expand Up @@ -758,7 +761,7 @@ catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpa elog(ERROR, "Cannot read file \"%s\"", from_fullpath); } file->read_size =rc * BLCKSZ; file->read_size =((int64)rc) * BLCKSZ; /* Determine that file didn`t changed in case of incremental catchup */ if (backup_mode != BACKUP_MODE_FULL && Expand Down Expand Up @@ -908,7 +911,8 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, if (use_headers && tmp_file->n_headers > 0) headers = get_data_file_headers(&(backup->hdr_map), tmp_file, parse_program_version(backup->program_version), true); true, backup->large_file); if (use_headers && !headers && tmp_file->n_headers > 0) elog(ERROR, "Failed to get page headers for file \"%s\"", from_fullpath); Expand Down Expand Up @@ -952,7 +956,7 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, */ size_t restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version, const char *from_fullpath, const char *to_fullpath,int nblocks, const char *from_fullpath, const char *to_fullpath,int64 nblocks, datapagemap_t *map, PageState *checksum_map, int checksum_version, datapagemap_t *lsn_map, BackupPageHeader2 *headers) { Expand Down Expand Up @@ -1075,7 +1079,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers if (fio_fseek(out, 0) < 0) elog(ERROR, "Cannot seek to the start of file \"%s\": %s", to_fullpath, strerror(errno)); if (fio_ftruncate(out, blknum * BLCKSZ) != 0) if (fio_ftruncate(out,((int64) blknum) * BLCKSZ) != 0) elog(ERROR, "Cannot truncate file \"%s\": %s", to_fullpath, strerror(errno)); break; Expand Down Expand Up @@ -1126,7 +1130,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers cur_pos_in != headers[n_hdr].pos) { if (fseek(in, headers[n_hdr].pos, SEEK_SET) != 0) elog(ERROR, "Cannot seek to offset%u of \"%s\": %s", elog(ERROR, "Cannot seek to offset" INT64_FORMAT " of \"%s\": %s", headers[n_hdr].pos, from_fullpath, strerror(errno)); cur_pos_in = headers[n_hdr].pos; Expand Down Expand Up @@ -1161,7 +1165,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers * When restoring file from FULL backup, pages are written sequentially, * so there is no need to issue fseek for every page. */ write_pos = blknum * BLCKSZ; write_pos =((int64) blknum) * BLCKSZ; if (cur_pos_out != write_pos) { Expand Down Expand Up @@ -1675,7 +1679,7 @@ check_data_file(ConnectionArgs *arguments, pgFile *file, /* Valiate pages of datafile in backup one by one */ bool validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, uint32 checksum_version, uint32 backup_version, HeaderMap *hdr_map) uint32 checksum_version, uint32 backup_version, HeaderMap *hdr_map, bool large_file ) { size_tread_len = 0; boolis_valid = true; Expand All @@ -1696,7 +1700,7 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, elog(ERROR, "Cannot open file \"%s\": %s", fullpath, strerror(errno)); headers = get_data_file_headers(hdr_map, file, backup_version, false); headers = get_data_file_headers(hdr_map, file, backup_version, false, large_file ); if (!headers && file->n_headers > 0) { Expand Down Expand Up @@ -1745,7 +1749,7 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, elog(ERROR, "Cannot seek block %u of \"%s\": %s", blknum, fullpath, strerror(errno)); else elog(VERBOSE, "Seek to%u" , headers[n_hdr].pos); elog(VERBOSE, "Seek to" INT64_FORMAT , headers[n_hdr].pos); cur_pos_in = headers[n_hdr].pos; } Expand Down Expand Up @@ -1882,7 +1886,7 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, /* read local data file and construct map with block checksums */ PageState* get_checksum_map(const char *fullpath, uint32 checksum_version, int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno)int64 n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno){ PageState *checksum_map = NULL; FILE *in = NULL; Expand All @@ -1897,7 +1901,7 @@ get_checksum_map(const char *fullpath, uint32 checksum_version, /* truncate up to blocks */ if (ftruncate(fileno(in), n_blocks * BLCKSZ) != 0) elog(ERROR, "Cannot truncate file to blknum%u \"%s\": %s", elog(ERROR, "Cannot truncate file to blknum" INT64_FORMAT " \"%s\": %s", n_blocks, fullpath, strerror(errno)); setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); Expand Down Expand Up @@ -1951,7 +1955,7 @@ get_checksum_map(const char *fullpath, uint32 checksum_version, /* return bitmap of valid blocks, bitmap is empty, then NULL is returned */ datapagemap_t * get_lsn_map(const char *fullpath, uint32 checksum_version, int n_blocks, XLogRecPtr shift_lsn, BlockNumber segmentno)int64 n_blocks, XLogRecPtr shift_lsn, BlockNumber segmentno){ FILE *in = NULL; BlockNumber blknum = 0; Expand All @@ -1968,7 +1972,7 @@ get_lsn_map(const char *fullpath, uint32 checksum_version, /* truncate up to blocks */ if (ftruncate(fileno(in), n_blocks * BLCKSZ) != 0) elog(ERROR, "Cannot truncate file to blknum%u \"%s\": %s", elog(ERROR, "Cannot truncate file to blknum" INT64_FORMAT " \"%s\": %s", n_blocks, fullpath, strerror(errno)); setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); Expand Down Expand Up @@ -2035,10 +2039,10 @@ get_page_header(FILE *in, const char *fullpath, BackupPageHeader* bph, return false;/* EOF found */ else if (read_len != 0 && feof(in)) elog(ERROR, "Odd size page found at offset%ld of \"%s\"", "Odd size page found at offset" INT64_FORMAT " of \"%s\"", ftello(in), fullpath); else elog(ERROR, "Cannot read header at offset%ld of \"%s\": %s", elog(ERROR, "Cannot read header at offset" INT64_FORMAT " of \"%s\": %s", ftello(in), fullpath, strerror(errno)); } Expand Down Expand Up @@ -2299,9 +2303,9 @@ copy_pages(const char *to_fullpath, const char *from_fullpath, else if (rc == PageIsOk) { if (fseek(out, blknum * BLCKSZ, SEEK_SET) != 0) elog(ERROR, "Cannot seek to position %u in destination file \"%s\": %s", blknum * BLCKSZ, to_fullpath, strerror(errno)); if (fseek(out,((int64) blknum) * BLCKSZ, SEEK_SET) != 0) elog(ERROR, "Cannot seek to position %ld in destination file \"%s\": %s", ((int64) blknum) * BLCKSZ, to_fullpath, strerror(errno));if (write_page(file, out, curr_page) != BLCKSZ) elog(ERROR, "File: \"%s\", cannot write at block %u: %s", Expand Down Expand Up @@ -2361,20 +2365,143 @@ copy_pages(const char *to_fullpath, const char *from_fullpath, return n_blocks_read; } BackupPageHeader2* get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, bool strict, bool large_file) { bool success = false; FILE *in = NULL; size_t read_len = 0; pg_crc32 hdr_crc; BackupPageHeader2 *headers = NULL; /* header decompression */ int z_len = 0; char *zheaders = NULL; const char *errormsg = NULL; if (backup_version < 20400) return NULL; if (file->n_headers <= 0) return NULL; if(!large_file) { BackupPageHeader2_v1 *tmp_headers; read_len = (file->n_headers+1) * sizeof(BackupPageHeader2); tmp_headers = get_data_file_headers_v1(hdr_map, file, backup_version, strict); headers = pgut_malloc(read_len); memset(headers, 0, read_len); if(!tmp_headers) { return NULL; } for(int i=0; i<file->n_headers+1; i++) { headers[i].block = tmp_headers[i].block; headers[i].lsn = tmp_headers[i].lsn; headers[i].pos = tmp_headers[i].pos; headers[i].checksum = tmp_headers[i].checksum; } pg_free(tmp_headers); return headers; } /* TODO: consider to make this descriptor thread-specific */ in = fopen(hdr_map->path, PG_BINARY_R); if (!in) { elog(strict ? ERROR : WARNING, "Cannot open header file \"%s\": %s", hdr_map->path, strerror(errno)); return NULL; } /* disable buffering for header file */ setvbuf(in, NULL, _IONBF, 0); if (fseeko(in, file->hdr_off, SEEK_SET)) { elog(strict ? ERROR : WARNING, "Cannot seek to position %llu in page header map \"%s\": %s", file->hdr_off, hdr_map->path, strerror(errno)); goto cleanup; } /* * The actual number of headers in header file is n+1, last one is a dummy header, * used for calculation of read_len for actual last header. */ read_len = (file->n_headers+1) * sizeof(BackupPageHeader2); /* allocate memory for compressed headers */ zheaders = pgut_malloc(file->hdr_size); memset(zheaders, 0, file->hdr_size); if (fread(zheaders, 1, file->hdr_size, in) != file->hdr_size) { elog(strict ? ERROR : WARNING, "Cannot read header file at offset: %llu len: %i \"%s\": %s", file->hdr_off, file->hdr_size, hdr_map->path, strerror(errno)); goto cleanup; } /* allocate memory for uncompressed headers */ headers = pgut_malloc(read_len); memset(headers, 0, read_len); z_len = do_decompress(headers, read_len, zheaders, file->hdr_size, ZLIB_COMPRESS, &errormsg); if (z_len <= 0) { if (errormsg) elog(strict ? ERROR : WARNING, "An error occured during metadata decompression for file \"%s\": %s", file->rel_path, errormsg); else elog(strict ? ERROR : WARNING, "An error occured during metadata decompression for file \"%s\": %i", file->rel_path, z_len); goto cleanup; } /* validate checksum */ INIT_FILE_CRC32(true, hdr_crc); COMP_FILE_CRC32(true, hdr_crc, headers, read_len); FIN_FILE_CRC32(true, hdr_crc); if (hdr_crc != file->hdr_crc) { elog(strict ? ERROR : WARNING, "Header map for file \"%s\" crc mismatch \"%s\" " "offset: %llu, len: %lu, current: %u, expected: %u", file->rel_path, hdr_map->path, file->hdr_off, read_len, hdr_crc, file->hdr_crc); goto cleanup; } success = true; cleanup: pg_free(zheaders); if (in && fclose(in)) elog(ERROR, "Cannot close file \"%s\"", hdr_map->path); if (!success) { pg_free(headers); headers = NULL; } return headers; } /* * Attempt to open header file, read content and return as * array of headers. * TODO: some access optimizations would be great here: * less fseeks, buffering, descriptor sharing, etc. */ BackupPageHeader2 *get_data_file_headers (HeaderMap *hdr_map, pgFile *file, uint32 backup_version, bool strict)BackupPageHeader2_v1 *get_data_file_headers_v1 (HeaderMap *hdr_map, pgFile *file, uint32 backup_version, bool strict){ bool success = false; FILE *in = NULL; size_t read_len = 0; pg_crc32 hdr_crc; BackupPageHeader2 *headers = NULL;BackupPageHeader2_v1 *headers = NULL;/* header decompression */ int z_len = 0; char *zheaders = NULL; Expand Down Expand Up @@ -2408,7 +2535,7 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, b * The actual number of headers in header file is n+1, last one is a dummy header, * used for calculation of read_len for actual last header. */ read_len = (file->n_headers+1) * sizeof(BackupPageHeader2 ); read_len = (file->n_headers+1) * sizeof(BackupPageHeader2_v1 ); /* allocate memory for compressed headers */ zheaders = pgut_malloc(file->hdr_size); Expand Down