Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commita46fe6e

Browse files
committed
pg_rewind: Fix some problems when copying files >2GB.
When incrementally updating a file larger than 2GB, the old code couldeither fail outright (if the client asked the server for bytes beyondthe 2GB boundary) or fail to copy all the blocks that had actuallybeen modified (if the server reported a file size to the client inexcess of 2GB), resulting in data corruption. Generally, such fileswon't occur anyway, but they might if using a non-default segment sizeor if there the directory contains stray files unrelated toPostgreSQL. Fix by a more prudent choice of data types.Even with these improvements, this code still uses a mix of differenttypes (off_t, size_t, uint64, int64) to represent file sizes andoffsets, not all of which necessarily have the same width orsignedness, so further cleanup might be in order here. However, atleast now they all have the potential to be 64 bits wide on 64-bitplatforms.Kuntal Ghosh and Michael Paquier, with a tweak by me.Discussion:http://postgr.es/m/CAGz5QC+8gbkz=Brp0TgoKNqHWTzonbPtPex80U0O6Uh_bevbaA@mail.gmail.com
1 parent88f48b5 commita46fe6e

File tree

1 file changed

+35
-12
lines changed

1 file changed

+35
-12
lines changed

‎src/bin/pg_rewind/libpq_fetch.c

Lines changed: 35 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,7 @@ libpqProcessFileList(void)
194194
for (i=0;i<PQntuples(res);i++)
195195
{
196196
char*path=PQgetvalue(res,i,0);
197-
intfilesize=atoi(PQgetvalue(res,i,1));
197+
int64filesize=atol(PQgetvalue(res,i,1));
198198
boolisdir= (strcmp(PQgetvalue(res,i,2),"t")==0);
199199
char*link_target=PQgetvalue(res,i,3);
200200
file_type_ttype;
@@ -220,13 +220,35 @@ libpqProcessFileList(void)
220220
PQclear(res);
221221
}
222222

223+
/*
224+
* Converts an int64 from network byte order to native format.
225+
*/
226+
staticint64
227+
pg_recvint64(int64value)
228+
{
229+
union
230+
{
231+
int64i64;
232+
uint32i32[2];
233+
}swap;
234+
int64result;
235+
236+
swap.i64=value;
237+
238+
result= (uint32)ntohl(swap.i32[0]);
239+
result <<=32;
240+
result |= (uint32)ntohl(swap.i32[1]);
241+
242+
returnresult;
243+
}
244+
223245
/*----
224246
* Runs a query, which returns pieces of files from the remote source data
225247
* directory, and overwrites the corresponding parts of target files with
226248
* the received parts. The result set is expected to be of format:
227249
*
228250
* pathtext-- path in the data directory, e.g "base/1/123"
229-
* beginint4-- offset within the file
251+
* beginint8-- offset within the file
230252
* chunkbytea-- file content
231253
*----
232254
*/
@@ -247,7 +269,7 @@ receiveFileChunks(const char *sql)
247269
{
248270
char*filename;
249271
intfilenamelen;
250-
intchunkoff;
272+
int64chunkoff;
251273
intchunksize;
252274
char*chunk;
253275

@@ -270,7 +292,7 @@ receiveFileChunks(const char *sql)
270292
pg_fatal("unexpected result set size while fetching remote files\n");
271293

272294
if (PQftype(res,0)!=TEXTOID||
273-
PQftype(res,1)!=INT4OID||
295+
PQftype(res,1)!=INT8OID||
274296
PQftype(res,2)!=BYTEAOID)
275297
{
276298
pg_fatal("unexpected data types in result set while fetching remote files: %u %u %u\n",
@@ -290,12 +312,12 @@ receiveFileChunks(const char *sql)
290312
pg_fatal("unexpected null values in result while fetching remote files\n");
291313
}
292314

293-
if (PQgetlength(res,0,1)!=sizeof(int32))
315+
if (PQgetlength(res,0,1)!=sizeof(int64))
294316
pg_fatal("unexpected result length while fetching remote files\n");
295317

296318
/* Read result set to local variables */
297-
memcpy(&chunkoff,PQgetvalue(res,0,1),sizeof(int32));
298-
chunkoff=ntohl(chunkoff);
319+
memcpy(&chunkoff,PQgetvalue(res,0,1),sizeof(int64));
320+
chunkoff=pg_recvint64(chunkoff);
299321
chunksize=PQgetlength(res,0,2);
300322

301323
filenamelen=PQgetlength(res,0,0);
@@ -320,7 +342,7 @@ receiveFileChunks(const char *sql)
320342
continue;
321343
}
322344

323-
pg_log(PG_DEBUG,"received chunk for file \"%s\", offset%d, size %d\n",
345+
pg_log(PG_DEBUG,"received chunk for file \"%s\", offset"INT64_FORMAT", size %d\n",
324346
filename,chunkoff,chunksize);
325347

326348
open_target_file(filename, false);
@@ -380,7 +402,7 @@ libpqGetFile(const char *filename, size_t *filesize)
380402
* function to actually fetch the data.
381403
*/
382404
staticvoid
383-
fetch_file_range(constchar*path,unsignedintbegin,unsignedintend)
405+
fetch_file_range(constchar*path,uint64begin,uint64end)
384406
{
385407
charlinebuf[MAXPGPATH+23];
386408

@@ -389,12 +411,13 @@ fetch_file_range(const char *path, unsigned int begin, unsigned int end)
389411
{
390412
unsignedintlen;
391413

414+
/* Fine as long as CHUNKSIZE is not bigger than UINT32_MAX */
392415
if (end-begin>CHUNKSIZE)
393416
len=CHUNKSIZE;
394417
else
395-
len=end-begin;
418+
len=(unsignedint) (end-begin);
396419

397-
snprintf(linebuf,sizeof(linebuf),"%s\t%u\t%u\n",path,begin,len);
420+
snprintf(linebuf,sizeof(linebuf),"%s\t"UINT64_FORMAT"\t%u\n",path,begin,len);
398421

399422
if (PQputCopyData(conn,linebuf,strlen(linebuf))!=1)
400423
pg_fatal("could not send COPY data: %s",
@@ -419,7 +442,7 @@ libpq_executeFileMap(filemap_t *map)
419442
* First create a temporary table, and load it with the blocks that we
420443
* need to fetch.
421444
*/
422-
sql="CREATE TEMPORARY TABLE fetchchunks(path text, beginint4, len int4);";
445+
sql="CREATE TEMPORARY TABLE fetchchunks(path text, beginint8, len int4);";
423446
res=PQexec(conn,sql);
424447

425448
if (PQresultStatus(res)!=PGRES_COMMAND_OK)

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp