Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit73fbf3d

Browse files
committed
pg_rewind: Fix some problems when copying files >2GB.
When incrementally updating a file larger than 2GB, the old code couldeither fail outright (if the client asked the server for bytes beyondthe 2GB boundary) or fail to copy all the blocks that had actuallybeen modified (if the server reported a file size to the client inexcess of 2GB), resulting in data corruption. Generally, such fileswon't occur anyway, but they might if using a non-default segment sizeor if there the directory contains stray files unrelated toPostgreSQL. Fix by a more prudent choice of data types.Even with these improvements, this code still uses a mix of differenttypes (off_t, size_t, uint64, int64) to represent file sizes andoffsets, not all of which necessarily have the same width orsignedness, so further cleanup might be in order here. However, atleast now they all have the potential to be 64 bits wide on 64-bitplatforms.Kuntal Ghosh and Michael Paquier, with a tweak by me.Discussion:http://postgr.es/m/CAGz5QC+8gbkz=Brp0TgoKNqHWTzonbPtPex80U0O6Uh_bevbaA@mail.gmail.com
1 parentafd56b8 commit73fbf3d

File tree

1 file changed

+35
-12
lines changed

1 file changed

+35
-12
lines changed

‎src/bin/pg_rewind/libpq_fetch.c

Lines changed: 35 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -195,7 +195,7 @@ libpqProcessFileList(void)
195195
for (i=0;i<PQntuples(res);i++)
196196
{
197197
char*path=PQgetvalue(res,i,0);
198-
intfilesize=atoi(PQgetvalue(res,i,1));
198+
int64filesize=atol(PQgetvalue(res,i,1));
199199
boolisdir= (strcmp(PQgetvalue(res,i,2),"t")==0);
200200
char*link_target=PQgetvalue(res,i,3);
201201
file_type_ttype;
@@ -221,13 +221,35 @@ libpqProcessFileList(void)
221221
PQclear(res);
222222
}
223223

224+
/*
225+
* Converts an int64 from network byte order to native format.
226+
*/
227+
staticint64
228+
pg_recvint64(int64value)
229+
{
230+
union
231+
{
232+
int64i64;
233+
uint32i32[2];
234+
}swap;
235+
int64result;
236+
237+
swap.i64=value;
238+
239+
result= (uint32)ntohl(swap.i32[0]);
240+
result <<=32;
241+
result |= (uint32)ntohl(swap.i32[1]);
242+
243+
returnresult;
244+
}
245+
224246
/*----
225247
* Runs a query, which returns pieces of files from the remote source data
226248
* directory, and overwrites the corresponding parts of target files with
227249
* the received parts. The result set is expected to be of format:
228250
*
229251
* pathtext-- path in the data directory, e.g "base/1/123"
230-
* beginint4-- offset within the file
252+
* beginint8-- offset within the file
231253
* chunkbytea-- file content
232254
*----
233255
*/
@@ -248,7 +270,7 @@ receiveFileChunks(const char *sql)
248270
{
249271
char*filename;
250272
intfilenamelen;
251-
intchunkoff;
273+
int64chunkoff;
252274
intchunksize;
253275
char*chunk;
254276

@@ -271,7 +293,7 @@ receiveFileChunks(const char *sql)
271293
pg_fatal("unexpected result set size while fetching remote files\n");
272294

273295
if (PQftype(res,0)!=TEXTOID||
274-
PQftype(res,1)!=INT4OID||
296+
PQftype(res,1)!=INT8OID||
275297
PQftype(res,2)!=BYTEAOID)
276298
{
277299
pg_fatal("unexpected data types in result set while fetching remote files: %u %u %u\n",
@@ -291,12 +313,12 @@ receiveFileChunks(const char *sql)
291313
pg_fatal("unexpected null values in result while fetching remote files\n");
292314
}
293315

294-
if (PQgetlength(res,0,1)!=sizeof(int32))
316+
if (PQgetlength(res,0,1)!=sizeof(int64))
295317
pg_fatal("unexpected result length while fetching remote files\n");
296318

297319
/* Read result set to local variables */
298-
memcpy(&chunkoff,PQgetvalue(res,0,1),sizeof(int32));
299-
chunkoff=ntohl(chunkoff);
320+
memcpy(&chunkoff,PQgetvalue(res,0,1),sizeof(int64));
321+
chunkoff=pg_recvint64(chunkoff);
300322
chunksize=PQgetlength(res,0,2);
301323

302324
filenamelen=PQgetlength(res,0,0);
@@ -321,7 +343,7 @@ receiveFileChunks(const char *sql)
321343
continue;
322344
}
323345

324-
pg_log(PG_DEBUG,"received chunk for file \"%s\", offset%d, size %d\n",
346+
pg_log(PG_DEBUG,"received chunk for file \"%s\", offset"INT64_FORMAT", size %d\n",
325347
filename,chunkoff,chunksize);
326348

327349
open_target_file(filename, false);
@@ -381,7 +403,7 @@ libpqGetFile(const char *filename, size_t *filesize)
381403
* function to actually fetch the data.
382404
*/
383405
staticvoid
384-
fetch_file_range(constchar*path,unsignedintbegin,unsignedintend)
406+
fetch_file_range(constchar*path,uint64begin,uint64end)
385407
{
386408
charlinebuf[MAXPGPATH+23];
387409

@@ -390,12 +412,13 @@ fetch_file_range(const char *path, unsigned int begin, unsigned int end)
390412
{
391413
unsignedintlen;
392414

415+
/* Fine as long as CHUNKSIZE is not bigger than UINT32_MAX */
393416
if (end-begin>CHUNKSIZE)
394417
len=CHUNKSIZE;
395418
else
396-
len=end-begin;
419+
len=(unsignedint) (end-begin);
397420

398-
snprintf(linebuf,sizeof(linebuf),"%s\t%u\t%u\n",path,begin,len);
421+
snprintf(linebuf,sizeof(linebuf),"%s\t"UINT64_FORMAT"\t%u\n",path,begin,len);
399422

400423
if (PQputCopyData(conn,linebuf,strlen(linebuf))!=1)
401424
pg_fatal("could not send COPY data: %s",
@@ -420,7 +443,7 @@ libpq_executeFileMap(filemap_t *map)
420443
* First create a temporary table, and load it with the blocks that we
421444
* need to fetch.
422445
*/
423-
sql="CREATE TEMPORARY TABLE fetchchunks(path text, beginint4, len int4);";
446+
sql="CREATE TEMPORARY TABLE fetchchunks(path text, beginint8, len int4);";
424447
res=PQexec(conn,sql);
425448

426449
if (PQresultStatus(res)!=PGRES_COMMAND_OK)

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp