Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitea4e423

Browse files
committed
Merge branch 'master' into issue_449
2 parentsbbc7aec +758a32f commitea4e423

File tree

18 files changed

+343
-91
lines changed

18 files changed

+343
-91
lines changed

‎doc/pgprobackup.xml‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,7 @@ doc/src/sgml/pgprobackup.sgml
131131
<argchoice="plain"><option>archive-push</option></arg>
132132
<argchoice="plain"><option>-B</option> <replaceable>backup_dir</replaceable></arg>
133133
<argchoice="plain"><option>--instance</option> <replaceable>instance_name</replaceable></arg>
134+
<argchoice="plain"><option>--wal-file-path</option> <replaceable>wal_file_path</replaceable></arg>
134135
<argchoice="plain"><option>--wal-file-name</option> <replaceable>wal_file_name</replaceable></arg>
135136
<argrep="repeat"><replaceable>option</replaceable></arg>
136137
</cmdsynopsis>
@@ -5367,7 +5368,9 @@ pg_probackup catchup -b <replaceable>catchup_mode</replaceable>
53675368
Provides the path to the WAL file in
53685369
<parameter>archive_command</parameter> and
53695370
<parameter>restore_command</parameter>. Use the <literal>%p</literal>
5370-
variable as the value for this option for correct processing.
5371+
variable as the value for this option or explicitly specify the path to a file
5372+
outside of the data directory. If you skip this option, the path
5373+
specified in <filename>pg_probackup.conf</filename> will be used.
53715374
</para>
53725375
</listitem>
53735376
</varlistentry>
@@ -5380,6 +5383,8 @@ pg_probackup catchup -b <replaceable>catchup_mode</replaceable>
53805383
<parameter>archive_command</parameter> and
53815384
<parameter>restore_command</parameter>. Use the <literal>%f</literal>
53825385
variable as the value for this option for correct processing.
5386+
If the value of <option>--wal-file-path</option> is a path
5387+
outside of the data directory, explicitly specify the filename.
53835388
</para>
53845389
</listitem>
53855390
</varlistentry>

‎src/archive.c‎

Lines changed: 14 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* archive.c: - pg_probackup specific archive commands for archive backups.
44
*
55
*
6-
* Portions Copyright (c) 2018-2019, Postgres Professional
6+
* Portions Copyright (c) 2018-2021, Postgres Professional
77
*
88
*-------------------------------------------------------------------------
99
*/
@@ -140,15 +140,13 @@ get_xlogFileType(const char *filename)
140140
* Where archlog_path is $BACKUP_PATH/wal/instance_name
141141
*/
142142
void
143-
do_archive_push(InstanceState*instanceState,InstanceConfig*instance,char*wal_file_path,
143+
do_archive_push(InstanceState*instanceState,InstanceConfig*instance,char*pg_xlog_dir,
144144
char*wal_file_name,intbatch_size,booloverwrite,
145145
boolno_sync,boolno_ready_rename)
146146
{
147147
uint64i;
148-
charcurrent_dir[MAXPGPATH];
149-
charpg_xlog_dir[MAXPGPATH];
150-
chararchive_status_dir[MAXPGPATH];
151-
uint64system_id;
148+
/* usually instance pgdata/pg_wal/archive_status, empty if no_ready_rename or batch_size == 1 */
149+
chararchive_status_dir[MAXPGPATH]="";
152150
boolis_compress= false;
153151

154152
/* arrays with meta info for multi threaded backup */
@@ -169,31 +167,8 @@ do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wa
169167
parray*archive_subdirs=NULL;
170168
intn_threads;
171169

172-
if (wal_file_name==NULL)
173-
elog(ERROR,"Required parameter is not specified: --wal-file-name %%f");
174-
175-
if (!getcwd(current_dir,sizeof(current_dir)))
176-
elog(ERROR,"getcwd() error");
177-
178-
/* verify that archive-push --instance parameter is valid */
179-
system_id=get_system_identifier(current_dir,FIO_DB_HOST);
180-
181-
if (instance->pgdata==NULL)
182-
elog(ERROR,"Cannot read pg_probackup.conf for this instance");
183-
184-
if (system_id!=instance->system_identifier)
185-
elog(ERROR,"Refuse to push WAL segment %s into archive. Instance parameters mismatch."
186-
"Instance '%s' should have SYSTEM_ID = "UINT64_FORMAT" instead of "UINT64_FORMAT,
187-
wal_file_name,instanceState->instance_name,instance->system_identifier,system_id);
188-
189-
if (instance->compress_alg==PGLZ_COMPRESS)
190-
elog(ERROR,"Cannot use pglz for WAL compression");
191-
192-
join_path_components(pg_xlog_dir,current_dir,XLOGDIR);
193-
join_path_components(archive_status_dir,pg_xlog_dir,"archive_status");
194-
195-
/* Create 'archlog_path' directory. Do nothing if it already exists. */
196-
//fio_mkdir(instanceState->instance_wal_subdir_path, DIR_PERMISSION, FIO_BACKUP_HOST);
170+
if (!no_ready_rename||batch_size>1)
171+
join_path_components(archive_status_dir,pg_xlog_dir,"archive_status");
197172

198173
#ifdefHAVE_LIBZ
199174
if (instance->compress_alg==ZLIB_COMPRESS)
@@ -246,12 +221,13 @@ do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wa
246221
{
247222
intrc;
248223
WALSegno*xlogfile= (WALSegno*)parray_get(batch_files,i);
224+
boolfirst_wal=strcmp(xlogfile->name,wal_file_name)==0;
249225

250-
rc=push_file(xlogfile,archive_status_dir,
226+
rc=push_file(xlogfile,first_wal ?NULL :archive_status_dir,
251227
pg_xlog_dir,instanceState->instance_wal_subdir_path,
252228
overwrite,no_sync,
253229
instance->archive_timeout,
254-
no_ready_rename||(strcmp(xlogfile->name,wal_file_name)==0) ? true : false,
230+
no_ready_rename||first_wal,
255231
is_compress&&IsXLogFileName(xlogfile->name) ? true : false,
256232
instance->compress_level);
257233
if (rc==0)
@@ -275,7 +251,7 @@ do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wa
275251
arg->first_filename=wal_file_name;
276252
arg->archive_dir=instanceState->instance_wal_subdir_path;
277253
arg->pg_xlog_dir=pg_xlog_dir;
278-
arg->archive_status_dir=archive_status_dir;
254+
arg->archive_status_dir=(!no_ready_rename||batch_size>1) ?archive_status_dir :NULL;
279255
arg->overwrite=overwrite;
280256
arg->compress=is_compress;
281257
arg->no_sync=no_sync;
@@ -318,7 +294,7 @@ do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wa
318294

319295
/* Note, that we are leaking memory here,
320296
* because pushing into archive is a very
321-
* time-sensetive operation, so we skip freeing stuff.
297+
* time-sensitive operation, so we skip freeing stuff.
322298
*/
323299

324300
push_done:
@@ -398,9 +374,6 @@ push_file(WALSegno *xlogfile, const char *archive_status_dir,
398374
intcompress_level)
399375
{
400376
intrc;
401-
charwal_file_dummy[MAXPGPATH];
402-
403-
join_path_components(wal_file_dummy,archive_status_dir,xlogfile->name);
404377

405378
elog(LOG,"pushing file \"%s\"",xlogfile->name);
406379

@@ -417,11 +390,13 @@ push_file(WALSegno *xlogfile, const char *archive_status_dir,
417390
#endif
418391

419392
/* take '--no-ready-rename' flag into account */
420-
if (!no_ready_rename)
393+
if (!no_ready_rename&&archive_status_dir!=NULL)
421394
{
395+
charwal_file_dummy[MAXPGPATH];
422396
charwal_file_ready[MAXPGPATH];
423397
charwal_file_done[MAXPGPATH];
424398

399+
join_path_components(wal_file_dummy,archive_status_dir,xlogfile->name);
425400
snprintf(wal_file_ready,MAXPGPATH,"%s.%s",wal_file_dummy,"ready");
426401
snprintf(wal_file_done,MAXPGPATH,"%s.%s",wal_file_dummy,"done");
427402

‎src/backup.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -943,7 +943,7 @@ check_system_identifiers(PGconn *conn, const char *pgdata)
943943
uint64system_id_conn;
944944
uint64system_id_pgdata;
945945

946-
system_id_pgdata=get_system_identifier(pgdata,FIO_DB_HOST);
946+
system_id_pgdata=get_system_identifier(pgdata,FIO_DB_HOST, false);
947947
system_id_conn=get_remote_system_identifier(conn);
948948

949949
/* for checkdb check only system_id_pgdata and system_id_conn */

‎src/catchup.c‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ catchup_init_state(PGNodeInfo*source_node_info, const char *source_pgdata, cons
4848

4949
/* Get WAL segments size and system ID of source PG instance */
5050
instance_config.xlog_seg_size=get_xlog_seg_size(source_pgdata);
51-
instance_config.system_identifier=get_system_identifier(source_pgdata,FIO_DB_HOST);
51+
instance_config.system_identifier=get_system_identifier(source_pgdata,FIO_DB_HOST, false);
5252
current.start_time=time(NULL);
5353

5454
strlcpy(current.program_version,PROGRAM_VERSION,sizeof(current.program_version));
@@ -163,15 +163,15 @@ catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn,
163163
uint64source_conn_id,source_id,dest_id;
164164

165165
source_conn_id=get_remote_system_identifier(source_conn);
166-
source_id=get_system_identifier(source_pgdata,FIO_DB_HOST);/* same as instance_config.system_identifier */
166+
source_id=get_system_identifier(source_pgdata,FIO_DB_HOST, false);/* same as instance_config.system_identifier */
167167

168168
if (source_conn_id!=source_id)
169169
elog(ERROR,"Database identifiers mismatch: we connected to DB id %lu, but in \"%s\" we found id %lu",
170170
source_conn_id,source_pgdata,source_id);
171171

172172
if (current.backup_mode!=BACKUP_MODE_FULL)
173173
{
174-
dest_id=get_system_identifier(dest_pgdata,FIO_LOCAL_HOST);
174+
dest_id=get_system_identifier(dest_pgdata,FIO_LOCAL_HOST, false);
175175
if (source_conn_id!=dest_id)
176176
elog(ERROR,"Database identifiers mismatch: we connected to DB id %lu, but in \"%s\" we found id %lu",
177177
source_conn_id,dest_pgdata,dest_id);

‎src/help.c‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,7 @@ help_pg_probackup(void)
227227

228228
printf(_("\n %s archive-push -B backup-path --instance=instance_name\n"),PROGRAM_NAME);
229229
printf(_(" --wal-file-name=wal-file-name\n"));
230+
printf(_(" [--wal-file-path=wal-file-path]\n"));
230231
printf(_(" [-j num-threads] [--batch-size=batch_size]\n"));
231232
printf(_(" [--archive-timeout=timeout]\n"));
232233
printf(_(" [--no-ready-rename] [--no-sync]\n"));
@@ -937,6 +938,7 @@ help_archive_push(void)
937938
{
938939
printf(_("\n%s archive-push -B backup-path --instance=instance_name\n"),PROGRAM_NAME);
939940
printf(_(" --wal-file-name=wal-file-name\n"));
941+
printf(_(" [--wal-file-path=wal-file-path]\n"));
940942
printf(_(" [-j num-threads] [--batch-size=batch_size]\n"));
941943
printf(_(" [--archive-timeout=timeout]\n"));
942944
printf(_(" [--no-ready-rename] [--no-sync]\n"));
@@ -951,6 +953,8 @@ help_archive_push(void)
951953
printf(_(" --instance=instance_name name of the instance to delete\n"));
952954
printf(_(" --wal-file-name=wal-file-name\n"));
953955
printf(_(" name of the file to copy into WAL archive\n"));
956+
printf(_(" --wal-file-path=wal-file-path\n"));
957+
printf(_(" relative destination path of the WAL archive\n"));
954958
printf(_(" -j, --threads=NUM number of parallel threads\n"));
955959
printf(_(" --batch-size=NUM number of files to be copied\n"));
956960
printf(_(" --archive-timeout=timeout wait timeout before discarding stale temp file(default: 5min)\n"));
@@ -981,8 +985,8 @@ static void
981985
help_archive_get(void)
982986
{
983987
printf(_("\n%s archive-get -B backup-path --instance=instance_name\n"),PROGRAM_NAME);
984-
printf(_(" --wal-file-path=wal-file-path\n"));
985988
printf(_(" --wal-file-name=wal-file-name\n"));
989+
printf(_(" [--wal-file-path=wal-file-path]\n"));
986990
printf(_(" [-j num-threads] [--batch-size=batch_size]\n"));
987991
printf(_(" [--no-validate-wal]\n"));
988992
printf(_(" [--remote-proto] [--remote-host]\n"));

‎src/init.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ do_add_instance(InstanceState *instanceState, InstanceConfig *instance)
5757
"(-D, --pgdata)");
5858

5959
/* Read system_identifier from PGDATA */
60-
instance->system_identifier=get_system_identifier(instance->pgdata,FIO_DB_HOST);
60+
instance->system_identifier=get_system_identifier(instance->pgdata,FIO_DB_HOST, false);
6161
/* Starting from PostgreSQL 11 read WAL segment size from PGDATA */
6262
instance->xlog_seg_size=get_xlog_seg_size(instance->pgdata);
6363

‎src/pg_probackup.c‎

Lines changed: 95 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@
3535
* which includes info about pgdata directory and connection.
3636
*
3737
* Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION
38-
* Portions Copyright (c) 2015-2019, Postgres Professional
38+
* Portions Copyright (c) 2015-2021, Postgres Professional
3939
*
4040
*-------------------------------------------------------------------------
4141
*/
@@ -151,6 +151,7 @@ static char *wal_file_path;
151151
staticchar*wal_file_name;
152152
staticboolfile_overwrite= false;
153153
staticboolno_ready_rename= false;
154+
staticchararchive_push_xlog_dir[MAXPGPATH]="";
154155

155156
/* archive get options */
156157
staticchar*prefetch_dir;
@@ -789,14 +790,105 @@ main(int argc, char *argv[])
789790
current.stream=stream_wal= true;
790791
if (instance_config.external_dir_str)
791792
elog(ERROR,"external directories not supported fom \"%s\" command",get_subcmd_name(backup_subcmd));
792-
// TODOпроверить instance_config.conn_opt
793+
// TODOcheck instance_config.conn_opt
793794
}
794795

795796
/* sanity */
796797
if (backup_subcmd==VALIDATE_CMD&&restore_params->no_validate)
797798
elog(ERROR,"You cannot specify \"--no-validate\" option with the \"%s\" command",
798799
get_subcmd_name(backup_subcmd));
799800

801+
if (backup_subcmd==ARCHIVE_PUSH_CMD)
802+
{
803+
/* Check archive-push parameters and construct archive_push_xlog_dir
804+
*
805+
* There are 4 cases:
806+
* 1. no --wal-file-path specified -- use cwd, ./PG_XLOG_DIR for wal files
807+
* (and ./PG_XLOG_DIR/archive_status for .done files inside do_archive_push())
808+
* in this case we can use batches and threads
809+
* 2. --wal-file-path is specified and it is the same dir as stored in pg_probackup.conf (instance_config.pgdata)
810+
* in this case we can use this path, as well as batches and thread
811+
* 3. --wal-file-path is specified and it isn't same dir as stored in pg_probackup.conf but control file present with correct system_id
812+
* in this case we can use this path, as well as batches and thread
813+
* (replica for example, see test_archive_push_sanity)
814+
* 4. --wal-file-path is specified and it is different from instance_config.pgdata and no control file found
815+
* disable optimizations and work with user specified path
816+
*/
817+
boolcheck_system_id= true;
818+
uint64system_id;
819+
charcurrent_dir[MAXPGPATH];
820+
821+
if (wal_file_name==NULL)
822+
elog(ERROR,"Required parameter is not specified: --wal-file-name %%f");
823+
824+
if (instance_config.pgdata==NULL)
825+
elog(ERROR,"Cannot read pg_probackup.conf for this instance");
826+
827+
/* TODO may be remove in preference of checking inside compress_init()? */
828+
if (instance_config.compress_alg==PGLZ_COMPRESS)
829+
elog(ERROR,"Cannot use pglz for WAL compression");
830+
831+
if (!getcwd(current_dir,sizeof(current_dir)))
832+
elog(ERROR,"getcwd() error");
833+
834+
if (wal_file_path==NULL)
835+
{
836+
/* 1st case */
837+
system_id=get_system_identifier(current_dir,FIO_DB_HOST, false);
838+
join_path_components(archive_push_xlog_dir,current_dir,XLOGDIR);
839+
}
840+
else
841+
{
842+
/*
843+
* Usually we get something like
844+
* wal_file_path = "pg_wal/0000000100000000000000A1"
845+
* wal_file_name = "0000000100000000000000A1"
846+
* instance_config.pgdata = "/pgdata/.../node/data"
847+
* We need to strip wal_file_name from wal_file_path, add XLOGDIR to instance_config.pgdata
848+
* and compare this directories.
849+
* Note, that pg_wal can be symlink (see test_waldir_outside_pgdata_archiving)
850+
*/
851+
char*stripped_wal_file_path=pgut_str_strip_trailing_filename(wal_file_path,wal_file_name);
852+
join_path_components(archive_push_xlog_dir,instance_config.pgdata,XLOGDIR);
853+
if (fio_is_same_file(stripped_wal_file_path,archive_push_xlog_dir, true,FIO_DB_HOST))
854+
{
855+
/* 2nd case */
856+
system_id=get_system_identifier(instance_config.pgdata,FIO_DB_HOST, false);
857+
/* archive_push_xlog_dir already have right value */
858+
}
859+
else
860+
{
861+
if (strlen(stripped_wal_file_path)<MAXPGPATH)
862+
strncpy(archive_push_xlog_dir,stripped_wal_file_path,MAXPGPATH);
863+
else
864+
elog(ERROR,"Value specified to --wal_file_path is too long");
865+
866+
system_id=get_system_identifier(current_dir,FIO_DB_HOST, true);
867+
/* 3rd case if control file present -- i.e. system_id != 0 */
868+
869+
if (system_id==0)
870+
{
871+
/* 4th case */
872+
check_system_id= false;
873+
874+
if (batch_size>1||num_threads>1|| !no_ready_rename)
875+
{
876+
elog(WARNING,"Supplied --wal_file_path is outside pgdata, force safe values for options: --batch-size=1 -j 1 --no-ready-rename");
877+
batch_size=1;
878+
num_threads=1;
879+
no_ready_rename= true;
880+
}
881+
}
882+
}
883+
pfree(stripped_wal_file_path);
884+
}
885+
886+
if (check_system_id&&system_id!=instance_config.system_identifier)
887+
elog(ERROR,"Refuse to push WAL segment %s into archive. Instance parameters mismatch."
888+
"Instance '%s' should have SYSTEM_ID = "UINT64_FORMAT" instead of "UINT64_FORMAT,
889+
wal_file_name,instanceState->instance_name,instance_config.system_identifier,system_id);
890+
}
891+
800892
#ifPG_VERSION_NUM >=100000
801893
if (temp_slot&&perm_slot)
802894
elog(ERROR,"You cannot specify \"--perm-slot\" option with the \"--temp-slot\" option");
@@ -820,7 +912,7 @@ main(int argc, char *argv[])
820912
switch (backup_subcmd)
821913
{
822914
caseARCHIVE_PUSH_CMD:
823-
do_archive_push(instanceState,&instance_config,wal_file_path,wal_file_name,
915+
do_archive_push(instanceState,&instance_config,archive_push_xlog_dir,wal_file_name,
824916
batch_size,file_overwrite,no_sync,no_ready_rename);
825917
break;
826918
caseARCHIVE_GET_CMD:

‎src/pg_probackup.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -895,7 +895,7 @@ extern int do_init(CatalogState *catalogState);
895895
externintdo_add_instance(InstanceState*instanceState,InstanceConfig*instance);
896896

897897
/* in archive.c */
898-
externvoiddo_archive_push(InstanceState*instanceState,InstanceConfig*instance,char*wal_file_path,
898+
externvoiddo_archive_push(InstanceState*instanceState,InstanceConfig*instance,char*pg_xlog_dir,
899899
char*wal_file_name,intbatch_size,booloverwrite,
900900
boolno_sync,boolno_ready_rename);
901901
externvoiddo_archive_get(InstanceState*instanceState,InstanceConfig*instance,constchar*prefetch_dir_arg,char*wal_file_path,
@@ -1163,7 +1163,7 @@ extern XLogRecPtr get_next_record_lsn(const char *archivedir, XLogSegNosegno, T
11631163
externTimeLineIDget_current_timeline(PGconn*conn);
11641164
externTimeLineIDget_current_timeline_from_control(constchar*pgdata_path,fio_locationlocation,boolsafe);
11651165
externXLogRecPtrget_checkpoint_location(PGconn*conn);
1166-
externuint64get_system_identifier(constchar*pgdata_path,fio_locationlocation);
1166+
externuint64get_system_identifier(constchar*pgdata_path,fio_locationlocation,boolsafe);
11671167
externuint64get_remote_system_identifier(PGconn*conn);
11681168
externuint32get_data_checksum_version(boolsafe);
11691169
externpg_crc32cget_pgcontrol_checksum(constchar*pgdata_path);

‎src/restore.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2186,7 +2186,7 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier,
21862186
*/
21872187
elog(INFO,"Trying to read pg_control file in destination directory");
21882188

2189-
system_id_pgdata=get_system_identifier(pgdata,FIO_DB_HOST);
2189+
system_id_pgdata=get_system_identifier(pgdata,FIO_DB_HOST, false);
21902190

21912191
if (system_id_pgdata==instance_config.system_identifier)
21922192
system_id_match= true;

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp