Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitb87ca18

Browse files
dlepikhovakulaginmindrups
authored
[Issue#265][PGPRO-5421] archive-push backward compatibility (#437)
Restore the --wal-file-path option of the archive-push command (it was ignored sincea196073) Co-authored-by: Mikhail A. Kulagin <m.kulagin@postgrespro.ru>Co-authored-by: Elena Indrupskaya <e.indrupskaya@postgrespro.ru>
1 parent5b6ca62 commitb87ca18

17 files changed

+338
-86
lines changed

‎doc/pgprobackup.xml

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,7 @@ doc/src/sgml/pgprobackup.sgml
131131
<argchoice="plain"><option>archive-push</option></arg>
132132
<argchoice="plain"><option>-B</option> <replaceable>backup_dir</replaceable></arg>
133133
<argchoice="plain"><option>--instance</option> <replaceable>instance_name</replaceable></arg>
134+
<argchoice="plain"><option>--wal-file-path</option> <replaceable>wal_file_path</replaceable></arg>
134135
<argchoice="plain"><option>--wal-file-name</option> <replaceable>wal_file_name</replaceable></arg>
135136
<argrep="repeat"><replaceable>option</replaceable></arg>
136137
</cmdsynopsis>
@@ -5367,7 +5368,9 @@ pg_probackup catchup -b <replaceable>catchup_mode</replaceable>
53675368
Provides the path to the WAL file in
53685369
<parameter>archive_command</parameter> and
53695370
<parameter>restore_command</parameter>. Use the <literal>%p</literal>
5370-
variable as the value for this option for correct processing.
5371+
variable as the value for this option or explicitly specify the path to a file
5372+
outside of the data directory. If you skip this option, the path
5373+
specified in <filename>pg_probackup.conf</filename> will be used.
53715374
</para>
53725375
</listitem>
53735376
</varlistentry>
@@ -5380,6 +5383,8 @@ pg_probackup catchup -b <replaceable>catchup_mode</replaceable>
53805383
<parameter>archive_command</parameter> and
53815384
<parameter>restore_command</parameter>. Use the <literal>%f</literal>
53825385
variable as the value for this option for correct processing.
5386+
If the value of <option>--wal-file-path</option> is a path
5387+
outside of the data directory, explicitly specify the filename.
53835388
</para>
53845389
</listitem>
53855390
</varlistentry>

‎src/archive.c

Lines changed: 14 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* archive.c: - pg_probackup specific archive commands for archive backups.
44
*
55
*
6-
* Portions Copyright (c) 2018-2019, Postgres Professional
6+
* Portions Copyright (c) 2018-2021, Postgres Professional
77
*
88
*-------------------------------------------------------------------------
99
*/
@@ -113,15 +113,13 @@ static parray *setup_push_filelist(const char *archive_status_dir,
113113
* Where archlog_path is $BACKUP_PATH/wal/instance_name
114114
*/
115115
void
116-
do_archive_push(InstanceState*instanceState,InstanceConfig*instance,char*wal_file_path,
116+
do_archive_push(InstanceState*instanceState,InstanceConfig*instance,char*pg_xlog_dir,
117117
char*wal_file_name,intbatch_size,booloverwrite,
118118
boolno_sync,boolno_ready_rename)
119119
{
120120
uint64i;
121-
charcurrent_dir[MAXPGPATH];
122-
charpg_xlog_dir[MAXPGPATH];
123-
chararchive_status_dir[MAXPGPATH];
124-
uint64system_id;
121+
/* usually instance pgdata/pg_wal/archive_status, empty if no_ready_rename or batch_size == 1 */
122+
chararchive_status_dir[MAXPGPATH]="";
125123
boolis_compress= false;
126124

127125
/* arrays with meta info for multi threaded backup */
@@ -141,31 +139,8 @@ do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wa
141139
parray*batch_files=NULL;
142140
intn_threads;
143141

144-
if (wal_file_name==NULL)
145-
elog(ERROR,"Required parameter is not specified: --wal-file-name %%f");
146-
147-
if (!getcwd(current_dir,sizeof(current_dir)))
148-
elog(ERROR,"getcwd() error");
149-
150-
/* verify that archive-push --instance parameter is valid */
151-
system_id=get_system_identifier(current_dir,FIO_DB_HOST);
152-
153-
if (instance->pgdata==NULL)
154-
elog(ERROR,"Cannot read pg_probackup.conf for this instance");
155-
156-
if (system_id!=instance->system_identifier)
157-
elog(ERROR,"Refuse to push WAL segment %s into archive. Instance parameters mismatch."
158-
"Instance '%s' should have SYSTEM_ID = "UINT64_FORMAT" instead of "UINT64_FORMAT,
159-
wal_file_name,instanceState->instance_name,instance->system_identifier,system_id);
160-
161-
if (instance->compress_alg==PGLZ_COMPRESS)
162-
elog(ERROR,"Cannot use pglz for WAL compression");
163-
164-
join_path_components(pg_xlog_dir,current_dir,XLOGDIR);
165-
join_path_components(archive_status_dir,pg_xlog_dir,"archive_status");
166-
167-
/* Create 'archlog_path' directory. Do nothing if it already exists. */
168-
//fio_mkdir(instanceState->instance_wal_subdir_path, DIR_PERMISSION, FIO_BACKUP_HOST);
142+
if (!no_ready_rename||batch_size>1)
143+
join_path_components(archive_status_dir,pg_xlog_dir,"archive_status");
169144

170145
#ifdefHAVE_LIBZ
171146
if (instance->compress_alg==ZLIB_COMPRESS)
@@ -204,12 +179,13 @@ do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wa
204179
{
205180
intrc;
206181
WALSegno*xlogfile= (WALSegno*)parray_get(batch_files,i);
182+
boolfirst_wal=strcmp(xlogfile->name,wal_file_name)==0;
207183

208-
rc=push_file(xlogfile,archive_status_dir,
184+
rc=push_file(xlogfile,first_wal ?NULL :archive_status_dir,
209185
pg_xlog_dir,instanceState->instance_wal_subdir_path,
210186
overwrite,no_sync,
211187
instance->archive_timeout,
212-
no_ready_rename||(strcmp(xlogfile->name,wal_file_name)==0) ? true : false,
188+
no_ready_rename||first_wal,
213189
is_compress&&IsXLogFileName(xlogfile->name) ? true : false,
214190
instance->compress_level);
215191
if (rc==0)
@@ -233,7 +209,7 @@ do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wa
233209
arg->first_filename=wal_file_name;
234210
arg->archive_dir=instanceState->instance_wal_subdir_path;
235211
arg->pg_xlog_dir=pg_xlog_dir;
236-
arg->archive_status_dir=archive_status_dir;
212+
arg->archive_status_dir=(!no_ready_rename||batch_size>1) ?archive_status_dir :NULL;
237213
arg->overwrite=overwrite;
238214
arg->compress=is_compress;
239215
arg->no_sync=no_sync;
@@ -276,7 +252,7 @@ do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wa
276252

277253
/* Note, that we are leaking memory here,
278254
* because pushing into archive is a very
279-
* time-sensetive operation, so we skip freeing stuff.
255+
* time-sensitive operation, so we skip freeing stuff.
280256
*/
281257

282258
push_done:
@@ -356,9 +332,6 @@ push_file(WALSegno *xlogfile, const char *archive_status_dir,
356332
intcompress_level)
357333
{
358334
intrc;
359-
charwal_file_dummy[MAXPGPATH];
360-
361-
join_path_components(wal_file_dummy,archive_status_dir,xlogfile->name);
362335

363336
elog(LOG,"pushing file \"%s\"",xlogfile->name);
364337

@@ -375,11 +348,13 @@ push_file(WALSegno *xlogfile, const char *archive_status_dir,
375348
#endif
376349

377350
/* take '--no-ready-rename' flag into account */
378-
if (!no_ready_rename)
351+
if (!no_ready_rename&&archive_status_dir!=NULL)
379352
{
353+
charwal_file_dummy[MAXPGPATH];
380354
charwal_file_ready[MAXPGPATH];
381355
charwal_file_done[MAXPGPATH];
382356

357+
join_path_components(wal_file_dummy,archive_status_dir,xlogfile->name);
383358
snprintf(wal_file_ready,MAXPGPATH,"%s.%s",wal_file_dummy,"ready");
384359
snprintf(wal_file_done,MAXPGPATH,"%s.%s",wal_file_dummy,"done");
385360

‎src/backup.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -943,7 +943,7 @@ check_system_identifiers(PGconn *conn, const char *pgdata)
943943
uint64system_id_conn;
944944
uint64system_id_pgdata;
945945

946-
system_id_pgdata=get_system_identifier(pgdata,FIO_DB_HOST);
946+
system_id_pgdata=get_system_identifier(pgdata,FIO_DB_HOST, false);
947947
system_id_conn=get_remote_system_identifier(conn);
948948

949949
/* for checkdb check only system_id_pgdata and system_id_conn */

‎src/catchup.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ catchup_init_state(PGNodeInfo*source_node_info, const char *source_pgdata, cons
4848

4949
/* Get WAL segments size and system ID of source PG instance */
5050
instance_config.xlog_seg_size=get_xlog_seg_size(source_pgdata);
51-
instance_config.system_identifier=get_system_identifier(source_pgdata,FIO_DB_HOST);
51+
instance_config.system_identifier=get_system_identifier(source_pgdata,FIO_DB_HOST, false);
5252
current.start_time=time(NULL);
5353

5454
strlcpy(current.program_version,PROGRAM_VERSION,sizeof(current.program_version));
@@ -163,15 +163,15 @@ catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn,
163163
uint64source_conn_id,source_id,dest_id;
164164

165165
source_conn_id=get_remote_system_identifier(source_conn);
166-
source_id=get_system_identifier(source_pgdata,FIO_DB_HOST);/* same as instance_config.system_identifier */
166+
source_id=get_system_identifier(source_pgdata,FIO_DB_HOST, false);/* same as instance_config.system_identifier */
167167

168168
if (source_conn_id!=source_id)
169169
elog(ERROR,"Database identifiers mismatch: we connected to DB id %lu, but in \"%s\" we found id %lu",
170170
source_conn_id,source_pgdata,source_id);
171171

172172
if (current.backup_mode!=BACKUP_MODE_FULL)
173173
{
174-
dest_id=get_system_identifier(dest_pgdata,FIO_LOCAL_HOST);
174+
dest_id=get_system_identifier(dest_pgdata,FIO_LOCAL_HOST, false);
175175
if (source_conn_id!=dest_id)
176176
elog(ERROR,"Database identifiers mismatch: we connected to DB id %lu, but in \"%s\" we found id %lu",
177177
source_conn_id,dest_pgdata,dest_id);

‎src/help.c

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,7 @@ help_pg_probackup(void)
227227

228228
printf(_("\n %s archive-push -B backup-path --instance=instance_name\n"),PROGRAM_NAME);
229229
printf(_(" --wal-file-name=wal-file-name\n"));
230+
printf(_(" [--wal-file-path=wal-file-path]\n"));
230231
printf(_(" [-j num-threads] [--batch-size=batch_size]\n"));
231232
printf(_(" [--archive-timeout=timeout]\n"));
232233
printf(_(" [--no-ready-rename] [--no-sync]\n"));
@@ -937,6 +938,7 @@ help_archive_push(void)
937938
{
938939
printf(_("\n%s archive-push -B backup-path --instance=instance_name\n"),PROGRAM_NAME);
939940
printf(_(" --wal-file-name=wal-file-name\n"));
941+
printf(_(" [--wal-file-path=wal-file-path]\n"));
940942
printf(_(" [-j num-threads] [--batch-size=batch_size]\n"));
941943
printf(_(" [--archive-timeout=timeout]\n"));
942944
printf(_(" [--no-ready-rename] [--no-sync]\n"));
@@ -951,6 +953,8 @@ help_archive_push(void)
951953
printf(_(" --instance=instance_name name of the instance to delete\n"));
952954
printf(_(" --wal-file-name=wal-file-name\n"));
953955
printf(_(" name of the file to copy into WAL archive\n"));
956+
printf(_(" --wal-file-path=wal-file-path\n"));
957+
printf(_(" relative destination path of the WAL archive\n"));
954958
printf(_(" -j, --threads=NUM number of parallel threads\n"));
955959
printf(_(" --batch-size=NUM number of files to be copied\n"));
956960
printf(_(" --archive-timeout=timeout wait timeout before discarding stale temp file(default: 5min)\n"));
@@ -981,8 +985,8 @@ static void
981985
help_archive_get(void)
982986
{
983987
printf(_("\n%s archive-get -B backup-path --instance=instance_name\n"),PROGRAM_NAME);
984-
printf(_(" --wal-file-path=wal-file-path\n"));
985988
printf(_(" --wal-file-name=wal-file-name\n"));
989+
printf(_(" [--wal-file-path=wal-file-path]\n"));
986990
printf(_(" [-j num-threads] [--batch-size=batch_size]\n"));
987991
printf(_(" [--no-validate-wal]\n"));
988992
printf(_(" [--remote-proto] [--remote-host]\n"));

‎src/init.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ do_add_instance(InstanceState *instanceState, InstanceConfig *instance)
5757
"(-D, --pgdata)");
5858

5959
/* Read system_identifier from PGDATA */
60-
instance->system_identifier=get_system_identifier(instance->pgdata,FIO_DB_HOST);
60+
instance->system_identifier=get_system_identifier(instance->pgdata,FIO_DB_HOST, false);
6161
/* Starting from PostgreSQL 11 read WAL segment size from PGDATA */
6262
instance->xlog_seg_size=get_xlog_seg_size(instance->pgdata);
6363

‎src/pg_probackup.c

Lines changed: 95 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@
3535
* which includes info about pgdata directory and connection.
3636
*
3737
* Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION
38-
* Portions Copyright (c) 2015-2019, Postgres Professional
38+
* Portions Copyright (c) 2015-2021, Postgres Professional
3939
*
4040
*-------------------------------------------------------------------------
4141
*/
@@ -151,6 +151,7 @@ static char *wal_file_path;
151151
staticchar*wal_file_name;
152152
staticboolfile_overwrite= false;
153153
staticboolno_ready_rename= false;
154+
staticchararchive_push_xlog_dir[MAXPGPATH]="";
154155

155156
/* archive get options */
156157
staticchar*prefetch_dir;
@@ -788,14 +789,105 @@ main(int argc, char *argv[])
788789
current.stream=stream_wal= true;
789790
if (instance_config.external_dir_str)
790791
elog(ERROR,"external directories not supported fom \"%s\" command",get_subcmd_name(backup_subcmd));
791-
// TODOпроверить instance_config.conn_opt
792+
// TODOcheck instance_config.conn_opt
792793
}
793794

794795
/* sanity */
795796
if (backup_subcmd==VALIDATE_CMD&&restore_params->no_validate)
796797
elog(ERROR,"You cannot specify \"--no-validate\" option with the \"%s\" command",
797798
get_subcmd_name(backup_subcmd));
798799

800+
if (backup_subcmd==ARCHIVE_PUSH_CMD)
801+
{
802+
/* Check archive-push parameters and construct archive_push_xlog_dir
803+
*
804+
* There are 4 cases:
805+
* 1. no --wal-file-path specified -- use cwd, ./PG_XLOG_DIR for wal files
806+
* (and ./PG_XLOG_DIR/archive_status for .done files inside do_archive_push())
807+
* in this case we can use batches and threads
808+
* 2. --wal-file-path is specified and it is the same dir as stored in pg_probackup.conf (instance_config.pgdata)
809+
* in this case we can use this path, as well as batches and thread
810+
* 3. --wal-file-path is specified and it isn't same dir as stored in pg_probackup.conf but control file present with correct system_id
811+
* in this case we can use this path, as well as batches and thread
812+
* (replica for example, see test_archive_push_sanity)
813+
* 4. --wal-file-path is specified and it is different from instance_config.pgdata and no control file found
814+
* disable optimizations and work with user specified path
815+
*/
816+
boolcheck_system_id= true;
817+
uint64system_id;
818+
charcurrent_dir[MAXPGPATH];
819+
820+
if (wal_file_name==NULL)
821+
elog(ERROR,"Required parameter is not specified: --wal-file-name %%f");
822+
823+
if (instance_config.pgdata==NULL)
824+
elog(ERROR,"Cannot read pg_probackup.conf for this instance");
825+
826+
/* TODO may be remove in preference of checking inside compress_init()? */
827+
if (instance_config.compress_alg==PGLZ_COMPRESS)
828+
elog(ERROR,"Cannot use pglz for WAL compression");
829+
830+
if (!getcwd(current_dir,sizeof(current_dir)))
831+
elog(ERROR,"getcwd() error");
832+
833+
if (wal_file_path==NULL)
834+
{
835+
/* 1st case */
836+
system_id=get_system_identifier(current_dir,FIO_DB_HOST, false);
837+
join_path_components(archive_push_xlog_dir,current_dir,XLOGDIR);
838+
}
839+
else
840+
{
841+
/*
842+
* Usually we get something like
843+
* wal_file_path = "pg_wal/0000000100000000000000A1"
844+
* wal_file_name = "0000000100000000000000A1"
845+
* instance_config.pgdata = "/pgdata/.../node/data"
846+
* We need to strip wal_file_name from wal_file_path, add XLOGDIR to instance_config.pgdata
847+
* and compare this directories.
848+
* Note, that pg_wal can be symlink (see test_waldir_outside_pgdata_archiving)
849+
*/
850+
char*stripped_wal_file_path=pgut_str_strip_trailing_filename(wal_file_path,wal_file_name);
851+
join_path_components(archive_push_xlog_dir,instance_config.pgdata,XLOGDIR);
852+
if (fio_is_same_file(stripped_wal_file_path,archive_push_xlog_dir, true,FIO_DB_HOST))
853+
{
854+
/* 2nd case */
855+
system_id=get_system_identifier(instance_config.pgdata,FIO_DB_HOST, false);
856+
/* archive_push_xlog_dir already have right value */
857+
}
858+
else
859+
{
860+
if (strlen(stripped_wal_file_path)<MAXPGPATH)
861+
strncpy(archive_push_xlog_dir,stripped_wal_file_path,MAXPGPATH);
862+
else
863+
elog(ERROR,"Value specified to --wal_file_path is too long");
864+
865+
system_id=get_system_identifier(current_dir,FIO_DB_HOST, true);
866+
/* 3rd case if control file present -- i.e. system_id != 0 */
867+
868+
if (system_id==0)
869+
{
870+
/* 4th case */
871+
check_system_id= false;
872+
873+
if (batch_size>1||num_threads>1|| !no_ready_rename)
874+
{
875+
elog(WARNING,"Supplied --wal_file_path is outside pgdata, force safe values for options: --batch-size=1 -j 1 --no-ready-rename");
876+
batch_size=1;
877+
num_threads=1;
878+
no_ready_rename= true;
879+
}
880+
}
881+
}
882+
pfree(stripped_wal_file_path);
883+
}
884+
885+
if (check_system_id&&system_id!=instance_config.system_identifier)
886+
elog(ERROR,"Refuse to push WAL segment %s into archive. Instance parameters mismatch."
887+
"Instance '%s' should have SYSTEM_ID = "UINT64_FORMAT" instead of "UINT64_FORMAT,
888+
wal_file_name,instanceState->instance_name,instance_config.system_identifier,system_id);
889+
}
890+
799891
#ifPG_VERSION_NUM >=100000
800892
if (temp_slot&&perm_slot)
801893
elog(ERROR,"You cannot specify \"--perm-slot\" option with the \"--temp-slot\" option");
@@ -819,7 +911,7 @@ main(int argc, char *argv[])
819911
switch (backup_subcmd)
820912
{
821913
caseARCHIVE_PUSH_CMD:
822-
do_archive_push(instanceState,&instance_config,wal_file_path,wal_file_name,
914+
do_archive_push(instanceState,&instance_config,archive_push_xlog_dir,wal_file_name,
823915
batch_size,file_overwrite,no_sync,no_ready_rename);
824916
break;
825917
caseARCHIVE_GET_CMD:

‎src/pg_probackup.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -889,7 +889,7 @@ extern int do_init(CatalogState *catalogState);
889889
externintdo_add_instance(InstanceState*instanceState,InstanceConfig*instance);
890890

891891
/* in archive.c */
892-
externvoiddo_archive_push(InstanceState*instanceState,InstanceConfig*instance,char*wal_file_path,
892+
externvoiddo_archive_push(InstanceState*instanceState,InstanceConfig*instance,char*pg_xlog_dir,
893893
char*wal_file_name,intbatch_size,booloverwrite,
894894
boolno_sync,boolno_ready_rename);
895895
externvoiddo_archive_get(InstanceState*instanceState,InstanceConfig*instance,constchar*prefetch_dir_arg,char*wal_file_path,
@@ -1153,7 +1153,7 @@ extern XLogRecPtr get_next_record_lsn(const char *archivedir, XLogSegNosegno, T
11531153
externTimeLineIDget_current_timeline(PGconn*conn);
11541154
externTimeLineIDget_current_timeline_from_control(constchar*pgdata_path,fio_locationlocation,boolsafe);
11551155
externXLogRecPtrget_checkpoint_location(PGconn*conn);
1156-
externuint64get_system_identifier(constchar*pgdata_path,fio_locationlocation);
1156+
externuint64get_system_identifier(constchar*pgdata_path,fio_locationlocation,boolsafe);
11571157
externuint64get_remote_system_identifier(PGconn*conn);
11581158
externuint32get_data_checksum_version(boolsafe);
11591159
externpg_crc32cget_pgcontrol_checksum(constchar*pgdata_path);

‎src/restore.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2186,7 +2186,7 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier,
21862186
*/
21872187
elog(INFO,"Trying to read pg_control file in destination directory");
21882188

2189-
system_id_pgdata=get_system_identifier(pgdata,FIO_DB_HOST);
2189+
system_id_pgdata=get_system_identifier(pgdata,FIO_DB_HOST, false);
21902190

21912191
if (system_id_pgdata==instance_config.system_identifier)
21922192
system_id_match= true;

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp