Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commiteac3d8a

Browse files
committed
Merge branch 'master' into issue_92
2 parents265837b +0e99b6f commiteac3d8a

File tree

11 files changed

+238
-76
lines changed

11 files changed

+238
-76
lines changed

‎Documentation.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ Current version - 2.1.3
107107

108108
As compared to other backup solutions, pg_probackup offers the following benefits that can help you implement different backup strategies and deal with large amounts of data:
109109

110-
- Incremental backup: page-level incremental backup allows yousave to disk space, speed up backup and restore. With three different incremental modes you can plan the backup strategy in accordance with your data flow
110+
- Incremental backup: page-level incremental backup allows youto save disk space, speed up backup and restore. With three different incremental modes you can plan the backup strategy in accordance with your data flow
111111
- Validation: Automatic data consistency checks and on-demand backup validation without actual data recovery
112112
- Verification: On-demand verification of PostgreSQL instance via dedicated command`checkdb`
113113
- Retention: Managing backups in accordance with retention policies - Time and/or Redundancy based, with two retention methods:`delete expired` and`merge expired`
@@ -778,7 +778,7 @@ Specifies remote host user for SSH connection. If you omit this option, the curr
778778
Specifies pg_probackup installation directory on the remote system.
779779

780780
--ssh-options
781-
Specifies a string of SSH command-line options.
781+
Specifies a string of SSH command-line options. For example, the following options can used to set keep-alive for ssh connections opened by pg_probackup:`--ssh-options='-o ServerAliveCountMax=5 -o ServerAliveInterval=60'`. Full list of possible options can be found here: (https://linux.die.net/man/5/ssh_config)[https://linux.die.net/man/5/ssh_config]
782782

783783
####Replica Options
784784

‎src/backup.c‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -344,6 +344,9 @@ do_backup_instance(PGconn *backup_conn)
344344
dir_list_file(backup_files_list,parray_get(external_dirs,i),
345345
false, true, false,i+1,FIO_DB_HOST);
346346

347+
/* close ssh session in main thread */
348+
fio_disconnect();
349+
347350
/* Sanity check for backup_files_list, thank you, Windows:
348351
* https://github.com/postgrespro/pg_probackup/issues/48
349352
*/
@@ -512,6 +515,9 @@ do_backup_instance(PGconn *backup_conn)
512515
parray_free(prev_backup_filelist);
513516
}
514517

518+
/* Notify end of backup */
519+
pg_stop_backup(&current,pg_startbackup_conn);
520+
515521
/* In case of backup from replica >= 9.6 we must fix minRecPoint,
516522
* First we must find pg_control in backup_files_list.
517523
*/
@@ -532,13 +538,16 @@ do_backup_instance(PGconn *backup_conn)
532538
break;
533539
}
534540
}
535-
}
536541

537-
/* Notify end of backup */
538-
pg_stop_backup(&current,pg_startbackup_conn);
542+
if (!pg_control)
543+
elog(ERROR,"Failed to find file \"%s\" in backup filelist.",
544+
pg_control_path);
539545

540-
if (current.from_replica&& !exclusive_backup)
541546
set_min_recovery_point(pg_control,database_path,current.stop_lsn);
547+
}
548+
549+
/* close ssh session in main thread */
550+
fio_disconnect();
542551

543552
/* Add archived xlog files into the list of files of this backup */
544553
if (stream_wal)
@@ -2143,6 +2152,9 @@ backup_files(void *arg)
21432152
elog(WARNING,"unexpected file type %d",buf.st_mode);
21442153
}
21452154

2155+
/* ssh connection to longer needed */
2156+
fio_disconnect();
2157+
21462158
/* Close connection */
21472159
if (arguments->conn_arg.conn)
21482160
pgut_disconnect(arguments->conn_arg.conn);

‎src/pg_probackup.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -578,6 +578,7 @@ extern bool in_backup_list(parray *backup_list, pgBackup *target_backup);
578578
externintget_backup_index_number(parray*backup_list,pgBackup*backup);
579579
externboollaunch_agent(void);
580580
externvoidlaunch_ssh(char*argv[]);
581+
externvoidwait_ssh(void);
581582

582583
#defineCOMPRESS_ALG_DEFAULT NOT_DEFINED_COMPRESS
583584
#defineCOMPRESS_LEVEL_DEFAULT 1

‎src/utils/file.c‎

Lines changed: 36 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -333,21 +333,52 @@ int fio_open(char const* path, int mode, fio_location location)
333333
returnfd;
334334
}
335335

336+
337+
/* Close ssh session */
338+
void
339+
fio_disconnect(void)
340+
{
341+
if (fio_stdin)
342+
{
343+
SYS_CHECK(close(fio_stdin));
344+
SYS_CHECK(close(fio_stdout));
345+
fio_stdin=0;
346+
fio_stdout=0;
347+
wait_ssh();
348+
}
349+
}
350+
336351
/* Open stdio file */
337352
FILE*fio_fopen(charconst*path,charconst*mode,fio_locationlocation)
338353
{
339354
FILE*f=NULL;
340355

341356
if (fio_is_remote(location))
342357
{
343-
intflags=O_RDWR|O_CREAT;
358+
intflags=0;
344359
intfd;
345360
if (strcmp(mode,PG_BINARY_W)==0) {
346-
flags |=O_TRUNC|PG_BINARY;
347-
}elseif (strncmp(mode,PG_BINARY_R,strlen(PG_BINARY_R))==0) {
348-
flags |=PG_BINARY;
361+
flags=O_TRUNC|PG_BINARY|O_RDWR|O_CREAT;
362+
}elseif (strcmp(mode,"w")==0) {
363+
flags=O_TRUNC|O_RDWR|O_CREAT;
364+
}elseif (strcmp(mode,PG_BINARY_R)==0) {
365+
flags=O_RDONLY|PG_BINARY;
366+
}elseif (strcmp(mode,"r")==0) {
367+
flags=O_RDONLY;
368+
}elseif (strcmp(mode,PG_BINARY_R"+")==0) {
369+
/* stdio fopen("rb+") actually doesn't create unexisted file, but probackup frequently
370+
* needs to open existed file or create new one if not exists.
371+
* In stdio it can be done using two fopen calls: fopen("r+") and if failed then fopen("w").
372+
* But to eliminate extra call which especially critical in case of remote connection
373+
* we change r+ semantic to create file if not exists.
374+
*/
375+
flags=O_RDWR|O_CREAT|PG_BINARY;
376+
}elseif (strcmp(mode,"r+")==0) {/* see comment above */
377+
flags |=O_RDWR|O_CREAT;
349378
}elseif (strcmp(mode,"a")==0) {
350-
flags |=O_APPEND;
379+
flags |=O_CREAT|O_RDWR|O_APPEND;
380+
}else {
381+
Assert(false);
351382
}
352383
fd=fio_open(path,flags,location);
353384
if (fd >=0)

‎src/utils/file.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@ extern int fio_seek(int fd, off_t offs);
9090
externintfio_fstat(intfd,structstat*st);
9191
externintfio_truncate(intfd,off_tsize);
9292
externintfio_close(intfd);
93+
externvoidfio_disconnect(void);
9394

9495
externintfio_rename(charconst*old_path,charconst*new_path,fio_locationlocation);
9596
externintfio_symlink(charconst*target,charconst*link_path,fio_locationlocation);

‎src/utils/remote.c‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,12 @@
55
#include<sys/wait.h>
66
#include<signal.h>
77

8+
#ifdefWIN32
9+
#define__thread __declspec(thread)
10+
#else
11+
#include<pthread.h>
12+
#endif
13+
814
#include"pg_probackup.h"
915
#include"file.h"
1016

@@ -52,14 +58,23 @@ static int split_options(int argc, char* argv[], int max_options, char* options)
5258
returnargc;
5359
}
5460

55-
staticintchild_pid;
61+
static __threadintchild_pid;
62+
5663
#if0
5764
staticvoidkill_child(void)
5865
{
5966
kill(child_pid,SIGTERM);
6067
}
6168
#endif
6269

70+
71+
voidwait_ssh(void)
72+
{
73+
intstatus;
74+
waitpid(child_pid,&status,0);
75+
elog(LOG,"SSH process %d is terminated with status %d",child_pid,status);
76+
}
77+
6378
#ifdefWIN32
6479
voidlaunch_ssh(char*argv[])
6580
{

‎tests/__init__.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,9 @@
1212
defload_tests(loader,tests,pattern):
1313
suite=unittest.TestSuite()
1414

15-
ifos.environ['PG_PROBACKUP_TEST_BASIC']=='ON':
16-
loader.testMethodPrefix='test_basic'
15+
if'PG_PROBACKUP_TEST_BASIC'inos.environ:
16+
ifos.environ['PG_PROBACKUP_TEST_BASIC']=='ON':
17+
loader.testMethodPrefix='test_basic'
1718

1819
# suite.addTests(loader.loadTestsFromModule(auth_test))
1920
suite.addTests(loader.loadTestsFromModule(archive))

‎tests/archive.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -452,22 +452,23 @@ def test_archive_push_partial_file_exists(self):
452452
filename=filename_orig+'.partial'
453453
file=os.path.join(wals_dir,filename)
454454

455+
# emulate stale .partial file
455456
withopen(file,'a')asf:
456457
f.write(b"blahblah")
457458
f.flush()
458459
f.close()
459460

460461
self.switch_wal_segment(node)
461-
sleep(15)
462+
sleep(20)
462463

463464
# check that segment is archived
464465
ifself.archive_compress:
465466
filename_orig=filename_orig+'.gz'
466467

467468
file=os.path.join(wals_dir,filename_orig)
468-
469469
self.assertTrue(os.path.isfile(file))
470470

471+
# successful validate means that archive-push reused stale wal segment
471472
self.validate_pb(
472473
backup_dir,'node',
473474
options=['--recovery-target-xid={0}'.format(xid)])

‎tests/backup.py‎

Lines changed: 72 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -469,16 +469,74 @@ def test_backup_detect_corruption(self):
469469
"\n Output: {0}\n CMD: {1}".format(
470470
repr(self.output),self.cmd))
471471
exceptProbackupExceptionase:
472-
self.assertIn(
473-
'WARNING: Corruption detected in file',
474-
e.message,
475-
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
476-
repr(e.message),self.cmd))
477-
self.assertIn(
478-
'ERROR: Data file corruption',
479-
e.message,
480-
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
481-
repr(e.message),self.cmd))
472+
ifself.remote:
473+
self.assertTrue(
474+
"ERROR: Failed to read file"ine.messageand
475+
"data file checksum mismatch"ine.message,
476+
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
477+
repr(e.message),self.cmd))
478+
else:
479+
self.assertIn(
480+
'WARNING: Corruption detected in file',
481+
e.message,
482+
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
483+
repr(e.message),self.cmd))
484+
self.assertIn(
485+
'ERROR: Data file corruption',
486+
e.message,
487+
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
488+
repr(e.message),self.cmd))
489+
490+
# Clean after yourself
491+
self.del_test_dir(module_name,fname)
492+
493+
# @unittest.skip("skip")
494+
deftest_backup_truncate_misaligned(self):
495+
"""
496+
make node, truncate file to size not even to BLCKSIZE,
497+
take backup
498+
"""
499+
fname=self.id().split('.')[3]
500+
node=self.make_simple_node(
501+
base_dir=os.path.join(module_name,fname,'node'),
502+
set_replication=True,
503+
initdb_params=['--data-checksums'])
504+
505+
backup_dir=os.path.join(self.tmp_path,module_name,fname,'backup')
506+
507+
self.init_pb(backup_dir)
508+
self.add_instance(backup_dir,'node',node)
509+
node.slow_start()
510+
511+
node.safe_psql(
512+
"postgres",
513+
"create table t_heap as select 1 as id, md5(i::text) as text, "
514+
"md5(repeat(i::text,10))::tsvector as tsvector "
515+
"from generate_series(0,100000) i")
516+
517+
node.safe_psql(
518+
"postgres",
519+
"CHECKPOINT;")
520+
521+
heap_path=node.safe_psql(
522+
"postgres",
523+
"select pg_relation_filepath('t_heap')").rstrip()
524+
525+
heap_size=node.safe_psql(
526+
"postgres",
527+
"select pg_relation_size('t_heap')")
528+
529+
withopen(os.path.join(node.data_dir,heap_path),"rb+",0)asf:
530+
f.truncate(int(heap_size)-4096)
531+
f.flush()
532+
f.close
533+
534+
output=self.backup_node(
535+
backup_dir,'node',node,backup_type="full",
536+
options=["-j","4","--stream"],return_id=False)
537+
538+
self.assertIn("WARNING: File",output)
539+
self.assertIn("invalid file size",output)
482540

483541
# Clean after yourself
484542
self.del_test_dir(module_name,fname)
@@ -1450,16 +1508,16 @@ def test_backup_with_least_privileges_role(self):
14501508

14511509
node.safe_psql(
14521510
'backupdb',
1453-
"REVOKETEMPORARY ON DATABASE backupdbFROM PUBLIC;"
1454-
"REVOKE ALLon SCHEMA public from PUBLIC; "
1511+
"REVOKEALL ON DATABASE backupdbfrom PUBLIC;"
1512+
"REVOKE ALLON SCHEMA public from PUBLIC; "
14551513
"REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; "
14561514
"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; "
14571515
"REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; "
1458-
"REVOKE ALLon SCHEMA pg_catalog from PUBLIC; "
1516+
"REVOKE ALLON SCHEMA pg_catalog from PUBLIC; "
14591517
"REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; "
14601518
"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; "
14611519
"REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; "
1462-
"REVOKE ALLon SCHEMA information_schema from PUBLIC; "
1520+
"REVOKE ALLON SCHEMA information_schema from PUBLIC; "
14631521
"REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; "
14641522
"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; "
14651523
"REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; "

‎tests/compatibility.py‎

Lines changed: 0 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -532,49 +532,3 @@ def test_backward_compatibility_merge(self):
532532

533533
# Clean after yourself
534534
self.del_test_dir(module_name,fname)
535-
536-
# @unittest.skip("skip")
537-
deftest_backup_concurrent_drop_table(self):
538-
""""""
539-
fname=self.id().split('.')[3]
540-
backup_dir=os.path.join(self.tmp_path,module_name,fname,'backup')
541-
node=self.make_simple_node(
542-
base_dir=os.path.join(module_name,fname,'node'),
543-
set_replication=True,
544-
initdb_params=['--data-checksums'])
545-
546-
self.init_pb(backup_dir)
547-
self.add_instance(backup_dir,'node',node,old_binary=True)
548-
node.slow_start()
549-
550-
node.pgbench_init(scale=1)
551-
552-
# FULL backup
553-
gdb=self.backup_node(
554-
backup_dir,'node',node,
555-
options=['--stream','--compress','--log-level-file=VERBOSE'],
556-
gdb=True,old_binary=True)
557-
558-
gdb.set_breakpoint('backup_data_file')
559-
gdb.run_until_break()
560-
561-
node.safe_psql(
562-
'postgres',
563-
'DROP TABLE pgbench_accounts')
564-
565-
# do checkpoint to guarantee filenode removal
566-
node.safe_psql(
567-
'postgres',
568-
'CHECKPOINT')
569-
570-
gdb.remove_all_breakpoints()
571-
gdb.continue_execution_until_exit()
572-
573-
# show_backup = self.show_pb(backup_dir, 'node')[0]
574-
# self.assertEqual(show_backup['status'], "OK")
575-
576-
# validate with fresh binary, it MUST be successful
577-
self.validate_pb(backup_dir)
578-
579-
# Clean after yourself
580-
self.del_test_dir(module_name,fname)

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp