Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit9a6c7f7

Browse files
committed
Merge branch 'release_2_5' into release_2_5-remove-ptrack1
2 parentseba5a47 +dbbfee4 commit9a6c7f7

File tree

8 files changed

+151
-86
lines changed

8 files changed

+151
-86
lines changed

‎.travis.yml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -42,11 +42,11 @@ env:
4242
-PG_VERSION=9.6 PG_BRANCH=REL9_6_STABLE
4343
-PG_VERSION=9.5 PG_BRANCH=REL9_5_STABLE
4444

45-
jobs:
46-
allow_failures:
47-
-if:env(MODE) IN (archive, backup, delta, locking, merge, replica, retention, restore)
45+
#jobs:
46+
# allow_failures:
47+
# - if: env(MODE) IN (archive, backup, delta, locking, merge, replica, retention, restore)
4848

4949
# Only run CI for master branch commits to limit our travis usage
50-
branches:
51-
only:
52-
-master
50+
#branches:
51+
# only:
52+
# - master

‎src/utils/file.c‎

Lines changed: 39 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -776,6 +776,34 @@ fio_fwrite(FILE* f, void const* buf, size_t size)
776776
returnfwrite(buf,1,size,f);
777777
}
778778

779+
/*
780+
* Write buffer to descriptor by calling write(),
781+
* If size of written data is less than buffer size,
782+
* then try to write what is left.
783+
* We do this to get honest errno if there are some problems
784+
* with filesystem, since writing less than buffer size
785+
* is not considered an error.
786+
*/
787+
staticssize_t
788+
durable_write(intfd,constchar*buf,size_tsize)
789+
{
790+
off_tcurrent_pos=0;
791+
size_tbytes_left=size;
792+
793+
while (bytes_left>0)
794+
{
795+
intrc=write(fd,buf+current_pos,bytes_left);
796+
797+
if (rc <=0)
798+
returnrc;
799+
800+
bytes_left-=rc;
801+
current_pos+=rc;
802+
}
803+
804+
returnsize;
805+
}
806+
779807
/* Write data to the file synchronously */
780808
ssize_t
781809
fio_write(intfd,voidconst*buf,size_tsize)
@@ -805,7 +833,7 @@ fio_write(int fd, void const* buf, size_t size)
805833
}
806834
else
807835
{
808-
returnwrite(fd,buf,size);
836+
returndurable_write(fd,buf,size);
809837
}
810838
}
811839

@@ -815,7 +843,7 @@ fio_write_impl(int fd, void const* buf, size_t size, int out)
815843
intrc;
816844
fio_headerhdr;
817845

818-
rc=write(fd,buf,size);
846+
rc=durable_write(fd,buf,size);
819847

820848
hdr.arg=0;
821849
hdr.size=0;
@@ -837,34 +865,6 @@ fio_fwrite_async(FILE* f, void const* buf, size_t size)
837865
:fwrite(buf,1,size,f);
838866
}
839867

840-
/*
841-
* Write buffer to descriptor by calling write(),
842-
* If size of written data is less than buffer size,
843-
* then try to write what is left.
844-
* We do this to get honest errno if there are some problems
845-
* with filesystem, since writing less than buffer size
846-
* is not considered an error.
847-
*/
848-
staticssize_t
849-
durable_write(intfd,constchar*buf,size_tsize)
850-
{
851-
off_tcurrent_pos=0;
852-
size_tbytes_left=size;
853-
854-
while (bytes_left>0)
855-
{
856-
intrc=write(fd,buf+current_pos,bytes_left);
857-
858-
if (rc <=0)
859-
returnrc;
860-
861-
bytes_left-=rc;
862-
current_pos+=rc;
863-
}
864-
865-
returnsize;
866-
}
867-
868868
/* Write data to the file */
869869
/* TODO: support async report error */
870870
ssize_t
@@ -949,23 +949,22 @@ fio_fwrite_async_compressed(FILE* f, void const* buf, size_t size, int compress_
949949
}
950950
else
951951
{
952-
charuncompressed_buf[BLCKSZ];
953952
char*errormsg=NULL;
954-
int32uncompressed_size=fio_decompress(uncompressed_buf,buf,size,compress_alg,&errormsg);
953+
chardecompressed_buf[BLCKSZ];
954+
int32decompressed_size=fio_decompress(decompressed_buf,buf,size,compress_alg,&errormsg);
955955

956-
if (uncompressed_size<0)
956+
if (decompressed_size<0)
957957
elog(ERROR,"%s",errormsg);
958958

959-
returnfwrite(uncompressed_buf,1,uncompressed_size,f);
959+
returnfwrite(decompressed_buf,1,decompressed_size,f);
960960
}
961961
}
962962

963963
staticvoid
964964
fio_write_compressed_impl(intfd,voidconst*buf,size_tsize,intcompress_alg)
965965
{
966-
intrc;
967-
int32uncompressed_size;
968-
charuncompressed_buf[BLCKSZ];
966+
int32decompressed_size;
967+
chardecompressed_buf[BLCKSZ];
969968

970969
/* If the previous command already have failed,
971970
* then there is no point in bashing a head against the wall
@@ -974,14 +973,12 @@ fio_write_compressed_impl(int fd, void const* buf, size_t size, int compress_alg
974973
return;
975974

976975
/* decompress chunk */
977-
uncompressed_size=fio_decompress(uncompressed_buf,buf,size,compress_alg,&async_errormsg);
976+
decompressed_size=fio_decompress(decompressed_buf,buf,size,compress_alg,&async_errormsg);
978977

979-
if (uncompressed_size<0)
978+
if (decompressed_size<0)
980979
return;
981980

982-
rc=write(fd,uncompressed_buf,uncompressed_size);
983-
984-
if (rc <=0)
981+
if (durable_write(fd,decompressed_buf,decompressed_size) <=0)
985982
{
986983
async_errormsg=pgut_malloc(ERRMSG_MAX_LEN);
987984
snprintf(async_errormsg,ERRMSG_MAX_LEN,"%s",strerror(errno));

‎tests/backup.py‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2106,7 +2106,6 @@ def test_backup_with_least_privileges_role(self):
21062106
)
21072107

21082108
ifself.ptrack:
2109-
ifnode.major_version>=11:
21102109
fnames= [
21112110
'ptrack.ptrack_get_pagemapset(pg_lsn)',
21122111
'ptrack.ptrack_init_lsn()'
@@ -2121,8 +2120,6 @@ def test_backup_with_least_privileges_role(self):
21212120
"backupdb",
21222121
"GRANT EXECUTE ON FUNCTION {0} "
21232122
"TO backup".format(fname))
2124-
else:
2125-
self.skipTest("skip --- we do not support ptrack 1.* anymore")
21262123

21272124
ifProbackupTest.enterprise:
21282125
node.safe_psql(

‎tests/helpers/ptrack_helpers.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -312,6 +312,12 @@ def __init__(self, *args, **kwargs):
312312

313313
os.environ["PGAPPNAME"]="pg_probackup"
314314

315+
ifself.ptrack:
316+
self.assertGreaterEqual(
317+
self.pg_config_version,
318+
self.version_to_num('11.0'),
319+
"ptrack testing require PostgreSQL >= 11")
320+
315321
@property
316322
defpg_config_version(self):
317323
returnself.version_to_num(
@@ -2004,6 +2010,9 @@ def stopped_in_breakpoint(self):
20042010
returnTrue
20052011
returnFalse
20062012

2013+
defquit(self):
2014+
self.proc.terminate()
2015+
20072016
# use for breakpoint, run, continue
20082017
def_execute(self,cmd,running=True):
20092018
output= []

‎tests/ptrack.py‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -481,16 +481,15 @@ def test_ptrack_unprivileged(self):
481481
"GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;"
482482
)
483483

484-
ifnode.major_version>=11:
485-
node.safe_psql(
486-
"backupdb",
487-
"CREATE SCHEMA ptrack")
488-
node.safe_psql(
489-
"backupdb",
490-
"CREATE EXTENSION ptrack WITH SCHEMA ptrack")
491-
node.safe_psql(
492-
"backupdb",
493-
"GRANT USAGE ON SCHEMA ptrack TO backup")
484+
node.safe_psql(
485+
"backupdb",
486+
"CREATE SCHEMA ptrack")
487+
node.safe_psql(
488+
"backupdb",
489+
"CREATE EXTENSION ptrack WITH SCHEMA ptrack")
490+
node.safe_psql(
491+
"backupdb",
492+
"GRANT USAGE ON SCHEMA ptrack TO backup")
494493

495494
node.safe_psql(
496495
"backupdb",

‎tests/restore.py‎

Lines changed: 72 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@
99
importshutil
1010
importjson
1111
fromshutilimportcopyfile
12-
fromtestgresimportQueryException
12+
fromtestgresimportQueryException,StartNodeException
13+
fromstatimportS_ISDIR
1314

1415

1516
module_name='restore'
@@ -3297,23 +3298,13 @@ def test_missing_database_map(self):
32973298
)
32983299

32993300
ifself.ptrack:
3300-
fnames= []
3301-
ifnode.major_version>=11:
3302-
# TODO why backup works without these grants ?
3303-
# fnames += [
3304-
# 'pg_ptrack_get_pagemapset(pg_lsn)',
3305-
# 'pg_ptrack_control_lsn()',
3306-
# ]
3307-
node.safe_psql(
3308-
"backupdb",
3309-
"CREATE EXTENSION ptrack WITH SCHEMA pg_catalog")
3310-
else:
3311-
self.skipTest("skip --- we do not support ptrack 1.* anymore")
3312-
3313-
forfnameinfnames:
3314-
node.safe_psql(
3315-
"backupdb",
3316-
"GRANT EXECUTE ON FUNCTION {0} TO backup".format(fname))
3301+
# TODO why backup works without these grants ?
3302+
# 'pg_ptrack_get_pagemapset(pg_lsn)',
3303+
# 'pg_ptrack_control_lsn()',
3304+
# because PUBLIC
3305+
node.safe_psql(
3306+
"backupdb",
3307+
"CREATE EXTENSION ptrack WITH SCHEMA pg_catalog")
33173308

33183309
ifProbackupTest.enterprise:
33193310
node.safe_psql(
@@ -3847,3 +3838,66 @@ def test_concurrent_restore(self):
38473838

38483839
# Clean after yourself
38493840
self.del_test_dir(module_name,fname)
3841+
3842+
# @unittest.skip("skip")
3843+
deftest_restore_issue_313(self):
3844+
"""
3845+
Check that partially restored PostgreSQL instance cannot be started
3846+
"""
3847+
fname=self.id().split('.')[3]
3848+
backup_dir=os.path.join(self.tmp_path,module_name,fname,'backup')
3849+
node=self.make_simple_node(
3850+
base_dir=os.path.join(module_name,fname,'node'),
3851+
set_replication=True,
3852+
initdb_params=['--data-checksums'])
3853+
3854+
self.init_pb(backup_dir)
3855+
self.add_instance(backup_dir,'node',node)
3856+
self.set_archiving(backup_dir,'node',node)
3857+
node.slow_start()
3858+
3859+
# FULL backup
3860+
backup_id=self.backup_node(backup_dir,'node',node)
3861+
node.cleanup()
3862+
3863+
count=0
3864+
filelist=self.get_backup_filelist(backup_dir,'node',backup_id)
3865+
forfileinfilelist:
3866+
# count only nondata files
3867+
ifint(filelist[file]['is_datafile'])==0andint(filelist[file]['size'])>0:
3868+
count+=1
3869+
3870+
node_restored=self.make_simple_node(
3871+
base_dir=os.path.join(module_name,fname,'node_restored'))
3872+
node_restored.cleanup()
3873+
self.restore_node(backup_dir,'node',node_restored)
3874+
3875+
gdb=self.restore_node(backup_dir,'node',node,gdb=True,options=['--progress'])
3876+
gdb.verbose=False
3877+
gdb.set_breakpoint('restore_non_data_file')
3878+
gdb.run_until_break()
3879+
gdb.continue_execution_until_break(count-2)
3880+
gdb.quit()
3881+
3882+
# emulate the user or HA taking care of PG configuration
3883+
forfnameinos.listdir(node_restored.data_dir):
3884+
iffname.endswith('.conf'):
3885+
os.rename(
3886+
os.path.join(node_restored.data_dir,fname),
3887+
os.path.join(node.data_dir,fname))
3888+
3889+
try:
3890+
node.slow_start()
3891+
# we should die here because exception is what we expect to happen
3892+
self.assertEqual(
3893+
1,0,
3894+
"Expecting Error because backup is not fully restored")
3895+
exceptStartNodeExceptionase:
3896+
self.assertIn(
3897+
'Cannot start node',
3898+
e.message,
3899+
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(
3900+
repr(e.message),self.cmd))
3901+
3902+
# Clean after yourself
3903+
self.del_test_dir(module_name,fname)

‎travis/Dockerfile.in‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@ FROM ololobus/postgres-dev:stretch
22

33
USER root
44
RUN apt-get update
5-
RUN apt-get -yq installpython python-pip
5+
RUN apt-get -yq installpython3 python3-pip
66

77
# RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
88
# RUN python2 get-pip.py
9-
RUNpython2 -m pip install virtualenv
9+
RUNpython3 -m pip install virtualenv
1010

1111
# Environment
1212
ENV PG_MAJOR=${PG_VERSION} PG_BRANCH=${PG_BRANCH}

‎travis/run_tests.sh‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,16 @@
33
#
44
# Copyright (c) 2019-2020, Postgres Professional
55
#
6+
set -xe
67

8+
sudo su -c'mkdir /run/sshd'
9+
sudo su -c'apt-get update -y'
10+
sudo su -c'apt-get install openssh-client openssh-server -y'
11+
/etc/init.d/ssh start
12+
13+
ssh-keygen -t rsa -q -N""
14+
cat~/.ssh/id_rsa.pub>~/.ssh/authorized_keys
15+
ssh-keyscan -H localhost>>~/.ssh/known_hosts
716

817
PG_SRC=$PWD/postgres
918

@@ -60,17 +69,17 @@ make USE_PGXS=1 top_srcdir=$PG_SRC install
6069

6170
# Setup python environment
6271
echo"############### Setting up python env:"
63-
python2 -m virtualenv pyenv
72+
python3 -m virtualenv pyenv
6473
source pyenv/bin/activate
65-
pip install testgres==1.8.2
74+
pip3 install testgres
6675

6776
echo"############### Testing:"
6877
if ["$MODE"="basic" ];then
6978
export PG_PROBACKUP_TEST_BASIC=ON
70-
python -m unittest -v tests
71-
python -m unittest -v tests.init
79+
python3 -m unittest -v tests
80+
python3 -m unittest -v tests.init
7281
else
73-
python -m unittest -v tests.$MODE
82+
python3 -m unittest -v tests.$MODE
7483
fi
7584

7685
# Generate *.gcov files

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp