Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit25856d5

Browse files
author
vshepard
committed
Add s3 tests
1 parent2b9c9b1 commit25856d5

File tree

59 files changed

+14614
-20420
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

59 files changed

+14614
-20420
lines changed

‎s3/__init__.py

Whitespace-only changes.

‎s3/test_utils/__init__.py

Whitespace-only changes.

‎s3/test_utils/config_provider.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
importconfigparser
2+
3+
4+
defread_config(s3_config_file):
5+
config=configparser.ConfigParser()
6+
config.read_string('[fake-section]\n'+open(s3_config_file).read())
7+
8+
returnconfig['fake-section']

‎s3/test_utils/s3_backup.py

Lines changed: 208 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
importos
2+
importio
3+
importsys
4+
5+
importminio
6+
fromminioimportMinio
7+
fromminio.deleteobjectsimportDeleteObject
8+
importurllib3
9+
frompg_probackup2.storage.fs_backupimportTestBackupDir
10+
frompg_probackup2.init_helpersimportinit_params
11+
froms3.test_utilsimportconfig_provider
12+
13+
root=os.path.realpath(os.path.join(os.path.dirname(__file__),'../..'))
14+
ifrootnotinsys.path:
15+
sys.path.append(root)
16+
17+
status_forcelist= [413,# RequestBodyTooLarge
18+
429,# TooManyRequests
19+
500,# InternalError
20+
503,# ServerBusy
21+
]
22+
23+
DEFAULT_CONF_FILE='s3/tests/s3.conf'
24+
25+
26+
classS3TestBackupDir(TestBackupDir):
27+
is_file_based=False
28+
29+
def__init__(self,*,rel_path,backup):
30+
self.access_key=None
31+
self.secret_key=None
32+
self.s3_type=None
33+
self.tmp_path=None
34+
self.host=None
35+
self.port=None
36+
self.bucket_name=None
37+
self.region=None
38+
self.bucket=None
39+
self.path_suffix=None
40+
self.https=None
41+
self.s3_config_file=None
42+
self.ca_certificate=None
43+
44+
self.set_s3_config_file()
45+
self.setup_s3_env()
46+
47+
path="pg_probackup"
48+
ifself.path_suffix:
49+
path+="_"+self.path_suffix
50+
ifself.tmp_path==''oros.path.isabs(self.tmp_path):
51+
self.path=f"{path}{self.tmp_path}/{rel_path}/{backup}"
52+
else:
53+
self.path=f"{path}/{self.tmp_path}/{rel_path}/{backup}"
54+
55+
secure:bool=False
56+
self.versioning:bool=False
57+
ifself.httpsin ['ON','HTTPS']:
58+
secure=True
59+
ifself.httpsandself.ca_certificate:
60+
http_client=urllib3.PoolManager(cert_reqs='CERT_REQUIRED',
61+
ca_certs=self.ca_certificate,
62+
retries=urllib3.Retry(total=5,
63+
backoff_factor=1,
64+
status_forcelist=status_forcelist))
65+
else:
66+
http_client=urllib3.PoolManager(retries=urllib3.Retry(total=5,
67+
backoff_factor=1,
68+
status_forcelist=status_forcelist))
69+
70+
self.conn=Minio(self.host+":"+self.port,secure=secure,access_key=self.access_key,
71+
secret_key=self.secret_key,http_client=http_client)
72+
ifnotself.conn.bucket_exists(self.bucket):
73+
raiseException(f"Test bucket{self.bucket} does not exist.")
74+
75+
try:
76+
config=self.conn.get_bucket_versioning(self.bucket)
77+
ifconfig.status.lower()=="enabled"orconfig.status.lower()=="suspended":
78+
self.versioning=True
79+
else:
80+
self.versioning=False
81+
exceptExceptionase:
82+
if"NotImplemented"inrepr(e):
83+
self.versioning=False
84+
else:
85+
raisee
86+
self.pb_args= ('-B','/'+self.path,f'--s3={init_params.s3_type}')
87+
ifself.s3_config_file:
88+
self.pb_args+= (f'--s3-config-file={self.s3_config_file}',)
89+
return
90+
91+
defsetup_s3_env(self,s3_config=None):
92+
self.tmp_path=os.environ.get('PGPROBACKUP_TMP_DIR',default='')
93+
self.host=os.environ.get('PG_PROBACKUP_S3_HOST',default='')
94+
95+
# If environment variables are not setup, use from config
96+
ifself.s3_config_fileors3_config:
97+
minio_config=config_provider.read_config(self.s3_config_fileors3_config)
98+
self.access_key=minio_config['access-key']
99+
self.secret_key=minio_config['secret-key']
100+
self.host=minio_config['s3-host']
101+
self.port=minio_config['s3-port']
102+
self.bucket=minio_config['s3-bucket']
103+
self.region=minio_config['s3-region']
104+
self.https=minio_config['s3-secure']
105+
init_params.s3_type='minio'
106+
else:
107+
self.access_key=os.environ.get('PG_PROBACKUP_S3_ACCESS_KEY')
108+
self.secret_key=os.environ.get('PG_PROBACKUP_S3_SECRET_ACCESS_KEY')
109+
self.host=os.environ.get('PG_PROBACKUP_S3_HOST')
110+
self.port=os.environ.get('PG_PROBACKUP_S3_PORT')
111+
self.bucket=os.environ.get('PG_PROBACKUP_S3_BUCKET_NAME')
112+
self.region=os.environ.get('PG_PROBACKUP_S3_REGION')
113+
self.https=os.environ.get('PG_PROBACKUP_S3_HTTPS')
114+
self.ca_certificate=os.environ.get('PG_PROBACKUP_S3_CA_CERTIFICATE')
115+
init_params.s3_type=os.environ.get('PG_PROBACKUP_S3_TEST')
116+
117+
# multi-url case
118+
# remove all urls from string except the first one
119+
if';'inself.host:
120+
self.host=self.host[:self.host.find(';')]
121+
if':'inself.host:# also change port if it was overridden in multihost string
122+
self.port=self.host[self.host.find(':')+1:]
123+
self.host=self.host[:self.host.find(':')]
124+
125+
defset_s3_config_file(self):
126+
s3_config=os.environ.get('PG_PROBACKUP_S3_CONFIG_FILE')
127+
ifs3_configisnotNoneands3_config.strip().lower()=="true":
128+
self.s3_config_file=DEFAULT_CONF_FILE
129+
else:
130+
self.s3_config_file=s3_config
131+
132+
deflist_instance_backups(self,instance):
133+
full_path=os.path.join(self.path,'backups',instance)
134+
candidates=self.conn.list_objects(self.bucket,prefix=full_path,recursive=True)
135+
return [os.path.basename(os.path.dirname(x.object_name))
136+
forxincandidatesifx.object_name.endswith('backup.control')]
137+
138+
deflist_files(self,sub_dir,recursive=False):
139+
full_path=os.path.join(self.path,sub_dir)
140+
# Need '/' in the end to find inside the folder
141+
full_path_dir=full_pathiffull_path[-1]=='/'elsefull_path+'/'
142+
object_list=self.conn.list_objects(self.bucket,prefix=full_path_dir,recursive=recursive)
143+
return [obj.object_name.replace(full_path_dir,'',1)
144+
forobjinobject_list
145+
ifnotobj.is_dir]
146+
147+
deflist_dirs(self,sub_dir):
148+
full_path=os.path.join(self.path,sub_dir)
149+
# Need '/' in the end to find inside the folder
150+
full_path_dir=full_pathiffull_path[-1]=='/'elsefull_path+'/'
151+
object_list=self.conn.list_objects(self.bucket,prefix=full_path_dir,recursive=False)
152+
return [obj.object_name.replace(full_path_dir,'',1).rstrip('\\/')
153+
forobjinobject_list
154+
ifobj.is_dir]
155+
156+
defread_file(self,sub_path,*,text=True):
157+
full_path=os.path.join(self.path,sub_path)
158+
bytes=self.conn.get_object(self.bucket,full_path).read()
159+
ifnottext:
160+
returnbytes
161+
returnbytes.decode('utf-8')
162+
163+
defwrite_file(self,sub_path,data,*,text=True):
164+
full_path=os.path.join(self.path,sub_path)
165+
iftext:
166+
data=data.encode('utf-8')
167+
self.conn.put_object(self.bucket,full_path,io.BytesIO(data),length=len(data))
168+
169+
defcleanup(self,dir=''):
170+
self.remove_dir(dir)
171+
172+
defremove_file(self,sub_path):
173+
full_path=os.path.join(self.path,sub_path)
174+
self.conn.remove_object(self.bucket,full_path)
175+
176+
defremove_dir(self,sub_path):
177+
ifsub_path:
178+
full_path=os.path.join(self.path,sub_path)
179+
else:
180+
full_path=self.path
181+
objs=self.conn.list_objects(self.bucket,prefix=full_path,recursive=True,
182+
include_version=self.versioning)
183+
delobjs= (DeleteObject(o.object_name,o.version_id)foroinobjs)
184+
errs=list(self.conn.remove_objects(self.bucket,delobjs))
185+
iferrs:
186+
strerrs="; ".join(str(err)forerrinerrs)
187+
raiseException("There were errors: {0}".format(strerrs))
188+
189+
defexists(self,sub_path):
190+
full_path=os.path.join(self.path,sub_path)
191+
try:
192+
self.conn.stat_object(self.bucket,full_path)
193+
returnTrue
194+
exceptminio.error.S3Errorass3err:
195+
ifs3err.code=='NoSuchKey':
196+
returnFalse
197+
raises3err
198+
exceptExceptionaserr:
199+
raiseerr
200+
201+
def__str__(self):
202+
return'/'+self.path
203+
204+
def__repr__(self):
205+
return"S3TestBackupDir"+str(self.path)
206+
207+
def__fspath__(self):
208+
returnself.path

‎s3/tests/__init__.py

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
importunittest
2+
importos
3+
4+
from .importauth_test,param_test
5+
6+
7+
defload_tests(loader,tests,pattern):
8+
suite=unittest.TestSuite()
9+
10+
if'PG_PROBACKUP_TEST_BASIC'inos.environ:
11+
ifos.environ['PG_PROBACKUP_TEST_BASIC']=='ON':
12+
loader.testMethodPrefix='test_basic'
13+
14+
suite.addTests(loader.loadTestsFromModule(auth_test))
15+
suite.addTests(loader.loadTestsFromModule(param_test))
16+
17+
returnsuite

‎s3/tests/auth_test.py

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
importos
2+
importsys
3+
4+
root=os.path.realpath(os.path.join(os.path.dirname(__file__),'../..'))
5+
ifrootnotinsys.path:
6+
sys.path.append(root)
7+
8+
fromtests.helpers.ptrack_helpersimportProbackupTest
9+
10+
11+
classAuthorizationTest(ProbackupTest):
12+
"""
13+
Check connect to S3 via pre_start_checks() function
14+
calling pg_probackup init --s3
15+
16+
test that s3 keys allow to connect to all types of storages
17+
"""
18+
19+
defs3_auth_test(self):
20+
console_output=self.pb.init(options=["--log-level-console=VERBOSE"])
21+
22+
self.assertNotIn(': 403',console_output)# Because we can have just '403' substring in timestamp
23+
self.assertMessage(console_output,contains='S3_pre_start_check successful')
24+
self.assertMessage(console_output,contains='HTTP response: 200')
25+
self.assertIn(
26+
f"INFO: Backup catalog '{self.backup_dir}' successfully initialized",
27+
console_output)
28+
29+
deftest_log_level_file_requires_log_directory(self):
30+
console_output=self.pb.init(options=["--log-level-file=VERBOSE"],
31+
skip_log_directory=True,
32+
expect_error=True)
33+
34+
self.assertMessage(console_output,
35+
contains='ERROR: Cannot save S3 logs to a file. You must specify --log-directory option when'
36+
' running backup with --log-level-file option enabled')

‎s3/tests/pytest.ini

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
[pytest]
2+
log_cli = true
3+
log_cli_level = INFO
4+
log_format = %(asctime)s %(levelname)s %(message)s
5+
log_date_format = %Y-%m-%d %H:%M:%S
6+
testpaths = tests

‎tests/CVE_2018_1058_test.py

Lines changed: 18 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,16 @@
11
importos
22
importunittest
3-
from .helpers.ptrack_helpersimportProbackupTest,ProbackupException
3+
from .helpers.ptrack_helpersimportProbackupTest
44

5-
classCVE_2018_1058(ProbackupTest,unittest.TestCase):
5+
classCVE_2018_1058(ProbackupTest):
66

77
# @unittest.skip("skip")
88
deftest_basic_default_search_path(self):
99
""""""
10-
backup_dir=os.path.join(self.tmp_path,self.module_name,self.fname,'backup')
11-
node=self.make_simple_node(
12-
base_dir=os.path.join(self.module_name,self.fname,'node'),
13-
set_replication=True)
10+
node=self.pg_node.make_simple('node',checksum=False,set_replication=True)
1411

15-
self.init_pb(backup_dir)
16-
self.add_instance(backup_dir,'node',node)
12+
self.pb.init()
13+
self.pb.add_instance('node',node)
1714
node.slow_start()
1815

1916
node.safe_psql(
@@ -26,19 +23,16 @@ def test_basic_default_search_path(self):
2623
"END "
2724
"$$ LANGUAGE plpgsql")
2825

29-
self.backup_node(backup_dir,'node',node,backup_type='full',options=['--stream'])
26+
self.pb.backup_node('node',node,backup_type='full',options=['--stream'])
3027

3128
# @unittest.skip("skip")
3229
deftest_basic_backup_modified_search_path(self):
3330
""""""
34-
backup_dir=os.path.join(self.tmp_path,self.module_name,self.fname,'backup')
35-
node=self.make_simple_node(
36-
base_dir=os.path.join(self.module_name,self.fname,'node'),
37-
set_replication=True)
38-
self.set_auto_conf(node,options={'search_path':'public,pg_catalog'})
31+
node=self.pg_node.make_simple('node',checksum=False,set_replication=True)
32+
node.set_auto_conf(options={'search_path':'public,pg_catalog'})
3933

40-
self.init_pb(backup_dir)
41-
self.add_instance(backup_dir,'node',node)
34+
self.pb.init()
35+
self.pb.add_instance('node',node)
4236
node.slow_start()
4337

4438
node.safe_psql(
@@ -62,7 +56,7 @@ def test_basic_backup_modified_search_path(self):
6256
"$$ LANGUAGE plpgsql; "
6357
"CREATE VIEW public.pg_proc AS SELECT proname FROM public.pg_proc()")
6458

65-
self.backup_node(backup_dir,'node',node,backup_type='full',options=['--stream'])
59+
self.pb.backup_node('node',node,backup_type='full',options=['--stream'])
6660

6761
log_file=os.path.join(node.logs_dir,'postgresql.log')
6862
withopen(log_file,'r')asf:
@@ -73,10 +67,8 @@ def test_basic_backup_modified_search_path(self):
7367
# @unittest.skip("skip")
7468
deftest_basic_checkdb_modified_search_path(self):
7569
""""""
76-
node=self.make_simple_node(
77-
base_dir=os.path.join(self.module_name,self.fname,'node'),
78-
initdb_params=['--data-checksums'])
79-
self.set_auto_conf(node,options={'search_path':'public,pg_catalog'})
70+
node=self.pg_node.make_simple('node')
71+
node.set_auto_conf(options={'search_path':'public,pg_catalog'})
8072
node.slow_start()
8173

8274
node.safe_psql(
@@ -110,20 +102,11 @@ def test_basic_checkdb_modified_search_path(self):
110102
"CREATE VIEW public.pg_namespace AS SELECT * FROM public.pg_namespace();"
111103
)
112104

113-
try:
114-
self.checkdb_node(
105+
self.pb.checkdb_node(
115106
options=[
116107
'--amcheck',
117108
'--skip-block-validation',
118-
'-d','postgres','-p',str(node.port)])
119-
self.assertEqual(
120-
1,0,
121-
"Expecting Error because amcheck{,_next} not installed\n"
122-
" Output: {0}\n CMD: {1}".format(
123-
repr(self.output),self.cmd))
124-
exceptProbackupExceptionase:
125-
self.assertIn(
126-
"WARNING: Extension 'amcheck' or 'amcheck_next' are not installed in database postgres",
127-
e.message,
128-
"\n Unexpected Error Message: {0}\n CMD: {1}".format(
129-
repr(e.message),self.cmd))
109+
'-d','postgres','-p',str(node.port)],
110+
expect_error="because amcheck{,_next} not installed")
111+
self.assertMessage(contains=
112+
"WARNING: Extension 'amcheck' or 'amcheck_next' are not installed in database postgres")

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp