Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit6ba3da3

Browse files
committed
network recovery tests infrastructure.#6
1 parent14a9ad8 commit6ba3da3

File tree

10 files changed

+538
-0
lines changed

10 files changed

+538
-0
lines changed

‎tests2/.gitignore

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
.blockade
2+
.vagrant
3+
*.swp
4+
*.pyc

‎tests2/Dockerfile

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
# vim:set ft=dockerfile:
2+
FROM debian:jessie
3+
4+
# explicitly set user/group IDs
5+
RUN groupadd -r postgres --gid=999 && useradd -r -g postgres --uid=999 postgres
6+
7+
# make the "en_US.UTF-8" locale so postgres will be utf-8 enabled by default
8+
RUN apt-get update && apt-get install -y locales && rm -rf /var/lib/apt/lists/* \
9+
&& localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
10+
ENV LANG en_US.utf8
11+
12+
# use git to fetch sources
13+
RUN apt-get update \
14+
&& apt-get install -y git \
15+
&& rm -rf /var/lib/apt/lists/*
16+
17+
# postgres build deps
18+
RUN apt-get update && apt-get install -y \
19+
make \
20+
gcc \
21+
libreadline-dev \
22+
bison \
23+
flex \
24+
zlib1g-dev\
25+
&& rm -rf /var/lib/apt/lists/*
26+
27+
RUN mkdir /pg
28+
RUN chown postgres:postgres /pg
29+
30+
USER postgres
31+
WORKDIR /pg
32+
ENV CFLAGS -O0
33+
RUN git clone https://github.com/postgrespro/postgres_cluster.git --depth 1
34+
WORKDIR /pg/postgres_cluster
35+
RUN ./configure --enable-cassert --enable-debug --prefix=/usr/local/
36+
RUN make -j 4
37+
38+
USER root
39+
RUN make install
40+
RUN cd /pg/postgres_cluster/contrib/pg_tsdtm && make install
41+
RUN cd /pg/postgres_cluster/contrib/raftable && make install
42+
RUN cd /pg/postgres_cluster/contrib/mmts && make install
43+
RUN cd /pg/postgres_cluster/contrib/postgres_fdw && make install
44+
RUN mkdir -p /var/lib/postgresql/data && chown -R postgres /var/lib/postgresql/data
45+
RUN mkdir -p /run/postgresql && chown -R postgres /run/postgresql
46+
47+
USER postgres
48+
ENV PATH /usr/local/bin:$PATH
49+
ENV PGDATA /var/lib/postgresql/data
50+
VOLUME /var/lib/postgresql/data
51+
52+
COPY docker-entrypoint.sh /
53+
54+
ENTRYPOINT ["/docker-entrypoint.sh"]
55+
56+
EXPOSE 5432
57+
CMD ["postgres"]
58+
59+
60+

‎tests2/Vagrantfile

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
script=<<SCRIPT
2+
#!/bin/bash -e
3+
4+
if [ ! -f /etc/default/docker ]; then
5+
echo "/etc/default/docker not found -- is docker installed?" >&2
6+
exit 1
7+
fi
8+
9+
apt-get -y install python-pip python-virtualenv
10+
apt-get -y install postgresql-server-dev-all python-dev
11+
12+
cd /vagrant
13+
14+
export PIP_DOWNLOAD_CACHE=/vagrant/.pip_download_cache
15+
16+
SCRIPT
17+
18+
Vagrant.configure(2)do |config|
19+
20+
config.vm.box="ubuntu/trusty64"
21+
22+
ifVagrant.has_plugin?("vagrant-cachier")
23+
config.cache.scope=:box
24+
end
25+
26+
config.vm.provider:virtualboxdo |vb,override|
27+
vb.memory=2048
28+
end
29+
30+
# there are obviously vagrant versions with a
31+
# broken 'docker' provisioner that can be fixed
32+
# by invoking an 'apt-get update' *before* docker itself
33+
config.vm.provision"shell",inline:"apt-get update"
34+
35+
# fetch the ubuntu:latest image that is required for
36+
# the test suite
37+
config.vm.provision"docker",images:["ubuntu:trusty"]
38+
39+
# kick off the tests automatically
40+
config.vm.provision"shell",inline:script
41+
end
42+

‎tests2/blockade.yml

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
version:'2'
2+
3+
containers:
4+
5+
node1:
6+
container_name:node1
7+
image:pgmmts
8+
environment:
9+
POSTGRES_USER:'postgres'
10+
NODE_ID:1
11+
ports:
12+
5432:5432
13+
14+
node2:
15+
container_name:node2
16+
image:pgmmts
17+
environment:
18+
POSTGRES_USER:'postgres'
19+
NODE_ID:2
20+
ports:
21+
5433:5432
22+
23+
node3:
24+
container_name:node3
25+
image:pgmmts
26+
environment:
27+
POSTGRES_USER:'postgres'
28+
NODE_ID:3
29+
ports:
30+
5434:5432
31+
32+

‎tests2/docker-entrypoint.sh

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
#!/bin/bash
2+
set -e
3+
4+
# wtf is that?
5+
if ["${1:0:1}"='-' ];then
6+
set -- postgres"$@"
7+
fi
8+
9+
if ["$1"='postgres' ];then
10+
mkdir -p"$PGDATA"
11+
chmod 700"$PGDATA"
12+
chown -R postgres"$PGDATA"
13+
14+
# look specifically for PG_VERSION, as it is expected in the DB dir
15+
if [!-s"$PGDATA/PG_VERSION" ];then
16+
initdb
17+
18+
if ["$POSTGRES_PASSWORD" ];then
19+
pass="PASSWORD '$POSTGRES_PASSWORD'"
20+
authMethod=md5
21+
else
22+
pass=
23+
authMethod=trust
24+
fi
25+
26+
{echo;echo"host all all 0.0.0.0/0$authMethod"; }>>"$PGDATA/pg_hba.conf"
27+
{echo;echo"host replication all 0.0.0.0/0$authMethod"; }>>"$PGDATA/pg_hba.conf"
28+
29+
############################################################################
30+
31+
# internal start of server in order to allow set-up using psql-client
32+
# does not listen on TCP/IP and waits until start finishes
33+
pg_ctl -D"$PGDATA" \
34+
-o"-c listen_addresses=''" \
35+
-w start
36+
37+
:${POSTGRES_USER:=postgres}
38+
:${POSTGRES_DB:=$POSTGRES_USER}
39+
export POSTGRES_USER POSTGRES_DB
40+
41+
psql=( psql -v ON_ERROR_STOP=1 )
42+
43+
if ["$POSTGRES_DB"!='postgres' ];then
44+
"${psql[@]}" --username postgres<<-EOSQL
45+
CREATE DATABASE "$POSTGRES_DB" ;
46+
EOSQL
47+
echo
48+
fi
49+
50+
if ["$POSTGRES_USER"='postgres' ];then
51+
op='ALTER'
52+
else
53+
op='CREATE'
54+
fi
55+
"${psql[@]}" --username postgres<<-EOSQL
56+
$op USER "$POSTGRES_USER" WITH SUPERUSER$pass ;
57+
EOSQL
58+
echo
59+
60+
############################################################################
61+
62+
CONNSTRS='dbname=postgres host=node1, dbname=postgres host=node2, dbname=postgres host=node3'
63+
RAFT_PEERS='1:172.18.0.2:6666, 2:172.18.0.4:6666, 3:172.18.0.3:6666'
64+
65+
cat<<-EOF >>$PGDATA/postgresql.conf
66+
listen_addresses='*'
67+
max_prepared_transactions = 100
68+
synchronous_commit = off
69+
wal_level = logical
70+
max_worker_processes = 15
71+
max_replication_slots = 10
72+
max_wal_senders = 10
73+
shared_preload_libraries = 'raftable,multimaster'
74+
raftable.id =$NODE_ID
75+
raftable.peers = '$RAFT_PEERS'
76+
multimaster.workers=4
77+
multimaster.use_raftable=true
78+
multimaster.queue_size=52857600
79+
multimaster.ignore_tables_without_pk=1
80+
multimaster.node_id =$NODE_ID
81+
multimaster.conn_strings = '$CONNSTRS'
82+
EOF
83+
84+
tail -n 20$PGDATA/postgresql.conf
85+
86+
pg_ctl -D"$PGDATA" -m fast -w stop
87+
88+
echo
89+
echo'PostgreSQL init process complete; ready for start up.'
90+
echo
91+
fi
92+
fi
93+
94+
exec"$@"
95+

‎tests2/lib/__init__.py

Whitespace-only changes.

‎tests2/lib/bank_client.py

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
importpsycopg2
2+
importrandom
3+
frommultiprocessingimportProcess,Value,Queue
4+
importtime
5+
fromevent_historyimport*
6+
7+
classClientCollection(object):
8+
def__init__(self,connstrs):
9+
self._clients= []
10+
11+
forcsinconnstrs:
12+
b=BankClient(cs)
13+
self._clients.append(b)
14+
# glob_clients.append(b)
15+
16+
self._clients[0].initialize()
17+
18+
@property
19+
defclients(self):
20+
returnself._clients
21+
22+
def__getitem__(self,index):
23+
returnself._clients[index]
24+
25+
defstart(self):
26+
forclientinself._clients:
27+
client.start()
28+
29+
defstop(self):
30+
forclientinself._clients:
31+
client.stop()
32+
# client.cleanup()
33+
34+
35+
classBankClient(object):
36+
37+
def__init__(self,connstr):
38+
self.connstr=connstr
39+
self.run=Value('b',True)
40+
self._history=EventHistory()
41+
self.accounts=10000
42+
43+
definitialize(self):
44+
# initialize database
45+
conn=psycopg2.connect(self.connstr)
46+
cur=conn.cursor()
47+
cur.execute('create extension if not exists multimaster')
48+
conn.commit()
49+
50+
cur.execute('create table bank_test(uid int primary key, amount int)')
51+
52+
cur.execute('''
53+
insert into bank_test
54+
select *, 0 from generate_series(0, %s)''',
55+
(self.accounts,))
56+
conn.commit()
57+
cur.close()
58+
conn.close()
59+
60+
@property
61+
defhistory(self):
62+
returnself._history
63+
64+
defcheck_total(self):
65+
conn=psycopg2.connect(self.connstr)
66+
cur=conn.cursor();
67+
whileself.run.value:
68+
cur.execute('select sum(amount) from bank_test')
69+
res=cur.fetchone()
70+
ifres[0]!=0:
71+
print("Isolation error, total = %d"% (res[0],))
72+
raiseBaseException#
73+
74+
cur.close()
75+
conn.close()
76+
77+
deftransfer_money(self):
78+
print(self.connstr)
79+
conn=psycopg2.connect(self.connstr)
80+
cur=conn.cursor()
81+
82+
i=0
83+
whileself.run.value:
84+
i+=1
85+
amount=1
86+
from_uid=random.randrange(1,self.accounts+1)
87+
to_uid=random.randrange(1,self.accounts+1)
88+
89+
event_id=self.history.register_start('tx')
90+
91+
#cur.execute('begin')
92+
cur.execute('''update bank_test
93+
set amount = amount - %s
94+
where uid = %s''',
95+
(amount,from_uid))
96+
cur.execute('''update bank_test
97+
set amount = amount + %s
98+
where uid = %s''',
99+
(amount,to_uid))
100+
#cur.execute('commit')
101+
102+
try:
103+
conn.commit()
104+
except:
105+
self.history.register_finish(event_id,'rollback')
106+
else:
107+
self.history.register_finish(event_id,'commit')
108+
109+
#print("T", i)
110+
111+
cur.close()
112+
conn.close()
113+
114+
defwatchdog(self):
115+
whileself.run.value:
116+
time.sleep(1)
117+
print('watchdog: ',self.history.aggregate())
118+
119+
defstart(self):
120+
self.transfer_process=Process(target=self.transfer_money,args=())
121+
self.transfer_process.start()
122+
123+
self.total_process=Process(target=self.check_total,args=())
124+
self.total_process.start()
125+
126+
self.total_process=Process(target=self.watchdog,args=())
127+
self.total_process.start()
128+
129+
130+
return
131+
132+
defstop(self):
133+
self.run.value=False
134+
# self.total_process.join()
135+
# self.transfer_process.join()
136+
return
137+
138+
defcleanup(self):
139+
conn=psycopg2.connect(self.connstr)
140+
cur=conn.cursor()
141+
cur.execute('drop table bank_test')
142+
conn.commit()
143+
cur.close()
144+
conn.close()
145+
146+
147+

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp