use strict;

use warnings;

use Proc::ProcessTable;

use PostgresNode;

use TestLib;

use Test::More;

return stopnode($self->{nodes}->[$idx]);

}

{

my$root =shift;

diag("killtree$root\n");

my$t = new Proc::ProcessTable;

my%parent = ();

foreachmy$p (@{$t->table}) {

$parent{$p->pid} =$p->ppid;

}

if (!defined$root) {

return;

}

my@queue = ($root);

my@killist = ();

while (scalar@queue) {

my$victim =shift@queue;

while (my ($pid,$ppid) =each%parent) {

if ($ppid ==$victim) {

push@queue,$pid;

}

}

diag("SIGSTOP to$victim");

kill'STOP',$victim;

unshift@killist,$victim;

}

diag("SIGKILL to" .join('',@killist));

kill'KILL',@killist;

}

{

my ($self,$mode) =@_;

.PHONY: all

EXTRA_INSTALL=contrib/mmts

all: multimaster.so

PG_CPPFLAGS = -I$(libpq_srcdir)

include$(top_builddir)/src/Makefile.global

include$(top_srcdir)/contrib/contrib-global.mk

check: temp-install

&&Mtm->nodes[MtmNodeId-1].lastStatusChangeTime+MSEC_TO_USEC(MtmNodeDisableDelay)<MtmGetSystemTime())

{

MTM_ELOG(WARNING,"Node %d thinks that I amdead, while I am %s (message %s)",resp->node,MtmNodeStatusMnem[Mtm->status],MtmMessageKindMnem[resp->code]);

MTM_ELOG(WARNING,"Node %d thinks that I'mdead, while I'm %s (message %s)",resp->node,MtmNodeStatusMnem[Mtm->status],MtmMessageKindMnem[resp->code]);

BIT_SET(Mtm->disabledNodeMask,MtmNodeId-1);

Mtm->nConfigChanges+=1;

MtmSwitchClusterMode(MTM_RECOVERY);

}

voidMtmCheckHeartbeat()

{

BIT_SET(busy_mask,node);

while (true) {

if (sockets[node] >=0&&BIT_CHECK(Mtm->reconnectMask,node)) {

MTM_ELOG(WARNING,"Arbiter is forced to reconnect to node %d",node+1);

Assert(*msg->gid);

tm= (MtmTransMap*)hash_search(MtmGid2State,msg->gid,HASH_FIND,NULL);

if (tm==NULL||tm->state==NULL) {

MTM_ELOG(WARNING,"Response forunexisted transaction %s from node %d",msg->gid,node);

MTM_ELOG(WARNING,"Response fornon-existing transaction %s from node %d",msg->gid,node);

}else {

ts=tm->state;

BIT_SET(ts->votedMask,node-1);

}

ts= (MtmTransState*)hash_search(MtmXid2State,&msg->dxid,HASH_FIND,NULL);

if (ts==NULL) {

MTM_ELOG(WARNING,"Ignore response forunexisted transaction %llu from node %d", (long64)msg->dxid,node);

MTM_ELOG(WARNING,"Ignore response fornon-existing transaction %llu from node %d", (long64)msg->dxid,node);

continue;

}

Assert(msg->code==MSG_ABORTED||strcmp(msg->gid,ts->gid)==0);

}else {

Assert(ts->status==TRANSACTION_STATUS_ABORTED);

MTM_ELOG(WARNING,"Receive PRECOMMITTED response for aborted transaction %s (%llu) from node %d",

ts->gid, (long64)ts->xid,node);// How it can happen?SHould we use assert here?

ts->gid, (long64)ts->xid,node);// How it can happen?Should we use assert here?

if ((ts->participantsMask& ~Mtm->disabledNodeMask& ~ts->votedMask)==0) {

MtmWakeUpBackend(ts);

}

}

if (Mtm->status==MTM_ONLINE) {

now=MtmGetSystemTime();

if (n==0) {

```multimaster.node_id```Multimaster node ID,unique number identifyingthis node. Nodes should be numbered by natural numbers startingfrom 1without gaps (e.g. 1, 2, 3, ...). node_id is also used as an offset in```multimaster.conn_strings```, thus i-thnode's connection string expectedtobe on i-th position in```multimaster.conn_strings```. Mandatory.

```multimaster.node_id```Node ID - auniquenaturalnumber identifyingthe node of a multi-master cluster. You must start node numberingfrom 1and cannot have any gaps in numbering. For example, for a cluster of five nodes, setnode IDsto1, 2, 3, 4, and 5.

```multimaster.conn_strings```Multimasternodeconnection stringsseparated by commas, i.e.'dbname=mydb host=node1, dbname=mydb host=node2, dbname=mydb host=node3'.Order here is important and should be consistent with```multimaster.node_id```. Multimaster allows to specify custom arbiter_port value for all connection strings. Also thisparameteris expected tobe identical on all nodes.Mandatory.

```multimaster.conn_strings```Connection strings for eachnodeof a multi-master cluster,separated by commas. Each connection string must include the name of the database to replicate and the cluster node domain name. For example,'dbname=mydb host=node1, dbname=mydb host=node2, dbname=mydb host=node3'.Connection strings must appear in the order of the node IDs specified in the```multimaster.node_id``` variable. Connection string for the i-th node must be on the i-th position. Thisparametermustbe identical on all nodes.You can specify a custom port for all connection strings using the`multimaster.arbiter_port` variable.

```multimaster.arbiter_port``` Port for arbiter process to listen on. Default to 5433.

```multimaster.arbiter_port``` Port for the arbiter process to listen on.

Default: 5433

```multimaster.heartbeat_send_timeout```Period of broadcasting heartbeat messages byarbiter to all nodes. In milliseconds. Default to 1000.

```multimaster.heartbeat_send_timeout```Time interval between heartbeat messages, in milliseconds. Anarbiterprocess broadcasts heartbeat messagesto all nodes to detect connection problems. Default: 1000.

```multimaster.heartbeat_recv_timeout``` If no heartbeat message is received from node within this period, it assumed to be dead. In milliseconds. Default to 10000.

```multimaster.heartbeat_recv_timeout``` Timeout, in milliseconds. If no heartbeat message is received from the node within this timeframe, the node is excluded from the cluster.

Default: 10000

```multimaster.min_recovery_lag``` Minimal lag of WAL-sender performing recovery after which cluster is locked until recovery is completed. When wal-sender almost catch-up WAL current position we need to stop 'Achilles tortile competition' and temporary stop commit of new transactions until node will be completely repared. In bytes. Default to 100000.

```multimaster.max_recovery_lag``` Maximal lag of replication slot of failed node after which this slot is dropped to avoid transaction log overflow. Dropping slot makes it not possible to recover node using logical replication mechanism, it will be necessary to completely copy content of some alive node using pg_basebackup or similar tool. Zero value of parameter disable slot dropping. In bytes. Default to 100000000.

```multimaster.min_recovery_lag``` Minimal WAL lag between the current cluster state and the node to be restored, in bytes. When this threshold is reached during node recovery, the cluster is locked for write transactions until the recovery is complete.

Default: 100000

```multimaster.max_recovery_lag``` Maximal WAL lag size, in bytes. When a node is disconnected from the cluster, other nodes copy WALs for all new trasactions into the replication slot of this node. Upon reaching the`multimaster.max_recovery_lag` value, the replication slot for the disconnected node is deleted to avoid overflow. At this point, automatic recovery of the node is no longer possible. In this case, you can restore the node manually by cloning the data from one of the alive nodes using`pg_basebackup` or a similar tool. If you set this variable to zero, replication slot will not be deleted.

Default: 10000000

```multimaster.ignore_tables_without_pk``` Boolean. This variable enables/disables replication of tables without primary keys. By default, replication of tables without primary keys is disabled because of the logical replication restrictions. To enable replication, you can set this variable to false. However, take into account that`multimaster` does not allow update operations on such tables. Default: true

```multimaster.cluster_name``` Name of the cluster. If you set this variable,`multimaster` checks that the cluster name is the same for all the cluster nodes.

```multimaster.ignore_tables_without_pk``` Do not replicate tables withpout primary key. Boolean.

```multimaster.cluster_name``` Name of the cluster, desn't affect anything. Just in case. If set that mmts will check name correspondence.

```multimaster.gc_period``` Number of distributed transactions after which garbage collection is started. Multimaster is building xid->csn hash map which has to be cleaned to avoid hash overflow. This parameter specifies interval of invoking garbage collector for this map. default = MTM_HASH_SIZE/10

```multimaster.max_node``` Maximal number of cluster nodes. This parameters allows to add new nodes to the cluster, default value 0 restricts number of nodes to one specified in multimaster.conn_strings (May be just set that to 64 and allow user to add node when trey need without restart?) default = 0

```multimaster.node_disable_delay``` Minimal amount of time (msec) between node status change. This delay is used to avoid false detection of node failure and to prevent blinking of node status node. default = 2000. (We can just increase heartbeat_recv_timeout)

```multimaster.connect_timeout``` Multimaster nodes connect timeout. Interval in milliseconds for establishing connection with cluster node. default = 10000, /* 10 seconds*/

SnapshotMtmGetSnapshot(Snapshotsnapshot)

{

snapshot=PgGetSnapshotData(snapshot);

if (snapshot!=&CatalogSnapshotData) {

RecentGlobalDataXmin=RecentGlobalXmin=Mtm->oldestXid;

}

returnsnapshot;

snapshot=PgGetSnapshotData(snapshot);

RecentGlobalDataXmin=RecentGlobalXmin=Mtm->oldestXid;

returnsnapshot;

}