|
| 1 | +package Cluster; |
| 2 | + |
| 3 | +use strict; |
| 4 | +use warnings; |
| 5 | + |
| 6 | +use Proc::ProcessTable; |
| 7 | +use PostgresNode; |
| 8 | +use TestLib; |
| 9 | +use Test::More; |
| 10 | +use Cwd; |
| 11 | + |
| 12 | +my %allocated_ports = (); |
| 13 | +sub allocate_ports |
| 14 | +{ |
| 15 | + my @allocated_now = (); |
| 16 | + my ($host, $ports_to_alloc) = @_; |
| 17 | + |
| 18 | + while ($ports_to_alloc > 0) |
| 19 | + { |
| 20 | + my $port = int(rand() * 16384) + 49152; |
| 21 | + next if $allocated_ports{$port}; |
| 22 | + diag("checking for port $port\n"); |
| 23 | + if (!TestLib::run_log(['pg_isready', '-h', $host, '-p', $port])) |
| 24 | + { |
| 25 | + $allocated_ports{$port} = 1; |
| 26 | + push(@allocated_now, $port); |
| 27 | + $ports_to_alloc--; |
| 28 | + } |
| 29 | + } |
| 30 | + |
| 31 | + return @allocated_now; |
| 32 | +} |
| 33 | + |
| 34 | +sub new |
| 35 | +{ |
| 36 | + my ($class, $nodenum) = @_; |
| 37 | + |
| 38 | + my $nodes = []; |
| 39 | + |
| 40 | + foreach my $i (1..$nodenum) |
| 41 | + { |
| 42 | + my $host = "127.0.0.1"; |
| 43 | + my ($pgport, $arbiter_port) = allocate_ports($host, 2); |
| 44 | + my $node = new PostgresNode("node$i", $host, $pgport); |
| 45 | + $node->{id} = $i; |
| 46 | + $node->{arbiter_port} = $arbiter_port; |
| 47 | + push(@$nodes, $node); |
| 48 | + } |
| 49 | + |
| 50 | + my $self = { |
| 51 | + nodenum => $nodenum, |
| 52 | + nodes => $nodes, |
| 53 | + }; |
| 54 | + |
| 55 | + bless $self, $class; |
| 56 | + return $self; |
| 57 | +} |
| 58 | + |
| 59 | +sub init |
| 60 | +{ |
| 61 | + my ($self) = @_; |
| 62 | + my $nodes = $self->{nodes}; |
| 63 | + |
| 64 | + foreach my $node (@$nodes) |
| 65 | + { |
| 66 | + $node->init(hba_permit_replication => 0); |
| 67 | + } |
| 68 | +} |
| 69 | + |
| 70 | +sub configure |
| 71 | +{ |
| 72 | + my ($self) = @_; |
| 73 | + my $nodes = $self->{nodes}; |
| 74 | + my $nnodes = scalar @{ $nodes }; |
| 75 | + |
| 76 | + my $connstr = join(', ', map { "${ \$_->connstr('postgres') } arbiter_port=${ \$_->{arbiter_port} }" } @$nodes); |
| 77 | + |
| 78 | + foreach my $node (@$nodes) |
| 79 | + { |
| 80 | + my $id = $node->{id}; |
| 81 | + my $host = $node->host; |
| 82 | + my $pgport = $node->port; |
| 83 | + my $arbiter_port = $node->{arbiter_port}; |
| 84 | + |
| 85 | + $node->append_conf("postgresql.conf", qq( |
| 86 | + log_statement = none |
| 87 | + listen_addresses = '$host' |
| 88 | + unix_socket_directories = '' |
| 89 | + port = $pgport |
| 90 | + max_prepared_transactions = 200 |
| 91 | + max_connections = 200 |
| 92 | + max_worker_processes = 100 |
| 93 | + wal_level = logical |
| 94 | + fsync = off |
| 95 | + max_wal_senders = 10 |
| 96 | + wal_sender_timeout = 0 |
| 97 | + default_transaction_isolation = 'repeatable read' |
| 98 | + max_replication_slots = 10 |
| 99 | + shared_preload_libraries = 'multimaster' |
| 100 | +
|
| 101 | + multimaster.arbiter_port = $arbiter_port |
| 102 | + multimaster.workers = 10 |
| 103 | + multimaster.queue_size = 10485760 # 10mb |
| 104 | + multimaster.node_id = $id |
| 105 | + multimaster.conn_strings = '$connstr' |
| 106 | + multimaster.heartbeat_recv_timeout = 1000 |
| 107 | + multimaster.heartbeat_send_timeout = 250 |
| 108 | + multimaster.max_nodes = $nnodes |
| 109 | + multimaster.ignore_tables_without_pk = true |
| 110 | + multimaster.twopc_min_timeout = 50000 |
| 111 | + multimaster.min_2pc_timeout = 50000 |
| 112 | + log_line_prefix = '%t: ' |
| 113 | + )); |
| 114 | + |
| 115 | + $node->append_conf("pg_hba.conf", qq( |
| 116 | + local replication all trust |
| 117 | + host replication all 127.0.0.1/32 trust |
| 118 | + host replication all ::1/128 trust |
| 119 | + )); |
| 120 | + } |
| 121 | +} |
| 122 | + |
| 123 | +sub start |
| 124 | +{ |
| 125 | + my ($self) = @_; |
| 126 | + my $nodes = $self->{nodes}; |
| 127 | + |
| 128 | + foreach my $node (@$nodes) |
| 129 | + { |
| 130 | + $node->start(); |
| 131 | + } |
| 132 | +} |
| 133 | + |
| 134 | +sub stopnode |
| 135 | +{ |
| 136 | + my ($node, $mode) = @_; |
| 137 | + return 1 unless defined $node->{_pid}; |
| 138 | + $mode = 'fast' unless defined $mode; |
| 139 | + my $name = $node->name; |
| 140 | + diag("stopping $name ${mode}ly"); |
| 141 | + |
| 142 | + if ($mode eq 'kill') { |
| 143 | + killtree($node->{_pid}); |
| 144 | + return 1; |
| 145 | + } |
| 146 | + |
| 147 | + my $pgdata = $node->data_dir; |
| 148 | + my $ret = TestLib::system_log('pg_ctl', '-D', $pgdata, '-m', 'fast', 'stop'); |
| 149 | + my $pidfile = $node->data_dir . "/postmaster.pid"; |
| 150 | + diag("unlink $pidfile"); |
| 151 | + unlink $pidfile; |
| 152 | + $node->{_pid} = undef; |
| 153 | + $node->_update_pid; |
| 154 | + |
| 155 | + if ($ret != 0) { |
| 156 | + diag("$name failed to stop ${mode}ly"); |
| 157 | + return 0; |
| 158 | + } |
| 159 | + |
| 160 | + return 1; |
| 161 | +} |
| 162 | + |
| 163 | +sub stopid |
| 164 | +{ |
| 165 | + my ($self, $idx, $mode) = @_; |
| 166 | + return stopnode($self->{nodes}->[$idx]); |
| 167 | +} |
| 168 | + |
| 169 | +sub killtree |
| 170 | +{ |
| 171 | + my $root = shift; |
| 172 | + diag("killtree $root\n"); |
| 173 | + |
| 174 | + my $t = new Proc::ProcessTable; |
| 175 | + |
| 176 | + my %parent = (); |
| 177 | + #my %cmd = (); |
| 178 | + foreach my $p (@{$t->table}) { |
| 179 | + $parent{$p->pid} = $p->ppid; |
| 180 | + # $cmd{$p->pid} = $p->cmndline; |
| 181 | + } |
| 182 | + |
| 183 | + if (!defined $root) { |
| 184 | + return; |
| 185 | + } |
| 186 | + my @queue = ($root); |
| 187 | + my @killist = (); |
| 188 | + |
| 189 | + while (scalar @queue) { |
| 190 | + my $victim = shift @queue; |
| 191 | + while (my ($pid, $ppid) = each %parent) { |
| 192 | + if ($ppid == $victim) { |
| 193 | + push @queue, $pid; |
| 194 | + } |
| 195 | + } |
| 196 | + diag("SIGSTOP to $victim"); |
| 197 | + kill 'STOP', $victim; |
| 198 | + unshift @killist, $victim; |
| 199 | + } |
| 200 | + |
| 201 | + diag("SIGKILL to " . join(' ', @killist)); |
| 202 | + kill 'KILL', @killist; |
| 203 | + #foreach my $victim (@killist) { |
| 204 | + # print("kill $victim " . $cmd{$victim} . "\n"); |
| 205 | + #} |
| 206 | +} |
| 207 | + |
| 208 | +sub stop |
| 209 | +{ |
| 210 | + my ($self, $mode) = @_; |
| 211 | + my $nodes = $self->{nodes}; |
| 212 | + $mode = 'fast' unless defined $mode; |
| 213 | + |
| 214 | + diag("Dumping logs:"); |
| 215 | + foreach my $node (@$nodes) { |
| 216 | + diag("##################################################################"); |
| 217 | + diag($node->{_logfile}); |
| 218 | + diag("##################################################################"); |
| 219 | + my $filename = $node->{_logfile}; |
| 220 | + open my $fh, '<', $filename or die "error opening $filename: $!"; |
| 221 | + my $data = do { local $/; <$fh> }; |
| 222 | + diag($data); |
| 223 | + diag("##################################################################\n\n"); |
| 224 | + } |
| 225 | + |
| 226 | + my $ok = 1; |
| 227 | + diag("stopping cluster ${mode}ly"); |
| 228 | + |
| 229 | + foreach my $node (@$nodes) { |
| 230 | + if (!stopnode($node, $mode)) { |
| 231 | + $ok = 0; |
| 232 | + if (!stopnode($node, 'kill')) { |
| 233 | + my $name = $node->name; |
| 234 | + BAIL_OUT("failed to kill $name"); |
| 235 | + } |
| 236 | + } |
| 237 | + } |
| 238 | + sleep(2); |
| 239 | + return $ok; |
| 240 | +} |
| 241 | + |
| 242 | +sub teardown |
| 243 | +{ |
| 244 | + my ($self) = @_; |
| 245 | + my $nodes = $self->{nodes}; |
| 246 | + |
| 247 | + foreach my $node (@$nodes) |
| 248 | + { |
| 249 | + $node->teardown(); |
| 250 | + } |
| 251 | +} |
| 252 | + |
| 253 | +sub psql |
| 254 | +{ |
| 255 | + my ($self, $index, @args) = @_; |
| 256 | + my $node = $self->{nodes}->[$index]; |
| 257 | + return $node->psql(@args); |
| 258 | +} |
| 259 | + |
| 260 | +sub poll |
| 261 | +{ |
| 262 | + my ($self, $poller, $dbname, $pollee, $tries, $delay) = @_; |
| 263 | + my $node = $self->{nodes}->[$poller]; |
| 264 | + for (my $i = 0; $i < $tries; $i++) { |
| 265 | + my $psql_out; |
| 266 | + my $pollee_plus_1 = $pollee + 1; |
| 267 | + $self->psql($poller, $dbname, "select mtm.poll_node($pollee_plus_1, true);", stdout => \$psql_out); |
| 268 | + if ($psql_out eq "t") { |
| 269 | + return 1; |
| 270 | + } |
| 271 | + my $tries_left = $tries - $i - 1; |
| 272 | + diag("$poller poll for $pollee failed [$tries_left tries left]"); |
| 273 | + sleep($delay); |
| 274 | + } |
| 275 | + return 0; |
| 276 | +} |
| 277 | + |
| 278 | +1; |
0 commit comments