diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..2877737 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2017 HelloTech + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md index f2fa614..a13826c 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,64 @@ -# Azure Patroni Deployment - - + - \ No newline at end of file +http://armviz.io/#/?load=https%3A%2F%2Fraw.githubusercontent.com%2Fhellotech%2Fazure_postgres%2Fmaster%2Ftemplate.json" target="_blank"> + + +# Highly-Available PostgreSQL Cluster (Patroni) on Azure + +This one click deployment of a Highly-Available PostgreSQL Cluster on Azure has automated replication, server management and leader election. This project contains a modified deployment version of Haufe's [blog post](http://dev.haufe.com/PostgreSQL-Cluster-Azure/). + +This Azure template generates two sets of machines both running Ubuntu 16.04 LTS. The first set of three machines are used to run the distributed configuration store [ZooKeeper](https://zookeeper.apache.org/), while the second set is running PostgreSQL 9.6 together with [Patroni](https://github.com/zalando/patroni) providing a high-availability customizable PostgreSQL cluster deployment. This deployment also includes [PL/V8 v.2.0.0](https://github.com/plv8/plv8). + +# Parameters + +This template provides the following parameters, so you can customize your deployment to your needs: + + +* clusterName: What your cluster will be named. + +* _artifactsLocation: The repo name from which you are deploying. Default("https://raw.githubusercontent.com/HelloTech/azure_postgres/master") + +* newVnet: Whether or not you want to create a new vnet. Default("yes") + +* vnetGroup: The resource group in which the vnet is located. If you select yes on newVnet this needs to be set to the resource group in which you are deploying. + +* lbType: Whether you want to create an external or internal load balancer. If set to internal the load balancer will only be accessible from inside the virtual network. Default("internal") + +* vnetName: If newVnet is set to yes, this will be name of the created vnet. Otherwise, this is the name of already existing vnet to which the deployment will be associated. + +* zookeeperNetName: If newVnet is set to yes, this will be name of the created subnet for the zookeeper machines, otherwise this is the name of already existing subnet that will contain the zookeeper vm's. + +* postgresNetName: If newVnet is set to yes, this will be name of the created subnet for the postgres machines, otherwise this is the name of already existing subnet that will contain the postgres vm's. + +* zookeeperVMSize: The size of zookeeper vms. + +* postgresVMSize: The size of the postgres vms. + +* postgresDataSize: The size of the ssd used for pg_data for the postgres vms. + +* instanceCount: The number of postgres vms. + +* adminUsername: The Ubuntu username. + +* adminPassword: The Ubuntu password. + + +# Scripts + +### Zookeeper +The creation of the zookeeper instances is handled by the [zookeeper_startup.sh](https://github.com/HelloTech/azure_postgres/blob/custom_deploy/zookeeper_startup.sh) script. If you would like to make any changes to what is installed on those vm's modify this file. + +### Mounts +The mounting of the data ssds is handled by the [autopart.sh](https://github.com/HelloTech/azure_postgres/blob/custom_deploy/autopart.sh) script. + +### Postgres +The creation of the postgres instance is handles by the [postgres_startup.sh](https://github.com/HelloTech/azure_postgres/blob/custom_deploy/postgres_startup.sh) script. If you would like to make any changes to what is installed on those vm's modify that script. + +### Notes +This deployment can take anywhere from 90 to 120 minutes to complete due to long build time for plv8. + +# License + +Released under the MIT license. See the LICENSE file for more info. \ No newline at end of file diff --git a/backup.json b/backup.json index ba2c0a3..ab89f52 100644 --- a/backup.json +++ b/backup.json @@ -224,7 +224,7 @@ "typeHandlerVersion": "1.2", "settings": { "fileUris": [ - "https://raw.githubusercontent.com/HelloTech/azure_patroni/master/zookeeper_startup.sh" + "https://raw.githubusercontent.com/HelloTech/azure_postgres/master/zookeeper_startup.sh" ], "commandToExecute": "[concat('sh zookeeper_startup.sh ', copyIndex(), ' ', variables('zookeeperInstanceCount'))]" } @@ -353,8 +353,8 @@ "typeHandlerVersion": "1.2", "settings": { "fileUris": [ - "https://raw.githubusercontent.com/HelloTech/azure_patroni/master/autopart.sh", - "https://raw.githubusercontent.com/HelloTech/azure_patroni/master/postgres_startup.sh" + "https://raw.githubusercontent.com/HelloTech/azure_postgres/master/autopart.sh", + "https://raw.githubusercontent.com/HelloTech/azure_postgres/master/postgres_startup.sh" ], "commandToExecute": "[concat('sh postgres_startup.sh ', parameters('clusterName'), ' ', variables('zookeeperInstanceCount'), ' ', variables('postgresInstanceCount'), ' ', copyIndex(), ' ', parameters('adminUsername'), ' \"', parameters('adminPassword'), '\" ')]" } diff --git a/postgres_startup.sh b/postgres_startup.sh index d9b3ff0..71c9d83 100644 --- a/postgres_startup.sh +++ b/postgres_startup.sh @@ -25,6 +25,8 @@ sudo chmod +x ./autopart.sh >> /usr/local/startup.log sudo ./autopart.sh >> /usr/local/startup.log sudo mkdir /media/data1/data sudo chmod 777 /media/data1/data +echo "/media/data1/data partition created" >> /usr/local/startup.log +echo "" >> /usr/local/startup.log # create RAID #sudo apt-get --assume-yes -qq install mdadm @@ -38,7 +40,10 @@ sudo chmod 777 /media/data1/data sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" >> /etc/apt/sources.list.d/pgdg.list' wget -q https://www.postgresql.org/media/keys/ACCC4CF8.asc -O - | sudo apt-key add - sudo apt-get -qq update +sudo apt-get --assume-yes -qq install jq echo "apt-get update exited with: $?" +echo "updated packages" >> /usr/local/startup.log +echo "" >> /usr/local/startup.log # install python @@ -72,16 +77,30 @@ sudo pip -q install tzlocal echo "pip tzlocal exited with: $?" sudo pip -q install python-dateutil echo "pip python-dateutil exited with: $?" - +echo "installed python" >> /usr/local/startup.log +echo "" >> /usr/local/startup.log # install postgres # prevent postgres from autostart #sudo touch /usr/sbin/policy-rc.d #sudo chmod 777 /usr/sbin/policy-rc.d #echo exit 101 > /usr/sbin/policy-rc.d -sudo apt-get --assume-yes --force-yes -qq install postgresql postgresql-contrib -echo "apt-get postgresql postgresql-contrib exited with: $?" +sudo apt-get --assume-yes --force-yes -qq install postgresql postgresql-contrib postgresql-server-dev-9.6 +echo "apt-get postgresql postgresql-server-dev-9.6 postgresql-contrib exited with: $?" export PATH=/usr/lib/postgresql/9.6/bin:$PATH +echo "installed postgres" >> /usr/local/startup.log +echo "" >> /usr/local/startup.log + +#install plv8 +wget https://github.com/plv8/plv8/archive/v2.0.0.tar.gz +tar -xvzf v2.0.0.tar.gz +cd plv8-2.0.0 +make static +sudo cp plv8.so /usr/lib/postgresql/9.6/lib/ +sudo cp plv8.control /usr/share/postgresql/9.6/extension/ +sudo cp plv8--2.0.0.sql /usr/share/postgresql/9.6/extension/ +echo "installed plv8" >> /usr/local/startup.log +echo "" >> /usr/local/startup.log # download patroni sudo apt-get --assume-yes -qq install unzip @@ -90,6 +109,8 @@ cd /usr/local sudo wget -O /usr/local/patroni-master.zip https://github.com/zalando/patroni/archive/master.zip sudo unzip patroni-master.zip cd patroni-master +echo "download patroni" >> /usr/local/startup.log +echo "" >> /usr/local/startup.log # write configuration sudo touch $patroniCfg @@ -161,6 +182,7 @@ fi echo " listen: '*:5433'" >> $patroniCfg echo " connect_address: 10.0.101.$(($myIndex + 10)):5433" >> $patroniCfg echo " data_dir: /media/data1/data/postgresql" >> $patroniCfg +echo " bin_dir: /usr/lib/postgresql/9.6/bin" >> $patroniCfg echo " pgpass: /tmp/pgpass" >> $patroniCfg if [ $myIndex -ne 0 ] then @@ -210,6 +232,8 @@ if [ $myIndex -ne 0 ] echo " parameters:" >> $patroniCfg echo " unix_socket_directories: '.'" >> $patroniCfg fi +echo "setup patroni configuration" >> /usr/local/startup.log +echo "" >> /usr/local/startup.log # install HA PROXY @@ -242,15 +266,46 @@ do echo " server Postgres$i 10.0.101.$(($i + 10)):5433 maxconn 100 check port 8008" >> $hacfgFile i=$(($i+1)) done +echo "installed haproxy" >> /usr/local/startup.log +echo "" >> /usr/local/startup.log # start ha as deamon sudo haproxy -D -f $hacfgFile +echo "started haproxy" >> /usr/local/startup.log +echo "" >> /usr/local/startup.log # prepare patroni to restart after reboot -sudo chmod 777 /etc/rc.local -echo "sudo haproxy -D -f /usr/local/patroni-master/$hacfgFile" > /etc/rc.local -echo "su -s /bin/bash -c 'export PATH=/usr/lib/postgresql/9.6/bin:\$PATH; /usr/local/patroni-master/patroni.py /usr/local/patroni-master/$patroniCfg >> /usr/local/startup.log' $adminUsername" >> /etc/rc.local -echo "exit 0" >> /etc/rc.local +sudo touch /etc/systemd/system/patroni.service +sudo chmod 777 /etc/systemd/system/patroni.service +echo "[Unit]" > /etc/systemd/system/patroni.service +echo "Description=patroni script" >> /etc/systemd/system/patroni.service +echo "" >> /etc/systemd/system/patroni.service +echo "[Service]" >> /etc/systemd/system/patroni.service +echo "User=$adminUsername" >> /etc/systemd/system/patroni.service +echo "WorkingDirectory=usr/local/patroni-master" >> /etc/systemd/system/patroni.service +echo "ExecStart=/usr/bin/python /usr/local/patroni-master/patroni.py /usr/local/patroni-master/postgres.yml >> /usr/local/startup.log" >> /etc/systemd/system/patroni.service +echo "Restart=always" >> /etc/systemd/system/patroni.service +echo "" >> /etc/systemd/system/patroni.service +echo "[Install]" >> /etc/systemd/system/patroni.service +echo "WantedBy=multi-user.target" >> /etc/systemd/system/patroni.service +sudo systemctl enable patroni.service +echo "setup reboot script for patroni" >> /usr/local/startup.log +echo "" >> /usr/local/startup.log + +# prepare haproxy to restart after reboot +sudo chmod 777 /etc/default/haproxy +echo "ENABLED=1" >> /etc/default/haproxy +echo "CONFIG=\"/usr/local/patroni-master/postgresha.cfg\"" >> /etc/default/haproxy +sudo chmod 644 /etc/default/haproxy +echo "setup reboot config for haproxy" >> /usr/local/startup.log +echo "" >> /usr/local/startup.log + +# start haproxy +sudo haproxy -D -f /usr/local/patroni-master/postgresha.cfg +echo "started haproxy" >> /usr/local/startup.log +echo "" >> /usr/local/startup.log # start patroni -su -c "export PATH=/usr/lib/postgresql/9.6/bin:$PATH; ./patroni.py $patroniCfg >> /usr/local/startup.log" $adminUsername & echo "spawned" +sudo systemctl start patroni.service +echo "started patroni" >> /usr/local/startup.log +echo "" >> /usr/local/startup.log \ No newline at end of file diff --git a/template.json b/template.json index 6252be5..76adb15 100644 --- a/template.json +++ b/template.json @@ -7,7 +7,7 @@ }, "_artifactsLocation": { "type": "string", - "defaultValue": "https://raw.githubusercontent.com/HelloTech/azure_patroni/custom_deploy", + "defaultValue": "https://raw.githubusercontent.com/HelloTech/azure_postgres/master", "metadata": { "description": "Change this value to your repo name if deploying from a fork" }