From 3091f8d2d8bef1528c96ea8080f08bc086d3d6fc Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Wed, 1 Feb 2017 17:54:26 -0800 Subject: [PATCH 01/51] created conditional template for network --- nestedtemplates/empty-resources.json | 26 +++++++ nestedtemplates/network-resources.json | 56 +++++++++++++++ template.json | 98 ++++++++++++++++++++++++-- 3 files changed, 174 insertions(+), 6 deletions(-) create mode 100644 nestedtemplates/empty-resources.json create mode 100644 nestedtemplates/network-resources.json diff --git a/nestedtemplates/empty-resources.json b/nestedtemplates/empty-resources.json new file mode 100644 index 0000000..ee32b84 --- /dev/null +++ b/nestedtemplates/empty-resources.json @@ -0,0 +1,26 @@ + +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "vnetName": { + "type": "String", + "defaultValue": "" + }, + "zookeeperNetName": { + "type": "String", + "defaultValue": "" + }, + "postgresNetName": { + "type": "String", + "defaultValue": "" + }, + "clusterName": { + "type": "String", + "defaultValue": "" + } + }, + "variables": {}, + "resources": [], + "outputs": {} +} diff --git a/nestedtemplates/network-resources.json b/nestedtemplates/network-resources.json new file mode 100644 index 0000000..9f315e1 --- /dev/null +++ b/nestedtemplates/network-resources.json @@ -0,0 +1,56 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "vnetName": { + "type": "String", + "defaultValue": "" + }, + "zookeeperNetName": { + "type": "String", + "defaultValue": "" + }, + "postgresNetName": { + "type": "String", + "defaultValue": "" + }, + "clusterName": { + "type": "String", + "defaultValue": "" + } + }, + "variables": { + "vnetName": "[concat(parameters('clusterName'), '-net')]" + }, + "resources": [ + { + "type": "Microsoft.Network/virtualNetworks", + "name": "[variables('vnetName')]", + "apiVersion": "2015-06-15", + "location": "[resourceGroup().location]", + "tags": {}, + "properties": { + "addressSpace": { + "addressPrefixes": [ + "10.0.0.0/16" + ] + }, + "subnets": [ + { + "name": "[parameters('zookeeperNetName')]", + "properties": { + "addressPrefix": "10.0.100.0/24" + } + }, + { + "name": "[parameters('postgresNetName')]", + "properties": { + "addressPrefix": "10.0.101.0/24" + } + } + ] + }, + "dependsOn": [] + } + ] +} \ No newline at end of file diff --git a/template.json b/template.json index e97d107..801ed88 100644 --- a/template.json +++ b/template.json @@ -5,6 +5,63 @@ "clusterName": { "type": "String" }, + "_artifactsLocation": { + "type": "string", + "defaultValue": "https://raw.githubusercontent.com/HelloTech/azure_patroni/master", + "metadata": { + "description": "Change this value to your repo name if deploying from a fork" + } + }, + "vnetGroup": { + "defaultValue": "", + "type": "String", + "metadata": { + "description": "VNET Resource Group." + } + }, + "newVnet": { + "type": "string", + "defaultValue": "yes", + "allowedValues": [ + "yes", + "no" + ], + "metadata": { + "description": "Would you like to create a new vnet?" + } + }, + "lbType": { + "type": "string", + "defaultValue": "yes", + "allowedValues": [ + "yes", + "no" + ], + "metadata": { + "description": "Would you like to create an internal or external load balancer?" + } + }, + "vnetName": { + "defaultValue": "", + "type": "String", + "metadata": { + "description": "Name of the vnet you would like to create, or that you would like to deploy this into." + } + }, + "zookeeperNetName": { + "defaultValue": "dcos-agentPublicSubnet", + "type": "String", + "metadata": { + "description": "Name you would like to give to zookeeper's subnet." + } + }, + "postgresNetName": { + "defaultValue": "dcos-agentPublicSubnet", + "type": "String", + "metadata": { + "description": "Name you would like to give to postgres' subnet." + } + }, "instanceCount": { "defaultValue": 2, "minValue": 2, @@ -30,15 +87,18 @@ "s", "y" ], + "nestedTemplates": "[concat(parameters('_artifactsLocation'), '/nestedtemplates/']", "storageAccountType": "Standard_LRS", "lbName": "postgresLoadBalancer", + "vnetFile": { + "yes": "network-resources.json", + "no": "empty-resources.json" + }, "lbId": "[resourceId('Microsoft.Network/loadBalancers/', variables('lbName'))]", "vnetName": "dcos-vnet-F0547AD4", "vnetGroup": "testht", - "vnetId": "[resourceId(variables('vnetGroup'), 'Microsoft.Network/virtualNetworks', variables('vnetName'))]", - "zookeeperNetName": "dcos-agentPublicSubnet", + "vnetId": "[resourceId(parameters('vnetGroup'), 'Microsoft.Network/virtualNetworks', parameters('vnetName'))]", "zookeeperInstanceCount": 3, - "postgresNetName": "dcos-agentPublicSubnet", "postgresInstanceCount": "[parameters('instanceCount')]", "osType": { "publisher": "Canonical", @@ -48,6 +108,32 @@ } }, "resources": [ + { + "type": "Microsoft.Network/virtualNetworks", + "name": "[variables('vnetName')]", + "apiVersion": "2015-06-15", + "properties": { + "mode": "Incremental", + "templateLink": { + "uri": "[concat(variables('sharedTemplateFileName'), variables('vnetFile')[parameters('newVnet')])]", + "contentVersion": "1.0.0.0" + }, + "parameters": { + "vnetName": { + "value": "[variables('vnetName')]" + }, + "zookeeperNetName": { + "value": "[parameters('zookeeperNetName')]" + }, + "postgresNetName": { + "value": "[parameters('postgresNetName')]" + }, + "clusterName": { + "value": "[variables('clusterName')]" + } + } + } + }, { "type": "Microsoft.Storage/storageAccounts", "name": "[concat(variables('storageAccountPrefix')[copyIndex()], variables('storageAccountSuffix'))]", @@ -105,7 +191,7 @@ "privateIPAllocationMethod": "Static", "privateIPAddress": "[concat('10.0.100.', copyIndex(10))]", "subnet": { - "id": "[concat(variables('vnetId'), '/subnets/', variables('zookeeperNetName'))]" + "id": "[concat(variables('vnetId'), '/subnets/', parameters('zookeeperNetName'))]" } } } @@ -207,7 +293,7 @@ "privateIPAllocationMethod": "Static", "privateIPAddress": "[concat('10.0.101.', copyIndex(10))]", "subnet": { - "id": "[concat(variables('vnetId'), '/subnets/', variables('postgresNetName'))]" + "id": "[concat(variables('vnetId'), '/subnets/', parameters('postgresNetName'))]" }, "loadBalancerBackendAddressPools": [ { @@ -335,7 +421,7 @@ "privateIPAddress": "10.0.101.4", "privateIPAllocationMethod": "Static", "subnet": { - "id": "[concat(variables('vnetId'), '/subnets/', variables('zookeeperNetName'))]" + "id": "[concat(variables('vnetId'), '/subnets/', parameters('zookeeperNetName'))]" } } } From bfb822654af5dd53bb743733d8c54fb72a61a2e5 Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Wed, 1 Feb 2017 18:06:45 -0800 Subject: [PATCH 02/51] added internal and external lb --- nestedtemplates/external-lb.json | 204 +++++++++++++++++++++++++++++++ nestedtemplates/internal-lb.json | 0 template.json | 27 ++++ 3 files changed, 231 insertions(+) create mode 100644 nestedtemplates/external-lb.json create mode 100644 nestedtemplates/internal-lb.json diff --git a/nestedtemplates/external-lb.json b/nestedtemplates/external-lb.json new file mode 100644 index 0000000..4b6c787 --- /dev/null +++ b/nestedtemplates/external-lb.json @@ -0,0 +1,204 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "clusterName": { + "type": "String", + "defaultValue": "" + }, + "lbName": { + "type": "String", + "defaultValue": "" + }, + "lbId": { + "type": "String", + "defaultValue": "" + } + }, + "variables": {}, + "resources": [ + { + "type": "Microsoft.Network/publicIPAddresses", + "name": "[concat(parameters('clusterName'), '-ip')]", + "apiVersion": "2016-03-30", + "location": "[resourceGroup().location]", + "properties": { + "publicIPAllocationMethod": "Dynamic", + "dnsSettings": { + "domainNameLabel": "[parameters('clusterName')]" + } + } + }, + { + "type": "Microsoft.Network/loadBalancers", + "name": "[parameters('lbName')]", + "apiVersion": "2015-06-15", + "location": "[resourceGroup().location]", + "properties": { + "availabilitySet": { + "id": "[resourceId('Microsoft.Compute/availabilitySets', 'postgresHaAvailabilitySet')]" + }, + "frontendIPConfigurations": [ + { + "name": "loadBalancerFrontEnd", + "properties": { + "publicIPAddress": { + "id": "[parameters('ipId')]" + } + } + } + ], + "backendAddressPools": [ + { + "name": "loadBalancerBackEnd" + } + ], + "loadBalancingRules": [ + { + "name": "postgresLbRule", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "backendAddressPool": { + "id": "[concat(parameters('lbId'), '/backendAddressPools/loadBalancerBackEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 5432, + "backendPort": 5000, + "probe": { + "id": "[concat(parameters('lbId'), '/probes/postgresProbe')]" + } + } + } + ], + "probes": [ + { + "name": "postgresProbe", + "properties": { + "protocol": "Tcp", + "port": 5000, + "intervalInSeconds": 5, + "numberOfProbes": 2 + } + } + ], + "inboundNatRules": [ + { + "name": "postgresSsh0", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10110, + "backendPort": 22 + } + }, + { + "name": "postgresSsh1", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10111, + "backendPort": 22 + } + }, + { + "name": "postgresSsh2", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10112, + "backendPort": 22 + } + }, + { + "name": "postgresSsh3", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10113, + "backendPort": 22 + } + }, + { + "name": "postgresSsh4", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10114, + "backendPort": 22 + } + }, + { + "name": "postgresSsh5", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10115, + "backendPort": 22 + } + }, + { + "name": "postgresSsh6", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10116, + "backendPort": 22 + } + }, + { + "name": "postgresSsh7", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10117, + "backendPort": 22 + } + }, + { + "name": "postgresSsh8", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10118, + "backendPort": 22 + } + }, + { + "name": "postgresSsh9", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10119, + "backendPort": 22 + } + } + ] + }, + "dependsOn": [ + "[variables('ipId')]" + ] + } + ] +} \ No newline at end of file diff --git a/nestedtemplates/internal-lb.json b/nestedtemplates/internal-lb.json new file mode 100644 index 0000000..e69de29 diff --git a/template.json b/template.json index 801ed88..2686b53 100644 --- a/template.json +++ b/template.json @@ -94,6 +94,10 @@ "yes": "network-resources.json", "no": "empty-resources.json" }, + "lbFile": { + "yes": "external-lb.json", + "no": "internal-lb.json" + }, "lbId": "[resourceId('Microsoft.Network/loadBalancers/', variables('lbName'))]", "vnetName": "dcos-vnet-F0547AD4", "vnetGroup": "testht", @@ -405,6 +409,29 @@ "[concat('Microsoft.Compute/virtualMachines/', 'postgres', copyIndex())]" ] }, + { + "type": "Microsoft.Network/loadBalancers", + "name": "[variables('lbName')]", + "apiVersion": "2015-06-15", + "properties": { + "mode": "Incremental", + "templateLink": { + "uri": "[concat(variables('sharedTemplateFileName'), variables('lbFile')[parameters('lbType')])]", + "contentVersion": "1.0.0.0" + }, + "parameters": { + "clusterName": { + "value": "[parameters('clusterName')]" + }, + "lbName": { + "value": "[variables('lbName')]" + }, + "lbId": { + "value": "[variables('lbId')]" + } + } + } + }, { "type": "Microsoft.Network/loadBalancers", "name": "[variables('lbName')]", From 3c19c0dccbbe67cb9773046104e9286579d7a843 Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Wed, 1 Feb 2017 18:08:26 -0800 Subject: [PATCH 03/51] added internal lb template --- nestedtemplates/internal-lb.json | 191 +++++++++++++++++++++++++++++++ template.json | 170 --------------------------- 2 files changed, 191 insertions(+), 170 deletions(-) diff --git a/nestedtemplates/internal-lb.json b/nestedtemplates/internal-lb.json index e69de29..37c447f 100644 --- a/nestedtemplates/internal-lb.json +++ b/nestedtemplates/internal-lb.json @@ -0,0 +1,191 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "clusterName": { + "type": "String", + "defaultValue": "" + }, + "lbName": { + "type": "String", + "defaultValue": "" + }, + "lbId": { + "type": "String", + "defaultValue": "" + } + }, + "variables": {}, + "resources": [ + { + "type": "Microsoft.Network/loadBalancers", + "name": "[variables('lbName')]", + "apiVersion": "2015-06-15", + "location": "[resourceGroup().location]", + "properties": { + "availabilitySet": { + "id": "[resourceId('Microsoft.Compute/availabilitySets', 'postgresHaAvailabilitySet')]" + }, + "frontendIPConfigurations": [ + { + "name": "LoadBalancerFrontEnd", + "properties": { + "privateIPAddress": "10.0.101.4", + "privateIPAllocationMethod": "Static", + "subnet": { + "id": "[concat(variables('vnetId'), '/subnets/', parameters('zookeeperNetName'))]" + } + } + } + ], + "backendAddressPools": [ + { + "name": "loadBalancerBackEnd" + } + ], + "loadBalancingRules": [ + { + "name": "postgresLbRule", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "backendAddressPool": { + "id": "[concat(variables('lbId'), '/backendAddressPools/loadBalancerBackEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 5432, + "backendPort": 5000, + "probe": { + "id": "[concat(variables('lbId'), '/probes/postgresProbe')]" + } + } + } + ], + "probes": [ + { + "name": "postgresProbe", + "properties": { + "protocol": "Tcp", + "port": 5000, + "intervalInSeconds": 5, + "numberOfProbes": 2 + } + } + ], + "inboundNatRules": [ + { + "name": "postgresSsh0", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10110, + "backendPort": 22 + } + }, + { + "name": "postgresSsh1", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10111, + "backendPort": 22 + } + }, + { + "name": "postgresSsh2", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10112, + "backendPort": 22 + } + }, + { + "name": "postgresSsh3", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10113, + "backendPort": 22 + } + }, + { + "name": "postgresSsh4", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10114, + "backendPort": 22 + } + }, + { + "name": "postgresSsh5", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10115, + "backendPort": 22 + } + }, + { + "name": "postgresSsh6", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10116, + "backendPort": 22 + } + }, + { + "name": "postgresSsh7", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10117, + "backendPort": 22 + } + }, + { + "name": "postgresSsh8", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10118, + "backendPort": 22 + } + }, + { + "name": "postgresSsh9", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10119, + "backendPort": 22 + } + } + ] + } + } + ] +} \ No newline at end of file diff --git a/template.json b/template.json index 2686b53..934589c 100644 --- a/template.json +++ b/template.json @@ -431,176 +431,6 @@ } } } - }, - { - "type": "Microsoft.Network/loadBalancers", - "name": "[variables('lbName')]", - "apiVersion": "2015-06-15", - "location": "[resourceGroup().location]", - "properties": { - "availabilitySet": { - "id": "[resourceId('Microsoft.Compute/availabilitySets', 'postgresHaAvailabilitySet')]" - }, - "frontendIPConfigurations": [ - { - "name": "LoadBalancerFrontEnd", - "properties": { - "privateIPAddress": "10.0.101.4", - "privateIPAllocationMethod": "Static", - "subnet": { - "id": "[concat(variables('vnetId'), '/subnets/', parameters('zookeeperNetName'))]" - } - } - } - ], - "backendAddressPools": [ - { - "name": "loadBalancerBackEnd" - } - ], - "loadBalancingRules": [ - { - "name": "postgresLbRule", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "backendAddressPool": { - "id": "[concat(variables('lbId'), '/backendAddressPools/loadBalancerBackEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 5432, - "backendPort": 5000, - "probe": { - "id": "[concat(variables('lbId'), '/probes/postgresProbe')]" - } - } - } - ], - "probes": [ - { - "name": "postgresProbe", - "properties": { - "protocol": "Tcp", - "port": 5000, - "intervalInSeconds": 5, - "numberOfProbes": 2 - } - } - ], - "inboundNatRules": [ - { - "name": "postgresSsh0", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10110, - "backendPort": 22 - } - }, - { - "name": "postgresSsh1", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10111, - "backendPort": 22 - } - }, - { - "name": "postgresSsh2", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10112, - "backendPort": 22 - } - }, - { - "name": "postgresSsh3", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10113, - "backendPort": 22 - } - }, - { - "name": "postgresSsh4", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10114, - "backendPort": 22 - } - }, - { - "name": "postgresSsh5", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10115, - "backendPort": 22 - } - }, - { - "name": "postgresSsh6", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10116, - "backendPort": 22 - } - }, - { - "name": "postgresSsh7", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10117, - "backendPort": 22 - } - }, - { - "name": "postgresSsh8", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10118, - "backendPort": 22 - } - }, - { - "name": "postgresSsh9", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10119, - "backendPort": 22 - } - } - ] - } } ], "outputs": { From 61769baf968275096b4548dcbc7134c610781bf7 Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Wed, 1 Feb 2017 18:11:16 -0800 Subject: [PATCH 04/51] switched default base branch location --- template.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/template.json b/template.json index 934589c..85f4960 100644 --- a/template.json +++ b/template.json @@ -271,7 +271,7 @@ "typeHandlerVersion": "1.2", "settings": { "fileUris": [ - "https://raw.githubusercontent.com/HelloTech/azure_patroni/master/zookeeper_startup.sh" + "concat(variables('sharedTemplateFileName'), '/zookeeper_startup.sh'" ], "commandToExecute": "[concat('sh zookeeper_startup.sh ', copyIndex(), ' ', variables('zookeeperInstanceCount'))]" } @@ -399,8 +399,8 @@ "typeHandlerVersion": "1.2", "settings": { "fileUris": [ - "https://raw.githubusercontent.com/HelloTech/azure_patroni/master/autopart.sh", - "https://raw.githubusercontent.com/HelloTech/azure_patroni/master/postgres_startup.sh" + "concat(variables('sharedTemplateFileName'), '/autopart.sh'", + "concat(variables('sharedTemplateFileName'), '/postgres_startup.sh'" ], "commandToExecute": "[concat('sh postgres_startup.sh ', parameters('clusterName'), ' ', variables('zookeeperInstanceCount'), ' ', variables('postgresInstanceCount'), ' ', copyIndex(), ' ', parameters('adminUsername'), ' \"', parameters('adminPassword'), '\" ')]" } From 6459de01182eb8092bee38e9e7be2d4d2d59f10e Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Thu, 2 Feb 2017 10:57:17 -0800 Subject: [PATCH 05/51] update branch deploy name --- README.md | 2 +- template.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9abeeb7..f2fa614 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Azure Patroni Deployment - + Date: Thu, 2 Feb 2017 11:07:18 -0800 Subject: [PATCH 06/51] fixed typos --- template.json | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/template.json b/template.json index 4edb26d..b50257c 100644 --- a/template.json +++ b/template.json @@ -87,7 +87,7 @@ "s", "y" ], - "nestedTemplates": "[concat(parameters('_artifactsLocation'), '/nestedtemplates/']", + "nestedTemplates": "[concat(parameters('_artifactsLocation'), '/nestedtemplates/')]", "storageAccountType": "Standard_LRS", "lbName": "postgresLoadBalancer", "vnetFile": { @@ -99,7 +99,6 @@ "no": "internal-lb.json" }, "lbId": "[resourceId('Microsoft.Network/loadBalancers/', variables('lbName'))]", - "vnetName": "dcos-vnet-F0547AD4", "vnetGroup": "testht", "vnetId": "[resourceId(parameters('vnetGroup'), 'Microsoft.Network/virtualNetworks', parameters('vnetName'))]", "zookeeperInstanceCount": 3, @@ -114,7 +113,7 @@ "resources": [ { "type": "Microsoft.Network/virtualNetworks", - "name": "[variables('vnetName')]", + "name": "[parameters('vnetName')]", "apiVersion": "2015-06-15", "properties": { "mode": "Incremental", @@ -124,7 +123,7 @@ }, "parameters": { "vnetName": { - "value": "[variables('vnetName')]" + "value": "[parameters('vnetName')]" }, "zookeeperNetName": { "value": "[parameters('zookeeperNetName')]" From 5def857024bc18b8e088e68ab6ae2a7c4ed290a5 Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Thu, 2 Feb 2017 11:17:38 -0800 Subject: [PATCH 07/51] fixed variable name error --- template.json | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/template.json b/template.json index b50257c..ab4bb81 100644 --- a/template.json +++ b/template.json @@ -34,8 +34,8 @@ "type": "string", "defaultValue": "yes", "allowedValues": [ - "yes", - "no" + "internal", + "external" ], "metadata": { "description": "Would you like to create an internal or external load balancer?" @@ -95,8 +95,8 @@ "no": "empty-resources.json" }, "lbFile": { - "yes": "external-lb.json", - "no": "internal-lb.json" + "external": "external-lb.json", + "internal": "internal-lb.json" }, "lbId": "[resourceId('Microsoft.Network/loadBalancers/', variables('lbName'))]", "vnetGroup": "testht", @@ -118,7 +118,7 @@ "properties": { "mode": "Incremental", "templateLink": { - "uri": "[concat(variables('sharedTemplateFileName'), variables('vnetFile')[parameters('newVnet')])]", + "uri": "[concat(variables('nestedTemplates'), variables('vnetFile')[parameters('newVnet')])]", "contentVersion": "1.0.0.0" }, "parameters": { @@ -270,7 +270,7 @@ "typeHandlerVersion": "1.2", "settings": { "fileUris": [ - "concat(variables('sharedTemplateFileName'), '/zookeeper_startup.sh'" + "concat(variables('nestedTemplates'), '/zookeeper_startup.sh'" ], "commandToExecute": "[concat('sh zookeeper_startup.sh ', copyIndex(), ' ', variables('zookeeperInstanceCount'))]" } @@ -398,8 +398,8 @@ "typeHandlerVersion": "1.2", "settings": { "fileUris": [ - "concat(variables('sharedTemplateFileName'), '/autopart.sh'", - "concat(variables('sharedTemplateFileName'), '/postgres_startup.sh'" + "concat(variables('nestedTemplates'), '/autopart.sh'", + "concat(variables('nestedTemplates'), '/postgres_startup.sh'" ], "commandToExecute": "[concat('sh postgres_startup.sh ', parameters('clusterName'), ' ', variables('zookeeperInstanceCount'), ' ', variables('postgresInstanceCount'), ' ', copyIndex(), ' ', parameters('adminUsername'), ' \"', parameters('adminPassword'), '\" ')]" } @@ -415,7 +415,7 @@ "properties": { "mode": "Incremental", "templateLink": { - "uri": "[concat(variables('sharedTemplateFileName'), variables('lbFile')[parameters('lbType')])]", + "uri": "[concat(variables('nestedTemplates'), variables('lbFile')[parameters('lbType')])]", "contentVersion": "1.0.0.0" }, "parameters": { From 9892c91e7e6d9d58bbfa4efc0f5a00751aa0b2b9 Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Thu, 2 Feb 2017 11:39:02 -0800 Subject: [PATCH 08/51] update formatting and missing resource pointers --- .idea/codeStyleSettings.xml | 34 + backup.json | 1054 ++++++++++++------------ nestedtemplates/empty-resources.json | 45 +- nestedtemplates/external-lb.json | 394 ++++----- nestedtemplates/internal-lb.json | 372 ++++----- nestedtemplates/network-resources.json | 102 +-- template.json | 6 +- 7 files changed, 1022 insertions(+), 985 deletions(-) create mode 100644 .idea/codeStyleSettings.xml diff --git a/.idea/codeStyleSettings.xml b/.idea/codeStyleSettings.xml new file mode 100644 index 0000000..efebc00 --- /dev/null +++ b/.idea/codeStyleSettings.xml @@ -0,0 +1,34 @@ + + + + + + \ No newline at end of file diff --git a/backup.json b/backup.json index 3a8ad51..ba2c0a3 100644 --- a/backup.json +++ b/backup.json @@ -1,556 +1,556 @@ { - "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "clusterName": { - "type": "String" - }, - "instanceCount": { - "defaultValue": 2, - "minValue": 2, - "maxValue": 5, - "type": "Int", - "metadata": { - "description": "Number of postgreSQL servers in the cluster." - } - }, - "adminUsername": { - "type": "String" - }, - "adminPassword": { - "type": "SecureString" - } - }, - "variables": { - "storageAccountSuffix": "[concat('vmds', parameters('clusterName'))]", - "storageAccountPrefix": [ - "a", - "g", - "m", - "s", - "y" - ], - "storageAccountType": "Standard_LRS", - "ipName": "[concat(parameters('clusterName'), '-ip')]", - "ipId": "[resourceId('Microsoft.Network/publicIPAddresses/', variables('ipName'))]", - "lbName": "postgresLoadBalancer", - "lbId": "[resourceId('Microsoft.Network/loadBalancers/', variables('lbName'))]", - "vnetName": "[concat(parameters('clusterName'), '-net')]", - "vnetId": "[resourceId('Microsoft.Network/virtualNetworks', variables('vnetName'))]", - "zookeeperNetName": "zookeeper-subnet", - "zookeeperInstanceCount": 3, - "postgresNetName": "postgres-subnet", - "postgresInstanceCount": "[parameters('instanceCount')]", - "osType": { - "publisher": "Canonical", - "offer": "UbuntuServer", - "sku": "16.04.0-LTS", - "version": "latest" - } - }, - "resources": [ - { - "type": "Microsoft.Network/virtualNetworks", - "name": "[variables('vnetName')]", - "apiVersion": "2015-06-15", - "location": "[resourceGroup().location]", - "tags": {}, - "properties": { - "addressSpace": { - "addressPrefixes": [ - "10.0.0.0/16" - ] + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "clusterName": { + "type": "String" }, - "subnets": [ - { - "name": "[variables('zookeeperNetName')]", - "properties": { - "addressPrefix": "10.0.100.0/24" - } - }, - { - "name": "[variables('postgresNetName')]", - "properties": { - "addressPrefix": "10.0.101.0/24" - } - } - ] - }, - "dependsOn": [] - }, - { - "type": "Microsoft.Storage/storageAccounts", - "name": "[concat(variables('storageAccountPrefix')[copyIndex()], variables('storageAccountSuffix'))]", - "apiVersion": "2015-06-15", - "location": "[resourceGroup().location]", - "copy": { - "name": "storageLoop", - "count": "[length(variables('storageAccountPrefix'))]" - }, - "properties": { - "accountType": "[variables('storageAccountType')]" - } - }, - { - "type": "Microsoft.Storage/storageAccounts", - "name": "[concat(variables('storageAccountPrefix')[copyIndex()], 'd', variables('storageAccountSuffix'))]", - "apiVersion": "2015-06-15", - "location": "[resourceGroup().location]", - "copy": { - "name": "storageDataLoop", - "count": "[length(variables('storageAccountPrefix'))]" - }, - "properties": { - "accountType": "Premium_LRS" - } - }, - { - "type": "Microsoft.Network/publicIPAddresses", - "name": "[variables('ipName')]", - "apiVersion": "2016-03-30", - "location": "[resourceGroup().location]", - "properties": { - "publicIPAllocationMethod": "Dynamic", - "dnsSettings": { - "domainNameLabel": "[parameters('clusterName')]" - } - } - }, - { - "type": "Microsoft.Compute/availabilitySets", - "name": "postgresAvailabilitySet", - "apiVersion": "2015-05-01-preview", - "location": "[resourceGroup().location]", - "properties": {} - }, - { - "type": "Microsoft.Compute/availabilitySets", - "name": "zookeeperAvailabilitySet", - "apiVersion": "2015-05-01-preview", - "location": "[resourceGroup().location]", - "properties": {} - }, - { - "type": "Microsoft.Network/networkInterfaces", - "name": "[concat('zookeeper', copyIndex(), '-nic')]", - "apiVersion": "2015-05-01-preview", - "location": "[resourceGroup().location]", - "copy": { - "name": "zookeeperNicLoop", - "count": "[variables('zookeeperInstanceCount')]" - }, - "properties": { - "ipConfigurations": [ - { - "name": "zookeeperIp", - "properties": { - "privateIPAllocationMethod": "Static", - "privateIPAddress": "[concat('10.0.100.', copyIndex(10))]", - "subnet": { - "id": "[concat(variables('vnetId'), '/subnets/', variables('zookeeperNetName'))]" - } + "instanceCount": { + "defaultValue": 2, + "minValue": 2, + "maxValue": 5, + "type": "Int", + "metadata": { + "description": "Number of postgreSQL servers in the cluster." } - } - ] - }, - "dependsOn": [ - "[variables('vnetId')]", - "[variables('lbId')]" - ] - }, - { - "type": "Microsoft.Compute/virtualMachines", - "name": "[concat('zookeeper', copyIndex())]", - "apiVersion": "2015-05-01-preview", - "location": "[resourceGroup().location]", - "copy": { - "name": "zookeeperVmLoop", - "count": "[variables('zookeeperInstanceCount')]" - }, - "properties": { - "availabilitySet": { - "id": "[resourceId('Microsoft.Compute/availabilitySets', 'zookeeperAvailabilitySet')]" - }, - "hardwareProfile": { - "vmSize": "Standard_A2" }, - "osProfile": { - "computerName": "[concat('zookeeper', copyIndex())]", - "adminUsername": "[parameters('adminUserName')]", - "adminPassword": "[parameters('adminPassword')]" - }, - "storageProfile": { - "imageReference": { - "publisher": "Canonical", - "offer": "UbuntuServer", - "sku": "16.04.0-LTS", - "version": "latest" - }, - "osDisk": { - "name": "osdisk", - "vhd": { - "uri": "[concat(reference(concat('Microsoft.Storage/storageAccounts/', variables('storageAccountPrefix')[copyIndex()], variables('storageAccountSuffix')), providers('Microsoft.Storage', 'storageAccounts').apiVersions[0]).primaryEndpoints.blob, 'vhds/', 'zookeeperosdisk', copyIndex(), '.vhd')]" - }, - "caching": "ReadWrite", - "createOption": "FromImage" - } + "adminUsername": { + "type": "String" }, - "networkProfile": { - "networkInterfaces": [ - { - "id": "[resourceId('Microsoft.Network/networkInterfaces', concat('zookeeper', copyIndex(), '-nic'))]" - } - ] - } - }, - "dependsOn": [ - "[concat('Microsoft.Storage/storageAccounts/', variables('storageAccountPrefix')[copyIndex()], variables('storageAccountSuffix'))]", - "[concat('Microsoft.Network/networkInterfaces/', 'zookeeper', copyIndex(), '-nic')]", - "[concat('Microsoft.Compute/availabilitySets/', 'zookeeperAvailabilitySet')]" - ] - }, - { - "type": "Microsoft.Compute/virtualMachines/extensions", - "name": "[concat('zookeeper', copyIndex(), '/zookeeperExtension')]", - "apiVersion": "2015-05-01-preview", - "location": "[resourceGroup().location]", - "copy": { - "name": "zookeeperExtLoop", - "count": "[variables('zookeeperInstanceCount')]" - }, - "properties": { - "publisher": "Microsoft.OSTCExtensions", - "type": "CustomScriptForLinux", - "typeHandlerVersion": "1.2", - "settings": { - "fileUris": [ - "https://raw.githubusercontent.com/HelloTech/azure_patroni/master/zookeeper_startup.sh" - ], - "commandToExecute": "[concat('sh zookeeper_startup.sh ', copyIndex(), ' ', variables('zookeeperInstanceCount'))]" + "adminPassword": { + "type": "SecureString" } - }, - "dependsOn": [ - "[concat('Microsoft.Compute/virtualMachines/', 'zookeeper', copyIndex())]" - ] }, - { - "type": "Microsoft.Network/networkInterfaces", - "name": "[concat('postgres', copyIndex(), '-nic')]", - "apiVersion": "2015-05-01-preview", - "location": "[resourceGroup().location]", - "copy": { - "name": "postgresNicLoop", - "count": "[variables('postgresInstanceCount')]" - }, - "properties": { - "ipConfigurations": [ - { - "name": "postgresIp", - "properties": { - "privateIPAllocationMethod": "Static", - "privateIPAddress": "[concat('10.0.101.', copyIndex(10))]", - "subnet": { - "id": "[concat(variables('vnetId'), '/subnets/', variables('postgresNetName'))]" - }, - "loadBalancerBackendAddressPools": [ - { - "id": "[concat(variables('lbId'), '/backendAddressPools/loadBalancerBackEnd')]" - } - ], - "loadBalancerInboundNatRules": [ - { - "id": "[concat(variables('lbId'), '/inboundNatRules/postgresSsh', copyIndex())]" - } - ] - } - } - ] - }, - "dependsOn": [ - "[variables('vnetId')]", - "[variables('lbId')]" - ] - }, - { - "type": "Microsoft.Compute/virtualMachines", - "name": "[concat('postgres', copyIndex())]", - "apiVersion": "2015-05-01-preview", - "location": "[resourceGroup().location]", - "copy": { - "name": "postgresVmLoop", - "count": "[variables('postgresInstanceCount')]" - }, - "properties": { - "availabilitySet": { - "id": "[resourceId('Microsoft.Compute/availabilitySets', 'postgresAvailabilitySet')]" - }, - "hardwareProfile": { - "vmSize": "Standard_DS1" - }, - "osProfile": { - "computerName": "[concat('postgres', copyIndex())]", - "adminUsername": "[parameters('adminUserName')]", - "adminPassword": "[parameters('adminPassword')]" - }, - "storageProfile": { - "imageReference": { + "variables": { + "storageAccountSuffix": "[concat('vmds', parameters('clusterName'))]", + "storageAccountPrefix": [ + "a", + "g", + "m", + "s", + "y" + ], + "storageAccountType": "Standard_LRS", + "ipName": "[concat(parameters('clusterName'), '-ip')]", + "ipId": "[resourceId('Microsoft.Network/publicIPAddresses/', variables('ipName'))]", + "lbName": "postgresLoadBalancer", + "lbId": "[resourceId('Microsoft.Network/loadBalancers/', variables('lbName'))]", + "vnetName": "[concat(parameters('clusterName'), '-net')]", + "vnetId": "[resourceId('Microsoft.Network/virtualNetworks', variables('vnetName'))]", + "zookeeperNetName": "zookeeper-subnet", + "zookeeperInstanceCount": 3, + "postgresNetName": "postgres-subnet", + "postgresInstanceCount": "[parameters('instanceCount')]", + "osType": { "publisher": "Canonical", "offer": "UbuntuServer", "sku": "16.04.0-LTS", "version": "latest" - }, - "osDisk": { - "name": "osdisk", - "vhd": { - "uri": "[concat(reference(concat('Microsoft.Storage/storageAccounts/', variables('storageAccountPrefix')[copyIndex()], 'd', variables('storageAccountSuffix')), providers('Microsoft.Storage', 'storageAccounts').apiVersions[0]).primaryEndpoints.blob, 'vhds/', 'postgresosdisk', copyIndex(), '.vhd')]" - }, - "caching": "ReadWrite", - "createOption": "FromImage" - }, - "dataDisks": [ - { - "name": "data0", - "diskSizeGB": "128", - "lun": 0, - "vhd": { - "uri": "[concat(reference(concat('Microsoft.Storage/storageAccounts/', variables('storageAccountPrefix')[copyIndex()], 'd', variables('storageAccountSuffix')), providers('Microsoft.Storage', 'storageAccounts').apiVersions[0]).primaryEndpoints.blob, 'vhds/', 'data0', copyIndex(), '.vhd')]" - }, - "createOption": "Empty" - } - ] - }, - "networkProfile": { - "networkInterfaces": [ - { - "id": "[resourceId('Microsoft.Network/networkInterfaces', concat('postgres', copyIndex(), '-nic'))]" - } - ] } - }, - "dependsOn": [ - "storageLoop", - "storageDataLoop", - "zookeeperVmLoop", - "zookeeperExtLoop", - "[concat('Microsoft.Storage/storageAccounts/', variables('storageAccountPrefix')[copyIndex()], variables('storageAccountSuffix'))]", - "[concat('Microsoft.Storage/storageAccounts/', variables('storageAccountPrefix')[copyIndex()], 'd', variables('storageAccountSuffix'))]", - "[concat('Microsoft.Network/networkInterfaces/', 'postgres', copyIndex(), '-nic')]", - "[concat('Microsoft.Compute/availabilitySets/', 'postgresAvailabilitySet')]" - ] }, - { - "type": "Microsoft.Compute/virtualMachines/extensions", - "name": "[concat('postgres', copyIndex(), '/postgresExtension')]", - "apiVersion": "2015-05-01-preview", - "location": "[resourceGroup().location]", - "copy": { - "name": "postgresExtLoop", - "count": "[variables('postgresInstanceCount')]" - }, - "properties": { - "publisher": "Microsoft.OSTCExtensions", - "type": "CustomScriptForLinux", - "typeHandlerVersion": "1.2", - "settings": { - "fileUris": [ - "https://raw.githubusercontent.com/HelloTech/azure_patroni/master/autopart.sh", - "https://raw.githubusercontent.com/HelloTech/azure_patroni/master/postgres_startup.sh" - ], - "commandToExecute": "[concat('sh postgres_startup.sh ', parameters('clusterName'), ' ', variables('zookeeperInstanceCount'), ' ', variables('postgresInstanceCount'), ' ', copyIndex(), ' ', parameters('adminUsername'), ' \"', parameters('adminPassword'), '\" ')]" - } - }, - "dependsOn": [ - "[concat('Microsoft.Compute/virtualMachines/', 'postgres', copyIndex())]" - ] - }, - { - "type": "Microsoft.Network/loadBalancers", - "name": "[variables('lbName')]", - "apiVersion": "2015-06-15", - "location": "[resourceGroup().location]", - "properties": { - "availabilitySet": { - "id": "[resourceId('Microsoft.Compute/availabilitySets', 'postgresHaAvailabilitySet')]" - }, - "frontendIPConfigurations": [ - { - "name": "loadBalancerFrontEnd", - "properties": { - "publicIPAddress": { - "id": "[variables('ipId')]" - } - } - } - ], - "backendAddressPools": [ - { - "name": "loadBalancerBackEnd" - } - ], - "loadBalancingRules": [ - { - "name": "postgresLbRule", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "backendAddressPool": { - "id": "[concat(variables('lbId'), '/backendAddressPools/loadBalancerBackEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 5432, - "backendPort": 5000, - "probe": { - "id": "[concat(variables('lbId'), '/probes/postgresProbe')]" - } - } - } - ], - "probes": [ - { - "name": "postgresProbe", + "resources": [ + { + "type": "Microsoft.Network/virtualNetworks", + "name": "[variables('vnetName')]", + "apiVersion": "2015-06-15", + "location": "[resourceGroup().location]", + "tags": {}, "properties": { - "protocol": "Tcp", - "port": 5000, - "intervalInSeconds": 5, - "numberOfProbes": 2 - } - } - ], - "inboundNatRules": [ - { - "name": "postgresSsh0", + "addressSpace": { + "addressPrefixes": [ + "10.0.0.0/16" + ] + }, + "subnets": [ + { + "name": "[variables('zookeeperNetName')]", + "properties": { + "addressPrefix": "10.0.100.0/24" + } + }, + { + "name": "[variables('postgresNetName')]", + "properties": { + "addressPrefix": "10.0.101.0/24" + } + } + ] + }, + "dependsOn": [] + }, + { + "type": "Microsoft.Storage/storageAccounts", + "name": "[concat(variables('storageAccountPrefix')[copyIndex()], variables('storageAccountSuffix'))]", + "apiVersion": "2015-06-15", + "location": "[resourceGroup().location]", + "copy": { + "name": "storageLoop", + "count": "[length(variables('storageAccountPrefix'))]" + }, "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10110, - "backendPort": 22 + "accountType": "[variables('storageAccountType')]" } - }, - { - "name": "postgresSsh1", + }, + { + "type": "Microsoft.Storage/storageAccounts", + "name": "[concat(variables('storageAccountPrefix')[copyIndex()], 'd', variables('storageAccountSuffix'))]", + "apiVersion": "2015-06-15", + "location": "[resourceGroup().location]", + "copy": { + "name": "storageDataLoop", + "count": "[length(variables('storageAccountPrefix'))]" + }, "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10111, - "backendPort": 22 + "accountType": "Premium_LRS" } - }, - { - "name": "postgresSsh2", + }, + { + "type": "Microsoft.Network/publicIPAddresses", + "name": "[variables('ipName')]", + "apiVersion": "2016-03-30", + "location": "[resourceGroup().location]", "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10112, - "backendPort": 22 + "publicIPAllocationMethod": "Dynamic", + "dnsSettings": { + "domainNameLabel": "[parameters('clusterName')]" + } } - }, - { - "name": "postgresSsh3", + }, + { + "type": "Microsoft.Compute/availabilitySets", + "name": "postgresAvailabilitySet", + "apiVersion": "2015-05-01-preview", + "location": "[resourceGroup().location]", + "properties": {} + }, + { + "type": "Microsoft.Compute/availabilitySets", + "name": "zookeeperAvailabilitySet", + "apiVersion": "2015-05-01-preview", + "location": "[resourceGroup().location]", + "properties": {} + }, + { + "type": "Microsoft.Network/networkInterfaces", + "name": "[concat('zookeeper', copyIndex(), '-nic')]", + "apiVersion": "2015-05-01-preview", + "location": "[resourceGroup().location]", + "copy": { + "name": "zookeeperNicLoop", + "count": "[variables('zookeeperInstanceCount')]" + }, "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10113, - "backendPort": 22 - } - }, - { - "name": "postgresSsh4", + "ipConfigurations": [ + { + "name": "zookeeperIp", + "properties": { + "privateIPAllocationMethod": "Static", + "privateIPAddress": "[concat('10.0.100.', copyIndex(10))]", + "subnet": { + "id": "[concat(variables('vnetId'), '/subnets/', variables('zookeeperNetName'))]" + } + } + } + ] + }, + "dependsOn": [ + "[variables('vnetId')]", + "[variables('lbId')]" + ] + }, + { + "type": "Microsoft.Compute/virtualMachines", + "name": "[concat('zookeeper', copyIndex())]", + "apiVersion": "2015-05-01-preview", + "location": "[resourceGroup().location]", + "copy": { + "name": "zookeeperVmLoop", + "count": "[variables('zookeeperInstanceCount')]" + }, "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10114, - "backendPort": 22 - } - }, - { - "name": "postgresSsh5", + "availabilitySet": { + "id": "[resourceId('Microsoft.Compute/availabilitySets', 'zookeeperAvailabilitySet')]" + }, + "hardwareProfile": { + "vmSize": "Standard_A2" + }, + "osProfile": { + "computerName": "[concat('zookeeper', copyIndex())]", + "adminUsername": "[parameters('adminUserName')]", + "adminPassword": "[parameters('adminPassword')]" + }, + "storageProfile": { + "imageReference": { + "publisher": "Canonical", + "offer": "UbuntuServer", + "sku": "16.04.0-LTS", + "version": "latest" + }, + "osDisk": { + "name": "osdisk", + "vhd": { + "uri": "[concat(reference(concat('Microsoft.Storage/storageAccounts/', variables('storageAccountPrefix')[copyIndex()], variables('storageAccountSuffix')), providers('Microsoft.Storage', 'storageAccounts').apiVersions[0]).primaryEndpoints.blob, 'vhds/', 'zookeeperosdisk', copyIndex(), '.vhd')]" + }, + "caching": "ReadWrite", + "createOption": "FromImage" + } + }, + "networkProfile": { + "networkInterfaces": [ + { + "id": "[resourceId('Microsoft.Network/networkInterfaces', concat('zookeeper', copyIndex(), '-nic'))]" + } + ] + } + }, + "dependsOn": [ + "[concat('Microsoft.Storage/storageAccounts/', variables('storageAccountPrefix')[copyIndex()], variables('storageAccountSuffix'))]", + "[concat('Microsoft.Network/networkInterfaces/', 'zookeeper', copyIndex(), '-nic')]", + "[concat('Microsoft.Compute/availabilitySets/', 'zookeeperAvailabilitySet')]" + ] + }, + { + "type": "Microsoft.Compute/virtualMachines/extensions", + "name": "[concat('zookeeper', copyIndex(), '/zookeeperExtension')]", + "apiVersion": "2015-05-01-preview", + "location": "[resourceGroup().location]", + "copy": { + "name": "zookeeperExtLoop", + "count": "[variables('zookeeperInstanceCount')]" + }, "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10115, - "backendPort": 22 - } - }, - { - "name": "postgresSsh6", + "publisher": "Microsoft.OSTCExtensions", + "type": "CustomScriptForLinux", + "typeHandlerVersion": "1.2", + "settings": { + "fileUris": [ + "https://raw.githubusercontent.com/HelloTech/azure_patroni/master/zookeeper_startup.sh" + ], + "commandToExecute": "[concat('sh zookeeper_startup.sh ', copyIndex(), ' ', variables('zookeeperInstanceCount'))]" + } + }, + "dependsOn": [ + "[concat('Microsoft.Compute/virtualMachines/', 'zookeeper', copyIndex())]" + ] + }, + { + "type": "Microsoft.Network/networkInterfaces", + "name": "[concat('postgres', copyIndex(), '-nic')]", + "apiVersion": "2015-05-01-preview", + "location": "[resourceGroup().location]", + "copy": { + "name": "postgresNicLoop", + "count": "[variables('postgresInstanceCount')]" + }, "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10116, - "backendPort": 22 - } - }, - { - "name": "postgresSsh7", + "ipConfigurations": [ + { + "name": "postgresIp", + "properties": { + "privateIPAllocationMethod": "Static", + "privateIPAddress": "[concat('10.0.101.', copyIndex(10))]", + "subnet": { + "id": "[concat(variables('vnetId'), '/subnets/', variables('postgresNetName'))]" + }, + "loadBalancerBackendAddressPools": [ + { + "id": "[concat(variables('lbId'), '/backendAddressPools/loadBalancerBackEnd')]" + } + ], + "loadBalancerInboundNatRules": [ + { + "id": "[concat(variables('lbId'), '/inboundNatRules/postgresSsh', copyIndex())]" + } + ] + } + } + ] + }, + "dependsOn": [ + "[variables('vnetId')]", + "[variables('lbId')]" + ] + }, + { + "type": "Microsoft.Compute/virtualMachines", + "name": "[concat('postgres', copyIndex())]", + "apiVersion": "2015-05-01-preview", + "location": "[resourceGroup().location]", + "copy": { + "name": "postgresVmLoop", + "count": "[variables('postgresInstanceCount')]" + }, "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10117, - "backendPort": 22 - } - }, - { - "name": "postgresSsh8", + "availabilitySet": { + "id": "[resourceId('Microsoft.Compute/availabilitySets', 'postgresAvailabilitySet')]" + }, + "hardwareProfile": { + "vmSize": "Standard_DS1" + }, + "osProfile": { + "computerName": "[concat('postgres', copyIndex())]", + "adminUsername": "[parameters('adminUserName')]", + "adminPassword": "[parameters('adminPassword')]" + }, + "storageProfile": { + "imageReference": { + "publisher": "Canonical", + "offer": "UbuntuServer", + "sku": "16.04.0-LTS", + "version": "latest" + }, + "osDisk": { + "name": "osdisk", + "vhd": { + "uri": "[concat(reference(concat('Microsoft.Storage/storageAccounts/', variables('storageAccountPrefix')[copyIndex()], 'd', variables('storageAccountSuffix')), providers('Microsoft.Storage', 'storageAccounts').apiVersions[0]).primaryEndpoints.blob, 'vhds/', 'postgresosdisk', copyIndex(), '.vhd')]" + }, + "caching": "ReadWrite", + "createOption": "FromImage" + }, + "dataDisks": [ + { + "name": "data0", + "diskSizeGB": "128", + "lun": 0, + "vhd": { + "uri": "[concat(reference(concat('Microsoft.Storage/storageAccounts/', variables('storageAccountPrefix')[copyIndex()], 'd', variables('storageAccountSuffix')), providers('Microsoft.Storage', 'storageAccounts').apiVersions[0]).primaryEndpoints.blob, 'vhds/', 'data0', copyIndex(), '.vhd')]" + }, + "createOption": "Empty" + } + ] + }, + "networkProfile": { + "networkInterfaces": [ + { + "id": "[resourceId('Microsoft.Network/networkInterfaces', concat('postgres', copyIndex(), '-nic'))]" + } + ] + } + }, + "dependsOn": [ + "storageLoop", + "storageDataLoop", + "zookeeperVmLoop", + "zookeeperExtLoop", + "[concat('Microsoft.Storage/storageAccounts/', variables('storageAccountPrefix')[copyIndex()], variables('storageAccountSuffix'))]", + "[concat('Microsoft.Storage/storageAccounts/', variables('storageAccountPrefix')[copyIndex()], 'd', variables('storageAccountSuffix'))]", + "[concat('Microsoft.Network/networkInterfaces/', 'postgres', copyIndex(), '-nic')]", + "[concat('Microsoft.Compute/availabilitySets/', 'postgresAvailabilitySet')]" + ] + }, + { + "type": "Microsoft.Compute/virtualMachines/extensions", + "name": "[concat('postgres', copyIndex(), '/postgresExtension')]", + "apiVersion": "2015-05-01-preview", + "location": "[resourceGroup().location]", + "copy": { + "name": "postgresExtLoop", + "count": "[variables('postgresInstanceCount')]" + }, "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10118, - "backendPort": 22 - } - }, - { - "name": "postgresSsh9", + "publisher": "Microsoft.OSTCExtensions", + "type": "CustomScriptForLinux", + "typeHandlerVersion": "1.2", + "settings": { + "fileUris": [ + "https://raw.githubusercontent.com/HelloTech/azure_patroni/master/autopart.sh", + "https://raw.githubusercontent.com/HelloTech/azure_patroni/master/postgres_startup.sh" + ], + "commandToExecute": "[concat('sh postgres_startup.sh ', parameters('clusterName'), ' ', variables('zookeeperInstanceCount'), ' ', variables('postgresInstanceCount'), ' ', copyIndex(), ' ', parameters('adminUsername'), ' \"', parameters('adminPassword'), '\" ')]" + } + }, + "dependsOn": [ + "[concat('Microsoft.Compute/virtualMachines/', 'postgres', copyIndex())]" + ] + }, + { + "type": "Microsoft.Network/loadBalancers", + "name": "[variables('lbName')]", + "apiVersion": "2015-06-15", + "location": "[resourceGroup().location]", "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10119, - "backendPort": 22 - } - } - ] - }, - "dependsOn": [ - "[variables('ipId')]" - ] - } - ], - "outputs": { - "fqdn": { - "type": "String", - "value": "[reference(variables('ipName')).dnsSettings.fqdn]" - }, - "port": { - "type": "Int", - "value": 5432 - }, - "postgresUser": { - "type": "String", - "value": "admin" - }, - "postgresUserPassword": { - "type": "SecureString", - "value": "[parameters('adminPassword')]" + "availabilitySet": { + "id": "[resourceId('Microsoft.Compute/availabilitySets', 'postgresHaAvailabilitySet')]" + }, + "frontendIPConfigurations": [ + { + "name": "loadBalancerFrontEnd", + "properties": { + "publicIPAddress": { + "id": "[variables('ipId')]" + } + } + } + ], + "backendAddressPools": [ + { + "name": "loadBalancerBackEnd" + } + ], + "loadBalancingRules": [ + { + "name": "postgresLbRule", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "backendAddressPool": { + "id": "[concat(variables('lbId'), '/backendAddressPools/loadBalancerBackEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 5432, + "backendPort": 5000, + "probe": { + "id": "[concat(variables('lbId'), '/probes/postgresProbe')]" + } + } + } + ], + "probes": [ + { + "name": "postgresProbe", + "properties": { + "protocol": "Tcp", + "port": 5000, + "intervalInSeconds": 5, + "numberOfProbes": 2 + } + } + ], + "inboundNatRules": [ + { + "name": "postgresSsh0", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10110, + "backendPort": 22 + } + }, + { + "name": "postgresSsh1", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10111, + "backendPort": 22 + } + }, + { + "name": "postgresSsh2", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10112, + "backendPort": 22 + } + }, + { + "name": "postgresSsh3", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10113, + "backendPort": 22 + } + }, + { + "name": "postgresSsh4", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10114, + "backendPort": 22 + } + }, + { + "name": "postgresSsh5", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10115, + "backendPort": 22 + } + }, + { + "name": "postgresSsh6", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10116, + "backendPort": 22 + } + }, + { + "name": "postgresSsh7", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10117, + "backendPort": 22 + } + }, + { + "name": "postgresSsh8", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10118, + "backendPort": 22 + } + }, + { + "name": "postgresSsh9", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10119, + "backendPort": 22 + } + } + ] + }, + "dependsOn": [ + "[variables('ipId')]" + ] + } + ], + "outputs": { + "fqdn": { + "type": "String", + "value": "[reference(variables('ipName')).dnsSettings.fqdn]" + }, + "port": { + "type": "Int", + "value": 5432 + }, + "postgresUser": { + "type": "String", + "value": "admin" + }, + "postgresUserPassword": { + "type": "SecureString", + "value": "[parameters('adminPassword')]" + } } - } } \ No newline at end of file diff --git a/nestedtemplates/empty-resources.json b/nestedtemplates/empty-resources.json index ee32b84..0454288 100644 --- a/nestedtemplates/empty-resources.json +++ b/nestedtemplates/empty-resources.json @@ -1,26 +1,25 @@ - { - "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "vnetName": { - "type": "String", - "defaultValue": "" + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "vnetName": { + "type": "String", + "defaultValue": "" + }, + "zookeeperNetName": { + "type": "String", + "defaultValue": "" + }, + "postgresNetName": { + "type": "String", + "defaultValue": "" + }, + "clusterName": { + "type": "String", + "defaultValue": "" + } }, - "zookeeperNetName": { - "type": "String", - "defaultValue": "" - }, - "postgresNetName": { - "type": "String", - "defaultValue": "" - }, - "clusterName": { - "type": "String", - "defaultValue": "" - } - }, - "variables": {}, - "resources": [], - "outputs": {} + "variables": {}, + "resources": [], + "outputs": {} } diff --git a/nestedtemplates/external-lb.json b/nestedtemplates/external-lb.json index 4b6c787..6e39bfa 100644 --- a/nestedtemplates/external-lb.json +++ b/nestedtemplates/external-lb.json @@ -1,204 +1,204 @@ { - "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "clusterName": { - "type": "String", - "defaultValue": "" - }, - "lbName": { - "type": "String", - "defaultValue": "" - }, - "lbId": { - "type": "String", - "defaultValue": "" - } - }, - "variables": {}, - "resources": [ - { - "type": "Microsoft.Network/publicIPAddresses", - "name": "[concat(parameters('clusterName'), '-ip')]", - "apiVersion": "2016-03-30", - "location": "[resourceGroup().location]", - "properties": { - "publicIPAllocationMethod": "Dynamic", - "dnsSettings": { - "domainNameLabel": "[parameters('clusterName')]" + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "clusterName": { + "type": "String", + "defaultValue": "" + }, + "lbName": { + "type": "String", + "defaultValue": "" + }, + "lbId": { + "type": "String", + "defaultValue": "" } - } }, - { - "type": "Microsoft.Network/loadBalancers", - "name": "[parameters('lbName')]", - "apiVersion": "2015-06-15", - "location": "[resourceGroup().location]", - "properties": { - "availabilitySet": { - "id": "[resourceId('Microsoft.Compute/availabilitySets', 'postgresHaAvailabilitySet')]" - }, - "frontendIPConfigurations": [ - { - "name": "loadBalancerFrontEnd", - "properties": { - "publicIPAddress": { - "id": "[parameters('ipId')]" - } - } - } - ], - "backendAddressPools": [ - { - "name": "loadBalancerBackEnd" - } - ], - "loadBalancingRules": [ - { - "name": "postgresLbRule", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "backendAddressPool": { - "id": "[concat(parameters('lbId'), '/backendAddressPools/loadBalancerBackEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 5432, - "backendPort": 5000, - "probe": { - "id": "[concat(parameters('lbId'), '/probes/postgresProbe')]" - } - } - } - ], - "probes": [ - { - "name": "postgresProbe", - "properties": { - "protocol": "Tcp", - "port": 5000, - "intervalInSeconds": 5, - "numberOfProbes": 2 - } - } - ], - "inboundNatRules": [ - { - "name": "postgresSsh0", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10110, - "backendPort": 22 - } - }, - { - "name": "postgresSsh1", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10111, - "backendPort": 22 - } - }, - { - "name": "postgresSsh2", + "variables": {}, + "resources": [ + { + "type": "Microsoft.Network/publicIPAddresses", + "name": "[concat(parameters('clusterName'), '-ip')]", + "apiVersion": "2016-03-30", + "location": "[resourceGroup().location]", "properties": { - "frontendIPConfiguration": { - "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10112, - "backendPort": 22 + "publicIPAllocationMethod": "Dynamic", + "dnsSettings": { + "domainNameLabel": "[parameters('clusterName')]" + } } - }, - { - "name": "postgresSsh3", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10113, - "backendPort": 22 - } - }, - { - "name": "postgresSsh4", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10114, - "backendPort": 22 - } - }, - { - "name": "postgresSsh5", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10115, - "backendPort": 22 - } - }, - { - "name": "postgresSsh6", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10116, - "backendPort": 22 - } - }, - { - "name": "postgresSsh7", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10117, - "backendPort": 22 - } - }, - { - "name": "postgresSsh8", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10118, - "backendPort": 22 - } - }, - { - "name": "postgresSsh9", + }, + { + "type": "Microsoft.Network/loadBalancers", + "name": "[parameters('lbName')]", + "apiVersion": "2015-06-15", + "location": "[resourceGroup().location]", "properties": { - "frontendIPConfiguration": { - "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10119, - "backendPort": 22 - } - } - ] - }, - "dependsOn": [ - "[variables('ipId')]" - ] - } - ] + "availabilitySet": { + "id": "[resourceId('Microsoft.Compute/availabilitySets', 'postgresHaAvailabilitySet')]" + }, + "frontendIPConfigurations": [ + { + "name": "loadBalancerFrontEnd", + "properties": { + "publicIPAddress": { + "id": "[parameters('ipId')]" + } + } + } + ], + "backendAddressPools": [ + { + "name": "loadBalancerBackEnd" + } + ], + "loadBalancingRules": [ + { + "name": "postgresLbRule", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "backendAddressPool": { + "id": "[concat(parameters('lbId'), '/backendAddressPools/loadBalancerBackEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 5432, + "backendPort": 5000, + "probe": { + "id": "[concat(parameters('lbId'), '/probes/postgresProbe')]" + } + } + } + ], + "probes": [ + { + "name": "postgresProbe", + "properties": { + "protocol": "Tcp", + "port": 5000, + "intervalInSeconds": 5, + "numberOfProbes": 2 + } + } + ], + "inboundNatRules": [ + { + "name": "postgresSsh0", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10110, + "backendPort": 22 + } + }, + { + "name": "postgresSsh1", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10111, + "backendPort": 22 + } + }, + { + "name": "postgresSsh2", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10112, + "backendPort": 22 + } + }, + { + "name": "postgresSsh3", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10113, + "backendPort": 22 + } + }, + { + "name": "postgresSsh4", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10114, + "backendPort": 22 + } + }, + { + "name": "postgresSsh5", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10115, + "backendPort": 22 + } + }, + { + "name": "postgresSsh6", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10116, + "backendPort": 22 + } + }, + { + "name": "postgresSsh7", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10117, + "backendPort": 22 + } + }, + { + "name": "postgresSsh8", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10118, + "backendPort": 22 + } + }, + { + "name": "postgresSsh9", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10119, + "backendPort": 22 + } + } + ] + }, + "dependsOn": [ + "[variables('ipId')]" + ] + } + ] } \ No newline at end of file diff --git a/nestedtemplates/internal-lb.json b/nestedtemplates/internal-lb.json index 37c447f..2e6b1cc 100644 --- a/nestedtemplates/internal-lb.json +++ b/nestedtemplates/internal-lb.json @@ -1,191 +1,191 @@ { - "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "clusterName": { - "type": "String", - "defaultValue": "" - }, - "lbName": { - "type": "String", - "defaultValue": "" - }, - "lbId": { - "type": "String", - "defaultValue": "" - } - }, - "variables": {}, - "resources": [ - { - "type": "Microsoft.Network/loadBalancers", - "name": "[variables('lbName')]", - "apiVersion": "2015-06-15", - "location": "[resourceGroup().location]", - "properties": { - "availabilitySet": { - "id": "[resourceId('Microsoft.Compute/availabilitySets', 'postgresHaAvailabilitySet')]" + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "clusterName": { + "type": "String", + "defaultValue": "" }, - "frontendIPConfigurations": [ - { - "name": "LoadBalancerFrontEnd", - "properties": { - "privateIPAddress": "10.0.101.4", - "privateIPAllocationMethod": "Static", - "subnet": { - "id": "[concat(variables('vnetId'), '/subnets/', parameters('zookeeperNetName'))]" - } - } - } - ], - "backendAddressPools": [ - { - "name": "loadBalancerBackEnd" - } - ], - "loadBalancingRules": [ - { - "name": "postgresLbRule", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "backendAddressPool": { - "id": "[concat(variables('lbId'), '/backendAddressPools/loadBalancerBackEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 5432, - "backendPort": 5000, - "probe": { - "id": "[concat(variables('lbId'), '/probes/postgresProbe')]" - } - } - } - ], - "probes": [ - { - "name": "postgresProbe", - "properties": { - "protocol": "Tcp", - "port": 5000, - "intervalInSeconds": 5, - "numberOfProbes": 2 - } - } - ], - "inboundNatRules": [ - { - "name": "postgresSsh0", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10110, - "backendPort": 22 - } - }, - { - "name": "postgresSsh1", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10111, - "backendPort": 22 - } - }, - { - "name": "postgresSsh2", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10112, - "backendPort": 22 - } - }, - { - "name": "postgresSsh3", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10113, - "backendPort": 22 - } - }, - { - "name": "postgresSsh4", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10114, - "backendPort": 22 - } - }, - { - "name": "postgresSsh5", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10115, - "backendPort": 22 - } - }, - { - "name": "postgresSsh6", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10116, - "backendPort": 22 - } - }, - { - "name": "postgresSsh7", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10117, - "backendPort": 22 - } - }, - { - "name": "postgresSsh8", - "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10118, - "backendPort": 22 - } - }, - { - "name": "postgresSsh9", + "lbName": { + "type": "String", + "defaultValue": "" + }, + "lbId": { + "type": "String", + "defaultValue": "" + } + }, + "variables": {}, + "resources": [ + { + "type": "Microsoft.Network/loadBalancers", + "name": "[variables('lbName')]", + "apiVersion": "2015-06-15", + "location": "[resourceGroup().location]", "properties": { - "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" - }, - "protocol": "Tcp", - "frontendPort": 10119, - "backendPort": 22 + "availabilitySet": { + "id": "[resourceId('Microsoft.Compute/availabilitySets', 'postgresHaAvailabilitySet')]" + }, + "frontendIPConfigurations": [ + { + "name": "LoadBalancerFrontEnd", + "properties": { + "privateIPAddress": "10.0.101.4", + "privateIPAllocationMethod": "Static", + "subnet": { + "id": "[concat(variables('vnetId'), '/subnets/', parameters('zookeeperNetName'))]" + } + } + } + ], + "backendAddressPools": [ + { + "name": "loadBalancerBackEnd" + } + ], + "loadBalancingRules": [ + { + "name": "postgresLbRule", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "backendAddressPool": { + "id": "[concat(variables('lbId'), '/backendAddressPools/loadBalancerBackEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 5432, + "backendPort": 5000, + "probe": { + "id": "[concat(variables('lbId'), '/probes/postgresProbe')]" + } + } + } + ], + "probes": [ + { + "name": "postgresProbe", + "properties": { + "protocol": "Tcp", + "port": 5000, + "intervalInSeconds": 5, + "numberOfProbes": 2 + } + } + ], + "inboundNatRules": [ + { + "name": "postgresSsh0", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10110, + "backendPort": 22 + } + }, + { + "name": "postgresSsh1", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10111, + "backendPort": 22 + } + }, + { + "name": "postgresSsh2", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10112, + "backendPort": 22 + } + }, + { + "name": "postgresSsh3", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10113, + "backendPort": 22 + } + }, + { + "name": "postgresSsh4", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10114, + "backendPort": 22 + } + }, + { + "name": "postgresSsh5", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10115, + "backendPort": 22 + } + }, + { + "name": "postgresSsh6", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10116, + "backendPort": 22 + } + }, + { + "name": "postgresSsh7", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10117, + "backendPort": 22 + } + }, + { + "name": "postgresSsh8", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10118, + "backendPort": 22 + } + }, + { + "name": "postgresSsh9", + "properties": { + "frontendIPConfiguration": { + "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + }, + "protocol": "Tcp", + "frontendPort": 10119, + "backendPort": 22 + } + } + ] } - } - ] - } - } - ] + } + ] } \ No newline at end of file diff --git a/nestedtemplates/network-resources.json b/nestedtemplates/network-resources.json index 9f315e1..896243f 100644 --- a/nestedtemplates/network-resources.json +++ b/nestedtemplates/network-resources.json @@ -1,56 +1,56 @@ { - "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "vnetName": { - "type": "String", - "defaultValue": "" - }, - "zookeeperNetName": { - "type": "String", - "defaultValue": "" + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "vnetName": { + "type": "String", + "defaultValue": "" + }, + "zookeeperNetName": { + "type": "String", + "defaultValue": "" + }, + "postgresNetName": { + "type": "String", + "defaultValue": "" + }, + "clusterName": { + "type": "String", + "defaultValue": "" + } }, - "postgresNetName": { - "type": "String", - "defaultValue": "" + "variables": { + "vnetName": "[concat(parameters('clusterName'), '-net')]" }, - "clusterName": { - "type": "String", - "defaultValue": "" - } - }, - "variables": { - "vnetName": "[concat(parameters('clusterName'), '-net')]" - }, - "resources": [ - { - "type": "Microsoft.Network/virtualNetworks", - "name": "[variables('vnetName')]", - "apiVersion": "2015-06-15", - "location": "[resourceGroup().location]", - "tags": {}, - "properties": { - "addressSpace": { - "addressPrefixes": [ - "10.0.0.0/16" - ] - }, - "subnets": [ - { - "name": "[parameters('zookeeperNetName')]", - "properties": { - "addressPrefix": "10.0.100.0/24" - } - }, - { - "name": "[parameters('postgresNetName')]", + "resources": [ + { + "type": "Microsoft.Network/virtualNetworks", + "name": "[variables('vnetName')]", + "apiVersion": "2015-06-15", + "location": "[resourceGroup().location]", + "tags": {}, "properties": { - "addressPrefix": "10.0.101.0/24" - } - } - ] - }, - "dependsOn": [] - } - ] + "addressSpace": { + "addressPrefixes": [ + "10.0.0.0/16" + ] + }, + "subnets": [ + { + "name": "[parameters('zookeeperNetName')]", + "properties": { + "addressPrefix": "10.0.100.0/24" + } + }, + { + "name": "[parameters('postgresNetName')]", + "properties": { + "addressPrefix": "10.0.101.0/24" + } + } + ] + }, + "dependsOn": [] + } + ] } \ No newline at end of file diff --git a/template.json b/template.json index ab4bb81..d821333 100644 --- a/template.json +++ b/template.json @@ -114,6 +114,7 @@ { "type": "Microsoft.Network/virtualNetworks", "name": "[parameters('vnetName')]", + "location": "[resourceGroup().location]", "apiVersion": "2015-06-15", "properties": { "mode": "Incremental", @@ -132,7 +133,7 @@ "value": "[parameters('postgresNetName')]" }, "clusterName": { - "value": "[variables('clusterName')]" + "value": "[parameters('clusterName')]" } } } @@ -201,6 +202,7 @@ ] }, "dependsOn": [ + "[variables('vnetId')]", "[variables('lbId')]" ] }, @@ -313,6 +315,7 @@ ] }, "dependsOn": [ + "[variables('vnetId')]", "[variables('lbId')]" ] }, @@ -411,6 +414,7 @@ { "type": "Microsoft.Network/loadBalancers", "name": "[variables('lbName')]", + "location": "[resourceGroup().location]", "apiVersion": "2015-06-15", "properties": { "mode": "Incremental", From 12a3b519e1ec49a3adb24bbc893bf7ab4e15e3bb Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Thu, 2 Feb 2017 11:56:09 -0800 Subject: [PATCH 09/51] more parameter fixes --- nestedtemplates/external-lb.json | 4 ++++ nestedtemplates/internal-lb.json | 6 +++++- nestedtemplates/network-resources.json | 7 +------ template.json | 6 ++++-- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/nestedtemplates/external-lb.json b/nestedtemplates/external-lb.json index 6e39bfa..f9d1999 100644 --- a/nestedtemplates/external-lb.json +++ b/nestedtemplates/external-lb.json @@ -13,6 +13,10 @@ "lbId": { "type": "String", "defaultValue": "" + }, + "vnetId": { + "type": "String", + "defaultValue": "" } }, "variables": {}, diff --git a/nestedtemplates/internal-lb.json b/nestedtemplates/internal-lb.json index 2e6b1cc..7cbd0de 100644 --- a/nestedtemplates/internal-lb.json +++ b/nestedtemplates/internal-lb.json @@ -13,6 +13,10 @@ "lbId": { "type": "String", "defaultValue": "" + }, + "vnetId": { + "type": "String", + "defaultValue": "" } }, "variables": {}, @@ -33,7 +37,7 @@ "privateIPAddress": "10.0.101.4", "privateIPAllocationMethod": "Static", "subnet": { - "id": "[concat(variables('vnetId'), '/subnets/', parameters('zookeeperNetName'))]" + "id": "[concat(parameters('vnetId'), '/subnets/', parameters('zookeeperNetName'))]" } } } diff --git a/nestedtemplates/network-resources.json b/nestedtemplates/network-resources.json index 896243f..2a504ef 100644 --- a/nestedtemplates/network-resources.json +++ b/nestedtemplates/network-resources.json @@ -13,19 +13,14 @@ "postgresNetName": { "type": "String", "defaultValue": "" - }, - "clusterName": { - "type": "String", - "defaultValue": "" } }, "variables": { - "vnetName": "[concat(parameters('clusterName'), '-net')]" }, "resources": [ { "type": "Microsoft.Network/virtualNetworks", - "name": "[variables('vnetName')]", + "name": "[parameters('vnetName')]", "apiVersion": "2015-06-15", "location": "[resourceGroup().location]", "tags": {}, diff --git a/template.json b/template.json index d821333..d5b018d 100644 --- a/template.json +++ b/template.json @@ -32,7 +32,7 @@ }, "lbType": { "type": "string", - "defaultValue": "yes", + "defaultValue": "internal", "allowedValues": [ "internal", "external" @@ -99,7 +99,6 @@ "internal": "internal-lb.json" }, "lbId": "[resourceId('Microsoft.Network/loadBalancers/', variables('lbName'))]", - "vnetGroup": "testht", "vnetId": "[resourceId(parameters('vnetGroup'), 'Microsoft.Network/virtualNetworks', parameters('vnetName'))]", "zookeeperInstanceCount": 3, "postgresInstanceCount": "[parameters('instanceCount')]", @@ -134,6 +133,9 @@ }, "clusterName": { "value": "[parameters('clusterName')]" + }, + "vnetId": { + "value": "[variables('vnetId')]" } } } From b432fc4e5f3a7192362a8014ca33c8746bbd15c0 Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Thu, 2 Feb 2017 12:07:17 -0800 Subject: [PATCH 10/51] more template fixes --- nestedtemplates/internal-lb.json | 2 +- nestedtemplates/network-resources.json | 2 +- template.json | 12 +++++------- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/nestedtemplates/internal-lb.json b/nestedtemplates/internal-lb.json index 7cbd0de..8c1896b 100644 --- a/nestedtemplates/internal-lb.json +++ b/nestedtemplates/internal-lb.json @@ -23,7 +23,7 @@ "resources": [ { "type": "Microsoft.Network/loadBalancers", - "name": "[variables('lbName')]", + "name": "[parameters('lbName')]", "apiVersion": "2015-06-15", "location": "[resourceGroup().location]", "properties": { diff --git a/nestedtemplates/network-resources.json b/nestedtemplates/network-resources.json index 2a504ef..cdb9014 100644 --- a/nestedtemplates/network-resources.json +++ b/nestedtemplates/network-resources.json @@ -21,7 +21,7 @@ { "type": "Microsoft.Network/virtualNetworks", "name": "[parameters('vnetName')]", - "apiVersion": "2015-06-15", + "apiVersion": "2016-03-30", "location": "[resourceGroup().location]", "tags": {}, "properties": { diff --git a/template.json b/template.json index d5b018d..27d23dd 100644 --- a/template.json +++ b/template.json @@ -111,10 +111,9 @@ }, "resources": [ { - "type": "Microsoft.Network/virtualNetworks", + "type": "Microsoft.Resources/deployments", "name": "[parameters('vnetName')]", - "location": "[resourceGroup().location]", - "apiVersion": "2015-06-15", + "apiVersion": "2015-01-01", "properties": { "mode": "Incremental", "templateLink": { @@ -414,10 +413,9 @@ ] }, { - "type": "Microsoft.Network/loadBalancers", - "name": "[variables('lbName')]", - "location": "[resourceGroup().location]", - "apiVersion": "2015-06-15", + "type": "Microsoft.Resources/deployments", + "name": "lb-temp", + "apiVersion": "2015-01-01", "properties": { "mode": "Incremental", "templateLink": { From 797497a4bb258141d6720a051c2e4f7c3845bc8d Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Thu, 2 Feb 2017 12:13:26 -0800 Subject: [PATCH 11/51] add dependencies --- template.json | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/template.json b/template.json index 27d23dd..e1f4b36 100644 --- a/template.json +++ b/template.json @@ -112,7 +112,7 @@ "resources": [ { "type": "Microsoft.Resources/deployments", - "name": "[parameters('vnetName')]", + "name": "vnet", "apiVersion": "2015-01-01", "properties": { "mode": "Incremental", @@ -203,6 +203,7 @@ ] }, "dependsOn": [ + "[resourceId('Microsoft.Resources/deployments', 'vnet')]", "[variables('vnetId')]", "[variables('lbId')]" ] @@ -255,6 +256,7 @@ "dependsOn": [ "[concat('Microsoft.Storage/storageAccounts/', variables('storageAccountPrefix')[copyIndex()], variables('storageAccountSuffix'))]", "[concat('Microsoft.Network/networkInterfaces/', 'zookeeper', copyIndex(), '-nic')]", + "[resourceId('Microsoft.Resources/deployments', 'vnet')]", "[concat('Microsoft.Compute/availabilitySets/', 'zookeeperAvailabilitySet')]" ] }, @@ -316,6 +318,7 @@ ] }, "dependsOn": [ + "[resourceId('Microsoft.Resources/deployments', 'vnet')]", "[variables('vnetId')]", "[variables('lbId')]" ] @@ -433,7 +436,10 @@ "value": "[variables('lbId')]" } } - } + }, + "dependsOn": [ + "[resourceId('Microsoft.Resources/deployments', 'vnet')]" + ] } ], "outputs": { From 2871e4c4ba2f35fac51fe768ac4ae42dde359a20 Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Thu, 2 Feb 2017 12:23:11 -0800 Subject: [PATCH 12/51] remove wrong dependency --- template.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/template.json b/template.json index e1f4b36..2505b56 100644 --- a/template.json +++ b/template.json @@ -204,7 +204,6 @@ }, "dependsOn": [ "[resourceId('Microsoft.Resources/deployments', 'vnet')]", - "[variables('vnetId')]", "[variables('lbId')]" ] }, @@ -319,7 +318,6 @@ }, "dependsOn": [ "[resourceId('Microsoft.Resources/deployments', 'vnet')]", - "[variables('vnetId')]", "[variables('lbId')]" ] }, From 4f079056107621471c054f18ed49e8b993828bd9 Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Thu, 2 Feb 2017 12:30:32 -0800 Subject: [PATCH 13/51] removed parameters which are not used --- nestedtemplates/empty-resources.json | 4 ---- template.json | 10 ++-------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/nestedtemplates/empty-resources.json b/nestedtemplates/empty-resources.json index 0454288..23a54f2 100644 --- a/nestedtemplates/empty-resources.json +++ b/nestedtemplates/empty-resources.json @@ -13,10 +13,6 @@ "postgresNetName": { "type": "String", "defaultValue": "" - }, - "clusterName": { - "type": "String", - "defaultValue": "" } }, "variables": {}, diff --git a/template.json b/template.json index 2505b56..badf520 100644 --- a/template.json +++ b/template.json @@ -129,12 +129,6 @@ }, "postgresNetName": { "value": "[parameters('postgresNetName')]" - }, - "clusterName": { - "value": "[parameters('clusterName')]" - }, - "vnetId": { - "value": "[variables('vnetId')]" } } } @@ -204,7 +198,7 @@ }, "dependsOn": [ "[resourceId('Microsoft.Resources/deployments', 'vnet')]", - "[variables('lbId')]" + "[resourceId('Microsoft.Resources/deployments', 'lb-temp')]" ] }, { @@ -318,7 +312,7 @@ }, "dependsOn": [ "[resourceId('Microsoft.Resources/deployments', 'vnet')]", - "[variables('lbId')]" + "[resourceId('Microsoft.Resources/deployments', 'lb-temp')]" ] }, { From f06b2bb8b15e1eaf37a6b2dc9198af079fdd5ead Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Thu, 2 Feb 2017 12:36:18 -0800 Subject: [PATCH 14/51] update dependencies --- nestedtemplates/internal-lb.json | 4 ++++ template.json | 3 +++ 2 files changed, 7 insertions(+) diff --git a/nestedtemplates/internal-lb.json b/nestedtemplates/internal-lb.json index 8c1896b..937bf34 100644 --- a/nestedtemplates/internal-lb.json +++ b/nestedtemplates/internal-lb.json @@ -17,6 +17,10 @@ "vnetId": { "type": "String", "defaultValue": "" + }, + "zookeeperNetName": { + "defaultValue": "", + "type": "String" } }, "variables": {}, diff --git a/template.json b/template.json index badf520..37f17b4 100644 --- a/template.json +++ b/template.json @@ -426,6 +426,9 @@ }, "lbId": { "value": "[variables('lbId')]" + }, + "zookeeperNetName": { + "value": "[parameters('zookeeperNetName')]" } } }, From 3e3ed92070a3a356d22ff468ef216d0aaa790e1e Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Thu, 2 Feb 2017 12:41:38 -0800 Subject: [PATCH 15/51] fix another parameter error --- nestedtemplates/internal-lb.json | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/nestedtemplates/internal-lb.json b/nestedtemplates/internal-lb.json index 937bf34..c1cdffc 100644 --- a/nestedtemplates/internal-lb.json +++ b/nestedtemplates/internal-lb.json @@ -56,16 +56,16 @@ "name": "postgresLbRule", "properties": { "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" }, "backendAddressPool": { - "id": "[concat(variables('lbId'), '/backendAddressPools/loadBalancerBackEnd')]" + "id": "[concat(parameters('lbId'), '/backendAddressPools/loadBalancerBackEnd')]" }, "protocol": "Tcp", "frontendPort": 5432, "backendPort": 5000, "probe": { - "id": "[concat(variables('lbId'), '/probes/postgresProbe')]" + "id": "[concat(parameters('lbId'), '/probes/postgresProbe')]" } } } @@ -86,7 +86,7 @@ "name": "postgresSsh0", "properties": { "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" }, "protocol": "Tcp", "frontendPort": 10110, @@ -97,7 +97,7 @@ "name": "postgresSsh1", "properties": { "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" }, "protocol": "Tcp", "frontendPort": 10111, @@ -108,7 +108,7 @@ "name": "postgresSsh2", "properties": { "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" }, "protocol": "Tcp", "frontendPort": 10112, @@ -119,7 +119,7 @@ "name": "postgresSsh3", "properties": { "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" }, "protocol": "Tcp", "frontendPort": 10113, @@ -130,7 +130,7 @@ "name": "postgresSsh4", "properties": { "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" }, "protocol": "Tcp", "frontendPort": 10114, @@ -141,7 +141,7 @@ "name": "postgresSsh5", "properties": { "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" }, "protocol": "Tcp", "frontendPort": 10115, @@ -152,7 +152,7 @@ "name": "postgresSsh6", "properties": { "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" }, "protocol": "Tcp", "frontendPort": 10116, @@ -163,7 +163,7 @@ "name": "postgresSsh7", "properties": { "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" }, "protocol": "Tcp", "frontendPort": 10117, @@ -174,7 +174,7 @@ "name": "postgresSsh8", "properties": { "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" }, "protocol": "Tcp", "frontendPort": 10118, @@ -185,7 +185,7 @@ "name": "postgresSsh9", "properties": { "frontendIPConfiguration": { - "id": "[concat(variables('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" + "id": "[concat(parameters('lbId'), '/frontendIPConfigurations/loadBalancerFrontEnd')]" }, "protocol": "Tcp", "frontendPort": 10119, From 8d3435ec9b55f8b6fd440ba7abffbbcf4eadcb14 Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Thu, 2 Feb 2017 12:57:26 -0800 Subject: [PATCH 16/51] fix another parameter error --- nestedtemplates/external-lb.json | 4 ++++ template.json | 3 +++ 2 files changed, 7 insertions(+) diff --git a/nestedtemplates/external-lb.json b/nestedtemplates/external-lb.json index f9d1999..554fbf5 100644 --- a/nestedtemplates/external-lb.json +++ b/nestedtemplates/external-lb.json @@ -17,6 +17,10 @@ "vnetId": { "type": "String", "defaultValue": "" + }, + "zookeeperNetName": { + "defaultValue": "", + "type": "String" } }, "variables": {}, diff --git a/template.json b/template.json index 37f17b4..a5589cc 100644 --- a/template.json +++ b/template.json @@ -427,6 +427,9 @@ "lbId": { "value": "[variables('lbId')]" }, + "vnetId": { + "value": "[variables('vnetId')]" + }, "zookeeperNetName": { "value": "[parameters('zookeeperNetName')]" } From 2de281cc4d950418ee431296333f5a50d075102a Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Thu, 2 Feb 2017 13:02:46 -0800 Subject: [PATCH 17/51] fixed load balancer error --- nestedtemplates/external-lb.json | 2 +- nestedtemplates/internal-lb.json | 4 ++-- template.json | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nestedtemplates/external-lb.json b/nestedtemplates/external-lb.json index 554fbf5..f389d34 100644 --- a/nestedtemplates/external-lb.json +++ b/nestedtemplates/external-lb.json @@ -18,7 +18,7 @@ "type": "String", "defaultValue": "" }, - "zookeeperNetName": { + "postgresNetName": { "defaultValue": "", "type": "String" } diff --git a/nestedtemplates/internal-lb.json b/nestedtemplates/internal-lb.json index c1cdffc..5be73e4 100644 --- a/nestedtemplates/internal-lb.json +++ b/nestedtemplates/internal-lb.json @@ -18,7 +18,7 @@ "type": "String", "defaultValue": "" }, - "zookeeperNetName": { + "postgresNetName": { "defaultValue": "", "type": "String" } @@ -41,7 +41,7 @@ "privateIPAddress": "10.0.101.4", "privateIPAllocationMethod": "Static", "subnet": { - "id": "[concat(parameters('vnetId'), '/subnets/', parameters('zookeeperNetName'))]" + "id": "[concat(parameters('vnetId'), '/subnets/', parameters('postgresNetName'))]" } } } diff --git a/template.json b/template.json index a5589cc..9cc78eb 100644 --- a/template.json +++ b/template.json @@ -430,8 +430,8 @@ "vnetId": { "value": "[variables('vnetId')]" }, - "zookeeperNetName": { - "value": "[parameters('zookeeperNetName')]" + "postgresNetName": { + "value": "[parameters('postgresNetName')]" } } }, From 74c007a17c1f736f3123c722291d60e29e3e3593 Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Thu, 2 Feb 2017 13:32:03 -0800 Subject: [PATCH 18/51] fix script locations --- template.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/template.json b/template.json index 9cc78eb..2f57544 100644 --- a/template.json +++ b/template.json @@ -268,7 +268,7 @@ "typeHandlerVersion": "1.2", "settings": { "fileUris": [ - "concat(variables('nestedTemplates'), '/zookeeper_startup.sh'" + "concat(parameters('_artifactsLocation'), '/zookeeper_startup.sh'" ], "commandToExecute": "[concat('sh zookeeper_startup.sh ', copyIndex(), ' ', variables('zookeeperInstanceCount'))]" } @@ -397,8 +397,8 @@ "typeHandlerVersion": "1.2", "settings": { "fileUris": [ - "concat(variables('nestedTemplates'), '/autopart.sh'", - "concat(variables('nestedTemplates'), '/postgres_startup.sh'" + "concat(parameters('_artifactsLocation'), '/autopart.sh'", + "concat(parameters('_artifactsLocation'), '/postgres_startup.sh'" ], "commandToExecute": "[concat('sh postgres_startup.sh ', parameters('clusterName'), ' ', variables('zookeeperInstanceCount'), ' ', variables('postgresInstanceCount'), ' ', copyIndex(), ' ', parameters('adminUsername'), ' \"', parameters('adminPassword'), '\" ')]" } From b697b3ae2afd0804cc8c23c483377359363ca688 Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Thu, 2 Feb 2017 14:26:29 -0800 Subject: [PATCH 19/51] fixed concat method --- parameters.json | 39 +++++++++++++++++++++++++++++++++++++++ template.json | 6 +++--- 2 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 parameters.json diff --git a/parameters.json b/parameters.json new file mode 100644 index 0000000..eff1543 --- /dev/null +++ b/parameters.json @@ -0,0 +1,39 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "clusterName": { + "value": "htpgtest3" + }, + "_artifactsLocation": { + "value": "https://raw.githubusercontent.com/HelloTech/azure_patroni/custom_deploy" + }, + "vnetGroup": { + "value": "htpgtest3" + }, + "newVnet": { + "value": "no" + }, + "lbType": { + "value": "internal" + }, + "vnetName": { + "value": "htpgtest3" + }, + "zookeeperNetName": { + "value": "zhtpgtest3" + }, + "postgresNetName": { + "value": "phtpgtest3" + }, + "instanceCount": { + "value": 2 + }, + "adminUsername": { + "value": "hellotech" + }, + "adminPassword": { + "value": "BigMonday2015" + } + } +} \ No newline at end of file diff --git a/template.json b/template.json index 2f57544..1f255e8 100644 --- a/template.json +++ b/template.json @@ -268,7 +268,7 @@ "typeHandlerVersion": "1.2", "settings": { "fileUris": [ - "concat(parameters('_artifactsLocation'), '/zookeeper_startup.sh'" + "[concat(parameters('_artifactsLocation'), '/zookeeper_startup.sh']" ], "commandToExecute": "[concat('sh zookeeper_startup.sh ', copyIndex(), ' ', variables('zookeeperInstanceCount'))]" } @@ -397,8 +397,8 @@ "typeHandlerVersion": "1.2", "settings": { "fileUris": [ - "concat(parameters('_artifactsLocation'), '/autopart.sh'", - "concat(parameters('_artifactsLocation'), '/postgres_startup.sh'" + "[concat(parameters('_artifactsLocation'), '/autopart.sh']", + "[concat(parameters('_artifactsLocation'), '/postgres_startup.sh']" ], "commandToExecute": "[concat('sh postgres_startup.sh ', parameters('clusterName'), ' ', variables('zookeeperInstanceCount'), ' ', variables('postgresInstanceCount'), ' ', copyIndex(), ' ', parameters('adminUsername'), ' \"', parameters('adminPassword'), '\" ')]" } From 32ae7d7008d62cff4dec0d85cf8eca41956dfe4d Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Thu, 2 Feb 2017 14:30:26 -0800 Subject: [PATCH 20/51] fixed concat method --- template.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/template.json b/template.json index 1f255e8..914dc37 100644 --- a/template.json +++ b/template.json @@ -268,7 +268,7 @@ "typeHandlerVersion": "1.2", "settings": { "fileUris": [ - "[concat(parameters('_artifactsLocation'), '/zookeeper_startup.sh']" + "[concat(parameters('_artifactsLocation'), '/zookeeper_startup.sh')]" ], "commandToExecute": "[concat('sh zookeeper_startup.sh ', copyIndex(), ' ', variables('zookeeperInstanceCount'))]" } @@ -397,8 +397,8 @@ "typeHandlerVersion": "1.2", "settings": { "fileUris": [ - "[concat(parameters('_artifactsLocation'), '/autopart.sh']", - "[concat(parameters('_artifactsLocation'), '/postgres_startup.sh']" + "[concat(parameters('_artifactsLocation'), '/autopart.sh')]", + "[concat(parameters('_artifactsLocation'), '/postgres_startup.sh')]" ], "commandToExecute": "[concat('sh postgres_startup.sh ', parameters('clusterName'), ' ', variables('zookeeperInstanceCount'), ' ', variables('postgresInstanceCount'), ' ', copyIndex(), ' ', parameters('adminUsername'), ' \"', parameters('adminPassword'), '\" ')]" } From fd1b5dc3c821c543fd99b3cb62e164b797a43b05 Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Thu, 2 Feb 2017 14:58:52 -0800 Subject: [PATCH 21/51] delete --- parameters.json | 39 --------------------------------------- 1 file changed, 39 deletions(-) delete mode 100644 parameters.json diff --git a/parameters.json b/parameters.json deleted file mode 100644 index eff1543..0000000 --- a/parameters.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "clusterName": { - "value": "htpgtest3" - }, - "_artifactsLocation": { - "value": "https://raw.githubusercontent.com/HelloTech/azure_patroni/custom_deploy" - }, - "vnetGroup": { - "value": "htpgtest3" - }, - "newVnet": { - "value": "no" - }, - "lbType": { - "value": "internal" - }, - "vnetName": { - "value": "htpgtest3" - }, - "zookeeperNetName": { - "value": "zhtpgtest3" - }, - "postgresNetName": { - "value": "phtpgtest3" - }, - "instanceCount": { - "value": 2 - }, - "adminUsername": { - "value": "hellotech" - }, - "adminPassword": { - "value": "BigMonday2015" - } - } -} \ No newline at end of file From ac44796d5c05903ed56764c10d6fc59642dace96 Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Thu, 2 Feb 2017 15:32:52 -0800 Subject: [PATCH 22/51] replace tabs with spaces --- postgres_startup.sh | 140 ++++++++++++++++++++++---------------------- 1 file changed, 70 insertions(+), 70 deletions(-) diff --git a/postgres_startup.sh b/postgres_startup.sh index 3aae507..d9b3ff0 100644 --- a/postgres_startup.sh +++ b/postgres_startup.sh @@ -120,32 +120,32 @@ echo "" >> $patroniCfg if [ $myIndex -eq 0 ] then echo "bootstrap:" >> $patroniCfg - echo " dcs:" >> $patroniCfg - echo " ttl: *ttl" >> $patroniCfg - echo " loop_wait: *loop_wait" >> $patroniCfg - echo " retry_timeout: *loop_wait" >> $patroniCfg - echo " maximum_lag_on_failover: 1048576" >> $patroniCfg - echo " postgresql:" >> $patroniCfg - echo " use_pg_rewind: true" >> $patroniCfg - echo " use_slots: true" >> $patroniCfg - echo " parameters:" >> $patroniCfg - echo " archive_mode: \"on\"" >> $patroniCfg - echo " archive_timeout: 1800s" >> $patroniCfg - echo " archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f" >> $patroniCfg - echo " recovery_conf:" >> $patroniCfg - echo " restore_command: cp ../wal_archive/%f %p" >> $patroniCfg - echo " initdb:" >> $patroniCfg - echo " - encoding: UTF8" >> $patroniCfg - echo " - data-checksums" >> $patroniCfg - echo " pg_hba:" >> $patroniCfg - echo " - host replication all 0.0.0.0/0 md5" >> $patroniCfg - echo " - host all all 0.0.0.0/0 md5" >> $patroniCfg - echo " users:" >> $patroniCfg - echo " admin:" >> $patroniCfg - echo " password: \"$adminPassword\"" >> $patroniCfg - echo " options:" >> $patroniCfg - echo " - createrole" >> $patroniCfg - echo " - createdb" >> $patroniCfg + echo " dcs:" >> $patroniCfg + echo " ttl: *ttl" >> $patroniCfg + echo " loop_wait: *loop_wait" >> $patroniCfg + echo " retry_timeout: *loop_wait" >> $patroniCfg + echo " maximum_lag_on_failover: 1048576" >> $patroniCfg + echo " postgresql:" >> $patroniCfg + echo " use_pg_rewind: true" >> $patroniCfg + echo " use_slots: true" >> $patroniCfg + echo " parameters:" >> $patroniCfg + echo " archive_mode: \"on\"" >> $patroniCfg + echo " archive_timeout: 1800s" >> $patroniCfg + echo " archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f" >> $patroniCfg + echo " recovery_conf:" >> $patroniCfg + echo " restore_command: cp ../wal_archive/%f %p" >> $patroniCfg + echo " initdb:" >> $patroniCfg + echo " - encoding: UTF8" >> $patroniCfg + echo " - data-checksums" >> $patroniCfg + echo " pg_hba:" >> $patroniCfg + echo " - host replication all 0.0.0.0/0 md5" >> $patroniCfg + echo " - host all all 0.0.0.0/0 md5" >> $patroniCfg + echo " users:" >> $patroniCfg + echo " admin:" >> $patroniCfg + echo " password: \"$adminPassword\"" >> $patroniCfg + echo " options:" >> $patroniCfg + echo " - createrole" >> $patroniCfg + echo " - createdb" >> $patroniCfg fi echo "" >> $patroniCfg echo "tags:" >> $patroniCfg @@ -165,53 +165,53 @@ echo " pgpass: /tmp/pgpass" >> $patroniCfg if [ $myIndex -ne 0 ] then echo " maximum_lag_on_failover: 1048576" >> $patroniCfg - echo " use_slots: true" >> $patroniCfg - echo " initdb:" >> $patroniCfg - echo " - encoding: UTF8" >> $patroniCfg - echo " - data-checksums" >> $patroniCfg - echo " pg_rewind:" >> $patroniCfg - echo " username: postgres" >> $patroniCfg - echo " password: \"$adminPassword\"" >> $patroniCfg - echo " pg_hba:" >> $patroniCfg - echo " - host replication all 0.0.0.0/0 md5" >> $patroniCfg - echo " - host all all 0.0.0.0/0 md5" >> $patroniCfg - echo " replication:" >> $patroniCfg - echo " username: replicator" >> $patroniCfg - echo " password: \"$adminPassword\"" >> $patroniCfg - echo " superuser:" >> $patroniCfg - echo " username: postgres" >> $patroniCfg - echo " password: \"$adminPassword\"" >> $patroniCfg - echo " admin:" >> $patroniCfg - echo " username: admin" >> $patroniCfg - echo " password: \"$adminPassword\"" >> $patroniCfg - echo " create_replica_method:" >> $patroniCfg - echo " - basebackup" >> $patroniCfg - echo " recovery_conf:" >> $patroniCfg - echo " restore_command: cp ../wal_archive/%f %p" >> $patroniCfg - echo " parameters:" >> $patroniCfg - echo " archive_mode: \"on\"" >> $patroniCfg - echo " wal_level: hot_standby" >> $patroniCfg - echo " archive_command: mkdir -r ../wal_archive && test ! -f ../wal_archive/%f && cp %cp ../wal_archive/%f" >> $patroniCfg - echo " max_wal_senders: 10" >> $patroniCfg - echo " wal_keep_segments: 8" >> $patroniCfg - echo " archive_timeout: 1800s" >> $patroniCfg - echo " max_replication_slots: 10" >> $patroniCfg - echo " hot_standby: \"on\"" >> $patroniCfg - echo " wal_log_hints: \"on\"" >> $patroniCfg - echo " unix_socket_directories: '.'" >> $patroniCfg + echo " use_slots: true" >> $patroniCfg + echo " initdb:" >> $patroniCfg + echo " - encoding: UTF8" >> $patroniCfg + echo " - data-checksums" >> $patroniCfg + echo " pg_rewind:" >> $patroniCfg + echo " username: postgres" >> $patroniCfg + echo " password: \"$adminPassword\"" >> $patroniCfg + echo " pg_hba:" >> $patroniCfg + echo " - host replication all 0.0.0.0/0 md5" >> $patroniCfg + echo " - host all all 0.0.0.0/0 md5" >> $patroniCfg + echo " replication:" >> $patroniCfg + echo " username: replicator" >> $patroniCfg + echo " password: \"$adminPassword\"" >> $patroniCfg + echo " superuser:" >> $patroniCfg + echo " username: postgres" >> $patroniCfg + echo " password: \"$adminPassword\"" >> $patroniCfg + echo " admin:" >> $patroniCfg + echo " username: admin" >> $patroniCfg + echo " password: \"$adminPassword\"" >> $patroniCfg + echo " create_replica_method:" >> $patroniCfg + echo " - basebackup" >> $patroniCfg + echo " recovery_conf:" >> $patroniCfg + echo " restore_command: cp ../wal_archive/%f %p" >> $patroniCfg + echo " parameters:" >> $patroniCfg + echo " archive_mode: \"on\"" >> $patroniCfg + echo " wal_level: hot_standby" >> $patroniCfg + echo " archive_command: mkdir -r ../wal_archive && test ! -f ../wal_archive/%f && cp %cp ../wal_archive/%f" >> $patroniCfg + echo " max_wal_senders: 10" >> $patroniCfg + echo " wal_keep_segments: 8" >> $patroniCfg + echo " archive_timeout: 1800s" >> $patroniCfg + echo " max_replication_slots: 10" >> $patroniCfg + echo " hot_standby: \"on\"" >> $patroniCfg + echo " wal_log_hints: \"on\"" >> $patroniCfg + echo " unix_socket_directories: '.'" >> $patroniCfg else echo " authentication:" >> $patroniCfg - echo " replication:" >> $patroniCfg - echo " username: replicator" >> $patroniCfg - echo " password: \"$adminPassword\"" >> $patroniCfg - echo " superuser:" >> $patroniCfg - echo " username: postgres" >> $patroniCfg - echo " password: \"$adminPassword\"" >> $patroniCfg - echo " parameters:" >> $patroniCfg - echo " unix_socket_directories: '.'" >> $patroniCfg + echo " replication:" >> $patroniCfg + echo " username: replicator" >> $patroniCfg + echo " password: \"$adminPassword\"" >> $patroniCfg + echo " superuser:" >> $patroniCfg + echo " username: postgres" >> $patroniCfg + echo " password: \"$adminPassword\"" >> $patroniCfg + echo " parameters:" >> $patroniCfg + echo " unix_socket_directories: '.'" >> $patroniCfg fi - - + + # install HA PROXY sudo apt-get --assume-yes install haproxy # write configuration From c505ede5c27cbf1c8b5f2dbdbc4a1f5522f0aadc Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Thu, 2 Feb 2017 15:36:45 -0800 Subject: [PATCH 23/51] remove dependency --- nestedtemplates/external-lb.json | 5 +---- template.json | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/nestedtemplates/external-lb.json b/nestedtemplates/external-lb.json index f389d34..d5324c6 100644 --- a/nestedtemplates/external-lb.json +++ b/nestedtemplates/external-lb.json @@ -203,10 +203,7 @@ } } ] - }, - "dependsOn": [ - "[variables('ipId')]" - ] + } } ] } \ No newline at end of file diff --git a/template.json b/template.json index 914dc37..7c5ccef 100644 --- a/template.json +++ b/template.json @@ -354,7 +354,7 @@ "dataDisks": [ { "name": "data0", - "diskSizeGB": "128", + "diskSizeGB": "256", "lun": 0, "vhd": { "uri": "[concat(reference(concat('Microsoft.Storage/storageAccounts/', variables('storageAccountPrefix')[copyIndex()], 'd', variables('storageAccountSuffix')), providers('Microsoft.Storage', 'storageAccounts').apiVersions[0]).primaryEndpoints.blob, 'vhds/', 'data0', copyIndex(), '.vhd')]" From 5b1949d4ee2f0fde0183c0269170d74965f2592c Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Thu, 2 Feb 2017 15:43:43 -0800 Subject: [PATCH 24/51] added missing params --- nestedtemplates/external-lb.json | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/nestedtemplates/external-lb.json b/nestedtemplates/external-lb.json index d5324c6..3c57da1 100644 --- a/nestedtemplates/external-lb.json +++ b/nestedtemplates/external-lb.json @@ -23,11 +23,14 @@ "type": "String" } }, - "variables": {}, + "variables": { + "ipName": "[concat(parameters('clusterName'), '-ip')]", + "ipId": "[resourceId('Microsoft.Network/publicIPAddresses/', variables('ipName'))]" + }, "resources": [ { "type": "Microsoft.Network/publicIPAddresses", - "name": "[concat(parameters('clusterName'), '-ip')]", + "name": "[variables('ipName')]", "apiVersion": "2016-03-30", "location": "[resourceGroup().location]", "properties": { @@ -51,7 +54,7 @@ "name": "loadBalancerFrontEnd", "properties": { "publicIPAddress": { - "id": "[parameters('ipId')]" + "id": "[variables('ipId')]" } } } @@ -203,7 +206,10 @@ } } ] - } + }, + "dependsOn": [ + "[variables('ipId')]" + ] } ] } \ No newline at end of file From d28c2d6fb239f4fee3ccfa907b7f3f3a98bb9df4 Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Thu, 2 Feb 2017 16:11:49 -0800 Subject: [PATCH 25/51] updates to allow choosing size of vm's and data disks --- template.json | 64 +++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 60 insertions(+), 4 deletions(-) diff --git a/template.json b/template.json index 7c5ccef..6d39ffd 100644 --- a/template.json +++ b/template.json @@ -13,7 +13,6 @@ } }, "vnetGroup": { - "defaultValue": "", "type": "String", "metadata": { "description": "VNET Resource Group." @@ -62,6 +61,63 @@ "description": "Name you would like to give to postgres' subnet." } }, + "zookeeperVMSize": { + "type": "string", + "defaultValue": "Standard_A3 ", + "allowedValues": [ + "Standard_D2_v2", + "Standard_D3_v2", + "Standard_D4_v2", + "Standard_D12_v2", + "Standard_A2", + "Standard_A3", + "Standard_A4", + "Standard_A5", + "Standard_A6", + "Standard_A7", + "Standard_DS2", + "Standard_DS3", + "Standard_DS4", + "Standard_DS12", + "Standard_DS13", + "Standard_DS14" + ], + "metadata": { + "description": "Size of the zookeeper vm's" + } + }, + "postgresVMSize": { + "type": "string", + "defaultValue": "Standard_DS12 ", + "allowedValues": [ + "Standard_D2_v2", + "Standard_D3_v2", + "Standard_D4_v2", + "Standard_D12_v2", + "Standard_A2", + "Standard_A3", + "Standard_A4", + "Standard_A5", + "Standard_A6", + "Standard_A7", + "Standard_DS2", + "Standard_DS3", + "Standard_DS4", + "Standard_DS12", + "Standard_DS13", + "Standard_DS14" + ], + "metadata": { + "description": "Size of the postgres vm's" + } + }, + "postgresDataSize": { + "defaultValue": 256, + "type": "Int", + "metadata": { + "description": "Size of the postgres data ssd in GB." + } + }, "instanceCount": { "defaultValue": 2, "minValue": 2, @@ -215,7 +271,7 @@ "id": "[resourceId('Microsoft.Compute/availabilitySets', 'zookeeperAvailabilitySet')]" }, "hardwareProfile": { - "vmSize": "Standard_A2" + "vmSize": "[parameters('zookeeperVMSize')]" }, "osProfile": { "computerName": "[concat('zookeeper', copyIndex())]", @@ -329,7 +385,7 @@ "id": "[resourceId('Microsoft.Compute/availabilitySets', 'postgresAvailabilitySet')]" }, "hardwareProfile": { - "vmSize": "Standard_DS1" + "vmSize": "[parameters('postgresVMSize')]" }, "osProfile": { "computerName": "[concat('postgres', copyIndex())]", @@ -354,7 +410,7 @@ "dataDisks": [ { "name": "data0", - "diskSizeGB": "256", + "diskSizeGB": "[parameters('postgresDataSize')]", "lun": 0, "vhd": { "uri": "[concat(reference(concat('Microsoft.Storage/storageAccounts/', variables('storageAccountPrefix')[copyIndex()], 'd', variables('storageAccountSuffix')), providers('Microsoft.Storage', 'storageAccounts').apiVersions[0]).primaryEndpoints.blob, 'vhds/', 'data0', copyIndex(), '.vhd')]" From d7d4b7a9a83123333549e4054c83e2d1736768f1 Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Thu, 2 Feb 2017 16:57:47 -0800 Subject: [PATCH 26/51] fixed default values --- template.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/template.json b/template.json index 6d39ffd..6252be5 100644 --- a/template.json +++ b/template.json @@ -63,7 +63,7 @@ }, "zookeeperVMSize": { "type": "string", - "defaultValue": "Standard_A3 ", + "defaultValue": "Standard_A3", "allowedValues": [ "Standard_D2_v2", "Standard_D3_v2", @@ -88,7 +88,7 @@ }, "postgresVMSize": { "type": "string", - "defaultValue": "Standard_DS12 ", + "defaultValue": "Standard_DS12", "allowedValues": [ "Standard_D2_v2", "Standard_D3_v2", From a0d2b99b546244c4b7622e4faf6018be2078f547 Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Thu, 2 Feb 2017 16:59:42 -0800 Subject: [PATCH 27/51] switch default deployment back to master --- README.md | 2 +- template.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f2fa614..9abeeb7 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Azure Patroni Deployment - + Date: Thu, 2 Feb 2017 17:27:49 -0800 Subject: [PATCH 28/51] added license and readme info --- LICENSE | 21 +++++++++++++++++++++ README.md | 47 ++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..2877737 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2017 HelloTech + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md index 9abeeb7..251ea37 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,49 @@ -# Azure Patroni Deployment - - \ No newline at end of file + + +# Azure Patroni Deployment + +One click deployment of a Highly-Available PostgreSQL Cluster on Azure. With automated replication, server management and leader election. This project contains a modified deployment version of Haufe's [blog post](http://dev.haufe.com/PostgreSQL-Cluster-Azure/). + +This azure template generates two sets of machines both running Ubuntu 16.04 LTS. The first set of three machines are used to run the distributed configuration store [ZooKeeper](https://zookeeper.apache.org/), while the second set is running PostgreSQL 9.6 together with [Patroni](https://github.com/zalando/patroni) providing a high-availability customizable PostgreSQL cluster deployment. + +# Parameters + +This template provides the following parameters, so you can customize your deployment to your needs: + + +* clusterName: What your cluster will be named. + +* _artifactsLocation: The repo name from which you are deploying. Default("https://raw.githubusercontent.com/HelloTech/azure_patroni/master") + +* newVnet: Whether or not you want to create a new vnet. Default("yes") + +* vnetGroup: The resource group in which the vnet is located. If you select yes on newVnet this needs to be set to the resource group in which you are deploying. + +* lbType: Whether you want to create an external or internal load balancer. If set to internal the load balancer will only be accessible from inside the virtual network. Default("internal") + +* vnetName: If newVnet is set to yet the this will be name of the created vnet otherwise this is the name of already existing vnet to which the deployment will be associated. + +* zookeeperNetName: If newVnet is set to yes the this will be name of the created subnet for the zookeeper machines, otherwise this is the name of already existing subnet that will contain the zookeeper vm's. + +* postgresNetName: If newVnet is set to yes the this will be name of the created subnet for the postgres machines, otherwise this is the name of already existing subnet that will contain the postgres vm's. + +* zookeeperVMSize: The size of zookeeper vms. + +* postgresVMSize: The size of the postgres vms. + +* postgresDataSize: The size of the ssd used for pg_data for the postgres vms. + +* instanceCount: The number of postgres vms. + +* adminUsername: The Ubuntu username. + +* adminPassword: The Ubuntu password. + +# License + +Released under the MIT license. See the LICENSE file for more info. \ No newline at end of file From f72bf251a586ca313135ada684874bf95b09523a Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Thu, 2 Feb 2017 17:35:16 -0800 Subject: [PATCH 29/51] added script info --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 251ea37..ff493e6 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,18 @@ This template provides the following parameters, so you can customize your deplo * adminPassword: The Ubuntu password. + +# Scripts + +### Zookeeper +The creation of the zookeeper instances is handled by the [zookeeper_startup.sh](https://github.com/HelloTech/azure_patroni/blob/custom_deploy/zookeeper_startup.sh) script. If you would like to make any changes to what is installed on those vm's that's the file you should modify. + +### Mounts +The mounting of the data ssds is handled by the [autopart.sh](https://github.com/HelloTech/azure_patroni/blob/custom_deploy/autopart.sh) script. + +### Postgres +The creation of the postgres instances is handles by the [postgres_startup.sh](https://github.com/HelloTech/azure_patroni/blob/custom_deploy/postgres_startup.sh) script. If you would like to make any changes to what is installed on those vm's that's the file you should modify. + # License Released under the MIT license. See the LICENSE file for more info. \ No newline at end of file From be6fc64e31a9345d8e1bb61817218754c845e63c Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Thu, 2 Feb 2017 19:55:38 -0800 Subject: [PATCH 30/51] added plv8 --- README.md | 14 +++++++------- postgres_startup.sh | 13 +++++++++++-- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index ff493e6..edc36bd 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,9 @@ http://armviz.io/#/?load=https%3A%2F%2Fraw.githubusercontent.com%2Fhellotech%2Fa # Azure Patroni Deployment -One click deployment of a Highly-Available PostgreSQL Cluster on Azure. With automated replication, server management and leader election. This project contains a modified deployment version of Haufe's [blog post](http://dev.haufe.com/PostgreSQL-Cluster-Azure/). +This one click deployment of a Highly-Available PostgreSQL Cluster on Azure has automated replication, server management and leader election. This project contains a modified deployment version of Haufe's [blog post](http://dev.haufe.com/PostgreSQL-Cluster-Azure/). -This azure template generates two sets of machines both running Ubuntu 16.04 LTS. The first set of three machines are used to run the distributed configuration store [ZooKeeper](https://zookeeper.apache.org/), while the second set is running PostgreSQL 9.6 together with [Patroni](https://github.com/zalando/patroni) providing a high-availability customizable PostgreSQL cluster deployment. +This Azure template generates two sets of machines both running Ubuntu 16.04 LTS. The first set of three machines are used to run the distributed configuration store [ZooKeeper](https://zookeeper.apache.org/), while the second set is running PostgreSQL 9.6 together with [Patroni](https://github.com/zalando/patroni) providing a high-availability customizable PostgreSQL cluster deployment. # Parameters @@ -26,11 +26,11 @@ This template provides the following parameters, so you can customize your deplo * lbType: Whether you want to create an external or internal load balancer. If set to internal the load balancer will only be accessible from inside the virtual network. Default("internal") -* vnetName: If newVnet is set to yet the this will be name of the created vnet otherwise this is the name of already existing vnet to which the deployment will be associated. +* vnetName: If newVnet is set to yes, this will be name of the created vnet. Otherwise, this is the name of already existing vnet to which the deployment will be associated. -* zookeeperNetName: If newVnet is set to yes the this will be name of the created subnet for the zookeeper machines, otherwise this is the name of already existing subnet that will contain the zookeeper vm's. +* zookeeperNetName: If newVnet is set to yes, this will be name of the created subnet for the zookeeper machines, otherwise this is the name of already existing subnet that will contain the zookeeper vm's. -* postgresNetName: If newVnet is set to yes the this will be name of the created subnet for the postgres machines, otherwise this is the name of already existing subnet that will contain the postgres vm's. +* postgresNetName: If newVnet is set to yes, this will be name of the created subnet for the postgres machines, otherwise this is the name of already existing subnet that will contain the postgres vm's. * zookeeperVMSize: The size of zookeeper vms. @@ -48,13 +48,13 @@ This template provides the following parameters, so you can customize your deplo # Scripts ### Zookeeper -The creation of the zookeeper instances is handled by the [zookeeper_startup.sh](https://github.com/HelloTech/azure_patroni/blob/custom_deploy/zookeeper_startup.sh) script. If you would like to make any changes to what is installed on those vm's that's the file you should modify. +The creation of the zookeeper instances is handled by the [zookeeper_startup.sh](https://github.com/HelloTech/azure_patroni/blob/custom_deploy/zookeeper_startup.sh) script. If you would like to make any changes to what is installed on those vm's modify this file. ### Mounts The mounting of the data ssds is handled by the [autopart.sh](https://github.com/HelloTech/azure_patroni/blob/custom_deploy/autopart.sh) script. ### Postgres -The creation of the postgres instances is handles by the [postgres_startup.sh](https://github.com/HelloTech/azure_patroni/blob/custom_deploy/postgres_startup.sh) script. If you would like to make any changes to what is installed on those vm's that's the file you should modify. +The creation of the postgres instance is handles by the [postgres_startup.sh](https://github.com/HelloTech/azure_patroni/blob/custom_deploy/postgres_startup.sh) script. If you would like to make any changes to what is installed on those vm's modify that script. # License diff --git a/postgres_startup.sh b/postgres_startup.sh index d9b3ff0..b7beb45 100644 --- a/postgres_startup.sh +++ b/postgres_startup.sh @@ -79,10 +79,19 @@ echo "pip python-dateutil exited with: $?" #sudo touch /usr/sbin/policy-rc.d #sudo chmod 777 /usr/sbin/policy-rc.d #echo exit 101 > /usr/sbin/policy-rc.d -sudo apt-get --assume-yes --force-yes -qq install postgresql postgresql-contrib -echo "apt-get postgresql postgresql-contrib exited with: $?" +sudo apt-get --assume-yes --force-yes -qq install postgresql postgresql-contrib postgresql-server-dev-9.6 +echo "apt-get postgresql postgresql-server-dev-9.6 postgresql-contrib exited with: $?" export PATH=/usr/lib/postgresql/9.6/bin:$PATH +#install plv8 +wget https://github.com/plv8/plv8/archive/v2.0.0.tar.gz +tar -xvzf v2.0.0.tar.gz +cd plv8-2.0.0 +make static +sudo cp plv8.so /usr/lib/postgresql/9.6/lib/ +sudo cp plv8.control /usr/share/postgresql/9.6/extension/ +sudo cp plv8--2.0.0.sql /usr/share/postgresql/9.6/extension/ + # download patroni sudo apt-get --assume-yes -qq install unzip echo "apt-get unzip exited with: $0" From fe80e0c6ff3add3d00bdc0c4bac57a6c1d80f436 Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Thu, 2 Feb 2017 19:58:56 -0800 Subject: [PATCH 31/51] added plv8 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index edc36bd..544195b 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ http://armviz.io/#/?load=https%3A%2F%2Fraw.githubusercontent.com%2Fhellotech%2Fa This one click deployment of a Highly-Available PostgreSQL Cluster on Azure has automated replication, server management and leader election. This project contains a modified deployment version of Haufe's [blog post](http://dev.haufe.com/PostgreSQL-Cluster-Azure/). -This Azure template generates two sets of machines both running Ubuntu 16.04 LTS. The first set of three machines are used to run the distributed configuration store [ZooKeeper](https://zookeeper.apache.org/), while the second set is running PostgreSQL 9.6 together with [Patroni](https://github.com/zalando/patroni) providing a high-availability customizable PostgreSQL cluster deployment. +This Azure template generates two sets of machines both running Ubuntu 16.04 LTS. The first set of three machines are used to run the distributed configuration store [ZooKeeper](https://zookeeper.apache.org/), while the second set is running PostgreSQL 9.6 together with [Patroni](https://github.com/zalando/patroni) providing a high-availability customizable PostgreSQL cluster deployment. This deployment also includes [PL/V8 v.2.0.0](https://github.com/plv8/plv8). # Parameters From 08b4441f7ae7d621529724cb110a094b2304d442 Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Thu, 2 Feb 2017 20:03:48 -0800 Subject: [PATCH 32/51] rename --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 544195b..b97c479 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ http://armviz.io/#/?load=https%3A%2F%2Fraw.githubusercontent.com%2Fhellotech%2Fazure_patroni%2Fmaster%2Ftemplate.json" target="_blank"> -# Azure Patroni Deployment +# Highly-Available PostgreSQL Cluster (Patroni) on Azure This one click deployment of a Highly-Available PostgreSQL Cluster on Azure has automated replication, server management and leader election. This project contains a modified deployment version of Haufe's [blog post](http://dev.haufe.com/PostgreSQL-Cluster-Azure/). From a6892e7a1cab6b69a6640fb2c03cc974b99ff55b Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Thu, 2 Feb 2017 20:08:13 -0800 Subject: [PATCH 33/51] renaming --- README.md | 12 ++++++------ backup.json | 6 +++--- template.json | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index b97c479..c38caba 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ - + +http://armviz.io/#/?load=https%3A%2F%2Fraw.githubusercontent.com%2Fhellotech%2Fazure_postgres%2Fmaster%2Ftemplate.json" target="_blank"> # Highly-Available PostgreSQL Cluster (Patroni) on Azure @@ -18,7 +18,7 @@ This template provides the following parameters, so you can customize your deplo * clusterName: What your cluster will be named. -* _artifactsLocation: The repo name from which you are deploying. Default("https://raw.githubusercontent.com/HelloTech/azure_patroni/master") +* _artifactsLocation: The repo name from which you are deploying. Default("https://raw.githubusercontent.com/HelloTech/azure_postgres/master") * newVnet: Whether or not you want to create a new vnet. Default("yes") @@ -48,13 +48,13 @@ This template provides the following parameters, so you can customize your deplo # Scripts ### Zookeeper -The creation of the zookeeper instances is handled by the [zookeeper_startup.sh](https://github.com/HelloTech/azure_patroni/blob/custom_deploy/zookeeper_startup.sh) script. If you would like to make any changes to what is installed on those vm's modify this file. +The creation of the zookeeper instances is handled by the [zookeeper_startup.sh](https://github.com/HelloTech/azure_postgres/blob/custom_deploy/zookeeper_startup.sh) script. If you would like to make any changes to what is installed on those vm's modify this file. ### Mounts -The mounting of the data ssds is handled by the [autopart.sh](https://github.com/HelloTech/azure_patroni/blob/custom_deploy/autopart.sh) script. +The mounting of the data ssds is handled by the [autopart.sh](https://github.com/HelloTech/azure_postgres/blob/custom_deploy/autopart.sh) script. ### Postgres -The creation of the postgres instance is handles by the [postgres_startup.sh](https://github.com/HelloTech/azure_patroni/blob/custom_deploy/postgres_startup.sh) script. If you would like to make any changes to what is installed on those vm's modify that script. +The creation of the postgres instance is handles by the [postgres_startup.sh](https://github.com/HelloTech/azure_postgres/blob/custom_deploy/postgres_startup.sh) script. If you would like to make any changes to what is installed on those vm's modify that script. # License diff --git a/backup.json b/backup.json index ba2c0a3..ab89f52 100644 --- a/backup.json +++ b/backup.json @@ -224,7 +224,7 @@ "typeHandlerVersion": "1.2", "settings": { "fileUris": [ - "https://raw.githubusercontent.com/HelloTech/azure_patroni/master/zookeeper_startup.sh" + "https://raw.githubusercontent.com/HelloTech/azure_postgres/master/zookeeper_startup.sh" ], "commandToExecute": "[concat('sh zookeeper_startup.sh ', copyIndex(), ' ', variables('zookeeperInstanceCount'))]" } @@ -353,8 +353,8 @@ "typeHandlerVersion": "1.2", "settings": { "fileUris": [ - "https://raw.githubusercontent.com/HelloTech/azure_patroni/master/autopart.sh", - "https://raw.githubusercontent.com/HelloTech/azure_patroni/master/postgres_startup.sh" + "https://raw.githubusercontent.com/HelloTech/azure_postgres/master/autopart.sh", + "https://raw.githubusercontent.com/HelloTech/azure_postgres/master/postgres_startup.sh" ], "commandToExecute": "[concat('sh postgres_startup.sh ', parameters('clusterName'), ' ', variables('zookeeperInstanceCount'), ' ', variables('postgresInstanceCount'), ' ', copyIndex(), ' ', parameters('adminUsername'), ' \"', parameters('adminPassword'), '\" ')]" } diff --git a/template.json b/template.json index beefc25..76adb15 100644 --- a/template.json +++ b/template.json @@ -7,7 +7,7 @@ }, "_artifactsLocation": { "type": "string", - "defaultValue": "https://raw.githubusercontent.com/HelloTech/azure_patroni/master", + "defaultValue": "https://raw.githubusercontent.com/HelloTech/azure_postgres/master", "metadata": { "description": "Change this value to your repo name if deploying from a fork" } From eb52bb926335ed061b51f34b01889c9bd3127027 Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Thu, 2 Feb 2017 23:39:23 -0800 Subject: [PATCH 34/51] update script to us systemd instead of rc.local --- postgres_startup.sh | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/postgres_startup.sh b/postgres_startup.sh index b7beb45..de8f087 100644 --- a/postgres_startup.sh +++ b/postgres_startup.sh @@ -170,6 +170,7 @@ fi echo " listen: '*:5433'" >> $patroniCfg echo " connect_address: 10.0.101.$(($myIndex + 10)):5433" >> $patroniCfg echo " data_dir: /media/data1/data/postgresql" >> $patroniCfg +echo " bin_dir: /usr/lib/postgresql/9.6/bin" >> $patroniCfg echo " pgpass: /tmp/pgpass" >> $patroniCfg if [ $myIndex -ne 0 ] then @@ -256,10 +257,20 @@ done sudo haproxy -D -f $hacfgFile # prepare patroni to restart after reboot -sudo chmod 777 /etc/rc.local -echo "sudo haproxy -D -f /usr/local/patroni-master/$hacfgFile" > /etc/rc.local -echo "su -s /bin/bash -c 'export PATH=/usr/lib/postgresql/9.6/bin:\$PATH; /usr/local/patroni-master/patroni.py /usr/local/patroni-master/$patroniCfg >> /usr/local/startup.log' $adminUsername" >> /etc/rc.local -echo "exit 0" >> /etc/rc.local +sudo touch /etc/systemd/system/patroni.service +sudo chmod 777 /etc/systemd/system/patroni.service +echo "[Unit]" > /etc/systemd/system/patroni.service +echo "Description=patroni script" >> /etc/systemd/system/patroni.service +echo "" >> /etc/systemd/system/patroni.service +echo "[Service]" >> /etc/systemd/system/patroni.service +echo "User=$adminUsername" >> /etc/systemd/system/patroni.service +echo "WorkingDirectory=usr/local/patroni-master" >> /etc/systemd/system/patroni.service +echo "ExecStart=/usr/bin/python /usr/local/patroni-master/patroni.py /usr/local/patroni-master/postgres.yml >> /usr/local/startup.log" >> /etc/systemd/system/patroni.service +echo "Restart=always" >> /etc/systemd/system/patroni.service +echo "" >> /etc/systemd/system/patroni.service +echo "[Install]" >> /etc/systemd/system/patroni.service +echo "WantedBy=multi-user.target" >> /etc/systemd/system/patroni.service +sudo systemctl enable patroni.service # start patroni -su -c "export PATH=/usr/lib/postgresql/9.6/bin:$PATH; ./patroni.py $patroniCfg >> /usr/local/startup.log" $adminUsername & echo "spawned" +sudo systemctl start patroni.service From b2fe4f05a78b5332833eaebc3516121843db17b2 Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Fri, 3 Feb 2017 02:26:48 -0800 Subject: [PATCH 35/51] added note regarding plv8 build time --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index c38caba..a13826c 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,9 @@ The mounting of the data ssds is handled by the [autopart.sh](https://github.com ### Postgres The creation of the postgres instance is handles by the [postgres_startup.sh](https://github.com/HelloTech/azure_postgres/blob/custom_deploy/postgres_startup.sh) script. If you would like to make any changes to what is installed on those vm's modify that script. +### Notes +This deployment can take anywhere from 90 to 120 minutes to complete due to long build time for plv8. + # License Released under the MIT license. See the LICENSE file for more info. \ No newline at end of file From 4a45efcd9bfee555312c5b92659da9c2cdc35d6b Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Fri, 3 Feb 2017 11:08:57 -0800 Subject: [PATCH 36/51] add package updates to build process --- postgres_startup.sh | 1 + zookeeper_startup.sh | 2 ++ 2 files changed, 3 insertions(+) diff --git a/postgres_startup.sh b/postgres_startup.sh index de8f087..112076d 100644 --- a/postgres_startup.sh +++ b/postgres_startup.sh @@ -38,6 +38,7 @@ sudo chmod 777 /media/data1/data sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" >> /etc/apt/sources.list.d/pgdg.list' wget -q https://www.postgresql.org/media/keys/ACCC4CF8.asc -O - | sudo apt-key add - sudo apt-get -qq update +sudo apt-get -qq -y upgrade echo "apt-get update exited with: $?" diff --git a/zookeeper_startup.sh b/zookeeper_startup.sh index 7ee47dd..6a2284b 100644 --- a/zookeeper_startup.sh +++ b/zookeeper_startup.sh @@ -1,4 +1,6 @@ #!/bin/bash +sudo apt update +sudo apt upgrade -y wget --no-check-certificate --no-cookies --header "Cookie: oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/7u75-b13/jdk-7u75-linux-x64.tar.gz" tar -xvf jdk-7* mkdir /usr/lib/jvm From 546d29524ba43b0fb477043aa9fb67b17b164df0 Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Fri, 3 Feb 2017 11:13:37 -0800 Subject: [PATCH 37/51] added updates to postrgres log --- postgres_startup.sh | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/postgres_startup.sh b/postgres_startup.sh index 112076d..6a53580 100644 --- a/postgres_startup.sh +++ b/postgres_startup.sh @@ -25,6 +25,8 @@ sudo chmod +x ./autopart.sh >> /usr/local/startup.log sudo ./autopart.sh >> /usr/local/startup.log sudo mkdir /media/data1/data sudo chmod 777 /media/data1/data +echo "/media/data1/data partition created" >> /usr/local/startup.log +echo "" >> /usr/local/startup.log # create RAID #sudo apt-get --assume-yes -qq install mdadm @@ -40,6 +42,8 @@ wget -q https://www.postgresql.org/media/keys/ACCC4CF8.asc -O - | sudo apt-key a sudo apt-get -qq update sudo apt-get -qq -y upgrade echo "apt-get update exited with: $?" +echo "updated packages" >> /usr/local/startup.log +echo "" >> /usr/local/startup.log # install python @@ -73,7 +77,8 @@ sudo pip -q install tzlocal echo "pip tzlocal exited with: $?" sudo pip -q install python-dateutil echo "pip python-dateutil exited with: $?" - +echo "installed python" >> /usr/local/startup.log +echo "" >> /usr/local/startup.log # install postgres # prevent postgres from autostart @@ -83,6 +88,8 @@ echo "pip python-dateutil exited with: $?" sudo apt-get --assume-yes --force-yes -qq install postgresql postgresql-contrib postgresql-server-dev-9.6 echo "apt-get postgresql postgresql-server-dev-9.6 postgresql-contrib exited with: $?" export PATH=/usr/lib/postgresql/9.6/bin:$PATH +echo "installed postgres" >> /usr/local/startup.log +echo "" >> /usr/local/startup.log #install plv8 wget https://github.com/plv8/plv8/archive/v2.0.0.tar.gz @@ -92,6 +99,8 @@ make static sudo cp plv8.so /usr/lib/postgresql/9.6/lib/ sudo cp plv8.control /usr/share/postgresql/9.6/extension/ sudo cp plv8--2.0.0.sql /usr/share/postgresql/9.6/extension/ +echo "installed plv8" >> /usr/local/startup.log +echo "" >> /usr/local/startup.log # download patroni sudo apt-get --assume-yes -qq install unzip @@ -100,6 +109,8 @@ cd /usr/local sudo wget -O /usr/local/patroni-master.zip https://github.com/zalando/patroni/archive/master.zip sudo unzip patroni-master.zip cd patroni-master +echo "download patroni" >> /usr/local/startup.log +echo "" >> /usr/local/startup.log # write configuration sudo touch $patroniCfg @@ -221,6 +232,8 @@ if [ $myIndex -ne 0 ] echo " parameters:" >> $patroniCfg echo " unix_socket_directories: '.'" >> $patroniCfg fi +echo "setup patroni configuration" >> /usr/local/startup.log +echo "" >> /usr/local/startup.log # install HA PROXY @@ -253,9 +266,13 @@ do echo " server Postgres$i 10.0.101.$(($i + 10)):5433 maxconn 100 check port 8008" >> $hacfgFile i=$(($i+1)) done +echo "installed haproxy" >> /usr/local/startup.log +echo "" >> /usr/local/startup.log # start ha as deamon sudo haproxy -D -f $hacfgFile +echo "started haproxy" >> /usr/local/startup.log +echo "" >> /usr/local/startup.log # prepare patroni to restart after reboot sudo touch /etc/systemd/system/patroni.service @@ -272,6 +289,10 @@ echo "" >> /etc/systemd/system/patroni.service echo "[Install]" >> /etc/systemd/system/patroni.service echo "WantedBy=multi-user.target" >> /etc/systemd/system/patroni.service sudo systemctl enable patroni.service +echo "setup reboot script for patroni" >> /usr/local/startup.log +echo "" >> /usr/local/startup.log # start patroni sudo systemctl start patroni.service +echo "started patroni" >> /usr/local/startup.log +echo "" >> /usr/local/startup.log \ No newline at end of file From be6802c8a2528e433c981cba952b156a468448b0 Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Fri, 3 Feb 2017 11:25:53 -0800 Subject: [PATCH 38/51] added jq installation to both init scripts --- postgres_startup.sh | 1 + zookeeper_startup.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/postgres_startup.sh b/postgres_startup.sh index 6a53580..84e1ecd 100644 --- a/postgres_startup.sh +++ b/postgres_startup.sh @@ -41,6 +41,7 @@ sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs` wget -q https://www.postgresql.org/media/keys/ACCC4CF8.asc -O - | sudo apt-key add - sudo apt-get -qq update sudo apt-get -qq -y upgrade +sudo apt-get install -y jq echo "apt-get update exited with: $?" echo "updated packages" >> /usr/local/startup.log echo "" >> /usr/local/startup.log diff --git a/zookeeper_startup.sh b/zookeeper_startup.sh index 6a2284b..9089b7e 100644 --- a/zookeeper_startup.sh +++ b/zookeeper_startup.sh @@ -1,6 +1,7 @@ #!/bin/bash sudo apt update sudo apt upgrade -y +sudo apt install -y jq wget --no-check-certificate --no-cookies --header "Cookie: oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/7u75-b13/jdk-7u75-linux-x64.tar.gz" tar -xvf jdk-7* mkdir /usr/lib/jvm From 37d630574972afae792a64e997fb9e519771f194 Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Fri, 3 Feb 2017 14:59:36 -0800 Subject: [PATCH 39/51] add restart true so apps are updated once service starts --- template.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/template.json b/template.json index 76adb15..a164762 100644 --- a/template.json +++ b/template.json @@ -326,7 +326,8 @@ "fileUris": [ "[concat(parameters('_artifactsLocation'), '/zookeeper_startup.sh')]" ], - "commandToExecute": "[concat('sh zookeeper_startup.sh ', copyIndex(), ' ', variables('zookeeperInstanceCount'))]" + "commandToExecute": "[concat('sh zookeeper_startup.sh ', copyIndex(), ' ', variables('zookeeperInstanceCount'))]", + "Restart": "true" } }, "dependsOn": [ @@ -456,7 +457,8 @@ "[concat(parameters('_artifactsLocation'), '/autopart.sh')]", "[concat(parameters('_artifactsLocation'), '/postgres_startup.sh')]" ], - "commandToExecute": "[concat('sh postgres_startup.sh ', parameters('clusterName'), ' ', variables('zookeeperInstanceCount'), ' ', variables('postgresInstanceCount'), ' ', copyIndex(), ' ', parameters('adminUsername'), ' \"', parameters('adminPassword'), '\" ')]" + "commandToExecute": "[concat('sh postgres_startup.sh ', parameters('clusterName'), ' ', variables('zookeeperInstanceCount'), ' ', variables('postgresInstanceCount'), ' ', copyIndex(), ' ', parameters('adminUsername'), ' \"', parameters('adminPassword'), '\" ')]", + "Restart": "true" } }, "dependsOn": [ From 1c10d3facc40ede996f1cad3a8fe09d68f5a1bb3 Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Fri, 3 Feb 2017 15:27:32 -0800 Subject: [PATCH 40/51] replace apt with apt-get --- zookeeper_startup.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zookeeper_startup.sh b/zookeeper_startup.sh index 9089b7e..884fa1f 100644 --- a/zookeeper_startup.sh +++ b/zookeeper_startup.sh @@ -1,7 +1,7 @@ #!/bin/bash -sudo apt update -sudo apt upgrade -y -sudo apt install -y jq +sudo apt-get update +sudo apt-get upgrade -y +sudo apt-get install -y jq wget --no-check-certificate --no-cookies --header "Cookie: oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/7u75-b13/jdk-7u75-linux-x64.tar.gz" tar -xvf jdk-7* mkdir /usr/lib/jvm From ae165873e7c0919a72c392e077ad4589a23a953d Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Fri, 3 Feb 2017 15:51:16 -0800 Subject: [PATCH 41/51] another apt-get update --- zookeeper_startup.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zookeeper_startup.sh b/zookeeper_startup.sh index 884fa1f..eac8f7b 100644 --- a/zookeeper_startup.sh +++ b/zookeeper_startup.sh @@ -1,7 +1,7 @@ #!/bin/bash -sudo apt-get update -sudo apt-get upgrade -y -sudo apt-get install -y jq +sudo apt-get --assume-yes -qq update +sudo apt-get --assume-yes -qq upgrade +sudo apt-get --assume-yes -qq install jq wget --no-check-certificate --no-cookies --header "Cookie: oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/7u75-b13/jdk-7u75-linux-x64.tar.gz" tar -xvf jdk-7* mkdir /usr/lib/jvm From 5d1e920cb43753611fdf90956d88f852552b7eca Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Fri, 3 Feb 2017 15:52:49 -0800 Subject: [PATCH 42/51] add assume-yes --- postgres_startup.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/postgres_startup.sh b/postgres_startup.sh index 84e1ecd..10cd891 100644 --- a/postgres_startup.sh +++ b/postgres_startup.sh @@ -39,9 +39,9 @@ echo "" >> /usr/local/startup.log # update package lists for PostgreSQL 9.6 sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" >> /etc/apt/sources.list.d/pgdg.list' wget -q https://www.postgresql.org/media/keys/ACCC4CF8.asc -O - | sudo apt-key add - -sudo apt-get -qq update -sudo apt-get -qq -y upgrade -sudo apt-get install -y jq +sudo apt-get --assume-yes -qq update +sudo apt-get --assume-yes -qq upgrade +sudo apt-get --assume-yes -qq install jq echo "apt-get update exited with: $?" echo "updated packages" >> /usr/local/startup.log echo "" >> /usr/local/startup.log From ddb0c6c819b761561d7c579396251d13bcebd5f6 Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Fri, 3 Feb 2017 16:18:58 -0800 Subject: [PATCH 43/51] test --- postgres_startup.sh | 1 + zookeeper_startup.sh | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/postgres_startup.sh b/postgres_startup.sh index 10cd891..0e5216c 100644 --- a/postgres_startup.sh +++ b/postgres_startup.sh @@ -9,6 +9,7 @@ adminPassword=$6 hacfgFile=postgresha.cfg patroniCfg=postgres.yml +export DEBIAN_FRONTEND=noninteractive sudo touch /usr/local/startup.log sudo chmod 666 /usr/local/startup.log echo "Cluster name: $clusterName" >> /usr/local/startup.log diff --git a/zookeeper_startup.sh b/zookeeper_startup.sh index eac8f7b..0cf7cb2 100644 --- a/zookeeper_startup.sh +++ b/zookeeper_startup.sh @@ -1,7 +1,13 @@ #!/bin/bash +export DEBIAN_FRONTEND=noninteractive +sudo touch /usr/local/startup.log +sudo chmod 666 /usr/local/startup.log sudo apt-get --assume-yes -qq update +echo "ran update" >> /usr/local/startup.log sudo apt-get --assume-yes -qq upgrade +echo "ran upgrade" >> /usr/local/startup.log sudo apt-get --assume-yes -qq install jq +echo "ran install" >> /usr/local/startup.log wget --no-check-certificate --no-cookies --header "Cookie: oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/7u75-b13/jdk-7u75-linux-x64.tar.gz" tar -xvf jdk-7* mkdir /usr/lib/jvm From 04f30686f0ade6aaed12149c3fcbc9a797af6a28 Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Fri, 3 Feb 2017 16:40:38 -0800 Subject: [PATCH 44/51] update vm on separate script --- postgres_startup.sh | 3 --- template.json | 54 ++++++++++++++++++++++++++++++++++++++++---- zookeeper_startup.sh | 7 ------ 3 files changed, 50 insertions(+), 14 deletions(-) diff --git a/postgres_startup.sh b/postgres_startup.sh index 0e5216c..151ed27 100644 --- a/postgres_startup.sh +++ b/postgres_startup.sh @@ -9,7 +9,6 @@ adminPassword=$6 hacfgFile=postgresha.cfg patroniCfg=postgres.yml -export DEBIAN_FRONTEND=noninteractive sudo touch /usr/local/startup.log sudo chmod 666 /usr/local/startup.log echo "Cluster name: $clusterName" >> /usr/local/startup.log @@ -40,8 +39,6 @@ echo "" >> /usr/local/startup.log # update package lists for PostgreSQL 9.6 sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" >> /etc/apt/sources.list.d/pgdg.list' wget -q https://www.postgresql.org/media/keys/ACCC4CF8.asc -O - | sudo apt-key add - -sudo apt-get --assume-yes -qq update -sudo apt-get --assume-yes -qq upgrade sudo apt-get --assume-yes -qq install jq echo "apt-get update exited with: $?" echo "updated packages" >> /usr/local/startup.log diff --git a/template.json b/template.json index a164762..0e9dc67 100644 --- a/template.json +++ b/template.json @@ -326,8 +326,31 @@ "fileUris": [ "[concat(parameters('_artifactsLocation'), '/zookeeper_startup.sh')]" ], - "commandToExecute": "[concat('sh zookeeper_startup.sh ', copyIndex(), ' ', variables('zookeeperInstanceCount'))]", - "Restart": "true" + "commandToExecute": "[concat('sh zookeeper_startup.sh ', copyIndex(), ' ', variables('zookeeperInstanceCount'))]" + } + }, + "dependsOn": [ + "[concat('Microsoft.Compute/virtualMachines/', 'zookeeper', copyIndex())]" + ] + }, + { + "type": "Microsoft.Compute/virtualMachines/extensions", + "name": "[concat('zookeeper', copyIndex(), '/zookeeperExtension')]", + "apiVersion": "2015-05-01-preview", + "location": "[resourceGroup().location]", + "copy": { + "name": "zookeeperExtLoop", + "count": "[variables('zookeeperInstanceCount')]" + }, + "properties": { + "publisher": "Microsoft.OSTCExtensions", + "type": "OSPatchingForLinux", + "typeHandlerVersion": "2.9", + "Settings": { + "disabled": false, + "stop": false, + "category": "ImportantAndRecommended", + "oneoff": true } }, "dependsOn": [ @@ -457,8 +480,31 @@ "[concat(parameters('_artifactsLocation'), '/autopart.sh')]", "[concat(parameters('_artifactsLocation'), '/postgres_startup.sh')]" ], - "commandToExecute": "[concat('sh postgres_startup.sh ', parameters('clusterName'), ' ', variables('zookeeperInstanceCount'), ' ', variables('postgresInstanceCount'), ' ', copyIndex(), ' ', parameters('adminUsername'), ' \"', parameters('adminPassword'), '\" ')]", - "Restart": "true" + "commandToExecute": "[concat('sh postgres_startup.sh ', parameters('clusterName'), ' ', variables('zookeeperInstanceCount'), ' ', variables('postgresInstanceCount'), ' ', copyIndex(), ' ', parameters('adminUsername'), ' \"', parameters('adminPassword'), '\" ')]" + } + }, + "dependsOn": [ + "[concat('Microsoft.Compute/virtualMachines/', 'postgres', copyIndex())]" + ] + }, + { + "type": "Microsoft.Compute/virtualMachines/extensions", + "name": "[concat('postgres', copyIndex(), '/postgresExtension')]", + "apiVersion": "2015-05-01-preview", + "location": "[resourceGroup().location]", + "copy": { + "name": "postgresExtLoop", + "count": "[variables('postgresInstanceCount')]" + }, + "properties": { + "publisher": "Microsoft.OSTCExtensions", + "type": "OSPatchingForLinux", + "typeHandlerVersion": "2.9", + "Settings": { + "disabled": false, + "stop": false, + "category": "ImportantAndRecommended", + "oneoff": true } }, "dependsOn": [ diff --git a/zookeeper_startup.sh b/zookeeper_startup.sh index 0cf7cb2..e77362d 100644 --- a/zookeeper_startup.sh +++ b/zookeeper_startup.sh @@ -1,11 +1,4 @@ #!/bin/bash -export DEBIAN_FRONTEND=noninteractive -sudo touch /usr/local/startup.log -sudo chmod 666 /usr/local/startup.log -sudo apt-get --assume-yes -qq update -echo "ran update" >> /usr/local/startup.log -sudo apt-get --assume-yes -qq upgrade -echo "ran upgrade" >> /usr/local/startup.log sudo apt-get --assume-yes -qq install jq echo "ran install" >> /usr/local/startup.log wget --no-check-certificate --no-cookies --header "Cookie: oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/7u75-b13/jdk-7u75-linux-x64.tar.gz" From 73d56957dab7d681aee3a042f18f562c26e0a810 Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Fri, 3 Feb 2017 16:59:26 -0800 Subject: [PATCH 45/51] remove update extension --- template.json | 48 ------------------------------------------------ 1 file changed, 48 deletions(-) diff --git a/template.json b/template.json index 0e9dc67..76adb15 100644 --- a/template.json +++ b/template.json @@ -333,30 +333,6 @@ "[concat('Microsoft.Compute/virtualMachines/', 'zookeeper', copyIndex())]" ] }, - { - "type": "Microsoft.Compute/virtualMachines/extensions", - "name": "[concat('zookeeper', copyIndex(), '/zookeeperExtension')]", - "apiVersion": "2015-05-01-preview", - "location": "[resourceGroup().location]", - "copy": { - "name": "zookeeperExtLoop", - "count": "[variables('zookeeperInstanceCount')]" - }, - "properties": { - "publisher": "Microsoft.OSTCExtensions", - "type": "OSPatchingForLinux", - "typeHandlerVersion": "2.9", - "Settings": { - "disabled": false, - "stop": false, - "category": "ImportantAndRecommended", - "oneoff": true - } - }, - "dependsOn": [ - "[concat('Microsoft.Compute/virtualMachines/', 'zookeeper', copyIndex())]" - ] - }, { "type": "Microsoft.Network/networkInterfaces", "name": "[concat('postgres', copyIndex(), '-nic')]", @@ -487,30 +463,6 @@ "[concat('Microsoft.Compute/virtualMachines/', 'postgres', copyIndex())]" ] }, - { - "type": "Microsoft.Compute/virtualMachines/extensions", - "name": "[concat('postgres', copyIndex(), '/postgresExtension')]", - "apiVersion": "2015-05-01-preview", - "location": "[resourceGroup().location]", - "copy": { - "name": "postgresExtLoop", - "count": "[variables('postgresInstanceCount')]" - }, - "properties": { - "publisher": "Microsoft.OSTCExtensions", - "type": "OSPatchingForLinux", - "typeHandlerVersion": "2.9", - "Settings": { - "disabled": false, - "stop": false, - "category": "ImportantAndRecommended", - "oneoff": true - } - }, - "dependsOn": [ - "[concat('Microsoft.Compute/virtualMachines/', 'postgres', copyIndex())]" - ] - }, { "type": "Microsoft.Resources/deployments", "name": "lb-temp", From 2f5eacf41a887100444c13bd8fd6c9c3628af6dd Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Fri, 3 Feb 2017 17:06:06 -0800 Subject: [PATCH 46/51] test with update only no restart --- zookeeper_startup.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/zookeeper_startup.sh b/zookeeper_startup.sh index e77362d..6897071 100644 --- a/zookeeper_startup.sh +++ b/zookeeper_startup.sh @@ -1,4 +1,5 @@ #!/bin/bash +sudo apt-get --assume-yes -qq update sudo apt-get --assume-yes -qq install jq echo "ran install" >> /usr/local/startup.log wget --no-check-certificate --no-cookies --header "Cookie: oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/7u75-b13/jdk-7u75-linux-x64.tar.gz" From 64541dfa0e9359b0b894c7dbd01dde615db411ba Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Fri, 3 Feb 2017 17:14:44 -0800 Subject: [PATCH 47/51] test with export DEBIAN_FRONTEND=noninteractive --- template.json | 2 +- zookeeper_startup.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/template.json b/template.json index 76adb15..bec2240 100644 --- a/template.json +++ b/template.json @@ -326,7 +326,7 @@ "fileUris": [ "[concat(parameters('_artifactsLocation'), '/zookeeper_startup.sh')]" ], - "commandToExecute": "[concat('sh zookeeper_startup.sh ', copyIndex(), ' ', variables('zookeeperInstanceCount'))]" + "commandToExecute": "[concat('export DEBIAN_FRONTEND=noninteractive sh zookeeper_startup.sh ', copyIndex(), ' ', variables('zookeeperInstanceCount'))]" } }, "dependsOn": [ diff --git a/zookeeper_startup.sh b/zookeeper_startup.sh index 6897071..5d637b9 100644 --- a/zookeeper_startup.sh +++ b/zookeeper_startup.sh @@ -1,7 +1,6 @@ #!/bin/bash sudo apt-get --assume-yes -qq update sudo apt-get --assume-yes -qq install jq -echo "ran install" >> /usr/local/startup.log wget --no-check-certificate --no-cookies --header "Cookie: oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/7u75-b13/jdk-7u75-linux-x64.tar.gz" tar -xvf jdk-7* mkdir /usr/lib/jvm From b9864c5596a6c22c20a9bde93174913d5c6253d1 Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Fri, 3 Feb 2017 22:58:55 -0800 Subject: [PATCH 48/51] test with export DEBIAN_FRONTEND=noninteractive --- postgres_startup.sh | 3 ++- zookeeper_startup.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/postgres_startup.sh b/postgres_startup.sh index 151ed27..4369f15 100644 --- a/postgres_startup.sh +++ b/postgres_startup.sh @@ -39,7 +39,8 @@ echo "" >> /usr/local/startup.log # update package lists for PostgreSQL 9.6 sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" >> /etc/apt/sources.list.d/pgdg.list' wget -q https://www.postgresql.org/media/keys/ACCC4CF8.asc -O - | sudo apt-key add - -sudo apt-get --assume-yes -qq install jq +sudo apt-get -qq update +sudo apt-get --assume-yes -qq install jq echo "apt-get update exited with: $?" echo "updated packages" >> /usr/local/startup.log echo "" >> /usr/local/startup.log diff --git a/zookeeper_startup.sh b/zookeeper_startup.sh index 5d637b9..5faeac7 100644 --- a/zookeeper_startup.sh +++ b/zookeeper_startup.sh @@ -1,5 +1,5 @@ #!/bin/bash -sudo apt-get --assume-yes -qq update +#sudo apt-get --assume-yes -qq update sudo apt-get --assume-yes -qq install jq wget --no-check-certificate --no-cookies --header "Cookie: oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/7u75-b13/jdk-7u75-linux-x64.tar.gz" tar -xvf jdk-7* From a0719cbe74f897bc4c52252bd1c55a63b0e1b178 Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Fri, 3 Feb 2017 23:49:57 -0800 Subject: [PATCH 49/51] remove more code --- zookeeper_startup.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/zookeeper_startup.sh b/zookeeper_startup.sh index 5faeac7..7ee47dd 100644 --- a/zookeeper_startup.sh +++ b/zookeeper_startup.sh @@ -1,6 +1,4 @@ #!/bin/bash -#sudo apt-get --assume-yes -qq update -sudo apt-get --assume-yes -qq install jq wget --no-check-certificate --no-cookies --header "Cookie: oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/7u75-b13/jdk-7u75-linux-x64.tar.gz" tar -xvf jdk-7* mkdir /usr/lib/jvm From bf689c92134338412f5faf292cbf992524476dba Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Sat, 4 Feb 2017 00:14:20 -0800 Subject: [PATCH 50/51] fix command bug --- template.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template.json b/template.json index bec2240..76adb15 100644 --- a/template.json +++ b/template.json @@ -326,7 +326,7 @@ "fileUris": [ "[concat(parameters('_artifactsLocation'), '/zookeeper_startup.sh')]" ], - "commandToExecute": "[concat('export DEBIAN_FRONTEND=noninteractive sh zookeeper_startup.sh ', copyIndex(), ' ', variables('zookeeperInstanceCount'))]" + "commandToExecute": "[concat('sh zookeeper_startup.sh ', copyIndex(), ' ', variables('zookeeperInstanceCount'))]" } }, "dependsOn": [ From da71606583616de24c3b27548e9f8a4910b31238 Mon Sep 17 00:00:00 2001 From: Paulo Diniz Date: Tue, 7 Feb 2017 16:58:20 -0800 Subject: [PATCH 51/51] added configs to start haproxy on startup --- postgres_startup.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/postgres_startup.sh b/postgres_startup.sh index 4369f15..71c9d83 100644 --- a/postgres_startup.sh +++ b/postgres_startup.sh @@ -292,6 +292,19 @@ sudo systemctl enable patroni.service echo "setup reboot script for patroni" >> /usr/local/startup.log echo "" >> /usr/local/startup.log +# prepare haproxy to restart after reboot +sudo chmod 777 /etc/default/haproxy +echo "ENABLED=1" >> /etc/default/haproxy +echo "CONFIG=\"/usr/local/patroni-master/postgresha.cfg\"" >> /etc/default/haproxy +sudo chmod 644 /etc/default/haproxy +echo "setup reboot config for haproxy" >> /usr/local/startup.log +echo "" >> /usr/local/startup.log + +# start haproxy +sudo haproxy -D -f /usr/local/patroni-master/postgresha.cfg +echo "started haproxy" >> /usr/local/startup.log +echo "" >> /usr/local/startup.log + # start patroni sudo systemctl start patroni.service echo "started patroni" >> /usr/local/startup.log