Skip to content

Commit ee535e0

Browse files
lucperkinsmerlimat
lucperkins
authored and
merlimat
committed
Alter Terraform config to allow for variables (openmessaging#12)
* add .tfvars files and update READMEs * add variables for number of instances * fix terraform formatting
1 parent ee2ded1 commit ee535e0

File tree

6 files changed

+135
-57
lines changed

6 files changed

+135
-57
lines changed

‎driver-kafka/README.md

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ In addition, you will need to:
3434
* [Install the `aws` CLI tool](https://aws.amazon.com/cli/)
3535
* [Configure the `aws` CLI tool](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html)
3636

37-
Once those conditions are in place, you'll need to create an SSH public and private key at `~/.ssh/aws_pulsar` (private) and `~/.ssh/aws_pulsar.pub` (public), respectively.
37+
Once those conditions are in place, you'll need to create an SSH public and private key at `~/.ssh/kafka_aws` (private) and `~/.ssh/kafka_aws.pub` (public), respectively.
3838

3939
```bash
4040
$ ssh-keygen -f ~/.ssh/kafka_aws
@@ -64,6 +64,32 @@ When you run `terraform apply`, you will be prompted to type `yes`. Type `yes` t
6464

6565
Once the installation is complete, you will see a confirmation message listing the resources that have been installed.
6666

67+
### Variables
68+
69+
There's a handful of configurable parameters related to the Terraform deployment that you can alter by modifying the defaults in the `terraform.tfvars` file.
70+
71+
Variable | Description | Default
72+
:--------|:------------|:-------
73+
`region` | The AWS region in which the Kafka cluster will be deployed | `us-west-2`
74+
`public_key_path` | The path to the SSH public key that you've generated | `~/.ssh/kafka_aws.pub`
75+
`ami` | The [Amazon Machine Image](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html) (AWI) to be used by the cluster's machines | [`ami-9fa343e7`](https://access.redhat.com/articles/3135091)
76+
`instance_types` | The EC2 instance types used by the various components | `i3.4xlarge` (Kafka brokers), `t2.small` (ZooKeeper), `c4.8xlarge` (benchmarking client)
77+
78+
> If you modify the `public_key_path`, make sure that you point to the appropriate SSH key path when running the [Ansible playbook](#running-the-ansible-playbook).
79+
80+
### Running the Ansible playbook
81+
82+
With the appropriate infrastructure in place, you can install and start the Kafka cluster using Ansible with just one command:
83+
84+
```bash
85+
$ ansible-playbook \
86+
--user ec2-user \
87+
--inventory `which terraform-inventory` \
88+
deploy.yaml
89+
```
90+
91+
> If you're using an SSH private key path different from `~/.ssh/kafka_aws`, you can specify that path using the `--private-key` flag, for example `--private-key=~/.ssh/my_key`.
92+
6793
## SSHing into the client host
6894

6995
In the [output](https://www.terraform.io/intro/getting-started/outputs.html) produced by Terraform, there's a `client_ssh_host` variable that provides the IP address for the client EC2 host from which benchmarks can be run. You can SSH into that host using this command:

‎driver-kafka/deploy/provision-kafka-aws.tf

Lines changed: 25 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
variable "public_key_path" {
2-
default = "~/.ssh/kafka_aws.pub"
32
description = <<DESCRIPTION
43
Path to the SSH public key to be used for authentication.
54
Ensure this keypair is added to your local SSH agent so provisioners can
@@ -10,16 +9,16 @@ DESCRIPTION
109
}
1110

1211
variable "key_name" {
13-
default = "kafka-benchmark-key"
12+
default = "kafka-benchmark-key"
1413
description = "Desired name of AWS key pair"
1514
}
1615

17-
variable "region" {
18-
default = "us-west-2"
19-
}
16+
variable "region" {}
17+
18+
variable "ami" {}
2019

21-
variable "ami" {
22-
default = "ami-9fa343e7" // RHEL-7.4
20+
variable "instance_types" {
21+
type = "map"
2322
}
2423

2524
provider "aws" {
@@ -55,8 +54,8 @@ resource "aws_subnet" "benchmark_subnet" {
5554
}
5655

5756
resource "aws_security_group" "benchmark_security_group" {
58-
name = "terraform"
59-
vpc_id = "${aws_vpc.benchmark_vpc.id}"
57+
name = "terraform"
58+
vpc_id = "${aws_vpc.benchmark_vpc.id}"
6059

6160
# SSH access from anywhere
6261
ingress {
@@ -83,7 +82,7 @@ resource "aws_security_group" "benchmark_security_group" {
8382
}
8483

8584
tags {
86-
Name = "Benchmark-Security-Group"
85+
Name = "Benchmark-Security-Group"
8786
}
8887
}
8988

@@ -93,38 +92,38 @@ resource "aws_key_pair" "auth" {
9392
}
9493

9594
resource "aws_instance" "zookeeper" {
96-
ami = "${var.ami}"
97-
instance_type = "t2.small"
98-
key_name = "${aws_key_pair.auth.id}"
99-
subnet_id = "${aws_subnet.benchmark_subnet.id}"
95+
ami = "${var.ami}"
96+
instance_type = "${var.instance_types["zookeeper"]}"
97+
key_name = "${aws_key_pair.auth.id}"
98+
subnet_id = "${aws_subnet.benchmark_subnet.id}"
10099
vpc_security_group_ids = ["${aws_security_group.benchmark_security_group.id}"]
101-
count = 3
100+
count = "${var.num_instances["zookeeper"]}"
102101

103102
tags {
104103
Name = "kafka-zk-${count.index}"
105104
}
106105
}
107106

108107
resource "aws_instance" "kafka" {
109-
ami = "${var.ami}"
110-
instance_type = "i3.4xlarge"
111-
key_name = "${aws_key_pair.auth.id}"
112-
subnet_id = "${aws_subnet.benchmark_subnet.id}"
108+
ami = "${var.ami}"
109+
instance_type = "${var.instance_types["kafka"]}"
110+
key_name = "${aws_key_pair.auth.id}"
111+
subnet_id = "${aws_subnet.benchmark_subnet.id}"
113112
vpc_security_group_ids = ["${aws_security_group.benchmark_security_group.id}"]
114-
count = 3
113+
count = "${var.num_instances["kafka"]}"
115114

116115
tags {
117116
Name = "kafka-${count.index}"
118117
}
119118
}
120119

121120
resource "aws_instance" "client" {
122-
ami = "${var.ami}"
123-
instance_type = "c4.8xlarge"
124-
key_name = "${aws_key_pair.auth.id}"
125-
subnet_id = "${aws_subnet.benchmark_subnet.id}"
121+
ami = "${var.ami}"
122+
instance_type = "${var.instance_types["client"]}"
123+
key_name = "${aws_key_pair.auth.id}"
124+
subnet_id = "${aws_subnet.benchmark_subnet.id}"
126125
vpc_security_group_ids = ["${aws_security_group.benchmark_security_group.id}"]
127-
count = 1
126+
count = 1
128127

129128
tags {
130129
Name = "kafka-client-${count.index}"
@@ -133,4 +132,4 @@ resource "aws_instance" "client" {
133132

134133
output "client_ssh_host" {
135134
value = "${aws_instance.client.0.public_ip}"
136-
}
135+
}

‎driver-kafka/deploy/terraform.tfvars

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
public_key_path = "~/.ssh/kafka_aws.pub"
2+
region = "us-west-2"
3+
ami = "ami-9fa343e7" // RHEL-7.4
4+
5+
instance_types = {
6+
"kafka" = "i3.4xlarge"
7+
"zookeeper" = "t2.small"
8+
"client" = "c4.8xlarge"
9+
}
10+
11+
num_instances = {
12+
"kafka" = 3
13+
"zookeeper" = 3
14+
}

‎driver-pulsar/README.md

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,32 @@ When you run `terraform apply`, you will be prompted to type `yes`. Type `yes` t
6464

6565
Once the installation is complete, you will see a confirmation message listing the resources that have been installed.
6666

67+
### Variables
68+
69+
There's a handful of configurable parameters related to the Terraform deployment that you can alter by modifying the defaults in the `terraform.tfvars` file.
70+
71+
Variable | Description | Default
72+
:--------|:------------|:-------
73+
`region` | The AWS region in which the Pulsar cluster will be deployed | `us-west-2`
74+
`public_key_path` | The path to the SSH public key that you've generated | `~/.ssh/pulsar_aws.pub`
75+
`ami` | The [Amazon Machine Image](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html) (AWI) to be used by the cluster's machines | [`ami-9fa343e7`](https://access.redhat.com/articles/3135091)
76+
`instance_types` | The EC2 instance types used by the various components | `i3.4xlarge` (Pulsar brokers and BookKeeper bookies), `t2.small` (ZooKeeper), `c4.8xlarge` (benchmarking client)
77+
78+
> If you modify the `public_key_path`, make sure that you point to the appropriate SSH key path when running the [Ansible playbook](#running-the-ansible-playbook).
79+
80+
### Running the Ansible playbook
81+
82+
With the appropriate infrastructure in place, you can install and start the Pulsar cluster using Ansible with just one command:
83+
84+
```bash
85+
$ ansible-playbook \
86+
--user ec2-user \
87+
--inventory `which terraform-inventory` \
88+
deploy.yaml
89+
```
90+
91+
> If you're using an SSH private key path different from `~/.ssh/pulsar_aws`, you can specify that path using the `--private-key` flag, for example `--private-key=~/.ssh/my_key`.
92+
6793
## SSHing into the client host
6894

6995
In the [output](https://www.terraform.io/intro/getting-started/outputs.html) produced by Terraform, there's a `client_ssh_host` variable that provides the IP address for the client EC2 host from which benchmarks can be run. You can SSH into that host using this command:

‎driver-pulsar/deploy/provision-pulsar-aws.tf

Lines changed: 29 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,24 @@
11
variable "public_key_path" {
2-
default = "~/.ssh/pulsar_aws.pub"
32
description = <<DESCRIPTION
43
Path to the SSH public key to be used for authentication.
54
Ensure this keypair is added to your local SSH agent so provisioners can
65
connect.
76
8-
Example: ~/.ssh/terraform_pulsar.pub
7+
Example: ~/.ssh/pulsar_aws.pub
98
DESCRIPTION
109
}
1110

1211
variable "key_name" {
13-
default = "pulsar-benchmark-key"
12+
default = "pulsar-benchmark-key"
1413
description = "Desired name of AWS key pair"
1514
}
1615

17-
variable "region" {
18-
default = "us-west-2"
19-
}
16+
variable "region" {}
17+
18+
variable "ami" {}
2019

21-
variable "ami" {
22-
default = "ami-9fa343e7" // RHEL-7.4
20+
variable "instance_types" {
21+
type = "map"
2322
}
2423

2524
provider "aws" {
@@ -55,8 +54,8 @@ resource "aws_subnet" "benchmark_subnet" {
5554
}
5655

5756
resource "aws_security_group" "benchmark_security_group" {
58-
name = "terraform"
59-
vpc_id = "${aws_vpc.benchmark_vpc.id}"
57+
name = "terraform"
58+
vpc_id = "${aws_vpc.benchmark_vpc.id}"
6059

6160
# SSH access from anywhere
6261
ingress {
@@ -83,7 +82,7 @@ resource "aws_security_group" "benchmark_security_group" {
8382
}
8483

8584
tags {
86-
Name = "Benchmark-Security-Group"
85+
Name = "Benchmark-Security-Group"
8786
}
8887
}
8988

@@ -93,44 +92,44 @@ resource "aws_key_pair" "auth" {
9392
}
9493

9594
resource "aws_instance" "zookeeper" {
96-
ami = "${var.ami}"
97-
instance_type = "t2.small"
98-
key_name = "${aws_key_pair.auth.id}"
99-
subnet_id = "${aws_subnet.benchmark_subnet.id}"
95+
ami = "${var.ami}"
96+
instance_type = "${var.instance_types["zookeeper"]}"
97+
key_name = "${aws_key_pair.auth.id}"
98+
subnet_id = "${aws_subnet.benchmark_subnet.id}"
10099
vpc_security_group_ids = ["${aws_security_group.benchmark_security_group.id}"]
101-
count = 3
100+
count = "${var.num_instances["zookeeper"]}"
102101

103102
tags {
104-
Name = "zk-${count.index}"
103+
Name = "zk-${count.index}"
105104
}
106105
}
107106

108107
resource "aws_instance" "pulsar" {
109-
ami = "${var.ami}"
110-
instance_type = "i3.4xlarge"
111-
key_name = "${aws_key_pair.auth.id}"
112-
subnet_id = "${aws_subnet.benchmark_subnet.id}"
108+
ami = "${var.ami}"
109+
instance_type = "${var.instance_types["pulsar"]}"
110+
key_name = "${aws_key_pair.auth.id}"
111+
subnet_id = "${aws_subnet.benchmark_subnet.id}"
113112
vpc_security_group_ids = ["${aws_security_group.benchmark_security_group.id}"]
114-
count = 3
113+
count = "${var.num_instances["pulsar"]}"
115114

116115
tags {
117-
Name = "pulsar-${count.index}"
116+
Name = "pulsar-${count.index}"
118117
}
119118
}
120119

121120
resource "aws_instance" "client" {
122-
ami = "${var.ami}"
123-
instance_type = "c4.8xlarge"
124-
key_name = "${aws_key_pair.auth.id}"
125-
subnet_id = "${aws_subnet.benchmark_subnet.id}"
121+
ami = "${var.ami}"
122+
instance_type = "${var.instance_types["client"]}"
123+
key_name = "${aws_key_pair.auth.id}"
124+
subnet_id = "${aws_subnet.benchmark_subnet.id}"
126125
vpc_security_group_ids = ["${aws_security_group.benchmark_security_group.id}"]
127-
count = 1
126+
count = 1
128127

129128
tags {
130-
Name = "pulsar-client-${count.index}"
129+
Name = "pulsar-client-${count.index}"
131130
}
132131
}
133132

134133
output "client_ssh_host" {
135134
value = "${aws_instance.client.0.public_ip}"
136-
}
135+
}

‎driver-pulsar/deploy/terraform.tfvars

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
public_key_path = "~/.ssh/pulsar_aws.pub"
2+
region = "us-west-2"
3+
ami = "ami-9fa343e7" // RHEL-7.4
4+
5+
instance_types = {
6+
"pulsar" = "i3.4xlarge"
7+
"zookeeper" = "t2.small"
8+
"client" = "c4.8xlarge"
9+
}
10+
11+
num_instances = {
12+
"pulsar" = 3
13+
"zookeeper" = 3
14+
}

0 commit comments

Comments
 (0)