Skip to content

Commit 854e974

Browse files
johnstcnjohnstcn
authored
chore: add terraform for spinning up load test cluster (#7504)
Adds terraform configs for spinning up loadtest environments
1 parent dab1d1f commit 854e974

14 files changed

+786
-4
lines changed

‎.gitignore

+6-1
Original file line numberDiff line numberDiff line change
@@ -48,9 +48,14 @@ site/stats/
4848
*.lock.hcl
4949
.terraform/
5050

51-
/.coderv2/*
51+
**/.coderv2/*
5252
**/__debug_bin
5353

5454
# direnv
5555
.envrc
5656
*.test
57+
58+
# Loadtesting
59+
./scaletest/terraform/.terraform
60+
./scaletest/terraform/.terraform.lock.hcl
61+
terraform.tfstate.*

‎.prettierignore

+6-1
Original file line numberDiff line numberDiff line change
@@ -51,12 +51,17 @@ site/stats/
5151
*.lock.hcl
5252
.terraform/
5353

54-
/.coderv2/*
54+
**/.coderv2/*
5555
**/__debug_bin
5656

5757
# direnv
5858
.envrc
5959
*.test
60+
61+
# Loadtesting
62+
./scaletest/terraform/.terraform
63+
./scaletest/terraform/.terraform.lock.hcl
64+
terraform.tfstate.*
6065
# .prettierignore.include:
6166
# Helm templates contain variables that are invalid YAML and can't be formatted
6267
# by Prettier.

‎scaletest/terraform/README.md

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# Load Test Terraform
2+
3+
This folder contains Terraform code and scripts to aid in performing load tests of Coder.
4+
It does the following:
5+
6+
- Creates a GCP VPC.
7+
- Creates a CloudSQL instance with a global peering rule so it's accessible inside the VPC.
8+
- Creates a GKE cluster inside the VPC with separate nodegroups for Coder and workspaces.
9+
- Installs Coder in a new namespace, using the CloudSQL instance.
10+
11+
## Usage
12+
13+
> You must have an existing Google Cloud project available.
14+
15+
1. Create a file named `override.tfvars` with the following content, modifying as appropriate:
16+
17+
```terraform
18+
name = "some_unique_identifier"
19+
project_id = "some_google_project_id"
20+
```
21+
22+
1. Inspect `vars.tf` and override any other variables you deem necessary.
23+
24+
1. Run `terraform init`.
25+
26+
1. Run `terraform plan -var-file=override.tfvars` and inspect the output.
27+
If you are not satisfied, modify `override.tfvars` until you are.
28+
29+
1. Run `terraform apply -var-file=override.tfvars`. This will spin up a pre-configured environment
30+
and emit the Coder URL as an output.
31+
32+
1. Run `coder_init.sh <coder_url>` to setup an initial user and a pre-configured Kubernetes
33+
template. It will also download the Coder CLI from the Coder instance locally.
34+
35+
1. Do whatever you need to do with the Coder instance.
36+
37+
> To run Coder commands against the instance, you can use `coder_shim.sh <command>`.
38+
> You don't need to run `coder login` yourself.
39+
40+
1. When you are finished, you can run `terraform destroy -var-file=override.tfvars`.

‎scaletest/terraform/coder.tf

+250
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,250 @@
1+
data "google_client_config" "default" {}
2+
3+
locals {
4+
coder_helm_repo = "https://helm.coder.com/v2"
5+
coder_helm_chart = "coder"
6+
coder_release_name = var.name
7+
coder_namespace = "coder-${var.name}"
8+
coder_admin_email = "admin@coder.com"
9+
coder_admin_user = "coder"
10+
coder_address = google_compute_address.coder.address
11+
coder_url = "http://${google_compute_address.coder.address}"
12+
}
13+
14+
provider "kubernetes" {
15+
host = "https://${google_container_cluster.primary.endpoint}"
16+
cluster_ca_certificate = base64decode(google_container_cluster.primary.master_auth.0.cluster_ca_certificate)
17+
token = data.google_client_config.default.access_token
18+
}
19+
20+
provider "helm" {
21+
kubernetes {
22+
host = "https://${google_container_cluster.primary.endpoint}"
23+
cluster_ca_certificate = base64decode(google_container_cluster.primary.master_auth.0.cluster_ca_certificate)
24+
token = data.google_client_config.default.access_token
25+
}
26+
}
27+
28+
resource "kubernetes_namespace" "coder_namespace" {
29+
metadata {
30+
name = local.coder_namespace
31+
}
32+
depends_on = [
33+
google_container_node_pool.coder
34+
]
35+
}
36+
37+
resource "random_password" "postgres-admin-password" {
38+
length = 12
39+
}
40+
41+
resource "random_password" "coder-postgres-password" {
42+
length = 12
43+
}
44+
45+
resource "kubernetes_secret" "coder-db" {
46+
type = "" # Opaque
47+
metadata {
48+
name = "coder-db-url"
49+
namespace = kubernetes_namespace.coder_namespace.metadata.0.name
50+
}
51+
data = {
52+
url = "postgres://${google_sql_user.coder.name}:${urlencode(random_password.coder-postgres-password.result)}@${google_sql_database_instance.db.private_ip_address}/${google_sql_database.coder.name}?sslmode=disable"
53+
}
54+
}
55+
56+
resource "helm_release" "coder-chart" {
57+
repository = local.coder_helm_repo
58+
chart = local.coder_helm_chart
59+
name = local.coder_release_name
60+
version = var.coder_chart_version
61+
namespace = kubernetes_namespace.coder_namespace.metadata.0.name
62+
depends_on = [
63+
google_container_node_pool.coder,
64+
]
65+
values = [<<EOF
66+
coder:
67+
affinity:
68+
nodeAffinity:
69+
requiredDuringSchedulingIgnoredDuringExecution:
70+
nodeSelectorTerms:
71+
- matchExpressions:
72+
- key: "cloud.google.com/gke-nodepool"
73+
operator: "In"
74+
values: ["${google_container_node_pool.coder.name}"]
75+
podAntiAffinity:
76+
preferredDuringSchedulingIgnoredDuringExecution:
77+
- weight: 1
78+
podAffinityTerm:
79+
topologyKey: "kubernetes.io/hostname"
80+
labelSelector:
81+
matchExpressions:
82+
- key: "app.kubernetes.io/instance"
83+
operator: "In"
84+
values: ["${local.coder_release_name}"]
85+
env:
86+
- name: "CODER_CACHE_DIRECTORY"
87+
value: "/tmp/coder"
88+
- name: "CODER_ENABLE_TELEMETRY"
89+
value: "false"
90+
- name: "CODER_LOGGING_HUMAN"
91+
value: "/dev/null"
92+
- name: "CODER_LOGGING_STACKDRIVER"
93+
value: "/dev/stderr"
94+
- name: "CODER_PG_CONNECTION_URL"
95+
valueFrom:
96+
secretKeyRef:
97+
name: "${kubernetes_secret.coder-db.metadata.0.name}"
98+
key: url
99+
- name: "CODER_PROMETHEUS_ENABLE"
100+
value: "true"
101+
- name: "CODER_VERBOSE"
102+
value: "true"
103+
image:
104+
repo: ${var.coder_image_repo}
105+
tag: ${var.coder_image_tag}
106+
replicaCount: "${var.coder_replicas}"
107+
resources:
108+
requests:
109+
cpu: "${var.coder_cpu}"
110+
memory: "${var.coder_mem}"
111+
limits:
112+
cpu: "${var.coder_cpu}"
113+
memory: "${var.coder_mem}"
114+
securityContext:
115+
readOnlyRootFilesystem: true
116+
service:
117+
enable: true
118+
loadBalancerIP: "${local.coder_address}"
119+
volumeMounts:
120+
- mountPath: "/tmp"
121+
name: cache
122+
readOnly: false
123+
volumes:
124+
- emptyDir:
125+
sizeLimit: 1024Mi
126+
name: cache
127+
EOF
128+
]
129+
}
130+
131+
resource "local_file" "coder-monitoring-manifest" {
132+
filename = "${path.module}/.coderv2/coder-monitoring.yaml"
133+
content = <<EOF
134+
apiVersion: monitoring.googleapis.com/v1
135+
kind: PodMonitoring
136+
metadata:
137+
namespace: ${kubernetes_namespace.coder_namespace.metadata.0.name}
138+
name: coder-monitoring
139+
spec:
140+
selector:
141+
matchLabels:
142+
app.kubernetes.io/name: coder
143+
endpoints:
144+
- port: prometheus-http
145+
interval: 30s
146+
EOF
147+
}
148+
149+
resource "null_resource" "coder-monitoring-manifest_apply" {
150+
provisioner "local-exec" {
151+
working_dir = "${abspath(path.module)}/.coderv2"
152+
command = <<EOF
153+
KUBECONFIG=${var.name}-cluster.kubeconfig gcloud container clusters get-credentials ${var.name}-cluster --project=${var.project_id} --zone=${var.zone} && \
154+
KUBECONFIG=${var.name}-cluster.kubeconfig kubectl apply -f ${abspath(local_file.coder-monitoring-manifest.filename)}
155+
EOF
156+
}
157+
}
158+
159+
resource "local_file" "kubernetes_template" {
160+
filename = "${path.module}/.coderv2/templates/kubernetes/main.tf"
161+
content = <<EOF
162+
terraform {
163+
required_providers {
164+
coder = {
165+
source = "coder/coder"
166+
version = "~> 0.7.0"
167+
}
168+
kubernetes = {
169+
source = "hashicorp/kubernetes"
170+
version = "~> 2.18"
171+
}
172+
}
173+
}
174+
175+
provider "coder" {}
176+
177+
provider "kubernetes" {
178+
config_path = null # always use host
179+
}
180+
181+
data "coder_workspace" "me" {}
182+
183+
resource "coder_agent" "main" {
184+
os = "linux"
185+
arch = "amd64"
186+
startup_script_timeout = 180
187+
startup_script = ""
188+
}
189+
190+
resource "kubernetes_pod" "main" {
191+
count = data.coder_workspace.me.start_count
192+
metadata {
193+
name = "coder-$${lower(data.coder_workspace.me.owner)}-$${lower(data.coder_workspace.me.name)}"
194+
namespace = "${kubernetes_namespace.coder_namespace.metadata.0.name}"
195+
labels = {
196+
"app.kubernetes.io/name" = "coder-workspace"
197+
"app.kubernetes.io/instance" = "coder-workspace-$${lower(data.coder_workspace.me.owner)}-$${lower(data.coder_workspace.me.name)}"
198+
}
199+
}
200+
spec {
201+
security_context {
202+
run_as_user = "1000"
203+
fs_group = "1000"
204+
}
205+
container {
206+
name = "dev"
207+
image = "${var.workspace_image}"
208+
image_pull_policy = "Always"
209+
command = ["sh", "-c", coder_agent.main.init_script]
210+
security_context {
211+
run_as_user = "1000"
212+
}
213+
env {
214+
name = "CODER_AGENT_TOKEN"
215+
value = coder_agent.main.token
216+
}
217+
resources {
218+
requests = {
219+
"cpu" = "0.1"
220+
"memory" = "128Mi"
221+
}
222+
limits = {
223+
"cpu" = "1"
224+
"memory" = "1Gi"
225+
}
226+
}
227+
}
228+
229+
affinity {
230+
node_affinity {
231+
required_during_scheduling_ignored_during_execution {
232+
node_selector_term {
233+
match_expressions {
234+
key = "cloud.google.com/gke-nodepool"
235+
operator = "In"
236+
values = ["${google_container_node_pool.workspaces.name}"]
237+
}
238+
}
239+
}
240+
}
241+
}
242+
}
243+
}
244+
EOF
245+
}
246+
247+
output "coder_url" {
248+
description = "URL of the Coder deployment"
249+
value = local.coder_url
250+
}

‎scaletest/terraform/coder_init.sh

+51
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
if [[ $# -lt 1 ]]; then
6+
echo "Usage: $0 <coder URL>"
7+
exit 1
8+
fi
9+
10+
# Allow toggling verbose output
11+
[[ -n ${VERBOSE:-} ]] && set -x
12+
13+
CODER_URL=$1
14+
CONFIG_DIR="${PWD}/.coderv2"
15+
ARCH="$(arch)"
16+
if [[ "$ARCH" == "x86_64" ]]; then
17+
ARCH="amd64"
18+
fi
19+
PLATFORM="$(uname | tr '[:upper:]' '[:lower:]')"
20+
21+
mkdir -p "${CONFIG_DIR}"
22+
echo "Fetching Coder CLI for first-time setup!"
23+
curl -fsSLk "${CODER_URL}/bin/coder-${PLATFORM}-${ARCH}" -o "${CONFIG_DIR}/coder"
24+
chmod +x "${CONFIG_DIR}/coder"
25+
26+
set +o pipefail
27+
RANDOM_ADMIN_PASSWORD=$(tr </dev/urandom -dc _A-Z-a-z-0-9 | head -c16)
28+
set -o pipefail
29+
CODER_FIRST_USER_EMAIL="admin@coder.com"
30+
CODER_FIRST_USER_USERNAME="coder"
31+
CODER_FIRST_USER_PASSWORD="${RANDOM_ADMIN_PASSWORD}"
32+
CODER_FIRST_USER_TRIAL="false"
33+
echo "Running login command!"
34+
"${CONFIG_DIR}/coder" login "${CODER_URL}" \
35+
--global-config="${CONFIG_DIR}" \
36+
--first-user-username="${CODER_FIRST_USER_USERNAME}" \
37+
--first-user-email="${CODER_FIRST_USER_EMAIL}" \
38+
--first-user-password="${CODER_FIRST_USER_PASSWORD}" \
39+
--first-user-trial=false
40+
41+
echo "Writing credentials to ${CONFIG_DIR}/coder.env"
42+
cat <<EOF >"${CONFIG_DIR}/coder.env"
43+
CODER_FIRST_USER_EMAIL=admin@coder.com
44+
CODER_FIRST_USER_USERNAME=coder
45+
CODER_FIRST_USER_PASSWORD="${RANDOM_ADMIN_PASSWORD}"
46+
CODER_FIRST_USER_TRIAL="${CODER_FIRST_USER_TRIAL}"
47+
EOF
48+
49+
echo "Importing kubernetes template"
50+
"${CONFIG_DIR}/coder" templates create --global-config="${CONFIG_DIR}" \
51+
--directory "${CONFIG_DIR}/templates/kubernetes" --yes kubernetes

‎scaletest/terraform/coder_shim.sh

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/usr/bin/env bash
2+
3+
# This is a shim for easily executing Coder commands against a loadtest cluster
4+
# without having to overwrite your own session/URL
5+
SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}")
6+
CONFIG_DIR="${SCRIPT_DIR}/.coderv2"
7+
CODER_BIN="${CONFIG_DIR}/coder"
8+
exec "${CODER_BIN}" --global-config "${CONFIG_DIR}" "$@"

0 commit comments

Comments
 (0)