Skip to content

Commit a1f3a6b

Browse files
johnstcnjohnstcn
authored
refactor(scaletest/terraform): break up infra creation and k8s resource provisioning (#9824)
Breaks up scaletest/terraform into {infra, k8s}.
1 parent 84999cb commit a1f3a6b

File tree

11 files changed

+285
-153
lines changed

11 files changed

+285
-153
lines changed

‎scaletest/terraform/gcp_cluster.tf renamed to ‎scaletest/terraform/infra/gcp_cluster.tf

+3-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,9 @@ data "google_compute_default_service_account" "default" {
33
}
44

55
locals {
6-
cluster_kubeconfig_path = "${abspath(path.module)}/../.coderv2/${var.name}-cluster.kubeconfig"
6+
abs_module_path = abspath(path.module)
7+
rel_kubeconfig_path = "../../.coderv2/${var.name}-cluster.kubeconfig"
8+
cluster_kubeconfig_path = abspath("${local.abs_module_path}/${local.rel_kubeconfig_path}")
79
}
810

911
resource "google_container_cluster" "primary" {

‎scaletest/terraform/gcp_db.tf renamed to ‎scaletest/terraform/infra/gcp_db.tf

+12
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,14 @@ resource "google_sql_database" "coder" {
4242
deletion_policy = "ABANDON"
4343
}
4444

45+
resource "random_password" "coder-postgres-password" {
46+
length = 12
47+
}
48+
49+
resource "random_password" "prometheus-postgres-password" {
50+
length = 12
51+
}
52+
4553
resource "google_sql_user" "coder" {
4654
project = var.project_id
4755
instance = google_sql_database_instance.db.id
@@ -61,3 +69,7 @@ resource "google_sql_user" "prometheus" {
6169
# required for postgres, otherwise user fails to delete
6270
deletion_policy = "ABANDON"
6371
}
72+
73+
locals {
74+
coder_db_url = "postgres://${google_sql_user.coder.name}:${urlencode(random_password.coder-postgres-password.result)}@${google_sql_database_instance.db.private_ip_address}/${google_sql_database.coder.name}?sslmode=disable"
75+
}

‎scaletest/terraform/gcp_project.tf renamed to ‎scaletest/terraform/infra/gcp_project.tf

-5
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,3 @@
1-
provider "google" {
2-
region = var.region
3-
project = var.project_id
4-
}
5-
61
locals {
72
project_apis = [
83
"cloudtrace",

‎scaletest/terraform/gcp_vpc.tf renamed to ‎scaletest/terraform/infra/gcp_vpc.tf

File renamed without changes.

‎scaletest/terraform/infra/main.tf

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
terraform {
2+
required_providers {
3+
google = {
4+
source = "hashicorp/google"
5+
version = "~> 4.36"
6+
}
7+
8+
random = {
9+
source = "hashicorp/random"
10+
version = "~> 3.5"
11+
}
12+
}
13+
14+
required_version = "~> 1.5.0"
15+
}
16+
17+
provider "google" {
18+
region = var.region
19+
project = var.project_id
20+
}

‎scaletest/terraform/infra/outputs.tf

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
output "coder_db_url" {
2+
description = "URL of the database for Coder."
3+
value = local.coder_db_url
4+
sensitive = true
5+
}
6+
7+
output "coder_address" {
8+
description = "IP address to use for the Coder service."
9+
value = google_compute_address.coder.address
10+
}
11+
12+
output "kubernetes_kubeconfig_path" {
13+
description = "Kubeconfig path."
14+
value = local.cluster_kubeconfig_path
15+
}
16+
17+
output "kubernetes_nodepool_coder" {
18+
description = "Name of the nodepool on which to run Coder."
19+
value = google_container_node_pool.coder.name
20+
}
21+
22+
output "kubernetes_nodepool_misc" {
23+
description = "Name of the nodepool on which to run everything else."
24+
value = google_container_node_pool.misc.name
25+
}
26+
27+
output "kubernetes_nodepool_workspaces" {
28+
description = "Name of the nodepool on which to run workspaces."
29+
value = google_container_node_pool.workspaces.name
30+
}
31+
32+
output "prometheus_external_label_cluster" {
33+
description = "Value for the Prometheus external label named cluster."
34+
value = google_container_cluster.primary.name
35+
}
36+
37+
output "prometheus_postgres_dbname" {
38+
description = "Name of the database for Prometheus to monitor."
39+
value = google_sql_database.coder.name
40+
}
41+
42+
output "prometheus_postgres_host" {
43+
description = "Hostname of the database for Prometheus to connect to."
44+
value = google_sql_database_instance.db.private_ip_address
45+
}
46+
47+
output "prometheus_postgres_password" {
48+
description = "Postgres password for Prometheus."
49+
value = random_password.prometheus-postgres-password.result
50+
sensitive = true
51+
}
52+
53+
output "prometheus_postgres_user" {
54+
description = "Postgres username for Prometheus."
55+
value = google_sql_user.prometheus.name
56+
}
57+
58+
resource "local_file" "outputs" {
59+
filename = "${path.module}/../../.coderv2/infra_outputs.tfvars"
60+
content = <<EOF
61+
coder_db_url = "${local.coder_db_url}"
62+
coder_address = "${google_compute_address.coder.address}"
63+
kubernetes_kubeconfig_path = "${local.cluster_kubeconfig_path}"
64+
kubernetes_nodepool_coder = "${google_container_node_pool.coder.name}"
65+
kubernetes_nodepool_misc = "${google_container_node_pool.misc.name}"
66+
kubernetes_nodepool_workspaces = "${google_container_node_pool.workspaces.name}"
67+
prometheus_external_label_cluster = "${google_container_cluster.primary.name}"
68+
prometheus_postgres_dbname = "${google_sql_database.coder.name}"
69+
prometheus_postgres_host = "${google_sql_database_instance.db.private_ip_address}"
70+
prometheus_postgres_password = "${random_password.prometheus-postgres-password.result}"
71+
prometheus_postgres_user = "${google_sql_user.prometheus.name}"
72+
EOF
73+
}

‎scaletest/terraform/infra/vars.tf

+102
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
variable "state" {
2+
description = "The state of the cluster. Valid values are 'started', and 'stopped'."
3+
validation {
4+
condition = contains(["started", "stopped"], var.state)
5+
error_message = "value must be one of 'started' or 'stopped'"
6+
}
7+
default = "started"
8+
}
9+
10+
variable "project_id" {
11+
description = "The project in which to provision resources"
12+
}
13+
14+
variable "name" {
15+
description = "Adds a prefix to resources."
16+
}
17+
18+
variable "region" {
19+
description = "GCP region in which to provision resources."
20+
default = "us-east1"
21+
}
22+
23+
variable "zone" {
24+
description = "GCP zone in which to provision resources."
25+
default = "us-east1-c"
26+
}
27+
28+
variable "k8s_version" {
29+
description = "Kubernetes version to provision."
30+
default = "1.24"
31+
}
32+
33+
variable "node_disk_size_gb" {
34+
description = "Size of the root disk for cluster nodes."
35+
default = 100
36+
}
37+
38+
variable "node_image_type" {
39+
description = "Image type to use for cluster nodes."
40+
default = "cos_containerd"
41+
}
42+
43+
// Preemptible nodes are way cheaper, but can be pulled out
44+
// from under you at any time. Caveat emptor.
45+
variable "node_preemptible" {
46+
description = "Use preemptible nodes."
47+
default = false
48+
}
49+
50+
// We create three nodepools:
51+
// - One for the Coder control plane
52+
// - One for workspaces
53+
// - One for everything else (for example, load generation)
54+
55+
// These variables control the node pool dedicated to Coder.
56+
variable "nodepool_machine_type_coder" {
57+
description = "Machine type to use for Coder control plane nodepool."
58+
default = "t2d-standard-4"
59+
}
60+
61+
variable "nodepool_size_coder" {
62+
description = "Number of cluster nodes for the Coder control plane nodepool."
63+
default = 1
64+
}
65+
66+
// These variables control the node pool dedicated to workspaces.
67+
variable "nodepool_machine_type_workspaces" {
68+
description = "Machine type to use for the workspaces nodepool."
69+
default = "t2d-standard-4"
70+
}
71+
72+
variable "nodepool_size_workspaces" {
73+
description = "Number of cluster nodes for the workspaces nodepool."
74+
default = 1
75+
}
76+
77+
// These variables control the node pool for everything else.
78+
variable "nodepool_machine_type_misc" {
79+
description = "Machine type to use for the misc nodepool."
80+
default = "t2d-standard-4"
81+
}
82+
83+
variable "nodepool_size_misc" {
84+
description = "Number of cluster nodes for the misc nodepool."
85+
default = 1
86+
}
87+
88+
// These variables control the size of the database to be used by Coder.
89+
variable "cloudsql_version" {
90+
description = "CloudSQL version to provision"
91+
default = "POSTGRES_14"
92+
}
93+
94+
variable "cloudsql_tier" {
95+
description = "CloudSQL database tier."
96+
default = "db-f1-micro"
97+
}
98+
99+
variable "cloudsql_max_connections" {
100+
description = "CloudSQL database max_connections"
101+
default = 500
102+
}

‎scaletest/terraform/coder.tf renamed to ‎scaletest/terraform/k8s/coder.tf

+14-41
Original file line numberDiff line numberDiff line change
@@ -7,32 +7,14 @@ locals {
77
coder_namespace = "coder-${var.name}"
88
coder_admin_email = "admin@coder.com"
99
coder_admin_user = "coder"
10-
coder_address = google_compute_address.coder.address
11-
coder_url = "http://${google_compute_address.coder.address}"
12-
}
13-
14-
provider "kubernetes" {
15-
host = "https://${google_container_cluster.primary.endpoint}"
16-
cluster_ca_certificate = base64decode(google_container_cluster.primary.master_auth.0.cluster_ca_certificate)
17-
token = data.google_client_config.default.access_token
18-
}
19-
20-
provider "helm" {
21-
kubernetes {
22-
host = "https://${google_container_cluster.primary.endpoint}"
23-
cluster_ca_certificate = base64decode(google_container_cluster.primary.master_auth.0.cluster_ca_certificate)
24-
token = data.google_client_config.default.access_token
25-
}
10+
coder_access_url = "http://${var.coder_address}"
2611
}
2712

2813
resource "null_resource" "coder_namespace" {
2914
triggers = {
3015
namespace = local.coder_namespace
31-
kubeconfig_path = local.cluster_kubeconfig_path
16+
kubeconfig_path = var.kubernetes_kubeconfig_path
3217
}
33-
depends_on = [
34-
google_container_node_pool.coder
35-
]
3618
provisioner "local-exec" {
3719
when = create
3820
command = <<EOF
@@ -45,14 +27,6 @@ resource "null_resource" "coder_namespace" {
4527
}
4628
}
4729

48-
resource "random_password" "coder-postgres-password" {
49-
length = 12
50-
}
51-
52-
resource "random_password" "prometheus-postgres-password" {
53-
length = 12
54-
}
55-
5630
resource "kubernetes_secret" "coder-db" {
5731
type = "Opaque"
5832
metadata {
@@ -61,7 +35,7 @@ resource "kubernetes_secret" "coder-db" {
6135
}
6236
depends_on = [null_resource.coder_namespace]
6337
data = {
64-
url = "postgres://${google_sql_user.coder.name}:${urlencode(random_password.coder-postgres-password.result)}@${google_sql_database_instance.db.private_ip_address}/${google_sql_database.coder.name}?sslmode=disable"
38+
url = var.coder_db_url
6539
}
6640
}
6741

@@ -72,7 +46,6 @@ resource "helm_release" "coder-chart" {
7246
version = var.coder_chart_version
7347
namespace = local.coder_namespace
7448
depends_on = [
75-
google_container_node_pool.coder,
7649
null_resource.coder_namespace
7750
]
7851
values = [<<EOF
@@ -84,7 +57,7 @@ coder:
8457
- matchExpressions:
8558
- key: "cloud.google.com/gke-nodepool"
8659
operator: "In"
87-
values: ["${google_container_node_pool.coder.name}"]
60+
values: ["${var.kubernetes_nodepool_coder}"]
8861
podAntiAffinity:
8962
preferredDuringSchedulingIgnoredDuringExecution:
9063
- weight: 1
@@ -97,7 +70,7 @@ coder:
9770
values: ["${local.coder_release_name}"]
9871
env:
9972
- name: "CODER_ACCESS_URL"
100-
value: "${local.coder_url}"
73+
value: "${local.coder_access_url}"
10174
- name: "CODER_CACHE_DIRECTORY"
10275
value: "/tmp/coder"
10376
- name: "CODER_ENABLE_TELEMETRY"
@@ -144,7 +117,7 @@ coder:
144117
service:
145118
enable: true
146119
sessionAffinity: None
147-
loadBalancerIP: "${local.coder_address}"
120+
loadBalancerIP: "${var.coder_address}"
148121
volumeMounts:
149122
- mountPath: "/tmp"
150123
name: cache
@@ -234,7 +207,7 @@ resource "local_file" "kubernetes_template" {
234207
match_expressions {
235208
key = "cloud.google.com/gke-nodepool"
236209
operator = "In"
237-
values = ["${google_container_node_pool.workspaces.name}"]
210+
values = ["${var.kubernetes_nodepool_workspaces}"]
238211
}
239212
}
240213
}
@@ -284,7 +257,7 @@ spec:
284257
- key: cloud.google.com/gke-nodepool
285258
operator: In
286259
values:
287-
- ${google_container_node_pool.coder.name}
260+
- ${var.kubernetes_nodepool_coder}
288261
podAntiAffinity:
289262
preferredDuringSchedulingIgnoredDuringExecution:
290263
- podAffinityTerm:
@@ -307,7 +280,7 @@ spec:
307280
- name: CODER_PROMETHEUS_ADDRESS
308281
value: 0.0.0.0:2112
309282
- name: CODER_ACCESS_URL
310-
value: ${local.coder_url}
283+
value: ${local.coder_access_url}
311284
- name: CODER_CACHE_DIRECTORY
312285
value: /tmp/coder
313286
- name: CODER_ENABLE_TELEMETRY
@@ -395,9 +368,9 @@ spec:
395368
}
396369

397370
resource "null_resource" "provisionerd_deployment_apply" {
398-
depends_on = [helm_release.coder-chart, local_file.provisionerd_deployment, null_resource.cluster_kubeconfig]
371+
depends_on = [helm_release.coder-chart, local_file.provisionerd_deployment]
399372
triggers = {
400-
kubeconfig_path = local.cluster_kubeconfig_path
373+
kubeconfig_path = var.kubernetes_kubeconfig_path
401374
manifest_path = local_file.provisionerd_deployment.filename
402375
}
403376
provisioner "local-exec" {
@@ -408,11 +381,11 @@ resource "null_resource" "provisionerd_deployment_apply" {
408381
}
409382

410383
resource "local_file" "output_vars" {
411-
filename = "${path.module}/../.coderv2/url"
412-
content = local.coder_url
384+
filename = "${path.module}/../../.coderv2/url"
385+
content = local.coder_access_url
413386
}
414387

415388
output "coder_url" {
416389
description = "URL of the Coder deployment"
417-
value = local.coder_url
390+
value = local.coder_access_url
418391
}

0 commit comments

Comments
 (0)