From 8e3c9864967b1f6ec7ec9e4f87719bdbf3bfaf05 Mon Sep 17 00:00:00 2001
From: Cian Johnston <cian@coder.com>
Date: Fri, 22 Sep 2023 14:41:43 +0100
Subject: [PATCH 1/5] refactor(scaletest/terraform): break up infra creation
 and k8s resource provisioning

---
 .../terraform/{ => infra}/gcp_cluster.tf      |   4 +-
 scaletest/terraform/{ => infra}/gcp_db.tf     |  12 ++
 .../terraform/{ => infra}/gcp_project.tf      |   5 -
 scaletest/terraform/{ => infra}/gcp_vpc.tf    |   0
 scaletest/terraform/infra/main.tf             |  20 ++
 scaletest/terraform/infra/outputs.tf          |  73 +++++++
 scaletest/terraform/infra/vars.tf             | 102 ++++++++++
 scaletest/terraform/{ => k8s}/coder.tf        |  55 ++---
 scaletest/terraform/{ => k8s}/main.tf         |  20 +-
 scaletest/terraform/{ => k8s}/prometheus.tf   |  28 ++-
 scaletest/terraform/{ => k8s}/vars.tf         | 189 +++++++++++-------
 scaletest/terraform/scenario-test.tfvars      |   9 +
 12 files changed, 374 insertions(+), 143 deletions(-)
 rename scaletest/terraform/{ => infra}/gcp_cluster.tf (95%)
 rename scaletest/terraform/{ => infra}/gcp_db.tf (81%)
 rename scaletest/terraform/{ => infra}/gcp_project.tf (88%)
 rename scaletest/terraform/{ => infra}/gcp_vpc.tf (100%)
 create mode 100644 scaletest/terraform/infra/main.tf
 create mode 100644 scaletest/terraform/infra/outputs.tf
 create mode 100644 scaletest/terraform/infra/vars.tf
 rename scaletest/terraform/{ => k8s}/coder.tf (85%)
 rename scaletest/terraform/{ => k8s}/main.tf (70%)
 rename scaletest/terraform/{ => k8s}/prometheus.tf (86%)
 rename scaletest/terraform/{ => k8s}/vars.tf (54%)
 create mode 100644 scaletest/terraform/scenario-test.tfvars

diff --git a/scaletest/terraform/gcp_cluster.tf b/scaletest/terraform/infra/gcp_cluster.tf
similarity index 95%
rename from scaletest/terraform/gcp_cluster.tf
rename to scaletest/terraform/infra/gcp_cluster.tf
index d83ab99c8cb74..cb744545eee1f 100644
--- a/scaletest/terraform/gcp_cluster.tf
+++ b/scaletest/terraform/infra/gcp_cluster.tf
@@ -3,7 +3,9 @@ data "google_compute_default_service_account" "default" {
 }
 
 locals {
-  cluster_kubeconfig_path = "${abspath(path.module)}/../.coderv2/${var.name}-cluster.kubeconfig"
+  abs_module_path = "${abspath(path.module)}"
+  rel_kubeconfig_path = "../../.coderv2/${var.name}-cluster.kubeconfig"
+  cluster_kubeconfig_path = "${abspath("${local.abs_module_path}/${local.rel_kubeconfig_path}")}"
 }
 
 resource "google_container_cluster" "primary" {
diff --git a/scaletest/terraform/gcp_db.tf b/scaletest/terraform/infra/gcp_db.tf
similarity index 81%
rename from scaletest/terraform/gcp_db.tf
rename to scaletest/terraform/infra/gcp_db.tf
index 8855d24f6d82b..1a02324ce071f 100644
--- a/scaletest/terraform/gcp_db.tf
+++ b/scaletest/terraform/infra/gcp_db.tf
@@ -42,6 +42,14 @@ resource "google_sql_database" "coder" {
   deletion_policy = "ABANDON"
 }
 
+resource "random_password" "coder-postgres-password" {
+  length = 12
+}
+
+resource "random_password" "prometheus-postgres-password" {
+  length = 12
+}
+
 resource "google_sql_user" "coder" {
   project  = var.project_id
   instance = google_sql_database_instance.db.id
@@ -61,3 +69,7 @@ resource "google_sql_user" "prometheus" {
   # required for postgres, otherwise user fails to delete
   deletion_policy = "ABANDON"
 }
+
+locals {
+  coder_db_url = "postgres://${google_sql_user.coder.name}:${urlencode(random_password.coder-postgres-password.result)}@${google_sql_database_instance.db.private_ip_address}/${google_sql_database.coder.name}?sslmode=disable"
+}
diff --git a/scaletest/terraform/gcp_project.tf b/scaletest/terraform/infra/gcp_project.tf
similarity index 88%
rename from scaletest/terraform/gcp_project.tf
rename to scaletest/terraform/infra/gcp_project.tf
index c233042e66acb..1073a621c33e0 100644
--- a/scaletest/terraform/gcp_project.tf
+++ b/scaletest/terraform/infra/gcp_project.tf
@@ -1,8 +1,3 @@
-provider "google" {
-  region  = var.region
-  project = var.project_id
-}
-
 locals {
   project_apis = [
     "cloudtrace",
diff --git a/scaletest/terraform/gcp_vpc.tf b/scaletest/terraform/infra/gcp_vpc.tf
similarity index 100%
rename from scaletest/terraform/gcp_vpc.tf
rename to scaletest/terraform/infra/gcp_vpc.tf
diff --git a/scaletest/terraform/infra/main.tf b/scaletest/terraform/infra/main.tf
new file mode 100644
index 0000000000000..1724692b19f3a
--- /dev/null
+++ b/scaletest/terraform/infra/main.tf
@@ -0,0 +1,20 @@
+terraform {
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "~> 4.36"
+    }
+
+    random = {
+      source  = "hashicorp/random"
+      version = "~> 3.5"
+    }
+  }
+
+  required_version = "~> 1.5.0"
+}
+
+provider "google" {
+  region  = var.region
+  project = var.project_id
+}
diff --git a/scaletest/terraform/infra/outputs.tf b/scaletest/terraform/infra/outputs.tf
new file mode 100644
index 0000000000000..812283d52db35
--- /dev/null
+++ b/scaletest/terraform/infra/outputs.tf
@@ -0,0 +1,73 @@
+output "coder_db_url" {
+  description = "URL of the database for Coder."
+  value = local.coder_db_url
+  sensitive = true
+}
+
+output "coder_address" {
+  description = "IP address to use for the Coder service."
+  value = google_compute_address.coder.address
+}
+
+output "kubernetes_kubeconfig_path" {
+  description = "Kubeconfig path."
+  value = local.cluster_kubeconfig_path
+}
+
+output "kubernetes_nodepool_coder" {
+  description = "Name of the nodepool on which to run Coder."
+  value = google_container_node_pool.coder.name
+}
+
+output "kubernetes_nodepool_misc" {
+  description = "Name of the nodepool on which to run everything else."
+  value = google_container_node_pool.misc.name
+}
+
+output "kubernetes_nodepool_workspaces" {
+  description = "Name of the nodepool on which to run workspaces."
+  value = google_container_node_pool.workspaces.name
+}
+
+output "prometheus_external_label_cluster" {
+  description = "Value for the Prometheus external label named cluster."
+  value = google_container_cluster.primary.name
+}
+
+output "prometheus_postgres_dbname" {
+  description = "Name of the database for Prometheus to monitor."
+  value = google_sql_database.coder.name
+}
+
+output "prometheus_postgres_host" {
+  description = "Hostname of the database for Prometheus to connect to."
+  value = google_sql_database_instance.db.private_ip_address
+}
+
+output "prometheus_postgres_password" {
+  description = "Postgres password for Prometheus."
+  value = random_password.prometheus-postgres-password.result
+  sensitive = true
+}
+
+output "prometheus_postgres_user" {
+  description = "Postgres username for Prometheus."
+  value = google_sql_user.prometheus.name
+}
+
+resource "local_file" "outputs" {
+  filename = "${path.module}/../../.coderv2/infra_outputs.tfvars"
+  content = <<EOF
+  coder_db_url = "${local.coder_db_url}"
+  coder_address = "${google_compute_address.coder.address}"
+  kubernetes_kubeconfig_path = "${local.cluster_kubeconfig_path}"
+  kubernetes_nodepool_coder = "${google_container_node_pool.coder.name}"
+  kubernetes_nodepool_misc = "${google_container_node_pool.misc.name}"
+  kubernetes_nodepool_workspaces = "${google_container_node_pool.workspaces.name}"
+  prometheus_external_label_cluster = "${google_container_cluster.primary.name}"
+  prometheus_postgres_dbname = "${google_sql_database.coder.name}"
+  prometheus_postgres_host = "${google_sql_database_instance.db.private_ip_address}"
+  prometheus_postgres_password = "${random_password.prometheus-postgres-password.result}"
+  prometheus_postgres_user = "${google_sql_user.prometheus.name}"
+EOF
+}
diff --git a/scaletest/terraform/infra/vars.tf b/scaletest/terraform/infra/vars.tf
new file mode 100644
index 0000000000000..e93a4e9b7ab02
--- /dev/null
+++ b/scaletest/terraform/infra/vars.tf
@@ -0,0 +1,102 @@
+variable "state" {
+  description = "The state of the cluster. Valid values are 'started', and 'stopped'."
+  validation {
+    condition     = contains(["started", "stopped"], var.state)
+    error_message = "value must be one of 'started' or 'stopped'"
+  }
+  default = "started"
+}
+
+variable "project_id" {
+  description = "The project in which to provision resources"
+}
+
+variable "name" {
+  description = "Adds a prefix to resources."
+}
+
+variable "region" {
+  description = "GCP region in which to provision resources."
+  default     = "us-east1"
+}
+
+variable "zone" {
+  description = "GCP zone in which to provision resources."
+  default     = "us-east1-c"
+}
+
+variable "k8s_version" {
+  description = "Kubernetes vversion to provision."
+  default     = "1.24"
+}
+
+variable "node_disk_size_gb" {
+  description = "Size of the root disk for cluster nodes."
+  default     = 100
+}
+
+variable "node_image_type" {
+  description = "Image type to use for cluster nodes."
+  default     = "cos_containerd"
+}
+
+// Preemptible nodes are way cheaper, but can be pulled out
+// from under you at any time. Caveat emptor.
+variable "node_preemptible" {
+  description = "Use preemptible nodes."
+  default     = false
+}
+
+// We create three nodepools:
+// - One for the Coder control plane
+// - One for workspaces
+// - One for everything else (for example, load generation)
+
+// These variables control the node pool dedicated to Coder.
+variable "nodepool_machine_type_coder" {
+  description = "Machine type to use for Coder control plane nodepool."
+  default     = "t2d-standard-4"
+}
+
+variable "nodepool_size_coder" {
+  description = "Number of cluster nodes for the Coder control plane nodepool."
+  default     = 1
+}
+
+// These variables control the node pool dedicated to workspaces.
+variable "nodepool_machine_type_workspaces" {
+  description = "Machine type to use for the workspaces nodepool."
+  default     = "t2d-standard-4"
+}
+
+variable "nodepool_size_workspaces" {
+  description = "Number of cluster nodes for the workspaces nodepool."
+  default     = 1
+}
+
+// These variables control the node pool for everything else.
+variable "nodepool_machine_type_misc" {
+  description = "Machine type to use for the misc nodepool."
+  default     = "t2d-standard-4"
+}
+
+variable "nodepool_size_misc" {
+  description = "Number of cluster nodes for the misc nodepool."
+  default     = 1
+}
+
+// These variables control the size of the database to be used by Coder.
+variable "cloudsql_version" {
+  description = "CloudSQL version to provision"
+  default     = "POSTGRES_14"
+}
+
+variable "cloudsql_tier" {
+  description = "CloudSQL database tier."
+  default     = "db-f1-micro"
+}
+
+variable "cloudsql_max_connections" {
+  description = "CloudSQL database max_connections"
+  default     = 500
+}
diff --git a/scaletest/terraform/coder.tf b/scaletest/terraform/k8s/coder.tf
similarity index 85%
rename from scaletest/terraform/coder.tf
rename to scaletest/terraform/k8s/coder.tf
index 55a9e7a35721e..f6b9ae7d16a09 100644
--- a/scaletest/terraform/coder.tf
+++ b/scaletest/terraform/k8s/coder.tf
@@ -7,32 +7,14 @@ locals {
   coder_namespace    = "coder-${var.name}"
   coder_admin_email  = "admin@coder.com"
   coder_admin_user   = "coder"
-  coder_address      = google_compute_address.coder.address
-  coder_url          = "http://${google_compute_address.coder.address}"
-}
-
-provider "kubernetes" {
-  host                   = "https://${google_container_cluster.primary.endpoint}"
-  cluster_ca_certificate = base64decode(google_container_cluster.primary.master_auth.0.cluster_ca_certificate)
-  token                  = data.google_client_config.default.access_token
-}
-
-provider "helm" {
-  kubernetes {
-    host                   = "https://${google_container_cluster.primary.endpoint}"
-    cluster_ca_certificate = base64decode(google_container_cluster.primary.master_auth.0.cluster_ca_certificate)
-    token                  = data.google_client_config.default.access_token
-  }
+  coder_access_url   = "http://${var.coder_address}"
 }
 
 resource "null_resource" "coder_namespace" {
   triggers = {
     namespace       = local.coder_namespace
-    kubeconfig_path = local.cluster_kubeconfig_path
+    kubeconfig_path = var.kubernetes_kubeconfig_path
   }
-  depends_on = [
-    google_container_node_pool.coder
-  ]
   provisioner "local-exec" {
     when    = create
     command = <<EOF
@@ -45,14 +27,6 @@ resource "null_resource" "coder_namespace" {
   }
 }
 
-resource "random_password" "coder-postgres-password" {
-  length = 12
-}
-
-resource "random_password" "prometheus-postgres-password" {
-  length = 12
-}
-
 resource "kubernetes_secret" "coder-db" {
   type = "Opaque"
   metadata {
@@ -61,7 +35,7 @@ resource "kubernetes_secret" "coder-db" {
   }
   depends_on = [null_resource.coder_namespace]
   data = {
-    url = "postgres://${google_sql_user.coder.name}:${urlencode(random_password.coder-postgres-password.result)}@${google_sql_database_instance.db.private_ip_address}/${google_sql_database.coder.name}?sslmode=disable"
+    url = var.coder_db_url
   }
 }
 
@@ -72,7 +46,6 @@ resource "helm_release" "coder-chart" {
   version    = var.coder_chart_version
   namespace  = local.coder_namespace
   depends_on = [
-    google_container_node_pool.coder,
     null_resource.coder_namespace
   ]
   values = [<<EOF
@@ -84,7 +57,7 @@ coder:
         - matchExpressions:
           - key: "cloud.google.com/gke-nodepool"
             operator: "In"
-            values: ["${google_container_node_pool.coder.name}"]
+            values: ["${var.kubernetes_nodepool_coder}"]
     podAntiAffinity:
       preferredDuringSchedulingIgnoredDuringExecution:
       - weight: 1
@@ -97,7 +70,7 @@ coder:
               values:   ["${local.coder_release_name}"]
   env:
     - name: "CODER_ACCESS_URL"
-      value: "${local.coder_url}"
+      value: "${local.coder_access_url}"
     - name: "CODER_CACHE_DIRECTORY"
       value: "/tmp/coder"
     - name: "CODER_ENABLE_TELEMETRY"
@@ -144,7 +117,7 @@ coder:
   service:
     enable: true
     sessionAffinity: None
-    loadBalancerIP: "${local.coder_address}"
+    loadBalancerIP: "${var.coder_address}"
   volumeMounts:
   - mountPath: "/tmp"
     name: cache
@@ -234,7 +207,7 @@ resource "local_file" "kubernetes_template" {
                 match_expressions {
                   key = "cloud.google.com/gke-nodepool"
                   operator = "In"
-                  values = ["${google_container_node_pool.workspaces.name}"]
+                  values = ["${var.kubernetes_nodepool_workspaces}"]
                 }
               }
             }
@@ -284,7 +257,7 @@ spec:
               - key: cloud.google.com/gke-nodepool
                 operator: In
                 values:
-                - ${google_container_node_pool.coder.name}
+                - ${var.kubernetes_nodepool_coder}
         podAntiAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
           - podAffinityTerm:
@@ -307,7 +280,7 @@ spec:
         - name: CODER_PROMETHEUS_ADDRESS
           value: 0.0.0.0:2112
         - name: CODER_ACCESS_URL
-          value: ${local.coder_url}
+          value: ${local.coder_access_url}
         - name: CODER_CACHE_DIRECTORY
           value: /tmp/coder
         - name: CODER_ENABLE_TELEMETRY
@@ -395,9 +368,9 @@ spec:
 }
 
 resource "null_resource" "provisionerd_deployment_apply" {
-  depends_on = [helm_release.coder-chart, local_file.provisionerd_deployment, null_resource.cluster_kubeconfig]
+  depends_on = [helm_release.coder-chart, local_file.provisionerd_deployment]
   triggers = {
-    kubeconfig_path = local.cluster_kubeconfig_path
+    kubeconfig_path = var.kubernetes_kubeconfig_path
     manifest_path   = local_file.provisionerd_deployment.filename
   }
   provisioner "local-exec" {
@@ -408,11 +381,11 @@ resource "null_resource" "provisionerd_deployment_apply" {
 }
 
 resource "local_file" "output_vars" {
-  filename = "${path.module}/../.coderv2/url"
-  content  = local.coder_url
+  filename = "${path.module}/../../.coderv2/url"
+  content  = local.coder_access_url
 }
 
 output "coder_url" {
   description = "URL of the Coder deployment"
-  value       = local.coder_url
+  value       = local.coder_access_url
 }
diff --git a/scaletest/terraform/main.tf b/scaletest/terraform/k8s/main.tf
similarity index 70%
rename from scaletest/terraform/main.tf
rename to scaletest/terraform/k8s/main.tf
index c80478f513b64..a5c8c1085a5ce 100644
--- a/scaletest/terraform/main.tf
+++ b/scaletest/terraform/k8s/main.tf
@@ -1,10 +1,5 @@
 terraform {
   required_providers {
-    google = {
-      source  = "hashicorp/google"
-      version = "~> 4.36"
-    }
-
     kubernetes = {
       source  = "hashicorp/kubernetes"
       version = "~> 2.20"
@@ -24,12 +19,17 @@ terraform {
       source  = "hashicorp/tls"
       version = "~> 4.0"
     }
-
-    docker = {
-      source  = "kreuzwerker/docker"
-      version = "~> 3.0"
-    }
   }
 
   required_version = "~> 1.5.0"
 }
+
+provider "kubernetes" {
+  config_path = var.kubernetes_kubeconfig_path
+}
+
+provider "helm" {
+  kubernetes {
+    config_path = var.kubernetes_kubeconfig_path
+  }
+}
diff --git a/scaletest/terraform/prometheus.tf b/scaletest/terraform/k8s/prometheus.tf
similarity index 86%
rename from scaletest/terraform/prometheus.tf
rename to scaletest/terraform/k8s/prometheus.tf
index ade1289e38739..cd1459084c000 100644
--- a/scaletest/terraform/prometheus.tf
+++ b/scaletest/terraform/k8s/prometheus.tf
@@ -13,11 +13,9 @@ locals {
 resource "null_resource" "prometheus_namespace" {
   triggers = {
     namespace       = local.prometheus_namespace
-    kubeconfig_path = local.cluster_kubeconfig_path
+    kubeconfig_path = var.kubernetes_kubeconfig_path
   }
-  depends_on = [
-    google_container_node_pool.misc
-  ]
+  depends_on = []
   provisioner "local-exec" {
     when    = create
     command = <<EOF
@@ -64,7 +62,7 @@ blackboxExporter:
         - matchExpressions:
           - key: "cloud.google.com/gke-nodepool"
             operator: "In"
-            values: ["${google_container_node_pool.misc.name}"]
+            values: ["${var.kubernetes_nodepool_misc}"]
 operator:
   affinity:
     nodeAffinity:
@@ -73,7 +71,7 @@ operator:
         - matchExpressions:
           - key: "cloud.google.com/gke-nodepool"
             operator: "In"
-            values: ["${google_container_node_pool.misc.name}"]
+            values: ["${var.kubernetes_nodepool_misc}"]
 prometheus:
   affinity:
     nodeAffinity:
@@ -82,9 +80,9 @@ prometheus:
         - matchExpressions:
           - key: "cloud.google.com/gke-nodepool"
             operator: "In"
-            values: ["${google_container_node_pool.misc.name}"]
+            values: ["${var.kubernetes_nodepool_misc}"]
   externalLabels:
-    cluster: "${google_container_cluster.primary.name}"
+    cluster: "${var.prometheus_external_label_cluster}"
   persistence:
     enabled: true
     storageClass: standard
@@ -119,8 +117,8 @@ resource "kubernetes_secret" "prometheus-postgres-password" {
   }
   depends_on = [null_resource.prometheus_namespace]
   data = {
-    username = google_sql_user.prometheus.name
-    password = google_sql_user.prometheus.password
+    username = var.prometheus_postgres_user
+    password = var.prometheus_postgres_password
   }
 }
 
@@ -139,12 +137,12 @@ affinity:
     - matchExpressions:
       - key: "cloud.google.com/gke-nodepool"
         operator: "In"
-        values: ["${google_container_node_pool.misc.name}"]
+        values: ["${var.kubernetes_nodepool_misc}"]
 config:
   datasource:
-    host: "${google_sql_database_instance.db.private_ip_address}"
-    user: "${google_sql_user.prometheus.name}"
-    database: "${google_sql_database.coder.name}"
+    host: "${var.prometheus_postgres_host}"
+    user: "${var.prometheus_postgres_user}"
+    database: "${var.prometheus_postgres_dbname}"
     passwordSecret:
       name: "${kubernetes_secret.prometheus-postgres-password.metadata.0.name}"
       key: password
@@ -182,7 +180,7 @@ resource "null_resource" "coder-monitoring-manifest_apply" {
   provisioner "local-exec" {
     working_dir = "${abspath(path.module)}/../.coderv2"
     command     = <<EOF
-KUBECONFIG=${local.cluster_kubeconfig_path} kubectl apply -f ${abspath(local_file.coder-monitoring-manifest.filename)}
+KUBECONFIG=${var.kubernetes_kubeconfig_path} kubectl apply -f ${abspath(local_file.coder-monitoring-manifest.filename)}
     EOF
   }
   depends_on = [helm_release.prometheus-chart]
diff --git a/scaletest/terraform/vars.tf b/scaletest/terraform/k8s/vars.tf
similarity index 54%
rename from scaletest/terraform/vars.tf
rename to scaletest/terraform/k8s/vars.tf
index aa404ee614684..e7fc9007d8f53 100644
--- a/scaletest/terraform/vars.tf
+++ b/scaletest/terraform/k8s/vars.tf
@@ -7,45 +7,61 @@ variable "state" {
   default = "started"
 }
 
-variable "project_id" {
-  description = "The project in which to provision resources"
-}
+# variable "project_id" {
+#   description = "The project in which to provision resources"
+# }
 
 variable "name" {
   description = "Adds a prefix to resources."
 }
 
-variable "region" {
-  description = "GCP region in which to provision resources."
-  default     = "us-east1"
+variable "kubernetes_kubeconfig_path" {
+  description = "Path to kubeconfig to use to provision resources."
 }
 
-variable "zone" {
-  description = "GCP zone in which to provision resources."
-  default     = "us-east1-c"
+variable "kubernetes_nodepool_coder" {
+  description = "Name of the nodepool on which to run Coder."
 }
 
-variable "k8s_version" {
-  description = "Kubernetes vversion to provision."
-  default     = "1.24"
+variable "kubernetes_nodepool_workspaces" {
+  description = "Name of the nodepool on which to run workspaces."
 }
 
-variable "node_disk_size_gb" {
-  description = "Size of the root disk for cluster nodes."
-  default     = 100
+variable "kubernetes_nodepool_misc" {
+  description = "Name of the nodepool on which to run everything else."
 }
 
-variable "node_image_type" {
-  description = "Image type to use for cluster nodes."
-  default     = "cos_containerd"
-}
+# variable "region" {
+#   description = "GCP region in which to provision resources."
+#   default     = "us-east1"
+# }
 
-// Preemptible nodes are way cheaper, but can be pulled out
-// from under you at any time. Caveat emptor.
-variable "node_preemptible" {
-  description = "Use preemptible nodes."
-  default     = false
-}
+# variable "zone" {
+#   description = "GCP zone in which to provision resources."
+#   default     = "us-east1-c"
+# }
+
+# variable "k8s_version" {
+#   description = "Kubernetes vversion to provision."
+#   default     = "1.24"
+# }
+
+# variable "node_disk_size_gb" {
+#   description = "Size of the root disk for cluster nodes."
+#   default     = 100
+# }
+
+# variable "node_image_type" {
+#   description = "Image type to use for cluster nodes."
+#   default     = "cos_containerd"
+# }
+
+# // Preemptible nodes are way cheaper, but can be pulled out
+# // from under you at any time. Caveat emptor.
+# variable "node_preemptible" {
+#   description = "Use preemptible nodes."
+#   default     = false
+# }
 
 // We create three nodepools:
 // - One for the Coder control plane
@@ -53,58 +69,67 @@ variable "node_preemptible" {
 // - One for everything else (for example, load generation)
 
 // These variables control the node pool dedicated to Coder.
-variable "nodepool_machine_type_coder" {
-  description = "Machine type to use for Coder control plane nodepool."
-  default     = "t2d-standard-4"
-}
-
-variable "nodepool_size_coder" {
-  description = "Number of cluster nodes for the Coder control plane nodepool."
-  default     = 1
-}
-
-// These variables control the node pool dedicated to workspaces.
-variable "nodepool_machine_type_workspaces" {
-  description = "Machine type to use for the workspaces nodepool."
-  default     = "t2d-standard-4"
-}
-
-variable "nodepool_size_workspaces" {
-  description = "Number of cluster nodes for the workspaces nodepool."
-  default     = 1
-}
-
-// These variables control the node pool for everything else.
-variable "nodepool_machine_type_misc" {
-  description = "Machine type to use for the misc nodepool."
-  default     = "t2d-standard-4"
-}
+# variable "nodepool_machine_type_coder" {
+#   description = "Machine type to use for Coder control plane nodepool."
+#   default     = "t2d-standard-4"
+# }
+
+# variable "nodepool_size_coder" {
+#   description = "Number of cluster nodes for the Coder control plane nodepool."
+#   default     = 1
+# }
+
+# // These variables control the node pool dedicated to workspaces.
+# variable "nodepool_machine_type_workspaces" {
+#   description = "Machine type to use for the workspaces nodepool."
+#   default     = "t2d-standard-4"
+# }
+
+# variable "nodepool_size_workspaces" {
+#   description = "Number of cluster nodes for the workspaces nodepool."
+#   default     = 1
+# }
+
+# // These variables control the node pool for everything else.
+# variable "nodepool_machine_type_misc" {
+#   description = "Machine type to use for the misc nodepool."
+#   default     = "t2d-standard-4"
+# }
+
+# variable "nodepool_size_misc" {
+#   description = "Number of cluster nodes for the misc nodepool."
+#   default     = 1
+# }
+
+# // These variables control the size of the database to be used by Coder.
+# variable "cloudsql_version" {
+#   description = "CloudSQL version to provision"
+#   default     = "POSTGRES_14"
+# }
+
+# variable "cloudsql_tier" {
+#   description = "CloudSQL database tier."
+#   default     = "db-f1-micro"
+# }
+
+# variable "cloudsql_max_connections" {
+#   description = "CloudSQL database max_connections"
+#   default     = 500
+# }
 
-variable "nodepool_size_misc" {
-  description = "Number of cluster nodes for the misc nodepool."
+// These variables control the Coder deployment.
+variable "coder_replicas" {
+  description = "Number of Coder replicas to provision."
   default     = 1
 }
 
-// These variables control the size of the database to be used by Coder.
-variable "cloudsql_version" {
-  description = "CloudSQL version to provision"
-  default     = "POSTGRES_14"
-}
-
-variable "cloudsql_tier" {
-  description = "CloudSQL database tier."
-  default     = "db-f1-micro"
-}
-
-variable "cloudsql_max_connections" {
-  description = "CloudSQL database max_connections"
-  default     = 500
+variable "coder_address" {
+  description = "IP address to use for Coder service."
 }
 
-// These variables control the Coder deployment.
-variable "coder_replicas" {
-  description = "Number of Coder replicas to provision."
-  default     = 1
+variable "coder_db_url" {
+  description = "URL of the database for Coder to use."
+  sensitive = true
 }
 
 // Ensure that requests allow for at least two replicas to be scheduled
@@ -208,6 +233,27 @@ variable "workspace_mem_limit" {
 }
 
 // These variables control the Prometheus deployment.
+variable "prometheus_external_label_cluster" {
+  description = "Value for the Prometheus external label named cluster."
+}
+
+variable "prometheus_postgres_dbname" {
+  description = "Database for Postgres to monitor."
+}
+
+variable "prometheus_postgres_host" {
+  description = "Database hostname for Prometheus."
+}
+
+variable "prometheus_postgres_password" {
+  description = "Postgres password for Prometheus."
+  sensitive = true
+}
+
+variable "prometheus_postgres_user" {
+  description = "Postgres username for Prometheus."
+}
+
 variable "prometheus_remote_write_user" {
   description = "Username for Prometheus remote write."
   default     = ""
@@ -216,6 +262,7 @@ variable "prometheus_remote_write_user" {
 variable "prometheus_remote_write_password" {
   description = "Password for Prometheus remote write."
   default     = ""
+  sensitive = true
 }
 
 variable "prometheus_remote_write_url" {
diff --git a/scaletest/terraform/scenario-test.tfvars b/scaletest/terraform/scenario-test.tfvars
new file mode 100644
index 0000000000000..b35ae40b23742
--- /dev/null
+++ b/scaletest/terraform/scenario-test.tfvars
@@ -0,0 +1,9 @@
+nodepool_machine_type_coder      = "t2d-standard-2"
+nodepool_size_coder              = 1
+nodepool_machine_type_workspaces = "t2d-standard-2"
+cloudsql_tier                    = "db-custom-1-3840"
+coder_cpu_request                = "100m"
+coder_mem_request                = "2Gi"
+coder_cpu_limit                  = "1000m"
+coder_mem_limit                  = "2Gi"
+coder_replicas                   = 1

From 7ba1cd5e0227cce5a79a7e22c295ffa23e817bb6 Mon Sep 17 00:00:00 2001
From: Cian Johnston <cian@coder.com>
Date: Fri, 22 Sep 2023 14:42:28 +0100
Subject: [PATCH 2/5] fmt

---
 scaletest/terraform/infra/gcp_cluster.tf |  6 ++---
 scaletest/terraform/infra/outputs.tf     | 28 ++++++++++++------------
 scaletest/terraform/k8s/vars.tf          |  6 ++---
 3 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/scaletest/terraform/infra/gcp_cluster.tf b/scaletest/terraform/infra/gcp_cluster.tf
index cb744545eee1f..577894790d7ff 100644
--- a/scaletest/terraform/infra/gcp_cluster.tf
+++ b/scaletest/terraform/infra/gcp_cluster.tf
@@ -3,9 +3,9 @@ data "google_compute_default_service_account" "default" {
 }
 
 locals {
-  abs_module_path = "${abspath(path.module)}"
-  rel_kubeconfig_path = "../../.coderv2/${var.name}-cluster.kubeconfig"
-  cluster_kubeconfig_path = "${abspath("${local.abs_module_path}/${local.rel_kubeconfig_path}")}"
+  abs_module_path         = abspath(path.module)
+  rel_kubeconfig_path     = "../../.coderv2/${var.name}-cluster.kubeconfig"
+  cluster_kubeconfig_path = abspath("${local.abs_module_path}/${local.rel_kubeconfig_path}")
 }
 
 resource "google_container_cluster" "primary" {
diff --git a/scaletest/terraform/infra/outputs.tf b/scaletest/terraform/infra/outputs.tf
index 812283d52db35..f5e619eca384d 100644
--- a/scaletest/terraform/infra/outputs.tf
+++ b/scaletest/terraform/infra/outputs.tf
@@ -1,63 +1,63 @@
 output "coder_db_url" {
   description = "URL of the database for Coder."
-  value = local.coder_db_url
-  sensitive = true
+  value       = local.coder_db_url
+  sensitive   = true
 }
 
 output "coder_address" {
   description = "IP address to use for the Coder service."
-  value = google_compute_address.coder.address
+  value       = google_compute_address.coder.address
 }
 
 output "kubernetes_kubeconfig_path" {
   description = "Kubeconfig path."
-  value = local.cluster_kubeconfig_path
+  value       = local.cluster_kubeconfig_path
 }
 
 output "kubernetes_nodepool_coder" {
   description = "Name of the nodepool on which to run Coder."
-  value = google_container_node_pool.coder.name
+  value       = google_container_node_pool.coder.name
 }
 
 output "kubernetes_nodepool_misc" {
   description = "Name of the nodepool on which to run everything else."
-  value = google_container_node_pool.misc.name
+  value       = google_container_node_pool.misc.name
 }
 
 output "kubernetes_nodepool_workspaces" {
   description = "Name of the nodepool on which to run workspaces."
-  value = google_container_node_pool.workspaces.name
+  value       = google_container_node_pool.workspaces.name
 }
 
 output "prometheus_external_label_cluster" {
   description = "Value for the Prometheus external label named cluster."
-  value = google_container_cluster.primary.name
+  value       = google_container_cluster.primary.name
 }
 
 output "prometheus_postgres_dbname" {
   description = "Name of the database for Prometheus to monitor."
-  value = google_sql_database.coder.name
+  value       = google_sql_database.coder.name
 }
 
 output "prometheus_postgres_host" {
   description = "Hostname of the database for Prometheus to connect to."
-  value = google_sql_database_instance.db.private_ip_address
+  value       = google_sql_database_instance.db.private_ip_address
 }
 
 output "prometheus_postgres_password" {
   description = "Postgres password for Prometheus."
-  value = random_password.prometheus-postgres-password.result
-  sensitive = true
+  value       = random_password.prometheus-postgres-password.result
+  sensitive   = true
 }
 
 output "prometheus_postgres_user" {
   description = "Postgres username for Prometheus."
-  value = google_sql_user.prometheus.name
+  value       = google_sql_user.prometheus.name
 }
 
 resource "local_file" "outputs" {
   filename = "${path.module}/../../.coderv2/infra_outputs.tfvars"
-  content = <<EOF
+  content  = <<EOF
   coder_db_url = "${local.coder_db_url}"
   coder_address = "${google_compute_address.coder.address}"
   kubernetes_kubeconfig_path = "${local.cluster_kubeconfig_path}"
diff --git a/scaletest/terraform/k8s/vars.tf b/scaletest/terraform/k8s/vars.tf
index e7fc9007d8f53..b135c98bbc9c1 100644
--- a/scaletest/terraform/k8s/vars.tf
+++ b/scaletest/terraform/k8s/vars.tf
@@ -129,7 +129,7 @@ variable "coder_address" {
 
 variable "coder_db_url" {
   description = "URL of the database for Coder to use."
-  sensitive = true
+  sensitive   = true
 }
 
 // Ensure that requests allow for at least two replicas to be scheduled
@@ -247,7 +247,7 @@ variable "prometheus_postgres_host" {
 
 variable "prometheus_postgres_password" {
   description = "Postgres password for Prometheus."
-  sensitive = true
+  sensitive   = true
 }
 
 variable "prometheus_postgres_user" {
@@ -262,7 +262,7 @@ variable "prometheus_remote_write_user" {
 variable "prometheus_remote_write_password" {
   description = "Password for Prometheus remote write."
   default     = ""
-  sensitive = true
+  sensitive   = true
 }
 
 variable "prometheus_remote_write_url" {

From 6abe5e4745594c085bfc92be1904131492ce2943 Mon Sep 17 00:00:00 2001
From: Cian Johnston <cian@coder.com>
Date: Fri, 22 Sep 2023 14:43:08 +0100
Subject: [PATCH 3/5] rm unnecessary file

---
 scaletest/terraform/scenario-test.tfvars | 9 ---------
 1 file changed, 9 deletions(-)
 delete mode 100644 scaletest/terraform/scenario-test.tfvars

diff --git a/scaletest/terraform/scenario-test.tfvars b/scaletest/terraform/scenario-test.tfvars
deleted file mode 100644
index b35ae40b23742..0000000000000
--- a/scaletest/terraform/scenario-test.tfvars
+++ /dev/null
@@ -1,9 +0,0 @@
-nodepool_machine_type_coder      = "t2d-standard-2"
-nodepool_size_coder              = 1
-nodepool_machine_type_workspaces = "t2d-standard-2"
-cloudsql_tier                    = "db-custom-1-3840"
-coder_cpu_request                = "100m"
-coder_mem_request                = "2Gi"
-coder_cpu_limit                  = "1000m"
-coder_mem_limit                  = "2Gi"
-coder_replicas                   = 1

From 50f84ec89e5cf0aa71727beb3043d4bd84160acd Mon Sep 17 00:00:00 2001
From: Cian Johnston <cian@coder.com>
Date: Fri, 22 Sep 2023 14:44:14 +0100
Subject: [PATCH 4/5] rm unused vars

---
 scaletest/terraform/k8s/vars.tf | 90 ---------------------------------
 1 file changed, 90 deletions(-)

diff --git a/scaletest/terraform/k8s/vars.tf b/scaletest/terraform/k8s/vars.tf
index b135c98bbc9c1..5493e64739843 100644
--- a/scaletest/terraform/k8s/vars.tf
+++ b/scaletest/terraform/k8s/vars.tf
@@ -7,10 +7,6 @@ variable "state" {
   default = "started"
 }
 
-# variable "project_id" {
-#   description = "The project in which to provision resources"
-# }
-
 variable "name" {
   description = "Adds a prefix to resources."
 }
@@ -31,92 +27,6 @@ variable "kubernetes_nodepool_misc" {
   description = "Name of the nodepool on which to run everything else."
 }
 
-# variable "region" {
-#   description = "GCP region in which to provision resources."
-#   default     = "us-east1"
-# }
-
-# variable "zone" {
-#   description = "GCP zone in which to provision resources."
-#   default     = "us-east1-c"
-# }
-
-# variable "k8s_version" {
-#   description = "Kubernetes vversion to provision."
-#   default     = "1.24"
-# }
-
-# variable "node_disk_size_gb" {
-#   description = "Size of the root disk for cluster nodes."
-#   default     = 100
-# }
-
-# variable "node_image_type" {
-#   description = "Image type to use for cluster nodes."
-#   default     = "cos_containerd"
-# }
-
-# // Preemptible nodes are way cheaper, but can be pulled out
-# // from under you at any time. Caveat emptor.
-# variable "node_preemptible" {
-#   description = "Use preemptible nodes."
-#   default     = false
-# }
-
-// We create three nodepools:
-// - One for the Coder control plane
-// - One for workspaces
-// - One for everything else (for example, load generation)
-
-// These variables control the node pool dedicated to Coder.
-# variable "nodepool_machine_type_coder" {
-#   description = "Machine type to use for Coder control plane nodepool."
-#   default     = "t2d-standard-4"
-# }
-
-# variable "nodepool_size_coder" {
-#   description = "Number of cluster nodes for the Coder control plane nodepool."
-#   default     = 1
-# }
-
-# // These variables control the node pool dedicated to workspaces.
-# variable "nodepool_machine_type_workspaces" {
-#   description = "Machine type to use for the workspaces nodepool."
-#   default     = "t2d-standard-4"
-# }
-
-# variable "nodepool_size_workspaces" {
-#   description = "Number of cluster nodes for the workspaces nodepool."
-#   default     = 1
-# }
-
-# // These variables control the node pool for everything else.
-# variable "nodepool_machine_type_misc" {
-#   description = "Machine type to use for the misc nodepool."
-#   default     = "t2d-standard-4"
-# }
-
-# variable "nodepool_size_misc" {
-#   description = "Number of cluster nodes for the misc nodepool."
-#   default     = 1
-# }
-
-# // These variables control the size of the database to be used by Coder.
-# variable "cloudsql_version" {
-#   description = "CloudSQL version to provision"
-#   default     = "POSTGRES_14"
-# }
-
-# variable "cloudsql_tier" {
-#   description = "CloudSQL database tier."
-#   default     = "db-f1-micro"
-# }
-
-# variable "cloudsql_max_connections" {
-#   description = "CloudSQL database max_connections"
-#   default     = 500
-# }
-
 // These variables control the Coder deployment.
 variable "coder_replicas" {
   description = "Number of Coder replicas to provision."

From 6cee2afe95e0eb657ed2fba7ba0a5c1038b61996 Mon Sep 17 00:00:00 2001
From: Cian Johnston <cian@coder.com>
Date: Fri, 22 Sep 2023 15:06:23 +0100
Subject: [PATCH 5/5] fix typo

---
 scaletest/terraform/infra/vars.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scaletest/terraform/infra/vars.tf b/scaletest/terraform/infra/vars.tf
index e93a4e9b7ab02..e26e5fa54f7df 100644
--- a/scaletest/terraform/infra/vars.tf
+++ b/scaletest/terraform/infra/vars.tf
@@ -26,7 +26,7 @@ variable "zone" {
 }
 
 variable "k8s_version" {
-  description = "Kubernetes vversion to provision."
+  description = "Kubernetes version to provision."
   default     = "1.24"
 }