From 29016dc9397cf97c099d7ff093aeb615698b66ee Mon Sep 17 00:00:00 2001
From: Cian Johnston <cian@coder.com>
Date: Fri, 5 May 2023 17:35:31 +0100
Subject: [PATCH 01/26] Port over initial version of v1 loadtest infra tf code

---
 .gitignore                         |   4 +
 scaletest/README.md                |   3 +
 scaletest/terraform/coder.tf       | 151 +++++++++++++++++++++++++++++
 scaletest/terraform/gcp_cluster.tf | 127 ++++++++++++++++++++++++
 scaletest/terraform/gcp_db.tf      |  49 ++++++++++
 scaletest/terraform/gcp_project.tf |  32 ++++++
 scaletest/terraform/gcp_vpc.tf     |  24 +++++
 scaletest/terraform/main.tf        |  30 ++++++
 scaletest/terraform/vars.tf        | 119 +++++++++++++++++++++++
 9 files changed, 539 insertions(+)
 create mode 100644 scaletest/README.md
 create mode 100644 scaletest/terraform/coder.tf
 create mode 100644 scaletest/terraform/gcp_cluster.tf
 create mode 100644 scaletest/terraform/gcp_db.tf
 create mode 100644 scaletest/terraform/gcp_project.tf
 create mode 100644 scaletest/terraform/gcp_vpc.tf
 create mode 100644 scaletest/terraform/main.tf
 create mode 100644 scaletest/terraform/vars.tf

diff --git a/.gitignore b/.gitignore
index 9a565462e8a4f..6b530e25cc3a9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -54,3 +54,7 @@ site/stats/
 # direnv
 .envrc
 *.test
+
+# Loadtesting
+./scaletest/terraform/.terraform
+./scaletest/terraform/.terraform.lock.hcl
diff --git a/scaletest/README.md b/scaletest/README.md
new file mode 100644
index 0000000000000..f46c66f797538
--- /dev/null
+++ b/scaletest/README.md
@@ -0,0 +1,3 @@
+# Load Testing
+
+TODO: write something here.
diff --git a/scaletest/terraform/coder.tf b/scaletest/terraform/coder.tf
new file mode 100644
index 0000000000000..82160c943aa3b
--- /dev/null
+++ b/scaletest/terraform/coder.tf
@@ -0,0 +1,151 @@
+data "google_client_config" "default" {}
+
+locals {
+  coder_helm_repo    = "https://helm.coder.com/v2"
+  coder_helm_chart   = "coder"
+  coder_release_name = "coder-${var.name}"
+  coder_namespace    = "coder-${var.name}"
+}
+
+provider "kubernetes" {
+  host                   = "https://${google_container_cluster.primary.endpoint}"
+  cluster_ca_certificate = base64decode(google_container_cluster.primary.master_auth.0.cluster_ca_certificate)
+  token                  = data.google_client_config.default.access_token
+}
+
+provider "helm" {
+  kubernetes {
+    host                   = "https://${google_container_cluster.primary.endpoint}"
+    cluster_ca_certificate = base64decode(google_container_cluster.primary.master_auth.0.cluster_ca_certificate)
+    token                  = data.google_client_config.default.access_token
+  }
+}
+
+resource "kubernetes_namespace" "coder_namespace" {
+  metadata {
+    name = local.coder_namespace
+  }
+  depends_on = [
+    google_container_node_pool.coder
+  ]
+}
+
+resource "random_password" "postgres-admin-password" {
+  length = 12
+}
+
+resource "random_password" "coder-postgres-password" {
+  length = 12
+}
+
+resource "kubernetes_secret" "coder-db" {
+  type = "kubernetes.io/basic-auth"
+  metadata {
+    name      = "coder-db-url"
+    namespace = kubernetes_namespace.coder_namespace.metadata.0.name
+  }
+  data = {
+    url = "postgres://coder:${random_password.coder-postgres-password.result}@/${google_sql_database_instance.db.ip_address}?sslmode=disable"
+  }
+}
+
+resource "tls_private_key" "coder" {
+  algorithm = "ED25519"
+}
+
+resource "tls_self_signed_cert" "coder" {
+  private_key_pem = tls_private_key.coder.private_key_pem
+
+  subject {
+    common_name = "${local.coder_release_name}.${local.coder_namespace}.svc.cluster.local"
+  }
+
+  allowed_uses = ["server_auth", "digital_signature", "data_encipherment", "key_agreement", "key_encipherment"]
+
+  # 1 year
+  validity_period_hours = 8760
+
+  dns_names = [
+    "${local.coder_release_name}.${local.coder_namespace}.svc.cluster.local",
+    "${local.coder_release_name}.${local.coder_namespace}",
+    "${local.coder_release_name}",
+  ]
+}
+
+resource "kubernetes_secret" "coder-tls" {
+  type = "kubernetes.io/tls"
+  metadata {
+    name      = "coder-tls"
+    namespace = kubernetes_namespace.coder_namespace.metadata.0.name
+  }
+
+  data = {
+    "tls.crt" = tls_self_signed_cert.coder.cert_pem
+    "tls.key" = tls_private_key.coder.private_key_pem
+  }
+}
+
+resource "kubernetes_secret" "coder-ca" {
+  type = "Opaque"
+  metadata {
+    name      = "coder-ca"
+    namespace = kubernetes_namespace.coder_namespace.metadata.0.name
+  }
+  data = {
+    "ca.crt" = "${tls_self_signed_cert.coder.cert_pem}"
+  }
+}
+
+resource "helm_release" "coder-chart" {
+  repository = local.coder_helm_repo
+  chart      = local.coder_helm_chart
+  name       = local.coder_release_name
+  version    = var.coder_chart_version
+  namespace  = kubernetes_namespace.coder_namespace.metadata.0.name
+  depends_on = [
+    google_container_node_pool.coder,
+  ]
+  values = [<<EOF
+coder:
+  env:
+    - name: "CODER_CACHE_DIRECTORY"
+      value: "/tmp/coder"
+    - name: "CODER_LOGGING_HUMAN"
+      value: "/dev/null"
+    - name: "CODER_LOGGING_STACKDRIVER"
+      value: "/dev/stderr"
+    - name: "CODER_PG_CONNECTION_URL"
+      valueFrom:
+        secretKeyRef:
+          name: "${kubernetes_secret.coder-db.metadata.0.name}"
+          key: url
+    - name: "CODER_VERBOSE"
+      value: "true"
+  image: ${var.coder_image_repo}:${var.coder_image_tag}
+  replicaCount: "${var.coder_replicas}"
+  resources:
+    requests:
+      cpu: "${var.coder_cpu}"
+      memory: "${var.coder_mem}"
+    limits:
+      cpu: "${var.coder_cpu}"
+      memory: "${var.coder_mem}"
+  securityContext:
+    readOnlyRootFilesystem: true
+  service:
+    enable: true
+  tls:
+    secretNames:
+    - "${kubernetes_secret.coder-tls.metadata.0.name}"
+  volumeMounts:
+  - mountPath: "/tmp"
+    name: cache
+    readOnly: false
+  volumes:
+  - emptyDir:
+      sizeLimit: 1024Mi
+    name: cache
+
+EOF
+  ]
+}
diff --git a/scaletest/terraform/gcp_cluster.tf b/scaletest/terraform/gcp_cluster.tf
new file mode 100644
index 0000000000000..3a57d535762cf
--- /dev/null
+++ b/scaletest/terraform/gcp_cluster.tf
@@ -0,0 +1,127 @@
+data "google_compute_default_service_account" "default" {
+  project = var.project_id
+}
+
+resource "google_container_cluster" "primary" {
+  name                     = "${var.name}-cluster"
+  location                 = var.zone
+  project                  = var.project_id
+  network                  = google_compute_network.vpc.name
+  subnetwork               = google_compute_subnetwork.subnet.name
+  initial_node_count       = 1
+  remove_default_node_pool = true
+  network_policy {
+    enabled = true
+  }
+  depends_on = [
+    google_project_service.api["container.googleapis.com"]
+  ]
+  monitoring_config {
+    enable_components = []
+    managed_prometheus {
+      enabled = true
+    }
+  }
+}
+
+resource "google_container_node_pool" "coder" {
+  name       = "${var.name}-node-pool-coder"
+  location   = var.zone
+  project    = var.project_id
+  cluster    = google_container_cluster.primary.name
+  node_count = var.nodepool_size_coder
+  node_config {
+    oauth_scopes = [
+      "https://www.googleapis.com/auth/logging.write",
+      "https://www.googleapis.com/auth/monitoring",
+      "https://www.googleapis.com/auth/trace.append",
+      "https://www.googleapis.com/auth/devstorage.read_only",
+      "https://www.googleapis.com/auth/service.management.readonly",
+      "https://www.googleapis.com/auth/servicecontrol",
+    ]
+    disk_size_gb    = var.node_disk_size_gb
+    machine_type    = var.nodepool_machine_type_coder
+    image_type      = var.node_image_type
+    preemptible     = var.node_preemptible
+    service_account = data.google_compute_default_service_account.default.email
+    tags            = ["gke-node", "${var.project_id}-gke"]
+    labels = {
+      env = var.project_id
+    }
+    metadata = {
+      disable-legacy-endpoints = "true"
+    }
+  }
+
+  depends_on = [
+    google_container_cluster.primary
+  ]
+}
+
+resource "google_container_node_pool" "workspaces" {
+  name       = "${var.name}-node-pool-workspaces"
+  location   = var.zone
+  project    = var.project_id
+  cluster    = google_container_cluster.primary.name
+  node_count = var.nodepool_size_workspaces
+  node_config {
+    oauth_scopes = [
+      "https://www.googleapis.com/auth/logging.write",
+      "https://www.googleapis.com/auth/monitoring",
+      "https://www.googleapis.com/auth/trace.append",
+      "https://www.googleapis.com/auth/devstorage.read_only",
+      "https://www.googleapis.com/auth/service.management.readonly",
+      "https://www.googleapis.com/auth/servicecontrol",
+    ]
+    disk_size_gb    = var.node_disk_size_gb
+    machine_type    = var.nodepool_machine_type_workspaces
+    image_type      = var.node_image_type
+    preemptible     = var.node_preemptible
+    service_account = data.google_compute_default_service_account.default.email
+    tags            = ["gke-node", "${var.project_id}-gke"]
+    labels = {
+      env = var.project_id
+    }
+    metadata = {
+      disable-legacy-endpoints = "true"
+    }
+  }
+
+  depends_on = [
+    google_container_cluster.primary
+  ]
+}
+
+resource "google_container_node_pool" "misc" {
+  name       = "${var.name}-node-pool-misc"
+  location   = var.zone
+  project    = var.project_id
+  cluster    = google_container_cluster.primary.name
+  node_count = var.nodepool_size_misc
+  node_config {
+    oauth_scopes = [
+      "https://www.googleapis.com/auth/logging.write",
+      "https://www.googleapis.com/auth/monitoring",
+      "https://www.googleapis.com/auth/trace.append",
+      "https://www.googleapis.com/auth/devstorage.read_only",
+      "https://www.googleapis.com/auth/service.management.readonly",
+      "https://www.googleapis.com/auth/servicecontrol",
+    ]
+    disk_size_gb    = var.node_disk_size_gb
+    machine_type    = var.nodepool_machine_type_misc
+    image_type      = var.node_image_type
+    preemptible     = var.node_preemptible
+    service_account = data.google_compute_default_service_account.default.email
+    tags            = ["gke-node", "${var.project_id}-gke"]
+    labels = {
+      env = var.project_id
+    }
+    metadata = {
+      disable-legacy-endpoints = "true"
+    }
+  }
+
+  depends_on = [
+    google_container_cluster.primary
+  ]
+}
diff --git a/scaletest/terraform/gcp_db.tf b/scaletest/terraform/gcp_db.tf
new file mode 100644
index 0000000000000..3a727c92041de
--- /dev/null
+++ b/scaletest/terraform/gcp_db.tf
@@ -0,0 +1,49 @@
+data "google_compute_network" "default" {
+  project = var.project_id
+  name    = "default"
+}
+
+data "google_compute_global_address" "sql_peering" {
+  name = "sql-ip-address"
+}
+
+resource "google_service_networking_connection" "private_vpc_connection" {
+  network                 = data.google_compute_network.default.id
+  service                 = "servicenetworking.googleapis.com"
+  reserved_peering_ranges = [google_compute_global_address.sql_peering.name]
+}
+
+resource "google_sql_database_instance" "db" {
+  name             = "${var.name}-db"
+  region           = var.region
+  database_version = var.cloudsql_version
+
+  depends_on = [google_service_networking_connection.private_vpc_connection]
+
+  settings {
+    tier              = var.cloudsql_tier
+    activation_policy = "ALWAYS"
+    availability_type = "ZONAL"
+
+    location_preference {
+      zone = var.zone
+    }
+
+    database_flags {
+      name  = "max_connections"
+      value = "500"
+    }
+
+    ip_configuration {
+      ipv4_enabled    = false
+      private_network = data.google_compute_network.default.id
+    }
+
+    insights_config {
+      query_insights_enabled  = true
+      query_string_length     = 1024
+      record_application_tags = false
+      record_client_address   = false
+    }
+  }
+}
diff --git a/scaletest/terraform/gcp_project.tf b/scaletest/terraform/gcp_project.tf
new file mode 100644
index 0000000000000..c233042e66acb
--- /dev/null
+++ b/scaletest/terraform/gcp_project.tf
@@ -0,0 +1,32 @@
+provider "google" {
+  region  = var.region
+  project = var.project_id
+}
+
+locals {
+  project_apis = [
+    "cloudtrace",
+    "compute",
+    "container",
+    "logging",
+    "monitoring",
+    "servicemanagement",
+    "servicenetworking",
+    "sqladmin",
+    "stackdriver",
+    "storage-api",
+  ]
+}
+
+data "google_project" "project" {
+  project_id = var.project_id
+}
+
+resource "google_project_service" "api" {
+  for_each = toset(local.project_apis)
+  project  = data.google_project.project.project_id
+  service  = "${each.value}.googleapis.com"
+
+  disable_dependent_services = false
+  disable_on_destroy         = false
+}
diff --git a/scaletest/terraform/gcp_vpc.tf b/scaletest/terraform/gcp_vpc.tf
new file mode 100644
index 0000000000000..8923745be637a
--- /dev/null
+++ b/scaletest/terraform/gcp_vpc.tf
@@ -0,0 +1,24 @@
+resource "google_compute_network" "vpc" {
+  project                 = var.project_id
+  name                    = "${var.name}-vpc"
+  auto_create_subnetworks = "false"
+  depends_on = [
+    google_project_service.api["compute.googleapis.com"]
+  ]
+}
+
+resource "google_compute_subnetwork" "subnet" {
+  name          = "${var.name}-subnet"
+  project       = var.project_id
+  region        = var.region
+  network       = google_compute_network.vpc.name
+  ip_cidr_range = "10.10.0.0/24"
+}
+
+resource "google_compute_global_address" "sql_peering" {
+  name          = "${var.name}-sql-peering"
+  purpose       = "VPC_PEERING"
+  address_type  = "INTERNAL"
+  # prefix_length = 16
+  network       = google_compute_network.vpc.id
+}
diff --git a/scaletest/terraform/main.tf b/scaletest/terraform/main.tf
new file mode 100644
index 0000000000000..121e5f7602d2d
--- /dev/null
+++ b/scaletest/terraform/main.tf
@@ -0,0 +1,30 @@
+terraform {
+  required_providers {
+    google = {
+      source = "hashicorp/google"
+      version = "~> 4.36"
+    }
+
+    kubernetes = {
+      source = "hashicorp/kubernetes"
+      version = "~> 2.20"
+    }
+
+    helm = {
+      source = "hashicorp/helm"
+      version = "~> 2.9"
+    }
+
+    random = {
+      source = "hashicorp/random"
+      version = "~> 3.5"
+    }
+
+    tls = {
+      source  = "hashicorp/tls"
+      version = "~> 4.0"
+    }
+  }
+
+  required_version = "~> 1.4.0"
+}
diff --git a/scaletest/terraform/vars.tf b/scaletest/terraform/vars.tf
new file mode 100644
index 0000000000000..55d487ed3fb99
--- /dev/null
+++ b/scaletest/terraform/vars.tf
@@ -0,0 +1,119 @@
+variable "project_id" {
+  description = "The project in which to provision resources"
+}
+
+variable "name" {
+  description = "Adds a prefix to resources."
+}
+
+variable "region" {
+  description = "GCP region in which to provision resources."
+  default = "us-east1"
+}
+
+variable "zone" {
+  description = "GCP zone in which to provision resources."
+  default = "us-east1-c"
+}
+
+variable "k8s_version" {
+  description = "Kubernetes vversion to provision."
+  default = "1.24"
+}
+
+variable "node_disk_size_gb" {
+  description = "Size of the root disk for cluster nodes."
+  default = 100
+}
+
+variable "node_image_type" {
+  description = "Image type to use for cluster nodes."
+  default = "cos_containerd"
+}
+
+// Preemptible nodes are way cheaper, but can be pulled out
+// from under you at any time. Caveat emptor.
+variable "node_preemptible" {
+  description = "Use preemptible nodes."
+  default = false
+}
+
+// We create three nodepools:
+// - One for the Coder control plane
+// - One for workspaces
+// - One for everything else (for example, load generation)
+
+// These variables control the node pool dedicated to Coder.
+variable "nodepool_machine_type_coder" {
+  description = "Machine type to use for Coder control plane nodepool."
+  default = "t2d-standard-4"
+}
+
+variable "nodepool_size_coder" {
+  description = "Number of cluster nodes for the Coder control plane nodepool."
+  default = 1
+}
+
+// These variables control the node pool dedicated to workspaces.
+variable "nodepool_machine_type_workspaces" {
+  description = "Machine type to use for the workspaces nodepool."
+  default = "t2d-standard-4"
+}
+
+variable "nodepool_size_workspaces" {
+  description = "Number of cluster nodes for the workspaces nodepool."
+  default = 1
+}
+
+// These variables control the node pool for everything else.
+variable "nodepool_machine_type_misc" {
+  description = "Machine type to use for the misc nodepool."
+  default = "t2d-standard-4"
+}
+
+variable "nodepool_size_misc" {
+  description = "Number of cluster nodes for the misc nodepool."
+  default = 1
+}
+
+// These variables control the size of the database to be used by Coder.
+variable "cloudsql_version" {
+  description = "CloudSQL version to provision"
+  default = "POSTGRES_14"
+}
+
+variable "cloudsql_tier" {
+  description = "CloudSQL database tier."
+  default = "db-f1-micro"
+}
+
+// These variables control the Coder deployment.
+variable "coder_replicas" {
+  description = "Number of Coder replicas to provision"
+  default = 1
+}
+
+variable "coder_cpu" {
+  description = "CPU to allocate to Coder"
+  default = "1000m"
+}
+
+variable "coder_mem" {
+  description = "Memory to allocate to Coder"
+  default = "1024Mi"
+}
+
+variable "coder_chart_version" {
+  description = "Version of the Coder Helm chart to install. Defaults to latest."
+  default = null
+}
+
+variable "coder_image_repo" {
+  description = "Repository to use for Coder image."
+  default = "ghcr.io/coder/coder"
+}
+
+variable "coder_image_tag" {
+  description = "Tag to use for Coder image."
+  default = "latest"
+}

From 8759e23e9fc9fceaa9ec74bdc5238b048e7ab8db Mon Sep 17 00:00:00 2001
From: Cian Johnston <cian@coder.com>
Date: Mon, 8 May 2023 13:54:50 +0100
Subject: [PATCH 02/26] Fix VPC peering for CloudSQL

---
 scaletest/terraform/coder.tf   | 8 +++++---
 scaletest/terraform/gcp_db.tf  | 8 +-------
 scaletest/terraform/gcp_vpc.tf | 9 ++++++++-
 3 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/scaletest/terraform/coder.tf b/scaletest/terraform/coder.tf
index 82160c943aa3b..22ae75fa9d630 100644
--- a/scaletest/terraform/coder.tf
+++ b/scaletest/terraform/coder.tf
@@ -39,13 +39,13 @@ resource "random_password" "coder-postgres-password" {
 }
 
 resource "kubernetes_secret" "coder-db" {
-  type = "kubernetes.io/basic-auth"
+  type = "" # Opaque
   metadata {
     name      = "coder-db-url"
     namespace = kubernetes_namespace.coder_namespace.metadata.0.name
   }
   data = {
-    url = "postgres://coder:${random_password.coder-postgres-password.result}@/${google_sql_database_instance.db.ip_address}?sslmode=disable"
+    url = "postgres://coder:${urlencode(random_password.coder-postgres-password.result)}@${google_sql_database_instance.db.private_ip_address}?sslmode=disable"
   }
 }
 
@@ -121,7 +121,9 @@ coder:
           key: url
     - name: "CODER_VERBOSE"
       value: "true"
-  image: ${var.coder_image_repo}:${var.coder_image_tag}
+  image:
+    repo: ${var.coder_image_repo}
+    tag: ${var.coder_image_tag}
   replicaCount: "${var.coder_replicas}"
   resources:
     requests:
diff --git a/scaletest/terraform/gcp_db.tf b/scaletest/terraform/gcp_db.tf
index 3a727c92041de..20a6640a58992 100644
--- a/scaletest/terraform/gcp_db.tf
+++ b/scaletest/terraform/gcp_db.tf
@@ -7,12 +7,6 @@ data "google_compute_global_address" "sql_peering" {
   name = "sql-ip-address"
 }
 
-resource "google_service_networking_connection" "private_vpc_connection" {
-  network                 = data.google_compute_network.default.id
-  service                 = "servicenetworking.googleapis.com"
-  reserved_peering_ranges = [google_compute_global_address.sql_peering.name]
-}
-
 resource "google_sql_database_instance" "db" {
   name             = "${var.name}-db"
   region           = var.region
@@ -36,7 +30,7 @@ resource "google_sql_database_instance" "db" {
 
     ip_configuration {
       ipv4_enabled    = false
-      private_network = data.google_compute_network.default.id
+      private_network = google_compute_network.vpc.id
     }
 
     insights_config {
diff --git a/scaletest/terraform/gcp_vpc.tf b/scaletest/terraform/gcp_vpc.tf
index 8923745be637a..083feb07013bf 100644
--- a/scaletest/terraform/gcp_vpc.tf
+++ b/scaletest/terraform/gcp_vpc.tf
@@ -16,9 +16,16 @@ resource "google_compute_subnetwork" "subnet" {
 }
 
 resource "google_compute_global_address" "sql_peering" {
+  project       = var.project_id
   name          = "${var.name}-sql-peering"
   purpose       = "VPC_PEERING"
   address_type  = "INTERNAL"
-  # prefix_length = 16
+  prefix_length = 16
   network       = google_compute_network.vpc.id
 }
+
+resource "google_service_networking_connection" "private_vpc_connection" {
+  network = google_compute_network.vpc.id
+  service = "servicenetworking.googleapis.com"
+  reserved_peering_ranges = [ google_compute_global_address.sql_peering.name ]
+}

From bef9887219e86926bd5317947912f63537205e12 Mon Sep 17 00:00:00 2001
From: Cian Johnston <cian@coder.com>
Date: Mon, 8 May 2023 14:37:30 +0100
Subject: [PATCH 03/26] create clusters in vpc native networking mode

---
 scaletest/terraform/gcp_cluster.tf | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/scaletest/terraform/gcp_cluster.tf b/scaletest/terraform/gcp_cluster.tf
index 3a57d535762cf..c34f7f2167d07 100644
--- a/scaletest/terraform/gcp_cluster.tf
+++ b/scaletest/terraform/gcp_cluster.tf
@@ -8,6 +8,13 @@ resource "google_container_cluster" "primary" {
   project                  = var.project_id
   network                  = google_compute_network.vpc.name
   subnetwork               = google_compute_subnetwork.subnet.name
+  networking_mode          = "VPC_NATIVE"
+  ip_allocation_policy { # Required with networking_mode=VPC_NATIVE
+
+  }
+  release_channel {
+    channel = "STABLE"
+  }
   initial_node_count       = 1
   remove_default_node_pool = true
   network_policy {

From ea42b443c53aeada72befe263425d7becdb45497 Mon Sep 17 00:00:00 2001
From: Cian Johnston <cian@coder.com>
Date: Mon, 8 May 2023 14:38:15 +0100
Subject: [PATCH 04/26] gitignore tfstate

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 6b530e25cc3a9..0d02d29599031 100644
--- a/.gitignore
+++ b/.gitignore
@@ -58,3 +58,4 @@ site/stats/
 # Loadtesting
 ./scaletest/terraform/.terraform
 ./scaletest/terraform/.terraform.lock.hcl
+terraform.tfstate.*

From 9875f84075ccebfff48357568f09fd50dcdb4496 Mon Sep 17 00:00:00 2001
From: Cian Johnston <cian@coder.com>
Date: Mon, 8 May 2023 15:12:29 +0100
Subject: [PATCH 05/26] create pg database and user, fix db url

---
 scaletest/terraform/coder.tf  |  2 +-
 scaletest/terraform/gcp_db.tf | 15 +++++++++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/scaletest/terraform/coder.tf b/scaletest/terraform/coder.tf
index 22ae75fa9d630..adbf21ebc245f 100644
--- a/scaletest/terraform/coder.tf
+++ b/scaletest/terraform/coder.tf
@@ -45,7 +45,7 @@ resource "kubernetes_secret" "coder-db" {
     namespace = kubernetes_namespace.coder_namespace.metadata.0.name
   }
   data = {
-    url = "postgres://coder:${urlencode(random_password.coder-postgres-password.result)}@${google_sql_database_instance.db.private_ip_address}?sslmode=disable"
+    url = "postgres://coder:${urlencode(random_password.coder-postgres-password.result)}@${google_sql_database_instance.db.private_ip_address}/${google_sql_database.coder.name}?sslmode=disable"
   }
 }
 
diff --git a/scaletest/terraform/gcp_db.tf b/scaletest/terraform/gcp_db.tf
index 20a6640a58992..8055f4abdbe47 100644
--- a/scaletest/terraform/gcp_db.tf
+++ b/scaletest/terraform/gcp_db.tf
@@ -41,3 +41,18 @@ resource "google_sql_database_instance" "db" {
     }
   }
 }
+
+resource "google_sql_database" "coder" {
+  project = var.project_id
+  instance = google_sql_database_instance.db.id
+  name = "${var.name}-coder"
+  deletion_policy = "DELETE"
+}
+
+resource "google_sql_user" "coder" {
+  project = var.project_id
+  instance = google_sql_database_instance.db.id
+  name = "coder"
+  type = "BUILT_IN"
+  password = random_password.coder-postgres-password.result
+}

From 8fb3511cf15b97c2633bbafb4d365a22afb762a1 Mon Sep 17 00:00:00 2001
From: Cian Johnston <cian@coder.com>
Date: Mon, 8 May 2023 15:42:44 +0100
Subject: [PATCH 06/26] ensure db is destroyed properly with terraform destroy

---
 scaletest/terraform/gcp_db.tf | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scaletest/terraform/gcp_db.tf b/scaletest/terraform/gcp_db.tf
index 8055f4abdbe47..7e755db1ff208 100644
--- a/scaletest/terraform/gcp_db.tf
+++ b/scaletest/terraform/gcp_db.tf
@@ -11,6 +11,7 @@ resource "google_sql_database_instance" "db" {
   name             = "${var.name}-db"
   region           = var.region
   database_version = var.cloudsql_version
+  deletion_protection = false
 
   depends_on = [google_service_networking_connection.private_vpc_connection]
 
@@ -46,7 +47,8 @@ resource "google_sql_database" "coder" {
   project = var.project_id
   instance = google_sql_database_instance.db.id
   name = "${var.name}-coder"
-  deletion_policy = "DELETE"
+   # required for postgres, otherwise db fails to delete
+  deletion_policy = "ABANDON"
 }
 
 resource "google_sql_user" "coder" {
@@ -55,4 +57,6 @@ resource "google_sql_user" "coder" {
   name = "coder"
   type = "BUILT_IN"
   password = random_password.coder-postgres-password.result
+  # required for postgres, otherwise user fails to delete
+  deletion_policy = "ABANDON"
 }

From 13ca9e4bb4e0c3356c2003d901383760139b7475 Mon Sep 17 00:00:00 2001
From: Cian Johnston <cian@coder.com>
Date: Wed, 10 May 2023 19:35:25 +0100
Subject: [PATCH 07/26] enable prometheus, add podmonitor spec

---
 scaletest/terraform/coder.tf | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/scaletest/terraform/coder.tf b/scaletest/terraform/coder.tf
index adbf21ebc245f..da3912ba3c2bb 100644
--- a/scaletest/terraform/coder.tf
+++ b/scaletest/terraform/coder.tf
@@ -119,6 +119,8 @@ coder:
         secretKeyRef:
           name: "${kubernetes_secret.coder-db.metadata.0.name}"
           key: url
+    - name: "CODER_PROMETHEUS_ENABLE"
+      value: "true"
     - name: "CODER_VERBOSE"
       value: "true"
   image:
@@ -147,6 +149,20 @@ coder:
   - emptyDir:
       sizeLimit: 1024Mi
     name: cache
+  extraTemplates:
+  - |
+    apiVersion: monitoring.googleapis.com/v1
+    kind: PodMonitoring
+    metadata:
+      namespace: ${kubernetes_namespace.coder_namespace.metadata.0.name}
+      name: coder-monitoring
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: coder
+      endpoints:
+      - port: prometheus-http
+        interval: 30s
 
 EOF
   ]

From 3a3509bb6cad7163662d8a3fd626403bfcc05d5e Mon Sep 17 00:00:00 2001
From: Cian Johnston <cian@coder.com>
Date: Wed, 10 May 2023 21:22:42 +0100
Subject: [PATCH 08/26] add inline kubernetes template

---
 scaletest/terraform/coder.tf | 103 +++++++++++++++++++++++++++++++++++
 1 file changed, 103 insertions(+)

diff --git a/scaletest/terraform/coder.tf b/scaletest/terraform/coder.tf
index da3912ba3c2bb..94ae5ec2eb854 100644
--- a/scaletest/terraform/coder.tf
+++ b/scaletest/terraform/coder.tf
@@ -167,3 +167,106 @@ coder:
 EOF
   ]
 }
+
+resource "local_file" "kubernetes_template" {
+  filename = "${path.module}/templates/kubernetes/main.tf"
+  content = <<EOF
+    terraform {
+      required_providers {
+        coder = {
+          source  = "coder/coder"
+          version = "~> 0.7.0"
+        }
+        kubernetes = {
+          source  = "hashicorp/kubernetes"
+          version = "~> 2.18"
+        }
+      }
+    }
+
+    provider "coder" {}
+
+    provider "kubernetes" {
+      config_path = null # always use host
+    }
+
+    data "coder_workspace" "me" {}
+
+    resource "coder_agent" "main" {
+      os                     = "linux"
+      arch                   = "amd64"
+      startup_script_timeout = 180
+      startup_script         = ""
+    }
+
+    resource "kubernetes_pod" "main" {
+      count = data.coder_workspace.me.start_count
+      metadata {
+        name      = "coder-${lower(data.coder_workspace.me.owner)}-${lower(data.coder_workspace.me.name)}"
+        namespace = var.namespace
+        labels = {
+          "app.kubernetes.io/name"     = "coder-workspace"
+          "app.kubernetes.io/instance" = "coder-workspace-${lower(data.coder_workspace.me.owner)}-${lower(data.coder_workspace.me.name)}"
+        }
+      }
+      spec {
+        security_context {
+          run_as_user = "1000"
+          fs_group    = "1000"
+        }
+        container {
+          name              = "dev"
+          image             = "gcr.io/coder-dev-1/coder-cian/minimal:ubuntu"
+          image_pull_policy = "Always"
+          command           = ["sh", "-c", coder_agent.main.init_script]
+          security_context {
+            run_as_user = "1000"
+          }
+          env {
+            name  = "CODER_AGENT_TOKEN"
+            value = coder_agent.main.token
+          }
+          resources {
+            requests = {
+              "cpu"    = "1"
+              "memory" = "1Gi"
+            }
+            limits = {
+              "cpu"    = "1"
+              "memory" = "1Gi"
+            }
+          }
+        }
+
+        affinity {
+          node_affinity {
+            required_during_scheduling_ignored_during_execution {
+              node_selector_terms {
+                match_expressions {
+                  key = "cloud.google.com/gke-nodepool"
+                  operator = "in"
+                  values = ["${google_container_node_pool.workspaces.name}"]
+                }
+              }
+            }
+          }
+          pod_affinity {
+            preferred_during_scheduling_ignored_during_execution {
+              weight = 1
+              pod_affinity_term {
+                topology_key = "kubernetes.io/hostname"
+                label_selector {
+                  match_expressions {
+                    key      = "app.kubernetes.io/name"
+                    operator = "In"
+                    values   = ["coder-workspace"]
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  EOF
+}

From 41ab251250bdce6a6804247eda621b2260c899e9 Mon Sep 17 00:00:00 2001
From: Cian Johnston <cian@coder.com>
Date: Thu, 11 May 2023 13:09:31 +0100
Subject: [PATCH 09/26] add script to init coder instance and import template

---
 .gitignore                        |  2 +-
 scaletest/terraform/coder.tf      |  6 ++--
 scaletest/terraform/coder_init.sh | 48 +++++++++++++++++++++++++++++++
 3 files changed, 52 insertions(+), 4 deletions(-)
 create mode 100755 scaletest/terraform/coder_init.sh

diff --git a/.gitignore b/.gitignore
index 0d02d29599031..69b58c4cee458 100644
--- a/.gitignore
+++ b/.gitignore
@@ -48,7 +48,7 @@ site/stats/
 *.lock.hcl
 .terraform/
 
-/.coderv2/*
+**/.coderv2/*
 **/__debug_bin
 
 # direnv
diff --git a/scaletest/terraform/coder.tf b/scaletest/terraform/coder.tf
index 94ae5ec2eb854..df7edf74a86b1 100644
--- a/scaletest/terraform/coder.tf
+++ b/scaletest/terraform/coder.tf
@@ -169,7 +169,7 @@ EOF
 }
 
 resource "local_file" "kubernetes_template" {
-  filename = "${path.module}/templates/kubernetes/main.tf"
+  filename = "${path.module}/.coderv2/templates/kubernetes/main.tf"
   content = <<EOF
     terraform {
       required_providers {
@@ -202,11 +202,11 @@ resource "local_file" "kubernetes_template" {
     resource "kubernetes_pod" "main" {
       count = data.coder_workspace.me.start_count
       metadata {
-        name      = "coder-${lower(data.coder_workspace.me.owner)}-${lower(data.coder_workspace.me.name)}"
+        name      = "coder-$${lower(data.coder_workspace.me.owner)}-$${lower(data.coder_workspace.me.name)}"
         namespace = var.namespace
         labels = {
           "app.kubernetes.io/name"     = "coder-workspace"
-          "app.kubernetes.io/instance" = "coder-workspace-${lower(data.coder_workspace.me.owner)}-${lower(data.coder_workspace.me.name)}"
+          "app.kubernetes.io/instance" = "coder-workspace-$${lower(data.coder_workspace.me.owner)}-$${lower(data.coder_workspace.me.name)}"
         }
       }
       spec {
diff --git a/scaletest/terraform/coder_init.sh b/scaletest/terraform/coder_init.sh
new file mode 100755
index 0000000000000..dfef877468c49
--- /dev/null
+++ b/scaletest/terraform/coder_init.sh
@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+if [[ $# -lt 1 ]]; then
+	echo "Usage: $0 <coder URL>"
+	exit 1
+fi
+
+# Allow toggling verbose output
+[[ -n ${VERBOSE:-} ]] && set -x
+
+CODER_URL=$1
+CONFIG_DIR="${PWD}/.coderv2"
+ARCH="$(arch)"
+PLATFORM="$(uname | tr '[:upper:]' '[:lower:]')"
+
+mkdir -p "${CONFIG_DIR}"
+echo "Fetching Coder CLI for first-time setup!"
+curl -fsSL "${CODER_URL}/bin/coder-${PLATFORM}-${ARCH}" -o "${CONFIG_DIR}/coder"
+chmod +x "${CONFIG_DIR}/coder"
+
+set +o pipefail
+RANDOM_ADMIN_PASSWORD=$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c16)
+set -o pipefail
+CODER_FIRST_USER_EMAIL="admin@coder.com"
+CODER_FIRST_USER_USERNAME="coder"
+CODER_FIRST_USER_PASSWORD="${RANDOM_ADMIN_PASSWORD}"
+CODER_FIRST_USER_TRIAL=false
+echo "Running login command!"
+${CONFIG_DIR}/coder login "${CODER_URL}" \
+	--global-config="${CONFIG_DIR}" \
+	--first-user-username="${CODER_FIRST_USER_USERNAME}" \
+	--first-user-email="${CODER_FIRST_USER_EMAIL}" \
+	--first-user-password="${CODER_FIRST_USER_PASSWORD}" \
+	--first-user-trial=false
+
+echo "Writing credentials to coder.env"
+cat <<EOF > ./coder.env
+CODER_FIRST_USER_EMAIL=admin@coder.com
+CODER_FIRST_USER_USERNAME=coder
+CODER_FIRST_USER_PASSWORD="${RANDOM_ADMIN_PASSWORD}"
+CODER_FIRST_USER_TRIAL=false
+EOF
+
+echo "Importing kubernetes template"
+"${CONFIG_DIR}/coder" templates create --global-config="${CONFIG_DIR}" \
+	--directory "${CONFIG_DIR}/templates/kubernetes" --yes kubernetes

From 0bbf206024981cd5c6dd24c8cee45f31450c1c27 Mon Sep 17 00:00:00 2001
From: Cian Johnston <cian@coder.com>
Date: Thu, 11 May 2023 13:27:02 +0100
Subject: [PATCH 10/26] modify tls cert def

---
 scaletest/terraform/coder.tf | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scaletest/terraform/coder.tf b/scaletest/terraform/coder.tf
index df7edf74a86b1..dfdbf3d41f4fe 100644
--- a/scaletest/terraform/coder.tf
+++ b/scaletest/terraform/coder.tf
@@ -50,17 +50,18 @@ resource "kubernetes_secret" "coder-db" {
 }
 
 resource "tls_private_key" "coder" {
-  algorithm = "ED25519"
+  algorithm = "RSA"
 }
 
 resource "tls_self_signed_cert" "coder" {
   private_key_pem = tls_private_key.coder.private_key_pem
+  is_ca_certificate = true
 
   subject {
     common_name = "${local.coder_release_name}.${local.coder_namespace}.svc.cluster.local"
   }
 
-  allowed_uses = ["server_auth", "digital_signature", "data_encipherment", "key_agreement", "key_encipherment"]
+  allowed_uses = ["digital_signature", "cert_signing", "crl_signing"]
 
   # 1 year
   validity_period_hours = 8760

From d4b1fe68fcc83604b1d07c3e57c32f69c63906fd Mon Sep 17 00:00:00 2001
From: Cian Johnston <cian@coder.com>
Date: Thu, 11 May 2023 13:28:46 +0100
Subject: [PATCH 11/26] fixup template

---
 scaletest/terraform/coder.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scaletest/terraform/coder.tf b/scaletest/terraform/coder.tf
index dfdbf3d41f4fe..b545d2f6e9389 100644
--- a/scaletest/terraform/coder.tf
+++ b/scaletest/terraform/coder.tf
@@ -204,7 +204,7 @@ resource "local_file" "kubernetes_template" {
       count = data.coder_workspace.me.start_count
       metadata {
         name      = "coder-$${lower(data.coder_workspace.me.owner)}-$${lower(data.coder_workspace.me.name)}"
-        namespace = var.namespace
+        namespace = "${kubernetes_namespace.coder_namespace.metadata.0.name}"
         labels = {
           "app.kubernetes.io/name"     = "coder-workspace"
           "app.kubernetes.io/instance" = "coder-workspace-$${lower(data.coder_workspace.me.owner)}-$${lower(data.coder_workspace.me.name)}"

From 2b5a15b64277b4b90788cfdddb68cebbfdb0cf30 Mon Sep 17 00:00:00 2001
From: Cian Johnston <cian@coder.com>
Date: Thu, 11 May 2023 16:19:06 +0100
Subject: [PATCH 12/26] multiple fixes

---
 scaletest/terraform/coder.tf      | 40 ++++++++++++++++++++++++++++---
 scaletest/terraform/coder_init.sh |  5 +++-
 scaletest/terraform/gcp_db.tf     |  2 +-
 scaletest/terraform/gcp_vpc.tf    |  8 +++++++
 4 files changed, 50 insertions(+), 5 deletions(-)

diff --git a/scaletest/terraform/coder.tf b/scaletest/terraform/coder.tf
index b545d2f6e9389..972ce7e4bf6b7 100644
--- a/scaletest/terraform/coder.tf
+++ b/scaletest/terraform/coder.tf
@@ -5,6 +5,8 @@ locals {
   coder_helm_chart   = "coder"
   coder_release_name = "coder-${var.name}"
   coder_namespace    = "coder-${var.name}"
+  coder_admin_email  = "admin@coder.com"
+  coder_admin_user   = "coder"
 }
 
 provider "kubernetes" {
@@ -45,7 +47,7 @@ resource "kubernetes_secret" "coder-db" {
     namespace = kubernetes_namespace.coder_namespace.metadata.0.name
   }
   data = {
-    url = "postgres://coder:${urlencode(random_password.coder-postgres-password.result)}@${google_sql_database_instance.db.private_ip_address}/${google_sql_database.coder.name}?sslmode=disable"
+    url = "postgres://${google_sql_user.coder.name}:${urlencode(random_password.coder-postgres-password.result)}@${google_sql_database_instance.db.private_ip_address}/${google_sql_database.coder.name}?sslmode=disable"
   }
 }
 
@@ -71,6 +73,10 @@ resource "tls_self_signed_cert" "coder" {
     "${local.coder_release_name}.${local.coder_namespace}",
     "${local.coder_release_name}",
   ]
+
+  ip_addresses = [
+    google_compute_address.coder.address
+  ]
 }
 
 resource "kubernetes_secret" "coder-tls" {
@@ -108,9 +114,29 @@ resource "helm_release" "coder-chart" {
   ]
   values = [<<EOF
 coder:
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: "cloud.google.com/gke-nodepool"
+            operator: "In"
+            values: ["${google_container_node_pool.workspaces.name}"]
+    podAntiAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+      - weight: 1
+        podAffinityTerm:
+          topologyKey: "kubernetes.io/hostname"
+          labelSelector:
+            matchExpressions:
+            - key:      "app.kubernetes.io/instance"
+              operator: "In"
+              values:   ["${local.coder_release_name}"]
   env:
     - name: "CODER_CACHE_DIRECTORY"
       value: "/tmp/coder"
+    - name: "CODER_ENABLE_TELEMETRY"
+      value: "false"
     - name: "CODER_LOGGING_HUMAN"
       value: "/dev/null"
     - name: "CODER_LOGGING_STACKDRIVER"
@@ -122,6 +148,8 @@ coder:
           key: url
     - name: "CODER_PROMETHEUS_ENABLE"
       value: "true"
+    - name: "CODER_TLS_REDIRECT_HTTP_TO_HTTPS"
+      value: "true"
     - name: "CODER_VERBOSE"
       value: "true"
   image:
@@ -139,6 +167,7 @@ coder:
     readOnlyRootFilesystem: true
   service:
     enable: true
+    loadBalancerIP: "${google_compute_address.coder.address}"
   tls:
     secretNames:
     - "${kubernetes_secret.coder-tls.metadata.0.name}"
@@ -169,6 +198,11 @@ EOF
   ]
 }
 
+resource "local_file" "url" {
+  filename = "${path.module}/coder_url"
+  content = "https://${google_compute_address.coder.address}"
+}
+
 resource "local_file" "kubernetes_template" {
   filename = "${path.module}/.coderv2/templates/kubernetes/main.tf"
   content = <<EOF
@@ -242,10 +276,10 @@ resource "local_file" "kubernetes_template" {
         affinity {
           node_affinity {
             required_during_scheduling_ignored_during_execution {
-              node_selector_terms {
+              node_selector_term {
                 match_expressions {
                   key = "cloud.google.com/gke-nodepool"
-                  operator = "in"
+                  operator = "In"
                   values = ["${google_container_node_pool.workspaces.name}"]
                 }
               }
diff --git a/scaletest/terraform/coder_init.sh b/scaletest/terraform/coder_init.sh
index dfef877468c49..072a9687e69f7 100755
--- a/scaletest/terraform/coder_init.sh
+++ b/scaletest/terraform/coder_init.sh
@@ -13,11 +13,14 @@ fi
 CODER_URL=$1
 CONFIG_DIR="${PWD}/.coderv2"
 ARCH="$(arch)"
+if [[ "$ARCH" == "x86_64" ]]; then
+	ARCH="amd64"
+fi
 PLATFORM="$(uname | tr '[:upper:]' '[:lower:]')"
 
 mkdir -p "${CONFIG_DIR}"
 echo "Fetching Coder CLI for first-time setup!"
-curl -fsSL "${CODER_URL}/bin/coder-${PLATFORM}-${ARCH}" -o "${CONFIG_DIR}/coder"
+curl -fsSLk "${CODER_URL}/bin/coder-${PLATFORM}-${ARCH}" -o "${CONFIG_DIR}/coder"
 chmod +x "${CONFIG_DIR}/coder"
 
 set +o pipefail
diff --git a/scaletest/terraform/gcp_db.tf b/scaletest/terraform/gcp_db.tf
index 7e755db1ff208..b0fb1ca3ae94d 100644
--- a/scaletest/terraform/gcp_db.tf
+++ b/scaletest/terraform/gcp_db.tf
@@ -54,7 +54,7 @@ resource "google_sql_database" "coder" {
 resource "google_sql_user" "coder" {
   project = var.project_id
   instance = google_sql_database_instance.db.id
-  name = "coder"
+  name = "${var.name}-coder"
   type = "BUILT_IN"
   password = random_password.coder-postgres-password.result
   # required for postgres, otherwise user fails to delete
diff --git a/scaletest/terraform/gcp_vpc.tf b/scaletest/terraform/gcp_vpc.tf
index 083feb07013bf..6edc39e6a2241 100644
--- a/scaletest/terraform/gcp_vpc.tf
+++ b/scaletest/terraform/gcp_vpc.tf
@@ -24,6 +24,14 @@ resource "google_compute_global_address" "sql_peering" {
   network       = google_compute_network.vpc.id
 }
 
+resource "google_compute_address" "coder" {
+  project       = var.project_id
+  region        = var.region
+  name          = "${var.name}-coder"
+  address_type  = "EXTERNAL"
+  network_tier  = "PREMIUM"
+}
+
 resource "google_service_networking_connection" "private_vpc_connection" {
   network = google_compute_network.vpc.id
   service = "servicenetworking.googleapis.com"

From dbcfc6420a3b6c00eb7a9d3dc7d1ee97e1108c16 Mon Sep 17 00:00:00 2001
From: Cian Johnston <cian@coder.com>
Date: Thu, 11 May 2023 17:50:50 +0100
Subject: [PATCH 13/26] rebuild docker image with certs

---
 scaletest/terraform/coder.tf | 48 +++++++++++++++++++++++++++++-------
 scaletest/terraform/main.tf  |  5 ++++
 scaletest/terraform/vars.tf  |  5 ++++
 3 files changed, 49 insertions(+), 9 deletions(-)

diff --git a/scaletest/terraform/coder.tf b/scaletest/terraform/coder.tf
index 972ce7e4bf6b7..6be40ddf13fe2 100644
--- a/scaletest/terraform/coder.tf
+++ b/scaletest/terraform/coder.tf
@@ -1,12 +1,15 @@
 data "google_client_config" "default" {}
 
 locals {
-  coder_helm_repo    = "https://helm.coder.com/v2"
-  coder_helm_chart   = "coder"
-  coder_release_name = "coder-${var.name}"
-  coder_namespace    = "coder-${var.name}"
-  coder_admin_email  = "admin@coder.com"
-  coder_admin_user   = "coder"
+  coder_helm_repo            = "https://helm.coder.com/v2"
+  coder_helm_chart           = "coder"
+  coder_release_name         = "coder-${var.name}"
+  coder_namespace            = "coder-${var.name}"
+  coder_admin_email          = "admin@coder.com"
+  coder_admin_user           = "coder"
+  coder_address              = "${google_compute_address.coder.address}"
+  coder_url                  =  "https://${google_compute_address.coder.address}"
+  rebuilt_workspace_image    =  "gcr.io/coder-dev-1/v2-loadtest/${var.name}/workspace:latest"
 }
 
 provider "kubernetes" {
@@ -167,7 +170,7 @@ coder:
     readOnlyRootFilesystem: true
   service:
     enable: true
-    loadBalancerIP: "${google_compute_address.coder.address}"
+    loadBalancerIP: "${local.coder_address}"
   tls:
     secretNames:
     - "${kubernetes_secret.coder-tls.metadata.0.name}"
@@ -200,7 +203,34 @@ EOF
 
 resource "local_file" "url" {
   filename = "${path.module}/coder_url"
-  content = "https://${google_compute_address.coder.address}"
+  content = "${local.coder_url}"
+}
+
+# Because we use a self-signed certificate, we need to also rebuild the base image.
+resource "local_file" "workspace_dockerfile" {
+  filename = "${path.module}/.coderv2/dockerfile/workspace/Dockerfile"
+  content = <<EOF
+    FROM ${var.workspace_image}
+    USER root
+    RUN openssl s_client -connect ${local.coder_address}:443 -servername ${local.coder_url} </dev/null 2>/dev/null |\
+        sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' | tee /usr/local/share/ca-certificates/coder.crt && \
+        update-ca-certificates
+    USER coder
+  EOF
+}
+
+resource "docker_image" "workspace" {
+  name = local.rebuilt_workspace_image
+  build {
+    context = dirname(abspath(local_file.workspace_dockerfile.filename))
+  }
+}
+
+resource "null_resource" "push_workspace_image" {
+  depends_on = [ docker_image.workspace ]
+  provisioner "local-exec" {
+    command = "docker push ${local.rebuilt_workspace_image}"
+  }
 }
 
 resource "local_file" "kubernetes_template" {
@@ -251,7 +281,7 @@ resource "local_file" "kubernetes_template" {
         }
         container {
           name              = "dev"
-          image             = "gcr.io/coder-dev-1/coder-cian/minimal:ubuntu"
+          image             = "${local.rebuilt_workspace_image}"
           image_pull_policy = "Always"
           command           = ["sh", "-c", coder_agent.main.init_script]
           security_context {
diff --git a/scaletest/terraform/main.tf b/scaletest/terraform/main.tf
index 121e5f7602d2d..fe7f7c9b52593 100644
--- a/scaletest/terraform/main.tf
+++ b/scaletest/terraform/main.tf
@@ -24,6 +24,11 @@ terraform {
       source  = "hashicorp/tls"
       version = "~> 4.0"
     }
+
+    docker = {
+      source = "kreuzwerker/docker"
+      version = "~> 3.0"
+    }
   }
 
   required_version = "~> 1.4.0"
diff --git a/scaletest/terraform/vars.tf b/scaletest/terraform/vars.tf
index 55d487ed3fb99..f1ed862f57afa 100644
--- a/scaletest/terraform/vars.tf
+++ b/scaletest/terraform/vars.tf
@@ -117,3 +117,8 @@ variable "coder_image_tag" {
   description = "Tag to use for Coder image."
   default = "latest"
 }
+
+variable "workspace_image" {
+  description = "Image and tag to use for workspaces."
+  default = "docker.io/codercom/enterprise-minimal:ubuntu"
+}

From 69bdfd170b56069a219b32200903088a5e2854a7 Mon Sep 17 00:00:00 2001
From: Cian Johnston <cian@coder.com>
Date: Fri, 12 May 2023 12:49:48 +0100
Subject: [PATCH 14/26] remove self-signed https for now

---
 scaletest/terraform/coder.tf | 70 +++---------------------------------
 1 file changed, 5 insertions(+), 65 deletions(-)

diff --git a/scaletest/terraform/coder.tf b/scaletest/terraform/coder.tf
index 6be40ddf13fe2..68bb2611f9f1a 100644
--- a/scaletest/terraform/coder.tf
+++ b/scaletest/terraform/coder.tf
@@ -8,8 +8,7 @@ locals {
   coder_admin_email          = "admin@coder.com"
   coder_admin_user           = "coder"
   coder_address              = "${google_compute_address.coder.address}"
-  coder_url                  =  "https://${google_compute_address.coder.address}"
-  rebuilt_workspace_image    =  "gcr.io/coder-dev-1/v2-loadtest/${var.name}/workspace:latest"
+  coder_url                  =  "http://${google_compute_address.coder.address}"
 }
 
 provider "kubernetes" {
@@ -54,58 +53,6 @@ resource "kubernetes_secret" "coder-db" {
   }
 }
 
-resource "tls_private_key" "coder" {
-  algorithm = "RSA"
-}
-
-resource "tls_self_signed_cert" "coder" {
-  private_key_pem = tls_private_key.coder.private_key_pem
-  is_ca_certificate = true
-
-  subject {
-    common_name = "${local.coder_release_name}.${local.coder_namespace}.svc.cluster.local"
-  }
-
-  allowed_uses = ["digital_signature", "cert_signing", "crl_signing"]
-
-  # 1 year
-  validity_period_hours = 8760
-
-  dns_names = [
-    "${local.coder_release_name}.${local.coder_namespace}.svc.cluster.local",
-    "${local.coder_release_name}.${local.coder_namespace}",
-    "${local.coder_release_name}",
-  ]
-
-  ip_addresses = [
-    google_compute_address.coder.address
-  ]
-}
-
-resource "kubernetes_secret" "coder-tls" {
-  type = "kubernetes.io/tls"
-  metadata {
-    name      = "coder-tls"
-    namespace = kubernetes_namespace.coder_namespace.metadata.0.name
-  }
-
-  data = {
-    "tls.crt" = tls_self_signed_cert.coder.cert_pem
-    "tls.key" = tls_private_key.coder.private_key_pem
-  }
-}
-
-resource "kubernetes_secret" "coder-ca" {
-  type = "Opaque"
-  metadata {
-    name      = "coder-ca"
-    namespace = kubernetes_namespace.coder_namespace.metadata.0.name
-  }
-  data = {
-    "ca.crt" = "${tls_self_signed_cert.coder.cert_pem}"
-  }
-}
-
 resource "helm_release" "coder-chart" {
   repository = local.coder_helm_repo
   chart      = local.coder_helm_chart
@@ -151,8 +98,6 @@ coder:
           key: url
     - name: "CODER_PROMETHEUS_ENABLE"
       value: "true"
-    - name: "CODER_TLS_REDIRECT_HTTP_TO_HTTPS"
-      value: "true"
     - name: "CODER_VERBOSE"
       value: "true"
   image:
@@ -171,9 +116,6 @@ coder:
   service:
     enable: true
     loadBalancerIP: "${local.coder_address}"
-  tls:
-    secretNames:
-    - "${kubernetes_secret.coder-tls.metadata.0.name}"
   volumeMounts:
   - mountPath: "/tmp"
     name: cache
@@ -226,11 +168,9 @@ resource "docker_image" "workspace" {
   }
 }
 
-resource "null_resource" "push_workspace_image" {
-  depends_on = [ docker_image.workspace ]
-  provisioner "local-exec" {
-    command = "docker push ${local.rebuilt_workspace_image}"
-  }
+resource "local_file" "url" {
+  filename = "${path.module}/coder_url"
+  content = "${local.coder_url}"
 }
 
 resource "local_file" "kubernetes_template" {
@@ -281,7 +221,7 @@ resource "local_file" "kubernetes_template" {
         }
         container {
           name              = "dev"
-          image             = "${local.rebuilt_workspace_image}"
+          image             = "${var.workspace_image}"
           image_pull_policy = "Always"
           command           = ["sh", "-c", coder_agent.main.init_script]
           security_context {

From 99c0f3c421754cc9d3a4a59045013ee10442a06b Mon Sep 17 00:00:00 2001
From: Cian Johnston <cian@coder.com>
Date: Fri, 12 May 2023 12:51:25 +0100
Subject: [PATCH 15/26] move monitoring manifest out of helm chart

---
 scaletest/terraform/coder.tf | 54 ++++++++++++++----------------------
 1 file changed, 21 insertions(+), 33 deletions(-)

diff --git a/scaletest/terraform/coder.tf b/scaletest/terraform/coder.tf
index 68bb2611f9f1a..d17d0e3998360 100644
--- a/scaletest/terraform/coder.tf
+++ b/scaletest/terraform/coder.tf
@@ -124,47 +124,35 @@ coder:
   - emptyDir:
       sizeLimit: 1024Mi
     name: cache
-  extraTemplates:
-  - |
-    apiVersion: monitoring.googleapis.com/v1
-    kind: PodMonitoring
-    metadata:
-      namespace: ${kubernetes_namespace.coder_namespace.metadata.0.name}
-      name: coder-monitoring
-    spec:
-      selector:
-        matchLabels:
-          app.kubernetes.io/name: coder
-      endpoints:
-      - port: prometheus-http
-        interval: 30s
-
 EOF
   ]
 }
 
-resource "local_file" "url" {
-  filename = "${path.module}/coder_url"
-  content = "${local.coder_url}"
-}
-
-# Because we use a self-signed certificate, we need to also rebuild the base image.
-resource "local_file" "workspace_dockerfile" {
-  filename = "${path.module}/.coderv2/dockerfile/workspace/Dockerfile"
+resource "local_file" "coder-monitoring-manifest" {
+  filename = "${path.module}/.coderv2/coder-monitoring.yaml"
   content = <<EOF
-    FROM ${var.workspace_image}
-    USER root
-    RUN openssl s_client -connect ${local.coder_address}:443 -servername ${local.coder_url} </dev/null 2>/dev/null |\
-        sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' | tee /usr/local/share/ca-certificates/coder.crt && \
-        update-ca-certificates
-    USER coder
+apiVersion: monitoring.googleapis.com/v1
+kind: PodMonitoring
+metadata:
+  namespace: ${kubernetes_namespace.coder_namespace.metadata.0.name}
+  name: coder-monitoring
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: coder
+  endpoints:
+  - port: prometheus-http
+    interval: 30s
   EOF
 }
 
-resource "docker_image" "workspace" {
-  name = local.rebuilt_workspace_image
-  build {
-    context = dirname(abspath(local_file.workspace_dockerfile.filename))
+resource "null_resource" "coder-monitoring-manifest_apply" {
+  provisioner "local-exec" {
+    working_dir = abspath(path.module)
+    command = <<EOF
+KUBECONFIG=${var.name}-cluster.kubeconfig gcloud container clusters get-credentials ${var.name}-cluster --project=${var.project_id} --zone=${var.zone} && \
+KUBECONFIG=${var.name}-cluster.kubeconfig kubectl apply -f ${abspath(local_file.coder-monitoring-manifest.filename)}
+    EOF
   }
 }
 

From ccda05dcee0308d6521f4ddfcbc078b16beec71a Mon Sep 17 00:00:00 2001
From: Cian Johnston <cian@coder.com>
Date: Fri, 12 May 2023 13:16:15 +0100
Subject: [PATCH 16/26] move generated files into .coderv2, create shim script

---
 scaletest/terraform/coder.tf      | 12 ++++++------
 scaletest/terraform/coder_init.sh |  4 ++--
 scaletest/terraform/coder_shim.sh |  8 ++++++++
 3 files changed, 16 insertions(+), 8 deletions(-)
 create mode 100755 scaletest/terraform/coder_shim.sh

diff --git a/scaletest/terraform/coder.tf b/scaletest/terraform/coder.tf
index d17d0e3998360..28ef79dc5e16b 100644
--- a/scaletest/terraform/coder.tf
+++ b/scaletest/terraform/coder.tf
@@ -148,7 +148,7 @@ spec:
 
 resource "null_resource" "coder-monitoring-manifest_apply" {
   provisioner "local-exec" {
-    working_dir = abspath(path.module)
+    working_dir = "${abspath(path.module)}/.coderv2"
     command = <<EOF
 KUBECONFIG=${var.name}-cluster.kubeconfig gcloud container clusters get-credentials ${var.name}-cluster --project=${var.project_id} --zone=${var.zone} && \
 KUBECONFIG=${var.name}-cluster.kubeconfig kubectl apply -f ${abspath(local_file.coder-monitoring-manifest.filename)}
@@ -156,11 +156,6 @@ KUBECONFIG=${var.name}-cluster.kubeconfig kubectl apply -f ${abspath(local_file.
   }
 }
 
-resource "local_file" "url" {
-  filename = "${path.module}/coder_url"
-  content = "${local.coder_url}"
-}
-
 resource "local_file" "kubernetes_template" {
   filename = "${path.module}/.coderv2/templates/kubernetes/main.tf"
   content = <<EOF
@@ -263,3 +258,8 @@ resource "local_file" "kubernetes_template" {
     }
   EOF
 }
+
+output "coder_url" {
+  description = "URL of the Coder deployment"
+  value = local.coder_url
+}
diff --git a/scaletest/terraform/coder_init.sh b/scaletest/terraform/coder_init.sh
index 072a9687e69f7..16067e0380703 100755
--- a/scaletest/terraform/coder_init.sh
+++ b/scaletest/terraform/coder_init.sh
@@ -38,8 +38,8 @@ ${CONFIG_DIR}/coder login "${CODER_URL}" \
 	--first-user-password="${CODER_FIRST_USER_PASSWORD}" \
 	--first-user-trial=false
 
-echo "Writing credentials to coder.env"
-cat <<EOF > ./coder.env
+echo "Writing credentials to "${CONFIG_DIR}/coder.env"
+cat <<EOF > ${CONFIG_DIR}/coder.env
 CODER_FIRST_USER_EMAIL=admin@coder.com
 CODER_FIRST_USER_USERNAME=coder
 CODER_FIRST_USER_PASSWORD="${RANDOM_ADMIN_PASSWORD}"
diff --git a/scaletest/terraform/coder_shim.sh b/scaletest/terraform/coder_shim.sh
new file mode 100755
index 0000000000000..d62c5a952ecb3
--- /dev/null
+++ b/scaletest/terraform/coder_shim.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+# This is a shim for easily executing Coder commands against a loadtest cluster
+# without having to overwrite your own session/URL
+SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}")
+CONFIG_DIR="${SCRIPT_DIR}/.coderv2"
+CODER_BIN="${CONFIG_DIR}/coder"
+exec "${CODER_BIN}" --global-config "${CONFIG_DIR}" "$@"

From 6ace6192243507b6192af5a19af79c49d92ab22a Mon Sep 17 00:00:00 2001
From: Cian Johnston <cian@coder.com>
Date: Fri, 12 May 2023 14:33:29 +0100
Subject: [PATCH 17/26] adjust template limits

---
 scaletest/terraform/coder.tf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scaletest/terraform/coder.tf b/scaletest/terraform/coder.tf
index 28ef79dc5e16b..4ca9ccc44695e 100644
--- a/scaletest/terraform/coder.tf
+++ b/scaletest/terraform/coder.tf
@@ -216,8 +216,8 @@ resource "local_file" "kubernetes_template" {
           }
           resources {
             requests = {
-              "cpu"    = "1"
-              "memory" = "1Gi"
+              "cpu"    = "0.1"
+              "memory" = "128Mi"
             }
             limits = {
               "cpu"    = "1"

From 01c6d392762666f636e16f6188c46dc03071f1f6 Mon Sep 17 00:00:00 2001
From: Cian Johnston <cian@coder.com>
Date: Fri, 12 May 2023 14:35:32 +0100
Subject: [PATCH 18/26] make fmt

---
 scaletest/terraform/coder.tf       | 24 ++++++++---------
 scaletest/terraform/coder_init.sh  |  2 +-
 scaletest/terraform/gcp_cluster.tf | 12 ++++-----
 scaletest/terraform/gcp_db.tf      | 18 ++++++-------
 scaletest/terraform/gcp_vpc.tf     | 16 ++++++------
 scaletest/terraform/main.tf        | 10 +++----
 scaletest/terraform/vars.tf        | 42 +++++++++++++++---------------
 7 files changed, 62 insertions(+), 62 deletions(-)

diff --git a/scaletest/terraform/coder.tf b/scaletest/terraform/coder.tf
index 4ca9ccc44695e..36ff9aeab13ba 100644
--- a/scaletest/terraform/coder.tf
+++ b/scaletest/terraform/coder.tf
@@ -1,14 +1,14 @@
 data "google_client_config" "default" {}
 
 locals {
-  coder_helm_repo            = "https://helm.coder.com/v2"
-  coder_helm_chart           = "coder"
-  coder_release_name         = "coder-${var.name}"
-  coder_namespace            = "coder-${var.name}"
-  coder_admin_email          = "admin@coder.com"
-  coder_admin_user           = "coder"
-  coder_address              = "${google_compute_address.coder.address}"
-  coder_url                  =  "http://${google_compute_address.coder.address}"
+  coder_helm_repo    = "https://helm.coder.com/v2"
+  coder_helm_chart   = "coder"
+  coder_release_name = "coder-${var.name}"
+  coder_namespace    = "coder-${var.name}"
+  coder_admin_email  = "admin@coder.com"
+  coder_admin_user   = "coder"
+  coder_address      = google_compute_address.coder.address
+  coder_url          = "http://${google_compute_address.coder.address}"
 }
 
 provider "kubernetes" {
@@ -130,7 +130,7 @@ EOF
 
 resource "local_file" "coder-monitoring-manifest" {
   filename = "${path.module}/.coderv2/coder-monitoring.yaml"
-  content = <<EOF
+  content  = <<EOF
 apiVersion: monitoring.googleapis.com/v1
 kind: PodMonitoring
 metadata:
@@ -149,7 +149,7 @@ spec:
 resource "null_resource" "coder-monitoring-manifest_apply" {
   provisioner "local-exec" {
     working_dir = "${abspath(path.module)}/.coderv2"
-    command = <<EOF
+    command     = <<EOF
 KUBECONFIG=${var.name}-cluster.kubeconfig gcloud container clusters get-credentials ${var.name}-cluster --project=${var.project_id} --zone=${var.zone} && \
 KUBECONFIG=${var.name}-cluster.kubeconfig kubectl apply -f ${abspath(local_file.coder-monitoring-manifest.filename)}
     EOF
@@ -158,7 +158,7 @@ KUBECONFIG=${var.name}-cluster.kubeconfig kubectl apply -f ${abspath(local_file.
 
 resource "local_file" "kubernetes_template" {
   filename = "${path.module}/.coderv2/templates/kubernetes/main.tf"
-  content = <<EOF
+  content  = <<EOF
     terraform {
       required_providers {
         coder = {
@@ -261,5 +261,5 @@ resource "local_file" "kubernetes_template" {
 
 output "coder_url" {
   description = "URL of the Coder deployment"
-  value = local.coder_url
+  value       = local.coder_url
 }
diff --git a/scaletest/terraform/coder_init.sh b/scaletest/terraform/coder_init.sh
index 16067e0380703..01659c8b3d212 100755
--- a/scaletest/terraform/coder_init.sh
+++ b/scaletest/terraform/coder_init.sh
@@ -38,7 +38,7 @@ ${CONFIG_DIR}/coder login "${CODER_URL}" \
 	--first-user-password="${CODER_FIRST_USER_PASSWORD}" \
 	--first-user-trial=false
 
-echo "Writing credentials to "${CONFIG_DIR}/coder.env"
+echo "Writing credentials to ${CONFIG_DIR}/coder.env"
 cat <<EOF > ${CONFIG_DIR}/coder.env
 CODER_FIRST_USER_EMAIL=admin@coder.com
 CODER_FIRST_USER_USERNAME=coder
diff --git a/scaletest/terraform/gcp_cluster.tf b/scaletest/terraform/gcp_cluster.tf
index c34f7f2167d07..3d6aede6cc9d5 100644
--- a/scaletest/terraform/gcp_cluster.tf
+++ b/scaletest/terraform/gcp_cluster.tf
@@ -3,12 +3,12 @@ data "google_compute_default_service_account" "default" {
 }
 
 resource "google_container_cluster" "primary" {
-  name                     = "${var.name}-cluster"
-  location                 = var.zone
-  project                  = var.project_id
-  network                  = google_compute_network.vpc.name
-  subnetwork               = google_compute_subnetwork.subnet.name
-  networking_mode          = "VPC_NATIVE"
+  name            = "${var.name}-cluster"
+  location        = var.zone
+  project         = var.project_id
+  network         = google_compute_network.vpc.name
+  subnetwork      = google_compute_subnetwork.subnet.name
+  networking_mode = "VPC_NATIVE"
   ip_allocation_policy { # Required with networking_mode=VPC_NATIVE
 
   }
diff --git a/scaletest/terraform/gcp_db.tf b/scaletest/terraform/gcp_db.tf
index b0fb1ca3ae94d..3122f5c0f2223 100644
--- a/scaletest/terraform/gcp_db.tf
+++ b/scaletest/terraform/gcp_db.tf
@@ -8,9 +8,9 @@ data "google_compute_global_address" "sql_peering" {
 }
 
 resource "google_sql_database_instance" "db" {
-  name             = "${var.name}-db"
-  region           = var.region
-  database_version = var.cloudsql_version
+  name                = "${var.name}-db"
+  region              = var.region
+  database_version    = var.cloudsql_version
   deletion_protection = false
 
   depends_on = [google_service_networking_connection.private_vpc_connection]
@@ -44,18 +44,18 @@ resource "google_sql_database_instance" "db" {
 }
 
 resource "google_sql_database" "coder" {
-  project = var.project_id
+  project  = var.project_id
   instance = google_sql_database_instance.db.id
-  name = "${var.name}-coder"
-   # required for postgres, otherwise db fails to delete
+  name     = "${var.name}-coder"
+  # required for postgres, otherwise db fails to delete
   deletion_policy = "ABANDON"
 }
 
 resource "google_sql_user" "coder" {
-  project = var.project_id
+  project  = var.project_id
   instance = google_sql_database_instance.db.id
-  name = "${var.name}-coder"
-  type = "BUILT_IN"
+  name     = "${var.name}-coder"
+  type     = "BUILT_IN"
   password = random_password.coder-postgres-password.result
   # required for postgres, otherwise user fails to delete
   deletion_policy = "ABANDON"
diff --git a/scaletest/terraform/gcp_vpc.tf b/scaletest/terraform/gcp_vpc.tf
index 6edc39e6a2241..59c65a1355862 100644
--- a/scaletest/terraform/gcp_vpc.tf
+++ b/scaletest/terraform/gcp_vpc.tf
@@ -25,15 +25,15 @@ resource "google_compute_global_address" "sql_peering" {
 }
 
 resource "google_compute_address" "coder" {
-  project       = var.project_id
-  region        = var.region
-  name          = "${var.name}-coder"
-  address_type  = "EXTERNAL"
-  network_tier  = "PREMIUM"
+  project      = var.project_id
+  region       = var.region
+  name         = "${var.name}-coder"
+  address_type = "EXTERNAL"
+  network_tier = "PREMIUM"
 }
 
 resource "google_service_networking_connection" "private_vpc_connection" {
-  network = google_compute_network.vpc.id
-  service = "servicenetworking.googleapis.com"
-  reserved_peering_ranges = [ google_compute_global_address.sql_peering.name ]
+  network                 = google_compute_network.vpc.id
+  service                 = "servicenetworking.googleapis.com"
+  reserved_peering_ranges = [google_compute_global_address.sql_peering.name]
 }
diff --git a/scaletest/terraform/main.tf b/scaletest/terraform/main.tf
index fe7f7c9b52593..280420cecf267 100644
--- a/scaletest/terraform/main.tf
+++ b/scaletest/terraform/main.tf
@@ -1,22 +1,22 @@
 terraform {
   required_providers {
     google = {
-      source = "hashicorp/google"
+      source  = "hashicorp/google"
       version = "~> 4.36"
     }
 
     kubernetes = {
-      source = "hashicorp/kubernetes"
+      source  = "hashicorp/kubernetes"
       version = "~> 2.20"
     }
 
     helm = {
-      source = "hashicorp/helm"
+      source  = "hashicorp/helm"
       version = "~> 2.9"
     }
 
     random = {
-      source = "hashicorp/random"
+      source  = "hashicorp/random"
       version = "~> 3.5"
     }
 
@@ -26,7 +26,7 @@ terraform {
     }
 
     docker = {
-      source = "kreuzwerker/docker"
+      source  = "kreuzwerker/docker"
       version = "~> 3.0"
     }
   }
diff --git a/scaletest/terraform/vars.tf b/scaletest/terraform/vars.tf
index f1ed862f57afa..a42034719ebc7 100644
--- a/scaletest/terraform/vars.tf
+++ b/scaletest/terraform/vars.tf
@@ -8,34 +8,34 @@ variable "name" {
 
 variable "region" {
   description = "GCP region in which to provision resources."
-  default = "us-east1"
+  default     = "us-east1"
 }
 
 variable "zone" {
   description = "GCP zone in which to provision resources."
-  default = "us-east1-c"
+  default     = "us-east1-c"
 }
 
 variable "k8s_version" {
   description = "Kubernetes vversion to provision."
-  default = "1.24"
+  default     = "1.24"
 }
 
 variable "node_disk_size_gb" {
   description = "Size of the root disk for cluster nodes."
-  default = 100
+  default     = 100
 }
 
 variable "node_image_type" {
   description = "Image type to use for cluster nodes."
-  default = "cos_containerd"
+  default     = "cos_containerd"
 }
 
 // Preemptible nodes are way cheaper, but can be pulled out
 // from under you at any time. Caveat emptor.
 variable "node_preemptible" {
   description = "Use preemptible nodes."
-  default = false
+  default     = false
 }
 
 // We create three nodepools:
@@ -46,79 +46,79 @@ variable "node_preemptible" {
 // These variables control the node pool dedicated to Coder.
 variable "nodepool_machine_type_coder" {
   description = "Machine type to use for Coder control plane nodepool."
-  default = "t2d-standard-4"
+  default     = "t2d-standard-4"
 }
 
 variable "nodepool_size_coder" {
   description = "Number of cluster nodes for the Coder control plane nodepool."
-  default = 1
+  default     = 1
 }
 
 // These variables control the node pool dedicated to workspaces.
 variable "nodepool_machine_type_workspaces" {
   description = "Machine type to use for the workspaces nodepool."
-  default = "t2d-standard-4"
+  default     = "t2d-standard-4"
 }
 
 variable "nodepool_size_workspaces" {
   description = "Number of cluster nodes for the workspaces nodepool."
-  default = 1
+  default     = 1
 }
 
 // These variables control the node pool for everything else.
 variable "nodepool_machine_type_misc" {
   description = "Machine type to use for the misc nodepool."
-  default = "t2d-standard-4"
+  default     = "t2d-standard-4"
 }
 
 variable "nodepool_size_misc" {
   description = "Number of cluster nodes for the misc nodepool."
-  default = 1
+  default     = 1
 }
 
 // These variables control the size of the database to be used by Coder.
 variable "cloudsql_version" {
   description = "CloudSQL version to provision"
-  default = "POSTGRES_14"
+  default     = "POSTGRES_14"
 }
 
 variable "cloudsql_tier" {
   description = "CloudSQL database tier."
-  default = "db-f1-micro"
+  default     = "db-f1-micro"
 }
 
 // These variables control the Coder deployment.
 variable "coder_replicas" {
   description = "Number of Coder replicas to provision"
-  default = 1
+  default     = 1
 }
 
 variable "coder_cpu" {
   description = "CPU to allocate to Coder"
-  default = "1000m"
+  default     = "1000m"
 }
 
 variable "coder_mem" {
   description = "Memory to allocate to Coder"
-  default = "1024Mi"
+  default     = "1024Mi"
 }
 
 variable "coder_chart_version" {
   description = "Version of the Coder Helm chart to install. Defaults to latest."
-  default = null
+  default     = null
 }
 
 variable "coder_image_repo" {
   description = "Repository to use for Coder image."
-  default = "ghcr.io/coder/coder"
+  default     = "ghcr.io/coder/coder"
 }
 
 variable "coder_image_tag" {
   description = "Tag to use for Coder image."
-  default = "latest"
+  default     = "latest"
 }
 
 variable "workspace_image" {
   description = "Image and tag to use for workspaces."
-  default = "docker.io/codercom/enterprise-minimal:ubuntu"
+  default     = "docker.io/codercom/enterprise-minimal:ubuntu"
 }

From 9f7c1654c532c6af39ad38412be4eb48cad1f2be Mon Sep 17 00:00:00 2001
From: Cian Johnston <cian@coder.com>
Date: Fri, 12 May 2023 14:37:12 +0100
Subject: [PATCH 19/26] make lint

---
 scaletest/terraform/coder_init.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scaletest/terraform/coder_init.sh b/scaletest/terraform/coder_init.sh
index 01659c8b3d212..d3ec15cdda4d0 100755
--- a/scaletest/terraform/coder_init.sh
+++ b/scaletest/terraform/coder_init.sh
@@ -29,9 +29,9 @@ set -o pipefail
 CODER_FIRST_USER_EMAIL="admin@coder.com"
 CODER_FIRST_USER_USERNAME="coder"
 CODER_FIRST_USER_PASSWORD="${RANDOM_ADMIN_PASSWORD}"
-CODER_FIRST_USER_TRIAL=false
+CODER_FIRST_USER_TRIAL="false"
 echo "Running login command!"
-${CONFIG_DIR}/coder login "${CODER_URL}" \
+"${CONFIG_DIR}/coder" login "${CODER_URL}" \
 	--global-config="${CONFIG_DIR}" \
 	--first-user-username="${CODER_FIRST_USER_USERNAME}" \
 	--first-user-email="${CODER_FIRST_USER_EMAIL}" \
@@ -39,11 +39,11 @@ ${CONFIG_DIR}/coder login "${CODER_URL}" \
 	--first-user-trial=false
 
 echo "Writing credentials to ${CONFIG_DIR}/coder.env"
-cat <<EOF > ${CONFIG_DIR}/coder.env
+cat <<EOF > "${CONFIG_DIR}/coder.env"
 CODER_FIRST_USER_EMAIL=admin@coder.com
 CODER_FIRST_USER_USERNAME=coder
 CODER_FIRST_USER_PASSWORD="${RANDOM_ADMIN_PASSWORD}"
-CODER_FIRST_USER_TRIAL=false
+CODER_FIRST_USER_TRIAL="${CODER_FIRST_USER_TRIAL}"
 EOF
 
 echo "Importing kubernetes template"

From 660959c96aae3c4558d1add363e1eb6395da5d32 Mon Sep 17 00:00:00 2001
From: Cian Johnston <cian@coder.com>
Date: Fri, 12 May 2023 14:56:38 +0100
Subject: [PATCH 20/26] make gen

---
 .prettierignore                   | 7 ++++++-
 scaletest/terraform/coder_init.sh | 4 ++--
 site/.eslintignore                | 7 ++++++-
 site/.prettierignore              | 7 ++++++-
 4 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/.prettierignore b/.prettierignore
index f08db1c28a2e6..cc4a83b0231a8 100644
--- a/.prettierignore
+++ b/.prettierignore
@@ -51,12 +51,17 @@ site/stats/
 *.lock.hcl
 .terraform/
 
-/.coderv2/*
+**/.coderv2/*
 **/__debug_bin
 
 # direnv
 .envrc
 *.test
+
+# Loadtesting
+./scaletest/terraform/.terraform
+./scaletest/terraform/.terraform.lock.hcl
+terraform.tfstate.*
 # .prettierignore.include:
 # Helm templates contain variables that are invalid YAML and can't be formatted
 # by Prettier.
diff --git a/scaletest/terraform/coder_init.sh b/scaletest/terraform/coder_init.sh
index d3ec15cdda4d0..fe038a6d9aff7 100755
--- a/scaletest/terraform/coder_init.sh
+++ b/scaletest/terraform/coder_init.sh
@@ -24,7 +24,7 @@ curl -fsSLk "${CODER_URL}/bin/coder-${PLATFORM}-${ARCH}" -o "${CONFIG_DIR}/coder
 chmod +x "${CONFIG_DIR}/coder"
 
 set +o pipefail
-RANDOM_ADMIN_PASSWORD=$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c16)
+RANDOM_ADMIN_PASSWORD=$(tr </dev/urandom -dc _A-Z-a-z-0-9 | head -c16)
 set -o pipefail
 CODER_FIRST_USER_EMAIL="admin@coder.com"
 CODER_FIRST_USER_USERNAME="coder"
@@ -39,7 +39,7 @@ echo "Running login command!"
 	--first-user-trial=false
 
 echo "Writing credentials to ${CONFIG_DIR}/coder.env"
-cat <<EOF > "${CONFIG_DIR}/coder.env"
+cat <<EOF >"${CONFIG_DIR}/coder.env"
 CODER_FIRST_USER_EMAIL=admin@coder.com
 CODER_FIRST_USER_USERNAME=coder
 CODER_FIRST_USER_PASSWORD="${RANDOM_ADMIN_PASSWORD}"
diff --git a/site/.eslintignore b/site/.eslintignore
index f83b3caa434f0..865d1e7006067 100644
--- a/site/.eslintignore
+++ b/site/.eslintignore
@@ -51,12 +51,17 @@ stats/
 *.lock.hcl
 .terraform/
 
-../.coderv2/*
+**/.coderv2/*
 **/__debug_bin
 
 # direnv
 .envrc
 *.test
+
+# Loadtesting
+.././scaletest/terraform/.terraform
+.././scaletest/terraform/.terraform.lock.hcl
+terraform.tfstate.*
 # .prettierignore.include:
 # Helm templates contain variables that are invalid YAML and can't be formatted
 # by Prettier.
diff --git a/site/.prettierignore b/site/.prettierignore
index f83b3caa434f0..865d1e7006067 100644
--- a/site/.prettierignore
+++ b/site/.prettierignore
@@ -51,12 +51,17 @@ stats/
 *.lock.hcl
 .terraform/
 
-../.coderv2/*
+**/.coderv2/*
 **/__debug_bin
 
 # direnv
 .envrc
 *.test
+
+# Loadtesting
+.././scaletest/terraform/.terraform
+.././scaletest/terraform/.terraform.lock.hcl
+terraform.tfstate.*
 # .prettierignore.include:
 # Helm templates contain variables that are invalid YAML and can't be formatted
 # by Prettier.

From 34f8b0219265672915d407ce49bccec4375322a2 Mon Sep 17 00:00:00 2001
From: Cian Johnston <cian@coder.com>
Date: Fri, 12 May 2023 14:57:10 +0100
Subject: [PATCH 21/26] update README

---
 scaletest/README.md           |  3 ---
 scaletest/terraform/README.md | 40 +++++++++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+), 3 deletions(-)
 delete mode 100644 scaletest/README.md
 create mode 100644 scaletest/terraform/README.md

diff --git a/scaletest/README.md b/scaletest/README.md
deleted file mode 100644
index f46c66f797538..0000000000000
--- a/scaletest/README.md
+++ /dev/null
@@ -1,3 +0,0 @@
-# Load Testing
-
-TODO: write something here.
diff --git a/scaletest/terraform/README.md b/scaletest/terraform/README.md
new file mode 100644
index 0000000000000..a4bdefeccb394
--- /dev/null
+++ b/scaletest/terraform/README.md
@@ -0,0 +1,40 @@
+# Load Test Terraform
+
+This folder contains Terraform code and scripts to aid in performing load tests of Coder.
+It does the following:
+
+- Creates a GCP VPC.
+- Creates a CloudSQL instance with a global peering rule so it's accessible inside the VPC.
+- Creates a GKE cluster inside the VPC with separate nodegroups for Coder and workspaces.
+- Installs Coder in a new namespace, using the CloudSQL instance.
+
+## Usage
+
+> You must have an existing Google Cloud project available.
+
+1. Create a file named `override.tfvars` with the following content, modifying as appropriate:
+
+```terraform
+name = "some_unique_identifier"
+project_id = "some_google_project_id"
+```
+
+1. Inspect `vars.tf` and override any other variables you deem necessary.
+
+1. Run `terraform init`.
+
+1. Run `terraform plan -var-file=override.tfvars` and inspect the output.
+   If you are not satisfied, modify `override.tfvars` until you are.
+
+1. Run `terraform apply -var-file=override.tfvars`. This will spin up a pre-configured environment
+   and emit the Coder URL as an output.
+
+1. Run `coder_init.sh <coder_url>` to setup an initial user and a pre-configured Kubernetes
+   template. It will also down the Coder CLI locally.
+
+1. Do whatever you need to do with the Coder instance.
+
+   > To run Coder commands against the instance, you can use `coder_shim.sh <command>`.
+   > You don't need to run `coder login` yourself.
+
+1. When you are finished, you can run `terraform destroy -var-file=override.tfvars`.

From 75d174680474799563f21398390e1f715e6b3d72 Mon Sep 17 00:00:00 2001
From: Cian Johnston <cian@coder.com>
Date: Fri, 12 May 2023 14:58:03 +0100
Subject: [PATCH 22/26] fixup! update README

---
 scaletest/terraform/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scaletest/terraform/README.md b/scaletest/terraform/README.md
index a4bdefeccb394..f5a2bc376d9c2 100644
--- a/scaletest/terraform/README.md
+++ b/scaletest/terraform/README.md
@@ -30,7 +30,7 @@ project_id = "some_google_project_id"
    and emit the Coder URL as an output.
 
 1. Run `coder_init.sh <coder_url>` to setup an initial user and a pre-configured Kubernetes
-   template. It will also down the Coder CLI locally.
+   template. It will also download the Coder CLI from the Coder instance locally.
 
 1. Do whatever you need to do with the Coder instance.
 

From 5a3c8017ae8302b57f13c6eb840a2aa86cb0103a Mon Sep 17 00:00:00 2001
From: Cian Johnston <cian@coder.com>
Date: Fri, 12 May 2023 15:59:47 +0100
Subject: [PATCH 23/26] update cluster monitoriong and workload identity

---
 scaletest/terraform/gcp_cluster.tf | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scaletest/terraform/gcp_cluster.tf b/scaletest/terraform/gcp_cluster.tf
index 3d6aede6cc9d5..fa6bd19ebb8e3 100644
--- a/scaletest/terraform/gcp_cluster.tf
+++ b/scaletest/terraform/gcp_cluster.tf
@@ -24,11 +24,14 @@ resource "google_container_cluster" "primary" {
     google_project_service.api["container.googleapis.com"]
   ]
   monitoring_config {
-    enable_components = []
+    enable_components = ["SYSTEM_COMPONENTS"]
     managed_prometheus {
       enabled = true
     }
   }
+  workload_identity_config {
+    workload_pool = "${data.google_project.project.project_id}.svc.id.goog"
+  }
 }
 
 resource "google_container_node_pool" "coder" {

From caa04d4f2e7b8b7d313ac230bce09e4f79419493 Mon Sep 17 00:00:00 2001
From: Cian Johnston <cian@coder.com>
Date: Fri, 12 May 2023 17:08:37 +0100
Subject: [PATCH 24/26] fix coder depoyment node affinity

---
 scaletest/terraform/coder.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scaletest/terraform/coder.tf b/scaletest/terraform/coder.tf
index 36ff9aeab13ba..726383a1a0575 100644
--- a/scaletest/terraform/coder.tf
+++ b/scaletest/terraform/coder.tf
@@ -71,7 +71,7 @@ coder:
         - matchExpressions:
           - key: "cloud.google.com/gke-nodepool"
             operator: "In"
-            values: ["${google_container_node_pool.workspaces.name}"]
+            values: ["${google_container_node_pool.coder.name}"]
     podAntiAffinity:
       preferredDuringSchedulingIgnoredDuringExecution:
       - weight: 1

From 941970150c5cd7730dab1511abd65e0d90516d69 Mon Sep 17 00:00:00 2001
From: Cian Johnston <cian@coder.com>
Date: Mon, 15 May 2023 15:37:54 +0100
Subject: [PATCH 25/26] address PR comments

---
 scaletest/terraform/coder.tf       | 17 +----------------
 scaletest/terraform/gcp_cluster.tf | 20 ++++----------------
 scaletest/terraform/gcp_db.tf      | 13 ++-----------
 scaletest/terraform/gcp_vpc.tf     |  4 ++--
 scaletest/terraform/vars.tf        |  5 +++++
 5 files changed, 14 insertions(+), 45 deletions(-)

diff --git a/scaletest/terraform/coder.tf b/scaletest/terraform/coder.tf
index 726383a1a0575..d86aa2a7fe1ad 100644
--- a/scaletest/terraform/coder.tf
+++ b/scaletest/terraform/coder.tf
@@ -3,7 +3,7 @@ data "google_client_config" "default" {}
 locals {
   coder_helm_repo    = "https://helm.coder.com/v2"
   coder_helm_chart   = "coder"
-  coder_release_name = "coder-${var.name}"
+  coder_release_name = var.name
   coder_namespace    = "coder-${var.name}"
   coder_admin_email  = "admin@coder.com"
   coder_admin_user   = "coder"
@@ -238,21 +238,6 @@ resource "local_file" "kubernetes_template" {
               }
             }
           }
-          pod_affinity {
-            preferred_during_scheduling_ignored_during_execution {
-              weight = 1
-              pod_affinity_term {
-                topology_key = "kubernetes.io/hostname"
-                label_selector {
-                  match_expressions {
-                    key      = "app.kubernetes.io/name"
-                    operator = "In"
-                    values   = ["coder-workspace"]
-                  }
-                }
-              }
-            }
-          }
         }
       }
     }
diff --git a/scaletest/terraform/gcp_cluster.tf b/scaletest/terraform/gcp_cluster.tf
index fa6bd19ebb8e3..df4bd551c9d75 100644
--- a/scaletest/terraform/gcp_cluster.tf
+++ b/scaletest/terraform/gcp_cluster.tf
@@ -3,7 +3,7 @@ data "google_compute_default_service_account" "default" {
 }
 
 resource "google_container_cluster" "primary" {
-  name            = "${var.name}-cluster"
+  name            = var.name
   location        = var.zone
   project         = var.project_id
   network         = google_compute_network.vpc.name
@@ -35,7 +35,7 @@ resource "google_container_cluster" "primary" {
 }
 
 resource "google_container_node_pool" "coder" {
-  name       = "${var.name}-node-pool-coder"
+  name       = "${var.name}-coder"
   location   = var.zone
   project    = var.project_id
   cluster    = google_container_cluster.primary.name
@@ -62,14 +62,10 @@ resource "google_container_node_pool" "coder" {
       disable-legacy-endpoints = "true"
     }
   }
-
-  depends_on = [
-    google_container_cluster.primary
-  ]
 }
 
 resource "google_container_node_pool" "workspaces" {
-  name       = "${var.name}-node-pool-workspaces"
+  name       = "${var.name}-workspaces"
   location   = var.zone
   project    = var.project_id
   cluster    = google_container_cluster.primary.name
@@ -96,14 +92,10 @@ resource "google_container_node_pool" "workspaces" {
       disable-legacy-endpoints = "true"
     }
   }
-
-  depends_on = [
-    google_container_cluster.primary
-  ]
 }
 
 resource "google_container_node_pool" "misc" {
-  name       = "${var.name}-node-pool-misc"
+  name       = "${var.name}-misc"
   location   = var.zone
   project    = var.project_id
   cluster    = google_container_cluster.primary.name
@@ -130,8 +122,4 @@ resource "google_container_node_pool" "misc" {
       disable-legacy-endpoints = "true"
     }
   }
-
-  depends_on = [
-    google_container_cluster.primary
-  ]
 }
diff --git a/scaletest/terraform/gcp_db.tf b/scaletest/terraform/gcp_db.tf
index 3122f5c0f2223..b57002f2d2872 100644
--- a/scaletest/terraform/gcp_db.tf
+++ b/scaletest/terraform/gcp_db.tf
@@ -1,14 +1,5 @@
-data "google_compute_network" "default" {
-  project = var.project_id
-  name    = "default"
-}
-
-data "google_compute_global_address" "sql_peering" {
-  name = "sql-ip-address"
-}
-
 resource "google_sql_database_instance" "db" {
-  name                = "${var.name}-db"
+  name                = var.name
   region              = var.region
   database_version    = var.cloudsql_version
   deletion_protection = false
@@ -26,7 +17,7 @@ resource "google_sql_database_instance" "db" {
 
     database_flags {
       name  = "max_connections"
-      value = "500"
+      value = var.cloudsql_max_connections
     }
 
     ip_configuration {
diff --git a/scaletest/terraform/gcp_vpc.tf b/scaletest/terraform/gcp_vpc.tf
index 59c65a1355862..7ed76a00235e9 100644
--- a/scaletest/terraform/gcp_vpc.tf
+++ b/scaletest/terraform/gcp_vpc.tf
@@ -1,6 +1,6 @@
 resource "google_compute_network" "vpc" {
   project                 = var.project_id
-  name                    = "${var.name}-vpc"
+  name                    = var.name
   auto_create_subnetworks = "false"
   depends_on = [
     google_project_service.api["compute.googleapis.com"]
@@ -8,7 +8,7 @@ resource "google_compute_network" "vpc" {
 }
 
 resource "google_compute_subnetwork" "subnet" {
-  name          = "${var.name}-subnet"
+  name          = var.name
   project       = var.project_id
   region        = var.region
   network       = google_compute_network.vpc.name
diff --git a/scaletest/terraform/vars.tf b/scaletest/terraform/vars.tf
index a42034719ebc7..a5389dac7b335 100644
--- a/scaletest/terraform/vars.tf
+++ b/scaletest/terraform/vars.tf
@@ -87,6 +87,11 @@ variable "cloudsql_tier" {
   default     = "db-f1-micro"
 }
 
+variable "cloudsql_max_connections" {
+  description = "CloudSQL database max_connections"
+  default = 500
+}
+
 // These variables control the Coder deployment.
 variable "coder_replicas" {
   description = "Number of Coder replicas to provision"

From 435e74d22c933d9cf9d202503681846952b0b10e Mon Sep 17 00:00:00 2001
From: Cian Johnston <cian@coder.com>
Date: Mon, 15 May 2023 15:46:14 +0100
Subject: [PATCH 26/26] make fmt

---
 scaletest/terraform/vars.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scaletest/terraform/vars.tf b/scaletest/terraform/vars.tf
index a5389dac7b335..e312c4e542215 100644
--- a/scaletest/terraform/vars.tf
+++ b/scaletest/terraform/vars.tf
@@ -89,7 +89,7 @@ variable "cloudsql_tier" {
 
 variable "cloudsql_max_connections" {
   description = "CloudSQL database max_connections"
-  default = 500
+  default     = 500
 }
 
 // These variables control the Coder deployment.