From 61792edbda13492c65e620815b3d9937778bdaf6 Mon Sep 17 00:00:00 2001
From: Garrett Delfosse <garrett@coder.com>
Date: Thu, 14 Nov 2024 17:25:09 +0000
Subject: [PATCH 01/24] checkpoint

---
 scaletest/terraform/infra/gcp_cluster.tf |   1 +
 scaletest/terraform/infra/main.tf        |   2 +-
 scaletest/terraform/k8s/cert-manager.tf  |  78 +++++++++------
 scaletest/terraform/k8s/coder.tf         | 101 +++++++++++--------
 scaletest/terraform/k8s/main.tf          |  11 ++-
 scaletest/terraform/k8s/otel.tf          | 121 +++++++++++++++--------
 scaletest/terraform/k8s/prometheus.tf    |  64 +++++++-----
 7 files changed, 242 insertions(+), 136 deletions(-)

diff --git a/scaletest/terraform/infra/gcp_cluster.tf b/scaletest/terraform/infra/gcp_cluster.tf
index c37132c38071b..5032f2e6b93fa 100644
--- a/scaletest/terraform/infra/gcp_cluster.tf
+++ b/scaletest/terraform/infra/gcp_cluster.tf
@@ -1,5 +1,6 @@
 data "google_compute_default_service_account" "default" {
   project = var.project_id
+  depends_on = [ google_project_service.api["compute.googleapis.com"] ]
 }
 
 locals {
diff --git a/scaletest/terraform/infra/main.tf b/scaletest/terraform/infra/main.tf
index 1724692b19f3a..0c07534b1ebd2 100644
--- a/scaletest/terraform/infra/main.tf
+++ b/scaletest/terraform/infra/main.tf
@@ -11,7 +11,7 @@ terraform {
     }
   }
 
-  required_version = "~> 1.5.0"
+  required_version = "~> 1.9.0"
 }
 
 provider "google" {
diff --git a/scaletest/terraform/k8s/cert-manager.tf b/scaletest/terraform/k8s/cert-manager.tf
index cfcb324b3ea0b..f0d5f099241a9 100644
--- a/scaletest/terraform/k8s/cert-manager.tf
+++ b/scaletest/terraform/k8s/cert-manager.tf
@@ -36,32 +36,54 @@ EOF
   ]
 }
 
-resource "kubernetes_manifest" "cloudflare-cluster-issuer" {
-  manifest = {
-    apiVersion = "cert-manager.io/v1"
-    kind       = "ClusterIssuer"
-    metadata = {
-      name = "cloudflare-issuer"
-    }
-    spec = {
-      acme = {
-        email = var.cloudflare_email
-        privateKeySecretRef = {
-          name = local.cloudflare_issuer_private_key_secret_name
-        }
-        solvers = [
-          {
-            dns01 = {
-              cloudflare = {
-                apiTokenSecretRef = {
-                  name = kubernetes_secret.cloudflare-api-key.metadata.0.name
-                  key  = "api-token"
-                }
-              }
-            }
-          }
-        ]
-      }
-    }
-  }
+# resource "kubernetes_manifest" "cloudflare-cluster-issuer" {
+#   manifest = {
+#     apiVersion = "cert-manager.io/v1"
+#     kind       = "ClusterIssuer"
+#     metadata = {
+#       name = "cloudflare-issuer"
+#     }
+#     spec = {
+#       acme = {
+#         email = var.cloudflare_email
+#         privateKeySecretRef = {
+#           name = local.cloudflare_issuer_private_key_secret_name
+#         }
+#         solvers = [
+#           {
+#             dns01 = {
+#               cloudflare = {
+#                 apiTokenSecretRef = {
+#                   name = kubernetes_secret.cloudflare-api-key.metadata.0.name
+#                   key  = "api-token"
+#                 }
+#               }
+#             }
+#           }
+#         ]
+#       }
+#     }
+#   }
+# }
+
+resource "kubectl_manifest" "cloudflare-cluster-issuer" {
+  depends_on = [ helm_release.cert-manager ]
+    yaml_body = <<YAML
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: cloudflare-issuer
+spec:
+  acme:
+    email: ${var.cloudflare_email}
+    server: https://acme-staging-v02.api.letsencrypt.org/directory
+    privateKeySecretRef:
+      name: ${local.cloudflare_issuer_private_key_secret_name}
+    solvers:
+      - dns01:
+          cloudflare:
+            apiTokenSecretRef:
+              name: ${kubernetes_secret.cloudflare-api-key.metadata.0.name}
+              key: api-token
+YAML
 }
diff --git a/scaletest/terraform/k8s/coder.tf b/scaletest/terraform/k8s/coder.tf
index ea83317127fd8..7388f072a6798 100644
--- a/scaletest/terraform/k8s/coder.tf
+++ b/scaletest/terraform/k8s/coder.tf
@@ -10,6 +10,7 @@ locals {
   coder_release_name        = var.name
   provisionerd_helm_chart   = "coder-provisioner"
   provisionerd_release_name = "${var.name}-provisionerd"
+  dnsNames = regex("https?://([^/]+)", local.coder_url)
 }
 
 resource "kubernetes_namespace" "coder_namespace" {
@@ -61,23 +62,41 @@ data "kubernetes_secret" "coder_oidc" {
   }
 }
 
-resource "kubernetes_manifest" "coder_certificate" {
-  manifest = {
-    apiVersion = "cert-manager.io/v1"
-    kind       = "Certificate"
-    metadata = {
-      name      = "${var.name}"
-      namespace = kubernetes_namespace.coder_namespace.metadata.0.name
-    }
-    spec = {
-      secretName = "${var.name}-tls"
-      dnsNames   = regex("https?://([^/]+)", local.coder_url)
-      issuerRef = {
-        name = kubernetes_manifest.cloudflare-cluster-issuer.manifest.metadata.name
-        kind = "ClusterIssuer"
-      }
-    }
-  }
+# resource "kubernetes_manifest" "coder_certificate" {
+#   manifest = {
+#     apiVersion = "cert-manager.io/v1"
+#     kind       = "Certificate"
+#     metadata = {
+#       name      = "${var.name}"
+#       namespace = kubernetes_namespace.coder_namespace.metadata.0.name
+#     }
+#     spec = {
+#       secretName = "${var.name}-tls"
+#       dnsNames   = regex("https?://([^/]+)", local.coder_url)
+#       issuerRef = {
+#         name = "cloudflare-issuer"
+#         kind = "ClusterIssuer"
+#       }
+#     }
+#   }
+# }
+
+resource "kubectl_manifest" "coder_certificate" {
+  depends_on = [ helm_release.cert-manager ]
+  yaml_body = <<YAML
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: ${var.name}
+  namespace: ${kubernetes_namespace.coder_namespace.metadata.0.name}
+spec:
+  secretName: ${var.name}-tls
+  dnsNames:
+    - ${local.dnsNames.0}
+  issuerRef:
+    name: cloudflare-issuer
+    kind: ClusterIssuer
+YAML
 }
 
 data "kubernetes_secret" "coder_tls" {
@@ -85,7 +104,7 @@ data "kubernetes_secret" "coder_tls" {
     namespace = kubernetes_namespace.coder_namespace.metadata.0.name
     name      = "${var.name}-tls"
   }
-  depends_on = [kubernetes_manifest.coder_certificate]
+  depends_on = [kubectl_manifest.coder_certificate]
 }
 
 resource "helm_release" "coder-chart" {
@@ -153,29 +172,29 @@ coder:
           key: psk
           name: "${kubernetes_secret.provisionerd_psk.metadata.0.name}"
     # Enable OIDC
-    - name: "CODER_OIDC_ISSUER_URL"
-      valueFrom:
-        secretKeyRef:
-          key: issuer-url
-          name: "${data.kubernetes_secret.coder_oidc.metadata.0.name}"
-    - name: "CODER_OIDC_EMAIL_DOMAIN"
-      valueFrom:
-        secretKeyRef:
-          key: email-domain
-          name: "${data.kubernetes_secret.coder_oidc.metadata.0.name}"
-    - name: "CODER_OIDC_CLIENT_ID"
-      valueFrom:
-        secretKeyRef:
-          key: client-id
-          name: "${data.kubernetes_secret.coder_oidc.metadata.0.name}"
-    - name: "CODER_OIDC_CLIENT_SECRET"
-      valueFrom:
-        secretKeyRef:
-          key: client-secret
-          name: "${data.kubernetes_secret.coder_oidc.metadata.0.name}"
+    # - name: "CODER_OIDC_ISSUER_URL"
+    #   valueFrom:
+    #     secretKeyRef:
+    #       key: issuer-url
+    #       name: "${data.kubernetes_secret.coder_oidc.metadata.0.name}"
+    # - name: "CODER_OIDC_EMAIL_DOMAIN"
+    #   valueFrom:
+    #     secretKeyRef:
+    #       key: email-domain
+    #       name: "${data.kubernetes_secret.coder_oidc.metadata.0.name}"
+    # - name: "CODER_OIDC_CLIENT_ID"
+    #   valueFrom:
+    #     secretKeyRef:
+    #       key: client-id
+    #       name: "${data.kubernetes_secret.coder_oidc.metadata.0.name}"
+    # - name: "CODER_OIDC_CLIENT_SECRET"
+    #   valueFrom:
+    #     secretKeyRef:
+    #       key: client-secret
+    #       name: "${data.kubernetes_secret.coder_oidc.metadata.0.name}"
     # Send OTEL traces to the cluster-local collector to sample 10%
     - name: "OTEL_EXPORTER_OTLP_ENDPOINT"
-      value: "http://${kubernetes_manifest.otel-collector.manifest.metadata.name}-collector.${kubernetes_namespace.coder_namespace.metadata.0.name}.svc.cluster.local:4317"
+      value: "http://otel-collector.${kubernetes_namespace.coder_namespace.metadata.0.name}.svc.cluster.local:4317"
     - name: "OTEL_TRACES_SAMPLER"
       value: parentbased_traceidratio
     - name: "OTEL_TRACES_SAMPLER_ARG"
@@ -240,6 +259,8 @@ coder:
       value: "${local.coder_url}"
     - name: "CODER_VERBOSE"
       value: "true"
+    - name: "CODER_CONFIG_DIR"
+      value: "/tmp/config"
     - name: "CODER_CACHE_DIRECTORY"
       value: "/tmp/coder"
     - name: "CODER_TELEMETRY_ENABLE"
@@ -251,7 +272,7 @@ coder:
     - name: "CODER_PROMETHEUS_ENABLE"
       value: "true"
     - name: "CODER_PROVISIONERD_TAGS"
-      value = "socpe=organization"
+      value: "scope=organization"
   image:
     repo: ${var.provisionerd_image_repo}
     tag: ${var.provisionerd_image_tag}
diff --git a/scaletest/terraform/k8s/main.tf b/scaletest/terraform/k8s/main.tf
index a5c8c1085a5ce..959ffb8b872dd 100644
--- a/scaletest/terraform/k8s/main.tf
+++ b/scaletest/terraform/k8s/main.tf
@@ -5,6 +5,11 @@ terraform {
       version = "~> 2.20"
     }
 
+    kubectl = {
+      source  = "alekc/kubectl"
+      version = ">= 2.0.0"
+    }
+
     helm = {
       source  = "hashicorp/helm"
       version = "~> 2.9"
@@ -21,13 +26,17 @@ terraform {
     }
   }
 
-  required_version = "~> 1.5.0"
+  required_version = "~> 1.9.0"
 }
 
 provider "kubernetes" {
   config_path = var.kubernetes_kubeconfig_path
 }
 
+provider "kubectl" {
+  config_path = var.kubernetes_kubeconfig_path
+}
+
 provider "helm" {
   kubernetes {
     config_path = var.kubernetes_kubeconfig_path
diff --git a/scaletest/terraform/k8s/otel.tf b/scaletest/terraform/k8s/otel.tf
index 3b1657ee48cbc..96375db02dcfe 100644
--- a/scaletest/terraform/k8s/otel.tf
+++ b/scaletest/terraform/k8s/otel.tf
@@ -3,7 +3,7 @@
 locals {
   otel_namespace              = "opentelemetry-operator-system"
   otel_operator_helm_repo     = "https://open-telemetry.github.io/opentelemetry-helm-charts"
-  otel_operator_helm_chart    = "opentelemtry-operator"
+  otel_operator_helm_chart    = "opentelemetry-operator"
   otel_operator_release_name  = "opentelemetry-operator"
   otel_operator_chart_version = "0.34.1"
 }
@@ -22,48 +22,83 @@ resource "helm_release" "otel-operator" {
   chart      = local.otel_operator_helm_chart
   name       = local.otel_operator_release_name
   namespace  = kubernetes_namespace.otel-namespace.metadata.0.name
-  # Default values
-  values = []
+  values = [<<EOF
+manager:
+  collectorImage:
+    repository: otel/opentelemetry-collector-k8s
+EOF
+  ]
 }
 
-resource "kubernetes_manifest" "otel-collector" {
-  manifest = {
-    apiVersion = "opentelemetry.io/v1alpha1"
-    kind       = "OpenTelemetryCollector"
-    metadata = {
-      namespace = kubernetes_namespace.coder_namespace.metadata.0.name
-      name      = "otel"
-    }
-    spec = {
-      config = jsonencode({
-        receivers = {
-          otlp = {
-            protocols : {
-              grpc : {}
-              http : {}
-            }
-          }
-        }
-        exporters = {
-          googlecloud = {
-            logging = {
-              loglevel = "debug"
-            }
-          }
-        }
-        service = {
-          pipelines = {
-            traces = {
-              receivers  = ["otlp"]
-              processors = []
-              exporters  = ["logging", "googlecloud"]
-            }
-          }
-        }
-        image    = "otel/open-telemetry-collector-contrib:latest"
-        mode     = "deployment"
-        replicas = 1
-      })
-    }
-  }
+# resource "kubernetes_manifest" "otel-collector" {
+#   manifest = {
+#     apiVersion = "opentelemetry.io/v1alpha1"
+#     kind       = "OpenTelemetryCollector"
+#     metadata = {
+#       namespace = kubernetes_namespace.coder_namespace.metadata.0.name
+#       name      = "otel"
+#     }
+#     spec = {
+#       config = jsonencode({
+#         receivers = {
+#           otlp = {
+#             protocols : {
+#               grpc : {}
+#               http : {}
+#             }
+#           }
+#         }
+#         exporters = {
+#           googlecloud = {
+#             logging = {
+#               loglevel = "debug"
+#             }
+#           }
+#         }
+#         service = {
+#           pipelines = {
+#             traces = {
+#               receivers  = ["otlp"]
+#               processors = []
+#               exporters  = ["logging", "googlecloud"]
+#             }
+#           }
+#         }
+#         image    = "otel/open-telemetry-collector-contrib:latest"
+#         mode     = "deployment"
+#         replicas = 1
+#       })
+#     }
+#   }
+# }
+
+resource "kubectl_manifest" "otel-collector" {
+  depends_on = [ helm_release.otel-operator ]
+    yaml_body = <<YAML
+apiVersion: opentelemetry.io/v1alpha1
+kind: OpenTelemetryCollector
+metadata:
+  name: otel
+  namespace: ${kubernetes_namespace.coder_namespace.metadata.0.name}
+spec:
+  config: |
+    receivers:
+      otlp:
+        protocols:
+          grpc: {}
+          http: {}
+    exporters:
+      googlecloud:
+        logging:
+          loglevel: debug
+    service:
+      pipelines:
+        traces:
+          receivers: [otlp]
+          processors: []
+          exporters: [logging, googlecloud]
+    image: otel/open-telemetry-collector-contrib:latest
+    mode: deployment
+    replicas: 1
+YAML
 }
diff --git a/scaletest/terraform/k8s/prometheus.tf b/scaletest/terraform/k8s/prometheus.tf
index accf926727575..610453b86766d 100644
--- a/scaletest/terraform/k8s/prometheus.tf
+++ b/scaletest/terraform/k8s/prometheus.tf
@@ -147,27 +147,45 @@ serviceMonitor:
   ]
 }
 
-resource "kubernetes_manifest" "coder_monitoring" {
-  depends_on = [helm_release.prometheus-chart]
-  manifest = {
-    apiVersion = "monitoring.coreos.com/v1"
-    kind       = "PodMonitor"
-    metadata = {
-      namespace = kubernetes_namespace.coder_namespace.metadata.0.name
-      name      = "coder-monitoring"
-    }
-    spec = {
-      selector = {
-        matchLabels = {
-          "app.kubernetes.io/name" : "coder"
-        }
-      }
-      podMetricsEndpoints = [
-        {
-          port     = "prometheus-http"
-          interval = "30s"
-        }
-      ]
-    }
-  }
+# resource "kubernetes_manifest" "coder_monitoring" {
+#   depends_on = [helm_release.prometheus-chart]
+#   manifest = {
+#     apiVersion = "monitoring.coreos.com/v1"
+#     kind       = "PodMonitor"
+#     metadata = {
+#       namespace = kubernetes_namespace.coder_namespace.metadata.0.name
+#       name      = "coder-monitoring"
+#     }
+#     spec = {
+#       selector = {
+#         matchLabels = {
+#           "app.kubernetes.io/name" : "coder"
+#         }
+#       }
+#       podMetricsEndpoints = [
+#         {
+#           port     = "prometheus-http"
+#           interval = "30s"
+#         }
+#       ]
+#     }
+#   }
+# }
+
+resource "kubectl_manifest" "coder_monitoring" {
+  depends_on = [ helm_release.prometheus-chart ]
+    yaml_body = <<YAML
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: coder-monitoring
+  namespace: ${kubernetes_namespace.coder_namespace.metadata.0.name}
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: coder
+  podMetricsEndpoints:
+    - port: prometheus-http
+      interval: 30s
+YAML
 }

From 5994591edaef6dcbada17f81ffde5ffccc271dfb Mon Sep 17 00:00:00 2001
From: Garrett Delfosse <garrett@coder.com>
Date: Thu, 14 Nov 2024 20:17:48 +0000
Subject: [PATCH 02/24] baseline with iter

---
 scaletest/terraform/infra/gcp_cluster.tf | 140 +++++------------------
 scaletest/terraform/infra/gcp_db.tf      |  17 +--
 scaletest/terraform/infra/gcp_vpc.tf     |  24 ++--
 scaletest/terraform/infra/main.tf        |   2 -
 scaletest/terraform/infra/outputs.tf     | 124 ++++++++++----------
 scaletest/terraform/infra/vars.tf        |  29 +++--
 scaletest/terraform/k8s/prometheus.tf    |   4 +-
 7 files changed, 130 insertions(+), 210 deletions(-)

diff --git a/scaletest/terraform/infra/gcp_cluster.tf b/scaletest/terraform/infra/gcp_cluster.tf
index 5032f2e6b93fa..ca3d20fdf925f 100644
--- a/scaletest/terraform/infra/gcp_cluster.tf
+++ b/scaletest/terraform/infra/gcp_cluster.tf
@@ -4,17 +4,35 @@ data "google_compute_default_service_account" "default" {
 }
 
 locals {
-  abs_module_path         = abspath(path.module)
-  rel_kubeconfig_path     = "../../.coderv2/${var.name}-cluster.kubeconfig"
-  cluster_kubeconfig_path = abspath("${local.abs_module_path}/${local.rel_kubeconfig_path}")
+  node_pools = flatten([ for i, deployment in var.deployments : [
+    {
+      name = "${var.name}-${deployment.name}-coder"
+      zone = deployment.zone
+      size = deployment.coder_node_pool_size
+      cluster_i = i
+    },     
+    {
+      name = "${var.name}-${deployment.name}-workspaces"
+      zone = deployment.zone
+      size = deployment.workspaces_node_pool_size
+      cluster_i = i
+    },
+    {
+      name = "${var.name}-${deployment.name}-misc"
+      zone = deployment.zone
+      size = deployment.misc_node_pool_size
+      cluster_i = i
+    }
+  ] ])
 }
 
-resource "google_container_cluster" "primary" {
-  name                      = var.name
-  location                  = var.zone
+resource "google_container_cluster" "cluster" {
+  count                     = length(var.deployments)
+  name                      = "${var.name}-${var.deployments[count.index].name}"
+  location                  = var.deployments[count.index].zone
   project                   = var.project_id
   network                   = google_compute_network.vpc.name
-  subnetwork                = google_compute_subnetwork.subnet.name
+  subnetwork                = google_compute_subnetwork.subnet[count.index].name
   networking_mode           = "VPC_NATIVE"
   default_max_pods_per_node = 256
   ip_allocation_policy { # Required with networking_mode=VPC_NATIVE
@@ -53,14 +71,15 @@ resource "google_container_cluster" "primary" {
   }
 }
 
-resource "google_container_node_pool" "coder" {
-  name     = "${var.name}-coder"
-  location = var.zone
+resource "google_container_node_pool" "node_pool" {
+  count    = length(local.node_pools)
+  name     = local.node_pools[count.index].name
+  location = local.node_pools[count.index].zone
   project  = var.project_id
-  cluster  = google_container_cluster.primary.name
+  cluster  = google_container_cluster.cluster[local.node_pools[count.index].cluster_i].name
   autoscaling {
     min_node_count = 1
-    max_node_count = var.nodepool_size_coder
+    max_node_count = local.node_pools[count.index].size
   }
   node_config {
     oauth_scopes = [
@@ -88,100 +107,3 @@ resource "google_container_node_pool" "coder" {
     ignore_changes = [management[0].auto_repair, management[0].auto_upgrade, timeouts]
   }
 }
-
-resource "google_container_node_pool" "workspaces" {
-  name     = "${var.name}-workspaces"
-  location = var.zone
-  project  = var.project_id
-  cluster  = google_container_cluster.primary.name
-  autoscaling {
-    min_node_count       = 0
-    total_max_node_count = var.nodepool_size_workspaces
-  }
-  management {
-    auto_upgrade = false
-  }
-  node_config {
-    oauth_scopes = [
-      "https://www.googleapis.com/auth/logging.write",
-      "https://www.googleapis.com/auth/monitoring",
-      "https://www.googleapis.com/auth/trace.append",
-      "https://www.googleapis.com/auth/devstorage.read_only",
-      "https://www.googleapis.com/auth/service.management.readonly",
-      "https://www.googleapis.com/auth/servicecontrol",
-    ]
-    disk_size_gb    = var.node_disk_size_gb
-    machine_type    = var.nodepool_machine_type_workspaces
-    image_type      = var.node_image_type
-    preemptible     = var.node_preemptible
-    service_account = data.google_compute_default_service_account.default.email
-    tags            = ["gke-node", "${var.project_id}-gke"]
-    labels = {
-      env = var.project_id
-    }
-    metadata = {
-      disable-legacy-endpoints = "true"
-    }
-  }
-  lifecycle {
-    ignore_changes = [management[0].auto_repair, management[0].auto_upgrade, timeouts]
-  }
-}
-
-resource "google_container_node_pool" "misc" {
-  name       = "${var.name}-misc"
-  location   = var.zone
-  project    = var.project_id
-  cluster    = google_container_cluster.primary.name
-  node_count = var.state == "stopped" ? 0 : var.nodepool_size_misc
-  management {
-    auto_upgrade = false
-  }
-  node_config {
-    oauth_scopes = [
-      "https://www.googleapis.com/auth/logging.write",
-      "https://www.googleapis.com/auth/monitoring",
-      "https://www.googleapis.com/auth/trace.append",
-      "https://www.googleapis.com/auth/devstorage.read_only",
-      "https://www.googleapis.com/auth/service.management.readonly",
-      "https://www.googleapis.com/auth/servicecontrol",
-    ]
-    disk_size_gb    = var.node_disk_size_gb
-    machine_type    = var.nodepool_machine_type_misc
-    image_type      = var.node_image_type
-    preemptible     = var.node_preemptible
-    service_account = data.google_compute_default_service_account.default.email
-    tags            = ["gke-node", "${var.project_id}-gke"]
-    labels = {
-      env = var.project_id
-    }
-    metadata = {
-      disable-legacy-endpoints = "true"
-    }
-  }
-  lifecycle {
-    ignore_changes = [management[0].auto_repair, management[0].auto_upgrade, timeouts]
-  }
-}
-
-resource "null_resource" "cluster_kubeconfig" {
-  depends_on = [google_container_cluster.primary]
-  triggers = {
-    path       = local.cluster_kubeconfig_path
-    name       = google_container_cluster.primary.name
-    project_id = var.project_id
-    zone       = var.zone
-  }
-  provisioner "local-exec" {
-    command = <<EOF
-      KUBECONFIG=${self.triggers.path} gcloud container clusters get-credentials ${self.triggers.name} --project=${self.triggers.project_id} --zone=${self.triggers.zone}
-    EOF
-  }
-
-  provisioner "local-exec" {
-    when    = destroy
-    command = <<EOF
-      rm -f ${self.triggers.path}
-    EOF
-  }
-}
diff --git a/scaletest/terraform/infra/gcp_db.tf b/scaletest/terraform/infra/gcp_db.tf
index 4d13b262c615f..71d92c4a77c9f 100644
--- a/scaletest/terraform/infra/gcp_db.tf
+++ b/scaletest/terraform/infra/gcp_db.tf
@@ -1,6 +1,7 @@
 resource "google_sql_database_instance" "db" {
-  name                = var.name
-  region              = var.region
+  name                = "${var.name}-coder"
+  project             = var.project_id
+  region              = var.deployments[0].region
   database_version    = var.cloudsql_version
   deletion_protection = false
 
@@ -12,7 +13,7 @@ resource "google_sql_database_instance" "db" {
     availability_type = "ZONAL"
 
     location_preference {
-      zone = var.zone
+      zone = var.deployments[0].zone
     }
 
     database_flags {
@@ -49,11 +50,11 @@ resource "google_sql_database" "coder" {
   }
 }
 
-resource "random_password" "coder-postgres-password" {
+resource "random_password" "coder_postgres_password" {
   length = 12
 }
 
-resource "random_password" "prometheus-postgres-password" {
+resource "random_password" "prometheus_postgres_password" {
   length = 12
 }
 
@@ -62,7 +63,7 @@ resource "google_sql_user" "coder" {
   instance = google_sql_database_instance.db.id
   name     = "${var.name}-coder"
   type     = "BUILT_IN"
-  password = random_password.coder-postgres-password.result
+  password = random_password.coder_postgres_password.result
   # required for postgres, otherwise user fails to delete
   deletion_policy = "ABANDON"
   lifecycle {
@@ -75,7 +76,7 @@ resource "google_sql_user" "prometheus" {
   instance = google_sql_database_instance.db.id
   name     = "${var.name}-prometheus"
   type     = "BUILT_IN"
-  password = random_password.prometheus-postgres-password.result
+  password = random_password.prometheus_postgres_password.result
   # required for postgres, otherwise user fails to delete
   deletion_policy = "ABANDON"
   lifecycle {
@@ -84,5 +85,5 @@ resource "google_sql_user" "prometheus" {
 }
 
 locals {
-  coder_db_url = "postgres://${google_sql_user.coder.name}:${urlencode(random_password.coder-postgres-password.result)}@${google_sql_database_instance.db.private_ip_address}/${google_sql_database.coder.name}?sslmode=disable"
+  coder_db_url = "postgres://${google_sql_user.coder.name}:${urlencode(random_password.coder_postgres_password.result)}@${google_sql_database_instance.db.private_ip_address}/${google_sql_database.coder.name}?sslmode=disable"
 }
diff --git a/scaletest/terraform/infra/gcp_vpc.tf b/scaletest/terraform/infra/gcp_vpc.tf
index b125c60cfd25a..db557c5735b2d 100644
--- a/scaletest/terraform/infra/gcp_vpc.tf
+++ b/scaletest/terraform/infra/gcp_vpc.tf
@@ -8,11 +8,21 @@ resource "google_compute_network" "vpc" {
 }
 
 resource "google_compute_subnetwork" "subnet" {
-  name          = var.name
+  count         = length(var.deployments)
+  name          = "${var.name}-${var.deployments[count.index].name}"
   project       = var.project_id
-  region        = var.region
+  region        = var.deployments[count.index].region
   network       = google_compute_network.vpc.name
-  ip_cidr_range = var.subnet_cidr
+  ip_cidr_range = var.deployments[count.index].subnet_cidr
+}
+
+resource "google_compute_address" "coder" {
+  count        = length(var.deployments)
+  project      = var.project_id
+  region       = var.deployments[count.index].region
+  name         = "${var.name}-${var.deployments[count.index].name}-coder"
+  address_type = "EXTERNAL"
+  network_tier = "PREMIUM"
 }
 
 resource "google_compute_global_address" "sql_peering" {
@@ -24,14 +34,6 @@ resource "google_compute_global_address" "sql_peering" {
   network       = google_compute_network.vpc.id
 }
 
-resource "google_compute_address" "coder" {
-  project      = var.project_id
-  region       = var.region
-  name         = "${var.name}-coder"
-  address_type = "EXTERNAL"
-  network_tier = "PREMIUM"
-}
-
 resource "google_service_networking_connection" "private_vpc_connection" {
   network                 = google_compute_network.vpc.id
   service                 = "servicenetworking.googleapis.com"
diff --git a/scaletest/terraform/infra/main.tf b/scaletest/terraform/infra/main.tf
index 0c07534b1ebd2..dce0228138668 100644
--- a/scaletest/terraform/infra/main.tf
+++ b/scaletest/terraform/infra/main.tf
@@ -15,6 +15,4 @@ terraform {
 }
 
 provider "google" {
-  region  = var.region
-  project = var.project_id
 }
diff --git a/scaletest/terraform/infra/outputs.tf b/scaletest/terraform/infra/outputs.tf
index f5e619eca384d..5c8bb2f5eb57b 100644
--- a/scaletest/terraform/infra/outputs.tf
+++ b/scaletest/terraform/infra/outputs.tf
@@ -1,73 +1,73 @@
-output "coder_db_url" {
-  description = "URL of the database for Coder."
-  value       = local.coder_db_url
-  sensitive   = true
-}
+# output "coder_db_url" {
+#   description = "URL of the database for Coder."
+#   value       = local.coder_db_url
+#   sensitive   = true
+# }
 
-output "coder_address" {
-  description = "IP address to use for the Coder service."
-  value       = google_compute_address.coder.address
-}
+# output "coder_address" {
+#   description = "IP address to use for the Coder service."
+#   value       = google_compute_address.coder_primary.address
+# }
 
-output "kubernetes_kubeconfig_path" {
-  description = "Kubeconfig path."
-  value       = local.cluster_kubeconfig_path
-}
+# # output "kubernetes_kubeconfig_path" {
+# #   description = "Kubeconfig path."
+# #   value       = local.cluster_kubeconfig_path
+# # }
 
-output "kubernetes_nodepool_coder" {
-  description = "Name of the nodepool on which to run Coder."
-  value       = google_container_node_pool.coder.name
-}
+# output "kubernetes_nodepool_coder" {
+#   description = "Name of the nodepool on which to run Coder."
+#   value       = google_container_node_pool.coder.name
+# }
 
-output "kubernetes_nodepool_misc" {
-  description = "Name of the nodepool on which to run everything else."
-  value       = google_container_node_pool.misc.name
-}
+# output "kubernetes_nodepool_misc" {
+#   description = "Name of the nodepool on which to run everything else."
+#   value       = google_container_node_pool.misc.name
+# }
 
-output "kubernetes_nodepool_workspaces" {
-  description = "Name of the nodepool on which to run workspaces."
-  value       = google_container_node_pool.workspaces.name
-}
+# output "kubernetes_nodepool_workspaces" {
+#   description = "Name of the nodepool on which to run workspaces."
+#   value       = google_container_node_pool.workspaces.name
+# }
 
-output "prometheus_external_label_cluster" {
-  description = "Value for the Prometheus external label named cluster."
-  value       = google_container_cluster.primary.name
-}
+# output "prometheus_external_label_cluster" {
+#   description = "Value for the Prometheus external label named cluster."
+#   value       = google_container_cluster.primary.name
+# }
 
-output "prometheus_postgres_dbname" {
-  description = "Name of the database for Prometheus to monitor."
-  value       = google_sql_database.coder.name
-}
+# output "prometheus_postgres_dbname" {
+#   description = "Name of the database for Prometheus to monitor."
+#   value       = google_sql_database.coder.name
+# }
 
-output "prometheus_postgres_host" {
-  description = "Hostname of the database for Prometheus to connect to."
-  value       = google_sql_database_instance.db.private_ip_address
-}
+# output "prometheus_postgres_host" {
+#   description = "Hostname of the database for Prometheus to connect to."
+#   value       = google_sql_database_instance.db.private_ip_address
+# }
 
-output "prometheus_postgres_password" {
-  description = "Postgres password for Prometheus."
-  value       = random_password.prometheus-postgres-password.result
-  sensitive   = true
-}
+# output "prometheus_postgres_password" {
+#   description = "Postgres password for Prometheus."
+#   value       = random_password.prometheus_postgres_password.result
+#   sensitive   = true
+# }
 
-output "prometheus_postgres_user" {
-  description = "Postgres username for Prometheus."
-  value       = google_sql_user.prometheus.name
-}
+# output "prometheus_postgres_user" {
+#   description = "Postgres username for Prometheus."
+#   value       = google_sql_user.prometheus.name
+# }
 
-resource "local_file" "outputs" {
-  filename = "${path.module}/../../.coderv2/infra_outputs.tfvars"
-  content  = <<EOF
-  coder_db_url = "${local.coder_db_url}"
-  coder_address = "${google_compute_address.coder.address}"
-  kubernetes_kubeconfig_path = "${local.cluster_kubeconfig_path}"
-  kubernetes_nodepool_coder = "${google_container_node_pool.coder.name}"
-  kubernetes_nodepool_misc = "${google_container_node_pool.misc.name}"
-  kubernetes_nodepool_workspaces = "${google_container_node_pool.workspaces.name}"
-  prometheus_external_label_cluster = "${google_container_cluster.primary.name}"
-  prometheus_postgres_dbname = "${google_sql_database.coder.name}"
-  prometheus_postgres_host = "${google_sql_database_instance.db.private_ip_address}"
-  prometheus_postgres_password = "${random_password.prometheus-postgres-password.result}"
-  prometheus_postgres_user = "${google_sql_user.prometheus.name}"
-EOF
-}
+# resource "local_file" "outputs" {
+#   filename = "${path.module}/../../.coderv2/infra_outputs.tfvars"
+#   content  = <<EOF
+#   coder_db_url = "${local.coder_db_url}"
+#   coder_address = "${google_compute_address.coder_primary.address}"
+#   kubernetes_kubeconfig_path = "${local.cluster_kubeconfig_path}"
+#   kubernetes_nodepool_coder = "${google_container_node_pool.coder.name}"
+#   kubernetes_nodepool_misc = "${google_container_node_pool.misc.name}"
+#   kubernetes_nodepool_workspaces = "${google_container_node_pool.workspaces.name}"
+#   prometheus_external_label_cluster = "${google_container_cluster.primary.name}"
+#   prometheus_postgres_dbname = "${google_sql_database.coder.name}"
+#   prometheus_postgres_host = "${google_sql_database_instance.db.private_ip_address}"
+#   prometheus_postgres_password = "${random_password.prometheus_postgres_password.result}"
+#   prometheus_postgres_user = "${google_sql_user.prometheus.name}"
+# EOF
+# }
diff --git a/scaletest/terraform/infra/vars.tf b/scaletest/terraform/infra/vars.tf
index d9f5040918ba5..3b39f2142719a 100644
--- a/scaletest/terraform/infra/vars.tf
+++ b/scaletest/terraform/infra/vars.tf
@@ -12,22 +12,19 @@ variable "project_id" {
 }
 
 variable "name" {
-  description = "Adds a prefix to resources."
-}
-
-variable "region" {
-  description = "GCP region in which to provision resources."
-  default     = "us-east1"
-}
-
-variable "zone" {
-  description = "GCP zone in which to provision resources."
-  default     = "us-east1-c"
-}
-
-variable "subnet_cidr" {
-  description = "CIDR range for the subnet."
-  default     = "10.200.0.0/24"
+  description = "The name all resources will be prefixed with"
+}
+
+variable "deployments" {
+  type = list(object({
+    name = string
+    region = string
+    zone   = string
+    subnet_cidr = string
+    coder_node_pool_size = number
+    workspaces_node_pool_size = number
+    misc_node_pool_size = number
+  }))
 }
 
 variable "k8s_version" {
diff --git a/scaletest/terraform/k8s/prometheus.tf b/scaletest/terraform/k8s/prometheus.tf
index 610453b86766d..c92c6abe2df41 100644
--- a/scaletest/terraform/k8s/prometheus.tf
+++ b/scaletest/terraform/k8s/prometheus.tf
@@ -101,7 +101,7 @@ prometheus:
   ]
 }
 
-resource "kubernetes_secret" "prometheus-postgres-password" {
+resource "kubernetes_secret" "prometheus_postgres_password" {
   type = "kubernetes.io/basic-auth"
   metadata {
     name      = "prometheus-postgres"
@@ -138,7 +138,7 @@ config:
     user: "${var.prometheus_postgres_user}"
     database: "${var.prometheus_postgres_dbname}"
     passwordSecret:
-      name: "${kubernetes_secret.prometheus-postgres-password.metadata.0.name}"
+      name: "${kubernetes_secret.prometheus_postgres_password.metadata.0.name}"
       key: password
     autoDiscoverDatabases: true
 serviceMonitor:

From ece56e863eb78ff3003b9d792dad95c926d7a89e Mon Sep 17 00:00:00 2001
From: Garrett Delfosse <garrett@coder.com>
Date: Mon, 18 Nov 2024 18:12:14 +0000
Subject: [PATCH 03/24] move to new dir

---
 scaletest/terraform/infra/gcp_cluster.tf | 141 ++++++++++++++++++-----
 scaletest/terraform/infra/gcp_db.tf      |  17 ++-
 scaletest/terraform/infra/gcp_vpc.tf     |  24 ++--
 scaletest/terraform/infra/main.tf        |   4 +-
 scaletest/terraform/infra/outputs.tf     | 124 ++++++++++----------
 scaletest/terraform/infra/vars.tf        |  29 ++---
 scaletest/terraform/new/gcp_cluster.tf   | 109 ++++++++++++++++++
 scaletest/terraform/new/gcp_db.tf        |  89 ++++++++++++++
 scaletest/terraform/new/gcp_project.tf   |  27 +++++
 scaletest/terraform/new/gcp_vpc.tf       |  41 +++++++
 scaletest/terraform/new/main.tf          |  18 +++
 scaletest/terraform/new/vars.tf          |  63 ++++++++++
 12 files changed, 556 insertions(+), 130 deletions(-)
 create mode 100644 scaletest/terraform/new/gcp_cluster.tf
 create mode 100644 scaletest/terraform/new/gcp_db.tf
 create mode 100644 scaletest/terraform/new/gcp_project.tf
 create mode 100644 scaletest/terraform/new/gcp_vpc.tf
 create mode 100644 scaletest/terraform/new/main.tf
 create mode 100644 scaletest/terraform/new/vars.tf

diff --git a/scaletest/terraform/infra/gcp_cluster.tf b/scaletest/terraform/infra/gcp_cluster.tf
index ca3d20fdf925f..c37132c38071b 100644
--- a/scaletest/terraform/infra/gcp_cluster.tf
+++ b/scaletest/terraform/infra/gcp_cluster.tf
@@ -1,38 +1,19 @@
 data "google_compute_default_service_account" "default" {
   project = var.project_id
-  depends_on = [ google_project_service.api["compute.googleapis.com"] ]
 }
 
 locals {
-  node_pools = flatten([ for i, deployment in var.deployments : [
-    {
-      name = "${var.name}-${deployment.name}-coder"
-      zone = deployment.zone
-      size = deployment.coder_node_pool_size
-      cluster_i = i
-    },     
-    {
-      name = "${var.name}-${deployment.name}-workspaces"
-      zone = deployment.zone
-      size = deployment.workspaces_node_pool_size
-      cluster_i = i
-    },
-    {
-      name = "${var.name}-${deployment.name}-misc"
-      zone = deployment.zone
-      size = deployment.misc_node_pool_size
-      cluster_i = i
-    }
-  ] ])
+  abs_module_path         = abspath(path.module)
+  rel_kubeconfig_path     = "../../.coderv2/${var.name}-cluster.kubeconfig"
+  cluster_kubeconfig_path = abspath("${local.abs_module_path}/${local.rel_kubeconfig_path}")
 }
 
-resource "google_container_cluster" "cluster" {
-  count                     = length(var.deployments)
-  name                      = "${var.name}-${var.deployments[count.index].name}"
-  location                  = var.deployments[count.index].zone
+resource "google_container_cluster" "primary" {
+  name                      = var.name
+  location                  = var.zone
   project                   = var.project_id
   network                   = google_compute_network.vpc.name
-  subnetwork                = google_compute_subnetwork.subnet[count.index].name
+  subnetwork                = google_compute_subnetwork.subnet.name
   networking_mode           = "VPC_NATIVE"
   default_max_pods_per_node = 256
   ip_allocation_policy { # Required with networking_mode=VPC_NATIVE
@@ -71,15 +52,14 @@ resource "google_container_cluster" "cluster" {
   }
 }
 
-resource "google_container_node_pool" "node_pool" {
-  count    = length(local.node_pools)
-  name     = local.node_pools[count.index].name
-  location = local.node_pools[count.index].zone
+resource "google_container_node_pool" "coder" {
+  name     = "${var.name}-coder"
+  location = var.zone
   project  = var.project_id
-  cluster  = google_container_cluster.cluster[local.node_pools[count.index].cluster_i].name
+  cluster  = google_container_cluster.primary.name
   autoscaling {
     min_node_count = 1
-    max_node_count = local.node_pools[count.index].size
+    max_node_count = var.nodepool_size_coder
   }
   node_config {
     oauth_scopes = [
@@ -107,3 +87,100 @@ resource "google_container_node_pool" "node_pool" {
     ignore_changes = [management[0].auto_repair, management[0].auto_upgrade, timeouts]
   }
 }
+
+resource "google_container_node_pool" "workspaces" {
+  name     = "${var.name}-workspaces"
+  location = var.zone
+  project  = var.project_id
+  cluster  = google_container_cluster.primary.name
+  autoscaling {
+    min_node_count       = 0
+    total_max_node_count = var.nodepool_size_workspaces
+  }
+  management {
+    auto_upgrade = false
+  }
+  node_config {
+    oauth_scopes = [
+      "https://www.googleapis.com/auth/logging.write",
+      "https://www.googleapis.com/auth/monitoring",
+      "https://www.googleapis.com/auth/trace.append",
+      "https://www.googleapis.com/auth/devstorage.read_only",
+      "https://www.googleapis.com/auth/service.management.readonly",
+      "https://www.googleapis.com/auth/servicecontrol",
+    ]
+    disk_size_gb    = var.node_disk_size_gb
+    machine_type    = var.nodepool_machine_type_workspaces
+    image_type      = var.node_image_type
+    preemptible     = var.node_preemptible
+    service_account = data.google_compute_default_service_account.default.email
+    tags            = ["gke-node", "${var.project_id}-gke"]
+    labels = {
+      env = var.project_id
+    }
+    metadata = {
+      disable-legacy-endpoints = "true"
+    }
+  }
+  lifecycle {
+    ignore_changes = [management[0].auto_repair, management[0].auto_upgrade, timeouts]
+  }
+}
+
+resource "google_container_node_pool" "misc" {
+  name       = "${var.name}-misc"
+  location   = var.zone
+  project    = var.project_id
+  cluster    = google_container_cluster.primary.name
+  node_count = var.state == "stopped" ? 0 : var.nodepool_size_misc
+  management {
+    auto_upgrade = false
+  }
+  node_config {
+    oauth_scopes = [
+      "https://www.googleapis.com/auth/logging.write",
+      "https://www.googleapis.com/auth/monitoring",
+      "https://www.googleapis.com/auth/trace.append",
+      "https://www.googleapis.com/auth/devstorage.read_only",
+      "https://www.googleapis.com/auth/service.management.readonly",
+      "https://www.googleapis.com/auth/servicecontrol",
+    ]
+    disk_size_gb    = var.node_disk_size_gb
+    machine_type    = var.nodepool_machine_type_misc
+    image_type      = var.node_image_type
+    preemptible     = var.node_preemptible
+    service_account = data.google_compute_default_service_account.default.email
+    tags            = ["gke-node", "${var.project_id}-gke"]
+    labels = {
+      env = var.project_id
+    }
+    metadata = {
+      disable-legacy-endpoints = "true"
+    }
+  }
+  lifecycle {
+    ignore_changes = [management[0].auto_repair, management[0].auto_upgrade, timeouts]
+  }
+}
+
+resource "null_resource" "cluster_kubeconfig" {
+  depends_on = [google_container_cluster.primary]
+  triggers = {
+    path       = local.cluster_kubeconfig_path
+    name       = google_container_cluster.primary.name
+    project_id = var.project_id
+    zone       = var.zone
+  }
+  provisioner "local-exec" {
+    command = <<EOF
+      KUBECONFIG=${self.triggers.path} gcloud container clusters get-credentials ${self.triggers.name} --project=${self.triggers.project_id} --zone=${self.triggers.zone}
+    EOF
+  }
+
+  provisioner "local-exec" {
+    when    = destroy
+    command = <<EOF
+      rm -f ${self.triggers.path}
+    EOF
+  }
+}
diff --git a/scaletest/terraform/infra/gcp_db.tf b/scaletest/terraform/infra/gcp_db.tf
index 71d92c4a77c9f..4d13b262c615f 100644
--- a/scaletest/terraform/infra/gcp_db.tf
+++ b/scaletest/terraform/infra/gcp_db.tf
@@ -1,7 +1,6 @@
 resource "google_sql_database_instance" "db" {
-  name                = "${var.name}-coder"
-  project             = var.project_id
-  region              = var.deployments[0].region
+  name                = var.name
+  region              = var.region
   database_version    = var.cloudsql_version
   deletion_protection = false
 
@@ -13,7 +12,7 @@ resource "google_sql_database_instance" "db" {
     availability_type = "ZONAL"
 
     location_preference {
-      zone = var.deployments[0].zone
+      zone = var.zone
     }
 
     database_flags {
@@ -50,11 +49,11 @@ resource "google_sql_database" "coder" {
   }
 }
 
-resource "random_password" "coder_postgres_password" {
+resource "random_password" "coder-postgres-password" {
   length = 12
 }
 
-resource "random_password" "prometheus_postgres_password" {
+resource "random_password" "prometheus-postgres-password" {
   length = 12
 }
 
@@ -63,7 +62,7 @@ resource "google_sql_user" "coder" {
   instance = google_sql_database_instance.db.id
   name     = "${var.name}-coder"
   type     = "BUILT_IN"
-  password = random_password.coder_postgres_password.result
+  password = random_password.coder-postgres-password.result
   # required for postgres, otherwise user fails to delete
   deletion_policy = "ABANDON"
   lifecycle {
@@ -76,7 +75,7 @@ resource "google_sql_user" "prometheus" {
   instance = google_sql_database_instance.db.id
   name     = "${var.name}-prometheus"
   type     = "BUILT_IN"
-  password = random_password.prometheus_postgres_password.result
+  password = random_password.prometheus-postgres-password.result
   # required for postgres, otherwise user fails to delete
   deletion_policy = "ABANDON"
   lifecycle {
@@ -85,5 +84,5 @@ resource "google_sql_user" "prometheus" {
 }
 
 locals {
-  coder_db_url = "postgres://${google_sql_user.coder.name}:${urlencode(random_password.coder_postgres_password.result)}@${google_sql_database_instance.db.private_ip_address}/${google_sql_database.coder.name}?sslmode=disable"
+  coder_db_url = "postgres://${google_sql_user.coder.name}:${urlencode(random_password.coder-postgres-password.result)}@${google_sql_database_instance.db.private_ip_address}/${google_sql_database.coder.name}?sslmode=disable"
 }
diff --git a/scaletest/terraform/infra/gcp_vpc.tf b/scaletest/terraform/infra/gcp_vpc.tf
index db557c5735b2d..b125c60cfd25a 100644
--- a/scaletest/terraform/infra/gcp_vpc.tf
+++ b/scaletest/terraform/infra/gcp_vpc.tf
@@ -8,21 +8,11 @@ resource "google_compute_network" "vpc" {
 }
 
 resource "google_compute_subnetwork" "subnet" {
-  count         = length(var.deployments)
-  name          = "${var.name}-${var.deployments[count.index].name}"
+  name          = var.name
   project       = var.project_id
-  region        = var.deployments[count.index].region
+  region        = var.region
   network       = google_compute_network.vpc.name
-  ip_cidr_range = var.deployments[count.index].subnet_cidr
-}
-
-resource "google_compute_address" "coder" {
-  count        = length(var.deployments)
-  project      = var.project_id
-  region       = var.deployments[count.index].region
-  name         = "${var.name}-${var.deployments[count.index].name}-coder"
-  address_type = "EXTERNAL"
-  network_tier = "PREMIUM"
+  ip_cidr_range = var.subnet_cidr
 }
 
 resource "google_compute_global_address" "sql_peering" {
@@ -34,6 +24,14 @@ resource "google_compute_global_address" "sql_peering" {
   network       = google_compute_network.vpc.id
 }
 
+resource "google_compute_address" "coder" {
+  project      = var.project_id
+  region       = var.region
+  name         = "${var.name}-coder"
+  address_type = "EXTERNAL"
+  network_tier = "PREMIUM"
+}
+
 resource "google_service_networking_connection" "private_vpc_connection" {
   network                 = google_compute_network.vpc.id
   service                 = "servicenetworking.googleapis.com"
diff --git a/scaletest/terraform/infra/main.tf b/scaletest/terraform/infra/main.tf
index dce0228138668..1724692b19f3a 100644
--- a/scaletest/terraform/infra/main.tf
+++ b/scaletest/terraform/infra/main.tf
@@ -11,8 +11,10 @@ terraform {
     }
   }
 
-  required_version = "~> 1.9.0"
+  required_version = "~> 1.5.0"
 }
 
 provider "google" {
+  region  = var.region
+  project = var.project_id
 }
diff --git a/scaletest/terraform/infra/outputs.tf b/scaletest/terraform/infra/outputs.tf
index 5c8bb2f5eb57b..f5e619eca384d 100644
--- a/scaletest/terraform/infra/outputs.tf
+++ b/scaletest/terraform/infra/outputs.tf
@@ -1,73 +1,73 @@
-# output "coder_db_url" {
-#   description = "URL of the database for Coder."
-#   value       = local.coder_db_url
-#   sensitive   = true
-# }
+output "coder_db_url" {
+  description = "URL of the database for Coder."
+  value       = local.coder_db_url
+  sensitive   = true
+}
 
-# output "coder_address" {
-#   description = "IP address to use for the Coder service."
-#   value       = google_compute_address.coder_primary.address
-# }
+output "coder_address" {
+  description = "IP address to use for the Coder service."
+  value       = google_compute_address.coder.address
+}
 
-# # output "kubernetes_kubeconfig_path" {
-# #   description = "Kubeconfig path."
-# #   value       = local.cluster_kubeconfig_path
-# # }
+output "kubernetes_kubeconfig_path" {
+  description = "Kubeconfig path."
+  value       = local.cluster_kubeconfig_path
+}
 
-# output "kubernetes_nodepool_coder" {
-#   description = "Name of the nodepool on which to run Coder."
-#   value       = google_container_node_pool.coder.name
-# }
+output "kubernetes_nodepool_coder" {
+  description = "Name of the nodepool on which to run Coder."
+  value       = google_container_node_pool.coder.name
+}
 
-# output "kubernetes_nodepool_misc" {
-#   description = "Name of the nodepool on which to run everything else."
-#   value       = google_container_node_pool.misc.name
-# }
+output "kubernetes_nodepool_misc" {
+  description = "Name of the nodepool on which to run everything else."
+  value       = google_container_node_pool.misc.name
+}
 
-# output "kubernetes_nodepool_workspaces" {
-#   description = "Name of the nodepool on which to run workspaces."
-#   value       = google_container_node_pool.workspaces.name
-# }
+output "kubernetes_nodepool_workspaces" {
+  description = "Name of the nodepool on which to run workspaces."
+  value       = google_container_node_pool.workspaces.name
+}
 
-# output "prometheus_external_label_cluster" {
-#   description = "Value for the Prometheus external label named cluster."
-#   value       = google_container_cluster.primary.name
-# }
+output "prometheus_external_label_cluster" {
+  description = "Value for the Prometheus external label named cluster."
+  value       = google_container_cluster.primary.name
+}
 
-# output "prometheus_postgres_dbname" {
-#   description = "Name of the database for Prometheus to monitor."
-#   value       = google_sql_database.coder.name
-# }
+output "prometheus_postgres_dbname" {
+  description = "Name of the database for Prometheus to monitor."
+  value       = google_sql_database.coder.name
+}
 
-# output "prometheus_postgres_host" {
-#   description = "Hostname of the database for Prometheus to connect to."
-#   value       = google_sql_database_instance.db.private_ip_address
-# }
+output "prometheus_postgres_host" {
+  description = "Hostname of the database for Prometheus to connect to."
+  value       = google_sql_database_instance.db.private_ip_address
+}
 
-# output "prometheus_postgres_password" {
-#   description = "Postgres password for Prometheus."
-#   value       = random_password.prometheus_postgres_password.result
-#   sensitive   = true
-# }
+output "prometheus_postgres_password" {
+  description = "Postgres password for Prometheus."
+  value       = random_password.prometheus-postgres-password.result
+  sensitive   = true
+}
 
-# output "prometheus_postgres_user" {
-#   description = "Postgres username for Prometheus."
-#   value       = google_sql_user.prometheus.name
-# }
+output "prometheus_postgres_user" {
+  description = "Postgres username for Prometheus."
+  value       = google_sql_user.prometheus.name
+}
 
-# resource "local_file" "outputs" {
-#   filename = "${path.module}/../../.coderv2/infra_outputs.tfvars"
-#   content  = <<EOF
-#   coder_db_url = "${local.coder_db_url}"
-#   coder_address = "${google_compute_address.coder_primary.address}"
-#   kubernetes_kubeconfig_path = "${local.cluster_kubeconfig_path}"
-#   kubernetes_nodepool_coder = "${google_container_node_pool.coder.name}"
-#   kubernetes_nodepool_misc = "${google_container_node_pool.misc.name}"
-#   kubernetes_nodepool_workspaces = "${google_container_node_pool.workspaces.name}"
-#   prometheus_external_label_cluster = "${google_container_cluster.primary.name}"
-#   prometheus_postgres_dbname = "${google_sql_database.coder.name}"
-#   prometheus_postgres_host = "${google_sql_database_instance.db.private_ip_address}"
-#   prometheus_postgres_password = "${random_password.prometheus_postgres_password.result}"
-#   prometheus_postgres_user = "${google_sql_user.prometheus.name}"
-# EOF
-# }
+resource "local_file" "outputs" {
+  filename = "${path.module}/../../.coderv2/infra_outputs.tfvars"
+  content  = <<EOF
+  coder_db_url = "${local.coder_db_url}"
+  coder_address = "${google_compute_address.coder.address}"
+  kubernetes_kubeconfig_path = "${local.cluster_kubeconfig_path}"
+  kubernetes_nodepool_coder = "${google_container_node_pool.coder.name}"
+  kubernetes_nodepool_misc = "${google_container_node_pool.misc.name}"
+  kubernetes_nodepool_workspaces = "${google_container_node_pool.workspaces.name}"
+  prometheus_external_label_cluster = "${google_container_cluster.primary.name}"
+  prometheus_postgres_dbname = "${google_sql_database.coder.name}"
+  prometheus_postgres_host = "${google_sql_database_instance.db.private_ip_address}"
+  prometheus_postgres_password = "${random_password.prometheus-postgres-password.result}"
+  prometheus_postgres_user = "${google_sql_user.prometheus.name}"
+EOF
+}
diff --git a/scaletest/terraform/infra/vars.tf b/scaletest/terraform/infra/vars.tf
index 3b39f2142719a..d9f5040918ba5 100644
--- a/scaletest/terraform/infra/vars.tf
+++ b/scaletest/terraform/infra/vars.tf
@@ -12,19 +12,22 @@ variable "project_id" {
 }
 
 variable "name" {
-  description = "The name all resources will be prefixed with"
-}
-
-variable "deployments" {
-  type = list(object({
-    name = string
-    region = string
-    zone   = string
-    subnet_cidr = string
-    coder_node_pool_size = number
-    workspaces_node_pool_size = number
-    misc_node_pool_size = number
-  }))
+  description = "Adds a prefix to resources."
+}
+
+variable "region" {
+  description = "GCP region in which to provision resources."
+  default     = "us-east1"
+}
+
+variable "zone" {
+  description = "GCP zone in which to provision resources."
+  default     = "us-east1-c"
+}
+
+variable "subnet_cidr" {
+  description = "CIDR range for the subnet."
+  default     = "10.200.0.0/24"
 }
 
 variable "k8s_version" {
diff --git a/scaletest/terraform/new/gcp_cluster.tf b/scaletest/terraform/new/gcp_cluster.tf
new file mode 100644
index 0000000000000..ca3d20fdf925f
--- /dev/null
+++ b/scaletest/terraform/new/gcp_cluster.tf
@@ -0,0 +1,109 @@
+data "google_compute_default_service_account" "default" {
+  project = var.project_id
+  depends_on = [ google_project_service.api["compute.googleapis.com"] ]
+}
+
+locals {
+  node_pools = flatten([ for i, deployment in var.deployments : [
+    {
+      name = "${var.name}-${deployment.name}-coder"
+      zone = deployment.zone
+      size = deployment.coder_node_pool_size
+      cluster_i = i
+    },     
+    {
+      name = "${var.name}-${deployment.name}-workspaces"
+      zone = deployment.zone
+      size = deployment.workspaces_node_pool_size
+      cluster_i = i
+    },
+    {
+      name = "${var.name}-${deployment.name}-misc"
+      zone = deployment.zone
+      size = deployment.misc_node_pool_size
+      cluster_i = i
+    }
+  ] ])
+}
+
+resource "google_container_cluster" "cluster" {
+  count                     = length(var.deployments)
+  name                      = "${var.name}-${var.deployments[count.index].name}"
+  location                  = var.deployments[count.index].zone
+  project                   = var.project_id
+  network                   = google_compute_network.vpc.name
+  subnetwork                = google_compute_subnetwork.subnet[count.index].name
+  networking_mode           = "VPC_NATIVE"
+  default_max_pods_per_node = 256
+  ip_allocation_policy { # Required with networking_mode=VPC_NATIVE
+
+  }
+  release_channel {
+    # Setting release channel as STABLE can cause unexpected cluster upgrades.
+    channel = "UNSPECIFIED"
+  }
+  initial_node_count       = 1
+  remove_default_node_pool = true
+
+  network_policy {
+    enabled = true
+  }
+  depends_on = [
+    google_project_service.api["container.googleapis.com"]
+  ]
+  monitoring_config {
+    enable_components = ["SYSTEM_COMPONENTS"]
+    managed_prometheus {
+      enabled = false
+    }
+  }
+  workload_identity_config {
+    workload_pool = "${data.google_project.project.project_id}.svc.id.goog"
+  }
+
+
+  lifecycle {
+    ignore_changes = [
+      maintenance_policy,
+      release_channel,
+      remove_default_node_pool
+    ]
+  }
+}
+
+resource "google_container_node_pool" "node_pool" {
+  count    = length(local.node_pools)
+  name     = local.node_pools[count.index].name
+  location = local.node_pools[count.index].zone
+  project  = var.project_id
+  cluster  = google_container_cluster.cluster[local.node_pools[count.index].cluster_i].name
+  autoscaling {
+    min_node_count = 1
+    max_node_count = local.node_pools[count.index].size
+  }
+  node_config {
+    oauth_scopes = [
+      "https://www.googleapis.com/auth/logging.write",
+      "https://www.googleapis.com/auth/monitoring",
+      "https://www.googleapis.com/auth/trace.append",
+      "https://www.googleapis.com/auth/devstorage.read_only",
+      "https://www.googleapis.com/auth/service.management.readonly",
+      "https://www.googleapis.com/auth/servicecontrol",
+    ]
+    disk_size_gb    = var.node_disk_size_gb
+    machine_type    = var.nodepool_machine_type_coder
+    image_type      = var.node_image_type
+    preemptible     = var.node_preemptible
+    service_account = data.google_compute_default_service_account.default.email
+    tags            = ["gke-node", "${var.project_id}-gke"]
+    labels = {
+      env = var.project_id
+    }
+    metadata = {
+      disable-legacy-endpoints = "true"
+    }
+  }
+  lifecycle {
+    ignore_changes = [management[0].auto_repair, management[0].auto_upgrade, timeouts]
+  }
+}
diff --git a/scaletest/terraform/new/gcp_db.tf b/scaletest/terraform/new/gcp_db.tf
new file mode 100644
index 0000000000000..71d92c4a77c9f
--- /dev/null
+++ b/scaletest/terraform/new/gcp_db.tf
@@ -0,0 +1,89 @@
+resource "google_sql_database_instance" "db" {
+  name                = "${var.name}-coder"
+  project             = var.project_id
+  region              = var.deployments[0].region
+  database_version    = var.cloudsql_version
+  deletion_protection = false
+
+  depends_on = [google_service_networking_connection.private_vpc_connection]
+
+  settings {
+    tier              = var.cloudsql_tier
+    activation_policy = "ALWAYS"
+    availability_type = "ZONAL"
+
+    location_preference {
+      zone = var.deployments[0].zone
+    }
+
+    database_flags {
+      name  = "max_connections"
+      value = var.cloudsql_max_connections
+    }
+
+    ip_configuration {
+      ipv4_enabled    = false
+      private_network = google_compute_network.vpc.id
+    }
+
+    insights_config {
+      query_insights_enabled  = true
+      query_string_length     = 1024
+      record_application_tags = false
+      record_client_address   = false
+    }
+  }
+
+  lifecycle {
+    ignore_changes = [deletion_protection, timeouts]
+  }
+}
+
+resource "google_sql_database" "coder" {
+  project  = var.project_id
+  instance = google_sql_database_instance.db.id
+  name     = "${var.name}-coder"
+  # required for postgres, otherwise db fails to delete
+  deletion_policy = "ABANDON"
+  lifecycle {
+    ignore_changes = [deletion_policy]
+  }
+}
+
+resource "random_password" "coder_postgres_password" {
+  length = 12
+}
+
+resource "random_password" "prometheus_postgres_password" {
+  length = 12
+}
+
+resource "google_sql_user" "coder" {
+  project  = var.project_id
+  instance = google_sql_database_instance.db.id
+  name     = "${var.name}-coder"
+  type     = "BUILT_IN"
+  password = random_password.coder_postgres_password.result
+  # required for postgres, otherwise user fails to delete
+  deletion_policy = "ABANDON"
+  lifecycle {
+    ignore_changes = [deletion_policy, password]
+  }
+}
+
+resource "google_sql_user" "prometheus" {
+  project  = var.project_id
+  instance = google_sql_database_instance.db.id
+  name     = "${var.name}-prometheus"
+  type     = "BUILT_IN"
+  password = random_password.prometheus_postgres_password.result
+  # required for postgres, otherwise user fails to delete
+  deletion_policy = "ABANDON"
+  lifecycle {
+    ignore_changes = [deletion_policy, password]
+  }
+}
+
+locals {
+  coder_db_url = "postgres://${google_sql_user.coder.name}:${urlencode(random_password.coder_postgres_password.result)}@${google_sql_database_instance.db.private_ip_address}/${google_sql_database.coder.name}?sslmode=disable"
+}
diff --git a/scaletest/terraform/new/gcp_project.tf b/scaletest/terraform/new/gcp_project.tf
new file mode 100644
index 0000000000000..1073a621c33e0
--- /dev/null
+++ b/scaletest/terraform/new/gcp_project.tf
@@ -0,0 +1,27 @@
+locals {
+  project_apis = [
+    "cloudtrace",
+    "compute",
+    "container",
+    "logging",
+    "monitoring",
+    "servicemanagement",
+    "servicenetworking",
+    "sqladmin",
+    "stackdriver",
+    "storage-api",
+  ]
+}
+
+data "google_project" "project" {
+  project_id = var.project_id
+}
+
+resource "google_project_service" "api" {
+  for_each = toset(local.project_apis)
+  project  = data.google_project.project.project_id
+  service  = "${each.value}.googleapis.com"
+
+  disable_dependent_services = false
+  disable_on_destroy         = false
+}
diff --git a/scaletest/terraform/new/gcp_vpc.tf b/scaletest/terraform/new/gcp_vpc.tf
new file mode 100644
index 0000000000000..db557c5735b2d
--- /dev/null
+++ b/scaletest/terraform/new/gcp_vpc.tf
@@ -0,0 +1,41 @@
+resource "google_compute_network" "vpc" {
+  project                 = var.project_id
+  name                    = var.name
+  auto_create_subnetworks = "false"
+  depends_on = [
+    google_project_service.api["compute.googleapis.com"]
+  ]
+}
+
+resource "google_compute_subnetwork" "subnet" {
+  count         = length(var.deployments)
+  name          = "${var.name}-${var.deployments[count.index].name}"
+  project       = var.project_id
+  region        = var.deployments[count.index].region
+  network       = google_compute_network.vpc.name
+  ip_cidr_range = var.deployments[count.index].subnet_cidr
+}
+
+resource "google_compute_address" "coder" {
+  count        = length(var.deployments)
+  project      = var.project_id
+  region       = var.deployments[count.index].region
+  name         = "${var.name}-${var.deployments[count.index].name}-coder"
+  address_type = "EXTERNAL"
+  network_tier = "PREMIUM"
+}
+
+resource "google_compute_global_address" "sql_peering" {
+  project       = var.project_id
+  name          = "${var.name}-sql-peering"
+  purpose       = "VPC_PEERING"
+  address_type  = "INTERNAL"
+  prefix_length = 16
+  network       = google_compute_network.vpc.id
+}
+
+resource "google_service_networking_connection" "private_vpc_connection" {
+  network                 = google_compute_network.vpc.id
+  service                 = "servicenetworking.googleapis.com"
+  reserved_peering_ranges = [google_compute_global_address.sql_peering.name]
+}
diff --git a/scaletest/terraform/new/main.tf b/scaletest/terraform/new/main.tf
new file mode 100644
index 0000000000000..dce0228138668
--- /dev/null
+++ b/scaletest/terraform/new/main.tf
@@ -0,0 +1,18 @@
+terraform {
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "~> 4.36"
+    }
+
+    random = {
+      source  = "hashicorp/random"
+      version = "~> 3.5"
+    }
+  }
+
+  required_version = "~> 1.9.0"
+}
+
+provider "google" {
+}
diff --git a/scaletest/terraform/new/vars.tf b/scaletest/terraform/new/vars.tf
new file mode 100644
index 0000000000000..2a62a12ba7b3e
--- /dev/null
+++ b/scaletest/terraform/new/vars.tf
@@ -0,0 +1,63 @@
+variable "project_id" {
+  description = "The project in which to provision resources"
+}
+
+variable "name" {
+  description = "The name all resources will be prefixed with"
+}
+
+variable "deployments" {
+  type = list(object({
+    name = string
+    region = string
+    zone   = string
+    subnet_cidr = string
+    coder_node_pool_size = number
+    workspaces_node_pool_size = number
+    misc_node_pool_size = number
+  }))
+}
+
+variable "k8s_version" {
+  description = "Kubernetes version to provision."
+  default     = "1.24"
+}
+
+variable "node_disk_size_gb" {
+  description = "Size of the root disk for cluster nodes."
+  default     = 100
+}
+
+variable "node_image_type" {
+  description = "Image type to use for cluster nodes."
+  default     = "cos_containerd"
+}
+
+// Preemptible nodes are way cheaper, but can be pulled out
+// from under you at any time. Caveat emptor.
+variable "node_preemptible" {
+  description = "Use preemptible nodes."
+  default     = false
+}
+
+// These variables control the node pool dedicated to Coder.
+variable "nodepool_machine_type_coder" {
+  description = "Machine type to use for Coder control plane nodepool."
+  default     = "t2d-standard-4"
+}
+
+// These variables control the size of the database to be used by Coder.
+variable "cloudsql_version" {
+  description = "CloudSQL version to provision"
+  default     = "POSTGRES_14"
+}
+
+variable "cloudsql_tier" {
+  description = "CloudSQL database tier."
+  default     = "db-f1-micro"
+}
+
+variable "cloudsql_max_connections" {
+  description = "CloudSQL database max_connections"
+  default     = 500
+}

From ddbfc3eee034c156461aabc6fffd8e8e968e268c Mon Sep 17 00:00:00 2001
From: Garrett Delfosse <garrett@coder.com>
Date: Tue, 19 Nov 2024 15:58:36 +0000
Subject: [PATCH 04/24] coderd

---
 scaletest/terraform/new/k8s_certmanager.tf |  57 +++++
 scaletest/terraform/new/k8s_coder.tf       | 280 +++++++++++++++++++++
 scaletest/terraform/new/k8s_otel.tf        |  62 +++++
 scaletest/terraform/new/main.tf            |  45 ++++
 scaletest/terraform/new/vars.tf            | 141 ++++++++++-
 5 files changed, 577 insertions(+), 8 deletions(-)
 create mode 100644 scaletest/terraform/new/k8s_certmanager.tf
 create mode 100644 scaletest/terraform/new/k8s_coder.tf
 create mode 100644 scaletest/terraform/new/k8s_otel.tf

diff --git a/scaletest/terraform/new/k8s_certmanager.tf b/scaletest/terraform/new/k8s_certmanager.tf
new file mode 100644
index 0000000000000..c4ae7ac707025
--- /dev/null
+++ b/scaletest/terraform/new/k8s_certmanager.tf
@@ -0,0 +1,57 @@
+locals {
+  cert_manager_namespace                    = "cert-manager"
+  cert_manager_helm_repo                    = "https://charts.jetstack.io"
+  cert_manager_helm_chart                   = "cert-manager"
+  cert_manager_release_name                 = "cert-manager"
+  cert_manager_chart_version                = "1.12.2"
+  cloudflare_issuer_private_key_secret_name = "cloudflare-issuer-private-key"
+}
+
+resource "kubernetes_secret" "cloudflare-api-key" {
+  metadata {
+    name      = "cloudflare-api-key-secret"
+    namespace = local.cert_manager_namespace
+  }
+  data = {
+    api-token = var.cloudflare_api_token
+  }
+}
+
+resource "kubernetes_namespace" "cert-manager-namespace" {
+  metadata {
+    name = local.cert_manager_namespace
+  }
+}
+
+resource "helm_release" "cert-manager" {
+  repository = local.cert_manager_helm_repo
+  chart      = local.cert_manager_helm_chart
+  name       = local.cert_manager_release_name
+  namespace  = kubernetes_namespace.cert-manager-namespace.metadata.0.name
+  values = [<<EOF
+installCRDs: true
+EOF
+  ]
+}
+
+resource "kubectl_manifest" "cloudflare-cluster-issuer" {
+  depends_on = [ helm_release.cert-manager ]
+    yaml_body = <<YAML
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: cloudflare-issuer
+spec:
+  acme:
+    email: ${var.cloudflare_email}
+    server: https://acme-staging-v02.api.letsencrypt.org/directory
+    privateKeySecretRef:
+      name: ${local.cloudflare_issuer_private_key_secret_name}
+    solvers:
+      - dns01:
+          cloudflare:
+            apiTokenSecretRef:
+              name: ${kubernetes_secret.cloudflare-api-key.metadata.0.name}
+              key: api-token
+YAML
+}
diff --git a/scaletest/terraform/new/k8s_coder.tf b/scaletest/terraform/new/k8s_coder.tf
new file mode 100644
index 0000000000000..516d558699f5a
--- /dev/null
+++ b/scaletest/terraform/new/k8s_coder.tf
@@ -0,0 +1,280 @@
+data "google_client_config" "default" {}
+
+locals {
+  coder_url                 = "https://${var.name}-${var.deployments[0].name}-scaletest.${var.cloudflare_domain}"
+  coder_admin_email         = "admin@coder.com"
+  coder_admin_user          = "coder"
+  coder_helm_repo           = "https://helm.coder.com/v2"
+  coder_helm_chart          = "coder"
+  coder_namespace           = "coder-${var.name}"
+  coder_release_name        = var.name
+  provisionerd_helm_chart   = "coder-provisioner"
+  provisionerd_release_name = "${var.name}-provisionerd"
+  dnsNames = regex("https?://([^/]+)", local.coder_url)
+}
+
+resource "kubernetes_namespace" "coder_namespace" {
+  metadata {
+    name = local.coder_namespace
+  }
+  lifecycle {
+    ignore_changes = [timeouts, wait_for_default_service_account]
+  }
+}
+
+resource "random_password" "provisionerd_psk" {
+  length = 26
+}
+
+resource "kubernetes_secret" "coder-db" {
+  type = "Opaque"
+  metadata {
+    name      = "coder-db-url"
+    namespace = kubernetes_namespace.coder_namespace.metadata.0.name
+  }
+  data = {
+    url = local.coder_db_url
+  }
+  lifecycle {
+    ignore_changes = [timeouts, wait_for_service_account_token]
+  }
+}
+
+resource "kubernetes_secret" "provisionerd_psk" {
+  type = "Opaque"
+  metadata {
+    name      = "coder-provisioner-psk"
+    namespace = kubernetes_namespace.coder_namespace.metadata.0.name
+  }
+  data = {
+    psk = random_password.provisionerd_psk.result
+  }
+  lifecycle {
+    ignore_changes = [timeouts, wait_for_service_account_token]
+  }
+}
+
+# OIDC secret needs to be manually provisioned for now.
+data "kubernetes_secret" "coder_oidc" {
+  metadata {
+    namespace = kubernetes_namespace.coder_namespace.metadata.0.name
+    name      = "coder-oidc"
+  }
+}
+
+resource "kubectl_manifest" "coder_certificate" {
+  depends_on = [ helm_release.cert-manager ]
+  yaml_body = <<YAML
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: ${var.name}
+  namespace: ${kubernetes_namespace.coder_namespace.metadata.0.name}
+spec:
+  secretName: ${var.name}-tls
+  dnsNames:
+    - ${local.dnsNames.0}
+  issuerRef:
+    name: cloudflare-issuer
+    kind: ClusterIssuer
+YAML
+}
+
+data "kubernetes_secret" "coder_tls" {
+  metadata {
+    namespace = kubernetes_namespace.coder_namespace.metadata.0.name
+    name      = "${var.name}-tls"
+  }
+  depends_on = [kubectl_manifest.coder_certificate]
+}
+
+resource "helm_release" "coder-chart" {
+  repository = local.coder_helm_repo
+  chart      = local.coder_helm_chart
+  name       = local.coder_release_name
+  version    = var.coder_chart_version
+  namespace  = kubernetes_namespace.coder_namespace.metadata.0.name
+  values = [<<EOF
+coder:
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: "cloud.google.com/gke-nodepool"
+            operator: "In"
+            values: ["${google_container_node_pool.node_pool[0].name}"]
+    podAntiAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+      - weight: 1
+        podAffinityTerm:
+          topologyKey: "kubernetes.io/hostname"
+          labelSelector:
+            matchExpressions:
+            - key:      "app.kubernetes.io/instance"
+              operator: "In"
+              values:   ["${local.coder_release_name}"]
+  env:
+    - name: "CODER_ACCESS_URL"
+      value: "${local.coder_url}"
+    - name: "CODER_CACHE_DIRECTORY"
+      value: "/tmp/coder"
+    - name: "CODER_TELEMETRY_ENABLE"
+      value: "false"
+    - name: "CODER_LOGGING_HUMAN"
+      value: "/dev/null"
+    - name: "CODER_LOGGING_STACKDRIVER"
+      value: "/dev/stderr"
+    - name: "CODER_PG_CONNECTION_URL"
+      valueFrom:
+        secretKeyRef:
+          name: "${kubernetes_secret.coder-db.metadata.0.name}"
+          key: url
+    - name: "CODER_PPROF_ENABLE"
+      value: "true"
+    - name: "CODER_PROMETHEUS_ENABLE"
+      value: "true"
+    - name: "CODER_PROMETHEUS_COLLECT_AGENT_STATS"
+      value: "true"
+    - name: "CODER_PROMETHEUS_COLLECT_DB_METRICS"
+      value: "true"
+    - name: "CODER_VERBOSE"
+      value: "true"
+    - name: "CODER_EXPERIMENTS"
+      value: "${var.coder_experiments}"
+    - name: "CODER_DANGEROUS_DISABLE_RATE_LIMITS"
+      value: "true"
+    # Disabling built-in provisioner daemons
+    - name: "CODER_PROVISIONER_DAEMONS"
+      value: "0"
+    - name: CODER_PROVISIONER_DAEMON_PSK
+      valueFrom:
+        secretKeyRef:
+          key: psk
+          name: "${kubernetes_secret.provisionerd_psk.metadata.0.name}"
+    # Enable OIDC
+    # - name: "CODER_OIDC_ISSUER_URL"
+    #   valueFrom:
+    #     secretKeyRef:
+    #       key: issuer-url
+    #       name: "${data.kubernetes_secret.coder_oidc.metadata.0.name}"
+    # - name: "CODER_OIDC_EMAIL_DOMAIN"
+    #   valueFrom:
+    #     secretKeyRef:
+    #       key: email-domain
+    #       name: "${data.kubernetes_secret.coder_oidc.metadata.0.name}"
+    # - name: "CODER_OIDC_CLIENT_ID"
+    #   valueFrom:
+    #     secretKeyRef:
+    #       key: client-id
+    #       name: "${data.kubernetes_secret.coder_oidc.metadata.0.name}"
+    # - name: "CODER_OIDC_CLIENT_SECRET"
+    #   valueFrom:
+    #     secretKeyRef:
+    #       key: client-secret
+    #       name: "${data.kubernetes_secret.coder_oidc.metadata.0.name}"
+    # Send OTEL traces to the cluster-local collector to sample 10%
+    - name: "OTEL_EXPORTER_OTLP_ENDPOINT"
+      value: "http://otel-collector.${kubernetes_namespace.coder_namespace.metadata.0.name}.svc.cluster.local:4317"
+    - name: "OTEL_TRACES_SAMPLER"
+      value: parentbased_traceidratio
+    - name: "OTEL_TRACES_SAMPLER_ARG"
+      value: "0.1"
+  image:
+    repo: ${var.coder_image_repo}
+    tag: ${var.coder_image_tag}
+  replicaCount: "${var.coder_replicas}"
+  resources:
+    requests:
+      cpu: "${var.coder_cpu_request}"
+      memory: "${var.coder_mem_request}"
+    limits:
+      cpu: "${var.coder_cpu_limit}"
+      memory: "${var.coder_mem_limit}"
+  securityContext:
+    readOnlyRootFilesystem: true
+  service:
+    enable: true
+    sessionAffinity: None
+    loadBalancerIP: "${google_compute_address.coder[0].address}"
+  volumeMounts:
+  - mountPath: "/tmp"
+    name: cache
+    readOnly: false
+  volumes:
+  - emptyDir:
+      sizeLimit: 1024Mi
+    name: cache
+EOF
+  ]
+}
+
+resource "helm_release" "provisionerd-chart" {
+  repository = local.coder_helm_repo
+  chart      = local.provisionerd_helm_chart
+  name       = local.provisionerd_release_name
+  version    = var.provisionerd_chart_version
+  namespace  = kubernetes_namespace.coder_namespace.metadata.0.name
+  values = [<<EOF
+coder:
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: "cloud.google.com/gke-nodepool"
+            operator: "In"
+            values: ["${google_container_node_pool.node_pool[0].name}"]
+    podAntiAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+      - weight: 1
+        podAffinityTerm:
+          topologyKey: "kubernetes.io/hostname"
+          labelSelector:
+            matchExpressions:
+            - key:      "app.kubernetes.io/instance"
+              operator: "In"
+              values:   ["${local.coder_release_name}"]
+  env:
+    - name: "CODER_URL"
+      value: "${local.coder_url}"
+    - name: "CODER_VERBOSE"
+      value: "true"
+    - name: "CODER_CONFIG_DIR"
+      value: "/tmp/config"
+    - name: "CODER_CACHE_DIRECTORY"
+      value: "/tmp/coder"
+    - name: "CODER_TELEMETRY_ENABLE"
+      value: "false"
+    - name: "CODER_LOGGING_HUMAN"
+      value: "/dev/null"
+    - name: "CODER_LOGGING_STACKDRIVER"
+      value: "/dev/stderr"
+    - name: "CODER_PROMETHEUS_ENABLE"
+      value: "true"
+    - name: "CODER_PROVISIONERD_TAGS"
+      value: "scope=organization"
+  image:
+    repo: ${var.provisionerd_image_repo}
+    tag: ${var.provisionerd_image_tag}
+  replicaCount: "${var.provisionerd_replicas}"
+  resources:
+    requests:
+      cpu: "${var.provisionerd_cpu_request}"
+      memory: "${var.provisionerd_mem_request}"
+    limits:
+      cpu: "${var.provisionerd_cpu_limit}"
+      memory: "${var.provisionerd_mem_limit}"
+  securityContext:
+    readOnlyRootFilesystem: true
+  volumeMounts:
+  - mountPath: "/tmp"
+    name: cache
+    readOnly: false
+  volumes:
+  - emptyDir:
+      sizeLimit: 1024Mi
+    name: cache
+EOF
+  ]
+}
diff --git a/scaletest/terraform/new/k8s_otel.tf b/scaletest/terraform/new/k8s_otel.tf
new file mode 100644
index 0000000000000..7d66a1745700a
--- /dev/null
+++ b/scaletest/terraform/new/k8s_otel.tf
@@ -0,0 +1,62 @@
+# Terraform configuration for OpenTelemetry Operator
+
+locals {
+  otel_namespace              = "opentelemetry-operator-system"
+  otel_operator_helm_repo     = "https://open-telemetry.github.io/opentelemetry-helm-charts"
+  otel_operator_helm_chart    = "opentelemetry-operator"
+  otel_operator_release_name  = "opentelemetry-operator"
+  otel_operator_chart_version = "0.34.1"
+}
+
+resource "kubernetes_namespace" "otel-namespace" {
+  metadata {
+    name = local.otel_namespace
+  }
+  lifecycle {
+    ignore_changes = [timeouts, wait_for_default_service_account]
+  }
+}
+
+resource "helm_release" "otel-operator" {
+  repository = local.otel_operator_helm_repo
+  chart      = local.otel_operator_helm_chart
+  name       = local.otel_operator_release_name
+  namespace  = kubernetes_namespace.otel-namespace.metadata.0.name
+  values = [<<EOF
+manager:
+  collectorImage:
+    repository: otel/opentelemetry-collector-k8s
+EOF
+  ]
+}
+
+resource "kubectl_manifest" "otel-collector" {
+  depends_on = [ helm_release.otel-operator ]
+    yaml_body = <<YAML
+apiVersion: opentelemetry.io/v1alpha1
+kind: OpenTelemetryCollector
+metadata:
+  name: otel
+  namespace: ${kubernetes_namespace.coder_namespace.metadata.0.name}
+spec:
+  config: |
+    receivers:
+      otlp:
+        protocols:
+          grpc: {}
+          http: {}
+    exporters:
+      googlecloud:
+        logging:
+          loglevel: debug
+    service:
+      pipelines:
+        traces:
+          receivers: [otlp]
+          processors: []
+          exporters: [logging, googlecloud]
+    image: otel/open-telemetry-collector-contrib:latest
+    mode: deployment
+    replicas: 1
+YAML
+}
diff --git a/scaletest/terraform/new/main.tf b/scaletest/terraform/new/main.tf
index dce0228138668..c74035594b705 100644
--- a/scaletest/terraform/new/main.tf
+++ b/scaletest/terraform/new/main.tf
@@ -9,6 +9,30 @@ terraform {
       source  = "hashicorp/random"
       version = "~> 3.5"
     }
+
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = "~> 2.20"
+    }
+
+    // We use the kubectl provider to apply Custom Resources.
+    // The kubernetes provider requires the CRD is already present 
+    // and would require a separate apply step beforehand.
+    // https://github.com/hashicorp/terraform-provider-kubernetes/issues/1367
+    kubectl = {
+      source  = "alekc/kubectl"
+      version = ">= 2.0.0"
+    }
+
+    helm = {
+      source  = "hashicorp/helm"
+      version = "~> 2.9"
+    }
+
+    tls = {
+      source  = "hashicorp/tls"
+      version = "~> 4.0"
+    }
   }
 
   required_version = "~> 1.9.0"
@@ -16,3 +40,24 @@ terraform {
 
 provider "google" {
 }
+
+provider "kubernetes" {
+  host                   = "https://${google_container_cluster.cluster[0].endpoint}"
+  cluster_ca_certificate = base64decode(google_container_cluster.cluster[0].master_auth.0.cluster_ca_certificate)
+  token                  = data.google_client_config.default.access_token
+}
+
+provider "kubectl" {
+  host                   = "https://${google_container_cluster.cluster[0].endpoint}"
+  cluster_ca_certificate = base64decode(google_container_cluster.cluster[0].master_auth.0.cluster_ca_certificate)
+  token                  = data.google_client_config.default.access_token
+  load_config_file       = false
+}
+
+provider "helm" {
+  kubernetes {
+    host                   = "https://${google_container_cluster.cluster[0].endpoint}"
+    cluster_ca_certificate = base64decode(google_container_cluster.cluster[0].master_auth.0.cluster_ca_certificate)
+    token                  = data.google_client_config.default.access_token
+  }
+}
diff --git a/scaletest/terraform/new/vars.tf b/scaletest/terraform/new/vars.tf
index 2a62a12ba7b3e..cfb574d165130 100644
--- a/scaletest/terraform/new/vars.tf
+++ b/scaletest/terraform/new/vars.tf
@@ -1,11 +1,12 @@
-variable "project_id" {
-  description = "The project in which to provision resources"
-}
-
 variable "name" {
   description = "The name all resources will be prefixed with"
 }
 
+// GCP
+variable "project_id" {
+  description = "The project in which to provision resources"
+}
+
 variable "deployments" {
   type = list(object({
     name = string
@@ -33,20 +34,16 @@ variable "node_image_type" {
   default     = "cos_containerd"
 }
 
-// Preemptible nodes are way cheaper, but can be pulled out
-// from under you at any time. Caveat emptor.
 variable "node_preemptible" {
   description = "Use preemptible nodes."
   default     = false
 }
 
-// These variables control the node pool dedicated to Coder.
 variable "nodepool_machine_type_coder" {
   description = "Machine type to use for Coder control plane nodepool."
   default     = "t2d-standard-4"
 }
 
-// These variables control the size of the database to be used by Coder.
 variable "cloudsql_version" {
   description = "CloudSQL version to provision"
   default     = "POSTGRES_14"
@@ -61,3 +58,131 @@ variable "cloudsql_max_connections" {
   description = "CloudSQL database max_connections"
   default     = 500
 }
+
+// Cloudflare
+variable "cloudflare_api_token" {
+  description = "Cloudflare API token."
+  sensitive   = true
+}
+
+variable "cloudflare_email" {
+  description = "Cloudflare email address."
+  sensitive   = true
+}
+
+variable "cloudflare_domain" {
+  description = "Cloudflare coder domain."
+}
+
+// Coder
+variable "coder_chart_version" {
+  description = "Version of the Coder Helm chart to install. Defaults to latest."
+  default     = null
+}
+
+variable "coder_image_tag" {
+  description = "Tag to use for Coder image."
+  default     = "latest"
+}
+
+variable "coder_image_repo" {
+  description = "Repository to use for Coder image."
+  default     = "ghcr.io/coder/coder"
+}
+
+variable "coder_replicas" {
+  description = "Number of Coder replicas to provision."
+  default     = 1
+}
+
+variable "coder_cpu_request" {
+  description = "CPU request to allocate to Coder."
+  default     = "500m"
+}
+
+variable "coder_mem_request" {
+  description = "Memory request to allocate to Coder."
+  default     = "512Mi"
+}
+
+variable "coder_cpu_limit" {
+  description = "CPU limit to allocate to Coder."
+  default     = "1000m"
+}
+
+variable "coder_mem_limit" {
+  description = "Memory limit to allocate to Coder."
+  default     = "1024Mi"
+}
+
+variable "coder_experiments" {
+  description = "Coder Experiments to enable."
+  default     = ""
+}
+
+// Workspaces
+variable "workspace_image" {
+  description = "Image and tag to use for workspaces."
+  default     = "docker.io/codercom/enterprise-minimal:ubuntu"
+}
+
+variable "workspace_cpu_request" {
+  description = "CPU request to allocate to workspaces."
+  default     = "100m"
+}
+
+variable "workspace_cpu_limit" {
+  description = "CPU limit to allocate to workspaces."
+  default     = "100m"
+}
+
+variable "workspace_mem_request" {
+  description = "Memory request to allocate to workspaces."
+  default     = "128Mi"
+}
+
+variable "workspace_mem_limit" {
+  description = "Memory limit to allocate to workspaces."
+  default     = "128Mi"
+}
+
+// Provisioners
+variable "provisionerd_cpu_request" {
+  description = "CPU request to allocate to provisionerd."
+  default     = "100m"
+}
+
+variable "provisionerd_mem_request" {
+  description = "Memory request to allocate to provisionerd."
+  default     = "1Gi"
+}
+
+variable "provisionerd_cpu_limit" {
+  description = "CPU limit to allocate to provisionerd."
+  default     = "1000m"
+}
+
+variable "provisionerd_mem_limit" {
+  description = "Memory limit to allocate to provisionerd."
+  default     = "1Gi"
+}
+
+variable "provisionerd_replicas" {
+  description = "Number of Provisionerd replicas."
+  default     = 1
+}
+
+variable "provisionerd_chart_version" {
+  description = "Version of the Provisionerd Helm chart to install. Defaults to latest."
+  default     = null
+}
+
+variable "provisionerd_image_repo" {
+  description = "Repository to use for Provisionerd image."
+  default     = "ghcr.io/coder/coder"
+}
+
+variable "provisionerd_image_tag" {
+  description = "Tag to use for Provisionerd image."
+  default     = "latest"
+}

From 6719829da5b3c5b33ccbdc124b6fdb9e9412c820 Mon Sep 17 00:00:00 2001
From: Garrett Delfosse <garrett@coder.com>
Date: Thu, 21 Nov 2024 18:07:08 +0000
Subject: [PATCH 05/24] non-dynamic providers

---
 scaletest/terraform/new/cf_dns.tf          |   7 ++
 scaletest/terraform/new/gcp_cluster.tf     |  60 ++++++-----
 scaletest/terraform/new/gcp_db.tf          |   4 +-
 scaletest/terraform/new/gcp_vpc.tf         |  15 +--
 scaletest/terraform/new/k8s_certmanager.tf |   8 ++
 scaletest/terraform/new/k8s_coder.tf       |  76 +++++++++++++-
 scaletest/terraform/new/k8s_otel.tf        | 116 ++++++++++-----------
 scaletest/terraform/new/main.tf            |  24 +++--
 scaletest/terraform/new/vars.tf            |  32 ++++--
 9 files changed, 226 insertions(+), 116 deletions(-)
 create mode 100644 scaletest/terraform/new/cf_dns.tf

diff --git a/scaletest/terraform/new/cf_dns.tf b/scaletest/terraform/new/cf_dns.tf
new file mode 100644
index 0000000000000..e28aaf22c721f
--- /dev/null
+++ b/scaletest/terraform/new/cf_dns.tf
@@ -0,0 +1,7 @@
+resource "cloudflare_record" "coder" {
+  zone_id = var.cloudflare_zone_id
+  name    = local.coder_subdomain
+  content = google_compute_address.coder["primary"].address
+  type    = "A"
+  ttl     = 3600
+}
diff --git a/scaletest/terraform/new/gcp_cluster.tf b/scaletest/terraform/new/gcp_cluster.tf
index ca3d20fdf925f..f23411bdd7dbe 100644
--- a/scaletest/terraform/new/gcp_cluster.tf
+++ b/scaletest/terraform/new/gcp_cluster.tf
@@ -4,35 +4,39 @@ data "google_compute_default_service_account" "default" {
 }
 
 locals {
-  node_pools = flatten([ for i, deployment in var.deployments : [
-    {
-      name = "${var.name}-${deployment.name}-coder"
-      zone = deployment.zone
-      size = deployment.coder_node_pool_size
-      cluster_i = i
-    },     
-    {
-      name = "${var.name}-${deployment.name}-workspaces"
-      zone = deployment.zone
-      size = deployment.workspaces_node_pool_size
-      cluster_i = i
-    },
-    {
-      name = "${var.name}-${deployment.name}-misc"
-      zone = deployment.zone
-      size = deployment.misc_node_pool_size
-      cluster_i = i
+  clusters = {
+    primary = {
+      region = "us-east1"
+      zone   = "us-east1-c"
+      cidr   = "10.200.0.0/24"
     }
-  ] ])
+  }
+  node_pools = {
+    primary_coder = {
+      name = "coder"
+      cluster = "primary"
+      size = 1
+    }
+    primary_workspaces = {
+      name = "workspaces"
+      cluster = "primary"
+      size = 1
+    }
+    primary_misc = {
+      name = "misc"
+      cluster = "primary"
+      size = 1
+    }
+  }
 }
 
 resource "google_container_cluster" "cluster" {
-  count                     = length(var.deployments)
-  name                      = "${var.name}-${var.deployments[count.index].name}"
-  location                  = var.deployments[count.index].zone
+  for_each                  = local.clusters
+  name                      = "${var.name}-${each.key}"
+  location                  = each.value.zone
   project                   = var.project_id
   network                   = google_compute_network.vpc.name
-  subnetwork                = google_compute_subnetwork.subnet[count.index].name
+  subnetwork                = google_compute_subnetwork.subnet[each.key].name
   networking_mode           = "VPC_NATIVE"
   default_max_pods_per_node = 256
   ip_allocation_policy { # Required with networking_mode=VPC_NATIVE
@@ -72,14 +76,14 @@ resource "google_container_cluster" "cluster" {
 }
 
 resource "google_container_node_pool" "node_pool" {
-  count    = length(local.node_pools)
-  name     = local.node_pools[count.index].name
-  location = local.node_pools[count.index].zone
+  for_each = local.node_pools
+  name     = each.value.name
+  location = local.clusters[each.value.cluster].zone
   project  = var.project_id
-  cluster  = google_container_cluster.cluster[local.node_pools[count.index].cluster_i].name
+  cluster  = google_container_cluster.cluster[each.value.cluster].name
   autoscaling {
     min_node_count = 1
-    max_node_count = local.node_pools[count.index].size
+    max_node_count = each.value.size
   }
   node_config {
     oauth_scopes = [
diff --git a/scaletest/terraform/new/gcp_db.tf b/scaletest/terraform/new/gcp_db.tf
index 71d92c4a77c9f..4a394289b75bf 100644
--- a/scaletest/terraform/new/gcp_db.tf
+++ b/scaletest/terraform/new/gcp_db.tf
@@ -1,7 +1,7 @@
 resource "google_sql_database_instance" "db" {
   name                = "${var.name}-coder"
   project             = var.project_id
-  region              = var.deployments[0].region
+  region              = local.clusters.primary.region
   database_version    = var.cloudsql_version
   deletion_protection = false
 
@@ -13,7 +13,7 @@ resource "google_sql_database_instance" "db" {
     availability_type = "ZONAL"
 
     location_preference {
-      zone = var.deployments[0].zone
+      zone = local.clusters.primary.zone
     }
 
     database_flags {
diff --git a/scaletest/terraform/new/gcp_vpc.tf b/scaletest/terraform/new/gcp_vpc.tf
index db557c5735b2d..56c2910996138 100644
--- a/scaletest/terraform/new/gcp_vpc.tf
+++ b/scaletest/terraform/new/gcp_vpc.tf
@@ -1,3 +1,4 @@
+
 resource "google_compute_network" "vpc" {
   project                 = var.project_id
   name                    = var.name
@@ -8,19 +9,19 @@ resource "google_compute_network" "vpc" {
 }
 
 resource "google_compute_subnetwork" "subnet" {
-  count         = length(var.deployments)
-  name          = "${var.name}-${var.deployments[count.index].name}"
+  for_each      = local.clusters
+  name          = "${var.name}-${each.key}"
   project       = var.project_id
-  region        = var.deployments[count.index].region
+  region        = each.value.region
   network       = google_compute_network.vpc.name
-  ip_cidr_range = var.deployments[count.index].subnet_cidr
+  ip_cidr_range = each.value.cidr
 }
 
 resource "google_compute_address" "coder" {
-  count        = length(var.deployments)
+  for_each     = local.clusters
   project      = var.project_id
-  region       = var.deployments[count.index].region
-  name         = "${var.name}-${var.deployments[count.index].name}-coder"
+  region       = each.value.region
+  name         = "${var.name}-${each.key}-coder"
   address_type = "EXTERNAL"
   network_tier = "PREMIUM"
 }
diff --git a/scaletest/terraform/new/k8s_certmanager.tf b/scaletest/terraform/new/k8s_certmanager.tf
index c4ae7ac707025..25cf69cf363d6 100644
--- a/scaletest/terraform/new/k8s_certmanager.tf
+++ b/scaletest/terraform/new/k8s_certmanager.tf
@@ -8,6 +8,8 @@ locals {
 }
 
 resource "kubernetes_secret" "cloudflare-api-key" {
+  provider = kubernetes.primary
+
   metadata {
     name      = "cloudflare-api-key-secret"
     namespace = local.cert_manager_namespace
@@ -18,12 +20,16 @@ resource "kubernetes_secret" "cloudflare-api-key" {
 }
 
 resource "kubernetes_namespace" "cert-manager-namespace" {
+  provider = kubernetes.primary
+
   metadata {
     name = local.cert_manager_namespace
   }
 }
 
 resource "helm_release" "cert-manager" {
+  provider = helm.primary
+
   repository = local.cert_manager_helm_repo
   chart      = local.cert_manager_helm_chart
   name       = local.cert_manager_release_name
@@ -35,6 +41,8 @@ EOF
 }
 
 resource "kubectl_manifest" "cloudflare-cluster-issuer" {
+  provider = kubectl.primary
+
   depends_on = [ helm_release.cert-manager ]
     yaml_body = <<YAML
 apiVersion: cert-manager.io/v1
diff --git a/scaletest/terraform/new/k8s_coder.tf b/scaletest/terraform/new/k8s_coder.tf
index 516d558699f5a..0c6bdb9ec1fbf 100644
--- a/scaletest/terraform/new/k8s_coder.tf
+++ b/scaletest/terraform/new/k8s_coder.tf
@@ -1,9 +1,12 @@
 data "google_client_config" "default" {}
 
 locals {
-  coder_url                 = "https://${var.name}-${var.deployments[0].name}-scaletest.${var.cloudflare_domain}"
+  coder_subdomain           = "${var.name}-primary-scaletest"
+  coder_url                 = "https://${local.coder_subdomain}.${var.cloudflare_domain}"
   coder_admin_email         = "admin@coder.com"
+  coder_admin_full_name     = "Coder Admin"
   coder_admin_user          = "coder"
+  coder_admin_password      = "SomeSecurePassword!"
   coder_helm_repo           = "https://helm.coder.com/v2"
   coder_helm_chart          = "coder"
   coder_namespace           = "coder-${var.name}"
@@ -14,6 +17,8 @@ locals {
 }
 
 resource "kubernetes_namespace" "coder_namespace" {
+  provider = kubernetes.primary
+
   metadata {
     name = local.coder_namespace
   }
@@ -41,6 +46,8 @@ resource "kubernetes_secret" "coder-db" {
 }
 
 resource "kubernetes_secret" "provisionerd_psk" {
+  provider = kubernetes.primary
+
   type = "Opaque"
   metadata {
     name      = "coder-provisioner-psk"
@@ -56,6 +63,7 @@ resource "kubernetes_secret" "provisionerd_psk" {
 
 # OIDC secret needs to be manually provisioned for now.
 data "kubernetes_secret" "coder_oidc" {
+  provider = kubernetes.primary
   metadata {
     namespace = kubernetes_namespace.coder_namespace.metadata.0.name
     name      = "coder-oidc"
@@ -63,6 +71,8 @@ data "kubernetes_secret" "coder_oidc" {
 }
 
 resource "kubectl_manifest" "coder_certificate" {
+  provider = kubectl.primary
+
   depends_on = [ helm_release.cert-manager ]
   yaml_body = <<YAML
 apiVersion: cert-manager.io/v1
@@ -81,6 +91,8 @@ YAML
 }
 
 data "kubernetes_secret" "coder_tls" {
+  provider = kubernetes.primary
+
   metadata {
     namespace = kubernetes_namespace.coder_namespace.metadata.0.name
     name      = "${var.name}-tls"
@@ -89,6 +101,8 @@ data "kubernetes_secret" "coder_tls" {
 }
 
 resource "helm_release" "coder-chart" {
+  provider = helm.primary
+
   repository = local.coder_helm_repo
   chart      = local.coder_helm_chart
   name       = local.coder_release_name
@@ -103,7 +117,7 @@ coder:
         - matchExpressions:
           - key: "cloud.google.com/gke-nodepool"
             operator: "In"
-            values: ["${google_container_node_pool.node_pool[0].name}"]
+            values: ["${google_container_node_pool.node_pool["primary_coder"].name}"]
     podAntiAffinity:
       preferredDuringSchedulingIgnoredDuringExecution:
       - weight: 1
@@ -196,7 +210,7 @@ coder:
   service:
     enable: true
     sessionAffinity: None
-    loadBalancerIP: "${google_compute_address.coder[0].address}"
+    loadBalancerIP: "${google_compute_address.coder["primary"].address}"
   volumeMounts:
   - mountPath: "/tmp"
     name: cache
@@ -224,7 +238,7 @@ coder:
         - matchExpressions:
           - key: "cloud.google.com/gke-nodepool"
             operator: "In"
-            values: ["${google_container_node_pool.node_pool[0].name}"]
+            values: ["${google_container_node_pool.node_pool["primary_coder"].name}"]
     podAntiAffinity:
       preferredDuringSchedulingIgnoredDuringExecution:
       - weight: 1
@@ -278,3 +292,57 @@ coder:
 EOF
   ]
 }
+
+data "http" "coder_healthy" {
+  url = "http://${local.coder_subdomain}.${var.cloudflare_domain}"
+  // Wait up to 5 minutes for DNS to propogate
+  retry {
+    attempts = 30
+    min_delay_ms = 10000
+  }
+
+  lifecycle {
+    postcondition {
+        condition = self.status_code == 200
+        error_message = "${self.url} returned an unhealthy status code"
+    }
+  }
+
+  depends_on = [ helm_release.coder-chart, cloudflare_record.coder ]
+}
+
+resource "terraform_data" "proxy_tokens" {
+  count = 1
+  provisioner "local-exec" {
+    interpreter = [ "/bin/bash", "-c" ]
+    command = <<EOF
+curl 'http://${local.coder_subdomain}.${var.cloudflare_domain}/api/v2/users/first' \
+  --data-raw $'{"email":"${local.coder_admin_email}","password":"${local.coder_admin_password}","username":"${local.coder_admin_user}","name":"${local.coder_admin_full_name}","trial":false}' \
+  --insecure --silent --output /dev/null
+
+token=$(curl 'http://${local.coder_subdomain}.${var.cloudflare_domain}/api/v2/users/login' \
+  --data-raw $'{"email":"${local.coder_admin_email}","password":"${local.coder_admin_password}"}' \
+  --insecure --silent | jq -r .session_token)
+
+curl 'http://${local.coder_subdomain}.${var.cloudflare_domain}/api/v2/licenses' \
+  -H "Coder-Session-Token: $${token}" \
+  --data-raw '{"license":"${var.coder_license}"}' \
+  --insecure --silent --output /dev/null
+
+europe_token=$(curl 'http://${local.coder_subdomain}.${var.cloudflare_domain}/api/v2/workspaceproxies' \
+  -H "Coder-Session-Token: $${token}" \
+  --data-raw '{"name":"europe"}' \
+  --insecure --silent | jq -r .proxy_token)
+
+asia_token=$(curl 'http://${local.coder_subdomain}.${var.cloudflare_domain}/api/v2/workspaceproxies' \
+  -H "Coder-Session-Token: $${token}" \
+  --data-raw '{"name":"asia"}' \
+  --insecure --silent | jq -r .proxy_token)
+
+echo "{\"europe\": \"$${europe_token}\", \"asia\": \"$${asia_token}\"}"
+EOF
+  }
+
+  depends_on = [ data.http.coder_healthy ]
+}
+
diff --git a/scaletest/terraform/new/k8s_otel.tf b/scaletest/terraform/new/k8s_otel.tf
index 7d66a1745700a..ed48e5d01ea70 100644
--- a/scaletest/terraform/new/k8s_otel.tf
+++ b/scaletest/terraform/new/k8s_otel.tf
@@ -1,62 +1,62 @@
-# Terraform configuration for OpenTelemetry Operator
+# # Terraform configuration for OpenTelemetry Operator
 
-locals {
-  otel_namespace              = "opentelemetry-operator-system"
-  otel_operator_helm_repo     = "https://open-telemetry.github.io/opentelemetry-helm-charts"
-  otel_operator_helm_chart    = "opentelemetry-operator"
-  otel_operator_release_name  = "opentelemetry-operator"
-  otel_operator_chart_version = "0.34.1"
-}
+# locals {
+#   otel_namespace              = "opentelemetry-operator-system"
+#   otel_operator_helm_repo     = "https://open-telemetry.github.io/opentelemetry-helm-charts"
+#   otel_operator_helm_chart    = "opentelemetry-operator"
+#   otel_operator_release_name  = "opentelemetry-operator"
+#   otel_operator_chart_version = "0.34.1"
+# }
 
-resource "kubernetes_namespace" "otel-namespace" {
-  metadata {
-    name = local.otel_namespace
-  }
-  lifecycle {
-    ignore_changes = [timeouts, wait_for_default_service_account]
-  }
-}
+# resource "kubernetes_namespace" "otel-namespace" {
+#   metadata {
+#     name = local.otel_namespace
+#   }
+#   lifecycle {
+#     ignore_changes = [timeouts, wait_for_default_service_account]
+#   }
+# }
 
-resource "helm_release" "otel-operator" {
-  repository = local.otel_operator_helm_repo
-  chart      = local.otel_operator_helm_chart
-  name       = local.otel_operator_release_name
-  namespace  = kubernetes_namespace.otel-namespace.metadata.0.name
-  values = [<<EOF
-manager:
-  collectorImage:
-    repository: otel/opentelemetry-collector-k8s
-EOF
-  ]
-}
+# resource "helm_release" "otel-operator" {
+#   repository = local.otel_operator_helm_repo
+#   chart      = local.otel_operator_helm_chart
+#   name       = local.otel_operator_release_name
+#   namespace  = kubernetes_namespace.otel-namespace.metadata.0.name
+#   values = [<<EOF
+# manager:
+#   collectorImage:
+#     repository: otel/opentelemetry-collector-k8s
+# EOF
+#   ]
+# }
 
-resource "kubectl_manifest" "otel-collector" {
-  depends_on = [ helm_release.otel-operator ]
-    yaml_body = <<YAML
-apiVersion: opentelemetry.io/v1alpha1
-kind: OpenTelemetryCollector
-metadata:
-  name: otel
-  namespace: ${kubernetes_namespace.coder_namespace.metadata.0.name}
-spec:
-  config: |
-    receivers:
-      otlp:
-        protocols:
-          grpc: {}
-          http: {}
-    exporters:
-      googlecloud:
-        logging:
-          loglevel: debug
-    service:
-      pipelines:
-        traces:
-          receivers: [otlp]
-          processors: []
-          exporters: [logging, googlecloud]
-    image: otel/open-telemetry-collector-contrib:latest
-    mode: deployment
-    replicas: 1
-YAML
-}
+# resource "kubectl_manifest" "otel-collector" {
+#   depends_on = [ helm_release.otel-operator ]
+#     yaml_body = <<YAML
+# apiVersion: opentelemetry.io/v1alpha1
+# kind: OpenTelemetryCollector
+# metadata:
+#   name: otel
+#   namespace: ${kubernetes_namespace.coder_namespace.metadata.0.name}
+# spec:
+#   config: |
+#     receivers:
+#       otlp:
+#         protocols:
+#           grpc: {}
+#           http: {}
+#     exporters:
+#       googlecloud:
+#         logging:
+#           loglevel: debug
+#     service:
+#       pipelines:
+#         traces:
+#           receivers: [otlp]
+#           processors: []
+#           exporters: [logging, googlecloud]
+#     image: otel/open-telemetry-collector-contrib:latest
+#     mode: deployment
+#     replicas: 1
+# YAML
+# }
diff --git a/scaletest/terraform/new/main.tf b/scaletest/terraform/new/main.tf
index c74035594b705..8c752251abbea 100644
--- a/scaletest/terraform/new/main.tf
+++ b/scaletest/terraform/new/main.tf
@@ -33,6 +33,11 @@ terraform {
       source  = "hashicorp/tls"
       version = "~> 4.0"
     }
+
+    cloudflare = {
+      source  = "cloudflare/cloudflare"
+      version = "~> 4.0"
+    }
   }
 
   required_version = "~> 1.9.0"
@@ -42,22 +47,29 @@ provider "google" {
 }
 
 provider "kubernetes" {
-  host                   = "https://${google_container_cluster.cluster[0].endpoint}"
-  cluster_ca_certificate = base64decode(google_container_cluster.cluster[0].master_auth.0.cluster_ca_certificate)
+  alias = "primary"
+  host                   = "https://${google_container_cluster.cluster["primary"].endpoint}"
+  cluster_ca_certificate = base64decode(google_container_cluster.cluster["primary"].master_auth.0.cluster_ca_certificate)
   token                  = data.google_client_config.default.access_token
 }
 
 provider "kubectl" {
-  host                   = "https://${google_container_cluster.cluster[0].endpoint}"
-  cluster_ca_certificate = base64decode(google_container_cluster.cluster[0].master_auth.0.cluster_ca_certificate)
+  alias = "primary"
+  host                   = "https://${google_container_cluster.cluster["primary"].endpoint}"
+  cluster_ca_certificate = base64decode(google_container_cluster.cluster["primary"].master_auth.0.cluster_ca_certificate)
   token                  = data.google_client_config.default.access_token
   load_config_file       = false
 }
 
 provider "helm" {
+  alias = "primary"
   kubernetes {
-    host                   = "https://${google_container_cluster.cluster[0].endpoint}"
-    cluster_ca_certificate = base64decode(google_container_cluster.cluster[0].master_auth.0.cluster_ca_certificate)
+    host                   = "https://${google_container_cluster.cluster["primary"].endpoint}"
+    cluster_ca_certificate = base64decode(google_container_cluster.cluster["primary"].master_auth.0.cluster_ca_certificate)
     token                  = data.google_client_config.default.access_token
   }
 }
+
+provider "cloudflare" {
+  api_token = var.cloudflare_api_token
+}
diff --git a/scaletest/terraform/new/vars.tf b/scaletest/terraform/new/vars.tf
index cfb574d165130..513d142aebf1c 100644
--- a/scaletest/terraform/new/vars.tf
+++ b/scaletest/terraform/new/vars.tf
@@ -7,17 +7,17 @@ variable "project_id" {
   description = "The project in which to provision resources"
 }
 
-variable "deployments" {
-  type = list(object({
-    name = string
-    region = string
-    zone   = string
-    subnet_cidr = string
-    coder_node_pool_size = number
-    workspaces_node_pool_size = number
-    misc_node_pool_size = number
-  }))
-}
+# variable "deployments" {
+#   type = list(object({
+#     name = string
+#     region = string
+#     zone   = string
+#     subnet_cidr = string
+#     coder_node_pool_size = number
+#     workspaces_node_pool_size = number
+#     misc_node_pool_size = number
+#   }))
+# }
 
 variable "k8s_version" {
   description = "Kubernetes version to provision."
@@ -74,6 +74,10 @@ variable "cloudflare_domain" {
   description = "Cloudflare coder domain."
 }
 
+variable "cloudflare_zone_id" {
+  description = "Cloudflare zone ID."
+}
+
 // Coder
 variable "coder_chart_version" {
   description = "Version of the Coder Helm chart to install. Defaults to latest."
@@ -95,6 +99,12 @@ variable "coder_replicas" {
   default     = 1
 }
 
+variable "coder_license" {
+  description = "Coder license key."
+  # sensitive   = true
+  
+}
+
 variable "coder_cpu_request" {
   description = "CPU request to allocate to Coder."
   default     = "500m"

From 9b77ae514beccf1f970e583cab96614344c80a0f Mon Sep 17 00:00:00 2001
From: Garrett Delfosse <garrett@coder.com>
Date: Mon, 2 Dec 2024 15:39:52 +0000
Subject: [PATCH 06/24] workspace proxies registered

---
 scaletest/terraform/new/gcp_cluster.tf | 40 ++++++++++++++++++++++++++
 scaletest/terraform/new/k8s_coder.tf   |  4 +++
 2 files changed, 44 insertions(+)

diff --git a/scaletest/terraform/new/gcp_cluster.tf b/scaletest/terraform/new/gcp_cluster.tf
index f23411bdd7dbe..ba3c8c115eaf2 100644
--- a/scaletest/terraform/new/gcp_cluster.tf
+++ b/scaletest/terraform/new/gcp_cluster.tf
@@ -10,6 +10,16 @@ locals {
       zone   = "us-east1-c"
       cidr   = "10.200.0.0/24"
     }
+    europe = {
+      region = "europe-west1"
+      zone   = "europe-west1-b"
+      cidr   = "10.201.0.0/24"
+    }
+    asia = {
+      region = "asia-southeast1"
+      zone   = "asia-southeast1-a"
+      cidr   = "10.202.0.0/24"
+    }
   }
   node_pools = {
     primary_coder = {
@@ -27,6 +37,36 @@ locals {
       cluster = "primary"
       size = 1
     }
+    europe_coder = {
+      name = "coder"
+      cluster = "europe"
+      size = 1
+    }
+    europe_workspaces = {
+      name = "workspaces"
+      cluster = "europe"
+      size = 1
+    }
+    europe_misc = {
+      name = "misc"
+      cluster = "europe"
+      size = 1
+    }
+    asia_coder = {
+      name = "coder"
+      cluster = "asia"
+      size = 1
+    }
+    asia_workspaces = {
+      name = "workspaces"
+      cluster = "asia"
+      size = 1
+    }
+    asia_misc = {
+      name = "misc"
+      cluster = "asia"
+      size = 1
+    }
   }
 }
 
diff --git a/scaletest/terraform/new/k8s_coder.tf b/scaletest/terraform/new/k8s_coder.tf
index 0c6bdb9ec1fbf..c606988cd0a1f 100644
--- a/scaletest/terraform/new/k8s_coder.tf
+++ b/scaletest/terraform/new/k8s_coder.tf
@@ -32,6 +32,8 @@ resource "random_password" "provisionerd_psk" {
 }
 
 resource "kubernetes_secret" "coder-db" {
+  provider = kubernetes.primary
+
   type = "Opaque"
   metadata {
     name      = "coder-db-url"
@@ -224,6 +226,8 @@ EOF
 }
 
 resource "helm_release" "provisionerd-chart" {
+  provider = helm.primary
+  
   repository = local.coder_helm_repo
   chart      = local.provisionerd_helm_chart
   name       = local.provisionerd_release_name

From a8475335506be086326abf051b2668d9fa39654f Mon Sep 17 00:00:00 2001
From: Garrett Delfosse <garrett@coder.com>
Date: Mon, 2 Dec 2024 16:01:17 +0000
Subject: [PATCH 07/24] reset old dir

---
 scaletest/terraform/k8s/cert-manager.tf |  78 ++++++---------
 scaletest/terraform/k8s/coder.tf        | 101 ++++++++------------
 scaletest/terraform/k8s/main.tf         |  11 +--
 scaletest/terraform/k8s/otel.tf         | 121 +++++++++---------------
 scaletest/terraform/k8s/prometheus.tf   |  68 +++++--------
 5 files changed, 137 insertions(+), 242 deletions(-)

diff --git a/scaletest/terraform/k8s/cert-manager.tf b/scaletest/terraform/k8s/cert-manager.tf
index f0d5f099241a9..cfcb324b3ea0b 100644
--- a/scaletest/terraform/k8s/cert-manager.tf
+++ b/scaletest/terraform/k8s/cert-manager.tf
@@ -36,54 +36,32 @@ EOF
   ]
 }
 
-# resource "kubernetes_manifest" "cloudflare-cluster-issuer" {
-#   manifest = {
-#     apiVersion = "cert-manager.io/v1"
-#     kind       = "ClusterIssuer"
-#     metadata = {
-#       name = "cloudflare-issuer"
-#     }
-#     spec = {
-#       acme = {
-#         email = var.cloudflare_email
-#         privateKeySecretRef = {
-#           name = local.cloudflare_issuer_private_key_secret_name
-#         }
-#         solvers = [
-#           {
-#             dns01 = {
-#               cloudflare = {
-#                 apiTokenSecretRef = {
-#                   name = kubernetes_secret.cloudflare-api-key.metadata.0.name
-#                   key  = "api-token"
-#                 }
-#               }
-#             }
-#           }
-#         ]
-#       }
-#     }
-#   }
-# }
-
-resource "kubectl_manifest" "cloudflare-cluster-issuer" {
-  depends_on = [ helm_release.cert-manager ]
-    yaml_body = <<YAML
-apiVersion: cert-manager.io/v1
-kind: ClusterIssuer
-metadata:
-  name: cloudflare-issuer
-spec:
-  acme:
-    email: ${var.cloudflare_email}
-    server: https://acme-staging-v02.api.letsencrypt.org/directory
-    privateKeySecretRef:
-      name: ${local.cloudflare_issuer_private_key_secret_name}
-    solvers:
-      - dns01:
-          cloudflare:
-            apiTokenSecretRef:
-              name: ${kubernetes_secret.cloudflare-api-key.metadata.0.name}
-              key: api-token
-YAML
+resource "kubernetes_manifest" "cloudflare-cluster-issuer" {
+  manifest = {
+    apiVersion = "cert-manager.io/v1"
+    kind       = "ClusterIssuer"
+    metadata = {
+      name = "cloudflare-issuer"
+    }
+    spec = {
+      acme = {
+        email = var.cloudflare_email
+        privateKeySecretRef = {
+          name = local.cloudflare_issuer_private_key_secret_name
+        }
+        solvers = [
+          {
+            dns01 = {
+              cloudflare = {
+                apiTokenSecretRef = {
+                  name = kubernetes_secret.cloudflare-api-key.metadata.0.name
+                  key  = "api-token"
+                }
+              }
+            }
+          }
+        ]
+      }
+    }
+  }
 }
diff --git a/scaletest/terraform/k8s/coder.tf b/scaletest/terraform/k8s/coder.tf
index 7388f072a6798..ea83317127fd8 100644
--- a/scaletest/terraform/k8s/coder.tf
+++ b/scaletest/terraform/k8s/coder.tf
@@ -10,7 +10,6 @@ locals {
   coder_release_name        = var.name
   provisionerd_helm_chart   = "coder-provisioner"
   provisionerd_release_name = "${var.name}-provisionerd"
-  dnsNames = regex("https?://([^/]+)", local.coder_url)
 }
 
 resource "kubernetes_namespace" "coder_namespace" {
@@ -62,41 +61,23 @@ data "kubernetes_secret" "coder_oidc" {
   }
 }
 
-# resource "kubernetes_manifest" "coder_certificate" {
-#   manifest = {
-#     apiVersion = "cert-manager.io/v1"
-#     kind       = "Certificate"
-#     metadata = {
-#       name      = "${var.name}"
-#       namespace = kubernetes_namespace.coder_namespace.metadata.0.name
-#     }
-#     spec = {
-#       secretName = "${var.name}-tls"
-#       dnsNames   = regex("https?://([^/]+)", local.coder_url)
-#       issuerRef = {
-#         name = "cloudflare-issuer"
-#         kind = "ClusterIssuer"
-#       }
-#     }
-#   }
-# }
-
-resource "kubectl_manifest" "coder_certificate" {
-  depends_on = [ helm_release.cert-manager ]
-  yaml_body = <<YAML
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: ${var.name}
-  namespace: ${kubernetes_namespace.coder_namespace.metadata.0.name}
-spec:
-  secretName: ${var.name}-tls
-  dnsNames:
-    - ${local.dnsNames.0}
-  issuerRef:
-    name: cloudflare-issuer
-    kind: ClusterIssuer
-YAML
+resource "kubernetes_manifest" "coder_certificate" {
+  manifest = {
+    apiVersion = "cert-manager.io/v1"
+    kind       = "Certificate"
+    metadata = {
+      name      = "${var.name}"
+      namespace = kubernetes_namespace.coder_namespace.metadata.0.name
+    }
+    spec = {
+      secretName = "${var.name}-tls"
+      dnsNames   = regex("https?://([^/]+)", local.coder_url)
+      issuerRef = {
+        name = kubernetes_manifest.cloudflare-cluster-issuer.manifest.metadata.name
+        kind = "ClusterIssuer"
+      }
+    }
+  }
 }
 
 data "kubernetes_secret" "coder_tls" {
@@ -104,7 +85,7 @@ data "kubernetes_secret" "coder_tls" {
     namespace = kubernetes_namespace.coder_namespace.metadata.0.name
     name      = "${var.name}-tls"
   }
-  depends_on = [kubectl_manifest.coder_certificate]
+  depends_on = [kubernetes_manifest.coder_certificate]
 }
 
 resource "helm_release" "coder-chart" {
@@ -172,29 +153,29 @@ coder:
           key: psk
           name: "${kubernetes_secret.provisionerd_psk.metadata.0.name}"
     # Enable OIDC
-    # - name: "CODER_OIDC_ISSUER_URL"
-    #   valueFrom:
-    #     secretKeyRef:
-    #       key: issuer-url
-    #       name: "${data.kubernetes_secret.coder_oidc.metadata.0.name}"
-    # - name: "CODER_OIDC_EMAIL_DOMAIN"
-    #   valueFrom:
-    #     secretKeyRef:
-    #       key: email-domain
-    #       name: "${data.kubernetes_secret.coder_oidc.metadata.0.name}"
-    # - name: "CODER_OIDC_CLIENT_ID"
-    #   valueFrom:
-    #     secretKeyRef:
-    #       key: client-id
-    #       name: "${data.kubernetes_secret.coder_oidc.metadata.0.name}"
-    # - name: "CODER_OIDC_CLIENT_SECRET"
-    #   valueFrom:
-    #     secretKeyRef:
-    #       key: client-secret
-    #       name: "${data.kubernetes_secret.coder_oidc.metadata.0.name}"
+    - name: "CODER_OIDC_ISSUER_URL"
+      valueFrom:
+        secretKeyRef:
+          key: issuer-url
+          name: "${data.kubernetes_secret.coder_oidc.metadata.0.name}"
+    - name: "CODER_OIDC_EMAIL_DOMAIN"
+      valueFrom:
+        secretKeyRef:
+          key: email-domain
+          name: "${data.kubernetes_secret.coder_oidc.metadata.0.name}"
+    - name: "CODER_OIDC_CLIENT_ID"
+      valueFrom:
+        secretKeyRef:
+          key: client-id
+          name: "${data.kubernetes_secret.coder_oidc.metadata.0.name}"
+    - name: "CODER_OIDC_CLIENT_SECRET"
+      valueFrom:
+        secretKeyRef:
+          key: client-secret
+          name: "${data.kubernetes_secret.coder_oidc.metadata.0.name}"
     # Send OTEL traces to the cluster-local collector to sample 10%
     - name: "OTEL_EXPORTER_OTLP_ENDPOINT"
-      value: "http://otel-collector.${kubernetes_namespace.coder_namespace.metadata.0.name}.svc.cluster.local:4317"
+      value: "http://${kubernetes_manifest.otel-collector.manifest.metadata.name}-collector.${kubernetes_namespace.coder_namespace.metadata.0.name}.svc.cluster.local:4317"
     - name: "OTEL_TRACES_SAMPLER"
       value: parentbased_traceidratio
     - name: "OTEL_TRACES_SAMPLER_ARG"
@@ -259,8 +240,6 @@ coder:
       value: "${local.coder_url}"
     - name: "CODER_VERBOSE"
       value: "true"
-    - name: "CODER_CONFIG_DIR"
-      value: "/tmp/config"
     - name: "CODER_CACHE_DIRECTORY"
       value: "/tmp/coder"
     - name: "CODER_TELEMETRY_ENABLE"
@@ -272,7 +251,7 @@ coder:
     - name: "CODER_PROMETHEUS_ENABLE"
       value: "true"
     - name: "CODER_PROVISIONERD_TAGS"
-      value: "scope=organization"
+      value = "socpe=organization"
   image:
     repo: ${var.provisionerd_image_repo}
     tag: ${var.provisionerd_image_tag}
diff --git a/scaletest/terraform/k8s/main.tf b/scaletest/terraform/k8s/main.tf
index 959ffb8b872dd..a5c8c1085a5ce 100644
--- a/scaletest/terraform/k8s/main.tf
+++ b/scaletest/terraform/k8s/main.tf
@@ -5,11 +5,6 @@ terraform {
       version = "~> 2.20"
     }
 
-    kubectl = {
-      source  = "alekc/kubectl"
-      version = ">= 2.0.0"
-    }
-
     helm = {
       source  = "hashicorp/helm"
       version = "~> 2.9"
@@ -26,17 +21,13 @@ terraform {
     }
   }
 
-  required_version = "~> 1.9.0"
+  required_version = "~> 1.5.0"
 }
 
 provider "kubernetes" {
   config_path = var.kubernetes_kubeconfig_path
 }
 
-provider "kubectl" {
-  config_path = var.kubernetes_kubeconfig_path
-}
-
 provider "helm" {
   kubernetes {
     config_path = var.kubernetes_kubeconfig_path
diff --git a/scaletest/terraform/k8s/otel.tf b/scaletest/terraform/k8s/otel.tf
index 96375db02dcfe..3b1657ee48cbc 100644
--- a/scaletest/terraform/k8s/otel.tf
+++ b/scaletest/terraform/k8s/otel.tf
@@ -3,7 +3,7 @@
 locals {
   otel_namespace              = "opentelemetry-operator-system"
   otel_operator_helm_repo     = "https://open-telemetry.github.io/opentelemetry-helm-charts"
-  otel_operator_helm_chart    = "opentelemetry-operator"
+  otel_operator_helm_chart    = "opentelemtry-operator"
   otel_operator_release_name  = "opentelemetry-operator"
   otel_operator_chart_version = "0.34.1"
 }
@@ -22,83 +22,48 @@ resource "helm_release" "otel-operator" {
   chart      = local.otel_operator_helm_chart
   name       = local.otel_operator_release_name
   namespace  = kubernetes_namespace.otel-namespace.metadata.0.name
-  values = [<<EOF
-manager:
-  collectorImage:
-    repository: otel/opentelemetry-collector-k8s
-EOF
-  ]
+  # Default values
+  values = []
 }
 
-# resource "kubernetes_manifest" "otel-collector" {
-#   manifest = {
-#     apiVersion = "opentelemetry.io/v1alpha1"
-#     kind       = "OpenTelemetryCollector"
-#     metadata = {
-#       namespace = kubernetes_namespace.coder_namespace.metadata.0.name
-#       name      = "otel"
-#     }
-#     spec = {
-#       config = jsonencode({
-#         receivers = {
-#           otlp = {
-#             protocols : {
-#               grpc : {}
-#               http : {}
-#             }
-#           }
-#         }
-#         exporters = {
-#           googlecloud = {
-#             logging = {
-#               loglevel = "debug"
-#             }
-#           }
-#         }
-#         service = {
-#           pipelines = {
-#             traces = {
-#               receivers  = ["otlp"]
-#               processors = []
-#               exporters  = ["logging", "googlecloud"]
-#             }
-#           }
-#         }
-#         image    = "otel/open-telemetry-collector-contrib:latest"
-#         mode     = "deployment"
-#         replicas = 1
-#       })
-#     }
-#   }
-# }
-
-resource "kubectl_manifest" "otel-collector" {
-  depends_on = [ helm_release.otel-operator ]
-    yaml_body = <<YAML
-apiVersion: opentelemetry.io/v1alpha1
-kind: OpenTelemetryCollector
-metadata:
-  name: otel
-  namespace: ${kubernetes_namespace.coder_namespace.metadata.0.name}
-spec:
-  config: |
-    receivers:
-      otlp:
-        protocols:
-          grpc: {}
-          http: {}
-    exporters:
-      googlecloud:
-        logging:
-          loglevel: debug
-    service:
-      pipelines:
-        traces:
-          receivers: [otlp]
-          processors: []
-          exporters: [logging, googlecloud]
-    image: otel/open-telemetry-collector-contrib:latest
-    mode: deployment
-    replicas: 1
-YAML
+resource "kubernetes_manifest" "otel-collector" {
+  manifest = {
+    apiVersion = "opentelemetry.io/v1alpha1"
+    kind       = "OpenTelemetryCollector"
+    metadata = {
+      namespace = kubernetes_namespace.coder_namespace.metadata.0.name
+      name      = "otel"
+    }
+    spec = {
+      config = jsonencode({
+        receivers = {
+          otlp = {
+            protocols : {
+              grpc : {}
+              http : {}
+            }
+          }
+        }
+        exporters = {
+          googlecloud = {
+            logging = {
+              loglevel = "debug"
+            }
+          }
+        }
+        service = {
+          pipelines = {
+            traces = {
+              receivers  = ["otlp"]
+              processors = []
+              exporters  = ["logging", "googlecloud"]
+            }
+          }
+        }
+        image    = "otel/open-telemetry-collector-contrib:latest"
+        mode     = "deployment"
+        replicas = 1
+      })
+    }
+  }
 }
diff --git a/scaletest/terraform/k8s/prometheus.tf b/scaletest/terraform/k8s/prometheus.tf
index c92c6abe2df41..accf926727575 100644
--- a/scaletest/terraform/k8s/prometheus.tf
+++ b/scaletest/terraform/k8s/prometheus.tf
@@ -101,7 +101,7 @@ prometheus:
   ]
 }
 
-resource "kubernetes_secret" "prometheus_postgres_password" {
+resource "kubernetes_secret" "prometheus-postgres-password" {
   type = "kubernetes.io/basic-auth"
   metadata {
     name      = "prometheus-postgres"
@@ -138,7 +138,7 @@ config:
     user: "${var.prometheus_postgres_user}"
     database: "${var.prometheus_postgres_dbname}"
     passwordSecret:
-      name: "${kubernetes_secret.prometheus_postgres_password.metadata.0.name}"
+      name: "${kubernetes_secret.prometheus-postgres-password.metadata.0.name}"
       key: password
     autoDiscoverDatabases: true
 serviceMonitor:
@@ -147,45 +147,27 @@ serviceMonitor:
   ]
 }
 
-# resource "kubernetes_manifest" "coder_monitoring" {
-#   depends_on = [helm_release.prometheus-chart]
-#   manifest = {
-#     apiVersion = "monitoring.coreos.com/v1"
-#     kind       = "PodMonitor"
-#     metadata = {
-#       namespace = kubernetes_namespace.coder_namespace.metadata.0.name
-#       name      = "coder-monitoring"
-#     }
-#     spec = {
-#       selector = {
-#         matchLabels = {
-#           "app.kubernetes.io/name" : "coder"
-#         }
-#       }
-#       podMetricsEndpoints = [
-#         {
-#           port     = "prometheus-http"
-#           interval = "30s"
-#         }
-#       ]
-#     }
-#   }
-# }
-
-resource "kubectl_manifest" "coder_monitoring" {
-  depends_on = [ helm_release.prometheus-chart ]
-    yaml_body = <<YAML
-apiVersion: monitoring.coreos.com/v1
-kind: PodMonitor
-metadata:
-  name: coder-monitoring
-  namespace: ${kubernetes_namespace.coder_namespace.metadata.0.name}
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: coder
-  podMetricsEndpoints:
-    - port: prometheus-http
-      interval: 30s
-YAML
+resource "kubernetes_manifest" "coder_monitoring" {
+  depends_on = [helm_release.prometheus-chart]
+  manifest = {
+    apiVersion = "monitoring.coreos.com/v1"
+    kind       = "PodMonitor"
+    metadata = {
+      namespace = kubernetes_namespace.coder_namespace.metadata.0.name
+      name      = "coder-monitoring"
+    }
+    spec = {
+      selector = {
+        matchLabels = {
+          "app.kubernetes.io/name" : "coder"
+        }
+      }
+      podMetricsEndpoints = [
+        {
+          port     = "prometheus-http"
+          interval = "30s"
+        }
+      ]
+    }
+  }
 }

From 8f21ee4e5253cd8b457ea64761e78b470cd7bc8f Mon Sep 17 00:00:00 2001
From: Garrett Delfosse <garrett@coder.com>
Date: Tue, 3 Dec 2024 23:55:34 +0000
Subject: [PATCH 08/24] eu

---
 scaletest/terraform/new/k8s_certmanager.tf | 117 ++++----
 scaletest/terraform/new/k8s_coder.tf       | 324 ++++++++++++++++++---
 scaletest/terraform/new/main.tf            |  24 ++
 3 files changed, 374 insertions(+), 91 deletions(-)

diff --git a/scaletest/terraform/new/k8s_certmanager.tf b/scaletest/terraform/new/k8s_certmanager.tf
index 25cf69cf363d6..730dd6f99d7b9 100644
--- a/scaletest/terraform/new/k8s_certmanager.tf
+++ b/scaletest/terraform/new/k8s_certmanager.tf
@@ -1,65 +1,68 @@
-locals {
-  cert_manager_namespace                    = "cert-manager"
-  cert_manager_helm_repo                    = "https://charts.jetstack.io"
-  cert_manager_helm_chart                   = "cert-manager"
-  cert_manager_release_name                 = "cert-manager"
-  cert_manager_chart_version                = "1.12.2"
-  cloudflare_issuer_private_key_secret_name = "cloudflare-issuer-private-key"
-}
+# locals {
+#   cert_manager_namespace                    = "cert-manager"
+#   cert_manager_helm_repo                    = "https://charts.jetstack.io"
+#   cert_manager_helm_chart                   = "cert-manager"
+#   cert_manager_release_name                 = "cert-manager"
+#   cert_manager_chart_version                = "1.16.2"
+#   cloudflare_issuer_private_key_secret_name = "cloudflare-issuer-private-key"
+# }
 
-resource "kubernetes_secret" "cloudflare-api-key" {
-  provider = kubernetes.primary
+# resource "kubernetes_secret" "cloudflare-api-key" {
+#   provider = kubernetes.primary
 
-  metadata {
-    name      = "cloudflare-api-key-secret"
-    namespace = local.cert_manager_namespace
-  }
-  data = {
-    api-token = var.cloudflare_api_token
-  }
-}
+#   metadata {
+#     name      = "cloudflare-api-key-secret"
+#     namespace = local.cert_manager_namespace
+#   }
+#   data = {
+#     api-token = var.cloudflare_api_token
+#   }
+# }
 
-resource "kubernetes_namespace" "cert-manager-namespace" {
-  provider = kubernetes.primary
+# resource "kubernetes_namespace" "cert-manager-namespace" {
+#   provider = kubernetes.primary
 
-  metadata {
-    name = local.cert_manager_namespace
-  }
-}
+#   metadata {
+#     name = local.cert_manager_namespace
+#   }
+# }
 
-resource "helm_release" "cert-manager" {
-  provider = helm.primary
+# resource "helm_release" "cert-manager" {
+#   provider = helm.primary
 
-  repository = local.cert_manager_helm_repo
-  chart      = local.cert_manager_helm_chart
-  name       = local.cert_manager_release_name
-  namespace  = kubernetes_namespace.cert-manager-namespace.metadata.0.name
-  values = [<<EOF
-installCRDs: true
-EOF
-  ]
-}
+#   repository = local.cert_manager_helm_repo
+#   chart      = local.cert_manager_helm_chart
+#   name       = local.cert_manager_release_name
+#   namespace  = kubernetes_namespace.cert-manager-namespace.metadata.0.name
+#   values = [<<EOF
+# installCRDs: true
+# webhook:
+#   securePort: 10250
+# EOF
+#   ]
+# }
 
-resource "kubectl_manifest" "cloudflare-cluster-issuer" {
-  provider = kubectl.primary
+# resource "kubectl_manifest" "cloudflare-issuer" {
+#   provider = kubectl.primary
 
-  depends_on = [ helm_release.cert-manager ]
-    yaml_body = <<YAML
-apiVersion: cert-manager.io/v1
-kind: ClusterIssuer
-metadata:
-  name: cloudflare-issuer
-spec:
-  acme:
-    email: ${var.cloudflare_email}
-    server: https://acme-staging-v02.api.letsencrypt.org/directory
-    privateKeySecretRef:
-      name: ${local.cloudflare_issuer_private_key_secret_name}
-    solvers:
-      - dns01:
-          cloudflare:
-            apiTokenSecretRef:
-              name: ${kubernetes_secret.cloudflare-api-key.metadata.0.name}
-              key: api-token
-YAML
-}
+#   depends_on = [ helm_release.cert-manager ]
+#     yaml_body = <<YAML
+# apiVersion: cert-manager.io/v1
+# kind: Issuer
+# metadata:
+#   name: cloudflare-issuer
+#   namespace: ${kubernetes_namespace.cert-manager-namespace.metadata.0.name}
+# spec:
+#   acme:
+#     email: ${var.cloudflare_email}
+#     server: https://acme-staging-v02.api.letsencrypt.org/directory
+#     privateKeySecretRef:
+#       name: ${local.cloudflare_issuer_private_key_secret_name}
+#     solvers:
+#       - dns01:
+#           cloudflare:
+#             apiTokenSecretRef:
+#               name: ${kubernetes_secret.cloudflare-api-key.metadata.0.name}
+#               key: api-token
+# YAML
+# }
diff --git a/scaletest/terraform/new/k8s_coder.tf b/scaletest/terraform/new/k8s_coder.tf
index c606988cd0a1f..31b44967e6f45 100644
--- a/scaletest/terraform/new/k8s_coder.tf
+++ b/scaletest/terraform/new/k8s_coder.tf
@@ -2,7 +2,14 @@ data "google_client_config" "default" {}
 
 locals {
   coder_subdomain           = "${var.name}-primary-scaletest"
-  coder_url                 = "https://${local.coder_subdomain}.${var.cloudflare_domain}"
+  coder_url                 = "http://${local.coder_subdomain}.${var.cloudflare_domain}"
+  dnsNames                  = regex("http?://([^/]+)", local.coder_url)
+  coder_europe_subdomain    = "${var.name}-europe-scaletest"
+  coder_europe_url          = "http://${local.coder_europe_subdomain}.${var.cloudflare_domain}"
+  coder_asia_subdomain      = "${var.name}-asia-scaletest"
+  coder_asia_url            = "http://${local.coder_asia_subdomain}.${var.cloudflare_domain}"
+  # coder_url                 = "https://${local.coder_subdomain}.${var.cloudflare_domain}"
+  # dnsNames                  = regex("https?://([^/]+)", local.coder_url)
   coder_admin_email         = "admin@coder.com"
   coder_admin_full_name     = "Coder Admin"
   coder_admin_user          = "coder"
@@ -13,7 +20,7 @@ locals {
   coder_release_name        = var.name
   provisionerd_helm_chart   = "coder-provisioner"
   provisionerd_release_name = "${var.name}-provisionerd"
-  dnsNames = regex("https?://([^/]+)", local.coder_url)
+
 }
 
 resource "kubernetes_namespace" "coder_namespace" {
@@ -27,6 +34,17 @@ resource "kubernetes_namespace" "coder_namespace" {
   }
 }
 
+resource "kubernetes_namespace" "coder_europe" {
+  provider = kubernetes.europe
+
+  metadata {
+    name = local.coder_namespace
+  }
+  lifecycle {
+    ignore_changes = [timeouts, wait_for_default_service_account]
+  }
+}
+
 resource "random_password" "provisionerd_psk" {
   length = 26
 }
@@ -63,6 +81,22 @@ resource "kubernetes_secret" "provisionerd_psk" {
   }
 }
 
+resource "kubernetes_secret" "provisionerd_psk_europe" {
+  provider = kubernetes.europe
+
+  type = "Opaque"
+  metadata {
+    name      = "coder-provisioner-psk"
+    namespace = kubernetes_namespace.coder_namespace.metadata.0.name
+  }
+  data = {
+    psk = random_password.provisionerd_psk.result
+  }
+  lifecycle {
+    ignore_changes = [timeouts, wait_for_service_account_token]
+  }
+}
+
 # OIDC secret needs to be manually provisioned for now.
 data "kubernetes_secret" "coder_oidc" {
   provider = kubernetes.primary
@@ -72,36 +106,45 @@ data "kubernetes_secret" "coder_oidc" {
   }
 }
 
-resource "kubectl_manifest" "coder_certificate" {
-  provider = kubectl.primary
-
-  depends_on = [ helm_release.cert-manager ]
-  yaml_body = <<YAML
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: ${var.name}
-  namespace: ${kubernetes_namespace.coder_namespace.metadata.0.name}
-spec:
-  secretName: ${var.name}-tls
-  dnsNames:
-    - ${local.dnsNames.0}
-  issuerRef:
-    name: cloudflare-issuer
-    kind: ClusterIssuer
-YAML
-}
-
-data "kubernetes_secret" "coder_tls" {
-  provider = kubernetes.primary
-
+data "kubernetes_secret" "coder_oidc_europe" {
+  provider = kubernetes.europe
   metadata {
     namespace = kubernetes_namespace.coder_namespace.metadata.0.name
-    name      = "${var.name}-tls"
+    name      = "coder-oidc"
   }
-  depends_on = [kubectl_manifest.coder_certificate]
 }
 
+# resource "kubectl_manifest" "coder_certificate" {
+#   provider = kubectl.primary
+
+#   depends_on = [ helm_release.cert-manager ]
+#   yaml_body = <<YAML
+# apiVersion: cert-manager.io/v1
+# kind: Certificate
+# metadata:
+#   name: ${var.name}
+#   namespace: ${kubernetes_namespace.coder_namespace.metadata.0.name}
+# spec:
+#   secretName: ${var.name}-tls
+#   dnsNames:
+#   - ${local.coder_subdomain}.${var.cloudflare_domain}
+#   issuerRef:
+#     group: cert-manager.io
+#     name: cloudflare-issuer
+#     kind: ClusterIssuer
+# YAML
+# }
+
+# data "kubernetes_secret" "coder_tls" {
+#   provider = kubernetes.primary
+
+#   metadata {
+#     namespace = kubernetes_namespace.coder_namespace.metadata.0.name
+#     name      = "${var.name}-tls"
+#   }
+#   depends_on = [kubectl_manifest.coder_certificate]
+# }
+
 resource "helm_release" "coder-chart" {
   provider = helm.primary
 
@@ -225,6 +268,148 @@ EOF
   ]
 }
 
+
+resource "kubernetes_secret" "proxy_token_europe" {
+  provider = kubernetes.europe
+
+  type = "Opaque"
+  metadata {
+    name      = "coder-proxy-token"
+    namespace = kubernetes_namespace.coder_europe.metadata.0.name
+  }
+  data = {
+    token = terraform_data.proxy_tokens.output.europe
+  }
+  lifecycle {
+    ignore_changes = [timeouts, wait_for_service_account_token]
+  }
+}
+
+resource "helm_release" "coder_europe" {
+  provider = helm.europe
+
+  repository = local.coder_helm_repo
+  chart      = local.coder_helm_chart
+  name       = local.coder_release_name
+  version    = var.coder_chart_version
+  namespace  = kubernetes_namespace.coder_europe.metadata.0.name
+  values = [<<EOF
+coder:
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: "cloud.google.com/gke-nodepool"
+            operator: "In"
+            values: ["${google_container_node_pool.node_pool["europe_coder"].name}"]
+    podAntiAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+      - weight: 1
+        podAffinityTerm:
+          topologyKey: "kubernetes.io/hostname"
+          labelSelector:
+            matchExpressions:
+            - key:      "app.kubernetes.io/instance"
+              operator: "In"
+              values:   ["${local.coder_release_name}"]
+  env:
+    - name: CODER_PRIMARY_ACCESS_URL
+      value: "${local.coder_url}"
+    - name: CODER_PROXY_SESSION_TOKEN
+      valueFrom:
+        secretKeyRef:
+          key: europe
+          name: "${kubernetes_secret.proxy_token_europe.metadata.0.name}"
+    - name: "CODER_ACCESS_URL"
+      value: "${local.coder_europe_url}"
+    - name: "CODER_CACHE_DIRECTORY"
+      value: "/tmp/coder"
+    - name: "CODER_TELEMETRY_ENABLE"
+      value: "false"
+    - name: "CODER_LOGGING_HUMAN"
+      value: "/dev/null"
+    - name: "CODER_LOGGING_STACKDRIVER"
+      value: "/dev/stderr"
+    - name: "CODER_PPROF_ENABLE"
+      value: "true"
+    - name: "CODER_PROMETHEUS_ENABLE"
+      value: "true"
+    - name: "CODER_PROMETHEUS_COLLECT_AGENT_STATS"
+      value: "true"
+    - name: "CODER_PROMETHEUS_COLLECT_DB_METRICS"
+      value: "true"
+    - name: "CODER_VERBOSE"
+      value: "true"
+    - name: "CODER_EXPERIMENTS"
+      value: "${var.coder_experiments}"
+    - name: "CODER_DANGEROUS_DISABLE_RATE_LIMITS"
+      value: "true"
+    # Disabling built-in provisioner daemons
+    - name: "CODER_PROVISIONER_DAEMONS"
+      value: "0"
+    - name: CODER_PROVISIONER_DAEMON_PSK
+      valueFrom:
+        secretKeyRef:
+          key: psk
+          name: "${kubernetes_secret.provisionerd_psk_europe.metadata.0.name}"
+    # Enable OIDC
+    # - name: "CODER_OIDC_ISSUER_URL"
+    #   valueFrom:
+    #     secretKeyRef:
+    #       key: issuer-url
+    #       name: "${data.kubernetes_secret.coder_oidc_europe.metadata.0.name}"
+    # - name: "CODER_OIDC_EMAIL_DOMAIN"
+    #   valueFrom:
+    #     secretKeyRef:
+    #       key: email-domain
+    #       name: "${data.kubernetes_secret.coder_oidc_europe.metadata.0.name}"
+    # - name: "CODER_OIDC_CLIENT_ID"
+    #   valueFrom:
+    #     secretKeyRef:
+    #       key: client-id
+    #       name: "${data.kubernetes_secret.coder_oidc_europe.metadata.0.name}"
+    # - name: "CODER_OIDC_CLIENT_SECRET"
+    #   valueFrom:
+    #     secretKeyRef:
+    #       key: client-secret
+    #       name: "${data.kubernetes_secret.coder_oidc_europe.metadata.0.name}"
+    # Send OTEL traces to the cluster-local collector to sample 10%
+    - name: "OTEL_EXPORTER_OTLP_ENDPOINT"
+      value: "http://otel-collector.${kubernetes_namespace.coder_europe.metadata.0.name}.svc.cluster.local:4317"
+    - name: "OTEL_TRACES_SAMPLER"
+      value: parentbased_traceidratio
+    - name: "OTEL_TRACES_SAMPLER_ARG"
+      value: "0.1"
+  image:
+    repo: ${var.coder_image_repo}
+    tag: ${var.coder_image_tag}
+  replicaCount: "${var.coder_replicas}"
+  resources:
+    requests:
+      cpu: "${var.coder_cpu_request}"
+      memory: "${var.coder_mem_request}"
+    limits:
+      cpu: "${var.coder_cpu_limit}"
+      memory: "${var.coder_mem_limit}"
+  securityContext:
+    readOnlyRootFilesystem: true
+  service:
+    enable: true
+    sessionAffinity: None
+    loadBalancerIP: "${google_compute_address.coder["europe"].address}"
+  volumeMounts:
+  - mountPath: "/tmp"
+    name: cache
+    readOnly: false
+  volumes:
+  - emptyDir:
+      sizeLimit: 1024Mi
+    name: cache
+EOF
+  ]
+}
+
 resource "helm_release" "provisionerd-chart" {
   provider = helm.primary
   
@@ -297,8 +482,80 @@ EOF
   ]
 }
 
+resource "helm_release" "provisionerd_europe" {
+  provider = helm.europe
+  
+  repository = local.coder_helm_repo
+  chart      = local.provisionerd_helm_chart
+  name       = local.provisionerd_release_name
+  version    = var.provisionerd_chart_version
+  namespace  = kubernetes_namespace.coder_europe.metadata.0.name
+  values = [<<EOF
+coder:
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: "cloud.google.com/gke-nodepool"
+            operator: "In"
+            values: ["${google_container_node_pool.node_pool["europe_coder"].name}"]
+    podAntiAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+      - weight: 1
+        podAffinityTerm:
+          topologyKey: "kubernetes.io/hostname"
+          labelSelector:
+            matchExpressions:
+            - key:      "app.kubernetes.io/instance"
+              operator: "In"
+              values:   ["${local.coder_release_name}"]
+  env:
+    - name: "CODER_URL"
+      value: "${local.coder_url}"
+    - name: "CODER_VERBOSE"
+      value: "true"
+    - name: "CODER_CONFIG_DIR"
+      value: "/tmp/config"
+    - name: "CODER_CACHE_DIRECTORY"
+      value: "/tmp/coder"
+    - name: "CODER_TELEMETRY_ENABLE"
+      value: "false"
+    - name: "CODER_LOGGING_HUMAN"
+      value: "/dev/null"
+    - name: "CODER_LOGGING_STACKDRIVER"
+      value: "/dev/stderr"
+    - name: "CODER_PROMETHEUS_ENABLE"
+      value: "true"
+    - name: "CODER_PROVISIONERD_TAGS"
+      value: "scope=organization"
+  image:
+    repo: ${var.provisionerd_image_repo}
+    tag: ${var.provisionerd_image_tag}
+  replicaCount: "${var.provisionerd_replicas}"
+  resources:
+    requests:
+      cpu: "${var.provisionerd_cpu_request}"
+      memory: "${var.provisionerd_mem_request}"
+    limits:
+      cpu: "${var.provisionerd_cpu_limit}"
+      memory: "${var.provisionerd_mem_limit}"
+  securityContext:
+    readOnlyRootFilesystem: true
+  volumeMounts:
+  - mountPath: "/tmp"
+    name: cache
+    readOnly: false
+  volumes:
+  - emptyDir:
+      sizeLimit: 1024Mi
+    name: cache
+EOF
+  ]
+}
+
 data "http" "coder_healthy" {
-  url = "http://${local.coder_subdomain}.${var.cloudflare_domain}"
+  url = local.coder_url
   // Wait up to 5 minutes for DNS to propogate
   retry {
     attempts = 30
@@ -316,29 +573,28 @@ data "http" "coder_healthy" {
 }
 
 resource "terraform_data" "proxy_tokens" {
-  count = 1
   provisioner "local-exec" {
     interpreter = [ "/bin/bash", "-c" ]
     command = <<EOF
-curl 'http://${local.coder_subdomain}.${var.cloudflare_domain}/api/v2/users/first' \
+curl '${local.coder_url}/api/v2/users/first' \
   --data-raw $'{"email":"${local.coder_admin_email}","password":"${local.coder_admin_password}","username":"${local.coder_admin_user}","name":"${local.coder_admin_full_name}","trial":false}' \
   --insecure --silent --output /dev/null
 
-token=$(curl 'http://${local.coder_subdomain}.${var.cloudflare_domain}/api/v2/users/login' \
+token=$(curl '${local.coder_url}/api/v2/users/login' \
   --data-raw $'{"email":"${local.coder_admin_email}","password":"${local.coder_admin_password}"}' \
   --insecure --silent | jq -r .session_token)
 
-curl 'http://${local.coder_subdomain}.${var.cloudflare_domain}/api/v2/licenses' \
+curl '${local.coder_url}/api/v2/licenses' \
   -H "Coder-Session-Token: $${token}" \
   --data-raw '{"license":"${var.coder_license}"}' \
   --insecure --silent --output /dev/null
 
-europe_token=$(curl 'http://${local.coder_subdomain}.${var.cloudflare_domain}/api/v2/workspaceproxies' \
+europe_token=$(curl '${local.coder_url}/api/v2/workspaceproxies' \
   -H "Coder-Session-Token: $${token}" \
   --data-raw '{"name":"europe"}' \
   --insecure --silent | jq -r .proxy_token)
 
-asia_token=$(curl 'http://${local.coder_subdomain}.${var.cloudflare_domain}/api/v2/workspaceproxies' \
+asia_token=$(curl '${local.coder_url}/api/v2/workspaceproxies' \
   -H "Coder-Session-Token: $${token}" \
   --data-raw '{"name":"asia"}' \
   --insecure --silent | jq -r .proxy_token)
diff --git a/scaletest/terraform/new/main.tf b/scaletest/terraform/new/main.tf
index 8c752251abbea..d1009dcfefcc0 100644
--- a/scaletest/terraform/new/main.tf
+++ b/scaletest/terraform/new/main.tf
@@ -53,6 +53,13 @@ provider "kubernetes" {
   token                  = data.google_client_config.default.access_token
 }
 
+provider "kubernetes" {
+  alias = "europe"
+  host                   = "https://${google_container_cluster.cluster["europe"].endpoint}"
+  cluster_ca_certificate = base64decode(google_container_cluster.cluster["europe"].master_auth.0.cluster_ca_certificate)
+  token                  = data.google_client_config.default.access_token
+}
+
 provider "kubectl" {
   alias = "primary"
   host                   = "https://${google_container_cluster.cluster["primary"].endpoint}"
@@ -61,6 +68,14 @@ provider "kubectl" {
   load_config_file       = false
 }
 
+provider "kubectl" {
+  alias = "europe"
+  host                   = "https://${google_container_cluster.cluster["europe"].endpoint}"
+  cluster_ca_certificate = base64decode(google_container_cluster.cluster["europe"].master_auth.0.cluster_ca_certificate)
+  token                  = data.google_client_config.default.access_token
+  load_config_file       = false
+}
+
 provider "helm" {
   alias = "primary"
   kubernetes {
@@ -70,6 +85,15 @@ provider "helm" {
   }
 }
 
+provider "helm" {
+  alias = "europe"
+  kubernetes {
+    host                   = "https://${google_container_cluster.cluster["europe"].endpoint}"
+    cluster_ca_certificate = base64decode(google_container_cluster.cluster["europe"].master_auth.0.cluster_ca_certificate)
+    token                  = data.google_client_config.default.access_token
+  }
+}
+
 provider "cloudflare" {
   api_token = var.cloudflare_api_token
 }

From 2e7396a748a5a30a2daf748aa296c6fc9ef1e5ac Mon Sep 17 00:00:00 2001
From: Garrett Delfosse <garrett@coder.com>
Date: Fri, 6 Dec 2024 18:27:44 +0000
Subject: [PATCH 09/24] working eu

---
 scaletest/terraform/new/cf_dns.tf    |  8 +++++++
 scaletest/terraform/new/k8s_coder.tf | 36 ++++++++++++++++++++++++----
 2 files changed, 40 insertions(+), 4 deletions(-)

diff --git a/scaletest/terraform/new/cf_dns.tf b/scaletest/terraform/new/cf_dns.tf
index e28aaf22c721f..61c669801a36d 100644
--- a/scaletest/terraform/new/cf_dns.tf
+++ b/scaletest/terraform/new/cf_dns.tf
@@ -5,3 +5,11 @@ resource "cloudflare_record" "coder" {
   type    = "A"
   ttl     = 3600
 }
+
+resource "cloudflare_record" "coder_europe" {
+  zone_id = var.cloudflare_zone_id
+  name    = local.coder_europe_subdomain
+  content = google_compute_address.coder["europe"].address
+  type    = "A"
+  ttl     = 3600
+}
diff --git a/scaletest/terraform/new/k8s_coder.tf b/scaletest/terraform/new/k8s_coder.tf
index 31b44967e6f45..b9a48ea6013f5 100644
--- a/scaletest/terraform/new/k8s_coder.tf
+++ b/scaletest/terraform/new/k8s_coder.tf
@@ -278,7 +278,7 @@ resource "kubernetes_secret" "proxy_token_europe" {
     namespace = kubernetes_namespace.coder_europe.metadata.0.name
   }
   data = {
-    token = terraform_data.proxy_tokens.output.europe
+    token = trimspace(data.local_file.europe_proxy_token.content)
   }
   lifecycle {
     ignore_changes = [timeouts, wait_for_service_account_token]
@@ -295,6 +295,7 @@ resource "helm_release" "coder_europe" {
   namespace  = kubernetes_namespace.coder_europe.metadata.0.name
   values = [<<EOF
 coder:
+  workspaceProxy: true
   affinity:
     nodeAffinity:
       requiredDuringSchedulingIgnoredDuringExecution:
@@ -319,7 +320,7 @@ coder:
     - name: CODER_PROXY_SESSION_TOKEN
       valueFrom:
         secretKeyRef:
-          key: europe
+          key: token
           name: "${kubernetes_secret.proxy_token_europe.metadata.0.name}"
     - name: "CODER_ACCESS_URL"
       value: "${local.coder_europe_url}"
@@ -572,7 +573,7 @@ data "http" "coder_healthy" {
   depends_on = [ helm_release.coder-chart, cloudflare_record.coder ]
 }
 
-resource "terraform_data" "proxy_tokens" {
+resource "null_resource" "proxy_tokens" {
   provisioner "local-exec" {
     interpreter = [ "/bin/bash", "-c" ]
     command = <<EOF
@@ -599,10 +600,37 @@ asia_token=$(curl '${local.coder_url}/api/v2/workspaceproxies' \
   --data-raw '{"name":"asia"}' \
   --insecure --silent | jq -r .proxy_token)
 
-echo "{\"europe\": \"$${europe_token}\", \"asia\": \"$${asia_token}\"}"
+echo -n $${europe_token} > ${path.module}/europe_proxy_token
+echo -n $${asia_token} > ${path.module}/asia_proxy_token
 EOF
   }
 
   depends_on = [ data.http.coder_healthy ]
 }
 
+data "local_file" "europe_proxy_token" {
+  filename = "${path.module}/europe_proxy_token"
+  depends_on = [ null_resource.proxy_tokens ]
+}
+
+data "local_file" "asia_proxy_token" {
+  filename = "${path.module}/asia_proxy_token"
+  depends_on = [ null_resource.proxy_tokens ]
+}
+
+# data "external" "proxy_tokens" {
+#   program = ["bash", "${path.module}/workspace_proxies.sh"]
+#   query = {
+#     coder_url = local.coder_url
+#     coder_admin_email = local.coder_admin_email
+#     coder_admin_password = local.coder_admin_password
+#     coder_admin_user = local.coder_admin_user
+#     coder_admin_full_name = local.coder_admin_full_name
+#     coder_license = var.coder_license
+
+#     status_code = data.http.coder_healthy.status_code
+#   }
+
+#   depends_on = [ data.http.coder_healthy ]
+# }
+

From bb5c9a88692cc001d0c9fc9758156d6a2d7b9dec Mon Sep 17 00:00:00 2001
From: Garrett Delfosse <garrett@coder.com>
Date: Fri, 6 Dec 2024 20:59:43 +0000
Subject: [PATCH 10/24] all working

---
 scaletest/terraform/new/cf_dns.tf             |  13 +-
 scaletest/terraform/new/deployments.tf        |  25 +
 .../new/{gcp_cluster.tf => gcp_clusters.tf}   |  21 +-
 scaletest/terraform/new/gcp_db.tf             |   4 +-
 scaletest/terraform/new/gcp_vpc.tf            |   4 +-
 scaletest/terraform/new/k8s_certmanager.tf    |  68 --
 scaletest/terraform/new/k8s_coder.tf          | 636 ------------------
 scaletest/terraform/new/k8s_coder_asia.tf     | 212 ++++++
 scaletest/terraform/new/k8s_coder_europe.tf   | 212 ++++++
 scaletest/terraform/new/k8s_coder_primary.tf  | 229 +++++++
 scaletest/terraform/new/k8s_coder_proxies.tf  |  63 ++
 scaletest/terraform/new/k8s_otel.tf           |  62 --
 scaletest/terraform/new/main.tf               |  28 +-
 scaletest/terraform/new/vars.tf               |  23 +-
 14 files changed, 781 insertions(+), 819 deletions(-)
 create mode 100644 scaletest/terraform/new/deployments.tf
 rename scaletest/terraform/new/{gcp_cluster.tf => gcp_clusters.tf} (88%)
 delete mode 100644 scaletest/terraform/new/k8s_certmanager.tf
 delete mode 100644 scaletest/terraform/new/k8s_coder.tf
 create mode 100644 scaletest/terraform/new/k8s_coder_asia.tf
 create mode 100644 scaletest/terraform/new/k8s_coder_europe.tf
 create mode 100644 scaletest/terraform/new/k8s_coder_primary.tf
 create mode 100644 scaletest/terraform/new/k8s_coder_proxies.tf
 delete mode 100644 scaletest/terraform/new/k8s_otel.tf

diff --git a/scaletest/terraform/new/cf_dns.tf b/scaletest/terraform/new/cf_dns.tf
index 61c669801a36d..5906741a81c7f 100644
--- a/scaletest/terraform/new/cf_dns.tf
+++ b/scaletest/terraform/new/cf_dns.tf
@@ -1,15 +1,8 @@
 resource "cloudflare_record" "coder" {
+  for_each = local.deployments
   zone_id = var.cloudflare_zone_id
-  name    = local.coder_subdomain
-  content = google_compute_address.coder["primary"].address
-  type    = "A"
-  ttl     = 3600
-}
-
-resource "cloudflare_record" "coder_europe" {
-  zone_id = var.cloudflare_zone_id
-  name    = local.coder_europe_subdomain
-  content = google_compute_address.coder["europe"].address
+  name    = each.value.subdomain
+  content = google_compute_address.coder[each.key].address
   type    = "A"
   ttl     = 3600
 }
diff --git a/scaletest/terraform/new/deployments.tf b/scaletest/terraform/new/deployments.tf
new file mode 100644
index 0000000000000..0fed67ab7d2c9
--- /dev/null
+++ b/scaletest/terraform/new/deployments.tf
@@ -0,0 +1,25 @@
+locals {
+  deployments = {
+    primary = {
+      subdomain = "${var.name}-scaletest"
+      url = "http://${var.name}-scaletest.${var.cloudflare_domain}"
+      region = "us-east1"
+      zone   = "us-east1-c"
+      cidr   = "10.200.0.0/24"
+    }
+    europe = {
+      subdomain = "${var.name}-europe-scaletest"
+      url = "http://${var.name}-europe-scaletest.${var.cloudflare_domain}"
+      region = "europe-west1"
+      zone   = "europe-west1-b"
+      cidr   = "10.201.0.0/24"
+    }
+    asia = {
+      subdomain = "${var.name}-asia-scaletest"
+      url = "http://${var.name}-asia-scaletest.${var.cloudflare_domain}"
+      region = "asia-southeast1"
+      zone   = "asia-southeast1-a"
+      cidr   = "10.202.0.0/24"
+    }
+  }
+}
diff --git a/scaletest/terraform/new/gcp_cluster.tf b/scaletest/terraform/new/gcp_clusters.tf
similarity index 88%
rename from scaletest/terraform/new/gcp_cluster.tf
rename to scaletest/terraform/new/gcp_clusters.tf
index ba3c8c115eaf2..16865c9f5690b 100644
--- a/scaletest/terraform/new/gcp_cluster.tf
+++ b/scaletest/terraform/new/gcp_clusters.tf
@@ -4,23 +4,6 @@ data "google_compute_default_service_account" "default" {
 }
 
 locals {
-  clusters = {
-    primary = {
-      region = "us-east1"
-      zone   = "us-east1-c"
-      cidr   = "10.200.0.0/24"
-    }
-    europe = {
-      region = "europe-west1"
-      zone   = "europe-west1-b"
-      cidr   = "10.201.0.0/24"
-    }
-    asia = {
-      region = "asia-southeast1"
-      zone   = "asia-southeast1-a"
-      cidr   = "10.202.0.0/24"
-    }
-  }
   node_pools = {
     primary_coder = {
       name = "coder"
@@ -71,7 +54,7 @@ locals {
 }
 
 resource "google_container_cluster" "cluster" {
-  for_each                  = local.clusters
+  for_each                  = local.deployments
   name                      = "${var.name}-${each.key}"
   location                  = each.value.zone
   project                   = var.project_id
@@ -118,7 +101,7 @@ resource "google_container_cluster" "cluster" {
 resource "google_container_node_pool" "node_pool" {
   for_each = local.node_pools
   name     = each.value.name
-  location = local.clusters[each.value.cluster].zone
+  location = local.deployments[each.value.cluster].zone
   project  = var.project_id
   cluster  = google_container_cluster.cluster[each.value.cluster].name
   autoscaling {
diff --git a/scaletest/terraform/new/gcp_db.tf b/scaletest/terraform/new/gcp_db.tf
index 4a394289b75bf..aa5dc1c5b923a 100644
--- a/scaletest/terraform/new/gcp_db.tf
+++ b/scaletest/terraform/new/gcp_db.tf
@@ -1,7 +1,7 @@
 resource "google_sql_database_instance" "db" {
   name                = "${var.name}-coder"
   project             = var.project_id
-  region              = local.clusters.primary.region
+  region              = local.deployments.primary.region
   database_version    = var.cloudsql_version
   deletion_protection = false
 
@@ -13,7 +13,7 @@ resource "google_sql_database_instance" "db" {
     availability_type = "ZONAL"
 
     location_preference {
-      zone = local.clusters.primary.zone
+      zone = local.deployments.primary.zone
     }
 
     database_flags {
diff --git a/scaletest/terraform/new/gcp_vpc.tf b/scaletest/terraform/new/gcp_vpc.tf
index 56c2910996138..c9fd412aa3cb4 100644
--- a/scaletest/terraform/new/gcp_vpc.tf
+++ b/scaletest/terraform/new/gcp_vpc.tf
@@ -9,7 +9,7 @@ resource "google_compute_network" "vpc" {
 }
 
 resource "google_compute_subnetwork" "subnet" {
-  for_each      = local.clusters
+  for_each      = local.deployments
   name          = "${var.name}-${each.key}"
   project       = var.project_id
   region        = each.value.region
@@ -18,7 +18,7 @@ resource "google_compute_subnetwork" "subnet" {
 }
 
 resource "google_compute_address" "coder" {
-  for_each     = local.clusters
+  for_each     = local.deployments
   project      = var.project_id
   region       = each.value.region
   name         = "${var.name}-${each.key}-coder"
diff --git a/scaletest/terraform/new/k8s_certmanager.tf b/scaletest/terraform/new/k8s_certmanager.tf
deleted file mode 100644
index 730dd6f99d7b9..0000000000000
--- a/scaletest/terraform/new/k8s_certmanager.tf
+++ /dev/null
@@ -1,68 +0,0 @@
-# locals {
-#   cert_manager_namespace                    = "cert-manager"
-#   cert_manager_helm_repo                    = "https://charts.jetstack.io"
-#   cert_manager_helm_chart                   = "cert-manager"
-#   cert_manager_release_name                 = "cert-manager"
-#   cert_manager_chart_version                = "1.16.2"
-#   cloudflare_issuer_private_key_secret_name = "cloudflare-issuer-private-key"
-# }
-
-# resource "kubernetes_secret" "cloudflare-api-key" {
-#   provider = kubernetes.primary
-
-#   metadata {
-#     name      = "cloudflare-api-key-secret"
-#     namespace = local.cert_manager_namespace
-#   }
-#   data = {
-#     api-token = var.cloudflare_api_token
-#   }
-# }
-
-# resource "kubernetes_namespace" "cert-manager-namespace" {
-#   provider = kubernetes.primary
-
-#   metadata {
-#     name = local.cert_manager_namespace
-#   }
-# }
-
-# resource "helm_release" "cert-manager" {
-#   provider = helm.primary
-
-#   repository = local.cert_manager_helm_repo
-#   chart      = local.cert_manager_helm_chart
-#   name       = local.cert_manager_release_name
-#   namespace  = kubernetes_namespace.cert-manager-namespace.metadata.0.name
-#   values = [<<EOF
-# installCRDs: true
-# webhook:
-#   securePort: 10250
-# EOF
-#   ]
-# }
-
-# resource "kubectl_manifest" "cloudflare-issuer" {
-#   provider = kubectl.primary
-
-#   depends_on = [ helm_release.cert-manager ]
-#     yaml_body = <<YAML
-# apiVersion: cert-manager.io/v1
-# kind: Issuer
-# metadata:
-#   name: cloudflare-issuer
-#   namespace: ${kubernetes_namespace.cert-manager-namespace.metadata.0.name}
-# spec:
-#   acme:
-#     email: ${var.cloudflare_email}
-#     server: https://acme-staging-v02.api.letsencrypt.org/directory
-#     privateKeySecretRef:
-#       name: ${local.cloudflare_issuer_private_key_secret_name}
-#     solvers:
-#       - dns01:
-#           cloudflare:
-#             apiTokenSecretRef:
-#               name: ${kubernetes_secret.cloudflare-api-key.metadata.0.name}
-#               key: api-token
-# YAML
-# }
diff --git a/scaletest/terraform/new/k8s_coder.tf b/scaletest/terraform/new/k8s_coder.tf
deleted file mode 100644
index b9a48ea6013f5..0000000000000
--- a/scaletest/terraform/new/k8s_coder.tf
+++ /dev/null
@@ -1,636 +0,0 @@
-data "google_client_config" "default" {}
-
-locals {
-  coder_subdomain           = "${var.name}-primary-scaletest"
-  coder_url                 = "http://${local.coder_subdomain}.${var.cloudflare_domain}"
-  dnsNames                  = regex("http?://([^/]+)", local.coder_url)
-  coder_europe_subdomain    = "${var.name}-europe-scaletest"
-  coder_europe_url          = "http://${local.coder_europe_subdomain}.${var.cloudflare_domain}"
-  coder_asia_subdomain      = "${var.name}-asia-scaletest"
-  coder_asia_url            = "http://${local.coder_asia_subdomain}.${var.cloudflare_domain}"
-  # coder_url                 = "https://${local.coder_subdomain}.${var.cloudflare_domain}"
-  # dnsNames                  = regex("https?://([^/]+)", local.coder_url)
-  coder_admin_email         = "admin@coder.com"
-  coder_admin_full_name     = "Coder Admin"
-  coder_admin_user          = "coder"
-  coder_admin_password      = "SomeSecurePassword!"
-  coder_helm_repo           = "https://helm.coder.com/v2"
-  coder_helm_chart          = "coder"
-  coder_namespace           = "coder-${var.name}"
-  coder_release_name        = var.name
-  provisionerd_helm_chart   = "coder-provisioner"
-  provisionerd_release_name = "${var.name}-provisionerd"
-
-}
-
-resource "kubernetes_namespace" "coder_namespace" {
-  provider = kubernetes.primary
-
-  metadata {
-    name = local.coder_namespace
-  }
-  lifecycle {
-    ignore_changes = [timeouts, wait_for_default_service_account]
-  }
-}
-
-resource "kubernetes_namespace" "coder_europe" {
-  provider = kubernetes.europe
-
-  metadata {
-    name = local.coder_namespace
-  }
-  lifecycle {
-    ignore_changes = [timeouts, wait_for_default_service_account]
-  }
-}
-
-resource "random_password" "provisionerd_psk" {
-  length = 26
-}
-
-resource "kubernetes_secret" "coder-db" {
-  provider = kubernetes.primary
-
-  type = "Opaque"
-  metadata {
-    name      = "coder-db-url"
-    namespace = kubernetes_namespace.coder_namespace.metadata.0.name
-  }
-  data = {
-    url = local.coder_db_url
-  }
-  lifecycle {
-    ignore_changes = [timeouts, wait_for_service_account_token]
-  }
-}
-
-resource "kubernetes_secret" "provisionerd_psk" {
-  provider = kubernetes.primary
-
-  type = "Opaque"
-  metadata {
-    name      = "coder-provisioner-psk"
-    namespace = kubernetes_namespace.coder_namespace.metadata.0.name
-  }
-  data = {
-    psk = random_password.provisionerd_psk.result
-  }
-  lifecycle {
-    ignore_changes = [timeouts, wait_for_service_account_token]
-  }
-}
-
-resource "kubernetes_secret" "provisionerd_psk_europe" {
-  provider = kubernetes.europe
-
-  type = "Opaque"
-  metadata {
-    name      = "coder-provisioner-psk"
-    namespace = kubernetes_namespace.coder_namespace.metadata.0.name
-  }
-  data = {
-    psk = random_password.provisionerd_psk.result
-  }
-  lifecycle {
-    ignore_changes = [timeouts, wait_for_service_account_token]
-  }
-}
-
-# OIDC secret needs to be manually provisioned for now.
-data "kubernetes_secret" "coder_oidc" {
-  provider = kubernetes.primary
-  metadata {
-    namespace = kubernetes_namespace.coder_namespace.metadata.0.name
-    name      = "coder-oidc"
-  }
-}
-
-data "kubernetes_secret" "coder_oidc_europe" {
-  provider = kubernetes.europe
-  metadata {
-    namespace = kubernetes_namespace.coder_namespace.metadata.0.name
-    name      = "coder-oidc"
-  }
-}
-
-# resource "kubectl_manifest" "coder_certificate" {
-#   provider = kubectl.primary
-
-#   depends_on = [ helm_release.cert-manager ]
-#   yaml_body = <<YAML
-# apiVersion: cert-manager.io/v1
-# kind: Certificate
-# metadata:
-#   name: ${var.name}
-#   namespace: ${kubernetes_namespace.coder_namespace.metadata.0.name}
-# spec:
-#   secretName: ${var.name}-tls
-#   dnsNames:
-#   - ${local.coder_subdomain}.${var.cloudflare_domain}
-#   issuerRef:
-#     group: cert-manager.io
-#     name: cloudflare-issuer
-#     kind: ClusterIssuer
-# YAML
-# }
-
-# data "kubernetes_secret" "coder_tls" {
-#   provider = kubernetes.primary
-
-#   metadata {
-#     namespace = kubernetes_namespace.coder_namespace.metadata.0.name
-#     name      = "${var.name}-tls"
-#   }
-#   depends_on = [kubectl_manifest.coder_certificate]
-# }
-
-resource "helm_release" "coder-chart" {
-  provider = helm.primary
-
-  repository = local.coder_helm_repo
-  chart      = local.coder_helm_chart
-  name       = local.coder_release_name
-  version    = var.coder_chart_version
-  namespace  = kubernetes_namespace.coder_namespace.metadata.0.name
-  values = [<<EOF
-coder:
-  affinity:
-    nodeAffinity:
-      requiredDuringSchedulingIgnoredDuringExecution:
-        nodeSelectorTerms:
-        - matchExpressions:
-          - key: "cloud.google.com/gke-nodepool"
-            operator: "In"
-            values: ["${google_container_node_pool.node_pool["primary_coder"].name}"]
-    podAntiAffinity:
-      preferredDuringSchedulingIgnoredDuringExecution:
-      - weight: 1
-        podAffinityTerm:
-          topologyKey: "kubernetes.io/hostname"
-          labelSelector:
-            matchExpressions:
-            - key:      "app.kubernetes.io/instance"
-              operator: "In"
-              values:   ["${local.coder_release_name}"]
-  env:
-    - name: "CODER_ACCESS_URL"
-      value: "${local.coder_url}"
-    - name: "CODER_CACHE_DIRECTORY"
-      value: "/tmp/coder"
-    - name: "CODER_TELEMETRY_ENABLE"
-      value: "false"
-    - name: "CODER_LOGGING_HUMAN"
-      value: "/dev/null"
-    - name: "CODER_LOGGING_STACKDRIVER"
-      value: "/dev/stderr"
-    - name: "CODER_PG_CONNECTION_URL"
-      valueFrom:
-        secretKeyRef:
-          name: "${kubernetes_secret.coder-db.metadata.0.name}"
-          key: url
-    - name: "CODER_PPROF_ENABLE"
-      value: "true"
-    - name: "CODER_PROMETHEUS_ENABLE"
-      value: "true"
-    - name: "CODER_PROMETHEUS_COLLECT_AGENT_STATS"
-      value: "true"
-    - name: "CODER_PROMETHEUS_COLLECT_DB_METRICS"
-      value: "true"
-    - name: "CODER_VERBOSE"
-      value: "true"
-    - name: "CODER_EXPERIMENTS"
-      value: "${var.coder_experiments}"
-    - name: "CODER_DANGEROUS_DISABLE_RATE_LIMITS"
-      value: "true"
-    # Disabling built-in provisioner daemons
-    - name: "CODER_PROVISIONER_DAEMONS"
-      value: "0"
-    - name: CODER_PROVISIONER_DAEMON_PSK
-      valueFrom:
-        secretKeyRef:
-          key: psk
-          name: "${kubernetes_secret.provisionerd_psk.metadata.0.name}"
-    # Enable OIDC
-    # - name: "CODER_OIDC_ISSUER_URL"
-    #   valueFrom:
-    #     secretKeyRef:
-    #       key: issuer-url
-    #       name: "${data.kubernetes_secret.coder_oidc.metadata.0.name}"
-    # - name: "CODER_OIDC_EMAIL_DOMAIN"
-    #   valueFrom:
-    #     secretKeyRef:
-    #       key: email-domain
-    #       name: "${data.kubernetes_secret.coder_oidc.metadata.0.name}"
-    # - name: "CODER_OIDC_CLIENT_ID"
-    #   valueFrom:
-    #     secretKeyRef:
-    #       key: client-id
-    #       name: "${data.kubernetes_secret.coder_oidc.metadata.0.name}"
-    # - name: "CODER_OIDC_CLIENT_SECRET"
-    #   valueFrom:
-    #     secretKeyRef:
-    #       key: client-secret
-    #       name: "${data.kubernetes_secret.coder_oidc.metadata.0.name}"
-    # Send OTEL traces to the cluster-local collector to sample 10%
-    - name: "OTEL_EXPORTER_OTLP_ENDPOINT"
-      value: "http://otel-collector.${kubernetes_namespace.coder_namespace.metadata.0.name}.svc.cluster.local:4317"
-    - name: "OTEL_TRACES_SAMPLER"
-      value: parentbased_traceidratio
-    - name: "OTEL_TRACES_SAMPLER_ARG"
-      value: "0.1"
-  image:
-    repo: ${var.coder_image_repo}
-    tag: ${var.coder_image_tag}
-  replicaCount: "${var.coder_replicas}"
-  resources:
-    requests:
-      cpu: "${var.coder_cpu_request}"
-      memory: "${var.coder_mem_request}"
-    limits:
-      cpu: "${var.coder_cpu_limit}"
-      memory: "${var.coder_mem_limit}"
-  securityContext:
-    readOnlyRootFilesystem: true
-  service:
-    enable: true
-    sessionAffinity: None
-    loadBalancerIP: "${google_compute_address.coder["primary"].address}"
-  volumeMounts:
-  - mountPath: "/tmp"
-    name: cache
-    readOnly: false
-  volumes:
-  - emptyDir:
-      sizeLimit: 1024Mi
-    name: cache
-EOF
-  ]
-}
-
-
-resource "kubernetes_secret" "proxy_token_europe" {
-  provider = kubernetes.europe
-
-  type = "Opaque"
-  metadata {
-    name      = "coder-proxy-token"
-    namespace = kubernetes_namespace.coder_europe.metadata.0.name
-  }
-  data = {
-    token = trimspace(data.local_file.europe_proxy_token.content)
-  }
-  lifecycle {
-    ignore_changes = [timeouts, wait_for_service_account_token]
-  }
-}
-
-resource "helm_release" "coder_europe" {
-  provider = helm.europe
-
-  repository = local.coder_helm_repo
-  chart      = local.coder_helm_chart
-  name       = local.coder_release_name
-  version    = var.coder_chart_version
-  namespace  = kubernetes_namespace.coder_europe.metadata.0.name
-  values = [<<EOF
-coder:
-  workspaceProxy: true
-  affinity:
-    nodeAffinity:
-      requiredDuringSchedulingIgnoredDuringExecution:
-        nodeSelectorTerms:
-        - matchExpressions:
-          - key: "cloud.google.com/gke-nodepool"
-            operator: "In"
-            values: ["${google_container_node_pool.node_pool["europe_coder"].name}"]
-    podAntiAffinity:
-      preferredDuringSchedulingIgnoredDuringExecution:
-      - weight: 1
-        podAffinityTerm:
-          topologyKey: "kubernetes.io/hostname"
-          labelSelector:
-            matchExpressions:
-            - key:      "app.kubernetes.io/instance"
-              operator: "In"
-              values:   ["${local.coder_release_name}"]
-  env:
-    - name: CODER_PRIMARY_ACCESS_URL
-      value: "${local.coder_url}"
-    - name: CODER_PROXY_SESSION_TOKEN
-      valueFrom:
-        secretKeyRef:
-          key: token
-          name: "${kubernetes_secret.proxy_token_europe.metadata.0.name}"
-    - name: "CODER_ACCESS_URL"
-      value: "${local.coder_europe_url}"
-    - name: "CODER_CACHE_DIRECTORY"
-      value: "/tmp/coder"
-    - name: "CODER_TELEMETRY_ENABLE"
-      value: "false"
-    - name: "CODER_LOGGING_HUMAN"
-      value: "/dev/null"
-    - name: "CODER_LOGGING_STACKDRIVER"
-      value: "/dev/stderr"
-    - name: "CODER_PPROF_ENABLE"
-      value: "true"
-    - name: "CODER_PROMETHEUS_ENABLE"
-      value: "true"
-    - name: "CODER_PROMETHEUS_COLLECT_AGENT_STATS"
-      value: "true"
-    - name: "CODER_PROMETHEUS_COLLECT_DB_METRICS"
-      value: "true"
-    - name: "CODER_VERBOSE"
-      value: "true"
-    - name: "CODER_EXPERIMENTS"
-      value: "${var.coder_experiments}"
-    - name: "CODER_DANGEROUS_DISABLE_RATE_LIMITS"
-      value: "true"
-    # Disabling built-in provisioner daemons
-    - name: "CODER_PROVISIONER_DAEMONS"
-      value: "0"
-    - name: CODER_PROVISIONER_DAEMON_PSK
-      valueFrom:
-        secretKeyRef:
-          key: psk
-          name: "${kubernetes_secret.provisionerd_psk_europe.metadata.0.name}"
-    # Enable OIDC
-    # - name: "CODER_OIDC_ISSUER_URL"
-    #   valueFrom:
-    #     secretKeyRef:
-    #       key: issuer-url
-    #       name: "${data.kubernetes_secret.coder_oidc_europe.metadata.0.name}"
-    # - name: "CODER_OIDC_EMAIL_DOMAIN"
-    #   valueFrom:
-    #     secretKeyRef:
-    #       key: email-domain
-    #       name: "${data.kubernetes_secret.coder_oidc_europe.metadata.0.name}"
-    # - name: "CODER_OIDC_CLIENT_ID"
-    #   valueFrom:
-    #     secretKeyRef:
-    #       key: client-id
-    #       name: "${data.kubernetes_secret.coder_oidc_europe.metadata.0.name}"
-    # - name: "CODER_OIDC_CLIENT_SECRET"
-    #   valueFrom:
-    #     secretKeyRef:
-    #       key: client-secret
-    #       name: "${data.kubernetes_secret.coder_oidc_europe.metadata.0.name}"
-    # Send OTEL traces to the cluster-local collector to sample 10%
-    - name: "OTEL_EXPORTER_OTLP_ENDPOINT"
-      value: "http://otel-collector.${kubernetes_namespace.coder_europe.metadata.0.name}.svc.cluster.local:4317"
-    - name: "OTEL_TRACES_SAMPLER"
-      value: parentbased_traceidratio
-    - name: "OTEL_TRACES_SAMPLER_ARG"
-      value: "0.1"
-  image:
-    repo: ${var.coder_image_repo}
-    tag: ${var.coder_image_tag}
-  replicaCount: "${var.coder_replicas}"
-  resources:
-    requests:
-      cpu: "${var.coder_cpu_request}"
-      memory: "${var.coder_mem_request}"
-    limits:
-      cpu: "${var.coder_cpu_limit}"
-      memory: "${var.coder_mem_limit}"
-  securityContext:
-    readOnlyRootFilesystem: true
-  service:
-    enable: true
-    sessionAffinity: None
-    loadBalancerIP: "${google_compute_address.coder["europe"].address}"
-  volumeMounts:
-  - mountPath: "/tmp"
-    name: cache
-    readOnly: false
-  volumes:
-  - emptyDir:
-      sizeLimit: 1024Mi
-    name: cache
-EOF
-  ]
-}
-
-resource "helm_release" "provisionerd-chart" {
-  provider = helm.primary
-  
-  repository = local.coder_helm_repo
-  chart      = local.provisionerd_helm_chart
-  name       = local.provisionerd_release_name
-  version    = var.provisionerd_chart_version
-  namespace  = kubernetes_namespace.coder_namespace.metadata.0.name
-  values = [<<EOF
-coder:
-  affinity:
-    nodeAffinity:
-      requiredDuringSchedulingIgnoredDuringExecution:
-        nodeSelectorTerms:
-        - matchExpressions:
-          - key: "cloud.google.com/gke-nodepool"
-            operator: "In"
-            values: ["${google_container_node_pool.node_pool["primary_coder"].name}"]
-    podAntiAffinity:
-      preferredDuringSchedulingIgnoredDuringExecution:
-      - weight: 1
-        podAffinityTerm:
-          topologyKey: "kubernetes.io/hostname"
-          labelSelector:
-            matchExpressions:
-            - key:      "app.kubernetes.io/instance"
-              operator: "In"
-              values:   ["${local.coder_release_name}"]
-  env:
-    - name: "CODER_URL"
-      value: "${local.coder_url}"
-    - name: "CODER_VERBOSE"
-      value: "true"
-    - name: "CODER_CONFIG_DIR"
-      value: "/tmp/config"
-    - name: "CODER_CACHE_DIRECTORY"
-      value: "/tmp/coder"
-    - name: "CODER_TELEMETRY_ENABLE"
-      value: "false"
-    - name: "CODER_LOGGING_HUMAN"
-      value: "/dev/null"
-    - name: "CODER_LOGGING_STACKDRIVER"
-      value: "/dev/stderr"
-    - name: "CODER_PROMETHEUS_ENABLE"
-      value: "true"
-    - name: "CODER_PROVISIONERD_TAGS"
-      value: "scope=organization"
-  image:
-    repo: ${var.provisionerd_image_repo}
-    tag: ${var.provisionerd_image_tag}
-  replicaCount: "${var.provisionerd_replicas}"
-  resources:
-    requests:
-      cpu: "${var.provisionerd_cpu_request}"
-      memory: "${var.provisionerd_mem_request}"
-    limits:
-      cpu: "${var.provisionerd_cpu_limit}"
-      memory: "${var.provisionerd_mem_limit}"
-  securityContext:
-    readOnlyRootFilesystem: true
-  volumeMounts:
-  - mountPath: "/tmp"
-    name: cache
-    readOnly: false
-  volumes:
-  - emptyDir:
-      sizeLimit: 1024Mi
-    name: cache
-EOF
-  ]
-}
-
-resource "helm_release" "provisionerd_europe" {
-  provider = helm.europe
-  
-  repository = local.coder_helm_repo
-  chart      = local.provisionerd_helm_chart
-  name       = local.provisionerd_release_name
-  version    = var.provisionerd_chart_version
-  namespace  = kubernetes_namespace.coder_europe.metadata.0.name
-  values = [<<EOF
-coder:
-  affinity:
-    nodeAffinity:
-      requiredDuringSchedulingIgnoredDuringExecution:
-        nodeSelectorTerms:
-        - matchExpressions:
-          - key: "cloud.google.com/gke-nodepool"
-            operator: "In"
-            values: ["${google_container_node_pool.node_pool["europe_coder"].name}"]
-    podAntiAffinity:
-      preferredDuringSchedulingIgnoredDuringExecution:
-      - weight: 1
-        podAffinityTerm:
-          topologyKey: "kubernetes.io/hostname"
-          labelSelector:
-            matchExpressions:
-            - key:      "app.kubernetes.io/instance"
-              operator: "In"
-              values:   ["${local.coder_release_name}"]
-  env:
-    - name: "CODER_URL"
-      value: "${local.coder_url}"
-    - name: "CODER_VERBOSE"
-      value: "true"
-    - name: "CODER_CONFIG_DIR"
-      value: "/tmp/config"
-    - name: "CODER_CACHE_DIRECTORY"
-      value: "/tmp/coder"
-    - name: "CODER_TELEMETRY_ENABLE"
-      value: "false"
-    - name: "CODER_LOGGING_HUMAN"
-      value: "/dev/null"
-    - name: "CODER_LOGGING_STACKDRIVER"
-      value: "/dev/stderr"
-    - name: "CODER_PROMETHEUS_ENABLE"
-      value: "true"
-    - name: "CODER_PROVISIONERD_TAGS"
-      value: "scope=organization"
-  image:
-    repo: ${var.provisionerd_image_repo}
-    tag: ${var.provisionerd_image_tag}
-  replicaCount: "${var.provisionerd_replicas}"
-  resources:
-    requests:
-      cpu: "${var.provisionerd_cpu_request}"
-      memory: "${var.provisionerd_mem_request}"
-    limits:
-      cpu: "${var.provisionerd_cpu_limit}"
-      memory: "${var.provisionerd_mem_limit}"
-  securityContext:
-    readOnlyRootFilesystem: true
-  volumeMounts:
-  - mountPath: "/tmp"
-    name: cache
-    readOnly: false
-  volumes:
-  - emptyDir:
-      sizeLimit: 1024Mi
-    name: cache
-EOF
-  ]
-}
-
-data "http" "coder_healthy" {
-  url = local.coder_url
-  // Wait up to 5 minutes for DNS to propogate
-  retry {
-    attempts = 30
-    min_delay_ms = 10000
-  }
-
-  lifecycle {
-    postcondition {
-        condition = self.status_code == 200
-        error_message = "${self.url} returned an unhealthy status code"
-    }
-  }
-
-  depends_on = [ helm_release.coder-chart, cloudflare_record.coder ]
-}
-
-resource "null_resource" "proxy_tokens" {
-  provisioner "local-exec" {
-    interpreter = [ "/bin/bash", "-c" ]
-    command = <<EOF
-curl '${local.coder_url}/api/v2/users/first' \
-  --data-raw $'{"email":"${local.coder_admin_email}","password":"${local.coder_admin_password}","username":"${local.coder_admin_user}","name":"${local.coder_admin_full_name}","trial":false}' \
-  --insecure --silent --output /dev/null
-
-token=$(curl '${local.coder_url}/api/v2/users/login' \
-  --data-raw $'{"email":"${local.coder_admin_email}","password":"${local.coder_admin_password}"}' \
-  --insecure --silent | jq -r .session_token)
-
-curl '${local.coder_url}/api/v2/licenses' \
-  -H "Coder-Session-Token: $${token}" \
-  --data-raw '{"license":"${var.coder_license}"}' \
-  --insecure --silent --output /dev/null
-
-europe_token=$(curl '${local.coder_url}/api/v2/workspaceproxies' \
-  -H "Coder-Session-Token: $${token}" \
-  --data-raw '{"name":"europe"}' \
-  --insecure --silent | jq -r .proxy_token)
-
-asia_token=$(curl '${local.coder_url}/api/v2/workspaceproxies' \
-  -H "Coder-Session-Token: $${token}" \
-  --data-raw '{"name":"asia"}' \
-  --insecure --silent | jq -r .proxy_token)
-
-echo -n $${europe_token} > ${path.module}/europe_proxy_token
-echo -n $${asia_token} > ${path.module}/asia_proxy_token
-EOF
-  }
-
-  depends_on = [ data.http.coder_healthy ]
-}
-
-data "local_file" "europe_proxy_token" {
-  filename = "${path.module}/europe_proxy_token"
-  depends_on = [ null_resource.proxy_tokens ]
-}
-
-data "local_file" "asia_proxy_token" {
-  filename = "${path.module}/asia_proxy_token"
-  depends_on = [ null_resource.proxy_tokens ]
-}
-
-# data "external" "proxy_tokens" {
-#   program = ["bash", "${path.module}/workspace_proxies.sh"]
-#   query = {
-#     coder_url = local.coder_url
-#     coder_admin_email = local.coder_admin_email
-#     coder_admin_password = local.coder_admin_password
-#     coder_admin_user = local.coder_admin_user
-#     coder_admin_full_name = local.coder_admin_full_name
-#     coder_license = var.coder_license
-
-#     status_code = data.http.coder_healthy.status_code
-#   }
-
-#   depends_on = [ data.http.coder_healthy ]
-# }
-
diff --git a/scaletest/terraform/new/k8s_coder_asia.tf b/scaletest/terraform/new/k8s_coder_asia.tf
new file mode 100644
index 0000000000000..6067621f01d75
--- /dev/null
+++ b/scaletest/terraform/new/k8s_coder_asia.tf
@@ -0,0 +1,212 @@
+resource "kubernetes_namespace" "coder_asia" {
+  provider = kubernetes.asia
+
+  metadata {
+    name = local.coder_namespace
+  }
+  lifecycle {
+    ignore_changes = [timeouts, wait_for_default_service_account]
+  }
+}
+
+resource "kubernetes_secret" "provisionerd_psk_asia" {
+  provider = kubernetes.asia
+
+  type = "Opaque"
+  metadata {
+    name      = "coder-provisioner-psk"
+    namespace = kubernetes_namespace.coder_asia.metadata.0.name
+  }
+  data = {
+    psk = random_password.provisionerd_psk.result
+  }
+  lifecycle {
+    ignore_changes = [timeouts, wait_for_service_account_token]
+  }
+}
+
+resource "kubernetes_secret" "proxy_token_asia" {
+  provider = kubernetes.asia
+
+  type = "Opaque"
+  metadata {
+    name      = "coder-proxy-token"
+    namespace = kubernetes_namespace.coder_asia.metadata.0.name
+  }
+  data = {
+    token = trimspace(data.local_file.asia_proxy_token.content)
+  }
+  lifecycle {
+    ignore_changes = [timeouts, wait_for_service_account_token]
+  }
+}
+
+resource "helm_release" "coder_asia" {
+  provider = helm.asia
+
+  repository = local.coder_helm_repo
+  chart      = local.coder_helm_chart
+  name       = local.coder_release_name
+  version    = var.coder_chart_version
+  namespace  = kubernetes_namespace.coder_asia.metadata.0.name
+  values = [<<EOF
+coder:
+  workspaceProxy: true
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: "cloud.google.com/gke-nodepool"
+            operator: "In"
+            values: ["${google_container_node_pool.node_pool["asia_coder"].name}"]
+    podAntiAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+      - weight: 1
+        podAffinityTerm:
+          topologyKey: "kubernetes.io/hostname"
+          labelSelector:
+            matchExpressions:
+            - key:      "app.kubernetes.io/instance"
+              operator: "In"
+              values:   ["${local.coder_release_name}"]
+  env:
+    - name: CODER_PRIMARY_ACCESS_URL
+      value: "${local.deployments.primary.url}"
+    - name: CODER_PROXY_SESSION_TOKEN
+      valueFrom:
+        secretKeyRef:
+          key: token
+          name: "${kubernetes_secret.proxy_token_asia.metadata.0.name}"
+    - name: "CODER_ACCESS_URL"
+      value: "${local.deployments.asia.url}"
+    - name: "CODER_CACHE_DIRECTORY"
+      value: "/tmp/coder"
+    - name: "CODER_TELEMETRY_ENABLE"
+      value: "false"
+    - name: "CODER_LOGGING_HUMAN"
+      value: "/dev/null"
+    - name: "CODER_LOGGING_STACKDRIVER"
+      value: "/dev/stderr"
+    - name: "CODER_PPROF_ENABLE"
+      value: "true"
+    - name: "CODER_PROMETHEUS_ENABLE"
+      value: "true"
+    - name: "CODER_PROMETHEUS_COLLECT_AGENT_STATS"
+      value: "true"
+    - name: "CODER_PROMETHEUS_COLLECT_DB_METRICS"
+      value: "true"
+    - name: "CODER_VERBOSE"
+      value: "true"
+    - name: "CODER_EXPERIMENTS"
+      value: "${var.coder_experiments}"
+    - name: "CODER_DANGEROUS_DISABLE_RATE_LIMITS"
+      value: "true"
+    # Disabling built-in provisioner daemons
+    - name: "CODER_PROVISIONER_DAEMONS"
+      value: "0"
+    - name: CODER_PROVISIONER_DAEMON_PSK
+      valueFrom:
+        secretKeyRef:
+          key: psk
+          name: "${kubernetes_secret.provisionerd_psk_asia.metadata.0.name}"
+  image:
+    repo: ${var.coder_image_repo}
+    tag: ${var.coder_image_tag}
+  replicaCount: "${var.coder_replicas}"
+  resources:
+    requests:
+      cpu: "${var.coder_cpu_request}"
+      memory: "${var.coder_mem_request}"
+    limits:
+      cpu: "${var.coder_cpu_limit}"
+      memory: "${var.coder_mem_limit}"
+  securityContext:
+    readOnlyRootFilesystem: true
+  service:
+    enable: true
+    sessionAffinity: None
+    loadBalancerIP: "${google_compute_address.coder["asia"].address}"
+  volumeMounts:
+  - mountPath: "/tmp"
+    name: cache
+    readOnly: false
+  volumes:
+  - emptyDir:
+      sizeLimit: 1024Mi
+    name: cache
+EOF
+  ]
+}
+
+resource "helm_release" "provisionerd_asia" {
+  provider = helm.asia
+  
+  repository = local.coder_helm_repo
+  chart      = local.provisionerd_helm_chart
+  name       = local.provisionerd_release_name
+  version    = var.provisionerd_chart_version
+  namespace  = kubernetes_namespace.coder_asia.metadata.0.name
+  values = [<<EOF
+coder:
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: "cloud.google.com/gke-nodepool"
+            operator: "In"
+            values: ["${google_container_node_pool.node_pool["asia_coder"].name}"]
+    podAntiAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+      - weight: 1
+        podAffinityTerm:
+          topologyKey: "kubernetes.io/hostname"
+          labelSelector:
+            matchExpressions:
+            - key:      "app.kubernetes.io/instance"
+              operator: "In"
+              values:   ["${local.coder_release_name}"]
+  env:
+    - name: "CODER_URL"
+      value: "${local.deployments.primary.url}"
+    - name: "CODER_VERBOSE"
+      value: "true"
+    - name: "CODER_CONFIG_DIR"
+      value: "/tmp/config"
+    - name: "CODER_CACHE_DIRECTORY"
+      value: "/tmp/coder"
+    - name: "CODER_TELEMETRY_ENABLE"
+      value: "false"
+    - name: "CODER_LOGGING_HUMAN"
+      value: "/dev/null"
+    - name: "CODER_LOGGING_STACKDRIVER"
+      value: "/dev/stderr"
+    - name: "CODER_PROMETHEUS_ENABLE"
+      value: "true"
+    - name: "CODER_PROVISIONERD_TAGS"
+      value: "scope=organization"
+  image:
+    repo: ${var.provisionerd_image_repo}
+    tag: ${var.provisionerd_image_tag}
+  replicaCount: "${var.provisionerd_replicas}"
+  resources:
+    requests:
+      cpu: "${var.provisionerd_cpu_request}"
+      memory: "${var.provisionerd_mem_request}"
+    limits:
+      cpu: "${var.provisionerd_cpu_limit}"
+      memory: "${var.provisionerd_mem_limit}"
+  securityContext:
+    readOnlyRootFilesystem: true
+  volumeMounts:
+  - mountPath: "/tmp"
+    name: cache
+    readOnly: false
+  volumes:
+  - emptyDir:
+      sizeLimit: 1024Mi
+    name: cache
+EOF
+  ]
+}
diff --git a/scaletest/terraform/new/k8s_coder_europe.tf b/scaletest/terraform/new/k8s_coder_europe.tf
new file mode 100644
index 0000000000000..7609f6eb783e5
--- /dev/null
+++ b/scaletest/terraform/new/k8s_coder_europe.tf
@@ -0,0 +1,212 @@
+resource "kubernetes_namespace" "coder_europe" {
+  provider = kubernetes.europe
+
+  metadata {
+    name = local.coder_namespace
+  }
+  lifecycle {
+    ignore_changes = [timeouts, wait_for_default_service_account]
+  }
+}
+
+resource "kubernetes_secret" "provisionerd_psk_europe" {
+  provider = kubernetes.europe
+
+  type = "Opaque"
+  metadata {
+    name      = "coder-provisioner-psk"
+    namespace = kubernetes_namespace.coder_europe.metadata.0.name
+  }
+  data = {
+    psk = random_password.provisionerd_psk.result
+  }
+  lifecycle {
+    ignore_changes = [timeouts, wait_for_service_account_token]
+  }
+}
+
+resource "kubernetes_secret" "proxy_token_europe" {
+  provider = kubernetes.europe
+
+  type = "Opaque"
+  metadata {
+    name      = "coder-proxy-token"
+    namespace = kubernetes_namespace.coder_europe.metadata.0.name
+  }
+  data = {
+    token = trimspace(data.local_file.europe_proxy_token.content)
+  }
+  lifecycle {
+    ignore_changes = [timeouts, wait_for_service_account_token]
+  }
+}
+
+resource "helm_release" "coder_europe" {
+  provider = helm.europe
+
+  repository = local.coder_helm_repo
+  chart      = local.coder_helm_chart
+  name       = local.coder_release_name
+  version    = var.coder_chart_version
+  namespace  = kubernetes_namespace.coder_europe.metadata.0.name
+  values = [<<EOF
+coder:
+  workspaceProxy: true
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: "cloud.google.com/gke-nodepool"
+            operator: "In"
+            values: ["${google_container_node_pool.node_pool["europe_coder"].name}"]
+    podAntiAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+      - weight: 1
+        podAffinityTerm:
+          topologyKey: "kubernetes.io/hostname"
+          labelSelector:
+            matchExpressions:
+            - key:      "app.kubernetes.io/instance"
+              operator: "In"
+              values:   ["${local.coder_release_name}"]
+  env:
+    - name: CODER_PRIMARY_ACCESS_URL
+      value: "${local.deployments.primary.url}"
+    - name: CODER_PROXY_SESSION_TOKEN
+      valueFrom:
+        secretKeyRef:
+          key: token
+          name: "${kubernetes_secret.proxy_token_europe.metadata.0.name}"
+    - name: "CODER_ACCESS_URL"
+      value: "${local.deployments.europe.url}"
+    - name: "CODER_CACHE_DIRECTORY"
+      value: "/tmp/coder"
+    - name: "CODER_TELEMETRY_ENABLE"
+      value: "false"
+    - name: "CODER_LOGGING_HUMAN"
+      value: "/dev/null"
+    - name: "CODER_LOGGING_STACKDRIVER"
+      value: "/dev/stderr"
+    - name: "CODER_PPROF_ENABLE"
+      value: "true"
+    - name: "CODER_PROMETHEUS_ENABLE"
+      value: "true"
+    - name: "CODER_PROMETHEUS_COLLECT_AGENT_STATS"
+      value: "true"
+    - name: "CODER_PROMETHEUS_COLLECT_DB_METRICS"
+      value: "true"
+    - name: "CODER_VERBOSE"
+      value: "true"
+    - name: "CODER_EXPERIMENTS"
+      value: "${var.coder_experiments}"
+    - name: "CODER_DANGEROUS_DISABLE_RATE_LIMITS"
+      value: "true"
+    # Disabling built-in provisioner daemons
+    - name: "CODER_PROVISIONER_DAEMONS"
+      value: "0"
+    - name: CODER_PROVISIONER_DAEMON_PSK
+      valueFrom:
+        secretKeyRef:
+          key: psk
+          name: "${kubernetes_secret.provisionerd_psk_europe.metadata.0.name}"
+  image:
+    repo: ${var.coder_image_repo}
+    tag: ${var.coder_image_tag}
+  replicaCount: "${var.coder_replicas}"
+  resources:
+    requests:
+      cpu: "${var.coder_cpu_request}"
+      memory: "${var.coder_mem_request}"
+    limits:
+      cpu: "${var.coder_cpu_limit}"
+      memory: "${var.coder_mem_limit}"
+  securityContext:
+    readOnlyRootFilesystem: true
+  service:
+    enable: true
+    sessionAffinity: None
+    loadBalancerIP: "${google_compute_address.coder["europe"].address}"
+  volumeMounts:
+  - mountPath: "/tmp"
+    name: cache
+    readOnly: false
+  volumes:
+  - emptyDir:
+      sizeLimit: 1024Mi
+    name: cache
+EOF
+  ]
+}
+
+resource "helm_release" "provisionerd_europe" {
+  provider = helm.europe
+  
+  repository = local.coder_helm_repo
+  chart      = local.provisionerd_helm_chart
+  name       = local.provisionerd_release_name
+  version    = var.provisionerd_chart_version
+  namespace  = kubernetes_namespace.coder_europe.metadata.0.name
+  values = [<<EOF
+coder:
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: "cloud.google.com/gke-nodepool"
+            operator: "In"
+            values: ["${google_container_node_pool.node_pool["europe_coder"].name}"]
+    podAntiAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+      - weight: 1
+        podAffinityTerm:
+          topologyKey: "kubernetes.io/hostname"
+          labelSelector:
+            matchExpressions:
+            - key:      "app.kubernetes.io/instance"
+              operator: "In"
+              values:   ["${local.coder_release_name}"]
+  env:
+    - name: "CODER_URL"
+      value: "${local.deployments.primary.url}"
+    - name: "CODER_VERBOSE"
+      value: "true"
+    - name: "CODER_CONFIG_DIR"
+      value: "/tmp/config"
+    - name: "CODER_CACHE_DIRECTORY"
+      value: "/tmp/coder"
+    - name: "CODER_TELEMETRY_ENABLE"
+      value: "false"
+    - name: "CODER_LOGGING_HUMAN"
+      value: "/dev/null"
+    - name: "CODER_LOGGING_STACKDRIVER"
+      value: "/dev/stderr"
+    - name: "CODER_PROMETHEUS_ENABLE"
+      value: "true"
+    - name: "CODER_PROVISIONERD_TAGS"
+      value: "scope=organization"
+  image:
+    repo: ${var.provisionerd_image_repo}
+    tag: ${var.provisionerd_image_tag}
+  replicaCount: "${var.provisionerd_replicas}"
+  resources:
+    requests:
+      cpu: "${var.provisionerd_cpu_request}"
+      memory: "${var.provisionerd_mem_request}"
+    limits:
+      cpu: "${var.provisionerd_cpu_limit}"
+      memory: "${var.provisionerd_mem_limit}"
+  securityContext:
+    readOnlyRootFilesystem: true
+  volumeMounts:
+  - mountPath: "/tmp"
+    name: cache
+    readOnly: false
+  volumes:
+  - emptyDir:
+      sizeLimit: 1024Mi
+    name: cache
+EOF
+  ]
+}
diff --git a/scaletest/terraform/new/k8s_coder_primary.tf b/scaletest/terraform/new/k8s_coder_primary.tf
new file mode 100644
index 0000000000000..c91464c7d8795
--- /dev/null
+++ b/scaletest/terraform/new/k8s_coder_primary.tf
@@ -0,0 +1,229 @@
+data "google_client_config" "default" {}
+
+locals {
+  coder_admin_email         = "admin@coder.com"
+  coder_admin_full_name     = "Coder Admin"
+  coder_admin_user          = "coder"
+  coder_admin_password      = "SomeSecurePassword!"
+  coder_helm_repo           = "https://helm.coder.com/v2"
+  coder_helm_chart          = "coder"
+  coder_namespace           = "coder"
+  coder_release_name        = "${var.name}-coder"
+  provisionerd_helm_chart   = "coder-provisioner"
+  provisionerd_release_name = "${var.name}-provisionerd"
+
+}
+
+resource "random_password" "provisionerd_psk" {
+  length = 26
+}
+
+resource "kubernetes_namespace" "coder_primary" {
+  provider = kubernetes.primary
+
+  metadata {
+    name = local.coder_namespace
+  }
+  lifecycle {
+    ignore_changes = [timeouts, wait_for_default_service_account]
+  }
+}
+
+resource "kubernetes_secret" "coder_db" {
+  provider = kubernetes.primary
+
+  type = "Opaque"
+  metadata {
+    name      = "coder-db-url"
+    namespace = kubernetes_namespace.coder_primary.metadata.0.name
+  }
+  data = {
+    url = local.coder_db_url
+  }
+  lifecycle {
+    ignore_changes = [timeouts, wait_for_service_account_token]
+  }
+}
+
+resource "kubernetes_secret" "provisionerd_psk_primary" {
+  provider = kubernetes.primary
+
+  type = "Opaque"
+  metadata {
+    name      = "coder-provisioner-psk"
+    namespace = kubernetes_namespace.coder_primary.metadata.0.name
+  }
+  data = {
+    psk = random_password.provisionerd_psk.result
+  }
+  lifecycle {
+    ignore_changes = [timeouts, wait_for_service_account_token]
+  }
+}
+
+resource "helm_release" "coder_primary" {
+  provider = helm.primary
+
+  repository = local.coder_helm_repo
+  chart      = local.coder_helm_chart
+  name       = local.coder_release_name
+  version    = var.coder_chart_version
+  namespace  = kubernetes_namespace.coder_primary.metadata.0.name
+  values = [<<EOF
+coder:
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: "cloud.google.com/gke-nodepool"
+            operator: "In"
+            values: ["${google_container_node_pool.node_pool["primary_coder"].name}"]
+    podAntiAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+      - weight: 1
+        podAffinityTerm:
+          topologyKey: "kubernetes.io/hostname"
+          labelSelector:
+            matchExpressions:
+            - key:      "app.kubernetes.io/instance"
+              operator: "In"
+              values:   ["${local.coder_release_name}"]
+  env:
+    - name: "CODER_ACCESS_URL"
+      value: "${local.deployments.primary.url}"
+    - name: "CODER_CACHE_DIRECTORY"
+      value: "/tmp/coder"
+    - name: "CODER_TELEMETRY_ENABLE"
+      value: "false"
+    - name: "CODER_LOGGING_HUMAN"
+      value: "/dev/null"
+    - name: "CODER_LOGGING_STACKDRIVER"
+      value: "/dev/stderr"
+    - name: "CODER_PG_CONNECTION_URL"
+      valueFrom:
+        secretKeyRef:
+          name: "${kubernetes_secret.coder_db.metadata.0.name}"
+          key: url
+    - name: "CODER_PPROF_ENABLE"
+      value: "true"
+    - name: "CODER_PROMETHEUS_ENABLE"
+      value: "true"
+    - name: "CODER_PROMETHEUS_COLLECT_AGENT_STATS"
+      value: "true"
+    - name: "CODER_PROMETHEUS_COLLECT_DB_METRICS"
+      value: "true"
+    - name: "CODER_VERBOSE"
+      value: "true"
+    - name: "CODER_EXPERIMENTS"
+      value: "${var.coder_experiments}"
+    - name: "CODER_DANGEROUS_DISABLE_RATE_LIMITS"
+      value: "true"
+    # Disabling built-in provisioner daemons
+    - name: "CODER_PROVISIONER_DAEMONS"
+      value: "0"
+    - name: CODER_PROVISIONER_DAEMON_PSK
+      valueFrom:
+        secretKeyRef:
+          key: psk
+          name: "${kubernetes_secret.provisionerd_psk_primary.metadata.0.name}"
+  image:
+    repo: ${var.coder_image_repo}
+    tag: ${var.coder_image_tag}
+  replicaCount: "${var.coder_replicas}"
+  resources:
+    requests:
+      cpu: "${var.coder_cpu_request}"
+      memory: "${var.coder_mem_request}"
+    limits:
+      cpu: "${var.coder_cpu_limit}"
+      memory: "${var.coder_mem_limit}"
+  securityContext:
+    readOnlyRootFilesystem: true
+  service:
+    enable: true
+    sessionAffinity: None
+    loadBalancerIP: "${google_compute_address.coder["primary"].address}"
+  volumeMounts:
+  - mountPath: "/tmp"
+    name: cache
+    readOnly: false
+  volumes:
+  - emptyDir:
+      sizeLimit: 1024Mi
+    name: cache
+EOF
+  ]
+}
+
+resource "helm_release" "provisionerd_chart" {
+  provider = helm.primary
+  
+  repository = local.coder_helm_repo
+  chart      = local.provisionerd_helm_chart
+  name       = local.provisionerd_release_name
+  version    = var.provisionerd_chart_version
+  namespace  = kubernetes_namespace.coder_primary.metadata.0.name
+  values = [<<EOF
+coder:
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: "cloud.google.com/gke-nodepool"
+            operator: "In"
+            values: ["${google_container_node_pool.node_pool["primary_coder"].name}"]
+    podAntiAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+      - weight: 1
+        podAffinityTerm:
+          topologyKey: "kubernetes.io/hostname"
+          labelSelector:
+            matchExpressions:
+            - key:      "app.kubernetes.io/instance"
+              operator: "In"
+              values:   ["${local.coder_release_name}"]
+  env:
+    - name: "CODER_URL"
+      value: "${local.deployments.primary.url}"
+    - name: "CODER_VERBOSE"
+      value: "true"
+    - name: "CODER_CONFIG_DIR"
+      value: "/tmp/config"
+    - name: "CODER_CACHE_DIRECTORY"
+      value: "/tmp/coder"
+    - name: "CODER_TELEMETRY_ENABLE"
+      value: "false"
+    - name: "CODER_LOGGING_HUMAN"
+      value: "/dev/null"
+    - name: "CODER_LOGGING_STACKDRIVER"
+      value: "/dev/stderr"
+    - name: "CODER_PROMETHEUS_ENABLE"
+      value: "true"
+    - name: "CODER_PROVISIONERD_TAGS"
+      value: "scope=organization"
+  image:
+    repo: ${var.provisionerd_image_repo}
+    tag: ${var.provisionerd_image_tag}
+  replicaCount: "${var.provisionerd_replicas}"
+  resources:
+    requests:
+      cpu: "${var.provisionerd_cpu_request}"
+      memory: "${var.provisionerd_mem_request}"
+    limits:
+      cpu: "${var.provisionerd_cpu_limit}"
+      memory: "${var.provisionerd_mem_limit}"
+  securityContext:
+    readOnlyRootFilesystem: true
+  volumeMounts:
+  - mountPath: "/tmp"
+    name: cache
+    readOnly: false
+  volumes:
+  - emptyDir:
+      sizeLimit: 1024Mi
+    name: cache
+EOF
+  ]
+}
diff --git a/scaletest/terraform/new/k8s_coder_proxies.tf b/scaletest/terraform/new/k8s_coder_proxies.tf
new file mode 100644
index 0000000000000..cc9887957d598
--- /dev/null
+++ b/scaletest/terraform/new/k8s_coder_proxies.tf
@@ -0,0 +1,63 @@
+data "http" "coder_healthy" {
+  url = local.deployments.primary.url
+  // Wait up to 5 minutes for DNS to propogate
+  retry {
+    attempts = 30
+    min_delay_ms = 10000
+  }
+
+  lifecycle {
+    postcondition {
+        condition = self.status_code == 200
+        error_message = "${self.url} returned an unhealthy status code"
+    }
+  }
+
+  depends_on = [ helm_release.coder_primary, cloudflare_record.coder["primary"] ]
+}
+
+resource "null_resource" "proxy_tokens" {
+  provisioner "local-exec" {
+    interpreter = [ "/bin/bash", "-c" ]
+    command = <<EOF
+curl '${local.deployments.primary.url}/api/v2/users/first' \
+  --data-raw $'{"email":"${local.coder_admin_email}","password":"${local.coder_admin_password}","username":"${local.coder_admin_user}","name":"${local.coder_admin_full_name}","trial":false}' \
+  --insecure --silent --output /dev/null
+
+token=$(curl '${local.deployments.primary.url}/api/v2/users/login' \
+  --data-raw $'{"email":"${local.coder_admin_email}","password":"${local.coder_admin_password}"}' \
+  --insecure --silent | jq -r .session_token)
+
+curl '${local.deployments.primary.url}/api/v2/licenses' \
+  -H "Coder-Session-Token: $${token}" \
+  --data-raw '{"license":"${var.coder_license}"}' \
+  --insecure --silent --output /dev/null
+
+europe_token=$(curl '${local.deployments.primary.url}/api/v2/workspaceproxies' \
+  -H "Coder-Session-Token: $${token}" \
+  --data-raw '{"name":"europe"}' \
+  --insecure --silent | jq -r .proxy_token)
+
+asia_token=$(curl '${local.deployments.primary.url}/api/v2/workspaceproxies' \
+  -H "Coder-Session-Token: $${token}" \
+  --data-raw '{"name":"asia"}' \
+  --insecure --silent | jq -r .proxy_token)
+
+mkdir -p ${path.module}/.coderv2
+echo -n $${europe_token} > ${path.module}/.coderv2/europe_proxy_token
+echo -n $${asia_token} > ${path.module}/.coderv2/asia_proxy_token
+EOF
+  }
+
+  depends_on = [ data.http.coder_healthy ]
+}
+
+data "local_file" "europe_proxy_token" {
+  filename = "${path.module}/.coderv2/europe_proxy_token"
+  depends_on = [ null_resource.proxy_tokens ]
+}
+
+data "local_file" "asia_proxy_token" {
+  filename = "${path.module}/.coderv2/asia_proxy_token"
+  depends_on = [ null_resource.proxy_tokens ]
+}
diff --git a/scaletest/terraform/new/k8s_otel.tf b/scaletest/terraform/new/k8s_otel.tf
deleted file mode 100644
index ed48e5d01ea70..0000000000000
--- a/scaletest/terraform/new/k8s_otel.tf
+++ /dev/null
@@ -1,62 +0,0 @@
-# # Terraform configuration for OpenTelemetry Operator
-
-# locals {
-#   otel_namespace              = "opentelemetry-operator-system"
-#   otel_operator_helm_repo     = "https://open-telemetry.github.io/opentelemetry-helm-charts"
-#   otel_operator_helm_chart    = "opentelemetry-operator"
-#   otel_operator_release_name  = "opentelemetry-operator"
-#   otel_operator_chart_version = "0.34.1"
-# }
-
-# resource "kubernetes_namespace" "otel-namespace" {
-#   metadata {
-#     name = local.otel_namespace
-#   }
-#   lifecycle {
-#     ignore_changes = [timeouts, wait_for_default_service_account]
-#   }
-# }
-
-# resource "helm_release" "otel-operator" {
-#   repository = local.otel_operator_helm_repo
-#   chart      = local.otel_operator_helm_chart
-#   name       = local.otel_operator_release_name
-#   namespace  = kubernetes_namespace.otel-namespace.metadata.0.name
-#   values = [<<EOF
-# manager:
-#   collectorImage:
-#     repository: otel/opentelemetry-collector-k8s
-# EOF
-#   ]
-# }
-
-# resource "kubectl_manifest" "otel-collector" {
-#   depends_on = [ helm_release.otel-operator ]
-#     yaml_body = <<YAML
-# apiVersion: opentelemetry.io/v1alpha1
-# kind: OpenTelemetryCollector
-# metadata:
-#   name: otel
-#   namespace: ${kubernetes_namespace.coder_namespace.metadata.0.name}
-# spec:
-#   config: |
-#     receivers:
-#       otlp:
-#         protocols:
-#           grpc: {}
-#           http: {}
-#     exporters:
-#       googlecloud:
-#         logging:
-#           loglevel: debug
-#     service:
-#       pipelines:
-#         traces:
-#           receivers: [otlp]
-#           processors: []
-#           exporters: [logging, googlecloud]
-#     image: otel/open-telemetry-collector-contrib:latest
-#     mode: deployment
-#     replicas: 1
-# YAML
-# }
diff --git a/scaletest/terraform/new/main.tf b/scaletest/terraform/new/main.tf
index d1009dcfefcc0..8124e1fb0d854 100644
--- a/scaletest/terraform/new/main.tf
+++ b/scaletest/terraform/new/main.tf
@@ -46,6 +46,10 @@ terraform {
 provider "google" {
 }
 
+provider "cloudflare" {
+  api_token = var.cloudflare_api_token
+}
+
 provider "kubernetes" {
   alias = "primary"
   host                   = "https://${google_container_cluster.cluster["primary"].endpoint}"
@@ -60,6 +64,13 @@ provider "kubernetes" {
   token                  = data.google_client_config.default.access_token
 }
 
+provider "kubernetes" {
+  alias = "asia"
+  host                   = "https://${google_container_cluster.cluster["asia"].endpoint}"
+  cluster_ca_certificate = base64decode(google_container_cluster.cluster["asia"].master_auth.0.cluster_ca_certificate)
+  token                  = data.google_client_config.default.access_token
+}
+
 provider "kubectl" {
   alias = "primary"
   host                   = "https://${google_container_cluster.cluster["primary"].endpoint}"
@@ -76,6 +87,14 @@ provider "kubectl" {
   load_config_file       = false
 }
 
+provider "kubectl" {
+  alias = "asia"
+  host                   = "https://${google_container_cluster.cluster["asia"].endpoint}"
+  cluster_ca_certificate = base64decode(google_container_cluster.cluster["asia"].master_auth.0.cluster_ca_certificate)
+  token                  = data.google_client_config.default.access_token
+  load_config_file       = false
+}
+
 provider "helm" {
   alias = "primary"
   kubernetes {
@@ -94,6 +113,11 @@ provider "helm" {
   }
 }
 
-provider "cloudflare" {
-  api_token = var.cloudflare_api_token
+provider "helm" {
+  alias = "asia"
+  kubernetes {
+    host                   = "https://${google_container_cluster.cluster["asia"].endpoint}"
+    cluster_ca_certificate = base64decode(google_container_cluster.cluster["asia"].master_auth.0.cluster_ca_certificate)
+    token                  = data.google_client_config.default.access_token
+  }
 }
diff --git a/scaletest/terraform/new/vars.tf b/scaletest/terraform/new/vars.tf
index 513d142aebf1c..dc6de957b3c4d 100644
--- a/scaletest/terraform/new/vars.tf
+++ b/scaletest/terraform/new/vars.tf
@@ -7,18 +7,6 @@ variable "project_id" {
   description = "The project in which to provision resources"
 }
 
-# variable "deployments" {
-#   type = list(object({
-#     name = string
-#     region = string
-#     zone   = string
-#     subnet_cidr = string
-#     coder_node_pool_size = number
-#     workspaces_node_pool_size = number
-#     misc_node_pool_size = number
-#   }))
-# }
-
 variable "k8s_version" {
   description = "Kubernetes version to provision."
   default     = "1.24"
@@ -79,6 +67,11 @@ variable "cloudflare_zone_id" {
 }
 
 // Coder
+variable "coder_license" {
+  description = "Coder license key."
+  sensitive   = true
+}
+
 variable "coder_chart_version" {
   description = "Version of the Coder Helm chart to install. Defaults to latest."
   default     = null
@@ -99,12 +92,6 @@ variable "coder_replicas" {
   default     = 1
 }
 
-variable "coder_license" {
-  description = "Coder license key."
-  # sensitive   = true
-  
-}
-
 variable "coder_cpu_request" {
   description = "CPU request to allocate to Coder."
   default     = "500m"

From 152d7003706184c2e00feae7c7714b417c776e02 Mon Sep 17 00:00:00 2001
From: Garrett Delfosse <garrett@coder.com>
Date: Fri, 6 Dec 2024 21:01:13 +0000
Subject: [PATCH 11/24] fmt

---
 scaletest/terraform/new/cf_dns.tf            | 10 ++---
 scaletest/terraform/new/deployments.tf       | 24 ++++++------
 scaletest/terraform/new/gcp_clusters.tf      | 40 ++++++++++----------
 scaletest/terraform/new/k8s_coder_asia.tf    |  2 +-
 scaletest/terraform/new/k8s_coder_europe.tf  |  2 +-
 scaletest/terraform/new/k8s_coder_primary.tf |  2 +-
 scaletest/terraform/new/k8s_coder_proxies.tf | 22 +++++------
 scaletest/terraform/new/main.tf              | 12 +++---
 8 files changed, 57 insertions(+), 57 deletions(-)

diff --git a/scaletest/terraform/new/cf_dns.tf b/scaletest/terraform/new/cf_dns.tf
index 5906741a81c7f..eaaff28ce03a0 100644
--- a/scaletest/terraform/new/cf_dns.tf
+++ b/scaletest/terraform/new/cf_dns.tf
@@ -1,8 +1,8 @@
 resource "cloudflare_record" "coder" {
   for_each = local.deployments
-  zone_id = var.cloudflare_zone_id
-  name    = each.value.subdomain
-  content = google_compute_address.coder[each.key].address
-  type    = "A"
-  ttl     = 3600
+  zone_id  = var.cloudflare_zone_id
+  name     = each.value.subdomain
+  content  = google_compute_address.coder[each.key].address
+  type     = "A"
+  ttl      = 3600
 }
diff --git a/scaletest/terraform/new/deployments.tf b/scaletest/terraform/new/deployments.tf
index 0fed67ab7d2c9..938943a2a8c16 100644
--- a/scaletest/terraform/new/deployments.tf
+++ b/scaletest/terraform/new/deployments.tf
@@ -2,24 +2,24 @@ locals {
   deployments = {
     primary = {
       subdomain = "${var.name}-scaletest"
-      url = "http://${var.name}-scaletest.${var.cloudflare_domain}"
-      region = "us-east1"
-      zone   = "us-east1-c"
-      cidr   = "10.200.0.0/24"
+      url       = "http://${var.name}-scaletest.${var.cloudflare_domain}"
+      region    = "us-east1"
+      zone      = "us-east1-c"
+      cidr      = "10.200.0.0/24"
     }
     europe = {
       subdomain = "${var.name}-europe-scaletest"
-      url = "http://${var.name}-europe-scaletest.${var.cloudflare_domain}"
-      region = "europe-west1"
-      zone   = "europe-west1-b"
-      cidr   = "10.201.0.0/24"
+      url       = "http://${var.name}-europe-scaletest.${var.cloudflare_domain}"
+      region    = "europe-west1"
+      zone      = "europe-west1-b"
+      cidr      = "10.201.0.0/24"
     }
     asia = {
       subdomain = "${var.name}-asia-scaletest"
-      url = "http://${var.name}-asia-scaletest.${var.cloudflare_domain}"
-      region = "asia-southeast1"
-      zone   = "asia-southeast1-a"
-      cidr   = "10.202.0.0/24"
+      url       = "http://${var.name}-asia-scaletest.${var.cloudflare_domain}"
+      region    = "asia-southeast1"
+      zone      = "asia-southeast1-a"
+      cidr      = "10.202.0.0/24"
     }
   }
 }
diff --git a/scaletest/terraform/new/gcp_clusters.tf b/scaletest/terraform/new/gcp_clusters.tf
index 16865c9f5690b..7b8993fc0fd11 100644
--- a/scaletest/terraform/new/gcp_clusters.tf
+++ b/scaletest/terraform/new/gcp_clusters.tf
@@ -1,54 +1,54 @@
 data "google_compute_default_service_account" "default" {
-  project = var.project_id
-  depends_on = [ google_project_service.api["compute.googleapis.com"] ]
+  project    = var.project_id
+  depends_on = [google_project_service.api["compute.googleapis.com"]]
 }
 
 locals {
   node_pools = {
     primary_coder = {
-      name = "coder"
+      name    = "coder"
       cluster = "primary"
-      size = 1
+      size    = 1
     }
     primary_workspaces = {
-      name = "workspaces"
+      name    = "workspaces"
       cluster = "primary"
-      size = 1
+      size    = 1
     }
     primary_misc = {
-      name = "misc"
+      name    = "misc"
       cluster = "primary"
-      size = 1
+      size    = 1
     }
     europe_coder = {
-      name = "coder"
+      name    = "coder"
       cluster = "europe"
-      size = 1
+      size    = 1
     }
     europe_workspaces = {
-      name = "workspaces"
+      name    = "workspaces"
       cluster = "europe"
-      size = 1
+      size    = 1
     }
     europe_misc = {
-      name = "misc"
+      name    = "misc"
       cluster = "europe"
-      size = 1
+      size    = 1
     }
     asia_coder = {
-      name = "coder"
+      name    = "coder"
       cluster = "asia"
-      size = 1
+      size    = 1
     }
     asia_workspaces = {
-      name = "workspaces"
+      name    = "workspaces"
       cluster = "asia"
-      size = 1
+      size    = 1
     }
     asia_misc = {
-      name = "misc"
+      name    = "misc"
       cluster = "asia"
-      size = 1
+      size    = 1
     }
   }
 }
diff --git a/scaletest/terraform/new/k8s_coder_asia.tf b/scaletest/terraform/new/k8s_coder_asia.tf
index 6067621f01d75..bd237416f6e6a 100644
--- a/scaletest/terraform/new/k8s_coder_asia.tf
+++ b/scaletest/terraform/new/k8s_coder_asia.tf
@@ -141,7 +141,7 @@ EOF
 
 resource "helm_release" "provisionerd_asia" {
   provider = helm.asia
-  
+
   repository = local.coder_helm_repo
   chart      = local.provisionerd_helm_chart
   name       = local.provisionerd_release_name
diff --git a/scaletest/terraform/new/k8s_coder_europe.tf b/scaletest/terraform/new/k8s_coder_europe.tf
index 7609f6eb783e5..244ec4e8de6c2 100644
--- a/scaletest/terraform/new/k8s_coder_europe.tf
+++ b/scaletest/terraform/new/k8s_coder_europe.tf
@@ -141,7 +141,7 @@ EOF
 
 resource "helm_release" "provisionerd_europe" {
   provider = helm.europe
-  
+
   repository = local.coder_helm_repo
   chart      = local.provisionerd_helm_chart
   name       = local.provisionerd_release_name
diff --git a/scaletest/terraform/new/k8s_coder_primary.tf b/scaletest/terraform/new/k8s_coder_primary.tf
index c91464c7d8795..a18698b335788 100644
--- a/scaletest/terraform/new/k8s_coder_primary.tf
+++ b/scaletest/terraform/new/k8s_coder_primary.tf
@@ -158,7 +158,7 @@ EOF
 
 resource "helm_release" "provisionerd_chart" {
   provider = helm.primary
-  
+
   repository = local.coder_helm_repo
   chart      = local.provisionerd_helm_chart
   name       = local.provisionerd_release_name
diff --git a/scaletest/terraform/new/k8s_coder_proxies.tf b/scaletest/terraform/new/k8s_coder_proxies.tf
index cc9887957d598..7dfe4ca60da12 100644
--- a/scaletest/terraform/new/k8s_coder_proxies.tf
+++ b/scaletest/terraform/new/k8s_coder_proxies.tf
@@ -2,24 +2,24 @@ data "http" "coder_healthy" {
   url = local.deployments.primary.url
   // Wait up to 5 minutes for DNS to propogate
   retry {
-    attempts = 30
+    attempts     = 30
     min_delay_ms = 10000
   }
 
   lifecycle {
     postcondition {
-        condition = self.status_code == 200
-        error_message = "${self.url} returned an unhealthy status code"
+      condition     = self.status_code == 200
+      error_message = "${self.url} returned an unhealthy status code"
     }
   }
 
-  depends_on = [ helm_release.coder_primary, cloudflare_record.coder["primary"] ]
+  depends_on = [helm_release.coder_primary, cloudflare_record.coder["primary"]]
 }
 
 resource "null_resource" "proxy_tokens" {
   provisioner "local-exec" {
-    interpreter = [ "/bin/bash", "-c" ]
-    command = <<EOF
+    interpreter = ["/bin/bash", "-c"]
+    command     = <<EOF
 curl '${local.deployments.primary.url}/api/v2/users/first' \
   --data-raw $'{"email":"${local.coder_admin_email}","password":"${local.coder_admin_password}","username":"${local.coder_admin_user}","name":"${local.coder_admin_full_name}","trial":false}' \
   --insecure --silent --output /dev/null
@@ -49,15 +49,15 @@ echo -n $${asia_token} > ${path.module}/.coderv2/asia_proxy_token
 EOF
   }
 
-  depends_on = [ data.http.coder_healthy ]
+  depends_on = [data.http.coder_healthy]
 }
 
 data "local_file" "europe_proxy_token" {
-  filename = "${path.module}/.coderv2/europe_proxy_token"
-  depends_on = [ null_resource.proxy_tokens ]
+  filename   = "${path.module}/.coderv2/europe_proxy_token"
+  depends_on = [null_resource.proxy_tokens]
 }
 
 data "local_file" "asia_proxy_token" {
-  filename = "${path.module}/.coderv2/asia_proxy_token"
-  depends_on = [ null_resource.proxy_tokens ]
+  filename   = "${path.module}/.coderv2/asia_proxy_token"
+  depends_on = [null_resource.proxy_tokens]
 }
diff --git a/scaletest/terraform/new/main.tf b/scaletest/terraform/new/main.tf
index 8124e1fb0d854..57a294710c5b5 100644
--- a/scaletest/terraform/new/main.tf
+++ b/scaletest/terraform/new/main.tf
@@ -51,28 +51,28 @@ provider "cloudflare" {
 }
 
 provider "kubernetes" {
-  alias = "primary"
+  alias                  = "primary"
   host                   = "https://${google_container_cluster.cluster["primary"].endpoint}"
   cluster_ca_certificate = base64decode(google_container_cluster.cluster["primary"].master_auth.0.cluster_ca_certificate)
   token                  = data.google_client_config.default.access_token
 }
 
 provider "kubernetes" {
-  alias = "europe"
+  alias                  = "europe"
   host                   = "https://${google_container_cluster.cluster["europe"].endpoint}"
   cluster_ca_certificate = base64decode(google_container_cluster.cluster["europe"].master_auth.0.cluster_ca_certificate)
   token                  = data.google_client_config.default.access_token
 }
 
 provider "kubernetes" {
-  alias = "asia"
+  alias                  = "asia"
   host                   = "https://${google_container_cluster.cluster["asia"].endpoint}"
   cluster_ca_certificate = base64decode(google_container_cluster.cluster["asia"].master_auth.0.cluster_ca_certificate)
   token                  = data.google_client_config.default.access_token
 }
 
 provider "kubectl" {
-  alias = "primary"
+  alias                  = "primary"
   host                   = "https://${google_container_cluster.cluster["primary"].endpoint}"
   cluster_ca_certificate = base64decode(google_container_cluster.cluster["primary"].master_auth.0.cluster_ca_certificate)
   token                  = data.google_client_config.default.access_token
@@ -80,7 +80,7 @@ provider "kubectl" {
 }
 
 provider "kubectl" {
-  alias = "europe"
+  alias                  = "europe"
   host                   = "https://${google_container_cluster.cluster["europe"].endpoint}"
   cluster_ca_certificate = base64decode(google_container_cluster.cluster["europe"].master_auth.0.cluster_ca_certificate)
   token                  = data.google_client_config.default.access_token
@@ -88,7 +88,7 @@ provider "kubectl" {
 }
 
 provider "kubectl" {
-  alias = "asia"
+  alias                  = "asia"
   host                   = "https://${google_container_cluster.cluster["asia"].endpoint}"
   cluster_ca_certificate = base64decode(google_container_cluster.cluster["asia"].master_auth.0.cluster_ca_certificate)
   token                  = data.google_client_config.default.access_token

From a0da296239ff551a13e564c84df4514618d79f9c Mon Sep 17 00:00:00 2001
From: Garrett Delfosse <garrett@coder.com>
Date: Fri, 6 Dec 2024 21:06:14 +0000
Subject: [PATCH 12/24] rename

---
 scaletest/terraform/{new => action}/cf_dns.tf            | 0
 scaletest/terraform/{new => action}/deployments.tf       | 0
 scaletest/terraform/{new => action}/gcp_clusters.tf      | 0
 scaletest/terraform/{new => action}/gcp_db.tf            | 0
 scaletest/terraform/{new => action}/gcp_project.tf       | 0
 scaletest/terraform/{new => action}/gcp_vpc.tf           | 0
 scaletest/terraform/{new => action}/k8s_coder_asia.tf    | 0
 scaletest/terraform/{new => action}/k8s_coder_europe.tf  | 0
 scaletest/terraform/{new => action}/k8s_coder_primary.tf | 0
 scaletest/terraform/{new => action}/k8s_coder_proxies.tf | 0
 scaletest/terraform/{new => action}/main.tf              | 0
 scaletest/terraform/{new => action}/vars.tf              | 0
 12 files changed, 0 insertions(+), 0 deletions(-)
 rename scaletest/terraform/{new => action}/cf_dns.tf (100%)
 rename scaletest/terraform/{new => action}/deployments.tf (100%)
 rename scaletest/terraform/{new => action}/gcp_clusters.tf (100%)
 rename scaletest/terraform/{new => action}/gcp_db.tf (100%)
 rename scaletest/terraform/{new => action}/gcp_project.tf (100%)
 rename scaletest/terraform/{new => action}/gcp_vpc.tf (100%)
 rename scaletest/terraform/{new => action}/k8s_coder_asia.tf (100%)
 rename scaletest/terraform/{new => action}/k8s_coder_europe.tf (100%)
 rename scaletest/terraform/{new => action}/k8s_coder_primary.tf (100%)
 rename scaletest/terraform/{new => action}/k8s_coder_proxies.tf (100%)
 rename scaletest/terraform/{new => action}/main.tf (100%)
 rename scaletest/terraform/{new => action}/vars.tf (100%)

diff --git a/scaletest/terraform/new/cf_dns.tf b/scaletest/terraform/action/cf_dns.tf
similarity index 100%
rename from scaletest/terraform/new/cf_dns.tf
rename to scaletest/terraform/action/cf_dns.tf
diff --git a/scaletest/terraform/new/deployments.tf b/scaletest/terraform/action/deployments.tf
similarity index 100%
rename from scaletest/terraform/new/deployments.tf
rename to scaletest/terraform/action/deployments.tf
diff --git a/scaletest/terraform/new/gcp_clusters.tf b/scaletest/terraform/action/gcp_clusters.tf
similarity index 100%
rename from scaletest/terraform/new/gcp_clusters.tf
rename to scaletest/terraform/action/gcp_clusters.tf
diff --git a/scaletest/terraform/new/gcp_db.tf b/scaletest/terraform/action/gcp_db.tf
similarity index 100%
rename from scaletest/terraform/new/gcp_db.tf
rename to scaletest/terraform/action/gcp_db.tf
diff --git a/scaletest/terraform/new/gcp_project.tf b/scaletest/terraform/action/gcp_project.tf
similarity index 100%
rename from scaletest/terraform/new/gcp_project.tf
rename to scaletest/terraform/action/gcp_project.tf
diff --git a/scaletest/terraform/new/gcp_vpc.tf b/scaletest/terraform/action/gcp_vpc.tf
similarity index 100%
rename from scaletest/terraform/new/gcp_vpc.tf
rename to scaletest/terraform/action/gcp_vpc.tf
diff --git a/scaletest/terraform/new/k8s_coder_asia.tf b/scaletest/terraform/action/k8s_coder_asia.tf
similarity index 100%
rename from scaletest/terraform/new/k8s_coder_asia.tf
rename to scaletest/terraform/action/k8s_coder_asia.tf
diff --git a/scaletest/terraform/new/k8s_coder_europe.tf b/scaletest/terraform/action/k8s_coder_europe.tf
similarity index 100%
rename from scaletest/terraform/new/k8s_coder_europe.tf
rename to scaletest/terraform/action/k8s_coder_europe.tf
diff --git a/scaletest/terraform/new/k8s_coder_primary.tf b/scaletest/terraform/action/k8s_coder_primary.tf
similarity index 100%
rename from scaletest/terraform/new/k8s_coder_primary.tf
rename to scaletest/terraform/action/k8s_coder_primary.tf
diff --git a/scaletest/terraform/new/k8s_coder_proxies.tf b/scaletest/terraform/action/k8s_coder_proxies.tf
similarity index 100%
rename from scaletest/terraform/new/k8s_coder_proxies.tf
rename to scaletest/terraform/action/k8s_coder_proxies.tf
diff --git a/scaletest/terraform/new/main.tf b/scaletest/terraform/action/main.tf
similarity index 100%
rename from scaletest/terraform/new/main.tf
rename to scaletest/terraform/action/main.tf
diff --git a/scaletest/terraform/new/vars.tf b/scaletest/terraform/action/vars.tf
similarity index 100%
rename from scaletest/terraform/new/vars.tf
rename to scaletest/terraform/action/vars.tf

From 186d3163634e61a853280f05a47f02648a464249 Mon Sep 17 00:00:00 2001
From: Garrett Delfosse <garrett@coder.com>
Date: Fri, 6 Dec 2024 21:11:39 +0000
Subject: [PATCH 13/24] typo

---
 scaletest/terraform/action/k8s_coder_proxies.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scaletest/terraform/action/k8s_coder_proxies.tf b/scaletest/terraform/action/k8s_coder_proxies.tf
index 7dfe4ca60da12..1a7f8f5ca71dc 100644
--- a/scaletest/terraform/action/k8s_coder_proxies.tf
+++ b/scaletest/terraform/action/k8s_coder_proxies.tf
@@ -1,6 +1,6 @@
 data "http" "coder_healthy" {
   url = local.deployments.primary.url
-  // Wait up to 5 minutes for DNS to propogate
+  // Wait up to 5 minutes for DNS to propagate
   retry {
     attempts     = 30
     min_delay_ms = 10000

From 2751240f8a41f4f5698e1e93e4872e2e3249e738 Mon Sep 17 00:00:00 2001
From: Garrett Delfosse <garrett@coder.com>
Date: Mon, 9 Dec 2024 17:14:39 +0000
Subject: [PATCH 14/24] scenarios

---
 ...{k8s_coder_proxies.tf => coder_proxies.tf} |   0
 scaletest/terraform/action/deployments.tf     |  33 +++++
 scaletest/terraform/action/gcp_clusters.tf    |  21 +---
 scaletest/terraform/action/gcp_db.tf          |   6 +-
 scaletest/terraform/action/k8s_coder_asia.tf  |  20 +--
 .../terraform/action/k8s_coder_europe.tf      |  20 +--
 .../terraform/action/k8s_coder_primary.tf     |  20 +--
 scaletest/terraform/action/vars.tf            | 114 ++----------------
 8 files changed, 78 insertions(+), 156 deletions(-)
 rename scaletest/terraform/action/{k8s_coder_proxies.tf => coder_proxies.tf} (100%)

diff --git a/scaletest/terraform/action/k8s_coder_proxies.tf b/scaletest/terraform/action/coder_proxies.tf
similarity index 100%
rename from scaletest/terraform/action/k8s_coder_proxies.tf
rename to scaletest/terraform/action/coder_proxies.tf
diff --git a/scaletest/terraform/action/deployments.tf b/scaletest/terraform/action/deployments.tf
index 938943a2a8c16..d8237f858d374 100644
--- a/scaletest/terraform/action/deployments.tf
+++ b/scaletest/terraform/action/deployments.tf
@@ -22,4 +22,37 @@ locals {
       cidr      = "10.202.0.0/24"
     }
   }
+
+  scenarios = {
+    small = {
+      coder = {
+        nodepool_size = 1
+        machine_type = "t2d-standard-4"
+        replicas = 1
+        cpu_request = "1000m"
+        mem_request = "6Gi"
+        cpu_limit = "2000m"
+        mem_limit = "12Gi"
+      }
+      provisionerd = {
+        replicas = 1
+        cpu_request = "100m"
+        mem_request = "1Gi"
+        cpu_limit = "1000m"
+        mem_limit = "1Gi"
+      }
+      workspaces = {
+        nodepool_size = 1
+        machine_type = "t2d-standard-4"
+        cpu_request = "100m"
+        mem_request = "128Mi"
+        cpu_limit = "100m"
+        mem_limit = "128Mi"
+      }
+      cloudsql = {
+        tier = "db-f1-micro"
+        replicas = 1
+      }
+    }
+  }
 }
diff --git a/scaletest/terraform/action/gcp_clusters.tf b/scaletest/terraform/action/gcp_clusters.tf
index 7b8993fc0fd11..b16f289dca2a9 100644
--- a/scaletest/terraform/action/gcp_clusters.tf
+++ b/scaletest/terraform/action/gcp_clusters.tf
@@ -8,47 +8,38 @@ locals {
     primary_coder = {
       name    = "coder"
       cluster = "primary"
-      size    = 1
     }
     primary_workspaces = {
       name    = "workspaces"
       cluster = "primary"
-      size    = 1
     }
     primary_misc = {
       name    = "misc"
       cluster = "primary"
-      size    = 1
     }
     europe_coder = {
       name    = "coder"
       cluster = "europe"
-      size    = 1
     }
     europe_workspaces = {
       name    = "workspaces"
       cluster = "europe"
-      size    = 1
     }
     europe_misc = {
       name    = "misc"
       cluster = "europe"
-      size    = 1
     }
     asia_coder = {
       name    = "coder"
       cluster = "asia"
-      size    = 1
     }
     asia_workspaces = {
       name    = "workspaces"
       cluster = "asia"
-      size    = 1
     }
     asia_misc = {
       name    = "misc"
       cluster = "asia"
-      size    = 1
     }
   }
 }
@@ -104,10 +95,7 @@ resource "google_container_node_pool" "node_pool" {
   location = local.deployments[each.value.cluster].zone
   project  = var.project_id
   cluster  = google_container_cluster.cluster[each.value.cluster].name
-  autoscaling {
-    min_node_count = 1
-    max_node_count = each.value.size
-  }
+  node_count = local.scenarios[var.scenario][each.value.name].nodepool_size
   node_config {
     oauth_scopes = [
       "https://www.googleapis.com/auth/logging.write",
@@ -117,10 +105,9 @@ resource "google_container_node_pool" "node_pool" {
       "https://www.googleapis.com/auth/service.management.readonly",
       "https://www.googleapis.com/auth/servicecontrol",
     ]
-    disk_size_gb    = var.node_disk_size_gb
-    machine_type    = var.nodepool_machine_type_coder
-    image_type      = var.node_image_type
-    preemptible     = var.node_preemptible
+    disk_size_gb    = 100
+    machine_type    = local.scenarios[var.scenario][each.value.name].machine_type
+    image_type      = "cos_containerd"
     service_account = data.google_compute_default_service_account.default.email
     tags            = ["gke-node", "${var.project_id}-gke"]
     labels = {
diff --git a/scaletest/terraform/action/gcp_db.tf b/scaletest/terraform/action/gcp_db.tf
index aa5dc1c5b923a..0443fa771fe65 100644
--- a/scaletest/terraform/action/gcp_db.tf
+++ b/scaletest/terraform/action/gcp_db.tf
@@ -2,13 +2,13 @@ resource "google_sql_database_instance" "db" {
   name                = "${var.name}-coder"
   project             = var.project_id
   region              = local.deployments.primary.region
-  database_version    = var.cloudsql_version
+  database_version    = "POSTGRES_14"
   deletion_protection = false
 
   depends_on = [google_service_networking_connection.private_vpc_connection]
 
   settings {
-    tier              = var.cloudsql_tier
+    tier              = local.scenarios[var.scenario].cloudsql.tier
     activation_policy = "ALWAYS"
     availability_type = "ZONAL"
 
@@ -18,7 +18,7 @@ resource "google_sql_database_instance" "db" {
 
     database_flags {
       name  = "max_connections"
-      value = var.cloudsql_max_connections
+      value = local.scenarios[var.scenario].cloudsql.max_connections
     }
 
     ip_configuration {
diff --git a/scaletest/terraform/action/k8s_coder_asia.tf b/scaletest/terraform/action/k8s_coder_asia.tf
index bd237416f6e6a..0626326cc7fbe 100644
--- a/scaletest/terraform/action/k8s_coder_asia.tf
+++ b/scaletest/terraform/action/k8s_coder_asia.tf
@@ -113,14 +113,14 @@ coder:
   image:
     repo: ${var.coder_image_repo}
     tag: ${var.coder_image_tag}
-  replicaCount: "${var.coder_replicas}"
+  replicaCount: "${local.scenarios[var.scenario].coder.replicas}"
   resources:
     requests:
-      cpu: "${var.coder_cpu_request}"
-      memory: "${var.coder_mem_request}"
+      cpu: "${local.scenarios[var.scenario].coder.cpu_request}"
+      memory: "${local.scenarios[var.scenario].coder.mem_request}"
     limits:
-      cpu: "${var.coder_cpu_limit}"
-      memory: "${var.coder_mem_limit}"
+      cpu: "${local.scenarios[var.scenario].coder.cpu_limit}"
+      memory: "${local.scenarios[var.scenario].coder.mem_limit}"
   securityContext:
     readOnlyRootFilesystem: true
   service:
@@ -189,14 +189,14 @@ coder:
   image:
     repo: ${var.provisionerd_image_repo}
     tag: ${var.provisionerd_image_tag}
-  replicaCount: "${var.provisionerd_replicas}"
+  replicaCount: "${local.scenarios[var.scenario].provisionerd.replicas}"
   resources:
     requests:
-      cpu: "${var.provisionerd_cpu_request}"
-      memory: "${var.provisionerd_mem_request}"
+      cpu: "${local.scenarios[var.scenario].provisionerd.request}"
+      memory: "${local.scenarios[var.scenario].provisionerd.mem_request}"
     limits:
-      cpu: "${var.provisionerd_cpu_limit}"
-      memory: "${var.provisionerd_mem_limit}"
+      cpu: "${local.scenarios[var.scenario].provisionerd.cpu_limit}"
+      memory: "${local.scenarios[var.scenario].provisionerd.mem_limit}"
   securityContext:
     readOnlyRootFilesystem: true
   volumeMounts:
diff --git a/scaletest/terraform/action/k8s_coder_europe.tf b/scaletest/terraform/action/k8s_coder_europe.tf
index 244ec4e8de6c2..8e2691d90424d 100644
--- a/scaletest/terraform/action/k8s_coder_europe.tf
+++ b/scaletest/terraform/action/k8s_coder_europe.tf
@@ -113,14 +113,14 @@ coder:
   image:
     repo: ${var.coder_image_repo}
     tag: ${var.coder_image_tag}
-  replicaCount: "${var.coder_replicas}"
+  replicaCount: "${local.scenarios[var.scenario].coder.replicas}"
   resources:
     requests:
-      cpu: "${var.coder_cpu_request}"
-      memory: "${var.coder_mem_request}"
+      cpu: "${local.scenarios[var.scenario].coder.cpu_request}"
+      memory: "${local.scenarios[var.scenario].coder.mem_request}"
     limits:
-      cpu: "${var.coder_cpu_limit}"
-      memory: "${var.coder_mem_limit}"
+      cpu: "${local.scenarios[var.scenario].coder.cpu_limit}"
+      memory: "${local.scenarios[var.scenario].coder.mem_limit}"
   securityContext:
     readOnlyRootFilesystem: true
   service:
@@ -189,14 +189,14 @@ coder:
   image:
     repo: ${var.provisionerd_image_repo}
     tag: ${var.provisionerd_image_tag}
-  replicaCount: "${var.provisionerd_replicas}"
+  replicaCount: "${local.scenarios[var.scenario].provisionerd.replicas}"
   resources:
     requests:
-      cpu: "${var.provisionerd_cpu_request}"
-      memory: "${var.provisionerd_mem_request}"
+      cpu: "${local.scenarios[var.scenario].provisionerd.request}"
+      memory: "${local.scenarios[var.scenario].provisionerd.mem_request}"
     limits:
-      cpu: "${var.provisionerd_cpu_limit}"
-      memory: "${var.provisionerd_mem_limit}"
+      cpu: "${local.scenarios[var.scenario].provisionerd.cpu_limit}"
+      memory: "${local.scenarios[var.scenario].provisionerd.mem_limit}"
   securityContext:
     readOnlyRootFilesystem: true
   volumeMounts:
diff --git a/scaletest/terraform/action/k8s_coder_primary.tf b/scaletest/terraform/action/k8s_coder_primary.tf
index a18698b335788..6175421db69b9 100644
--- a/scaletest/terraform/action/k8s_coder_primary.tf
+++ b/scaletest/terraform/action/k8s_coder_primary.tf
@@ -130,14 +130,14 @@ coder:
   image:
     repo: ${var.coder_image_repo}
     tag: ${var.coder_image_tag}
-  replicaCount: "${var.coder_replicas}"
+  replicaCount: "${local.scenarios[var.scenario].coder.replicas}"
   resources:
     requests:
-      cpu: "${var.coder_cpu_request}"
-      memory: "${var.coder_mem_request}"
+      cpu: "${local.scenarios[var.scenario].coder.cpu_request}"
+      memory: "${local.scenarios[var.scenario].coder.mem_request}"
     limits:
-      cpu: "${var.coder_cpu_limit}"
-      memory: "${var.coder_mem_limit}"
+      cpu: "${local.scenarios[var.scenario].coder.cpu_limit}"
+      memory: "${local.scenarios[var.scenario].coder.mem_limit}"
   securityContext:
     readOnlyRootFilesystem: true
   service:
@@ -206,14 +206,14 @@ coder:
   image:
     repo: ${var.provisionerd_image_repo}
     tag: ${var.provisionerd_image_tag}
-  replicaCount: "${var.provisionerd_replicas}"
+  replicaCount: "${local.scenarios[var.scenario].provisionerd.replicas}"
   resources:
     requests:
-      cpu: "${var.provisionerd_cpu_request}"
-      memory: "${var.provisionerd_mem_request}"
+      cpu: "${local.scenarios[var.scenario].provisionerd.request}"
+      memory: "${local.scenarios[var.scenario].provisionerd.mem_request}"
     limits:
-      cpu: "${var.provisionerd_cpu_limit}"
-      memory: "${var.provisionerd_mem_limit}"
+      cpu: "${local.scenarios[var.scenario].provisionerd.cpu_limit}"
+      memory: "${local.scenarios[var.scenario].provisionerd.mem_limit}"
   securityContext:
     readOnlyRootFilesystem: true
   volumeMounts:
diff --git a/scaletest/terraform/action/vars.tf b/scaletest/terraform/action/vars.tf
index dc6de957b3c4d..d1ba508d06587 100644
--- a/scaletest/terraform/action/vars.tf
+++ b/scaletest/terraform/action/vars.tf
@@ -2,6 +2,14 @@ variable "name" {
   description = "The name all resources will be prefixed with"
 }
 
+variable "scenario" {
+  description = "The scenario to deploy"
+  validation {
+    condition = contains(["small", "medium", "large"], var.scenario)
+    error_message = "Scenario must be one of small, medium, or large"
+  }
+}
+
 // GCP
 variable "project_id" {
   description = "The project in which to provision resources"
@@ -12,41 +20,6 @@ variable "k8s_version" {
   default     = "1.24"
 }
 
-variable "node_disk_size_gb" {
-  description = "Size of the root disk for cluster nodes."
-  default     = 100
-}
-
-variable "node_image_type" {
-  description = "Image type to use for cluster nodes."
-  default     = "cos_containerd"
-}
-
-variable "node_preemptible" {
-  description = "Use preemptible nodes."
-  default     = false
-}
-
-variable "nodepool_machine_type_coder" {
-  description = "Machine type to use for Coder control plane nodepool."
-  default     = "t2d-standard-4"
-}
-
-variable "cloudsql_version" {
-  description = "CloudSQL version to provision"
-  default     = "POSTGRES_14"
-}
-
-variable "cloudsql_tier" {
-  description = "CloudSQL database tier."
-  default     = "db-f1-micro"
-}
-
-variable "cloudsql_max_connections" {
-  description = "CloudSQL database max_connections"
-  default     = 500
-}
-
 // Cloudflare
 variable "cloudflare_api_token" {
   description = "Cloudflare API token."
@@ -87,31 +60,6 @@ variable "coder_image_repo" {
   default     = "ghcr.io/coder/coder"
 }
 
-variable "coder_replicas" {
-  description = "Number of Coder replicas to provision."
-  default     = 1
-}
-
-variable "coder_cpu_request" {
-  description = "CPU request to allocate to Coder."
-  default     = "500m"
-}
-
-variable "coder_mem_request" {
-  description = "Memory request to allocate to Coder."
-  default     = "512Mi"
-}
-
-variable "coder_cpu_limit" {
-  description = "CPU limit to allocate to Coder."
-  default     = "1000m"
-}
-
-variable "coder_mem_limit" {
-  description = "Memory limit to allocate to Coder."
-  default     = "1024Mi"
-}
-
 variable "coder_experiments" {
   description = "Coder Experiments to enable."
   default     = ""
@@ -123,52 +71,6 @@ variable "workspace_image" {
   default     = "docker.io/codercom/enterprise-minimal:ubuntu"
 }
 
-variable "workspace_cpu_request" {
-  description = "CPU request to allocate to workspaces."
-  default     = "100m"
-}
-
-variable "workspace_cpu_limit" {
-  description = "CPU limit to allocate to workspaces."
-  default     = "100m"
-}
-
-variable "workspace_mem_request" {
-  description = "Memory request to allocate to workspaces."
-  default     = "128Mi"
-}
-
-variable "workspace_mem_limit" {
-  description = "Memory limit to allocate to workspaces."
-  default     = "128Mi"
-}
-
-// Provisioners
-variable "provisionerd_cpu_request" {
-  description = "CPU request to allocate to provisionerd."
-  default     = "100m"
-}
-
-variable "provisionerd_mem_request" {
-  description = "Memory request to allocate to provisionerd."
-  default     = "1Gi"
-}
-
-variable "provisionerd_cpu_limit" {
-  description = "CPU limit to allocate to provisionerd."
-  default     = "1000m"
-}
-
-variable "provisionerd_mem_limit" {
-  description = "Memory limit to allocate to provisionerd."
-  default     = "1Gi"
-}
-
-variable "provisionerd_replicas" {
-  description = "Number of Provisionerd replicas."
-  default     = 1
-}
-
 variable "provisionerd_chart_version" {
   description = "Version of the Provisionerd Helm chart to install. Defaults to latest."
   default     = null

From 1edb0fe980c90da5ff9323669688b54047f55fa5 Mon Sep 17 00:00:00 2001
From: Garrett Delfosse <garrett@coder.com>
Date: Mon, 9 Dec 2024 17:51:35 +0000
Subject: [PATCH 15/24] emojis

---
 scaletest/terraform/action/coder_proxies.tf     | 4 ++--
 scaletest/terraform/action/deployments.tf       | 6 +++++-
 scaletest/terraform/action/k8s_coder_asia.tf    | 2 +-
 scaletest/terraform/action/k8s_coder_europe.tf  | 2 +-
 scaletest/terraform/action/k8s_coder_primary.tf | 2 +-
 5 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/scaletest/terraform/action/coder_proxies.tf b/scaletest/terraform/action/coder_proxies.tf
index 1a7f8f5ca71dc..dc1c4e9e3ecaa 100644
--- a/scaletest/terraform/action/coder_proxies.tf
+++ b/scaletest/terraform/action/coder_proxies.tf
@@ -35,12 +35,12 @@ curl '${local.deployments.primary.url}/api/v2/licenses' \
 
 europe_token=$(curl '${local.deployments.primary.url}/api/v2/workspaceproxies' \
   -H "Coder-Session-Token: $${token}" \
-  --data-raw '{"name":"europe"}' \
+  --data-raw '{"name":"europe","display_name":"Europe","icon":"/emojis/1f35b.png"}' \
   --insecure --silent | jq -r .proxy_token)
 
 asia_token=$(curl '${local.deployments.primary.url}/api/v2/workspaceproxies' \
   -H "Coder-Session-Token: $${token}" \
-  --data-raw '{"name":"asia"}' \
+  --data-raw '{"name":"asia","display_name":"Asia","icon":"/emojis/1f950.png"}' \
   --insecure --silent | jq -r .proxy_token)
 
 mkdir -p ${path.module}/.coderv2
diff --git a/scaletest/terraform/action/deployments.tf b/scaletest/terraform/action/deployments.tf
index d8237f858d374..8b18e285ee28d 100644
--- a/scaletest/terraform/action/deployments.tf
+++ b/scaletest/terraform/action/deployments.tf
@@ -49,9 +49,13 @@ locals {
         cpu_limit = "100m"
         mem_limit = "128Mi"
       }
+      misc = {
+        nodepool_size = 1
+        machine_type = "t2d-standard-4"
+      }
       cloudsql = {
         tier = "db-f1-micro"
-        replicas = 1
+        max_connections = 500
       }
     }
   }
diff --git a/scaletest/terraform/action/k8s_coder_asia.tf b/scaletest/terraform/action/k8s_coder_asia.tf
index 0626326cc7fbe..f8f8b62180a80 100644
--- a/scaletest/terraform/action/k8s_coder_asia.tf
+++ b/scaletest/terraform/action/k8s_coder_asia.tf
@@ -192,7 +192,7 @@ coder:
   replicaCount: "${local.scenarios[var.scenario].provisionerd.replicas}"
   resources:
     requests:
-      cpu: "${local.scenarios[var.scenario].provisionerd.request}"
+      cpu: "${local.scenarios[var.scenario].provisionerd.cpu_request}"
       memory: "${local.scenarios[var.scenario].provisionerd.mem_request}"
     limits:
       cpu: "${local.scenarios[var.scenario].provisionerd.cpu_limit}"
diff --git a/scaletest/terraform/action/k8s_coder_europe.tf b/scaletest/terraform/action/k8s_coder_europe.tf
index 8e2691d90424d..32c2c851265fb 100644
--- a/scaletest/terraform/action/k8s_coder_europe.tf
+++ b/scaletest/terraform/action/k8s_coder_europe.tf
@@ -192,7 +192,7 @@ coder:
   replicaCount: "${local.scenarios[var.scenario].provisionerd.replicas}"
   resources:
     requests:
-      cpu: "${local.scenarios[var.scenario].provisionerd.request}"
+      cpu: "${local.scenarios[var.scenario].provisionerd.cpu_request}"
       memory: "${local.scenarios[var.scenario].provisionerd.mem_request}"
     limits:
       cpu: "${local.scenarios[var.scenario].provisionerd.cpu_limit}"
diff --git a/scaletest/terraform/action/k8s_coder_primary.tf b/scaletest/terraform/action/k8s_coder_primary.tf
index 6175421db69b9..66874a5a5eb08 100644
--- a/scaletest/terraform/action/k8s_coder_primary.tf
+++ b/scaletest/terraform/action/k8s_coder_primary.tf
@@ -209,7 +209,7 @@ coder:
   replicaCount: "${local.scenarios[var.scenario].provisionerd.replicas}"
   resources:
     requests:
-      cpu: "${local.scenarios[var.scenario].provisionerd.request}"
+      cpu: "${local.scenarios[var.scenario].provisionerd.cpu_request}"
       memory: "${local.scenarios[var.scenario].provisionerd.mem_request}"
     limits:
       cpu: "${local.scenarios[var.scenario].provisionerd.cpu_limit}"

From bbad08e0d43365c27d9a0734f33961c2080f502c Mon Sep 17 00:00:00 2001
From: Garrett Delfosse <garrett@coder.com>
Date: Mon, 9 Dec 2024 18:19:52 +0000
Subject: [PATCH 16/24] fmt

---
 scaletest/terraform/action/deployments.tf     | 32 +++++++++----------
 scaletest/terraform/action/gcp_clusters.tf    | 10 +++---
 scaletest/terraform/action/k8s_coder_asia.tf  |  2 ++
 .../terraform/action/k8s_coder_europe.tf      |  2 ++
 .../terraform/action/k8s_coder_primary.tf     |  2 ++
 scaletest/terraform/action/vars.tf            |  2 +-
 6 files changed, 28 insertions(+), 22 deletions(-)

diff --git a/scaletest/terraform/action/deployments.tf b/scaletest/terraform/action/deployments.tf
index 8b18e285ee28d..bd46b2629208d 100644
--- a/scaletest/terraform/action/deployments.tf
+++ b/scaletest/terraform/action/deployments.tf
@@ -27,34 +27,34 @@ locals {
     small = {
       coder = {
         nodepool_size = 1
-        machine_type = "t2d-standard-4"
-        replicas = 1
-        cpu_request = "1000m"
-        mem_request = "6Gi"
-        cpu_limit = "2000m"
-        mem_limit = "12Gi"
+        machine_type  = "t2d-standard-4"
+        replicas      = 1
+        cpu_request   = "1000m"
+        mem_request   = "6Gi"
+        cpu_limit     = "2000m"
+        mem_limit     = "12Gi"
       }
       provisionerd = {
-        replicas = 1
+        replicas    = 1
         cpu_request = "100m"
         mem_request = "1Gi"
-        cpu_limit = "1000m"
-        mem_limit = "1Gi"
+        cpu_limit   = "1000m"
+        mem_limit   = "1Gi"
       }
       workspaces = {
         nodepool_size = 1
-        machine_type = "t2d-standard-4"
-        cpu_request = "100m"
-        mem_request = "128Mi"
-        cpu_limit = "100m"
-        mem_limit = "128Mi"
+        machine_type  = "t2d-standard-4"
+        cpu_request   = "100m"
+        mem_request   = "128Mi"
+        cpu_limit     = "100m"
+        mem_limit     = "128Mi"
       }
       misc = {
         nodepool_size = 1
-        machine_type = "t2d-standard-4"
+        machine_type  = "t2d-standard-4"
       }
       cloudsql = {
-        tier = "db-f1-micro"
+        tier            = "db-f1-micro"
         max_connections = 500
       }
     }
diff --git a/scaletest/terraform/action/gcp_clusters.tf b/scaletest/terraform/action/gcp_clusters.tf
index b16f289dca2a9..e2e360c65043c 100644
--- a/scaletest/terraform/action/gcp_clusters.tf
+++ b/scaletest/terraform/action/gcp_clusters.tf
@@ -90,11 +90,11 @@ resource "google_container_cluster" "cluster" {
 }
 
 resource "google_container_node_pool" "node_pool" {
-  for_each = local.node_pools
-  name     = each.value.name
-  location = local.deployments[each.value.cluster].zone
-  project  = var.project_id
-  cluster  = google_container_cluster.cluster[each.value.cluster].name
+  for_each   = local.node_pools
+  name       = each.value.name
+  location   = local.deployments[each.value.cluster].zone
+  project    = var.project_id
+  cluster    = google_container_cluster.cluster[each.value.cluster].name
   node_count = local.scenarios[var.scenario][each.value.name].nodepool_size
   node_config {
     oauth_scopes = [
diff --git a/scaletest/terraform/action/k8s_coder_asia.tf b/scaletest/terraform/action/k8s_coder_asia.tf
index f8f8b62180a80..f2055632c89ca 100644
--- a/scaletest/terraform/action/k8s_coder_asia.tf
+++ b/scaletest/terraform/action/k8s_coder_asia.tf
@@ -7,6 +7,8 @@ resource "kubernetes_namespace" "coder_asia" {
   lifecycle {
     ignore_changes = [timeouts, wait_for_default_service_account]
   }
+
+  depends_on = [google_container_node_pool.node_pool["asia_misc"]]
 }
 
 resource "kubernetes_secret" "provisionerd_psk_asia" {
diff --git a/scaletest/terraform/action/k8s_coder_europe.tf b/scaletest/terraform/action/k8s_coder_europe.tf
index 32c2c851265fb..84c0d30d949b8 100644
--- a/scaletest/terraform/action/k8s_coder_europe.tf
+++ b/scaletest/terraform/action/k8s_coder_europe.tf
@@ -7,6 +7,8 @@ resource "kubernetes_namespace" "coder_europe" {
   lifecycle {
     ignore_changes = [timeouts, wait_for_default_service_account]
   }
+
+  depends_on = [google_container_node_pool.node_pool["europe_misc"]]
 }
 
 resource "kubernetes_secret" "provisionerd_psk_europe" {
diff --git a/scaletest/terraform/action/k8s_coder_primary.tf b/scaletest/terraform/action/k8s_coder_primary.tf
index 66874a5a5eb08..6e8eaa9db7b66 100644
--- a/scaletest/terraform/action/k8s_coder_primary.tf
+++ b/scaletest/terraform/action/k8s_coder_primary.tf
@@ -27,6 +27,8 @@ resource "kubernetes_namespace" "coder_primary" {
   lifecycle {
     ignore_changes = [timeouts, wait_for_default_service_account]
   }
+
+  depends_on = [google_container_node_pool.node_pool["primary_misc"]]
 }
 
 resource "kubernetes_secret" "coder_db" {
diff --git a/scaletest/terraform/action/vars.tf b/scaletest/terraform/action/vars.tf
index d1ba508d06587..264110e239845 100644
--- a/scaletest/terraform/action/vars.tf
+++ b/scaletest/terraform/action/vars.tf
@@ -5,7 +5,7 @@ variable "name" {
 variable "scenario" {
   description = "The scenario to deploy"
   validation {
-    condition = contains(["small", "medium", "large"], var.scenario)
+    condition     = contains(["small", "medium", "large"], var.scenario)
     error_message = "Scenario must be one of small, medium, or large"
   }
 }

From 98935253589f544436f0cc3e72fdb730ec184f47 Mon Sep 17 00:00:00 2001
From: Garrett Delfosse <garrett@coder.com>
Date: Mon, 9 Dec 2024 18:24:21 +0000
Subject: [PATCH 17/24] fmt:

---
 scaletest/terraform/action/deployments.tf |  37 --------
 scaletest/terraform/action/scenarios.tf   | 106 ++++++++++++++++++++++
 2 files changed, 106 insertions(+), 37 deletions(-)
 create mode 100644 scaletest/terraform/action/scenarios.tf

diff --git a/scaletest/terraform/action/deployments.tf b/scaletest/terraform/action/deployments.tf
index bd46b2629208d..938943a2a8c16 100644
--- a/scaletest/terraform/action/deployments.tf
+++ b/scaletest/terraform/action/deployments.tf
@@ -22,41 +22,4 @@ locals {
       cidr      = "10.202.0.0/24"
     }
   }
-
-  scenarios = {
-    small = {
-      coder = {
-        nodepool_size = 1
-        machine_type  = "t2d-standard-4"
-        replicas      = 1
-        cpu_request   = "1000m"
-        mem_request   = "6Gi"
-        cpu_limit     = "2000m"
-        mem_limit     = "12Gi"
-      }
-      provisionerd = {
-        replicas    = 1
-        cpu_request = "100m"
-        mem_request = "1Gi"
-        cpu_limit   = "1000m"
-        mem_limit   = "1Gi"
-      }
-      workspaces = {
-        nodepool_size = 1
-        machine_type  = "t2d-standard-4"
-        cpu_request   = "100m"
-        mem_request   = "128Mi"
-        cpu_limit     = "100m"
-        mem_limit     = "128Mi"
-      }
-      misc = {
-        nodepool_size = 1
-        machine_type  = "t2d-standard-4"
-      }
-      cloudsql = {
-        tier            = "db-f1-micro"
-        max_connections = 500
-      }
-    }
-  }
 }
diff --git a/scaletest/terraform/action/scenarios.tf b/scaletest/terraform/action/scenarios.tf
new file mode 100644
index 0000000000000..996c449d285bf
--- /dev/null
+++ b/scaletest/terraform/action/scenarios.tf
@@ -0,0 +1,106 @@
+locals {
+  scenarios = {
+    small = {
+      coder = {
+        nodepool_size = 1
+        machine_type  = "t2d-standard-4"
+        replicas      = 1
+        cpu_request   = "1000m"
+        mem_request   = "6Gi"
+        cpu_limit     = "2000m"
+        mem_limit     = "12Gi"
+      }
+      provisionerd = {
+        replicas    = 1
+        cpu_request = "100m"
+        mem_request = "1Gi"
+        cpu_limit   = "1000m"
+        mem_limit   = "1Gi"
+      }
+      workspaces = {
+        nodepool_size = 1
+        machine_type  = "t2d-standard-4"
+        cpu_request   = "100m"
+        mem_request   = "128Mi"
+        cpu_limit     = "100m"
+        mem_limit     = "128Mi"
+      }
+      misc = {
+        nodepool_size = 1
+        machine_type  = "t2d-standard-4"
+      }
+      cloudsql = {
+        tier            = "db-f1-micro"
+        max_connections = 500
+      }
+    }
+    medium = {
+      coder = {
+        nodepool_size = 1
+        machine_type  = "t2d-standard-8"
+        replicas      = 1
+        cpu_request   = "3000m"
+        mem_request   = "12Gi"
+        cpu_limit     = "6000m"
+        mem_limit     = "24Gi"
+      }
+      provisionerd = {
+        replicas    = 1
+        cpu_request = "100m"
+        mem_request = "1Gi"
+        cpu_limit   = "1000m"
+        mem_limit   = "1Gi"
+      }
+      workspaces = {
+        nodepool_size = 1
+        machine_type  = "t2d-standard-8"
+        cpu_request   = "100m"
+        mem_request   = "128Mi"
+        cpu_limit     = "100m"
+        mem_limit     = "128Mi"
+      }
+      misc = {
+        nodepool_size = 1
+        machine_type  = "t2d-standard-4"
+      }
+      cloudsql = {
+        tier            = "db-custom-1-3840"
+        max_connections = 500
+      }
+    }
+    large = {
+      coder = {
+        nodepool_size = 3
+        machine_type  = "t2d-standard-8"
+        replicas      = 3
+        cpu_request   = "1000m"
+        mem_request   = "6Gi"
+        cpu_limit     = "2000m"
+        mem_limit     = "12Gi"
+      }
+      provisionerd = {
+        replicas    = 1
+        cpu_request = "100m"
+        mem_request = "1Gi"
+        cpu_limit   = "1000m"
+        mem_limit   = "1Gi"
+      }
+      workspaces = {
+        nodepool_size = 1
+        machine_type  = "t2d-standard-8"
+        cpu_request   = "100m"
+        mem_request   = "128Mi"
+        cpu_limit     = "100m"
+        mem_limit     = "128Mi"
+      }
+      misc = {
+        nodepool_size = 1
+        machine_type  = "t2d-standard-4"
+      }
+      cloudsql = {
+        tier            = "db-custom-2-7680"
+        max_connections = 500
+      }
+    }
+  }
+}

From 486e0f246b04c4aea12c14559e064c1f5a08b140 Mon Sep 17 00:00:00 2001
From: Garrett Delfosse <garrett@coder.com>
Date: Mon, 9 Dec 2024 18:42:19 +0000
Subject: [PATCH 18/24] swap icons

---
 scaletest/terraform/action/coder_proxies.tf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scaletest/terraform/action/coder_proxies.tf b/scaletest/terraform/action/coder_proxies.tf
index dc1c4e9e3ecaa..6c905645efcbd 100644
--- a/scaletest/terraform/action/coder_proxies.tf
+++ b/scaletest/terraform/action/coder_proxies.tf
@@ -35,12 +35,12 @@ curl '${local.deployments.primary.url}/api/v2/licenses' \
 
 europe_token=$(curl '${local.deployments.primary.url}/api/v2/workspaceproxies' \
   -H "Coder-Session-Token: $${token}" \
-  --data-raw '{"name":"europe","display_name":"Europe","icon":"/emojis/1f35b.png"}' \
+  --data-raw '{"name":"europe","display_name":"Europe","icon":"/emojis/1f950.png"}' \
   --insecure --silent | jq -r .proxy_token)
 
 asia_token=$(curl '${local.deployments.primary.url}/api/v2/workspaceproxies' \
   -H "Coder-Session-Token: $${token}" \
-  --data-raw '{"name":"asia","display_name":"Asia","icon":"/emojis/1f950.png"}' \
+  --data-raw '{"name":"asia","display_name":"Asia","icon":"/emojis/1f35b.png"}' \
   --insecure --silent | jq -r .proxy_token)
 
 mkdir -p ${path.module}/.coderv2

From b2349d39a12d1835a7b68e3f4a1efb0e8db483c6 Mon Sep 17 00:00:00 2001
From: Garrett Delfosse <garrett@coder.com>
Date: Tue, 10 Dec 2024 17:47:25 +0000
Subject: [PATCH 19/24] start traffic

---
 scaletest/terraform/action/coder_proxies.tf   |  63 -------
 scaletest/terraform/action/coderd.tf          | 165 ++++++++++++++++++
 scaletest/terraform/action/deployments.tf     |  25 ---
 scaletest/terraform/action/gcp_clusters.tf    |  23 +++
 scaletest/terraform/action/k8s_coder_asia.tf  |   2 +-
 .../terraform/action/k8s_coder_europe.tf      |   2 +-
 scaletest/terraform/action/main.tf            |   5 +
 scaletest/terraform/action/vars.tf            |  21 +++
 .../terraform/action/workspace_traffic.tf     |  95 ++++++++++
 9 files changed, 311 insertions(+), 90 deletions(-)
 delete mode 100644 scaletest/terraform/action/coder_proxies.tf
 create mode 100644 scaletest/terraform/action/coderd.tf
 delete mode 100644 scaletest/terraform/action/deployments.tf
 create mode 100644 scaletest/terraform/action/workspace_traffic.tf

diff --git a/scaletest/terraform/action/coder_proxies.tf b/scaletest/terraform/action/coder_proxies.tf
deleted file mode 100644
index 6c905645efcbd..0000000000000
--- a/scaletest/terraform/action/coder_proxies.tf
+++ /dev/null
@@ -1,63 +0,0 @@
-data "http" "coder_healthy" {
-  url = local.deployments.primary.url
-  // Wait up to 5 minutes for DNS to propagate
-  retry {
-    attempts     = 30
-    min_delay_ms = 10000
-  }
-
-  lifecycle {
-    postcondition {
-      condition     = self.status_code == 200
-      error_message = "${self.url} returned an unhealthy status code"
-    }
-  }
-
-  depends_on = [helm_release.coder_primary, cloudflare_record.coder["primary"]]
-}
-
-resource "null_resource" "proxy_tokens" {
-  provisioner "local-exec" {
-    interpreter = ["/bin/bash", "-c"]
-    command     = <<EOF
-curl '${local.deployments.primary.url}/api/v2/users/first' \
-  --data-raw $'{"email":"${local.coder_admin_email}","password":"${local.coder_admin_password}","username":"${local.coder_admin_user}","name":"${local.coder_admin_full_name}","trial":false}' \
-  --insecure --silent --output /dev/null
-
-token=$(curl '${local.deployments.primary.url}/api/v2/users/login' \
-  --data-raw $'{"email":"${local.coder_admin_email}","password":"${local.coder_admin_password}"}' \
-  --insecure --silent | jq -r .session_token)
-
-curl '${local.deployments.primary.url}/api/v2/licenses' \
-  -H "Coder-Session-Token: $${token}" \
-  --data-raw '{"license":"${var.coder_license}"}' \
-  --insecure --silent --output /dev/null
-
-europe_token=$(curl '${local.deployments.primary.url}/api/v2/workspaceproxies' \
-  -H "Coder-Session-Token: $${token}" \
-  --data-raw '{"name":"europe","display_name":"Europe","icon":"/emojis/1f950.png"}' \
-  --insecure --silent | jq -r .proxy_token)
-
-asia_token=$(curl '${local.deployments.primary.url}/api/v2/workspaceproxies' \
-  -H "Coder-Session-Token: $${token}" \
-  --data-raw '{"name":"asia","display_name":"Asia","icon":"/emojis/1f35b.png"}' \
-  --insecure --silent | jq -r .proxy_token)
-
-mkdir -p ${path.module}/.coderv2
-echo -n $${europe_token} > ${path.module}/.coderv2/europe_proxy_token
-echo -n $${asia_token} > ${path.module}/.coderv2/asia_proxy_token
-EOF
-  }
-
-  depends_on = [data.http.coder_healthy]
-}
-
-data "local_file" "europe_proxy_token" {
-  filename   = "${path.module}/.coderv2/europe_proxy_token"
-  depends_on = [null_resource.proxy_tokens]
-}
-
-data "local_file" "asia_proxy_token" {
-  filename   = "${path.module}/.coderv2/asia_proxy_token"
-  depends_on = [null_resource.proxy_tokens]
-}
diff --git a/scaletest/terraform/action/coderd.tf b/scaletest/terraform/action/coderd.tf
new file mode 100644
index 0000000000000..02ae3f272f664
--- /dev/null
+++ b/scaletest/terraform/action/coderd.tf
@@ -0,0 +1,165 @@
+data "http" "coder_healthy" {
+  url = local.deployments.primary.url
+  // Wait up to 5 minutes for DNS to propagate
+  retry {
+    attempts     = 30
+    min_delay_ms = 10000
+  }
+
+  lifecycle {
+    postcondition {
+      condition     = self.status_code == 200
+      error_message = "${self.url} returned an unhealthy status code"
+    }
+  }
+
+  depends_on = [helm_release.coder_primary, cloudflare_record.coder["primary"]]
+}
+
+resource "null_resource" "api_key" {
+  provisioner "local-exec" {
+    interpreter = ["/bin/bash", "-c"]
+    command     = <<EOF
+curl '${local.deployments.primary.url}/api/v2/users/first' \
+  --data-raw $'{"email":"${local.coder_admin_email}","password":"${local.coder_admin_password}","username":"${local.coder_admin_user}","name":"${local.coder_admin_full_name}","trial":false}' \
+  --insecure --silent --output /dev/null
+
+session_token=$(curl '${local.deployments.primary.url}/api/v2/users/login' \
+  --data-raw $'{"email":"${local.coder_admin_email}","password":"${local.coder_admin_password}"}' \
+  --insecure --silent | jq -r .session_token)
+
+api_key=$(curl '${local.deployments.primary.url}/users/me/keys/tokens' \
+  -H "Coder-Session-Token: $${session_token}" \
+  --data-raw '{"token_name":"terraform","scope":"all"}' \
+  --insecure --silent | jq -r .key)
+
+mkdir -p ${path.module}/.coderv2
+echo -n $${api_key} > ${path.module}/.coderv2/api_key
+EOF
+  }
+
+  depends_on = [data.http.coder_healthy]
+}
+
+data "local_file" "api_key" {
+  filename   = "${path.module}/.coderv2/api_key"
+  depends_on = [null_resource.api_key]
+}
+
+resource "coderd_license" "license" {
+  license = var.coder_license
+  lifecycle {
+    create_before_destroy = true
+  }
+}
+
+resource "coderd_workspace_proxy" "europe" {
+  name         = "europe"
+  display_name = "Europe"
+  icon         = "/emojis/1f950.png"
+
+  depends_on = [coderd_license.license]
+}
+
+resource "coderd_workspace_proxy" "asia" {
+  name         = "asia"
+  display_name = "Asia"
+  icon         = "/emojis/1f35b.png"
+
+  depends_on = [coderd_license.license]
+}
+
+resource "local_file" "kubernetes_template" {
+  filename = "${path.module}/.coderv2/templates/kubernetes/main.tf"
+  content  = <<EOF
+    terraform {
+      required_providers {
+        coder = {
+          source  = "coder/coder"
+          version = "~> 0.23.0"
+        }
+        kubernetes = {
+          source  = "hashicorp/kubernetes"
+          version = "~> 2.30"
+        }
+      }
+    }
+
+    provider "coder" {}
+
+    provider "kubernetes" {
+      config_path = null # always use host
+    }
+
+    data "coder_workspace" "me" {}
+    data "coder_workspace_owner" "me" {}
+
+    resource "coder_agent" "main" {
+      os                     = "linux"
+      arch                   = "amd64"
+    }
+
+    resource "kubernetes_pod" "main" {
+      count = data.coder_workspace.me.start_count
+      metadata {
+        name      = "coder-$${lower(data.coder_workspace_owner.me.name)}-$${lower(data.coder_workspace.me.name)}"
+        namespace = "${local.coder_namespace}"
+        labels = {
+          "app.kubernetes.io/name"     = "coder-workspace"
+          "app.kubernetes.io/instance" = "coder-workspace-$${lower(data.coder_workspace_owner.me.name)}-$${lower(data.coder_workspace.me.name)}"
+        }
+      }
+      spec {
+        security_context {
+          run_as_user = "1000"
+          fs_group    = "1000"
+        }
+        container {
+          name              = "dev"
+          image             = "${var.workspace_image}"
+          image_pull_policy = "Always"
+          command           = ["sh", "-c", coder_agent.main.init_script]
+          security_context {
+            run_as_user = "1000"
+          }
+          env {
+            name  = "CODER_AGENT_TOKEN"
+            value = coder_agent.main.token
+          }
+          resources {
+            requests = {
+              "cpu"    = "${local.scenarios[var.scenario].workspace.cpu_request}"
+              "memory" = "${local.scenarios[var.scenario].workspace.mem_request}"
+            }
+            limits = {
+              "cpu"    = "${local.scenarios[var.scenario].workspace.cpu_limit}"
+              "memory" = "${local.scenarios[var.scenario].workspace.mem_limit}"
+            }
+          }
+        }
+
+        affinity {
+          node_affinity {
+            required_during_scheduling_ignored_during_execution {
+              node_selector_term {
+                match_expressions {
+                  key = "cloud.google.com/gke-nodepool"
+                  operator = "In"
+                  values = ["${google_container_node_pool.node_pool["primary_workspaces"].name}","${google_container_node_pool.node_pool["europe_workspaces"].name}","${google_container_node_pool.node_pool["asia_workspaces"].name}"]
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  EOF
+}
+
+resource "coderd_template" "kubernetes" {
+  name = "kubernetes"
+  versions = [{
+    directory = "${path.module}/.coderv2/templates/kubernetes"
+    active    = true
+  }]
+}
diff --git a/scaletest/terraform/action/deployments.tf b/scaletest/terraform/action/deployments.tf
deleted file mode 100644
index 938943a2a8c16..0000000000000
--- a/scaletest/terraform/action/deployments.tf
+++ /dev/null
@@ -1,25 +0,0 @@
-locals {
-  deployments = {
-    primary = {
-      subdomain = "${var.name}-scaletest"
-      url       = "http://${var.name}-scaletest.${var.cloudflare_domain}"
-      region    = "us-east1"
-      zone      = "us-east1-c"
-      cidr      = "10.200.0.0/24"
-    }
-    europe = {
-      subdomain = "${var.name}-europe-scaletest"
-      url       = "http://${var.name}-europe-scaletest.${var.cloudflare_domain}"
-      region    = "europe-west1"
-      zone      = "europe-west1-b"
-      cidr      = "10.201.0.0/24"
-    }
-    asia = {
-      subdomain = "${var.name}-asia-scaletest"
-      url       = "http://${var.name}-asia-scaletest.${var.cloudflare_domain}"
-      region    = "asia-southeast1"
-      zone      = "asia-southeast1-a"
-      cidr      = "10.202.0.0/24"
-    }
-  }
-}
diff --git a/scaletest/terraform/action/gcp_clusters.tf b/scaletest/terraform/action/gcp_clusters.tf
index e2e360c65043c..5ca3bac3f7161 100644
--- a/scaletest/terraform/action/gcp_clusters.tf
+++ b/scaletest/terraform/action/gcp_clusters.tf
@@ -4,6 +4,29 @@ data "google_compute_default_service_account" "default" {
 }
 
 locals {
+  deployments = {
+    primary = {
+      subdomain = "${var.name}-scaletest"
+      url       = "http://${var.name}-scaletest.${var.cloudflare_domain}"
+      region    = "us-east1"
+      zone      = "us-east1-c"
+      cidr      = "10.200.0.0/24"
+    }
+    europe = {
+      subdomain = "${var.name}-europe-scaletest"
+      url       = "http://${var.name}-europe-scaletest.${var.cloudflare_domain}"
+      region    = "europe-west1"
+      zone      = "europe-west1-b"
+      cidr      = "10.201.0.0/24"
+    }
+    asia = {
+      subdomain = "${var.name}-asia-scaletest"
+      url       = "http://${var.name}-asia-scaletest.${var.cloudflare_domain}"
+      region    = "asia-southeast1"
+      zone      = "asia-southeast1-a"
+      cidr      = "10.202.0.0/24"
+    }
+  }
   node_pools = {
     primary_coder = {
       name    = "coder"
diff --git a/scaletest/terraform/action/k8s_coder_asia.tf b/scaletest/terraform/action/k8s_coder_asia.tf
index f2055632c89ca..012ffb7bba49e 100644
--- a/scaletest/terraform/action/k8s_coder_asia.tf
+++ b/scaletest/terraform/action/k8s_coder_asia.tf
@@ -36,7 +36,7 @@ resource "kubernetes_secret" "proxy_token_asia" {
     namespace = kubernetes_namespace.coder_asia.metadata.0.name
   }
   data = {
-    token = trimspace(data.local_file.asia_proxy_token.content)
+    token = coderd_workspace_proxy.asia.session_token
   }
   lifecycle {
     ignore_changes = [timeouts, wait_for_service_account_token]
diff --git a/scaletest/terraform/action/k8s_coder_europe.tf b/scaletest/terraform/action/k8s_coder_europe.tf
index 84c0d30d949b8..004bc470cc132 100644
--- a/scaletest/terraform/action/k8s_coder_europe.tf
+++ b/scaletest/terraform/action/k8s_coder_europe.tf
@@ -36,7 +36,7 @@ resource "kubernetes_secret" "proxy_token_europe" {
     namespace = kubernetes_namespace.coder_europe.metadata.0.name
   }
   data = {
-    token = trimspace(data.local_file.europe_proxy_token.content)
+    token = coderd_workspace_proxy.europe.session_token
   }
   lifecycle {
     ignore_changes = [timeouts, wait_for_service_account_token]
diff --git a/scaletest/terraform/action/main.tf b/scaletest/terraform/action/main.tf
index 57a294710c5b5..4c7e5fdcbb217 100644
--- a/scaletest/terraform/action/main.tf
+++ b/scaletest/terraform/action/main.tf
@@ -38,6 +38,11 @@ terraform {
       source  = "cloudflare/cloudflare"
       version = "~> 4.0"
     }
+
+    coderd = {
+      source  = "coder/coderd"
+      version = "~> 0.0.8"
+    }
   }
 
   required_version = "~> 1.9.0"
diff --git a/scaletest/terraform/action/vars.tf b/scaletest/terraform/action/vars.tf
index 264110e239845..83c811b09d7db 100644
--- a/scaletest/terraform/action/vars.tf
+++ b/scaletest/terraform/action/vars.tf
@@ -85,3 +85,24 @@ variable "provisionerd_image_tag" {
   description = "Tag to use for Provisionerd image."
   default     = "latest"
 }
+
+// Traffic
+variable "traffic_bytes_per_tick" {
+  description = "Number of bytes to send per tick."
+  default     = 1024
+}
+
+variable "traffic_tick_interval" {
+  description = "Interval between ticks."
+  default     = "1s"
+}
+
+variable "workspace_count" {
+  description = "Number of workspaces to create."
+  default     = 10
+}
+
+variable "workspace_create_concurrency" {
+  description = "Number of workspaces to create concurrently."
+  default     = 1
+}
diff --git a/scaletest/terraform/action/workspace_traffic.tf b/scaletest/terraform/action/workspace_traffic.tf
new file mode 100644
index 0000000000000..184c960c6c07b
--- /dev/null
+++ b/scaletest/terraform/action/workspace_traffic.tf
@@ -0,0 +1,95 @@
+resource "kubernetes_pod" "create_workspaces" {
+  provider = kubernetes.primary
+
+  metadata {
+    name      = "${var.name}-create-workspaces"
+    namespace = kubernetes_namespace.coder.metadata.0.name
+    labels = {
+      "app.kubernetes.io/name" = "${var.name}-create-workspaces"
+    }
+  }
+  spec {
+    affinity {
+      node_affinity {
+        required_during_scheduling_ignored_during_execution {
+          node_selector_term {
+            match_expressions {
+              key      = "cloud.google.com/gke-nodepool"
+              operator = "In"
+              values   = ["${google_container_node_pool.node_pool["primary_misc"].name}"]
+            }
+          }
+        }
+      }
+    }
+    container {
+      name    = "cli"
+      image   = "${var.coder_image_repo}:${var.coder_image_tag}"
+      command = ["/opt/coder --verbose --url=${local.deployments.primary.url} --token=${trimspace(local_file.api_key.content)} exp scaletest create-workspaces --count ${var.workspace_count} --template=kubernetes --concurrency ${var.workspace_create_concurrency} --no-cleanup"]
+    }
+    restart_policy = "Never"
+  }
+}
+
+resource "time_sleep" "wait_for_baseline" {
+  depends_on = [kubernetes_pod.create_workspaces]
+
+  create_duration = "600s"
+}
+
+resource "kubernetes_pod" "workspace_traffic_primary" {
+  provider = kubernetes.primary
+
+  metadata {
+    name      = "${var.name}-traffic"
+    namespace = kubernetes_namespace.coder.metadata.0.name
+    labels = {
+      "app.kubernetes.io/name" = "${var.name}-traffic"
+    }
+  }
+  spec {
+    affinity {
+      node_affinity {
+        required_during_scheduling_ignored_during_execution {
+          node_selector_term {
+            match_expressions {
+              key      = "cloud.google.com/gke-nodepool"
+              operator = "In"
+              values   = ["${google_container_node_pool.node_pool["primary_misc"].name}"]
+            }
+          }
+        }
+      }
+    }
+    container {
+      name    = "cli"
+      image   = "${var.coder_image_repo}:${var.coder_image_tag}"
+      command = ["/opt/coder --verbose --url=${local.deployments.primary.url} --token=${trimspace(local_file.api_key.content)} exp scaletest workspace-traffic --concurrency=0 --bytes-per-tick=${var.traffic_bytes_per_tick} --tick-interval=${var.traffic_tick_interval} --scaletest-prometheus-wait=60s"]
+
+      env {
+        name  = "CODER_URL"
+        value = local.deployments.primary.url
+      }
+      env {
+        name  = "CODER_TOKEN"
+        value = trimspace(local_file.api_key.content)
+      }
+      env {
+        name  = "CODER_SCALETEST_PROMETHEUS_ADDRESS"
+        value = "0.0.0.0:21112"
+      }
+      env {
+        name  = "CODER_SCALETEST_JOB_TIMEOUT"
+        value = "30m"
+      }
+      port {
+        container_port = 21112
+        name           = "prometheus-http"
+        protocol       = "TCP"
+      }
+    }
+    restart_policy = "Never"
+  }
+
+  depends_on = [time_sleep.wait_for_baseline]
+}

From acc56b03e70c43c7df26b8e2267561de58e1001e Mon Sep 17 00:00:00 2001
From: Garrett Delfosse <garrett@coder.com>
Date: Tue, 10 Dec 2024 19:23:58 +0000
Subject: [PATCH 20/24] back to curl

---
 scaletest/terraform/action/coder_proxies.tf   |  91 +++++++++
 .../action/{coderd.tf => coder_templates.tf}  | 153 +++++++--------
 scaletest/terraform/action/k8s_coder_asia.tf  |   2 +-
 .../terraform/action/k8s_coder_europe.tf      |   2 +-
 scaletest/terraform/action/vars.tf            |  18 +-
 .../terraform/action/workspace_traffic.tf     | 178 +++++++++---------
 6 files changed, 264 insertions(+), 180 deletions(-)
 create mode 100644 scaletest/terraform/action/coder_proxies.tf
 rename scaletest/terraform/action/{coderd.tf => coder_templates.tf} (55%)

diff --git a/scaletest/terraform/action/coder_proxies.tf b/scaletest/terraform/action/coder_proxies.tf
new file mode 100644
index 0000000000000..126bd11efb1fc
--- /dev/null
+++ b/scaletest/terraform/action/coder_proxies.tf
@@ -0,0 +1,91 @@
+data "http" "coder_healthy" {
+  url = local.deployments.primary.url
+  // Wait up to 5 minutes for DNS to propagate
+  retry {
+    attempts     = 30
+    min_delay_ms = 10000
+  }
+
+  lifecycle {
+    postcondition {
+      condition     = self.status_code == 200
+      error_message = "${self.url} returned an unhealthy status code"
+    }
+  }
+
+  depends_on = [helm_release.coder_primary, cloudflare_record.coder["primary"]]
+}
+
+resource "null_resource" "api_key" {
+  provisioner "local-exec" {
+    interpreter = ["/bin/bash", "-c"]
+    command     = <<EOF
+curl '${local.deployments.primary.url}/api/v2/users/first' \
+  --data-raw $'{"email":"${local.coder_admin_email}","password":"${local.coder_admin_password}","username":"${local.coder_admin_user}","name":"${local.coder_admin_full_name}","trial":false}' \
+  --insecure --silent --output /dev/null
+
+session_token=$(curl '${local.deployments.primary.url}/api/v2/users/login' \
+  --data-raw $'{"email":"${local.coder_admin_email}","password":"${local.coder_admin_password}"}' \
+  --insecure --silent | jq -r .session_token)
+
+api_key=$(curl '${local.deployments.primary.url}/users/me/keys/tokens' \
+  -H "Coder-Session-Token: $${session_token}" \
+  --data-raw '{"token_name":"terraform","scope":"all"}' \
+  --insecure --silent | jq -r .key)
+
+mkdir -p ${path.module}/.coderv2
+echo -n $${api_key} > ${path.module}/.coderv2/api_key
+EOF
+  }
+
+  depends_on = [data.http.coder_healthy]
+}
+
+data "local_file" "api_key" {
+  filename   = "${path.module}/.coderv2/api_key"
+  depends_on = [null_resource.api_key]
+}
+
+resource "null_resource" "license" {
+  provisioner "local-exec" {
+    interpreter = ["/bin/bash", "-c"]
+    command     = <<EOF
+curl '${local.deployments.primary.url}/api/v2/licenses' \
+  -H "Coder-Session-Token: ${trimspace(data.local_file.api_key.content)}" \
+  --data-raw '{"license":"${var.coder_license}"}' \
+  --insecure --silent --output /dev/null
+EOF
+  }
+}
+
+resource "null_resource" "proxy_tokens" {
+  provisioner "local-exec" {
+    interpreter = ["/bin/bash", "-c"]
+    command     = <<EOF
+europe_token=$(curl '${local.deployments.primary.url}/api/v2/workspaceproxies' \
+  -H "Coder-Session-Token: ${trimspace(data.local_file.api_key.content)}" \
+  --data-raw '{"name":"europe","display_name":"Europe","icon":"/emojis/1f950.png"}' \
+  --insecure --silent | jq -r .proxy_token)
+
+asia_token=$(curl '${local.deployments.primary.url}/api/v2/workspaceproxies' \
+  -H "Coder-Session-Token: ${trimspace(data.local_file.api_key.content)}" \
+  --data-raw '{"name":"asia","display_name":"Asia","icon":"/emojis/1f35b.png"}' \
+  --insecure --silent | jq -r .proxy_token)
+
+echo -n $${europe_token} > ${path.module}/.coderv2/europe_proxy_token
+echo -n $${asia_token} > ${path.module}/.coderv2/asia_proxy_token
+EOF
+  }
+
+  depends_on = [data.http.coder_healthy]
+}
+
+data "local_file" "europe_proxy_token" {
+  filename   = "${path.module}/.coderv2/europe_proxy_token"
+  depends_on = [null_resource.proxy_tokens]
+}
+
+data "local_file" "asia_proxy_token" {
+  filename   = "${path.module}/.coderv2/asia_proxy_token"
+  depends_on = [null_resource.proxy_tokens]
+}
diff --git a/scaletest/terraform/action/coderd.tf b/scaletest/terraform/action/coder_templates.tf
similarity index 55%
rename from scaletest/terraform/action/coderd.tf
rename to scaletest/terraform/action/coder_templates.tf
index 02ae3f272f664..8b9254185e9df 100644
--- a/scaletest/terraform/action/coderd.tf
+++ b/scaletest/terraform/action/coder_templates.tf
@@ -1,74 +1,3 @@
-data "http" "coder_healthy" {
-  url = local.deployments.primary.url
-  // Wait up to 5 minutes for DNS to propagate
-  retry {
-    attempts     = 30
-    min_delay_ms = 10000
-  }
-
-  lifecycle {
-    postcondition {
-      condition     = self.status_code == 200
-      error_message = "${self.url} returned an unhealthy status code"
-    }
-  }
-
-  depends_on = [helm_release.coder_primary, cloudflare_record.coder["primary"]]
-}
-
-resource "null_resource" "api_key" {
-  provisioner "local-exec" {
-    interpreter = ["/bin/bash", "-c"]
-    command     = <<EOF
-curl '${local.deployments.primary.url}/api/v2/users/first' \
-  --data-raw $'{"email":"${local.coder_admin_email}","password":"${local.coder_admin_password}","username":"${local.coder_admin_user}","name":"${local.coder_admin_full_name}","trial":false}' \
-  --insecure --silent --output /dev/null
-
-session_token=$(curl '${local.deployments.primary.url}/api/v2/users/login' \
-  --data-raw $'{"email":"${local.coder_admin_email}","password":"${local.coder_admin_password}"}' \
-  --insecure --silent | jq -r .session_token)
-
-api_key=$(curl '${local.deployments.primary.url}/users/me/keys/tokens' \
-  -H "Coder-Session-Token: $${session_token}" \
-  --data-raw '{"token_name":"terraform","scope":"all"}' \
-  --insecure --silent | jq -r .key)
-
-mkdir -p ${path.module}/.coderv2
-echo -n $${api_key} > ${path.module}/.coderv2/api_key
-EOF
-  }
-
-  depends_on = [data.http.coder_healthy]
-}
-
-data "local_file" "api_key" {
-  filename   = "${path.module}/.coderv2/api_key"
-  depends_on = [null_resource.api_key]
-}
-
-resource "coderd_license" "license" {
-  license = var.coder_license
-  lifecycle {
-    create_before_destroy = true
-  }
-}
-
-resource "coderd_workspace_proxy" "europe" {
-  name         = "europe"
-  display_name = "Europe"
-  icon         = "/emojis/1f950.png"
-
-  depends_on = [coderd_license.license]
-}
-
-resource "coderd_workspace_proxy" "asia" {
-  name         = "asia"
-  display_name = "Asia"
-  icon         = "/emojis/1f35b.png"
-
-  depends_on = [coderd_license.license]
-}
-
 resource "local_file" "kubernetes_template" {
   filename = "${path.module}/.coderv2/templates/kubernetes/main.tf"
   content  = <<EOF
@@ -128,12 +57,12 @@ resource "local_file" "kubernetes_template" {
           }
           resources {
             requests = {
-              "cpu"    = "${local.scenarios[var.scenario].workspace.cpu_request}"
-              "memory" = "${local.scenarios[var.scenario].workspace.mem_request}"
+              "cpu"    = "${local.scenarios[var.scenario].workspaces.cpu_request}"
+              "memory" = "${local.scenarios[var.scenario].workspaces.mem_request}"
             }
             limits = {
-              "cpu"    = "${local.scenarios[var.scenario].workspace.cpu_limit}"
-              "memory" = "${local.scenarios[var.scenario].workspace.mem_limit}"
+              "cpu"    = "${local.scenarios[var.scenario].workspaces.cpu_limit}"
+              "memory" = "${local.scenarios[var.scenario].workspaces.mem_limit}"
             }
           }
         }
@@ -156,10 +85,72 @@ resource "local_file" "kubernetes_template" {
   EOF
 }
 
-resource "coderd_template" "kubernetes" {
-  name = "kubernetes"
-  versions = [{
-    directory = "${path.module}/.coderv2/templates/kubernetes"
-    active    = true
-  }]
+resource "kubernetes_config_map" "template" {
+  provider = kubernetes.primary
+
+  metadata {
+    name      = "coder-template"
+    namespace = kubernetes_namespace.coder_primary.metadata.0.name
+  }
+
+  data = {
+    "main.tf" = local_file.kubernetes_template.content
+  }
+}
+
+resource "kubernetes_pod" "push_template" {
+  provider = kubernetes.primary
+
+  metadata {
+    name      = "${var.name}-push-template"
+    namespace = kubernetes_namespace.coder_primary.metadata.0.name
+    labels = {
+      "app.kubernetes.io/name" = "${var.name}-push-template"
+    }
+  }
+  spec {
+    affinity {
+      node_affinity {
+        required_during_scheduling_ignored_during_execution {
+          node_selector_term {
+            match_expressions {
+              key      = "cloud.google.com/gke-nodepool"
+              operator = "In"
+              values   = ["${google_container_node_pool.node_pool["primary_misc"].name}"]
+            }
+          }
+        }
+      }
+    }
+    container {
+      name  = "cli"
+      image = "${var.coder_image_repo}:${var.coder_image_tag}"
+      command = [
+        "/opt/coder",
+        "--verbose",
+        "--url=${local.deployments.primary.url}",
+        "--token=${trimspace(data.local_file.api_key.content)}",
+        "templates",
+        "push",
+        "--directory=/template",
+        "--yes",
+        "kubernetes"
+      ]
+      volume_mount {
+        name       = "coder-template"
+        mount_path = "/template"
+      }
+    }
+    volume {
+      name = "coder-template"
+      config_map {
+        name = kubernetes_config_map.template.metadata.0.name
+        items {
+          key  = "main.tf"
+          path = "main.tf"
+        }
+      }
+    }
+    restart_policy = "Never"
+  }
 }
diff --git a/scaletest/terraform/action/k8s_coder_asia.tf b/scaletest/terraform/action/k8s_coder_asia.tf
index 012ffb7bba49e..f2055632c89ca 100644
--- a/scaletest/terraform/action/k8s_coder_asia.tf
+++ b/scaletest/terraform/action/k8s_coder_asia.tf
@@ -36,7 +36,7 @@ resource "kubernetes_secret" "proxy_token_asia" {
     namespace = kubernetes_namespace.coder_asia.metadata.0.name
   }
   data = {
-    token = coderd_workspace_proxy.asia.session_token
+    token = trimspace(data.local_file.asia_proxy_token.content)
   }
   lifecycle {
     ignore_changes = [timeouts, wait_for_service_account_token]
diff --git a/scaletest/terraform/action/k8s_coder_europe.tf b/scaletest/terraform/action/k8s_coder_europe.tf
index 004bc470cc132..84c0d30d949b8 100644
--- a/scaletest/terraform/action/k8s_coder_europe.tf
+++ b/scaletest/terraform/action/k8s_coder_europe.tf
@@ -36,7 +36,7 @@ resource "kubernetes_secret" "proxy_token_europe" {
     namespace = kubernetes_namespace.coder_europe.metadata.0.name
   }
   data = {
-    token = coderd_workspace_proxy.europe.session_token
+    token = trimspace(data.local_file.europe_proxy_token.content)
   }
   lifecycle {
     ignore_changes = [timeouts, wait_for_service_account_token]
diff --git a/scaletest/terraform/action/vars.tf b/scaletest/terraform/action/vars.tf
index 83c811b09d7db..ddd8162c7f570 100644
--- a/scaletest/terraform/action/vars.tf
+++ b/scaletest/terraform/action/vars.tf
@@ -87,15 +87,15 @@ variable "provisionerd_image_tag" {
 }
 
 // Traffic
-variable "traffic_bytes_per_tick" {
-  description = "Number of bytes to send per tick."
-  default     = 1024
-}
+# variable "traffic_bytes_per_tick" {
+#   description = "Number of bytes to send per tick."
+#   default     = 1024
+# }
 
-variable "traffic_tick_interval" {
-  description = "Interval between ticks."
-  default     = "1s"
-}
+# variable "traffic_tick_interval" {
+#   description = "Interval between ticks."
+#   default     = "10s"
+# }
 
 variable "workspace_count" {
   description = "Number of workspaces to create."
@@ -104,5 +104,5 @@ variable "workspace_count" {
 
 variable "workspace_create_concurrency" {
   description = "Number of workspaces to create concurrently."
-  default     = 1
+  default     = 10
 }
diff --git a/scaletest/terraform/action/workspace_traffic.tf b/scaletest/terraform/action/workspace_traffic.tf
index 184c960c6c07b..a58637979d166 100644
--- a/scaletest/terraform/action/workspace_traffic.tf
+++ b/scaletest/terraform/action/workspace_traffic.tf
@@ -1,95 +1,97 @@
-resource "kubernetes_pod" "create_workspaces" {
-  provider = kubernetes.primary
+# resource "kubernetes_pod" "create_workspaces" {
+#   provider = kubernetes.primary
 
-  metadata {
-    name      = "${var.name}-create-workspaces"
-    namespace = kubernetes_namespace.coder.metadata.0.name
-    labels = {
-      "app.kubernetes.io/name" = "${var.name}-create-workspaces"
-    }
-  }
-  spec {
-    affinity {
-      node_affinity {
-        required_during_scheduling_ignored_during_execution {
-          node_selector_term {
-            match_expressions {
-              key      = "cloud.google.com/gke-nodepool"
-              operator = "In"
-              values   = ["${google_container_node_pool.node_pool["primary_misc"].name}"]
-            }
-          }
-        }
-      }
-    }
-    container {
-      name    = "cli"
-      image   = "${var.coder_image_repo}:${var.coder_image_tag}"
-      command = ["/opt/coder --verbose --url=${local.deployments.primary.url} --token=${trimspace(local_file.api_key.content)} exp scaletest create-workspaces --count ${var.workspace_count} --template=kubernetes --concurrency ${var.workspace_create_concurrency} --no-cleanup"]
-    }
-    restart_policy = "Never"
-  }
-}
+#   metadata {
+#     name      = "${var.name}-create-workspaces"
+#     namespace = kubernetes_namespace.coder_primary.metadata.0.name
+#     labels = {
+#       "app.kubernetes.io/name" = "${var.name}-create-workspaces"
+#     }
+#   }
+#   spec {
+#     affinity {
+#       node_affinity {
+#         required_during_scheduling_ignored_during_execution {
+#           node_selector_term {
+#             match_expressions {
+#               key      = "cloud.google.com/gke-nodepool"
+#               operator = "In"
+#               values   = ["${google_container_node_pool.node_pool["primary_misc"].name}"]
+#             }
+#           }
+#         }
+#       }
+#     }
+#     container {
+#       name    = "cli"
+#       image   = "${var.coder_image_repo}:${var.coder_image_tag}"
+#       command = ["/opt/coder --verbose --url=${local.deployments.primary.url} --token=${trimspace(data.local_file.api_key.content)} exp scaletest create-workspaces --count ${var.workspace_count} --template=kubernetes --concurrency ${var.workspace_create_concurrency} --no-cleanup"]
+#     }
+#     restart_policy = "Never"
+#   }
 
-resource "time_sleep" "wait_for_baseline" {
-  depends_on = [kubernetes_pod.create_workspaces]
+#   depends_on = [ coderd_template.kubernetes ]
+# }
 
-  create_duration = "600s"
-}
+# resource "time_sleep" "wait_for_baseline" {
+#   depends_on = [kubernetes_pod.create_workspaces]
 
-resource "kubernetes_pod" "workspace_traffic_primary" {
-  provider = kubernetes.primary
+#   create_duration = "600s"
+# }
 
-  metadata {
-    name      = "${var.name}-traffic"
-    namespace = kubernetes_namespace.coder.metadata.0.name
-    labels = {
-      "app.kubernetes.io/name" = "${var.name}-traffic"
-    }
-  }
-  spec {
-    affinity {
-      node_affinity {
-        required_during_scheduling_ignored_during_execution {
-          node_selector_term {
-            match_expressions {
-              key      = "cloud.google.com/gke-nodepool"
-              operator = "In"
-              values   = ["${google_container_node_pool.node_pool["primary_misc"].name}"]
-            }
-          }
-        }
-      }
-    }
-    container {
-      name    = "cli"
-      image   = "${var.coder_image_repo}:${var.coder_image_tag}"
-      command = ["/opt/coder --verbose --url=${local.deployments.primary.url} --token=${trimspace(local_file.api_key.content)} exp scaletest workspace-traffic --concurrency=0 --bytes-per-tick=${var.traffic_bytes_per_tick} --tick-interval=${var.traffic_tick_interval} --scaletest-prometheus-wait=60s"]
+# resource "kubernetes_pod" "workspace_traffic_primary" {
+#   provider = kubernetes.primary
 
-      env {
-        name  = "CODER_URL"
-        value = local.deployments.primary.url
-      }
-      env {
-        name  = "CODER_TOKEN"
-        value = trimspace(local_file.api_key.content)
-      }
-      env {
-        name  = "CODER_SCALETEST_PROMETHEUS_ADDRESS"
-        value = "0.0.0.0:21112"
-      }
-      env {
-        name  = "CODER_SCALETEST_JOB_TIMEOUT"
-        value = "30m"
-      }
-      port {
-        container_port = 21112
-        name           = "prometheus-http"
-        protocol       = "TCP"
-      }
-    }
-    restart_policy = "Never"
-  }
+#   metadata {
+#     name      = "${var.name}-traffic"
+#     namespace = kubernetes_namespace.coder.metadata.0.name
+#     labels = {
+#       "app.kubernetes.io/name" = "${var.name}-traffic"
+#     }
+#   }
+#   spec {
+#     affinity {
+#       node_affinity {
+#         required_during_scheduling_ignored_during_execution {
+#           node_selector_term {
+#             match_expressions {
+#               key      = "cloud.google.com/gke-nodepool"
+#               operator = "In"
+#               values   = ["${google_container_node_pool.node_pool["primary_misc"].name}"]
+#             }
+#           }
+#         }
+#       }
+#     }
+#     container {
+#       name    = "cli"
+#       image   = "${var.coder_image_repo}:${var.coder_image_tag}"
+#       command = ["/opt/coder --verbose --url=${local.deployments.primary.url} --token=${trimspace(local_file.api_key.content)} exp scaletest workspace-traffic --concurrency=0 --bytes-per-tick=${var.traffic_bytes_per_tick} --tick-interval=${var.traffic_tick_interval} --scaletest-prometheus-wait=60s"]
 
-  depends_on = [time_sleep.wait_for_baseline]
-}
+#       env {
+#         name  = "CODER_URL"
+#         value = local.deployments.primary.url
+#       }
+#       env {
+#         name  = "CODER_TOKEN"
+#         value = trimspace(local_file.api_key.content)
+#       }
+#       env {
+#         name  = "CODER_SCALETEST_PROMETHEUS_ADDRESS"
+#         value = "0.0.0.0:21112"
+#       }
+#       env {
+#         name  = "CODER_SCALETEST_JOB_TIMEOUT"
+#         value = "30m"
+#       }
+#       port {
+#         container_port = 21112
+#         name           = "prometheus-http"
+#         protocol       = "TCP"
+#       }
+#     }
+#     restart_policy = "Never"
+#   }
+
+#   depends_on = [time_sleep.wait_for_baseline]
+# }

From 5385e883e4e6db1ec08d1e5a65533883042469ff Mon Sep 17 00:00:00 2001
From: Garrett Delfosse <garrett@coder.com>
Date: Wed, 11 Dec 2024 19:14:44 +0000
Subject: [PATCH 21/24] push template

---
 scaletest/terraform/action/coder_proxies.tf   | 43 ++++++----
 scaletest/terraform/action/coder_templates.tf | 80 ++++++++++---------
 .../terraform/action/workspace_traffic.tf     | 14 +++-
 3 files changed, 82 insertions(+), 55 deletions(-)

diff --git a/scaletest/terraform/action/coder_proxies.tf b/scaletest/terraform/action/coder_proxies.tf
index 126bd11efb1fc..6af3ef82bb392 100644
--- a/scaletest/terraform/action/coder_proxies.tf
+++ b/scaletest/terraform/action/coder_proxies.tf
@@ -20,6 +20,8 @@ resource "null_resource" "api_key" {
   provisioner "local-exec" {
     interpreter = ["/bin/bash", "-c"]
     command     = <<EOF
+set -e
+
 curl '${local.deployments.primary.url}/api/v2/users/first' \
   --data-raw $'{"email":"${local.coder_admin_email}","password":"${local.coder_admin_password}","username":"${local.coder_admin_user}","name":"${local.coder_admin_full_name}","trial":false}' \
   --insecure --silent --output /dev/null
@@ -28,12 +30,13 @@ session_token=$(curl '${local.deployments.primary.url}/api/v2/users/login' \
   --data-raw $'{"email":"${local.coder_admin_email}","password":"${local.coder_admin_password}"}' \
   --insecure --silent | jq -r .session_token)
 
-api_key=$(curl '${local.deployments.primary.url}/users/me/keys/tokens' \
+echo -n $${session_token} > ${path.module}/.coderv2/session_token
+
+api_key=$(curl '${local.deployments.primary.url}/api/v2/users/me/keys/tokens' \
   -H "Coder-Session-Token: $${session_token}" \
   --data-raw '{"token_name":"terraform","scope":"all"}' \
   --insecure --silent | jq -r .key)
 
-mkdir -p ${path.module}/.coderv2
 echo -n $${api_key} > ${path.module}/.coderv2/api_key
 EOF
   }
@@ -58,34 +61,42 @@ EOF
   }
 }
 
-resource "null_resource" "proxy_tokens" {
+resource "null_resource" "europe_proxy_token" {
   provisioner "local-exec" {
     interpreter = ["/bin/bash", "-c"]
     command     = <<EOF
-europe_token=$(curl '${local.deployments.primary.url}/api/v2/workspaceproxies' \
+curl '${local.deployments.primary.url}/api/v2/workspaceproxies' \
   -H "Coder-Session-Token: ${trimspace(data.local_file.api_key.content)}" \
   --data-raw '{"name":"europe","display_name":"Europe","icon":"/emojis/1f950.png"}' \
-  --insecure --silent | jq -r .proxy_token)
-
-asia_token=$(curl '${local.deployments.primary.url}/api/v2/workspaceproxies' \
-  -H "Coder-Session-Token: ${trimspace(data.local_file.api_key.content)}" \
-  --data-raw '{"name":"asia","display_name":"Asia","icon":"/emojis/1f35b.png"}' \
-  --insecure --silent | jq -r .proxy_token)
-
-echo -n $${europe_token} > ${path.module}/.coderv2/europe_proxy_token
-echo -n $${asia_token} > ${path.module}/.coderv2/asia_proxy_token
+  --insecure --silent \
+  | jq -r .proxy_token > ${path.module}/.coderv2/europe_proxy_token
 EOF
   }
 
-  depends_on = [data.http.coder_healthy]
+  depends_on = [null_resource.license]
 }
 
 data "local_file" "europe_proxy_token" {
   filename   = "${path.module}/.coderv2/europe_proxy_token"
-  depends_on = [null_resource.proxy_tokens]
+  depends_on = [null_resource.europe_proxy_token]
+}
+
+resource "null_resource" "asia_proxy_token" {
+  provisioner "local-exec" {
+    interpreter = ["/bin/bash", "-c"]
+    command     = <<EOF
+curl '${local.deployments.primary.url}/api/v2/workspaceproxies' \
+  -H "Coder-Session-Token: ${trimspace(data.local_file.api_key.content)}" \
+  --data-raw '{"name":"asia","display_name":"Asia","icon":"/emojis/1f35b.png"}' \
+  --insecure --silent \
+  | jq -r .proxy_token > ${path.module}/.coderv2/asia_proxy_token
+EOF
+  }
+
+  depends_on = [null_resource.license]
 }
 
 data "local_file" "asia_proxy_token" {
   filename   = "${path.module}/.coderv2/asia_proxy_token"
-  depends_on = [null_resource.proxy_tokens]
+  depends_on = [null_resource.asia_proxy_token]
 }
diff --git a/scaletest/terraform/action/coder_templates.tf b/scaletest/terraform/action/coder_templates.tf
index 8b9254185e9df..c2334a488a85a 100644
--- a/scaletest/terraform/action/coder_templates.tf
+++ b/scaletest/terraform/action/coder_templates.tf
@@ -98,7 +98,7 @@ resource "kubernetes_config_map" "template" {
   }
 }
 
-resource "kubernetes_pod" "push_template" {
+resource "kubernetes_job" "push_template" {
   provider = kubernetes.primary
 
   metadata {
@@ -109,48 +109,52 @@ resource "kubernetes_pod" "push_template" {
     }
   }
   spec {
-    affinity {
-      node_affinity {
-        required_during_scheduling_ignored_during_execution {
-          node_selector_term {
-            match_expressions {
-              key      = "cloud.google.com/gke-nodepool"
-              operator = "In"
-              values   = ["${google_container_node_pool.node_pool["primary_misc"].name}"]
+    completions = 1
+    template {
+      metadata {}
+      spec {
+        affinity {
+          node_affinity {
+            required_during_scheduling_ignored_during_execution {
+              node_selector_term {
+                match_expressions {
+                  key      = "cloud.google.com/gke-nodepool"
+                  operator = "In"
+                  values   = ["${google_container_node_pool.node_pool["primary_misc"].name}"]
+                }
+              }
             }
           }
         }
-      }
-    }
-    container {
-      name  = "cli"
-      image = "${var.coder_image_repo}:${var.coder_image_tag}"
-      command = [
-        "/opt/coder",
-        "--verbose",
-        "--url=${local.deployments.primary.url}",
-        "--token=${trimspace(data.local_file.api_key.content)}",
-        "templates",
-        "push",
-        "--directory=/template",
-        "--yes",
-        "kubernetes"
-      ]
-      volume_mount {
-        name       = "coder-template"
-        mount_path = "/template"
-      }
-    }
-    volume {
-      name = "coder-template"
-      config_map {
-        name = kubernetes_config_map.template.metadata.0.name
-        items {
-          key  = "main.tf"
-          path = "main.tf"
+        container {
+          name  = "cli"
+          image = "${var.coder_image_repo}:${var.coder_image_tag}"
+          command = [
+            "/opt/coder",
+            "--verbose",
+            "--url=${local.deployments.primary.url}",
+            "--token=${trimspace(data.local_file.api_key.content)}",
+            "templates",
+            "push",
+            "--directory=/home/coder/template",
+            "--yes",
+            "kubernetes"
+          ]
+          volume_mount {
+            name       = "coder-template"
+            mount_path = "/home/coder/template/main.tf"
+            sub_path   = "main.tf"
+          }
+        }
+        volume {
+          name = "coder-template"
+          config_map {
+            name = kubernetes_config_map.template.metadata.0.name
+          }
         }
+        restart_policy = "Never"
       }
     }
-    restart_policy = "Never"
   }
+  wait_for_completion = true
 }
diff --git a/scaletest/terraform/action/workspace_traffic.tf b/scaletest/terraform/action/workspace_traffic.tf
index a58637979d166..a0db132725456 100644
--- a/scaletest/terraform/action/workspace_traffic.tf
+++ b/scaletest/terraform/action/workspace_traffic.tf
@@ -25,7 +25,19 @@
 #     container {
 #       name    = "cli"
 #       image   = "${var.coder_image_repo}:${var.coder_image_tag}"
-#       command = ["/opt/coder --verbose --url=${local.deployments.primary.url} --token=${trimspace(data.local_file.api_key.content)} exp scaletest create-workspaces --count ${var.workspace_count} --template=kubernetes --concurrency ${var.workspace_create_concurrency} --no-cleanup"]
+#       command = [
+#         "/opt/coder",
+#         "--verbose",
+#         "--url=${local.deployments.primary.url}",
+#         "--token=${trimspace(data.local_file.api_key.content)}",
+#         "exp",
+#         "scaletest",
+#         "create-workspaces",
+#         "--count=${var.workspace_count}",
+#         "--template=kubernetes",
+#         "--concurrency=${var.workspace_create_concurrency}",
+#         "--no-cleanup"
+#       ]
 #     }
 #     restart_policy = "Never"
 #   }

From 769b5d311e7ee1a86983f8879605c3643357ec09 Mon Sep 17 00:00:00 2001
From: Garrett Delfosse <garrett@coder.com>
Date: Wed, 11 Dec 2024 21:26:33 +0000
Subject: [PATCH 22/24] tpl

---
 .../terraform/action/coder_helm_values.tftpl  | 105 ++++++++++
 scaletest/terraform/action/k8s_coder_asia.tf  | 190 ++++--------------
 .../terraform/action/k8s_coder_europe.tf      | 190 ++++--------------
 .../terraform/action/k8s_coder_primary.tf     | 187 ++++-------------
 4 files changed, 225 insertions(+), 447 deletions(-)
 create mode 100644 scaletest/terraform/action/coder_helm_values.tftpl

diff --git a/scaletest/terraform/action/coder_helm_values.tftpl b/scaletest/terraform/action/coder_helm_values.tftpl
new file mode 100644
index 0000000000000..7de0c598a1780
--- /dev/null
+++ b/scaletest/terraform/action/coder_helm_values.tftpl
@@ -0,0 +1,105 @@
+coder:
+  workspaceProxy: ${workspace_proxy}
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: "cloud.google.com/gke-nodepool"
+            operator: "In"
+            values: ["${node_pool}"]
+    podAntiAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+      - weight: 1
+        podAffinityTerm:
+          topologyKey: "kubernetes.io/hostname"
+          labelSelector:
+            matchExpressions:
+            - key:      "app.kubernetes.io/instance"
+              operator: "In"
+              values:   ["${release_name}"]
+  env:
+    %{~ if workspace_proxy ~}
+    - name: "CODER_ACCESS_URL"
+      value: "${access_url}"
+    - name: CODER_PRIMARY_ACCESS_URL
+      value: "${primary_url}"
+    - name: CODER_PROXY_SESSION_TOKEN
+      valueFrom:
+        secretKeyRef:
+          key: token
+          name: "${proxy_token}"
+    %{~ endif ~}
+    %{~ if provisionerd ~}
+    - name: "CODER_URL"
+      value: "${access_url}"
+    - name: "CODER_PROVISIONERD_TAGS"
+      value: "scope=organization"
+    - name: "CODER_CONFIG_DIR"
+      value: "/tmp/config"
+    %{~ endif ~}
+    %{~ if !workspace_proxy && !provisionerd ~}
+    - name: "CODER_ACCESS_URL"
+      value: "${access_url}"
+    - name: "CODER_PG_CONNECTION_URL"
+      valueFrom:
+        secretKeyRef:
+          name: "${db_secret}"
+          key: url
+    - name: "CODER_PROVISIONER_DAEMONS"
+      value: "0"
+    - name: CODER_PROVISIONER_DAEMON_PSK
+      valueFrom:
+        secretKeyRef:
+          key: psk
+          name: "${provisionerd_psk}"
+    - name: "CODER_PROMETHEUS_COLLECT_AGENT_STATS"
+      value: "true"
+    - name: "CODER_PROMETHEUS_COLLECT_DB_METRICS"
+      value: "true"
+    - name: "CODER_PPROF_ENABLE"
+      value: "true"
+    %{~ endif ~}
+    - name: "CODER_CACHE_DIRECTORY"
+      value: "/tmp/coder"
+    - name: "CODER_TELEMETRY_ENABLE"
+      value: "false"
+    - name: "CODER_LOGGING_HUMAN"
+      value: "/dev/null"
+    - name: "CODER_LOGGING_STACKDRIVER"
+      value: "/dev/stderr"
+    - name: "CODER_PROMETHEUS_ENABLE"
+      value: "true"
+    - name: "CODER_VERBOSE"
+      value: "true"
+    - name: "CODER_EXPERIMENTS"
+      value: "${experiments}"
+    - name: "CODER_DANGEROUS_DISABLE_RATE_LIMITS"
+      value: "true"
+  image:
+    repo: ${image_repo}
+    tag: ${image_tag}
+  replicaCount: "${replicas}"
+  resources:
+    requests:
+      cpu: "${cpu_request}"
+      memory: "${mem_request}"
+    limits:
+      cpu: "${cpu_limit}"
+      memory: "${mem_limit}"
+  securityContext:
+    readOnlyRootFilesystem: true
+  %{~ if !provisionerd ~}
+  service:
+    enable: true
+    sessionAffinity: None
+    loadBalancerIP: "${ip_address}"
+  %{~ endif ~}
+  volumeMounts:
+  - mountPath: "/tmp"
+    name: cache
+    readOnly: false
+  volumes:
+  - emptyDir:
+      sizeLimit: 1024Mi
+    name: cache
diff --git a/scaletest/terraform/action/k8s_coder_asia.tf b/scaletest/terraform/action/k8s_coder_asia.tf
index f2055632c89ca..b5b2323113adc 100644
--- a/scaletest/terraform/action/k8s_coder_asia.tf
+++ b/scaletest/terraform/action/k8s_coder_asia.tf
@@ -51,94 +51,26 @@ resource "helm_release" "coder_asia" {
   name       = local.coder_release_name
   version    = var.coder_chart_version
   namespace  = kubernetes_namespace.coder_asia.metadata.0.name
-  values = [<<EOF
-coder:
-  workspaceProxy: true
-  affinity:
-    nodeAffinity:
-      requiredDuringSchedulingIgnoredDuringExecution:
-        nodeSelectorTerms:
-        - matchExpressions:
-          - key: "cloud.google.com/gke-nodepool"
-            operator: "In"
-            values: ["${google_container_node_pool.node_pool["asia_coder"].name}"]
-    podAntiAffinity:
-      preferredDuringSchedulingIgnoredDuringExecution:
-      - weight: 1
-        podAffinityTerm:
-          topologyKey: "kubernetes.io/hostname"
-          labelSelector:
-            matchExpressions:
-            - key:      "app.kubernetes.io/instance"
-              operator: "In"
-              values:   ["${local.coder_release_name}"]
-  env:
-    - name: CODER_PRIMARY_ACCESS_URL
-      value: "${local.deployments.primary.url}"
-    - name: CODER_PROXY_SESSION_TOKEN
-      valueFrom:
-        secretKeyRef:
-          key: token
-          name: "${kubernetes_secret.proxy_token_asia.metadata.0.name}"
-    - name: "CODER_ACCESS_URL"
-      value: "${local.deployments.asia.url}"
-    - name: "CODER_CACHE_DIRECTORY"
-      value: "/tmp/coder"
-    - name: "CODER_TELEMETRY_ENABLE"
-      value: "false"
-    - name: "CODER_LOGGING_HUMAN"
-      value: "/dev/null"
-    - name: "CODER_LOGGING_STACKDRIVER"
-      value: "/dev/stderr"
-    - name: "CODER_PPROF_ENABLE"
-      value: "true"
-    - name: "CODER_PROMETHEUS_ENABLE"
-      value: "true"
-    - name: "CODER_PROMETHEUS_COLLECT_AGENT_STATS"
-      value: "true"
-    - name: "CODER_PROMETHEUS_COLLECT_DB_METRICS"
-      value: "true"
-    - name: "CODER_VERBOSE"
-      value: "true"
-    - name: "CODER_EXPERIMENTS"
-      value: "${var.coder_experiments}"
-    - name: "CODER_DANGEROUS_DISABLE_RATE_LIMITS"
-      value: "true"
-    # Disabling built-in provisioner daemons
-    - name: "CODER_PROVISIONER_DAEMONS"
-      value: "0"
-    - name: CODER_PROVISIONER_DAEMON_PSK
-      valueFrom:
-        secretKeyRef:
-          key: psk
-          name: "${kubernetes_secret.provisionerd_psk_asia.metadata.0.name}"
-  image:
-    repo: ${var.coder_image_repo}
-    tag: ${var.coder_image_tag}
-  replicaCount: "${local.scenarios[var.scenario].coder.replicas}"
-  resources:
-    requests:
-      cpu: "${local.scenarios[var.scenario].coder.cpu_request}"
-      memory: "${local.scenarios[var.scenario].coder.mem_request}"
-    limits:
-      cpu: "${local.scenarios[var.scenario].coder.cpu_limit}"
-      memory: "${local.scenarios[var.scenario].coder.mem_limit}"
-  securityContext:
-    readOnlyRootFilesystem: true
-  service:
-    enable: true
-    sessionAffinity: None
-    loadBalancerIP: "${google_compute_address.coder["asia"].address}"
-  volumeMounts:
-  - mountPath: "/tmp"
-    name: cache
-    readOnly: false
-  volumes:
-  - emptyDir:
-      sizeLimit: 1024Mi
-    name: cache
-EOF
-  ]
+  values = [templatefile("${path.module}/coder_helm_values.tftpl", {
+    workspace_proxy = true,
+    provisionerd = false,
+    primary_url = local.deployments.primary.url,
+    proxy_token = kubernetes_secret.proxy_token_asia.metadata.0.name,
+    db_secret = null,
+    ip_address = google_compute_address.coder["asia"].address,
+    provisionerd_psk = null,
+    access_url = local.deployments.asia.url,
+    node_pool = google_container_node_pool.node_pool["asia_coder"].name,
+    release_name = local.coder_release_name,
+    experiments = var.coder_experiments,
+    image_repo = var.coder_image_repo,
+    image_tag = var.coder_image_tag,
+    replicas = local.scenarios[var.scenario].coder.replicas,
+    cpu_request = local.scenarios[var.scenario].coder.cpu_request,
+    mem_request = local.scenarios[var.scenario].coder.mem_request,
+    cpu_limit = local.scenarios[var.scenario].coder.cpu_limit,
+    mem_limit = local.scenarios[var.scenario].coder.mem_limit,
+  })]
 }
 
 resource "helm_release" "provisionerd_asia" {
@@ -149,66 +81,24 @@ resource "helm_release" "provisionerd_asia" {
   name       = local.provisionerd_release_name
   version    = var.provisionerd_chart_version
   namespace  = kubernetes_namespace.coder_asia.metadata.0.name
-  values = [<<EOF
-coder:
-  affinity:
-    nodeAffinity:
-      requiredDuringSchedulingIgnoredDuringExecution:
-        nodeSelectorTerms:
-        - matchExpressions:
-          - key: "cloud.google.com/gke-nodepool"
-            operator: "In"
-            values: ["${google_container_node_pool.node_pool["asia_coder"].name}"]
-    podAntiAffinity:
-      preferredDuringSchedulingIgnoredDuringExecution:
-      - weight: 1
-        podAffinityTerm:
-          topologyKey: "kubernetes.io/hostname"
-          labelSelector:
-            matchExpressions:
-            - key:      "app.kubernetes.io/instance"
-              operator: "In"
-              values:   ["${local.coder_release_name}"]
-  env:
-    - name: "CODER_URL"
-      value: "${local.deployments.primary.url}"
-    - name: "CODER_VERBOSE"
-      value: "true"
-    - name: "CODER_CONFIG_DIR"
-      value: "/tmp/config"
-    - name: "CODER_CACHE_DIRECTORY"
-      value: "/tmp/coder"
-    - name: "CODER_TELEMETRY_ENABLE"
-      value: "false"
-    - name: "CODER_LOGGING_HUMAN"
-      value: "/dev/null"
-    - name: "CODER_LOGGING_STACKDRIVER"
-      value: "/dev/stderr"
-    - name: "CODER_PROMETHEUS_ENABLE"
-      value: "true"
-    - name: "CODER_PROVISIONERD_TAGS"
-      value: "scope=organization"
-  image:
-    repo: ${var.provisionerd_image_repo}
-    tag: ${var.provisionerd_image_tag}
-  replicaCount: "${local.scenarios[var.scenario].provisionerd.replicas}"
-  resources:
-    requests:
-      cpu: "${local.scenarios[var.scenario].provisionerd.cpu_request}"
-      memory: "${local.scenarios[var.scenario].provisionerd.mem_request}"
-    limits:
-      cpu: "${local.scenarios[var.scenario].provisionerd.cpu_limit}"
-      memory: "${local.scenarios[var.scenario].provisionerd.mem_limit}"
-  securityContext:
-    readOnlyRootFilesystem: true
-  volumeMounts:
-  - mountPath: "/tmp"
-    name: cache
-    readOnly: false
-  volumes:
-  - emptyDir:
-      sizeLimit: 1024Mi
-    name: cache
-EOF
-  ]
+  values = [templatefile("${path.module}/coder_helm_values.tftpl", {
+    workspace_proxy = false,
+    provisionerd = true,
+    primary_url = null,
+    proxy_token = null,
+    db_secret = null,
+    ip_address = null,
+    provisionerd_psk = kubernetes_secret.provisionerd_psk_asia.metadata.0.name,
+    access_url = local.deployments.primary.url,
+    node_pool = google_container_node_pool.node_pool["asia_coder"].name,
+    release_name = local.coder_release_name,
+    experiments = var.coder_experiments,
+    image_repo = var.coder_image_repo,
+    image_tag = var.coder_image_tag,
+    replicas = local.scenarios[var.scenario].provisionerd.replicas,
+    cpu_request = local.scenarios[var.scenario].provisionerd.cpu_request,
+    mem_request = local.scenarios[var.scenario].provisionerd.mem_request,
+    cpu_limit = local.scenarios[var.scenario].provisionerd.cpu_limit,
+    mem_limit = local.scenarios[var.scenario].provisionerd.mem_limit,
+  })]
 }
diff --git a/scaletest/terraform/action/k8s_coder_europe.tf b/scaletest/terraform/action/k8s_coder_europe.tf
index 84c0d30d949b8..8a3c8bcaa96d3 100644
--- a/scaletest/terraform/action/k8s_coder_europe.tf
+++ b/scaletest/terraform/action/k8s_coder_europe.tf
@@ -51,94 +51,26 @@ resource "helm_release" "coder_europe" {
   name       = local.coder_release_name
   version    = var.coder_chart_version
   namespace  = kubernetes_namespace.coder_europe.metadata.0.name
-  values = [<<EOF
-coder:
-  workspaceProxy: true
-  affinity:
-    nodeAffinity:
-      requiredDuringSchedulingIgnoredDuringExecution:
-        nodeSelectorTerms:
-        - matchExpressions:
-          - key: "cloud.google.com/gke-nodepool"
-            operator: "In"
-            values: ["${google_container_node_pool.node_pool["europe_coder"].name}"]
-    podAntiAffinity:
-      preferredDuringSchedulingIgnoredDuringExecution:
-      - weight: 1
-        podAffinityTerm:
-          topologyKey: "kubernetes.io/hostname"
-          labelSelector:
-            matchExpressions:
-            - key:      "app.kubernetes.io/instance"
-              operator: "In"
-              values:   ["${local.coder_release_name}"]
-  env:
-    - name: CODER_PRIMARY_ACCESS_URL
-      value: "${local.deployments.primary.url}"
-    - name: CODER_PROXY_SESSION_TOKEN
-      valueFrom:
-        secretKeyRef:
-          key: token
-          name: "${kubernetes_secret.proxy_token_europe.metadata.0.name}"
-    - name: "CODER_ACCESS_URL"
-      value: "${local.deployments.europe.url}"
-    - name: "CODER_CACHE_DIRECTORY"
-      value: "/tmp/coder"
-    - name: "CODER_TELEMETRY_ENABLE"
-      value: "false"
-    - name: "CODER_LOGGING_HUMAN"
-      value: "/dev/null"
-    - name: "CODER_LOGGING_STACKDRIVER"
-      value: "/dev/stderr"
-    - name: "CODER_PPROF_ENABLE"
-      value: "true"
-    - name: "CODER_PROMETHEUS_ENABLE"
-      value: "true"
-    - name: "CODER_PROMETHEUS_COLLECT_AGENT_STATS"
-      value: "true"
-    - name: "CODER_PROMETHEUS_COLLECT_DB_METRICS"
-      value: "true"
-    - name: "CODER_VERBOSE"
-      value: "true"
-    - name: "CODER_EXPERIMENTS"
-      value: "${var.coder_experiments}"
-    - name: "CODER_DANGEROUS_DISABLE_RATE_LIMITS"
-      value: "true"
-    # Disabling built-in provisioner daemons
-    - name: "CODER_PROVISIONER_DAEMONS"
-      value: "0"
-    - name: CODER_PROVISIONER_DAEMON_PSK
-      valueFrom:
-        secretKeyRef:
-          key: psk
-          name: "${kubernetes_secret.provisionerd_psk_europe.metadata.0.name}"
-  image:
-    repo: ${var.coder_image_repo}
-    tag: ${var.coder_image_tag}
-  replicaCount: "${local.scenarios[var.scenario].coder.replicas}"
-  resources:
-    requests:
-      cpu: "${local.scenarios[var.scenario].coder.cpu_request}"
-      memory: "${local.scenarios[var.scenario].coder.mem_request}"
-    limits:
-      cpu: "${local.scenarios[var.scenario].coder.cpu_limit}"
-      memory: "${local.scenarios[var.scenario].coder.mem_limit}"
-  securityContext:
-    readOnlyRootFilesystem: true
-  service:
-    enable: true
-    sessionAffinity: None
-    loadBalancerIP: "${google_compute_address.coder["europe"].address}"
-  volumeMounts:
-  - mountPath: "/tmp"
-    name: cache
-    readOnly: false
-  volumes:
-  - emptyDir:
-      sizeLimit: 1024Mi
-    name: cache
-EOF
-  ]
+  values = [templatefile("${path.module}/coder_helm_values.tftpl", {
+    workspace_proxy = true,
+    provisionerd = false,
+    primary_url = local.deployments.primary.url,
+    proxy_token = kubernetes_secret.proxy_token_europe.metadata.0.name,
+    db_secret = null,
+    ip_address = google_compute_address.coder["europe"].address,
+    provisionerd_psk = null,
+    access_url = local.deployments.europe.url,
+    node_pool = google_container_node_pool.node_pool["europe_coder"].name,
+    release_name = local.coder_release_name,
+    experiments = var.coder_experiments,
+    image_repo = var.coder_image_repo,
+    image_tag = var.coder_image_tag,
+    replicas = local.scenarios[var.scenario].coder.replicas,
+    cpu_request = local.scenarios[var.scenario].coder.cpu_request,
+    mem_request = local.scenarios[var.scenario].coder.mem_request,
+    cpu_limit = local.scenarios[var.scenario].coder.cpu_limit,
+    mem_limit = local.scenarios[var.scenario].coder.mem_limit,
+  })]
 }
 
 resource "helm_release" "provisionerd_europe" {
@@ -149,66 +81,24 @@ resource "helm_release" "provisionerd_europe" {
   name       = local.provisionerd_release_name
   version    = var.provisionerd_chart_version
   namespace  = kubernetes_namespace.coder_europe.metadata.0.name
-  values = [<<EOF
-coder:
-  affinity:
-    nodeAffinity:
-      requiredDuringSchedulingIgnoredDuringExecution:
-        nodeSelectorTerms:
-        - matchExpressions:
-          - key: "cloud.google.com/gke-nodepool"
-            operator: "In"
-            values: ["${google_container_node_pool.node_pool["europe_coder"].name}"]
-    podAntiAffinity:
-      preferredDuringSchedulingIgnoredDuringExecution:
-      - weight: 1
-        podAffinityTerm:
-          topologyKey: "kubernetes.io/hostname"
-          labelSelector:
-            matchExpressions:
-            - key:      "app.kubernetes.io/instance"
-              operator: "In"
-              values:   ["${local.coder_release_name}"]
-  env:
-    - name: "CODER_URL"
-      value: "${local.deployments.primary.url}"
-    - name: "CODER_VERBOSE"
-      value: "true"
-    - name: "CODER_CONFIG_DIR"
-      value: "/tmp/config"
-    - name: "CODER_CACHE_DIRECTORY"
-      value: "/tmp/coder"
-    - name: "CODER_TELEMETRY_ENABLE"
-      value: "false"
-    - name: "CODER_LOGGING_HUMAN"
-      value: "/dev/null"
-    - name: "CODER_LOGGING_STACKDRIVER"
-      value: "/dev/stderr"
-    - name: "CODER_PROMETHEUS_ENABLE"
-      value: "true"
-    - name: "CODER_PROVISIONERD_TAGS"
-      value: "scope=organization"
-  image:
-    repo: ${var.provisionerd_image_repo}
-    tag: ${var.provisionerd_image_tag}
-  replicaCount: "${local.scenarios[var.scenario].provisionerd.replicas}"
-  resources:
-    requests:
-      cpu: "${local.scenarios[var.scenario].provisionerd.cpu_request}"
-      memory: "${local.scenarios[var.scenario].provisionerd.mem_request}"
-    limits:
-      cpu: "${local.scenarios[var.scenario].provisionerd.cpu_limit}"
-      memory: "${local.scenarios[var.scenario].provisionerd.mem_limit}"
-  securityContext:
-    readOnlyRootFilesystem: true
-  volumeMounts:
-  - mountPath: "/tmp"
-    name: cache
-    readOnly: false
-  volumes:
-  - emptyDir:
-      sizeLimit: 1024Mi
-    name: cache
-EOF
-  ]
+  values = [templatefile("${path.module}/coder_helm_values.tftpl", {
+    workspace_proxy = false,
+    provisionerd = true,
+    primary_url = null,
+    proxy_token = null,
+    db_secret = null,
+    ip_address = null,
+    provisionerd_psk = kubernetes_secret.provisionerd_psk_europe.metadata.0.name,
+    access_url = local.deployments.primary.url,
+    node_pool = google_container_node_pool.node_pool["europe_coder"].name,
+    release_name = local.coder_release_name,
+    experiments = var.coder_experiments,
+    image_repo = var.coder_image_repo,
+    image_tag = var.coder_image_tag,
+    replicas = local.scenarios[var.scenario].provisionerd.replicas,
+    cpu_request = local.scenarios[var.scenario].provisionerd.cpu_request,
+    mem_request = local.scenarios[var.scenario].provisionerd.mem_request,
+    cpu_limit = local.scenarios[var.scenario].provisionerd.cpu_limit,
+    mem_limit = local.scenarios[var.scenario].provisionerd.mem_limit,
+  })]
 }
diff --git a/scaletest/terraform/action/k8s_coder_primary.tf b/scaletest/terraform/action/k8s_coder_primary.tf
index 6e8eaa9db7b66..5988bba450a26 100644
--- a/scaletest/terraform/action/k8s_coder_primary.tf
+++ b/scaletest/terraform/action/k8s_coder_primary.tf
@@ -71,91 +71,26 @@ resource "helm_release" "coder_primary" {
   name       = local.coder_release_name
   version    = var.coder_chart_version
   namespace  = kubernetes_namespace.coder_primary.metadata.0.name
-  values = [<<EOF
-coder:
-  affinity:
-    nodeAffinity:
-      requiredDuringSchedulingIgnoredDuringExecution:
-        nodeSelectorTerms:
-        - matchExpressions:
-          - key: "cloud.google.com/gke-nodepool"
-            operator: "In"
-            values: ["${google_container_node_pool.node_pool["primary_coder"].name}"]
-    podAntiAffinity:
-      preferredDuringSchedulingIgnoredDuringExecution:
-      - weight: 1
-        podAffinityTerm:
-          topologyKey: "kubernetes.io/hostname"
-          labelSelector:
-            matchExpressions:
-            - key:      "app.kubernetes.io/instance"
-              operator: "In"
-              values:   ["${local.coder_release_name}"]
-  env:
-    - name: "CODER_ACCESS_URL"
-      value: "${local.deployments.primary.url}"
-    - name: "CODER_CACHE_DIRECTORY"
-      value: "/tmp/coder"
-    - name: "CODER_TELEMETRY_ENABLE"
-      value: "false"
-    - name: "CODER_LOGGING_HUMAN"
-      value: "/dev/null"
-    - name: "CODER_LOGGING_STACKDRIVER"
-      value: "/dev/stderr"
-    - name: "CODER_PG_CONNECTION_URL"
-      valueFrom:
-        secretKeyRef:
-          name: "${kubernetes_secret.coder_db.metadata.0.name}"
-          key: url
-    - name: "CODER_PPROF_ENABLE"
-      value: "true"
-    - name: "CODER_PROMETHEUS_ENABLE"
-      value: "true"
-    - name: "CODER_PROMETHEUS_COLLECT_AGENT_STATS"
-      value: "true"
-    - name: "CODER_PROMETHEUS_COLLECT_DB_METRICS"
-      value: "true"
-    - name: "CODER_VERBOSE"
-      value: "true"
-    - name: "CODER_EXPERIMENTS"
-      value: "${var.coder_experiments}"
-    - name: "CODER_DANGEROUS_DISABLE_RATE_LIMITS"
-      value: "true"
-    # Disabling built-in provisioner daemons
-    - name: "CODER_PROVISIONER_DAEMONS"
-      value: "0"
-    - name: CODER_PROVISIONER_DAEMON_PSK
-      valueFrom:
-        secretKeyRef:
-          key: psk
-          name: "${kubernetes_secret.provisionerd_psk_primary.metadata.0.name}"
-  image:
-    repo: ${var.coder_image_repo}
-    tag: ${var.coder_image_tag}
-  replicaCount: "${local.scenarios[var.scenario].coder.replicas}"
-  resources:
-    requests:
-      cpu: "${local.scenarios[var.scenario].coder.cpu_request}"
-      memory: "${local.scenarios[var.scenario].coder.mem_request}"
-    limits:
-      cpu: "${local.scenarios[var.scenario].coder.cpu_limit}"
-      memory: "${local.scenarios[var.scenario].coder.mem_limit}"
-  securityContext:
-    readOnlyRootFilesystem: true
-  service:
-    enable: true
-    sessionAffinity: None
-    loadBalancerIP: "${google_compute_address.coder["primary"].address}"
-  volumeMounts:
-  - mountPath: "/tmp"
-    name: cache
-    readOnly: false
-  volumes:
-  - emptyDir:
-      sizeLimit: 1024Mi
-    name: cache
-EOF
-  ]
+  values = [templatefile("${path.module}/coder_helm_values.tftpl", {
+    workspace_proxy = false,
+    provisionerd = false,
+    primary_url = null,
+    proxy_token = null,
+    db_secret = kubernetes_secret.coder_db.metadata.0.name,
+    ip_address = google_compute_address.coder["primary"].address,
+    provisionerd_psk = kubernetes_secret.provisionerd_psk_primary.metadata.0.name,
+    access_url = local.deployments.primary.url,
+    node_pool = google_container_node_pool.node_pool["primary_coder"].name,
+    release_name = local.coder_release_name,
+    experiments = var.coder_experiments,
+    image_repo = var.coder_image_repo,
+    image_tag = var.coder_image_tag,
+    replicas = local.scenarios[var.scenario].coder.replicas,
+    cpu_request = local.scenarios[var.scenario].coder.cpu_request,
+    mem_request = local.scenarios[var.scenario].coder.mem_request,
+    cpu_limit = local.scenarios[var.scenario].coder.cpu_limit,
+    mem_limit = local.scenarios[var.scenario].coder.mem_limit,
+  })]
 }
 
 resource "helm_release" "provisionerd_chart" {
@@ -166,66 +101,24 @@ resource "helm_release" "provisionerd_chart" {
   name       = local.provisionerd_release_name
   version    = var.provisionerd_chart_version
   namespace  = kubernetes_namespace.coder_primary.metadata.0.name
-  values = [<<EOF
-coder:
-  affinity:
-    nodeAffinity:
-      requiredDuringSchedulingIgnoredDuringExecution:
-        nodeSelectorTerms:
-        - matchExpressions:
-          - key: "cloud.google.com/gke-nodepool"
-            operator: "In"
-            values: ["${google_container_node_pool.node_pool["primary_coder"].name}"]
-    podAntiAffinity:
-      preferredDuringSchedulingIgnoredDuringExecution:
-      - weight: 1
-        podAffinityTerm:
-          topologyKey: "kubernetes.io/hostname"
-          labelSelector:
-            matchExpressions:
-            - key:      "app.kubernetes.io/instance"
-              operator: "In"
-              values:   ["${local.coder_release_name}"]
-  env:
-    - name: "CODER_URL"
-      value: "${local.deployments.primary.url}"
-    - name: "CODER_VERBOSE"
-      value: "true"
-    - name: "CODER_CONFIG_DIR"
-      value: "/tmp/config"
-    - name: "CODER_CACHE_DIRECTORY"
-      value: "/tmp/coder"
-    - name: "CODER_TELEMETRY_ENABLE"
-      value: "false"
-    - name: "CODER_LOGGING_HUMAN"
-      value: "/dev/null"
-    - name: "CODER_LOGGING_STACKDRIVER"
-      value: "/dev/stderr"
-    - name: "CODER_PROMETHEUS_ENABLE"
-      value: "true"
-    - name: "CODER_PROVISIONERD_TAGS"
-      value: "scope=organization"
-  image:
-    repo: ${var.provisionerd_image_repo}
-    tag: ${var.provisionerd_image_tag}
-  replicaCount: "${local.scenarios[var.scenario].provisionerd.replicas}"
-  resources:
-    requests:
-      cpu: "${local.scenarios[var.scenario].provisionerd.cpu_request}"
-      memory: "${local.scenarios[var.scenario].provisionerd.mem_request}"
-    limits:
-      cpu: "${local.scenarios[var.scenario].provisionerd.cpu_limit}"
-      memory: "${local.scenarios[var.scenario].provisionerd.mem_limit}"
-  securityContext:
-    readOnlyRootFilesystem: true
-  volumeMounts:
-  - mountPath: "/tmp"
-    name: cache
-    readOnly: false
-  volumes:
-  - emptyDir:
-      sizeLimit: 1024Mi
-    name: cache
-EOF
-  ]
+  values = [templatefile("${path.module}/coder_helm_values.tftpl", {
+    workspace_proxy = false,
+    provisionerd = true,
+    primary_url = null,
+    proxy_token = null,
+    db_secret = null,
+    ip_address = null,
+    provisionerd_psk = kubernetes_secret.provisionerd_psk_primary.metadata.0.name,
+    access_url = local.deployments.primary.url,
+    node_pool = google_container_node_pool.node_pool["primary_coder"].name,
+    release_name = local.coder_release_name,
+    experiments = var.coder_experiments,
+    image_repo = var.coder_image_repo,
+    image_tag = var.coder_image_tag,
+    replicas = local.scenarios[var.scenario].provisionerd.replicas,
+    cpu_request = local.scenarios[var.scenario].provisionerd.cpu_request,
+    mem_request = local.scenarios[var.scenario].provisionerd.mem_request,
+    cpu_limit = local.scenarios[var.scenario].provisionerd.cpu_limit,
+    mem_limit = local.scenarios[var.scenario].provisionerd.mem_limit,
+  })]
 }

From 9c1a6e91324370d5385d8f8060a2e44aee6193dd Mon Sep 17 00:00:00 2001
From: Garrett Delfosse <garrett@coder.com>
Date: Wed, 11 Dec 2024 21:30:33 +0000
Subject: [PATCH 23/24] parity

---
 scaletest/terraform/action/main.tf            |   5 -
 scaletest/terraform/action/vars.tf            |  21 ----
 .../terraform/action/workspace_traffic.tf     | 109 ------------------
 3 files changed, 135 deletions(-)
 delete mode 100644 scaletest/terraform/action/workspace_traffic.tf

diff --git a/scaletest/terraform/action/main.tf b/scaletest/terraform/action/main.tf
index 4c7e5fdcbb217..57a294710c5b5 100644
--- a/scaletest/terraform/action/main.tf
+++ b/scaletest/terraform/action/main.tf
@@ -38,11 +38,6 @@ terraform {
       source  = "cloudflare/cloudflare"
       version = "~> 4.0"
     }
-
-    coderd = {
-      source  = "coder/coderd"
-      version = "~> 0.0.8"
-    }
   }
 
   required_version = "~> 1.9.0"
diff --git a/scaletest/terraform/action/vars.tf b/scaletest/terraform/action/vars.tf
index ddd8162c7f570..264110e239845 100644
--- a/scaletest/terraform/action/vars.tf
+++ b/scaletest/terraform/action/vars.tf
@@ -85,24 +85,3 @@ variable "provisionerd_image_tag" {
   description = "Tag to use for Provisionerd image."
   default     = "latest"
 }
-
-// Traffic
-# variable "traffic_bytes_per_tick" {
-#   description = "Number of bytes to send per tick."
-#   default     = 1024
-# }
-
-# variable "traffic_tick_interval" {
-#   description = "Interval between ticks."
-#   default     = "10s"
-# }
-
-variable "workspace_count" {
-  description = "Number of workspaces to create."
-  default     = 10
-}
-
-variable "workspace_create_concurrency" {
-  description = "Number of workspaces to create concurrently."
-  default     = 10
-}
diff --git a/scaletest/terraform/action/workspace_traffic.tf b/scaletest/terraform/action/workspace_traffic.tf
deleted file mode 100644
index a0db132725456..0000000000000
--- a/scaletest/terraform/action/workspace_traffic.tf
+++ /dev/null
@@ -1,109 +0,0 @@
-# resource "kubernetes_pod" "create_workspaces" {
-#   provider = kubernetes.primary
-
-#   metadata {
-#     name      = "${var.name}-create-workspaces"
-#     namespace = kubernetes_namespace.coder_primary.metadata.0.name
-#     labels = {
-#       "app.kubernetes.io/name" = "${var.name}-create-workspaces"
-#     }
-#   }
-#   spec {
-#     affinity {
-#       node_affinity {
-#         required_during_scheduling_ignored_during_execution {
-#           node_selector_term {
-#             match_expressions {
-#               key      = "cloud.google.com/gke-nodepool"
-#               operator = "In"
-#               values   = ["${google_container_node_pool.node_pool["primary_misc"].name}"]
-#             }
-#           }
-#         }
-#       }
-#     }
-#     container {
-#       name    = "cli"
-#       image   = "${var.coder_image_repo}:${var.coder_image_tag}"
-#       command = [
-#         "/opt/coder",
-#         "--verbose",
-#         "--url=${local.deployments.primary.url}",
-#         "--token=${trimspace(data.local_file.api_key.content)}",
-#         "exp",
-#         "scaletest",
-#         "create-workspaces",
-#         "--count=${var.workspace_count}",
-#         "--template=kubernetes",
-#         "--concurrency=${var.workspace_create_concurrency}",
-#         "--no-cleanup"
-#       ]
-#     }
-#     restart_policy = "Never"
-#   }
-
-#   depends_on = [ coderd_template.kubernetes ]
-# }
-
-# resource "time_sleep" "wait_for_baseline" {
-#   depends_on = [kubernetes_pod.create_workspaces]
-
-#   create_duration = "600s"
-# }
-
-# resource "kubernetes_pod" "workspace_traffic_primary" {
-#   provider = kubernetes.primary
-
-#   metadata {
-#     name      = "${var.name}-traffic"
-#     namespace = kubernetes_namespace.coder.metadata.0.name
-#     labels = {
-#       "app.kubernetes.io/name" = "${var.name}-traffic"
-#     }
-#   }
-#   spec {
-#     affinity {
-#       node_affinity {
-#         required_during_scheduling_ignored_during_execution {
-#           node_selector_term {
-#             match_expressions {
-#               key      = "cloud.google.com/gke-nodepool"
-#               operator = "In"
-#               values   = ["${google_container_node_pool.node_pool["primary_misc"].name}"]
-#             }
-#           }
-#         }
-#       }
-#     }
-#     container {
-#       name    = "cli"
-#       image   = "${var.coder_image_repo}:${var.coder_image_tag}"
-#       command = ["/opt/coder --verbose --url=${local.deployments.primary.url} --token=${trimspace(local_file.api_key.content)} exp scaletest workspace-traffic --concurrency=0 --bytes-per-tick=${var.traffic_bytes_per_tick} --tick-interval=${var.traffic_tick_interval} --scaletest-prometheus-wait=60s"]
-
-#       env {
-#         name  = "CODER_URL"
-#         value = local.deployments.primary.url
-#       }
-#       env {
-#         name  = "CODER_TOKEN"
-#         value = trimspace(local_file.api_key.content)
-#       }
-#       env {
-#         name  = "CODER_SCALETEST_PROMETHEUS_ADDRESS"
-#         value = "0.0.0.0:21112"
-#       }
-#       env {
-#         name  = "CODER_SCALETEST_JOB_TIMEOUT"
-#         value = "30m"
-#       }
-#       port {
-#         container_port = 21112
-#         name           = "prometheus-http"
-#         protocol       = "TCP"
-#       }
-#     }
-#     restart_policy = "Never"
-#   }
-
-#   depends_on = [time_sleep.wait_for_baseline]
-# }

From df36247465225cf771e3cc0cea3846a90109744f Mon Sep 17 00:00:00 2001
From: Garrett Delfosse <garrett@coder.com>
Date: Wed, 11 Dec 2024 21:37:28 +0000
Subject: [PATCH 24/24] fmt

---
 scaletest/terraform/action/k8s_coder_asia.tf  | 68 +++++++++----------
 .../terraform/action/k8s_coder_europe.tf      | 68 +++++++++----------
 .../terraform/action/k8s_coder_primary.tf     | 68 +++++++++----------
 3 files changed, 102 insertions(+), 102 deletions(-)

diff --git a/scaletest/terraform/action/k8s_coder_asia.tf b/scaletest/terraform/action/k8s_coder_asia.tf
index b5b2323113adc..2d22173498e85 100644
--- a/scaletest/terraform/action/k8s_coder_asia.tf
+++ b/scaletest/terraform/action/k8s_coder_asia.tf
@@ -52,24 +52,24 @@ resource "helm_release" "coder_asia" {
   version    = var.coder_chart_version
   namespace  = kubernetes_namespace.coder_asia.metadata.0.name
   values = [templatefile("${path.module}/coder_helm_values.tftpl", {
-    workspace_proxy = true,
-    provisionerd = false,
-    primary_url = local.deployments.primary.url,
-    proxy_token = kubernetes_secret.proxy_token_asia.metadata.0.name,
-    db_secret = null,
-    ip_address = google_compute_address.coder["asia"].address,
+    workspace_proxy  = true,
+    provisionerd     = false,
+    primary_url      = local.deployments.primary.url,
+    proxy_token      = kubernetes_secret.proxy_token_asia.metadata.0.name,
+    db_secret        = null,
+    ip_address       = google_compute_address.coder["asia"].address,
     provisionerd_psk = null,
-    access_url = local.deployments.asia.url,
-    node_pool = google_container_node_pool.node_pool["asia_coder"].name,
-    release_name = local.coder_release_name,
-    experiments = var.coder_experiments,
-    image_repo = var.coder_image_repo,
-    image_tag = var.coder_image_tag,
-    replicas = local.scenarios[var.scenario].coder.replicas,
-    cpu_request = local.scenarios[var.scenario].coder.cpu_request,
-    mem_request = local.scenarios[var.scenario].coder.mem_request,
-    cpu_limit = local.scenarios[var.scenario].coder.cpu_limit,
-    mem_limit = local.scenarios[var.scenario].coder.mem_limit,
+    access_url       = local.deployments.asia.url,
+    node_pool        = google_container_node_pool.node_pool["asia_coder"].name,
+    release_name     = local.coder_release_name,
+    experiments      = var.coder_experiments,
+    image_repo       = var.coder_image_repo,
+    image_tag        = var.coder_image_tag,
+    replicas         = local.scenarios[var.scenario].coder.replicas,
+    cpu_request      = local.scenarios[var.scenario].coder.cpu_request,
+    mem_request      = local.scenarios[var.scenario].coder.mem_request,
+    cpu_limit        = local.scenarios[var.scenario].coder.cpu_limit,
+    mem_limit        = local.scenarios[var.scenario].coder.mem_limit,
   })]
 }
 
@@ -82,23 +82,23 @@ resource "helm_release" "provisionerd_asia" {
   version    = var.provisionerd_chart_version
   namespace  = kubernetes_namespace.coder_asia.metadata.0.name
   values = [templatefile("${path.module}/coder_helm_values.tftpl", {
-    workspace_proxy = false,
-    provisionerd = true,
-    primary_url = null,
-    proxy_token = null,
-    db_secret = null,
-    ip_address = null,
+    workspace_proxy  = false,
+    provisionerd     = true,
+    primary_url      = null,
+    proxy_token      = null,
+    db_secret        = null,
+    ip_address       = null,
     provisionerd_psk = kubernetes_secret.provisionerd_psk_asia.metadata.0.name,
-    access_url = local.deployments.primary.url,
-    node_pool = google_container_node_pool.node_pool["asia_coder"].name,
-    release_name = local.coder_release_name,
-    experiments = var.coder_experiments,
-    image_repo = var.coder_image_repo,
-    image_tag = var.coder_image_tag,
-    replicas = local.scenarios[var.scenario].provisionerd.replicas,
-    cpu_request = local.scenarios[var.scenario].provisionerd.cpu_request,
-    mem_request = local.scenarios[var.scenario].provisionerd.mem_request,
-    cpu_limit = local.scenarios[var.scenario].provisionerd.cpu_limit,
-    mem_limit = local.scenarios[var.scenario].provisionerd.mem_limit,
+    access_url       = local.deployments.primary.url,
+    node_pool        = google_container_node_pool.node_pool["asia_coder"].name,
+    release_name     = local.coder_release_name,
+    experiments      = var.coder_experiments,
+    image_repo       = var.coder_image_repo,
+    image_tag        = var.coder_image_tag,
+    replicas         = local.scenarios[var.scenario].provisionerd.replicas,
+    cpu_request      = local.scenarios[var.scenario].provisionerd.cpu_request,
+    mem_request      = local.scenarios[var.scenario].provisionerd.mem_request,
+    cpu_limit        = local.scenarios[var.scenario].provisionerd.cpu_limit,
+    mem_limit        = local.scenarios[var.scenario].provisionerd.mem_limit,
   })]
 }
diff --git a/scaletest/terraform/action/k8s_coder_europe.tf b/scaletest/terraform/action/k8s_coder_europe.tf
index 8a3c8bcaa96d3..bb6140aef7ea6 100644
--- a/scaletest/terraform/action/k8s_coder_europe.tf
+++ b/scaletest/terraform/action/k8s_coder_europe.tf
@@ -52,24 +52,24 @@ resource "helm_release" "coder_europe" {
   version    = var.coder_chart_version
   namespace  = kubernetes_namespace.coder_europe.metadata.0.name
   values = [templatefile("${path.module}/coder_helm_values.tftpl", {
-    workspace_proxy = true,
-    provisionerd = false,
-    primary_url = local.deployments.primary.url,
-    proxy_token = kubernetes_secret.proxy_token_europe.metadata.0.name,
-    db_secret = null,
-    ip_address = google_compute_address.coder["europe"].address,
+    workspace_proxy  = true,
+    provisionerd     = false,
+    primary_url      = local.deployments.primary.url,
+    proxy_token      = kubernetes_secret.proxy_token_europe.metadata.0.name,
+    db_secret        = null,
+    ip_address       = google_compute_address.coder["europe"].address,
     provisionerd_psk = null,
-    access_url = local.deployments.europe.url,
-    node_pool = google_container_node_pool.node_pool["europe_coder"].name,
-    release_name = local.coder_release_name,
-    experiments = var.coder_experiments,
-    image_repo = var.coder_image_repo,
-    image_tag = var.coder_image_tag,
-    replicas = local.scenarios[var.scenario].coder.replicas,
-    cpu_request = local.scenarios[var.scenario].coder.cpu_request,
-    mem_request = local.scenarios[var.scenario].coder.mem_request,
-    cpu_limit = local.scenarios[var.scenario].coder.cpu_limit,
-    mem_limit = local.scenarios[var.scenario].coder.mem_limit,
+    access_url       = local.deployments.europe.url,
+    node_pool        = google_container_node_pool.node_pool["europe_coder"].name,
+    release_name     = local.coder_release_name,
+    experiments      = var.coder_experiments,
+    image_repo       = var.coder_image_repo,
+    image_tag        = var.coder_image_tag,
+    replicas         = local.scenarios[var.scenario].coder.replicas,
+    cpu_request      = local.scenarios[var.scenario].coder.cpu_request,
+    mem_request      = local.scenarios[var.scenario].coder.mem_request,
+    cpu_limit        = local.scenarios[var.scenario].coder.cpu_limit,
+    mem_limit        = local.scenarios[var.scenario].coder.mem_limit,
   })]
 }
 
@@ -82,23 +82,23 @@ resource "helm_release" "provisionerd_europe" {
   version    = var.provisionerd_chart_version
   namespace  = kubernetes_namespace.coder_europe.metadata.0.name
   values = [templatefile("${path.module}/coder_helm_values.tftpl", {
-    workspace_proxy = false,
-    provisionerd = true,
-    primary_url = null,
-    proxy_token = null,
-    db_secret = null,
-    ip_address = null,
+    workspace_proxy  = false,
+    provisionerd     = true,
+    primary_url      = null,
+    proxy_token      = null,
+    db_secret        = null,
+    ip_address       = null,
     provisionerd_psk = kubernetes_secret.provisionerd_psk_europe.metadata.0.name,
-    access_url = local.deployments.primary.url,
-    node_pool = google_container_node_pool.node_pool["europe_coder"].name,
-    release_name = local.coder_release_name,
-    experiments = var.coder_experiments,
-    image_repo = var.coder_image_repo,
-    image_tag = var.coder_image_tag,
-    replicas = local.scenarios[var.scenario].provisionerd.replicas,
-    cpu_request = local.scenarios[var.scenario].provisionerd.cpu_request,
-    mem_request = local.scenarios[var.scenario].provisionerd.mem_request,
-    cpu_limit = local.scenarios[var.scenario].provisionerd.cpu_limit,
-    mem_limit = local.scenarios[var.scenario].provisionerd.mem_limit,
+    access_url       = local.deployments.primary.url,
+    node_pool        = google_container_node_pool.node_pool["europe_coder"].name,
+    release_name     = local.coder_release_name,
+    experiments      = var.coder_experiments,
+    image_repo       = var.coder_image_repo,
+    image_tag        = var.coder_image_tag,
+    replicas         = local.scenarios[var.scenario].provisionerd.replicas,
+    cpu_request      = local.scenarios[var.scenario].provisionerd.cpu_request,
+    mem_request      = local.scenarios[var.scenario].provisionerd.mem_request,
+    cpu_limit        = local.scenarios[var.scenario].provisionerd.cpu_limit,
+    mem_limit        = local.scenarios[var.scenario].provisionerd.mem_limit,
   })]
 }
diff --git a/scaletest/terraform/action/k8s_coder_primary.tf b/scaletest/terraform/action/k8s_coder_primary.tf
index 5988bba450a26..68dc7fc80a561 100644
--- a/scaletest/terraform/action/k8s_coder_primary.tf
+++ b/scaletest/terraform/action/k8s_coder_primary.tf
@@ -72,24 +72,24 @@ resource "helm_release" "coder_primary" {
   version    = var.coder_chart_version
   namespace  = kubernetes_namespace.coder_primary.metadata.0.name
   values = [templatefile("${path.module}/coder_helm_values.tftpl", {
-    workspace_proxy = false,
-    provisionerd = false,
-    primary_url = null,
-    proxy_token = null,
-    db_secret = kubernetes_secret.coder_db.metadata.0.name,
-    ip_address = google_compute_address.coder["primary"].address,
+    workspace_proxy  = false,
+    provisionerd     = false,
+    primary_url      = null,
+    proxy_token      = null,
+    db_secret        = kubernetes_secret.coder_db.metadata.0.name,
+    ip_address       = google_compute_address.coder["primary"].address,
     provisionerd_psk = kubernetes_secret.provisionerd_psk_primary.metadata.0.name,
-    access_url = local.deployments.primary.url,
-    node_pool = google_container_node_pool.node_pool["primary_coder"].name,
-    release_name = local.coder_release_name,
-    experiments = var.coder_experiments,
-    image_repo = var.coder_image_repo,
-    image_tag = var.coder_image_tag,
-    replicas = local.scenarios[var.scenario].coder.replicas,
-    cpu_request = local.scenarios[var.scenario].coder.cpu_request,
-    mem_request = local.scenarios[var.scenario].coder.mem_request,
-    cpu_limit = local.scenarios[var.scenario].coder.cpu_limit,
-    mem_limit = local.scenarios[var.scenario].coder.mem_limit,
+    access_url       = local.deployments.primary.url,
+    node_pool        = google_container_node_pool.node_pool["primary_coder"].name,
+    release_name     = local.coder_release_name,
+    experiments      = var.coder_experiments,
+    image_repo       = var.coder_image_repo,
+    image_tag        = var.coder_image_tag,
+    replicas         = local.scenarios[var.scenario].coder.replicas,
+    cpu_request      = local.scenarios[var.scenario].coder.cpu_request,
+    mem_request      = local.scenarios[var.scenario].coder.mem_request,
+    cpu_limit        = local.scenarios[var.scenario].coder.cpu_limit,
+    mem_limit        = local.scenarios[var.scenario].coder.mem_limit,
   })]
 }
 
@@ -102,23 +102,23 @@ resource "helm_release" "provisionerd_chart" {
   version    = var.provisionerd_chart_version
   namespace  = kubernetes_namespace.coder_primary.metadata.0.name
   values = [templatefile("${path.module}/coder_helm_values.tftpl", {
-    workspace_proxy = false,
-    provisionerd = true,
-    primary_url = null,
-    proxy_token = null,
-    db_secret = null,
-    ip_address = null,
+    workspace_proxy  = false,
+    provisionerd     = true,
+    primary_url      = null,
+    proxy_token      = null,
+    db_secret        = null,
+    ip_address       = null,
     provisionerd_psk = kubernetes_secret.provisionerd_psk_primary.metadata.0.name,
-    access_url = local.deployments.primary.url,
-    node_pool = google_container_node_pool.node_pool["primary_coder"].name,
-    release_name = local.coder_release_name,
-    experiments = var.coder_experiments,
-    image_repo = var.coder_image_repo,
-    image_tag = var.coder_image_tag,
-    replicas = local.scenarios[var.scenario].provisionerd.replicas,
-    cpu_request = local.scenarios[var.scenario].provisionerd.cpu_request,
-    mem_request = local.scenarios[var.scenario].provisionerd.mem_request,
-    cpu_limit = local.scenarios[var.scenario].provisionerd.cpu_limit,
-    mem_limit = local.scenarios[var.scenario].provisionerd.mem_limit,
+    access_url       = local.deployments.primary.url,
+    node_pool        = google_container_node_pool.node_pool["primary_coder"].name,
+    release_name     = local.coder_release_name,
+    experiments      = var.coder_experiments,
+    image_repo       = var.coder_image_repo,
+    image_tag        = var.coder_image_tag,
+    replicas         = local.scenarios[var.scenario].provisionerd.replicas,
+    cpu_request      = local.scenarios[var.scenario].provisionerd.cpu_request,
+    mem_request      = local.scenarios[var.scenario].provisionerd.mem_request,
+    cpu_limit        = local.scenarios[var.scenario].provisionerd.cpu_limit,
+    mem_limit        = local.scenarios[var.scenario].provisionerd.mem_limit,
   })]
 }