coder · johnstcn · Sep 22, 2023 · Sep 22, 2023 · Sep 22, 2023 · Sep 22, 2023
diff --git a/scaletest/terraform/gcp_cluster.tf → scaletest/terraform/infra/gcp_cluster.tf b/scaletest/terraform/gcp_cluster.tf → scaletest/terraform/infra/gcp_cluster.tf
@@ -3,7 +3,9 @@ data "google_compute_default_service_account" "default" {
 }
 
 locals {
-  cluster_kubeconfig_path = "${abspath(path.module)}/../.coderv2/${var.name}-cluster.kubeconfig"
+  abs_module_path         = abspath(path.module)
+  rel_kubeconfig_path     = "../../.coderv2/${var.name}-cluster.kubeconfig"
+  cluster_kubeconfig_path = abspath("${local.abs_module_path}/${local.rel_kubeconfig_path}")
 }
 
 resource "google_container_cluster" "primary" {

diff --git a/scaletest/terraform/gcp_db.tf → scaletest/terraform/infra/gcp_db.tf b/scaletest/terraform/gcp_db.tf → scaletest/terraform/infra/gcp_db.tf
@@ -42,6 +42,14 @@ resource "google_sql_database" "coder" {
   deletion_policy = "ABANDON"
 }
 
+resource "random_password" "coder-postgres-password" {
+  length = 12
+}
+
+resource "random_password" "prometheus-postgres-password" {
+  length = 12
+}
+
 resource "google_sql_user" "coder" {
   project  = var.project_id
   instance = google_sql_database_instance.db.id
@@ -61,3 +69,7 @@ resource "google_sql_user" "prometheus" {
   # required for postgres, otherwise user fails to delete
   deletion_policy = "ABANDON"
 }
+
+locals {
+  coder_db_url = "postgres://${google_sql_user.coder.name}:${urlencode(random_password.coder-postgres-password.result)}@${google_sql_database_instance.db.private_ip_address}/${google_sql_database.coder.name}?sslmode=disable"
+}
diff --git a/scaletest/terraform/gcp_project.tf → scaletest/terraform/infra/gcp_project.tf b/scaletest/terraform/gcp_project.tf → scaletest/terraform/infra/gcp_project.tf
@@ -1,8 +1,3 @@
-provider "google" {
-  region  = var.region
-  project = var.project_id
-}
-
 locals {
   project_apis = [
     "cloudtrace",

diff --git a/scaletest/terraform/gcp_vpc.tf → scaletest/terraform/infra/gcp_vpc.tf b/scaletest/terraform/gcp_vpc.tf → scaletest/terraform/infra/gcp_vpc.tf
diff --git a/scaletest/terraform/infra/main.tf b/scaletest/terraform/infra/main.tf
@@ -0,0 +1,20 @@
+terraform {
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "~> 4.36"
+    }
+
+    random = {
+      source  = "hashicorp/random"
+      version = "~> 3.5"
+    }
+  }
+
+  required_version = "~> 1.5.0"
+}
+
+provider "google" {
+  region  = var.region
+  project = var.project_id
+}
diff --git a/scaletest/terraform/infra/outputs.tf b/scaletest/terraform/infra/outputs.tf
@@ -0,0 +1,73 @@
+output "coder_db_url" {
+  description = "URL of the database for Coder."
+  value       = local.coder_db_url
+  sensitive   = true
+}
+
+output "coder_address" {
+  description = "IP address to use for the Coder service."
+  value       = google_compute_address.coder.address
+}
+
+output "kubernetes_kubeconfig_path" {
+  description = "Kubeconfig path."
+  value       = local.cluster_kubeconfig_path
+}
+
+output "kubernetes_nodepool_coder" {
+  description = "Name of the nodepool on which to run Coder."
+  value       = google_container_node_pool.coder.name
+}
+
+output "kubernetes_nodepool_misc" {
+  description = "Name of the nodepool on which to run everything else."
+  value       = google_container_node_pool.misc.name
+}
+
+output "kubernetes_nodepool_workspaces" {
+  description = "Name of the nodepool on which to run workspaces."
+  value       = google_container_node_pool.workspaces.name
+}
+
+output "prometheus_external_label_cluster" {
+  description = "Value for the Prometheus external label named cluster."
+  value       = google_container_cluster.primary.name
+}
+
+output "prometheus_postgres_dbname" {
+  description = "Name of the database for Prometheus to monitor."
+  value       = google_sql_database.coder.name
+}
+
+output "prometheus_postgres_host" {
+  description = "Hostname of the database for Prometheus to connect to."
+  value       = google_sql_database_instance.db.private_ip_address
+}
+
+output "prometheus_postgres_password" {
+  description = "Postgres password for Prometheus."
+  value       = random_password.prometheus-postgres-password.result
+  sensitive   = true
+}
+
+output "prometheus_postgres_user" {
+  description = "Postgres username for Prometheus."
+  value       = google_sql_user.prometheus.name
+}
+
+resource "local_file" "outputs" {
+  filename = "${path.module}/../../.coderv2/infra_outputs.tfvars"
+  content  = <<EOF
+  coder_db_url = "${local.coder_db_url}"
+  coder_address = "${google_compute_address.coder.address}"
+  kubernetes_kubeconfig_path = "${local.cluster_kubeconfig_path}"
+  kubernetes_nodepool_coder = "${google_container_node_pool.coder.name}"
+  kubernetes_nodepool_misc = "${google_container_node_pool.misc.name}"
+  kubernetes_nodepool_workspaces = "${google_container_node_pool.workspaces.name}"
+  prometheus_external_label_cluster = "${google_container_cluster.primary.name}"
+  prometheus_postgres_dbname = "${google_sql_database.coder.name}"
+  prometheus_postgres_host = "${google_sql_database_instance.db.private_ip_address}"
+  prometheus_postgres_password = "${random_password.prometheus-postgres-password.result}"
+  prometheus_postgres_user = "${google_sql_user.prometheus.name}"
+EOF
+}
diff --git a/scaletest/terraform/infra/vars.tf b/scaletest/terraform/infra/vars.tf
@@ -0,0 +1,102 @@
+variable "state" {
+  description = "The state of the cluster. Valid values are 'started', and 'stopped'."
+  validation {
+    condition     = contains(["started", "stopped"], var.state)
+    error_message = "value must be one of 'started' or 'stopped'"
+  }
+  default = "started"
+}
+
+variable "project_id" {
+  description = "The project in which to provision resources"
+}
+
+variable "name" {
+  description = "Adds a prefix to resources."
+}
+
+variable "region" {
+  description = "GCP region in which to provision resources."
+  default     = "us-east1"
+}
+
+variable "zone" {
+  description = "GCP zone in which to provision resources."
+  default     = "us-east1-c"
+}
+
+variable "k8s_version" {
+  description = "Kubernetes version to provision."
+  default     = "1.24"
+}
+
+variable "node_disk_size_gb" {
+  description = "Size of the root disk for cluster nodes."
+  default     = 100
+}
+
+variable "node_image_type" {
+  description = "Image type to use for cluster nodes."
+  default     = "cos_containerd"
+}
+
+// Preemptible nodes are way cheaper, but can be pulled out
+// from under you at any time. Caveat emptor.
+variable "node_preemptible" {
+  description = "Use preemptible nodes."
+  default     = false
+}
+
+// We create three nodepools:
+// - One for the Coder control plane
+// - One for workspaces
+// - One for everything else (for example, load generation)
+
+// These variables control the node pool dedicated to Coder.
+variable "nodepool_machine_type_coder" {
+  description = "Machine type to use for Coder control plane nodepool."
+  default     = "t2d-standard-4"
+}
+
+variable "nodepool_size_coder" {
+  description = "Number of cluster nodes for the Coder control plane nodepool."
+  default     = 1
+}
+
+// These variables control the node pool dedicated to workspaces.
+variable "nodepool_machine_type_workspaces" {
+  description = "Machine type to use for the workspaces nodepool."
+  default     = "t2d-standard-4"
+}
+
+variable "nodepool_size_workspaces" {
+  description = "Number of cluster nodes for the workspaces nodepool."
+  default     = 1
+}
+
+// These variables control the node pool for everything else.
+variable "nodepool_machine_type_misc" {
+  description = "Machine type to use for the misc nodepool."
+  default     = "t2d-standard-4"
+}
+
+variable "nodepool_size_misc" {
+  description = "Number of cluster nodes for the misc nodepool."
+  default     = 1
+}
+
+// These variables control the size of the database to be used by Coder.
+variable "cloudsql_version" {
+  description = "CloudSQL version to provision"
+  default     = "POSTGRES_14"
+}
+
+variable "cloudsql_tier" {
+  description = "CloudSQL database tier."
+  default     = "db-f1-micro"
+}
+
+variable "cloudsql_max_connections" {
+  description = "CloudSQL database max_connections"
+  default     = 500
+}
diff --git a/scaletest/terraform/coder.tf → scaletest/terraform/k8s/coder.tf b/scaletest/terraform/coder.tf → scaletest/terraform/k8s/coder.tf
@@ -7,32 +7,14 @@ locals {
   coder_namespace    = "coder-${var.name}"
   coder_admin_email  = "admin@coder.com"
   coder_admin_user   = "coder"
-  coder_address      = google_compute_address.coder.address
-  coder_url          = "http://${google_compute_address.coder.address}"
-}
-
-provider "kubernetes" {
-  host                   = "https://${google_container_cluster.primary.endpoint}"
-  cluster_ca_certificate = base64decode(google_container_cluster.primary.master_auth.0.cluster_ca_certificate)
-  token                  = data.google_client_config.default.access_token
-}
-
-provider "helm" {
-  kubernetes {
-    host                   = "https://${google_container_cluster.primary.endpoint}"
-    cluster_ca_certificate = base64decode(google_container_cluster.primary.master_auth.0.cluster_ca_certificate)
-    token                  = data.google_client_config.default.access_token
-  }
+  coder_access_url   = "http://${var.coder_address}"
 }
 
 resource "null_resource" "coder_namespace" {
   triggers = {
     namespace       = local.coder_namespace
-    kubeconfig_path = local.cluster_kubeconfig_path
+    kubeconfig_path = var.kubernetes_kubeconfig_path
   }
-  depends_on = [
-    google_container_node_pool.coder
-  ]
   provisioner "local-exec" {
     when    = create
     command = <<EOF
@@ -45,14 +27,6 @@ resource "null_resource" "coder_namespace" {
   }
 }
 
-resource "random_password" "coder-postgres-password" {
-  length = 12
-}
-
-resource "random_password" "prometheus-postgres-password" {
-  length = 12
-}
-
 resource "kubernetes_secret" "coder-db" {
   type = "Opaque"
   metadata {
@@ -61,7 +35,7 @@ resource "kubernetes_secret" "coder-db" {
   }
   depends_on = [null_resource.coder_namespace]
   data = {
-    url = "postgres://${google_sql_user.coder.name}:${urlencode(random_password.coder-postgres-password.result)}@${google_sql_database_instance.db.private_ip_address}/${google_sql_database.coder.name}?sslmode=disable"
+    url = var.coder_db_url
   }
 }
 
@@ -72,7 +46,6 @@ resource "helm_release" "coder-chart" {
   version    = var.coder_chart_version
   namespace  = local.coder_namespace
   depends_on = [
-    google_container_node_pool.coder,
     null_resource.coder_namespace
   ]
   values = [<<EOF
@@ -84,7 +57,7 @@ coder:
         - matchExpressions:
           - key: "cloud.google.com/gke-nodepool"
             operator: "In"
-            values: ["${google_container_node_pool.coder.name}"]
+            values: ["${var.kubernetes_nodepool_coder}"]
     podAntiAffinity:
       preferredDuringSchedulingIgnoredDuringExecution:
       - weight: 1
@@ -97,7 +70,7 @@ coder:
               values:   ["${local.coder_release_name}"]
   env:
     - name: "CODER_ACCESS_URL"
-      value: "${local.coder_url}"
+      value: "${local.coder_access_url}"
     - name: "CODER_CACHE_DIRECTORY"
       value: "/tmp/coder"
     - name: "CODER_ENABLE_TELEMETRY"
@@ -144,7 +117,7 @@ coder:
   service:
     enable: true
     sessionAffinity: None
-    loadBalancerIP: "${local.coder_address}"
+    loadBalancerIP: "${var.coder_address}"
   volumeMounts:
   - mountPath: "/tmp"
     name: cache
@@ -234,7 +207,7 @@ resource "local_file" "kubernetes_template" {
                 match_expressions {
                   key = "cloud.google.com/gke-nodepool"
                   operator = "In"
-                  values = ["${google_container_node_pool.workspaces.name}"]
+                  values = ["${var.kubernetes_nodepool_workspaces}"]
                 }
               }
             }
@@ -284,7 +257,7 @@ spec:
               - key: cloud.google.com/gke-nodepool
                 operator: In
                 values:
-                - ${google_container_node_pool.coder.name}
+                - ${var.kubernetes_nodepool_coder}
         podAntiAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
           - podAffinityTerm:
@@ -307,7 +280,7 @@ spec:
         - name: CODER_PROMETHEUS_ADDRESS
           value: 0.0.0.0:2112
         - name: CODER_ACCESS_URL
-          value: ${local.coder_url}
+          value: ${local.coder_access_url}
         - name: CODER_CACHE_DIRECTORY
           value: /tmp/coder
         - name: CODER_ENABLE_TELEMETRY
@@ -395,9 +368,9 @@ spec:
 }
 
 resource "null_resource" "provisionerd_deployment_apply" {
-  depends_on = [helm_release.coder-chart, local_file.provisionerd_deployment, null_resource.cluster_kubeconfig]
+  depends_on = [helm_release.coder-chart, local_file.provisionerd_deployment]
   triggers = {
-    kubeconfig_path = local.cluster_kubeconfig_path
+    kubeconfig_path = var.kubernetes_kubeconfig_path
     manifest_path   = local_file.provisionerd_deployment.filename
   }
   provisioner "local-exec" {
@@ -408,11 +381,11 @@ resource "null_resource" "provisionerd_deployment_apply" {
 }
 
 resource "local_file" "output_vars" {
-  filename = "${path.module}/../.coderv2/url"
-  content  = local.coder_url
+  filename = "${path.module}/../../.coderv2/url"
+  content  = local.coder_access_url
 }
 
 output "coder_url" {
   description = "URL of the Coder deployment"
-  value       = local.coder_url
+  value       = local.coder_access_url
 }