non-dynamic providers

coder · f0ssel · Jan 13, 2025 · Nov 14, 2024 · Nov 14, 2024 · Nov 18, 2024
commit dc28627a578dd9b14bf0d9f0ea613a2565b85607
diff --git a/scaletest/terraform/new/cf_dns.tf b/scaletest/terraform/new/cf_dns.tf
@@ -0,0 +1,7 @@
+resource "cloudflare_record" "coder" {
+  zone_id = var.cloudflare_zone_id
+  name    = local.coder_subdomain
+  content = google_compute_address.coder["primary"].address
+  type    = "A"
+  ttl     = 3600
+}
diff --git a/scaletest/terraform/new/gcp_cluster.tf b/scaletest/terraform/new/gcp_cluster.tf
@@ -4,35 +4,39 @@ data "google_compute_default_service_account" "default" {
 }
 
 locals {
-  node_pools = flatten([ for i, deployment in var.deployments : [
-    {
-      name = "${var.name}-${deployment.name}-coder"
-      zone = deployment.zone
-      size = deployment.coder_node_pool_size
-      cluster_i = i
-    },     
-    {
-      name = "${var.name}-${deployment.name}-workspaces"
-      zone = deployment.zone
-      size = deployment.workspaces_node_pool_size
-      cluster_i = i
-    },
-    {
-      name = "${var.name}-${deployment.name}-misc"
-      zone = deployment.zone
-      size = deployment.misc_node_pool_size
-      cluster_i = i
+  clusters = {
+    primary = {
+      region = "us-east1"
+      zone   = "us-east1-c"
+      cidr   = "10.200.0.0/24"
     }
-  ] ])
+  }
+  node_pools = {
+    primary_coder = {
+      name = "coder"
+      cluster = "primary"
+      size = 1
+    }
+    primary_workspaces = {
+      name = "workspaces"
+      cluster = "primary"
+      size = 1
+    }
+    primary_misc = {
+      name = "misc"
+      cluster = "primary"
+      size = 1
+    }
+  }
 }
 
 resource "google_container_cluster" "cluster" {
-  count                     = length(var.deployments)
-  name                      = "${var.name}-${var.deployments[count.index].name}"
-  location                  = var.deployments[count.index].zone
+  for_each                  = local.clusters
+  name                      = "${var.name}-${each.key}"
+  location                  = each.value.zone
   project                   = var.project_id
   network                   = google_compute_network.vpc.name
-  subnetwork                = google_compute_subnetwork.subnet[count.index].name
+  subnetwork                = google_compute_subnetwork.subnet[each.key].name
   networking_mode           = "VPC_NATIVE"
   default_max_pods_per_node = 256
   ip_allocation_policy { # Required with networking_mode=VPC_NATIVE
@@ -72,14 +76,14 @@ resource "google_container_cluster" "cluster" {
 }
 
 resource "google_container_node_pool" "node_pool" {
-  count    = length(local.node_pools)
-  name     = local.node_pools[count.index].name
-  location = local.node_pools[count.index].zone
+  for_each = local.node_pools
+  name     = each.value.name
+  location = local.clusters[each.value.cluster].zone
   project  = var.project_id
-  cluster  = google_container_cluster.cluster[local.node_pools[count.index].cluster_i].name
+  cluster  = google_container_cluster.cluster[each.value.cluster].name
   autoscaling {
     min_node_count = 1
-    max_node_count = local.node_pools[count.index].size
+    max_node_count = each.value.size
   }
   node_config {
     oauth_scopes = [

diff --git a/scaletest/terraform/new/gcp_db.tf b/scaletest/terraform/new/gcp_db.tf
@@ -1,7 +1,7 @@
 resource "google_sql_database_instance" "db" {
   name                = "${var.name}-coder"
   project             = var.project_id
-  region              = var.deployments[0].region
+  region              = local.clusters.primary.region
   database_version    = var.cloudsql_version
   deletion_protection = false
 
@@ -13,7 +13,7 @@ resource "google_sql_database_instance" "db" {
     availability_type = "ZONAL"
 
     location_preference {
-      zone = var.deployments[0].zone
+      zone = local.clusters.primary.zone
     }
 
     database_flags {

diff --git a/scaletest/terraform/new/gcp_vpc.tf b/scaletest/terraform/new/gcp_vpc.tf
@@ -1,3 +1,4 @@
+
 resource "google_compute_network" "vpc" {
   project                 = var.project_id
   name                    = var.name
@@ -8,19 +9,19 @@ resource "google_compute_network" "vpc" {
 }
 
 resource "google_compute_subnetwork" "subnet" {
-  count         = length(var.deployments)
-  name          = "${var.name}-${var.deployments[count.index].name}"
+  for_each      = local.clusters
+  name          = "${var.name}-${each.key}"
   project       = var.project_id
-  region        = var.deployments[count.index].region
+  region        = each.value.region
   network       = google_compute_network.vpc.name
-  ip_cidr_range = var.deployments[count.index].subnet_cidr
+  ip_cidr_range = each.value.cidr
 }
 
 resource "google_compute_address" "coder" {
-  count        = length(var.deployments)
+  for_each     = local.clusters
   project      = var.project_id
-  region       = var.deployments[count.index].region
-  name         = "${var.name}-${var.deployments[count.index].name}-coder"
+  region       = each.value.region
+  name         = "${var.name}-${each.key}-coder"
   address_type = "EXTERNAL"
   network_tier = "PREMIUM"
 }

diff --git a/scaletest/terraform/new/k8s_certmanager.tf b/scaletest/terraform/new/k8s_certmanager.tf
@@ -8,6 +8,8 @@ locals {
 }
 
 resource "kubernetes_secret" "cloudflare-api-key" {
+  provider = kubernetes.primary
+
   metadata {
     name      = "cloudflare-api-key-secret"
     namespace = local.cert_manager_namespace
@@ -18,12 +20,16 @@ resource "kubernetes_secret" "cloudflare-api-key" {
 }
 
 resource "kubernetes_namespace" "cert-manager-namespace" {
+  provider = kubernetes.primary
+
   metadata {
     name = local.cert_manager_namespace
   }
 }
 
 resource "helm_release" "cert-manager" {
+  provider = helm.primary
+
   repository = local.cert_manager_helm_repo
   chart      = local.cert_manager_helm_chart
   name       = local.cert_manager_release_name
@@ -35,6 +41,8 @@ EOF
 }
 
 resource "kubectl_manifest" "cloudflare-cluster-issuer" {
+  provider = kubectl.primary
+
   depends_on = [ helm_release.cert-manager ]
     yaml_body = <<YAML
 apiVersion: cert-manager.io/v1

diff --git a/scaletest/terraform/new/k8s_coder.tf b/scaletest/terraform/new/k8s_coder.tf
@@ -1,9 +1,12 @@
 data "google_client_config" "default" {}
 
 locals {
-  coder_url                 = "https://${var.name}-${var.deployments[0].name}-scaletest.${var.cloudflare_domain}"
+  coder_subdomain           = "${var.name}-primary-scaletest"
+  coder_url                 = "https://${local.coder_subdomain}.${var.cloudflare_domain}"
   coder_admin_email         = "admin@coder.com"
+  coder_admin_full_name     = "Coder Admin"
   coder_admin_user          = "coder"
+  coder_admin_password      = "SomeSecurePassword!"
   coder_helm_repo           = "https://helm.coder.com/v2"
   coder_helm_chart          = "coder"
   coder_namespace           = "coder-${var.name}"
@@ -14,6 +17,8 @@ locals {
 }
 
 resource "kubernetes_namespace" "coder_namespace" {
+  provider = kubernetes.primary
+
   metadata {
     name = local.coder_namespace
   }
@@ -41,6 +46,8 @@ resource "kubernetes_secret" "coder-db" {
 }
 
 resource "kubernetes_secret" "provisionerd_psk" {
+  provider = kubernetes.primary
+
   type = "Opaque"
   metadata {
     name      = "coder-provisioner-psk"
@@ -56,13 +63,16 @@ resource "kubernetes_secret" "provisionerd_psk" {
 
 # OIDC secret needs to be manually provisioned for now.
 data "kubernetes_secret" "coder_oidc" {
+  provider = kubernetes.primary
   metadata {
     namespace = kubernetes_namespace.coder_namespace.metadata.0.name
     name      = "coder-oidc"
   }
 }
 
 resource "kubectl_manifest" "coder_certificate" {
+  provider = kubectl.primary
+
   depends_on = [ helm_release.cert-manager ]
   yaml_body = <<YAML
 apiVersion: cert-manager.io/v1
@@ -81,6 +91,8 @@ YAML
 }
 
 data "kubernetes_secret" "coder_tls" {
+  provider = kubernetes.primary
+
   metadata {
     namespace = kubernetes_namespace.coder_namespace.metadata.0.name
     name      = "${var.name}-tls"
@@ -89,6 +101,8 @@ data "kubernetes_secret" "coder_tls" {
 }
 
 resource "helm_release" "coder-chart" {
+  provider = helm.primary
+
   repository = local.coder_helm_repo
   chart      = local.coder_helm_chart
   name       = local.coder_release_name
@@ -103,7 +117,7 @@ coder:
         - matchExpressions:
           - key: "cloud.google.com/gke-nodepool"
             operator: "In"
-            values: ["${google_container_node_pool.node_pool[0].name}"]
+            values: ["${google_container_node_pool.node_pool["primary_coder"].name}"]
     podAntiAffinity:
       preferredDuringSchedulingIgnoredDuringExecution:
       - weight: 1
@@ -196,7 +210,7 @@ coder:
   service:
     enable: true
     sessionAffinity: None
-    loadBalancerIP: "${google_compute_address.coder[0].address}"
+    loadBalancerIP: "${google_compute_address.coder["primary"].address}"
   volumeMounts:
   - mountPath: "/tmp"
     name: cache
@@ -224,7 +238,7 @@ coder:
         - matchExpressions:
           - key: "cloud.google.com/gke-nodepool"
             operator: "In"
-            values: ["${google_container_node_pool.node_pool[0].name}"]
+            values: ["${google_container_node_pool.node_pool["primary_coder"].name}"]
     podAntiAffinity:
       preferredDuringSchedulingIgnoredDuringExecution:
       - weight: 1
@@ -278,3 +292,57 @@ coder:
 EOF
   ]
 }
+
+data "http" "coder_healthy" {
+  url = "http://${local.coder_subdomain}.${var.cloudflare_domain}"
+  // Wait up to 5 minutes for DNS to propogate
+  retry {
+    attempts = 30
+    min_delay_ms = 10000
+  }
+
+  lifecycle {
+    postcondition {
+        condition = self.status_code == 200
+        error_message = "${self.url} returned an unhealthy status code"
+    }
+  }
+
+  depends_on = [ helm_release.coder-chart, cloudflare_record.coder ]
+}
+
+resource "terraform_data" "proxy_tokens" {
+  count = 1
+  provisioner "local-exec" {
+    interpreter = [ "/bin/bash", "-c" ]
+    command = <<EOF
+curl 'http://${local.coder_subdomain}.${var.cloudflare_domain}/api/v2/users/first' \
+  --data-raw $'{"email":"${local.coder_admin_email}","password":"${local.coder_admin_password}","username":"${local.coder_admin_user}","name":"${local.coder_admin_full_name}","trial":false}' \
+  --insecure --silent --output /dev/null
+
+token=$(curl 'http://${local.coder_subdomain}.${var.cloudflare_domain}/api/v2/users/login' \
+  --data-raw $'{"email":"${local.coder_admin_email}","password":"${local.coder_admin_password}"}' \
+  --insecure --silent | jq -r .session_token)
+
+curl 'http://${local.coder_subdomain}.${var.cloudflare_domain}/api/v2/licenses' \
+  -H "Coder-Session-Token: $${token}" \
+  --data-raw '{"license":"${var.coder_license}"}' \
+  --insecure --silent --output /dev/null
+
+europe_token=$(curl 'http://${local.coder_subdomain}.${var.cloudflare_domain}/api/v2/workspaceproxies' \
+  -H "Coder-Session-Token: $${token}" \
+  --data-raw '{"name":"europe"}' \
+  --insecure --silent | jq -r .proxy_token)
+
+asia_token=$(curl 'http://${local.coder_subdomain}.${var.cloudflare_domain}/api/v2/workspaceproxies' \
+  -H "Coder-Session-Token: $${token}" \
+  --data-raw '{"name":"asia"}' \
+  --insecure --silent | jq -r .proxy_token)
+
+echo "{\"europe\": \"$${europe_token}\", \"asia\": \"$${asia_token}\"}"
+EOF
+  }
+
+  depends_on = [ data.http.coder_healthy ]
+}
+