3 create worksapces

coder · f0ssel · Jan 13, 2025 · Nov 14, 2024 · Nov 14, 2024 · Nov 18, 2024
commit 60c86215541cb97513046c79ac27137c6edeab45
diff --git a/scaletest/terraform/action/coder_workspaces.tf b/scaletest/terraform/action/coder_workspaces.tf
@@ -0,0 +1,177 @@
+locals {
+  create_workspace_timeout = "600s"
+}
+
+resource "kubernetes_job" "create_workspaces_primary" {
+  provider = kubernetes.primary
+
+  metadata {
+    name      = "${var.name}-create-workspaces"
+    namespace = kubernetes_namespace.coder_primary.metadata.0.name
+    labels = {
+      "app.kubernetes.io/name" = "${var.name}-create-workspaces"
+    }
+  }
+  spec {
+    completions = 1
+    template {
+      metadata {}
+      spec {
+        affinity {
+          node_affinity {
+            required_during_scheduling_ignored_during_execution {
+              node_selector_term {
+                match_expressions {
+                  key      = "cloud.google.com/gke-nodepool"
+                  operator = "In"
+                  values   = ["${google_container_node_pool.node_pool["primary_misc"].name}"]
+                }
+              }
+            }
+          }
+        }
+        container {
+          name  = "cli"
+          image = "${var.coder_image_repo}:${var.coder_image_tag}"
+          command = [
+            "/opt/coder",
+            "--verbose",
+            "--url=${local.deployments.primary.url}",
+            "--token=${trimspace(data.local_file.api_key.content)}",
+            "exp",
+            "scaletest",
+            "create-workspaces",
+            "--count=${local.scenarios[var.scenario].workspaces.count_per_deployment}",
+            "--template=kubernetes-primary",
+            "--concurrency=${var.workspace_create_concurrency}",
+            "--no-cleanup"
+          ]
+        }
+        restart_policy = "Never"
+      }
+    }
+  }
+  wait_for_completion = true
+
+  timeouts {
+    create = local.create_workspace_timeout
+  }
+
+  depends_on = [kubernetes_job.push_template_primary]
+}
+
+resource "kubernetes_job" "create_workspaces_europe" {
+  provider = kubernetes.europe
+
+  metadata {
+    name      = "${var.name}-create-workspaces"
+    namespace = kubernetes_namespace.coder_europe.metadata.0.name
+    labels = {
+      "app.kubernetes.io/name" = "${var.name}-create-workspaces"
+    }
+  }
+  spec {
+    completions = 1
+    template {
+      metadata {}
+      spec {
+        affinity {
+          node_affinity {
+            required_during_scheduling_ignored_during_execution {
+              node_selector_term {
+                match_expressions {
+                  key      = "cloud.google.com/gke-nodepool"
+                  operator = "In"
+                  values   = ["${google_container_node_pool.node_pool["europe_misc"].name}"]
+                }
+              }
+            }
+          }
+        }
+        container {
+          name  = "cli"
+          image = "${var.coder_image_repo}:${var.coder_image_tag}"
+          command = [
+            "/opt/coder",
+            "--verbose",
+            "--url=${local.deployments.primary.url}",
+            "--token=${trimspace(data.local_file.api_key.content)}",
+            "exp",
+            "scaletest",
+            "create-workspaces",
+            "--count=${local.scenarios[var.scenario].workspaces.count_per_deployment}",
+            "--template=kubernetes-europe",
+            "--concurrency=${var.workspace_create_concurrency}",
+            "--no-cleanup"
+          ]
+        }
+        restart_policy = "Never"
+      }
+    }
+  }
+  wait_for_completion = true
+
+  timeouts {
+    create = local.create_workspace_timeout
+  }
+
+  depends_on = [kubernetes_job.push_template_europe]
+}
+
+resource "kubernetes_job" "create_workspaces_asia" {
+  provider = kubernetes.asia
+
+  metadata {
+    name      = "${var.name}-create-workspaces"
+    namespace = kubernetes_namespace.coder_asia.metadata.0.name
+    labels = {
+      "app.kubernetes.io/name" = "${var.name}-create-workspaces"
+    }
+  }
+  spec {
+    completions = 1
+    template {
+      metadata {}
+      spec {
+        affinity {
+          node_affinity {
+            required_during_scheduling_ignored_during_execution {
+              node_selector_term {
+                match_expressions {
+                  key      = "cloud.google.com/gke-nodepool"
+                  operator = "In"
+                  values   = ["${google_container_node_pool.node_pool["asia_misc"].name}"]
+                }
+              }
+            }
+          }
+        }
+        container {
+          name  = "cli"
+          image = "${var.coder_image_repo}:${var.coder_image_tag}"
+          command = [
+            "/opt/coder",
+            "--verbose",
+            "--url=${local.deployments.primary.url}",
+            "--token=${trimspace(data.local_file.api_key.content)}",
+            "exp",
+            "scaletest",
+            "create-workspaces",
+            "--count=${local.scenarios[var.scenario].workspaces.count_per_deployment}",
+            "--template=kubernetes-asia",
+            "--concurrency=${var.workspace_create_concurrency}",
+            "--no-cleanup"
+          ]
+        }
+        restart_policy = "Never"
+      }
+    }
+  }
+  wait_for_completion = true
+
+  timeouts {
+    create = local.create_workspace_timeout
+  }
+
+  depends_on = [kubernetes_job.push_template_asia]
+}
diff --git a/scaletest/terraform/action/k8s_coder_asia.tf b/scaletest/terraform/action/k8s_coder_asia.tf
@@ -82,66 +82,25 @@ resource "helm_release" "provisionerd_asia" {
   name       = local.provisionerd_release_name
   version    = var.provisionerd_chart_version
   namespace  = kubernetes_namespace.coder_asia.metadata.0.name
-  values = [<<EOF
-coder:
-  affinity:
-    nodeAffinity:
-      requiredDuringSchedulingIgnoredDuringExecution:
-        nodeSelectorTerms:
-        - matchExpressions:
-          - key: "cloud.google.com/gke-nodepool"
-            operator: "In"
-            values: ["${google_container_node_pool.node_pool["asia_coder"].name}"]
-    podAntiAffinity:
-      preferredDuringSchedulingIgnoredDuringExecution:
-      - weight: 1
-        podAffinityTerm:
-          topologyKey: "kubernetes.io/hostname"
-          labelSelector:
-            matchExpressions:
-            - key:      "app.kubernetes.io/instance"
-              operator: "In"
-              values:   ["${local.coder_release_name}"]
-  env:
-    - name: "CODER_URL"
-      value: "${local.deployments.primary.url}"
-    - name: "CODER_VERBOSE"
-      value: "true"
-    - name: "CODER_CONFIG_DIR"
-      value: "/tmp/config"
-    - name: "CODER_CACHE_DIRECTORY"
-      value: "/tmp/coder"
-    - name: "CODER_TELEMETRY_ENABLE"
-      value: "false"
-    - name: "CODER_LOGGING_HUMAN"
-      value: "/dev/null"
-    - name: "CODER_LOGGING_STACKDRIVER"
-      value: "/dev/stderr"
-    - name: "CODER_PROMETHEUS_ENABLE"
-      value: "true"
-    - name: "CODER_PROVISIONERD_TAGS"
-      value: "scope=organization"
-  image:
-    repo: ${var.provisionerd_image_repo}
-    tag: ${var.provisionerd_image_tag}
-  replicaCount: "${local.scenarios[var.scenario].provisionerd.replicas}"
-  resources:
-    requests:
-      cpu: "${local.scenarios[var.scenario].provisionerd.cpu_request}"
-      memory: "${local.scenarios[var.scenario].provisionerd.mem_request}"
-    limits:
-      cpu: "${local.scenarios[var.scenario].provisionerd.cpu_limit}"
-      memory: "${local.scenarios[var.scenario].provisionerd.mem_limit}"
-  securityContext:
-    readOnlyRootFilesystem: true
-  volumeMounts:
-  - mountPath: "/tmp"
-    name: cache
-    readOnly: false
-  volumes:
-  - emptyDir:
-      sizeLimit: 1024Mi
-    name: cache
-EOF
-  ]
+  values = [templatefile("${path.module}/coder_helm_values.tftpl", {
+    workspace_proxy = false,
+    provisionerd = true,
+    primary_url = null,
+    proxy_token = null,
+    db_secret = null,
+    ip_address = null,
+    provisionerd_psk = kubernetes_secret.provisionerd_psk_asia.metadata.0.name,
+    access_url       = local.deployments.primary.url,
+    node_pool        = google_container_node_pool.node_pool["asia_coder"].name,
+    release_name     = local.coder_release_name,
+    experiments      = var.coder_experiments,
+    image_repo       = var.coder_image_repo,
+    image_tag        = var.coder_image_tag,
+    replicas         = local.scenarios[var.scenario].provisionerd.replicas,
+    cpu_request      = local.scenarios[var.scenario].provisionerd.cpu_request,
+    mem_request      = local.scenarios[var.scenario].provisionerd.mem_request,
+    cpu_limit        = local.scenarios[var.scenario].provisionerd.cpu_limit,
+    mem_limit        = local.scenarios[var.scenario].provisionerd.mem_limit,
+    deployment       = "asia",
+  })]
 }
diff --git a/scaletest/terraform/action/scenarios.tf b/scaletest/terraform/action/scenarios.tf
@@ -18,12 +18,13 @@ locals {
         mem_limit   = "1Gi"
       }
       workspaces = {
-        nodepool_size = 1
-        machine_type  = "t2d-standard-4"
-        cpu_request   = "100m"
-        mem_request   = "128Mi"
-        cpu_limit     = "100m"
-        mem_limit     = "128Mi"
+        count_per_deployment = 10
+        nodepool_size        = 1
+        machine_type         = "t2d-standard-4"
+        cpu_request          = "100m"
+        mem_request          = "128Mi"
+        cpu_limit            = "100m"
+        mem_limit            = "128Mi"
       }
       misc = {
         nodepool_size = 1
@@ -52,12 +53,13 @@ locals {
         mem_limit   = "1Gi"
       }
       workspaces = {
-        nodepool_size = 1
-        machine_type  = "t2d-standard-8"
-        cpu_request   = "100m"
-        mem_request   = "128Mi"
-        cpu_limit     = "100m"
-        mem_limit     = "128Mi"
+        count_per_deployment = 10
+        nodepool_size        = 1
+        machine_type         = "t2d-standard-8"
+        cpu_request          = "100m"
+        mem_request          = "128Mi"
+        cpu_limit            = "100m"
+        mem_limit            = "128Mi"
       }
       misc = {
         nodepool_size = 1
@@ -86,12 +88,13 @@ locals {
         mem_limit   = "1Gi"
       }
       workspaces = {
-        nodepool_size = 1
-        machine_type  = "t2d-standard-8"
-        cpu_request   = "100m"
-        mem_request   = "128Mi"
-        cpu_limit     = "100m"
-        mem_limit     = "128Mi"
+        count_per_deployment = 10
+        nodepool_size        = 1
+        machine_type         = "t2d-standard-8"
+        cpu_request          = "100m"
+        mem_request          = "128Mi"
+        cpu_limit            = "100m"
+        mem_limit            = "128Mi"
       }
       misc = {
         nodepool_size = 1

diff --git a/scaletest/terraform/action/vars.tf b/scaletest/terraform/action/vars.tf
@@ -85,3 +85,8 @@ variable "provisionerd_image_tag" {
   description = "Tag to use for Provisionerd image."
   default     = "latest"
 }
+
+variable "workspace_create_concurrency" {
+  description = "Number of concurrent workspace creation jobs to run."
+  default     = 10
+}