coder · f0ssel · Dec 12, 2024 · Nov 14, 2024 · Nov 14, 2024 · Nov 18, 2024
diff --git a/scaletest/terraform/action/cf_dns.tf b/scaletest/terraform/action/cf_dns.tf
@@ -0,0 +1,8 @@
+resource "cloudflare_record" "coder" {
+  for_each = local.deployments
+  zone_id  = var.cloudflare_zone_id
+  name     = each.value.subdomain
+  content  = google_compute_address.coder[each.key].address
+  type     = "A"
+  ttl      = 3600
+}
diff --git a/scaletest/terraform/action/coder_helm_values.tftpl b/scaletest/terraform/action/coder_helm_values.tftpl
@@ -0,0 +1,105 @@
+coder:
+  workspaceProxy: ${workspace_proxy}
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: "cloud.google.com/gke-nodepool"
+            operator: "In"
+            values: ["${node_pool}"]
+    podAntiAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+      - weight: 1
+        podAffinityTerm:
+          topologyKey: "kubernetes.io/hostname"
+          labelSelector:
+            matchExpressions:
+            - key:      "app.kubernetes.io/instance"
+              operator: "In"
+              values:   ["${release_name}"]
+  env:
+    %{~ if workspace_proxy ~}
+    - name: "CODER_ACCESS_URL"
+      value: "${access_url}"
+    - name: CODER_PRIMARY_ACCESS_URL
+      value: "${primary_url}"
+    - name: CODER_PROXY_SESSION_TOKEN
+      valueFrom:
+        secretKeyRef:
+          key: token
+          name: "${proxy_token}"
+    %{~ endif ~}
+    %{~ if provisionerd ~}
+    - name: "CODER_URL"
+      value: "${access_url}"
+    - name: "CODER_PROVISIONERD_TAGS"
+      value: "scope=organization"
+    - name: "CODER_CONFIG_DIR"
+      value: "/tmp/config"
+    %{~ endif ~}
+    %{~ if !workspace_proxy && !provisionerd ~}
+    - name: "CODER_ACCESS_URL"
+      value: "${access_url}"
+    - name: "CODER_PG_CONNECTION_URL"
+      valueFrom:
+        secretKeyRef:
+          name: "${db_secret}"
+          key: url
+    - name: "CODER_PROVISIONER_DAEMONS"
+      value: "0"
+    - name: CODER_PROVISIONER_DAEMON_PSK
+      valueFrom:
+        secretKeyRef:
+          key: psk
+          name: "${provisionerd_psk}"
+    - name: "CODER_PROMETHEUS_COLLECT_AGENT_STATS"
+      value: "true"
+    - name: "CODER_PROMETHEUS_COLLECT_DB_METRICS"
+      value: "true"
+    - name: "CODER_PPROF_ENABLE"
+      value: "true"
+    %{~ endif ~}
+    - name: "CODER_CACHE_DIRECTORY"
+      value: "/tmp/coder"
+    - name: "CODER_TELEMETRY_ENABLE"
+      value: "false"
+    - name: "CODER_LOGGING_HUMAN"
+      value: "/dev/null"
+    - name: "CODER_LOGGING_STACKDRIVER"
+      value: "/dev/stderr"
+    - name: "CODER_PROMETHEUS_ENABLE"
+      value: "true"
+    - name: "CODER_VERBOSE"
+      value: "true"
+    - name: "CODER_EXPERIMENTS"
+      value: "${experiments}"
+    - name: "CODER_DANGEROUS_DISABLE_RATE_LIMITS"
+      value: "true"
+  image:
+    repo: ${image_repo}
+    tag: ${image_tag}
+  replicaCount: "${replicas}"
+  resources:
+    requests:
+      cpu: "${cpu_request}"
+      memory: "${mem_request}"
+    limits:
+      cpu: "${cpu_limit}"
+      memory: "${mem_limit}"
+  securityContext:
+    readOnlyRootFilesystem: true
+  %{~ if !provisionerd ~}
+  service:
+    enable: true
+    sessionAffinity: None
+    loadBalancerIP: "${ip_address}"
+  %{~ endif ~}
+  volumeMounts:
+  - mountPath: "/tmp"
+    name: cache
+    readOnly: false
+  volumes:
+  - emptyDir:
+      sizeLimit: 1024Mi
+    name: cache
diff --git a/scaletest/terraform/action/coder_proxies.tf b/scaletest/terraform/action/coder_proxies.tf
@@ -0,0 +1,102 @@
+data "http" "coder_healthy" {
+  url = local.deployments.primary.url
+  // Wait up to 5 minutes for DNS to propagate
+  retry {
+    attempts     = 30
+    min_delay_ms = 10000
+  }
+
+  lifecycle {
+    postcondition {
+      condition     = self.status_code == 200
+      error_message = "${self.url} returned an unhealthy status code"
+    }
+  }
+
+  depends_on = [helm_release.coder_primary, cloudflare_record.coder["primary"]]
+}
+
+resource "null_resource" "api_key" {
+  provisioner "local-exec" {
+    interpreter = ["/bin/bash", "-c"]
+    command     = <<EOF
+set -e
+
+curl '${local.deployments.primary.url}/api/v2/users/first' \
+  --data-raw $'{"email":"${local.coder_admin_email}","password":"${local.coder_admin_password}","username":"${local.coder_admin_user}","name":"${local.coder_admin_full_name}","trial":false}' \
+  --insecure --silent --output /dev/null
+
+session_token=$(curl '${local.deployments.primary.url}/api/v2/users/login' \
+  --data-raw $'{"email":"${local.coder_admin_email}","password":"${local.coder_admin_password}"}' \
+  --insecure --silent | jq -r .session_token)
+
+echo -n $${session_token} > ${path.module}/.coderv2/session_token
+
+api_key=$(curl '${local.deployments.primary.url}/api/v2/users/me/keys/tokens' \
+  -H "Coder-Session-Token: $${session_token}" \
+  --data-raw '{"token_name":"terraform","scope":"all"}' \
+  --insecure --silent | jq -r .key)
+
+echo -n $${api_key} > ${path.module}/.coderv2/api_key
+EOF
+  }
+
+  depends_on = [data.http.coder_healthy]
+}
+
+data "local_file" "api_key" {
+  filename   = "${path.module}/.coderv2/api_key"
+  depends_on = [null_resource.api_key]
+}
+
+resource "null_resource" "license" {
+  provisioner "local-exec" {
+    interpreter = ["/bin/bash", "-c"]
+    command     = <<EOF
+curl '${local.deployments.primary.url}/api/v2/licenses' \
+  -H "Coder-Session-Token: ${trimspace(data.local_file.api_key.content)}" \
+  --data-raw '{"license":"${var.coder_license}"}' \
+  --insecure --silent --output /dev/null
+EOF
+  }
+}
+
+resource "null_resource" "europe_proxy_token" {
+  provisioner "local-exec" {
+    interpreter = ["/bin/bash", "-c"]
+    command     = <<EOF
+curl '${local.deployments.primary.url}/api/v2/workspaceproxies' \
+  -H "Coder-Session-Token: ${trimspace(data.local_file.api_key.content)}" \
+  --data-raw '{"name":"europe","display_name":"Europe","icon":"/emojis/1f950.png"}' \
+  --insecure --silent \
+  | jq -r .proxy_token > ${path.module}/.coderv2/europe_proxy_token
+EOF
+  }
+
+  depends_on = [null_resource.license]
+}
+
+data "local_file" "europe_proxy_token" {
+  filename   = "${path.module}/.coderv2/europe_proxy_token"
+  depends_on = [null_resource.europe_proxy_token]
+}
+
+resource "null_resource" "asia_proxy_token" {
+  provisioner "local-exec" {
+    interpreter = ["/bin/bash", "-c"]
+    command     = <<EOF
+curl '${local.deployments.primary.url}/api/v2/workspaceproxies' \
+  -H "Coder-Session-Token: ${trimspace(data.local_file.api_key.content)}" \
+  --data-raw '{"name":"asia","display_name":"Asia","icon":"/emojis/1f35b.png"}' \
+  --insecure --silent \
+  | jq -r .proxy_token > ${path.module}/.coderv2/asia_proxy_token
+EOF
+  }
+
+  depends_on = [null_resource.license]
+}
+
+data "local_file" "asia_proxy_token" {
+  filename   = "${path.module}/.coderv2/asia_proxy_token"
+  depends_on = [null_resource.asia_proxy_token]
+}
diff --git a/scaletest/terraform/action/coder_templates.tf b/scaletest/terraform/action/coder_templates.tf
@@ -0,0 +1,160 @@
+resource "local_file" "kubernetes_template" {
+  filename = "${path.module}/.coderv2/templates/kubernetes/main.tf"
+  content  = <<EOF
+    terraform {
+      required_providers {
+        coder = {
+          source  = "coder/coder"
+          version = "~> 0.23.0"
+        }
+        kubernetes = {
+          source  = "hashicorp/kubernetes"
+          version = "~> 2.30"
+        }
+      }
+    }
+
+    provider "coder" {}
+
+    provider "kubernetes" {
+      config_path = null # always use host
+    }
+
+    data "coder_workspace" "me" {}
+    data "coder_workspace_owner" "me" {}
+
+    resource "coder_agent" "main" {
+      os                     = "linux"
+      arch                   = "amd64"
+    }
+
+    resource "kubernetes_pod" "main" {
+      count = data.coder_workspace.me.start_count
+      metadata {
+        name      = "coder-$${lower(data.coder_workspace_owner.me.name)}-$${lower(data.coder_workspace.me.name)}"
+        namespace = "${local.coder_namespace}"
+        labels = {
+          "app.kubernetes.io/name"     = "coder-workspace"
+          "app.kubernetes.io/instance" = "coder-workspace-$${lower(data.coder_workspace_owner.me.name)}-$${lower(data.coder_workspace.me.name)}"
+        }
+      }
+      spec {
+        security_context {
+          run_as_user = "1000"
+          fs_group    = "1000"
+        }
+        container {
+          name              = "dev"
+          image             = "${var.workspace_image}"
+          image_pull_policy = "Always"
+          command           = ["sh", "-c", coder_agent.main.init_script]
+          security_context {
+            run_as_user = "1000"
+          }
+          env {
+            name  = "CODER_AGENT_TOKEN"
+            value = coder_agent.main.token
+          }
+          resources {
+            requests = {
+              "cpu"    = "${local.scenarios[var.scenario].workspaces.cpu_request}"
+              "memory" = "${local.scenarios[var.scenario].workspaces.mem_request}"
+            }
+            limits = {
+              "cpu"    = "${local.scenarios[var.scenario].workspaces.cpu_limit}"
+              "memory" = "${local.scenarios[var.scenario].workspaces.mem_limit}"
+            }
+          }
+        }
+
+        affinity {
+          node_affinity {
+            required_during_scheduling_ignored_during_execution {
+              node_selector_term {
+                match_expressions {
+                  key = "cloud.google.com/gke-nodepool"
+                  operator = "In"
+                  values = ["${google_container_node_pool.node_pool["primary_workspaces"].name}","${google_container_node_pool.node_pool["europe_workspaces"].name}","${google_container_node_pool.node_pool["asia_workspaces"].name}"]
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  EOF
+}
+
+resource "kubernetes_config_map" "template" {
+  provider = kubernetes.primary
+
+  metadata {
+    name      = "coder-template"
+    namespace = kubernetes_namespace.coder_primary.metadata.0.name
+  }
+
+  data = {
+    "main.tf" = local_file.kubernetes_template.content
+  }
+}
+
+resource "kubernetes_job" "push_template" {
+  provider = kubernetes.primary
+
+  metadata {
+    name      = "${var.name}-push-template"
+    namespace = kubernetes_namespace.coder_primary.metadata.0.name
+    labels = {
+      "app.kubernetes.io/name" = "${var.name}-push-template"
+    }
+  }
+  spec {
+    completions = 1
+    template {
+      metadata {}
+      spec {
+        affinity {
+          node_affinity {
+            required_during_scheduling_ignored_during_execution {
+              node_selector_term {
+                match_expressions {
+                  key      = "cloud.google.com/gke-nodepool"
+                  operator = "In"
+                  values   = ["${google_container_node_pool.node_pool["primary_misc"].name}"]
+                }
+              }
+            }
+          }
+        }
+        container {
+          name  = "cli"
+          image = "${var.coder_image_repo}:${var.coder_image_tag}"
+          command = [
+            "/opt/coder",
+            "--verbose",
+            "--url=${local.deployments.primary.url}",
+            "--token=${trimspace(data.local_file.api_key.content)}",
+            "templates",
+            "push",
+            "--directory=/home/coder/template",
+            "--yes",
+            "kubernetes"
+          ]
+          volume_mount {
+            name       = "coder-template"
+            mount_path = "/home/coder/template/main.tf"
+            sub_path   = "main.tf"
+          }
+        }
+        volume {
+          name = "coder-template"
+          config_map {
+            name = kubernetes_config_map.template.metadata.0.name
+          }
+        }
+        restart_policy = "Never"
+      }
+    }
+  }
+  wait_for_completion = true
+}