diff --git a/.gitignore b/.gitignore index 9a565462e8a4f..69b58c4cee458 100644 --- a/.gitignore +++ b/.gitignore @@ -48,9 +48,14 @@ site/stats/ *.lock.hcl .terraform/ -/.coderv2/* +**/.coderv2/* **/__debug_bin # direnv .envrc *.test + +# Loadtesting +./scaletest/terraform/.terraform +./scaletest/terraform/.terraform.lock.hcl +terraform.tfstate.* diff --git a/.prettierignore b/.prettierignore index f08db1c28a2e6..cc4a83b0231a8 100644 --- a/.prettierignore +++ b/.prettierignore @@ -51,12 +51,17 @@ site/stats/ *.lock.hcl .terraform/ -/.coderv2/* +**/.coderv2/* **/__debug_bin # direnv .envrc *.test + +# Loadtesting +./scaletest/terraform/.terraform +./scaletest/terraform/.terraform.lock.hcl +terraform.tfstate.* # .prettierignore.include: # Helm templates contain variables that are invalid YAML and can't be formatted # by Prettier. diff --git a/scaletest/terraform/README.md b/scaletest/terraform/README.md new file mode 100644 index 0000000000000..f5a2bc376d9c2 --- /dev/null +++ b/scaletest/terraform/README.md @@ -0,0 +1,40 @@ +# Load Test Terraform + +This folder contains Terraform code and scripts to aid in performing load tests of Coder. +It does the following: + +- Creates a GCP VPC. +- Creates a CloudSQL instance with a global peering rule so it's accessible inside the VPC. +- Creates a GKE cluster inside the VPC with separate nodegroups for Coder and workspaces. +- Installs Coder in a new namespace, using the CloudSQL instance. + +## Usage + +> You must have an existing Google Cloud project available. + +1. Create a file named `override.tfvars` with the following content, modifying as appropriate: + +```terraform +name = "some_unique_identifier" +project_id = "some_google_project_id" +``` + +1. Inspect `vars.tf` and override any other variables you deem necessary. + +1. Run `terraform init`. + +1. Run `terraform plan -var-file=override.tfvars` and inspect the output. + If you are not satisfied, modify `override.tfvars` until you are. + +1. Run `terraform apply -var-file=override.tfvars`. This will spin up a pre-configured environment + and emit the Coder URL as an output. + +1. Run `coder_init.sh ` to setup an initial user and a pre-configured Kubernetes + template. It will also download the Coder CLI from the Coder instance locally. + +1. Do whatever you need to do with the Coder instance. + + > To run Coder commands against the instance, you can use `coder_shim.sh `. + > You don't need to run `coder login` yourself. + +1. When you are finished, you can run `terraform destroy -var-file=override.tfvars`. diff --git a/scaletest/terraform/coder.tf b/scaletest/terraform/coder.tf new file mode 100644 index 0000000000000..d86aa2a7fe1ad --- /dev/null +++ b/scaletest/terraform/coder.tf @@ -0,0 +1,250 @@ +data "google_client_config" "default" {} + +locals { + coder_helm_repo = "https://helm.coder.com/v2" + coder_helm_chart = "coder" + coder_release_name = var.name + coder_namespace = "coder-${var.name}" + coder_admin_email = "admin@coder.com" + coder_admin_user = "coder" + coder_address = google_compute_address.coder.address + coder_url = "http://${google_compute_address.coder.address}" +} + +provider "kubernetes" { + host = "https://${google_container_cluster.primary.endpoint}" + cluster_ca_certificate = base64decode(google_container_cluster.primary.master_auth.0.cluster_ca_certificate) + token = data.google_client_config.default.access_token +} + +provider "helm" { + kubernetes { + host = "https://${google_container_cluster.primary.endpoint}" + cluster_ca_certificate = base64decode(google_container_cluster.primary.master_auth.0.cluster_ca_certificate) + token = data.google_client_config.default.access_token + } +} + +resource "kubernetes_namespace" "coder_namespace" { + metadata { + name = local.coder_namespace + } + depends_on = [ + google_container_node_pool.coder + ] +} + +resource "random_password" "postgres-admin-password" { + length = 12 +} + +resource "random_password" "coder-postgres-password" { + length = 12 +} + +resource "kubernetes_secret" "coder-db" { + type = "" # Opaque + metadata { + name = "coder-db-url" + namespace = kubernetes_namespace.coder_namespace.metadata.0.name + } + data = { + url = "postgres://${google_sql_user.coder.name}:${urlencode(random_password.coder-postgres-password.result)}@${google_sql_database_instance.db.private_ip_address}/${google_sql_database.coder.name}?sslmode=disable" + } +} + +resource "helm_release" "coder-chart" { + repository = local.coder_helm_repo + chart = local.coder_helm_chart + name = local.coder_release_name + version = var.coder_chart_version + namespace = kubernetes_namespace.coder_namespace.metadata.0.name + depends_on = [ + google_container_node_pool.coder, + ] + values = [<" + exit 1 +fi + +# Allow toggling verbose output +[[ -n ${VERBOSE:-} ]] && set -x + +CODER_URL=$1 +CONFIG_DIR="${PWD}/.coderv2" +ARCH="$(arch)" +if [[ "$ARCH" == "x86_64" ]]; then + ARCH="amd64" +fi +PLATFORM="$(uname | tr '[:upper:]' '[:lower:]')" + +mkdir -p "${CONFIG_DIR}" +echo "Fetching Coder CLI for first-time setup!" +curl -fsSLk "${CODER_URL}/bin/coder-${PLATFORM}-${ARCH}" -o "${CONFIG_DIR}/coder" +chmod +x "${CONFIG_DIR}/coder" + +set +o pipefail +RANDOM_ADMIN_PASSWORD=$(tr "${CONFIG_DIR}/coder.env" +CODER_FIRST_USER_EMAIL=admin@coder.com +CODER_FIRST_USER_USERNAME=coder +CODER_FIRST_USER_PASSWORD="${RANDOM_ADMIN_PASSWORD}" +CODER_FIRST_USER_TRIAL="${CODER_FIRST_USER_TRIAL}" +EOF + +echo "Importing kubernetes template" +"${CONFIG_DIR}/coder" templates create --global-config="${CONFIG_DIR}" \ + --directory "${CONFIG_DIR}/templates/kubernetes" --yes kubernetes diff --git a/scaletest/terraform/coder_shim.sh b/scaletest/terraform/coder_shim.sh new file mode 100755 index 0000000000000..d62c5a952ecb3 --- /dev/null +++ b/scaletest/terraform/coder_shim.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# This is a shim for easily executing Coder commands against a loadtest cluster +# without having to overwrite your own session/URL +SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") +CONFIG_DIR="${SCRIPT_DIR}/.coderv2" +CODER_BIN="${CONFIG_DIR}/coder" +exec "${CODER_BIN}" --global-config "${CONFIG_DIR}" "$@" diff --git a/scaletest/terraform/gcp_cluster.tf b/scaletest/terraform/gcp_cluster.tf new file mode 100644 index 0000000000000..df4bd551c9d75 --- /dev/null +++ b/scaletest/terraform/gcp_cluster.tf @@ -0,0 +1,125 @@ +data "google_compute_default_service_account" "default" { + project = var.project_id +} + +resource "google_container_cluster" "primary" { + name = var.name + location = var.zone + project = var.project_id + network = google_compute_network.vpc.name + subnetwork = google_compute_subnetwork.subnet.name + networking_mode = "VPC_NATIVE" + ip_allocation_policy { # Required with networking_mode=VPC_NATIVE + + } + release_channel { + channel = "STABLE" + } + initial_node_count = 1 + remove_default_node_pool = true + network_policy { + enabled = true + } + depends_on = [ + google_project_service.api["container.googleapis.com"] + ] + monitoring_config { + enable_components = ["SYSTEM_COMPONENTS"] + managed_prometheus { + enabled = true + } + } + workload_identity_config { + workload_pool = "${data.google_project.project.project_id}.svc.id.goog" + } +} + +resource "google_container_node_pool" "coder" { + name = "${var.name}-coder" + location = var.zone + project = var.project_id + cluster = google_container_cluster.primary.name + node_count = var.nodepool_size_coder + node_config { + oauth_scopes = [ + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring", + "https://www.googleapis.com/auth/trace.append", + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/service.management.readonly", + "https://www.googleapis.com/auth/servicecontrol", + ] + disk_size_gb = var.node_disk_size_gb + machine_type = var.nodepool_machine_type_coder + image_type = var.node_image_type + preemptible = var.node_preemptible + service_account = data.google_compute_default_service_account.default.email + tags = ["gke-node", "${var.project_id}-gke"] + labels = { + env = var.project_id + } + metadata = { + disable-legacy-endpoints = "true" + } + } +} + +resource "google_container_node_pool" "workspaces" { + name = "${var.name}-workspaces" + location = var.zone + project = var.project_id + cluster = google_container_cluster.primary.name + node_count = var.nodepool_size_workspaces + node_config { + oauth_scopes = [ + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring", + "https://www.googleapis.com/auth/trace.append", + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/service.management.readonly", + "https://www.googleapis.com/auth/servicecontrol", + ] + disk_size_gb = var.node_disk_size_gb + machine_type = var.nodepool_machine_type_workspaces + image_type = var.node_image_type + preemptible = var.node_preemptible + service_account = data.google_compute_default_service_account.default.email + tags = ["gke-node", "${var.project_id}-gke"] + labels = { + env = var.project_id + } + metadata = { + disable-legacy-endpoints = "true" + } + } +} + +resource "google_container_node_pool" "misc" { + name = "${var.name}-misc" + location = var.zone + project = var.project_id + cluster = google_container_cluster.primary.name + node_count = var.nodepool_size_misc + node_config { + oauth_scopes = [ + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring", + "https://www.googleapis.com/auth/trace.append", + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/service.management.readonly", + "https://www.googleapis.com/auth/servicecontrol", + ] + disk_size_gb = var.node_disk_size_gb + machine_type = var.nodepool_machine_type_misc + image_type = var.node_image_type + preemptible = var.node_preemptible + service_account = data.google_compute_default_service_account.default.email + tags = ["gke-node", "${var.project_id}-gke"] + labels = { + env = var.project_id + } + metadata = { + disable-legacy-endpoints = "true" + } + } +} diff --git a/scaletest/terraform/gcp_db.tf b/scaletest/terraform/gcp_db.tf new file mode 100644 index 0000000000000..b57002f2d2872 --- /dev/null +++ b/scaletest/terraform/gcp_db.tf @@ -0,0 +1,53 @@ +resource "google_sql_database_instance" "db" { + name = var.name + region = var.region + database_version = var.cloudsql_version + deletion_protection = false + + depends_on = [google_service_networking_connection.private_vpc_connection] + + settings { + tier = var.cloudsql_tier + activation_policy = "ALWAYS" + availability_type = "ZONAL" + + location_preference { + zone = var.zone + } + + database_flags { + name = "max_connections" + value = var.cloudsql_max_connections + } + + ip_configuration { + ipv4_enabled = false + private_network = google_compute_network.vpc.id + } + + insights_config { + query_insights_enabled = true + query_string_length = 1024 + record_application_tags = false + record_client_address = false + } + } +} + +resource "google_sql_database" "coder" { + project = var.project_id + instance = google_sql_database_instance.db.id + name = "${var.name}-coder" + # required for postgres, otherwise db fails to delete + deletion_policy = "ABANDON" +} + +resource "google_sql_user" "coder" { + project = var.project_id + instance = google_sql_database_instance.db.id + name = "${var.name}-coder" + type = "BUILT_IN" + password = random_password.coder-postgres-password.result + # required for postgres, otherwise user fails to delete + deletion_policy = "ABANDON" +} diff --git a/scaletest/terraform/gcp_project.tf b/scaletest/terraform/gcp_project.tf new file mode 100644 index 0000000000000..c233042e66acb --- /dev/null +++ b/scaletest/terraform/gcp_project.tf @@ -0,0 +1,32 @@ +provider "google" { + region = var.region + project = var.project_id +} + +locals { + project_apis = [ + "cloudtrace", + "compute", + "container", + "logging", + "monitoring", + "servicemanagement", + "servicenetworking", + "sqladmin", + "stackdriver", + "storage-api", + ] +} + +data "google_project" "project" { + project_id = var.project_id +} + +resource "google_project_service" "api" { + for_each = toset(local.project_apis) + project = data.google_project.project.project_id + service = "${each.value}.googleapis.com" + + disable_dependent_services = false + disable_on_destroy = false +} diff --git a/scaletest/terraform/gcp_vpc.tf b/scaletest/terraform/gcp_vpc.tf new file mode 100644 index 0000000000000..7ed76a00235e9 --- /dev/null +++ b/scaletest/terraform/gcp_vpc.tf @@ -0,0 +1,39 @@ +resource "google_compute_network" "vpc" { + project = var.project_id + name = var.name + auto_create_subnetworks = "false" + depends_on = [ + google_project_service.api["compute.googleapis.com"] + ] +} + +resource "google_compute_subnetwork" "subnet" { + name = var.name + project = var.project_id + region = var.region + network = google_compute_network.vpc.name + ip_cidr_range = "10.10.0.0/24" +} + +resource "google_compute_global_address" "sql_peering" { + project = var.project_id + name = "${var.name}-sql-peering" + purpose = "VPC_PEERING" + address_type = "INTERNAL" + prefix_length = 16 + network = google_compute_network.vpc.id +} + +resource "google_compute_address" "coder" { + project = var.project_id + region = var.region + name = "${var.name}-coder" + address_type = "EXTERNAL" + network_tier = "PREMIUM" +} + +resource "google_service_networking_connection" "private_vpc_connection" { + network = google_compute_network.vpc.id + service = "servicenetworking.googleapis.com" + reserved_peering_ranges = [google_compute_global_address.sql_peering.name] +} diff --git a/scaletest/terraform/main.tf b/scaletest/terraform/main.tf new file mode 100644 index 0000000000000..280420cecf267 --- /dev/null +++ b/scaletest/terraform/main.tf @@ -0,0 +1,35 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "~> 4.36" + } + + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.20" + } + + helm = { + source = "hashicorp/helm" + version = "~> 2.9" + } + + random = { + source = "hashicorp/random" + version = "~> 3.5" + } + + tls = { + source = "hashicorp/tls" + version = "~> 4.0" + } + + docker = { + source = "kreuzwerker/docker" + version = "~> 3.0" + } + } + + required_version = "~> 1.4.0" +} diff --git a/scaletest/terraform/vars.tf b/scaletest/terraform/vars.tf new file mode 100644 index 0000000000000..e312c4e542215 --- /dev/null +++ b/scaletest/terraform/vars.tf @@ -0,0 +1,129 @@ +variable "project_id" { + description = "The project in which to provision resources" +} + +variable "name" { + description = "Adds a prefix to resources." +} + +variable "region" { + description = "GCP region in which to provision resources." + default = "us-east1" +} + +variable "zone" { + description = "GCP zone in which to provision resources." + default = "us-east1-c" +} + +variable "k8s_version" { + description = "Kubernetes vversion to provision." + default = "1.24" +} + +variable "node_disk_size_gb" { + description = "Size of the root disk for cluster nodes." + default = 100 +} + +variable "node_image_type" { + description = "Image type to use for cluster nodes." + default = "cos_containerd" +} + +// Preemptible nodes are way cheaper, but can be pulled out +// from under you at any time. Caveat emptor. +variable "node_preemptible" { + description = "Use preemptible nodes." + default = false +} + +// We create three nodepools: +// - One for the Coder control plane +// - One for workspaces +// - One for everything else (for example, load generation) + +// These variables control the node pool dedicated to Coder. +variable "nodepool_machine_type_coder" { + description = "Machine type to use for Coder control plane nodepool." + default = "t2d-standard-4" +} + +variable "nodepool_size_coder" { + description = "Number of cluster nodes for the Coder control plane nodepool." + default = 1 +} + +// These variables control the node pool dedicated to workspaces. +variable "nodepool_machine_type_workspaces" { + description = "Machine type to use for the workspaces nodepool." + default = "t2d-standard-4" +} + +variable "nodepool_size_workspaces" { + description = "Number of cluster nodes for the workspaces nodepool." + default = 1 +} + +// These variables control the node pool for everything else. +variable "nodepool_machine_type_misc" { + description = "Machine type to use for the misc nodepool." + default = "t2d-standard-4" +} + +variable "nodepool_size_misc" { + description = "Number of cluster nodes for the misc nodepool." + default = 1 +} + +// These variables control the size of the database to be used by Coder. +variable "cloudsql_version" { + description = "CloudSQL version to provision" + default = "POSTGRES_14" +} + +variable "cloudsql_tier" { + description = "CloudSQL database tier." + default = "db-f1-micro" +} + +variable "cloudsql_max_connections" { + description = "CloudSQL database max_connections" + default = 500 +} + +// These variables control the Coder deployment. +variable "coder_replicas" { + description = "Number of Coder replicas to provision" + default = 1 +} + +variable "coder_cpu" { + description = "CPU to allocate to Coder" + default = "1000m" +} + +variable "coder_mem" { + description = "Memory to allocate to Coder" + default = "1024Mi" +} + +variable "coder_chart_version" { + description = "Version of the Coder Helm chart to install. Defaults to latest." + default = null +} + +variable "coder_image_repo" { + description = "Repository to use for Coder image." + default = "ghcr.io/coder/coder" +} + +variable "coder_image_tag" { + description = "Tag to use for Coder image." + default = "latest" +} + +variable "workspace_image" { + description = "Image and tag to use for workspaces." + default = "docker.io/codercom/enterprise-minimal:ubuntu" +} diff --git a/site/.eslintignore b/site/.eslintignore index f83b3caa434f0..865d1e7006067 100644 --- a/site/.eslintignore +++ b/site/.eslintignore @@ -51,12 +51,17 @@ stats/ *.lock.hcl .terraform/ -../.coderv2/* +**/.coderv2/* **/__debug_bin # direnv .envrc *.test + +# Loadtesting +.././scaletest/terraform/.terraform +.././scaletest/terraform/.terraform.lock.hcl +terraform.tfstate.* # .prettierignore.include: # Helm templates contain variables that are invalid YAML and can't be formatted # by Prettier. diff --git a/site/.prettierignore b/site/.prettierignore index f83b3caa434f0..865d1e7006067 100644 --- a/site/.prettierignore +++ b/site/.prettierignore @@ -51,12 +51,17 @@ stats/ *.lock.hcl .terraform/ -../.coderv2/* +**/.coderv2/* **/__debug_bin # direnv .envrc *.test + +# Loadtesting +.././scaletest/terraform/.terraform +.././scaletest/terraform/.terraform.lock.hcl +terraform.tfstate.* # .prettierignore.include: # Helm templates contain variables that are invalid YAML and can't be formatted # by Prettier.