From 63c884acb5f8e71a58f34e85a0c5563631f2dbd2 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 29 Jun 2023 13:28:06 +0100 Subject: [PATCH 1/4] feat: allow scaling down scaletest environments --- scaletest/scaletest.sh | 20 +++++++++++++++----- scaletest/terraform/gcp_cluster.tf | 6 +++--- scaletest/terraform/vars.tf | 9 +++++++++ 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/scaletest/scaletest.sh b/scaletest/scaletest.sh index 9db7d5a123e74..940d829ed7f8c 100755 --- a/scaletest/scaletest.sh +++ b/scaletest/scaletest.sh @@ -19,9 +19,10 @@ SCALETEST_SKIP_CLEANUP="${SCALETEST_SKIP_CLEANUP:-0}" SCALETEST_CREATE_CONCURRENCY="${SCALETEST_CREATE_CONCURRENCY:-10}" SCALETEST_TRAFFIC_BYTES_PER_TICK="${SCALETEST_TRAFFIC_BYTES_PER_TICK:-1024}" SCALETEST_TRAFFIC_TICK_INTERVAL="${SCALETEST_TRAFFIC_TICK_INTERVAL:-10}" +SCALETEST_DESTROY="${SCALETEST_DESTROY:-0}" script_name=$(basename "$0") -args="$(getopt -o "" -l create-concurrency:,dry-run,help,name:,num-workspaces:,project:,scenario:,skip-cleanup,traffic-bytes-per-tick:,traffic-tick-interval:, -- "$@")" +args="$(getopt -o "" -l create-concurrency:,destroy,dry-run,help,name:,num-workspaces:,project:,scenario:,skip-cleanup,traffic-bytes-per-tick:,traffic-tick-interval:, -- "$@")" eval set -- "$args" while true; do case "$1" in @@ -29,12 +30,16 @@ while true; do SCALETEST_CREATE_CONCURRENCY="$2" shift 2 ;; + --destroy) + SCALETEST_DESTROY=1 + shift + ;; --dry-run) DRY_RUN=1 shift ;; --help) - echo "Usage: $script_name --name --project --num-workspaces --scenario [--dry-run] [--skip-cleanup] [--create-concurrency=]" + echo "Usage: $script_name --name --project --num-workspaces --scenario [--destroy] [--dry-run] [--skip-cleanup] [--create-concurrency=]" exit 1 ;; --name) @@ -142,7 +147,7 @@ echo "Initializing terraform." maybedryrun "$DRY_RUN" terraform init echo "Setting up infrastructure." -maybedryrun "$DRY_RUN" terraform apply --var-file="${SCALETEST_SCENARIO_VARS}" --var-file="${SCALETEST_SECRETS}" --auto-approve +maybedryrun "$DRY_RUN" terraform apply --var-file="${SCALETEST_SCENARIO_VARS}" --var-file="${SCALETEST_SECRETS}" --var state=started --auto-approve if [[ "${DRY_RUN}" != 1 ]]; then SCALETEST_CODER_URL=$(<"${CONFIG_DIR}/url") @@ -212,5 +217,10 @@ if [[ "${SCALETEST_SKIP_CLEANUP}" == 1 ]]; then exit 0 fi -echo "Cleaning up" -maybedryrun "$DRY_RUN" terraform destroy --var-file="${SCALETEST_SCENARIO_VARS}" --var-file="${SCALETEST_SECRETS}" --auto-approve +if [[ "${SCALETEST_DESTROY}" == 1 ]]; then + echo "Destroying infrastructure" + maybedryrun "$DRY_RUN" terraform destroy --var-file="${SCALETEST_SCENARIO_VARS}" --var-file="${SCALETEST_SECRETS}" --auto-approve +else + echo "Scaling down infrastructure" + maybedryrun "$DRY_RUN" terraform apply --var-file="${SCALETEST_SCENARIO_VARS}" --var-file="${SCALETEST_SECRETS}" --var state=stopped --auto-approve +fi diff --git a/scaletest/terraform/gcp_cluster.tf b/scaletest/terraform/gcp_cluster.tf index 70223c8251e30..87dbab0386ecc 100644 --- a/scaletest/terraform/gcp_cluster.tf +++ b/scaletest/terraform/gcp_cluster.tf @@ -40,7 +40,7 @@ resource "google_container_node_pool" "coder" { location = var.zone project = var.project_id cluster = google_container_cluster.primary.name - node_count = var.nodepool_size_coder + node_count = var.state == "stopped" ? 0 : var.nodepool_size_coder node_config { oauth_scopes = [ "https://www.googleapis.com/auth/logging.write", @@ -70,7 +70,7 @@ resource "google_container_node_pool" "workspaces" { location = var.zone project = var.project_id cluster = google_container_cluster.primary.name - node_count = var.nodepool_size_workspaces + node_count = var.state == "stopped" ? 0 : var.nodepool_size_workspaces node_config { oauth_scopes = [ "https://www.googleapis.com/auth/logging.write", @@ -100,7 +100,7 @@ resource "google_container_node_pool" "misc" { location = var.zone project = var.project_id cluster = google_container_cluster.primary.name - node_count = var.nodepool_size_misc + node_count = var.state == "stopped" ? 0 : var.nodepool_size_misc node_config { oauth_scopes = [ "https://www.googleapis.com/auth/logging.write", diff --git a/scaletest/terraform/vars.tf b/scaletest/terraform/vars.tf index ad534f2f99b65..d65711f1c07d3 100644 --- a/scaletest/terraform/vars.tf +++ b/scaletest/terraform/vars.tf @@ -1,3 +1,12 @@ +variable "state" { + description = "The state of the cluster. Valid values are 'started', and 'stopped'." + validation { + condition = contains(["started", "stopped"], var.state) + error_message = "value must be one of 'started' or 'stopped'" + } + default = "started" +} + variable "project_id" { description = "The project in which to provision resources" } From 8f706d81f102f024433ac04f9e8b8d24e0ff7ab0 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 30 Jun 2023 13:18:14 +0100 Subject: [PATCH 2/4] fix bugged namespace deletion --- scaletest/terraform/coder.tf | 27 +++++++++++++++++++------ scaletest/terraform/gcp_cluster.tf | 26 ++++++++++++++++++++++++ scaletest/terraform/prometheus.tf | 32 +++++++++++++++++++++--------- 3 files changed, 70 insertions(+), 15 deletions(-) diff --git a/scaletest/terraform/coder.tf b/scaletest/terraform/coder.tf index ad93d100ed5e0..d0c9f5c882a3c 100644 --- a/scaletest/terraform/coder.tf +++ b/scaletest/terraform/coder.tf @@ -25,13 +25,26 @@ provider "helm" { } } -resource "kubernetes_namespace" "coder_namespace" { - metadata { - name = local.coder_namespace +resource "null_resource" "coder_namespace" { + triggers = { + namespace = local.coder_namespace + kubeconfig_path = local.cluster_kubeconfig_path } depends_on = [ google_container_node_pool.coder ] + provisioner "local-exec" { + when = create + command = < Date: Fri, 30 Jun 2023 15:15:58 +0100 Subject: [PATCH 3/4] misc fixes to scaletest.sh --- scaletest/scaletest.sh | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/scaletest/scaletest.sh b/scaletest/scaletest.sh index 940d829ed7f8c..fdb23d9271bbb 100755 --- a/scaletest/scaletest.sh +++ b/scaletest/scaletest.sh @@ -18,7 +18,7 @@ SCALETEST_CODER_LICENSE="${SCALETEST_CODER_LICENSE:-}" SCALETEST_SKIP_CLEANUP="${SCALETEST_SKIP_CLEANUP:-0}" SCALETEST_CREATE_CONCURRENCY="${SCALETEST_CREATE_CONCURRENCY:-10}" SCALETEST_TRAFFIC_BYTES_PER_TICK="${SCALETEST_TRAFFIC_BYTES_PER_TICK:-1024}" -SCALETEST_TRAFFIC_TICK_INTERVAL="${SCALETEST_TRAFFIC_TICK_INTERVAL:-10}" +SCALETEST_TRAFFIC_TICK_INTERVAL="${SCALETEST_TRAFFIC_TICK_INTERVAL:-10s}" SCALETEST_DESTROY="${SCALETEST_DESTROY:-0}" script_name=$(basename "$0") @@ -39,7 +39,7 @@ while true; do shift ;; --help) - echo "Usage: $script_name --name --project --num-workspaces --scenario [--destroy] [--dry-run] [--skip-cleanup] [--create-concurrency=]" + echo "Usage: $script_name --name --project --num-workspaces --scenario [--create-concurrency ] [--destroy] [--dry-run] [--skip-cleanup] [--traffic-bytes-per-tick ] [--traffic-tick-interval ]" exit 1 ;; --name) @@ -156,7 +156,21 @@ else fi KUBECONFIG="${PROJECT_ROOT}/scaletest/.coderv2/${SCALETEST_NAME}-cluster.kubeconfig" echo "Waiting for Coder deployment at ${SCALETEST_CODER_URL} to become ready" -maybedryrun "$DRY_RUN" kubectl --kubeconfig="${KUBECONFIG}" -n "coder-${SCALETEST_NAME}" rollout status deployment/coder +max_attempts=10 +for attempt in $(seq 1 $max_attempts); do + maybedryrun "$DRY_RUN" curl --silent --fail --output /dev/null "${SCALETEST_CODER_URL}/api/v2/buildinfo" + curl_status=$? + if [[ $curl_status -eq 0 ]]; then + break + fi + if attempt -eq $max_attempts; then + echo + echo "Coder deployment failed to become ready in time!" + exit 1 + fi + echo "Coder deployment not ready yet (${attempt}/${max_attempts}), sleeping 3 seconds" + maybedryrun "$DRY_RUN" sleep 3 +done echo "Initializing Coder deployment." DRY_RUN="$DRY_RUN" "${PROJECT_ROOT}/scaletest/lib/coder_init.sh" "${SCALETEST_CODER_URL}" From b75e235d41994324c4d825c02419ea1311d82722 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 30 Jun 2023 15:25:48 +0100 Subject: [PATCH 4/4] destroy namespaces is a no-op as the cluster will be gone anyway --- scaletest/terraform/coder.tf | 4 +--- scaletest/terraform/prometheus.tf | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/scaletest/terraform/coder.tf b/scaletest/terraform/coder.tf index d0c9f5c882a3c..13bd15205b6eb 100644 --- a/scaletest/terraform/coder.tf +++ b/scaletest/terraform/coder.tf @@ -41,9 +41,7 @@ resource "null_resource" "coder_namespace" { } provisioner "local-exec" { when = destroy - command = <