From 51b505453a2cf273b73ee44ea5b457bc8252eb22 Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Wed, 13 Sep 2023 12:40:15 +0000
Subject: [PATCH 01/36] feat(scaletest): add scaletest-runner template

---
 cli/exp_scaletest.go                          |   2 +-
 .../templates/scaletest-runner/README.md      |   9 +
 scaletest/templates/scaletest-runner/main.tf  | 438 ++++++++++++++++++
 .../scaletest-runner/metadata_phase.sh        |  15 +
 .../scaletest-runner/metadata_status.sh       |  13 +
 .../scaletest-runner/scripts/cleanup.sh       |  37 ++
 .../scaletest-runner/scripts/prepare.sh       |  52 +++
 .../templates/scaletest-runner/scripts/run.sh |  79 ++++
 .../templates/scaletest-runner/shutdown.sh    |   9 +
 .../templates/scaletest-runner/startup.sh     |  25 +
 10 files changed, 678 insertions(+), 1 deletion(-)
 create mode 100644 scaletest/templates/scaletest-runner/README.md
 create mode 100644 scaletest/templates/scaletest-runner/main.tf
 create mode 100755 scaletest/templates/scaletest-runner/metadata_phase.sh
 create mode 100755 scaletest/templates/scaletest-runner/metadata_status.sh
 create mode 100755 scaletest/templates/scaletest-runner/scripts/cleanup.sh
 create mode 100755 scaletest/templates/scaletest-runner/scripts/prepare.sh
 create mode 100755 scaletest/templates/scaletest-runner/scripts/run.sh
 create mode 100755 scaletest/templates/scaletest-runner/shutdown.sh
 create mode 100755 scaletest/templates/scaletest-runner/startup.sh

diff --git a/cli/exp_scaletest.go b/cli/exp_scaletest.go
index 5f0dc34bf68bc..a8e423d1fd9f2 100644
--- a/cli/exp_scaletest.go
+++ b/cli/exp_scaletest.go
@@ -858,7 +858,7 @@ func (r *RootCmd) scaletestCreateWorkspaces() *clibase.Cmd {
 			Flag:        "use-host-login",
 			Env:         "CODER_SCALETEST_USE_HOST_LOGIN",
 			Default:     "false",
-			Description: "Use the use logged in on the host machine, instead of creating users.",
+			Description: "Use the user logged in on the host machine, instead of creating users.",
 			Value:       clibase.BoolOf(&useHostUser),
 		},
 	}
diff --git a/scaletest/templates/scaletest-runner/README.md b/scaletest/templates/scaletest-runner/README.md
new file mode 100644
index 0000000000000..3bb65379915b0
--- /dev/null
+++ b/scaletest/templates/scaletest-runner/README.md
@@ -0,0 +1,9 @@
+---
+name: Scaletest Runner
+description: Run a scaletest.
+tags: [local]
+---
+
+# Scaletest Runner
+
+Run a scaletest.
\ No newline at end of file
diff --git a/scaletest/templates/scaletest-runner/main.tf b/scaletest/templates/scaletest-runner/main.tf
new file mode 100644
index 0000000000000..6721f7d6ecbaf
--- /dev/null
+++ b/scaletest/templates/scaletest-runner/main.tf
@@ -0,0 +1,438 @@
+terraform {
+  required_providers {
+    coder = {
+      source  = "coder/coder"
+      version = "~> 0.11"
+    }
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = "~> 2.22"
+    }
+  }
+}
+
+resource "time_static" "start_time" {
+  # We con't set `count = data.coder_workspace.me.start_count` here because then
+  # we can't use this value in `locals`. The permission check is recreated on
+  # start, which will update the timestamp.
+  triggers = {
+    count : length(null_resource.permission_check)
+  }
+}
+
+locals {
+  namespace              = "coder-big"
+  workspace_pod_name     = "coder-scaletest-runner-${lower(data.coder_workspace.me.owner)}-${lower(data.coder_workspace.me.name)}"
+  workspace_pod_instance = "coder-workspace-${lower(data.coder_workspace.me.owner)}-${lower(data.coder_workspace.me.name)}"
+  service_account_name   = "scaletest-sa"
+  cpu                    = 2
+  memory                 = 2
+  home_disk_size         = 10
+  scaletest_run_id       = "scaletest-${time_static.start_time.rfc3339}"
+  scaletest_run_dir      = "/home/coder/${local.scaletest_run_id}"
+}
+
+resource "null_resource" "permission_check" {
+  count = data.coder_workspace.me.start_count
+
+  # Limit which users can create a workspace in this template.
+  # The "default" user and workspace are present because they are needed
+  # for the plan, and consequently, updating the template.
+  lifecycle {
+    precondition {
+      condition     = can(regex("^(default|scaletest)$", data.coder_workspace.me.owner))
+      error_message = "User is not allowed, expected 'scaletest'."
+    }
+    precondition {
+      condition     = can(regex("^(default|runner)$", data.coder_workspace.me.name))
+      error_message = "Workspace name is not allowed, expected 'runner'."
+    }
+  }
+}
+
+data "coder_provisioner" "me" {
+}
+
+data "coder_workspace" "me" {
+}
+
+data "coder_parameter" "verbose" {
+  order       = 1
+  type        = "bool"
+  name        = "Verbose"
+  default     = false
+  description = "Show debug output."
+  mutable     = true
+  ephemeral   = true
+}
+
+data "coder_parameter" "dry_run" {
+  order       = 2
+  type        = "bool"
+  name        = "Dry-run"
+  default     = false
+  description = "Perform a dry-run to see what would happen."
+  mutable     = true
+  ephemeral   = true
+}
+
+data "coder_parameter" "workspace_template" {
+  order        = 3
+  name         = "workspace_template"
+  display_name = "Workspace Template"
+  description  = "The template used for workspace creation."
+  default      = "kubernetes-minimal"
+  icon         = "/emojis/1f4dc.png" # Scroll.
+  mutable      = true
+  ephemeral    = true
+  option {
+    name        = "Minimal"
+    value       = "kubernetes-minimal" # Feather.
+    icon        = "/emojis/1fab6.png"
+    description = "Sized to fit approx. 32 per t2d-standard-8 instance."
+  }
+  option {
+    name        = "Small"
+    value       = "kubernetes-small"
+    icon        = "/emojis/1f42d.png" # Mouse.
+    description = "Provisions a small-sized workspace with no persistent storage."
+  }
+  option {
+    name        = "Medium"
+    value       = "kubernetes-medium"
+    icon        = "/emojis/1f436.png" # Dog.
+    description = "Provisions a medium-sized workspace with no persistent storage."
+  }
+  option {
+    name        = "Large"
+    value       = "kubernetes-large"
+    icon        = "/emojis/1f434.png" # Horse.
+    description = "Provisions a large-sized workspace with no persistent storage."
+  }
+}
+
+data "coder_parameter" "num_workspaces" {
+  order       = 10
+  type        = "number"
+  name        = "Number of workspaces to create"
+  default     = 100
+  description = "The scaletest suite will create this numer of workspaces."
+  mutable     = true
+  ephemeral   = true
+
+  validation {
+    min = 0
+    max = 1000
+  }
+}
+
+data "archive_file" "scripts_zip" {
+  type        = "zip"
+  output_path = "${path.module}/scripts.zip"
+  source_dir  = "${path.module}/scripts"
+}
+
+resource "coder_agent" "main" {
+  arch = data.coder_provisioner.me.arch
+  os   = "linux"
+  env = {
+    VERBOSE : data.coder_parameter.verbose.value ? "1" : "0",
+    DRY_RUN : data.coder_parameter.dry_run.value ? "1" : "0",
+    CODER_CONFIG_DIR : "/home/coder/.config/coderv2",
+    CODER_USER_TOKEN : data.coder_workspace.me.owner_session_token,
+    CODER_URL : data.coder_workspace.me.access_url,
+    CODER_SCALETEST_PROMETHEUS_ADDRESS : "0.0.0.0:21112",
+    CODER_SCALETEST_PROMETHEUS_WAIT : "60s",
+
+    SCALETEST_RUN_ID : local.scaletest_run_id,
+    SCALETEST_RUN_DIR : local.scaletest_run_dir,
+    SCALETEST_TEMPLATE : data.coder_parameter.workspace_template.value,
+    SCALETEST_SKIP_CLEANUP : "1",
+    SCALETEST_NUM_WORKSPACES : data.coder_parameter.num_workspaces.value,
+    SCALETEST_CREATE_CONCURRENCY : "10",
+    SCALETEST_CLEANUP_CONCURRENCY : "10",
+
+    SCRIPTS_ZIP : filebase64(data.archive_file.scripts_zip.output_path),
+    SCRIPTS_DIR : "/tmp/scripts",
+  }
+  display_apps {
+    vscode     = false
+    ssh_helper = false
+  }
+  startup_script_timeout  = 3600
+  shutdown_script_timeout = 1800
+  startup_script_behavior = "blocking"
+  startup_script          = file("startup.sh")
+  shutdown_script         = file("shutdown.sh")
+
+  # Scaletest metadata.
+  metadata {
+    display_name = "Scaletest status"
+    key          = "00_scaletest_status"
+    script       = file("metadata_status.sh")
+    interval     = 1
+    timeout      = 1
+  }
+
+  metadata {
+    display_name = "Scaletest phase"
+    key          = "01_scaletest_phase"
+    script       = file("metadata_phase.sh")
+    interval     = 1
+    timeout      = 1
+  }
+
+  # Misc workspace metadata.
+  metadata {
+    display_name = "CPU Usage"
+    key          = "80_cpu_usage"
+    script       = "coder stat cpu"
+    interval     = 10
+    timeout      = 1
+  }
+
+  metadata {
+    display_name = "RAM Usage"
+    key          = "81_ram_usage"
+    script       = "coder stat mem"
+    interval     = 10
+    timeout      = 1
+  }
+
+  metadata {
+    display_name = "Home Disk"
+    key          = "82_home_disk"
+    script       = "coder stat disk --path $${HOME}"
+    interval     = 60
+    timeout      = 1
+  }
+
+  metadata {
+    display_name = "CPU Usage (Host)"
+    key          = "83_cpu_usage_host"
+    script       = "coder stat cpu --host"
+    interval     = 10
+    timeout      = 1
+  }
+
+  metadata {
+    display_name = "Memory Usage (Host)"
+    key          = "84_mem_usage_host"
+    script       = "coder stat mem --host"
+    interval     = 10
+    timeout      = 1
+  }
+
+  metadata {
+    display_name = "Load Average (Host)"
+    key          = "85_load_host"
+    # Get load avg scaled by number of cores.
+    script   = <<-EOS
+      echo "`cat /proc/loadavg | awk '{ print $1 }'` `nproc`" | awk '{ printf "%0.2f", $1/$2 }'
+    EOS
+    interval = 60
+    timeout  = 1
+  }
+}
+
+resource "coder_app" "grafana" {
+  agent_id     = coder_agent.main.id
+  slug         = "00-grafana"
+  display_name = "Grafana"
+  url          = "https://stats.dev.c8s.io/d/qLVSTR-Vz/coderv2-loadtest-dashboard?orgId=1&from=${time_static.start_time.unix * 1000}&to=now"
+  icon         = "https://grafana.com/static/assets/img/fav32.png"
+  external     = true
+}
+
+resource "coder_app" "prometheus" {
+  agent_id     = coder_agent.main.id
+  slug         = "01-prometheus"
+  display_name = "Prometheus"
+  // https://stats.dev.c8s.io:9443/classic/graph?g0.range_input=2h&g0.end_input=2023-09-08%2015%3A58&g0.stacked=0&g0.expr=rate(pg_stat_database_xact_commit%7Bcluster%3D%22big%22%2Cdatname%3D%22big-coder%22%7D%5B1m%5D)&g0.tab=0
+  url      = "https://stats.dev.c8s.io:9443"
+  icon     = "https://prometheus.io/assets/favicons/favicon-32x32.png"
+  external = true
+}
+
+resource "coder_app" "manual_cleanup" {
+  agent_id     = coder_agent.main.id
+  slug         = "02-manual-cleanup"
+  display_name = "Manual cleanup"
+  icon         = "/emojis/1f9f9.png"
+  command      = "/tmp/scripts/cleanup.sh manual"
+}
+
+resource "kubernetes_persistent_volume_claim" "home" {
+  depends_on = [null_resource.permission_check]
+  metadata {
+    name      = "${local.workspace_pod_name}-home"
+    namespace = local.namespace
+    labels = {
+      "app.kubernetes.io/name"     = "coder-pvc"
+      "app.kubernetes.io/instance" = "coder-pvc-${lower(data.coder_workspace.me.owner)}-${lower(data.coder_workspace.me.name)}"
+      "app.kubernetes.io/part-of"  = "coder"
+      // Coder specific labels.
+      "com.coder.resource"       = "true"
+      "com.coder.workspace.id"   = data.coder_workspace.me.id
+      "com.coder.workspace.name" = data.coder_workspace.me.name
+      "com.coder.user.id"        = data.coder_workspace.me.owner_id
+      "com.coder.user.username"  = data.coder_workspace.me.owner
+    }
+    annotations = {
+      "com.coder.user.email" = data.coder_workspace.me.owner_email
+    }
+  }
+  wait_until_bound = false
+  spec {
+    access_modes = ["ReadWriteOnce"]
+    resources {
+      requests = {
+        storage = "${local.home_disk_size}Gi"
+      }
+    }
+  }
+}
+
+resource "kubernetes_pod" "main" {
+  depends_on = [null_resource.permission_check]
+  count      = data.coder_workspace.me.start_count
+  metadata {
+    name      = local.workspace_pod_name
+    namespace = local.namespace
+    labels = {
+      "app.kubernetes.io/name"     = "coder-workspace"
+      "app.kubernetes.io/instance" = local.workspace_pod_instance
+      "app.kubernetes.io/part-of"  = "coder"
+      // Coder specific labels.
+      "com.coder.resource"       = "true"
+      "com.coder.workspace.id"   = data.coder_workspace.me.id
+      "com.coder.workspace.name" = data.coder_workspace.me.name
+      "com.coder.user.id"        = data.coder_workspace.me.owner_id
+      "com.coder.user.username"  = data.coder_workspace.me.owner
+    }
+    annotations = {
+      "com.coder.user.email" = data.coder_workspace.me.owner_email
+    }
+  }
+  # Set the pod delete timeout to termination_grace_period_seconds + 1m.
+  timeouts {
+    delete = "32m"
+  }
+  spec {
+    security_context {
+      run_as_user = "1000"
+      fs_group    = "1000"
+    }
+
+    # Allow this pod to perform scale tests.
+    service_account_name = local.service_account_name
+
+    # Allow the coder agent to perform graceful shutdown and cleanup of
+    # scaletest resources, 30 minutes (cleanup timeout) + 1 minute.
+    termination_grace_period_seconds = 1860
+
+    container {
+      name              = "dev"
+      image             = "codercom/enterprise-base:ubuntu"
+      image_pull_policy = "Always"
+      command           = ["sh", "-c", coder_agent.main.init_script]
+      security_context {
+        run_as_user = "1000"
+      }
+      env {
+        name  = "CODER_AGENT_TOKEN"
+        value = coder_agent.main.token
+      }
+      env {
+        name  = "CODER_AGENT_LOG_DIR"
+        value = "${local.scaletest_run_dir}/logs"
+      }
+      resources {
+        # Set requests and limits values such that we can do performant
+        # execution of `coder scaletest` commands.
+        requests = {
+          "cpu"    = "250m"
+          "memory" = "512Mi"
+        }
+        limits = {
+          "cpu"    = "${local.cpu}"
+          "memory" = "${local.memory}Gi"
+        }
+      }
+      volume_mount {
+        mount_path = "/home/coder"
+        name       = "home"
+        read_only  = false
+      }
+      port {
+        container_port = 21112
+        name           = "prometheus-http"
+        protocol       = "TCP"
+      }
+    }
+
+    volume {
+      name = "home"
+      persistent_volume_claim {
+        claim_name = kubernetes_persistent_volume_claim.home.metadata.0.name
+        read_only  = false
+      }
+    }
+
+    affinity {
+      pod_anti_affinity {
+        // This affinity attempts to spread out all workspace pods evenly across
+        // nodes.
+        preferred_during_scheduling_ignored_during_execution {
+          weight = 1
+          pod_affinity_term {
+            topology_key = "kubernetes.io/hostname"
+            label_selector {
+              match_expressions {
+                key      = "app.kubernetes.io/name"
+                operator = "In"
+                values   = ["coder-workspace"]
+              }
+            }
+          }
+        }
+      }
+      node_affinity {
+        required_during_scheduling_ignored_during_execution {
+          node_selector_term {
+            match_expressions {
+              key      = "cloud.google.com/gke-nodepool"
+              operator = "In"
+              values   = ["big-misc"] # Avoid placing on the same nodes as scaletest workspaces.
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+resource "kubernetes_manifest" "pod_monitor" {
+  count = data.coder_workspace.me.start_count
+  manifest = {
+    apiVersion = "monitoring.coreos.com/v1"
+    kind       = "PodMonitor"
+    metadata = {
+      namespace = local.namespace
+      name      = "podmonitor-${local.workspace_pod_name}"
+    }
+    spec = {
+      selector = {
+        matchLabels = {
+          "app.kubernetes.io/instance" : local.workspace_pod_instance
+        }
+      }
+      podMetricsEndpoints = [
+        {
+          port     = "prometheus-http"
+          interval = "15s"
+        }
+      ]
+    }
+  }
+}
diff --git a/scaletest/templates/scaletest-runner/metadata_phase.sh b/scaletest/templates/scaletest-runner/metadata_phase.sh
new file mode 100755
index 0000000000000..f79e94c0237ba
--- /dev/null
+++ b/scaletest/templates/scaletest-runner/metadata_phase.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+if [ -f /tmp/.scaletest_phase_creating_workspaces ]; then
+	echo "Creating workspaces"
+elif [ -f /tmp/.scaletest_phase_ssh ]; then
+	echo "SSH traffic"
+elif [ -f /tmp/.scaletest_phase_rpty ]; then
+	echo "RPTY traffic"
+elif [ -f /tmp/.scaletest_phase_dashboard ]; then
+	echo "Dashboard traffic"
+elif [ -f /tmp/.scaletest_phase_wait_baseline ]; then
+	echo "Waiting $(</tmp/.scaletest_phase_wait_baseline)m (establishing baseline)"
+else
+	echo "None"
+fi
diff --git a/scaletest/templates/scaletest-runner/metadata_status.sh b/scaletest/templates/scaletest-runner/metadata_status.sh
new file mode 100755
index 0000000000000..552a713cde3a7
--- /dev/null
+++ b/scaletest/templates/scaletest-runner/metadata_status.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+if [ -f /tmp/.scaletest_failed ]; then
+	echo "Failed"
+elif [ -f /tmp/.scaletest_complete ]; then
+	echo "Complete"
+elif [ -f /tmp/.scaletest_running ]; then
+	echo "Running"
+elif [ -f /tmp/.scaletest_preparing ]; then
+	echo "Preparing"
+else
+	echo "Not started"
+fi
diff --git a/scaletest/templates/scaletest-runner/scripts/cleanup.sh b/scaletest/templates/scaletest-runner/scripts/cleanup.sh
new file mode 100755
index 0000000000000..a8d5f5b7c36f7
--- /dev/null
+++ b/scaletest/templates/scaletest-runner/scripts/cleanup.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+set -euo pipefail
+
+[[ $VERBOSE == 1 ]] && set -x
+
+# shellcheck source=../../../../scripts/lib.sh
+. ~/coder/scripts/lib.sh
+
+event=${1:-}
+
+if [[ -z $event ]]; then
+	event=manual
+fi
+
+if [[ $event = manual ]]; then
+	echo -n 'WARNING: This will clean up all scaletest resources, continue? (y/n) '
+	read -r -n 1
+	if [[ $REPLY != [yY] ]]; then
+		echo $'\nAborting...'
+		exit 1
+	fi
+fi
+
+echo "Cleaning up scaletest resources (${event})..."
+
+maybedryrun "${DRY_RUN}" coder exp scaletest cleanup \
+	--cleanup-concurrency "${SCALETEST_CLEANUP_CONCURRENCY}" \
+	--cleanup-job-timeout 15m \
+	--cleanup-timeout 30m \
+	| tee "result-cleanup-${event}.txt"
+
+echo "Cleanup complete!"
+
+if [[ $event = manual ]]; then
+	echo 'Press any key to continue...'
+	read -s -r -n 1
+fi
diff --git a/scaletest/templates/scaletest-runner/scripts/prepare.sh b/scaletest/templates/scaletest-runner/scripts/prepare.sh
new file mode 100755
index 0000000000000..f8bcbbe24b481
--- /dev/null
+++ b/scaletest/templates/scaletest-runner/scripts/prepare.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+set -euo pipefail
+
+[[ $VERBOSE == 1 ]] && set -x
+
+echo "Preparing scaletest workspace environment..."
+touch /tmp/.scaletest_preparing
+
+mkdir -p "${SCALETEST_RUN_DIR}"
+
+echo "Installing prerequisities (terraform, envsubst, gcloud, jq and kubectl)..."
+
+wget --quiet -O /tmp/terraform.zip https://releases.hashicorp.com/terraform/1.5.7/terraform_1.5.7_linux_amd64.zip
+sudo unzip /tmp/terraform.zip -d /usr/local/bin
+terraform --version
+
+wget --quiet -O /tmp/envsubst "https://github.com/a8m/envsubst/releases/download/v1.2.0/envsubst-$(uname -s)-$(uname -m)"
+chmod +x /tmp/envsubst
+sudo mv /tmp/envsubst /usr/local/bin
+
+echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
+curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
+sudo apt-get update
+sudo apt-get install --yes \
+	google-cloud-cli \
+	jq \
+	kubectl
+
+echo "Cloning coder/coder repo..."
+
+if [[ ! -d coder ]]; then
+	git clone https://github.com/coder/coder.git ~/coder
+fi
+(cd ~/coder && git pull)
+
+echo "Creating coder CLI token (needed for cleanup during shutdown)..."
+
+mkdir -p "${CODER_CONFIG_DIR}"
+echo -n "${CODER_URL}" >"${CODER_CONFIG_DIR}/url"
+
+set +x # Avoid logging the token.
+# Persist configuration for shutdown script too since the
+# owner token is invalidated immediately on workspace stop.
+export CODER_SESSION_TOKEN=$CODER_USER_TOKEN
+coder tokens delete scaletest_runner >/dev/null 2>&1 || true
+# TODO(mafredri): Set TTL? This could interfere with delayed stop though.
+token=$(coder tokens create --name scaletest_runner)
+unset CODER_SESSION_TOKEN
+echo -n "${token}" >"${CODER_CONFIG_DIR}/session"
+[[ $VERBOSE == 1 ]] && set -x # Restore logging (if enabled).
+
+echo "Preparation complete!"
diff --git a/scaletest/templates/scaletest-runner/scripts/run.sh b/scaletest/templates/scaletest-runner/scripts/run.sh
new file mode 100755
index 0000000000000..ff1b72eea1fda
--- /dev/null
+++ b/scaletest/templates/scaletest-runner/scripts/run.sh
@@ -0,0 +1,79 @@
+#!/bin/bash
+set -euo pipefail
+
+[[ $VERBOSE == 1 ]] && set -x
+
+# shellcheck source=../../../../scripts/lib.sh
+. ~/coder/scripts/lib.sh
+
+coder() {
+	maybedryrun "$DRY_RUN" command coder "${@}"
+}
+
+show_json() {
+	maybedryrun "$DRY_RUN" jq 'del(.. | .logs?)' "${1}"
+}
+
+wait_baseline() {
+	s=${1:-2}
+	echo "Waiting ${s}m (establishing baseline)..."
+	echo "${s}" >/tmp/.scaletest_phase_wait_baseline
+	sleep $((s * 60))
+	rm /tmp/.scaletest_phase_wait_baseline
+}
+
+echo "Running scaletest..."
+touch /tmp/.scaletest_running
+
+touch /tmp/.scaletest_phase_creating_workspaces
+
+coder exp scaletest create-workspaces \
+	--count "${SCALETEST_NUM_WORKSPACES}" \
+	--template="${SCALETEST_TEMPLATE}" \
+	--concurrency "${SCALETEST_CREATE_CONCURRENCY}" \
+	--job-timeout 15m \
+	--no-cleanup \
+	--output json:"${SCALETEST_RUN_DIR}/result-create-workspaces.json"
+show_json "${SCALETEST_RUN_DIR}/result-create-workspaces.json"
+rm /tmp/.scaletest_phase_creating_workspaces
+
+wait_baseline 5
+
+touch /tmp/.scaletest_phase_ssh
+coder exp scaletest workspace-traffic \
+	--ssh \
+	--bytes-per-tick 10240 \
+	--tick-interval 1s \
+	--concurrency 0 \
+	--timeout 5m \
+	--output json:"${SCALETEST_RUN_DIR}/result-ssh.json"
+show_json "${SCALETEST_RUN_DIR}/result-ssh.json"
+rm /tmp/.scaletest_phase_ssh
+
+wait_baseline 5
+
+touch /tmp/.scaletest_phase_rpty
+coder exp scaletest workspace-traffic \
+	--bytes-per-tick 10240 \
+	--tick-interval 1s \
+	--concurrency 0 \
+	--timeout 5m \
+	--output json:"${SCALETEST_RUN_DIR}/result-rpty.json"
+show_json "${SCALETEST_RUN_DIR}/result-rpty.json"
+rm /tmp/.scaletest_phase_rpty
+
+wait_baseline 5
+
+touch /tmp/.scaletest_phase_dashboard
+coder exp scaletest dashboard \
+	--count "${SCALETEST_NUM_WORKSPACES}" \
+	--concurrency 0 \
+	--job-timeout 5m \
+	--output json:"${SCALETEST_RUN_DIR}/result-dashboard.json"
+show_json "${SCALETEST_RUN_DIR}/result-dashboard.json"
+rm /tmp/.scaletest_phase_dashboard
+
+wait_baseline 5
+
+echo "Scaletest complete!"
+touch /tmp/.scaletest_complete
diff --git a/scaletest/templates/scaletest-runner/shutdown.sh b/scaletest/templates/scaletest-runner/shutdown.sh
new file mode 100755
index 0000000000000..dc19d10eb2779
--- /dev/null
+++ b/scaletest/templates/scaletest-runner/shutdown.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+
+cleanup() {
+	coder tokens remove scaletest_runner >/dev/null 2>&1 || true
+}
+trap cleanup EXIT
+
+"${SCRIPTS_DIR}/cleanup.sh" shutdown
diff --git a/scaletest/templates/scaletest-runner/startup.sh b/scaletest/templates/scaletest-runner/startup.sh
new file mode 100755
index 0000000000000..c6d5eaa964c81
--- /dev/null
+++ b/scaletest/templates/scaletest-runner/startup.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+set -euo pipefail
+
+[[ $VERBOSE == 1 ]] && set -x
+
+# Unzip scripts and add to path.
+echo "Extracting scaletest scripts into ${SCRIPTS_DIR}..."
+base64 -d <<<"${SCRIPTS_ZIP}" >/tmp/scripts.zip
+rm -rf "${SCRIPTS_DIR}" || true
+mkdir -p "${SCRIPTS_DIR}"
+unzip -o /tmp/scripts.zip -d "${SCRIPTS_DIR}"
+rm /tmp/scripts.zip
+
+# Clean up any previous scaletest runs (in case /tmp persists).
+rm -rf /tmp/.scaletest_* || true
+
+# Show failure in the UI if script exits with error.
+failed() {
+	echo "Scaletest failed!"
+	touch /tmp/.scaletest_failed
+}
+trap failed ERR
+
+"${SCRIPTS_DIR}/prepare.sh"
+"${SCRIPTS_DIR}/run.sh"

From 9a579fb97a4e39a6e77d8c420e571f05a580f75d Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Wed, 13 Sep 2023 13:02:31 +0000
Subject: [PATCH 02/36] add setup/scaletest-sa.yaml

---
 scaletest/setup/scaletest-sa.yaml | 50 +++++++++++++++++++++++++++++++
 1 file changed, 50 insertions(+)
 create mode 100644 scaletest/setup/scaletest-sa.yaml

diff --git a/scaletest/setup/scaletest-sa.yaml b/scaletest/setup/scaletest-sa.yaml
new file mode 100644
index 0000000000000..b6aad5ae820aa
--- /dev/null
+++ b/scaletest/setup/scaletest-sa.yaml
@@ -0,0 +1,50 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: scaletest-sa
+  namespace: coder-big
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: scaletest-role
+  namespace: coder-big
+rules:
+  - apiGroups:
+    - ""
+    resources: ["*"]
+    verbs:
+    - get
+    - list
+    - watch
+    - create
+    - update
+    - patch
+    - delete
+    - deletecollection
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: scaletest-rolebinding
+  namespace: coder-big
+subjects:
+  - kind: ServiceAccount
+    name: scaletest-sa
+roleRef:
+  kind: Role
+  name: scaletest-role
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: coder-provisioner-podmonitor-rolebinding
+  namespace: coder-big
+subjects:
+  - kind: ServiceAccount
+    name: coder-provisioner
+roleRef:
+  kind: Role
+  name: coder-podmonitor
+---

From cde438898b1af935be75bd1be15e800ee5976fb7 Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Wed, 13 Sep 2023 13:07:00 +0000
Subject: [PATCH 03/36] fix typo

---
 scaletest/templates/scaletest-runner/scripts/prepare.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scaletest/templates/scaletest-runner/scripts/prepare.sh b/scaletest/templates/scaletest-runner/scripts/prepare.sh
index f8bcbbe24b481..31a224b52a45c 100755
--- a/scaletest/templates/scaletest-runner/scripts/prepare.sh
+++ b/scaletest/templates/scaletest-runner/scripts/prepare.sh
@@ -8,7 +8,7 @@ touch /tmp/.scaletest_preparing
 
 mkdir -p "${SCALETEST_RUN_DIR}"
 
-echo "Installing prerequisities (terraform, envsubst, gcloud, jq and kubectl)..."
+echo "Installing prerequisites (terraform, envsubst, gcloud, jq and kubectl)..."
 
 wget --quiet -O /tmp/terraform.zip https://releases.hashicorp.com/terraform/1.5.7/terraform_1.5.7_linux_amd64.zip
 sudo unzip /tmp/terraform.zip -d /usr/local/bin

From cd20cb83b73e5adb4c6fb271513fea6a9f31e2f2 Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Wed, 13 Sep 2023 13:10:29 +0000
Subject: [PATCH 04/36] shfmt

---
 scaletest/templates/scaletest-runner/scripts/cleanup.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scaletest/templates/scaletest-runner/scripts/cleanup.sh b/scaletest/templates/scaletest-runner/scripts/cleanup.sh
index a8d5f5b7c36f7..dce53e0613e16 100755
--- a/scaletest/templates/scaletest-runner/scripts/cleanup.sh
+++ b/scaletest/templates/scaletest-runner/scripts/cleanup.sh
@@ -26,8 +26,8 @@ echo "Cleaning up scaletest resources (${event})..."
 maybedryrun "${DRY_RUN}" coder exp scaletest cleanup \
 	--cleanup-concurrency "${SCALETEST_CLEANUP_CONCURRENCY}" \
 	--cleanup-job-timeout 15m \
-	--cleanup-timeout 30m \
-	| tee "result-cleanup-${event}.txt"
+	--cleanup-timeout 30m |
+	tee "result-cleanup-${event}.txt"
 
 echo "Cleanup complete!"
 

From 12ecd7140a60e79c0668f61e2b7cd2942186d1a5 Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Wed, 13 Sep 2023 13:14:17 +0000
Subject: [PATCH 05/36] fix typo

---
 scaletest/templates/scaletest-runner/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scaletest/templates/scaletest-runner/main.tf b/scaletest/templates/scaletest-runner/main.tf
index 6721f7d6ecbaf..a22be108bb0f2 100644
--- a/scaletest/templates/scaletest-runner/main.tf
+++ b/scaletest/templates/scaletest-runner/main.tf
@@ -116,7 +116,7 @@ data "coder_parameter" "num_workspaces" {
   type        = "number"
   name        = "Number of workspaces to create"
   default     = 100
-  description = "The scaletest suite will create this numer of workspaces."
+  description = "The scaletest suite will create this number of workspaces."
   mutable     = true
   ephemeral   = true
 

From 55d6fd823e4772efb7150ad48a87a75b1d282bdf Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Wed, 13 Sep 2023 13:17:05 +0000
Subject: [PATCH 06/36] fix shellcheck

---
 scaletest/templates/scaletest-runner/scripts/cleanup.sh | 2 +-
 scaletest/templates/scaletest-runner/scripts/run.sh     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/scaletest/templates/scaletest-runner/scripts/cleanup.sh b/scaletest/templates/scaletest-runner/scripts/cleanup.sh
index dce53e0613e16..909ac7c309df5 100755
--- a/scaletest/templates/scaletest-runner/scripts/cleanup.sh
+++ b/scaletest/templates/scaletest-runner/scripts/cleanup.sh
@@ -3,7 +3,7 @@ set -euo pipefail
 
 [[ $VERBOSE == 1 ]] && set -x
 
-# shellcheck source=../../../../scripts/lib.sh
+# shellcheck source=scripts/lib.sh
 . ~/coder/scripts/lib.sh
 
 event=${1:-}
diff --git a/scaletest/templates/scaletest-runner/scripts/run.sh b/scaletest/templates/scaletest-runner/scripts/run.sh
index ff1b72eea1fda..0a91b18809360 100755
--- a/scaletest/templates/scaletest-runner/scripts/run.sh
+++ b/scaletest/templates/scaletest-runner/scripts/run.sh
@@ -3,7 +3,7 @@ set -euo pipefail
 
 [[ $VERBOSE == 1 ]] && set -x
 
-# shellcheck source=../../../../scripts/lib.sh
+# shellcheck source=scripts/lib.sh
 . ~/coder/scripts/lib.sh
 
 coder() {

From b251af975c27b623596d22f4a0319ad1f7a7f61c Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Wed, 13 Sep 2023 14:01:35 +0000
Subject: [PATCH 07/36] fmt

---
 scaletest/setup/scaletest-sa.yaml             | 19 ++++++++++---------
 .../templates/scaletest-runner/README.md      |  2 +-
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/scaletest/setup/scaletest-sa.yaml b/scaletest/setup/scaletest-sa.yaml
index b6aad5ae820aa..96d9747d84b3e 100644
--- a/scaletest/setup/scaletest-sa.yaml
+++ b/scaletest/setup/scaletest-sa.yaml
@@ -12,17 +12,17 @@ metadata:
   namespace: coder-big
 rules:
   - apiGroups:
-    - ""
+      - ""
     resources: ["*"]
     verbs:
-    - get
-    - list
-    - watch
-    - create
-    - update
-    - patch
-    - delete
-    - deletecollection
+      - get
+      - list
+      - watch
+      - create
+      - update
+      - patch
+      - delete
+      - deletecollection
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
@@ -48,3 +48,4 @@ roleRef:
   kind: Role
   name: coder-podmonitor
 ---
+
diff --git a/scaletest/templates/scaletest-runner/README.md b/scaletest/templates/scaletest-runner/README.md
index 3bb65379915b0..6c048211e1ad4 100644
--- a/scaletest/templates/scaletest-runner/README.md
+++ b/scaletest/templates/scaletest-runner/README.md
@@ -6,4 +6,4 @@ tags: [local]
 
 # Scaletest Runner
 
-Run a scaletest.
\ No newline at end of file
+Run a scaletest.

From 1b1b4012f91e663605287ad03a07e6e530a997bb Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Wed, 13 Sep 2023 14:16:10 +0000
Subject: [PATCH 08/36] improve precondition

---
 scaletest/templates/scaletest-runner/main.tf | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/scaletest/templates/scaletest-runner/main.tf b/scaletest/templates/scaletest-runner/main.tf
index a22be108bb0f2..196140040059d 100644
--- a/scaletest/templates/scaletest-runner/main.tf
+++ b/scaletest/templates/scaletest-runner/main.tf
@@ -40,12 +40,8 @@ resource "null_resource" "permission_check" {
   # for the plan, and consequently, updating the template.
   lifecycle {
     precondition {
-      condition     = can(regex("^(default|scaletest)$", data.coder_workspace.me.owner))
-      error_message = "User is not allowed, expected 'scaletest'."
-    }
-    precondition {
-      condition     = can(regex("^(default|runner)$", data.coder_workspace.me.name))
-      error_message = "Workspace name is not allowed, expected 'runner'."
+      condition     = can(regex("^(default/default|scaletest/runner)$", "${data.coder_workspace.me.owner}/${data.coder_workspace.me.name}"))
+      error_message = "User and workspace name is not allowed, expected 'scaletest/runner'."
     }
   }
 }

From fa0de669292324eade65464e46da14e57fc40e20 Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Thu, 14 Sep 2023 12:09:17 +0000
Subject: [PATCH 09/36] turn namespace into param

---
 scaletest/templates/scaletest-runner/main.tf | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/scaletest/templates/scaletest-runner/main.tf b/scaletest/templates/scaletest-runner/main.tf
index 196140040059d..4a481cb12eaee 100644
--- a/scaletest/templates/scaletest-runner/main.tf
+++ b/scaletest/templates/scaletest-runner/main.tf
@@ -21,7 +21,6 @@ resource "time_static" "start_time" {
 }
 
 locals {
-  namespace              = "coder-big"
   workspace_pod_name     = "coder-scaletest-runner-${lower(data.coder_workspace.me.owner)}-${lower(data.coder_workspace.me.name)}"
   workspace_pod_instance = "coder-workspace-${lower(data.coder_workspace.me.owner)}-${lower(data.coder_workspace.me.name)}"
   service_account_name   = "scaletest-sa"
@@ -122,6 +121,14 @@ data "coder_parameter" "num_workspaces" {
   }
 }
 
+data "coder_parameter" "namespace" {
+  order       = 999
+  type        = "string"
+  name        = "namespace"
+  default     = "coder-big"
+  description = "The Kubernetes namespace to create the scaletest runner resources in."
+}
+
 data "archive_file" "scripts_zip" {
   type        = "zip"
   output_path = "${path.module}/scripts.zip"
@@ -262,7 +269,7 @@ resource "kubernetes_persistent_volume_claim" "home" {
   depends_on = [null_resource.permission_check]
   metadata {
     name      = "${local.workspace_pod_name}-home"
-    namespace = local.namespace
+    namespace = data.coder_parameter.namespace.value
     labels = {
       "app.kubernetes.io/name"     = "coder-pvc"
       "app.kubernetes.io/instance" = "coder-pvc-${lower(data.coder_workspace.me.owner)}-${lower(data.coder_workspace.me.name)}"
@@ -294,7 +301,7 @@ resource "kubernetes_pod" "main" {
   count      = data.coder_workspace.me.start_count
   metadata {
     name      = local.workspace_pod_name
-    namespace = local.namespace
+    namespace = data.coder_parameter.namespace.value
     labels = {
       "app.kubernetes.io/name"     = "coder-workspace"
       "app.kubernetes.io/instance" = local.workspace_pod_instance

From 77d575cf381e5ee881a70d340161c9a05a4f892a Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Thu, 14 Sep 2023 12:11:14 +0000
Subject: [PATCH 10/36] turn some params to non-ephemeral

---
 scaletest/templates/scaletest-runner/main.tf | 2 --
 1 file changed, 2 deletions(-)

diff --git a/scaletest/templates/scaletest-runner/main.tf b/scaletest/templates/scaletest-runner/main.tf
index 4a481cb12eaee..6fa2815fbb611 100644
--- a/scaletest/templates/scaletest-runner/main.tf
+++ b/scaletest/templates/scaletest-runner/main.tf
@@ -79,7 +79,6 @@ data "coder_parameter" "workspace_template" {
   default      = "kubernetes-minimal"
   icon         = "/emojis/1f4dc.png" # Scroll.
   mutable      = true
-  ephemeral    = true
   option {
     name        = "Minimal"
     value       = "kubernetes-minimal" # Feather.
@@ -113,7 +112,6 @@ data "coder_parameter" "num_workspaces" {
   default     = 100
   description = "The scaletest suite will create this number of workspaces."
   mutable     = true
-  ephemeral   = true
 
   validation {
     min = 0

From abab20f541e193089db3aa44db8225e10f6835c8 Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Thu, 14 Sep 2023 12:24:57 +0000
Subject: [PATCH 11/36] expose concurrency knobs as parameters

---
 scaletest/templates/scaletest-runner/main.tf  | 60 +++++++++++++++++--
 .../templates/scaletest-runner/scripts/run.sh |  5 +-
 2 files changed, 57 insertions(+), 8 deletions(-)

diff --git a/scaletest/templates/scaletest-runner/main.tf b/scaletest/templates/scaletest-runner/main.tf
index 6fa2815fbb611..6ad92496d714f 100644
--- a/scaletest/templates/scaletest-runner/main.tf
+++ b/scaletest/templates/scaletest-runner/main.tf
@@ -71,8 +71,56 @@ data "coder_parameter" "dry_run" {
   ephemeral   = true
 }
 
+data "coder_parameter" "create_concurrency" {
+  order       = 10
+  type        = "number"
+  name        = "Create concurrency"
+  default     = 10
+  description = "The number of workspaces to create concurrently."
+  mutable     = true
+
+  # Setting zero = unlimited, but perhaps not a good idea,
+  # we can raise this limit instead.
+  validation {
+    min = 1
+    max = 100
+  }
+}
+
+data "coder_parameter" "job_concurrency" {
+  order       = 11
+  type        = "number"
+  name        = "Scaletest job concurrency"
+  default     = 10
+  description = "The number of concurrent jobs (e.g. when producing workspace traffic)."
+  mutable     = true
+
+  # Setting zero = unlimited, but perhaps not a good idea,
+  # we can raise this limit instead.
+  validation {
+    min = 1
+    max = 100
+  }
+}
+
+data "coder_parameter" "cleanup_concurrency" {
+  order       = 12
+  type        = "number"
+  name        = "Cleanup concurrency"
+  default     = 10
+  description = "The number of concurrent cleanup jobs."
+  mutable     = true
+
+  # Setting zero = unlimited, but perhaps not a good idea,
+  # we can raise this limit instead.
+  validation {
+    min = 1
+    max = 100
+  }
+}
+
 data "coder_parameter" "workspace_template" {
-  order        = 3
+  order        = 20
   name         = "workspace_template"
   display_name = "Workspace Template"
   description  = "The template used for workspace creation."
@@ -106,7 +154,7 @@ data "coder_parameter" "workspace_template" {
 }
 
 data "coder_parameter" "num_workspaces" {
-  order       = 10
+  order       = 21
   type        = "number"
   name        = "Number of workspaces to create"
   default     = 100
@@ -142,16 +190,20 @@ resource "coder_agent" "main" {
     CODER_CONFIG_DIR : "/home/coder/.config/coderv2",
     CODER_USER_TOKEN : data.coder_workspace.me.owner_session_token,
     CODER_URL : data.coder_workspace.me.access_url,
+
+    # Global scaletest envs that may affect each `coder exp scaletest` invocation.
     CODER_SCALETEST_PROMETHEUS_ADDRESS : "0.0.0.0:21112",
     CODER_SCALETEST_PROMETHEUS_WAIT : "60s",
+    CODER_SCALETEST_CONCURRENCY : "${data.coder_parameter.job_concurrency.value}",
+    CODER_SCALETEST_CLEANUP_CONCURRENCY : "${data.coder_parameter.cleanup_concurrency.value}",
 
+    # Local envs passed as arguments to `coder exp scaletest` invocations.
     SCALETEST_RUN_ID : local.scaletest_run_id,
     SCALETEST_RUN_DIR : local.scaletest_run_dir,
     SCALETEST_TEMPLATE : data.coder_parameter.workspace_template.value,
     SCALETEST_SKIP_CLEANUP : "1",
     SCALETEST_NUM_WORKSPACES : data.coder_parameter.num_workspaces.value,
-    SCALETEST_CREATE_CONCURRENCY : "10",
-    SCALETEST_CLEANUP_CONCURRENCY : "10",
+    SCALETEST_CREATE_CONCURRENCY : "${data.coder_parameter.create_concurrency.value}",
 
     SCRIPTS_ZIP : filebase64(data.archive_file.scripts_zip.output_path),
     SCRIPTS_DIR : "/tmp/scripts",
diff --git a/scaletest/templates/scaletest-runner/scripts/run.sh b/scaletest/templates/scaletest-runner/scripts/run.sh
index 0a91b18809360..a000a4e9f25bc 100755
--- a/scaletest/templates/scaletest-runner/scripts/run.sh
+++ b/scaletest/templates/scaletest-runner/scripts/run.sh
@@ -29,7 +29,7 @@ touch /tmp/.scaletest_phase_creating_workspaces
 
 coder exp scaletest create-workspaces \
 	--count "${SCALETEST_NUM_WORKSPACES}" \
-	--template="${SCALETEST_TEMPLATE}" \
+	--template "${SCALETEST_TEMPLATE}" \
 	--concurrency "${SCALETEST_CREATE_CONCURRENCY}" \
 	--job-timeout 15m \
 	--no-cleanup \
@@ -44,7 +44,6 @@ coder exp scaletest workspace-traffic \
 	--ssh \
 	--bytes-per-tick 10240 \
 	--tick-interval 1s \
-	--concurrency 0 \
 	--timeout 5m \
 	--output json:"${SCALETEST_RUN_DIR}/result-ssh.json"
 show_json "${SCALETEST_RUN_DIR}/result-ssh.json"
@@ -56,7 +55,6 @@ touch /tmp/.scaletest_phase_rpty
 coder exp scaletest workspace-traffic \
 	--bytes-per-tick 10240 \
 	--tick-interval 1s \
-	--concurrency 0 \
 	--timeout 5m \
 	--output json:"${SCALETEST_RUN_DIR}/result-rpty.json"
 show_json "${SCALETEST_RUN_DIR}/result-rpty.json"
@@ -67,7 +65,6 @@ wait_baseline 5
 touch /tmp/.scaletest_phase_dashboard
 coder exp scaletest dashboard \
 	--count "${SCALETEST_NUM_WORKSPACES}" \
-	--concurrency 0 \
 	--job-timeout 5m \
 	--output json:"${SCALETEST_RUN_DIR}/result-dashboard.json"
 show_json "${SCALETEST_RUN_DIR}/result-dashboard.json"

From dac2ff8d4f103b0152e52a2dcfbaedeaf2c1e963 Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Thu, 14 Sep 2023 13:19:02 +0000
Subject: [PATCH 12/36] refactor phase, create lib

---
 .../scaletest-runner/metadata_phase.sh        | 20 ++++----
 .../scaletest-runner/metadata_status.sh       |  9 ++--
 .../scaletest-runner/scripts/cleanup.sh       | 16 +++----
 .../templates/scaletest-runner/scripts/lib.sh | 46 +++++++++++++++++++
 .../templates/scaletest-runner/scripts/run.sh | 45 ++++++------------
 .../templates/scaletest-runner/shutdown.sh    |  5 ++
 .../templates/scaletest-runner/startup.sh     |  4 ++
 7 files changed, 91 insertions(+), 54 deletions(-)
 create mode 100644 scaletest/templates/scaletest-runner/scripts/lib.sh

diff --git a/scaletest/templates/scaletest-runner/metadata_phase.sh b/scaletest/templates/scaletest-runner/metadata_phase.sh
index f79e94c0237ba..1bb8e02b409e3 100755
--- a/scaletest/templates/scaletest-runner/metadata_phase.sh
+++ b/scaletest/templates/scaletest-runner/metadata_phase.sh
@@ -1,15 +1,15 @@
 #!/bin/bash
 
-if [ -f /tmp/.scaletest_phase_creating_workspaces ]; then
-	echo "Creating workspaces"
-elif [ -f /tmp/.scaletest_phase_ssh ]; then
-	echo "SSH traffic"
-elif [ -f /tmp/.scaletest_phase_rpty ]; then
-	echo "RPTY traffic"
-elif [ -f /tmp/.scaletest_phase_dashboard ]; then
-	echo "Dashboard traffic"
-elif [ -f /tmp/.scaletest_phase_wait_baseline ]; then
-	echo "Waiting $(</tmp/.scaletest_phase_wait_baseline)m (establishing baseline)"
+# shellcheck disable=SC2153 source=scaletest/templates/scaletest-runner/scripts/lib.sh
+. "${SCRIPTS_DIR}/lib.sh"
+
+if [[ -f "${SCALETEST_PHASE_FILE}" ]]; then
+	phase_raw="$(tail -n1 "${SCALETEST_PHASE_FILE}")"
+	phase="$(echo "${phase_raw}" | cut -d' ' -f3-)"
+	if [[ ${phase_raw} == *"END:"* ]]; then
+		phase+=" (done)"
+	fi
+	echo "${phase}"
 else
 	echo "None"
 fi
diff --git a/scaletest/templates/scaletest-runner/metadata_status.sh b/scaletest/templates/scaletest-runner/metadata_status.sh
index 552a713cde3a7..73209303d5845 100755
--- a/scaletest/templates/scaletest-runner/metadata_status.sh
+++ b/scaletest/templates/scaletest-runner/metadata_status.sh
@@ -1,12 +1,13 @@
 #!/bin/bash
 
-if [ -f /tmp/.scaletest_failed ]; then
+# Order of importance (reverse of creation).
+if [[ -f /tmp/.scaletest_failed ]]; then
 	echo "Failed"
-elif [ -f /tmp/.scaletest_complete ]; then
+elif [[ -f /tmp/.scaletest_complete ]]; then
 	echo "Complete"
-elif [ -f /tmp/.scaletest_running ]; then
+elif [[ -f /tmp/.scaletest_running ]]; then
 	echo "Running"
-elif [ -f /tmp/.scaletest_preparing ]; then
+elif [[ -f /tmp/.scaletest_preparing ]]; then
 	echo "Preparing"
 else
 	echo "Not started"
diff --git a/scaletest/templates/scaletest-runner/scripts/cleanup.sh b/scaletest/templates/scaletest-runner/scripts/cleanup.sh
index 909ac7c309df5..f4ede63040776 100755
--- a/scaletest/templates/scaletest-runner/scripts/cleanup.sh
+++ b/scaletest/templates/scaletest-runner/scripts/cleanup.sh
@@ -3,8 +3,8 @@ set -euo pipefail
 
 [[ $VERBOSE == 1 ]] && set -x
 
-# shellcheck source=scripts/lib.sh
-. ~/coder/scripts/lib.sh
+# shellcheck disable=SC2153 source=scaletest/templates/scaletest-runner/scripts/lib.sh
+. "${SCRIPTS_DIR}/lib.sh"
 
 event=${1:-}
 
@@ -21,15 +21,13 @@ if [[ $event = manual ]]; then
 	fi
 fi
 
-echo "Cleaning up scaletest resources (${event})..."
-
-maybedryrun "${DRY_RUN}" coder exp scaletest cleanup \
+phase_start "Cleanup (${event})"
+coder exp scaletest cleanup \
 	--cleanup-concurrency "${SCALETEST_CLEANUP_CONCURRENCY}" \
 	--cleanup-job-timeout 15m \
-	--cleanup-timeout 30m |
-	tee "result-cleanup-${event}.txt"
-
-echo "Cleanup complete!"
+	--cleanup-timeout 30m \
+	| tee "result-cleanup-${event}.txt"
+end_phase
 
 if [[ $event = manual ]]; then
 	echo 'Press any key to continue...'
diff --git a/scaletest/templates/scaletest-runner/scripts/lib.sh b/scaletest/templates/scaletest-runner/scripts/lib.sh
new file mode 100644
index 0000000000000..9df42ef364433
--- /dev/null
+++ b/scaletest/templates/scaletest-runner/scripts/lib.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+set -euo pipefail
+
+# Only source this script once, this env comes from sourcing
+# scripts/lib.sh from coder/coder below.
+if [[ ${SCRIPTS_LIB_IS_SOURCED} == 1 ]]; then
+	return 0
+fi
+
+# Source scripts/lib.sh from coder/coder for common functions.
+# shellcheck source=scripts/lib.sh
+. ~/coder/scripts/lib.sh
+
+# Environment variables shared between scripts.
+SCALETEST_PHASE_FILE=/tmp/.scaletest_phase
+
+coder() {
+	maybedryrun "$DRY_RUN" command coder "${@}"
+}
+
+show_json() {
+	maybedryrun "$DRY_RUN" jq 'del(.. | .logs?)' "${1}"
+}
+
+phase_num=0
+start_phase() {
+	((phase_num++))
+	log "Start phase ${phase_num}: ${*}"
+	echo "$(date -Iseconds) START:${phase_num}: ${*}" >>"${SCALETEST_PHASE_FILE}"
+}
+end_phase() {
+	phase="$(tail -n 1 "${SCALETEST_PHASE_FILE}" | grep "START:${phase_num}:" | cut -d' ' -f3-)"
+	if [[ -z ${phase} ]]; then
+		log "BUG: Could not find start phase ${phase_num} in ${SCALETEST_PHASE_FILE}"
+		exit 1
+	fi
+	log "End phase ${phase_num}: ${phase}"
+	echo "$(date -Iseconds) END:${phase_num}: ${phase}" >>"${SCALETEST_PHASE_FILE}"
+}
+
+wait_baseline() {
+	s=${1:-2}
+	start_phase "Waiting ${s}m to establish baseline"
+	sleep $((s * 60))
+	end_phase
+}
diff --git a/scaletest/templates/scaletest-runner/scripts/run.sh b/scaletest/templates/scaletest-runner/scripts/run.sh
index a000a4e9f25bc..cae4ff1e40851 100755
--- a/scaletest/templates/scaletest-runner/scripts/run.sh
+++ b/scaletest/templates/scaletest-runner/scripts/run.sh
@@ -3,30 +3,13 @@ set -euo pipefail
 
 [[ $VERBOSE == 1 ]] && set -x
 
-# shellcheck source=scripts/lib.sh
-. ~/coder/scripts/lib.sh
+# shellcheck disable=SC2153 source=scaletest/templates/scaletest-runner/scripts/lib.sh
+. "${SCRIPTS_DIR}/lib.sh"
 
-coder() {
-	maybedryrun "$DRY_RUN" command coder "${@}"
-}
-
-show_json() {
-	maybedryrun "$DRY_RUN" jq 'del(.. | .logs?)' "${1}"
-}
-
-wait_baseline() {
-	s=${1:-2}
-	echo "Waiting ${s}m (establishing baseline)..."
-	echo "${s}" >/tmp/.scaletest_phase_wait_baseline
-	sleep $((s * 60))
-	rm /tmp/.scaletest_phase_wait_baseline
-}
-
-echo "Running scaletest..."
+log "Running scaletest..."
 touch /tmp/.scaletest_running
 
-touch /tmp/.scaletest_phase_creating_workspaces
-
+start_phase "Creating workspaces"
 coder exp scaletest create-workspaces \
 	--count "${SCALETEST_NUM_WORKSPACES}" \
 	--template "${SCALETEST_TEMPLATE}" \
@@ -35,11 +18,11 @@ coder exp scaletest create-workspaces \
 	--no-cleanup \
 	--output json:"${SCALETEST_RUN_DIR}/result-create-workspaces.json"
 show_json "${SCALETEST_RUN_DIR}/result-create-workspaces.json"
-rm /tmp/.scaletest_phase_creating_workspaces
+end_phase
 
 wait_baseline 5
 
-touch /tmp/.scaletest_phase_ssh
+start_phase "SSH traffic"
 coder exp scaletest workspace-traffic \
 	--ssh \
 	--bytes-per-tick 10240 \
@@ -47,30 +30,30 @@ coder exp scaletest workspace-traffic \
 	--timeout 5m \
 	--output json:"${SCALETEST_RUN_DIR}/result-ssh.json"
 show_json "${SCALETEST_RUN_DIR}/result-ssh.json"
-rm /tmp/.scaletest_phase_ssh
+end_phase
 
 wait_baseline 5
 
-touch /tmp/.scaletest_phase_rpty
+start_phase "ReconnectingPTY traffic"
 coder exp scaletest workspace-traffic \
 	--bytes-per-tick 10240 \
 	--tick-interval 1s \
 	--timeout 5m \
-	--output json:"${SCALETEST_RUN_DIR}/result-rpty.json"
-show_json "${SCALETEST_RUN_DIR}/result-rpty.json"
-rm /tmp/.scaletest_phase_rpty
+	--output json:"${SCALETEST_RUN_DIR}/result-reconnectingpty.json"
+show_json "${SCALETEST_RUN_DIR}/result-reconnectingpty.json"
+end_phase
 
 wait_baseline 5
 
-touch /tmp/.scaletest_phase_dashboard
+start_phase "Dashboard traffic"
 coder exp scaletest dashboard \
 	--count "${SCALETEST_NUM_WORKSPACES}" \
 	--job-timeout 5m \
 	--output json:"${SCALETEST_RUN_DIR}/result-dashboard.json"
 show_json "${SCALETEST_RUN_DIR}/result-dashboard.json"
-rm /tmp/.scaletest_phase_dashboard
+end_phase
 
 wait_baseline 5
 
-echo "Scaletest complete!"
+log "Scaletest complete!"
 touch /tmp/.scaletest_complete
diff --git a/scaletest/templates/scaletest-runner/shutdown.sh b/scaletest/templates/scaletest-runner/shutdown.sh
index dc19d10eb2779..fe621afe4c6c4 100755
--- a/scaletest/templates/scaletest-runner/shutdown.sh
+++ b/scaletest/templates/scaletest-runner/shutdown.sh
@@ -1,6 +1,11 @@
 #!/bin/bash
 set -e
 
+[[ $VERBOSE == 1 ]] && set -x
+
+# shellcheck disable=SC2153 source=scaletest/templates/scaletest-runner/scripts/lib.sh
+. "${SCRIPTS_DIR}/lib.sh"
+
 cleanup() {
 	coder tokens remove scaletest_runner >/dev/null 2>&1 || true
 }
diff --git a/scaletest/templates/scaletest-runner/startup.sh b/scaletest/templates/scaletest-runner/startup.sh
index c6d5eaa964c81..7e939f2f7cbb4 100755
--- a/scaletest/templates/scaletest-runner/startup.sh
+++ b/scaletest/templates/scaletest-runner/startup.sh
@@ -4,6 +4,7 @@ set -euo pipefail
 [[ $VERBOSE == 1 ]] && set -x
 
 # Unzip scripts and add to path.
+# shellcheck disable=SC2153
 echo "Extracting scaletest scripts into ${SCRIPTS_DIR}..."
 base64 -d <<<"${SCRIPTS_ZIP}" >/tmp/scripts.zip
 rm -rf "${SCRIPTS_DIR}" || true
@@ -11,6 +12,9 @@ mkdir -p "${SCRIPTS_DIR}"
 unzip -o /tmp/scripts.zip -d "${SCRIPTS_DIR}"
 rm /tmp/scripts.zip
 
+# shellcheck disable=SC2153 source=scaletest/templates/scaletest-runner/scripts/lib.sh
+. "${SCRIPTS_DIR}/lib.sh"
+
 # Clean up any previous scaletest runs (in case /tmp persists).
 rm -rf /tmp/.scaletest_* || true
 

From 059988e106b157c5079ddb2f51fc3b79a5e431bc Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Thu, 14 Sep 2023 13:33:52 +0000
Subject: [PATCH 13/36] refactor to use state dir in run dir

---
 .../scaletest-runner/metadata_phase.sh        | 11 +------
 .../scaletest-runner/metadata_status.sh       | 16 +++-------
 .../templates/scaletest-runner/scripts/lib.sh | 30 ++++++++++++++++++-
 .../scaletest-runner/scripts/prepare.sh       |  9 ++++--
 .../templates/scaletest-runner/scripts/run.sh |  4 +--
 .../templates/scaletest-runner/startup.sh     |  8 ++---
 6 files changed, 45 insertions(+), 33 deletions(-)

diff --git a/scaletest/templates/scaletest-runner/metadata_phase.sh b/scaletest/templates/scaletest-runner/metadata_phase.sh
index 1bb8e02b409e3..755a8ba084db7 100755
--- a/scaletest/templates/scaletest-runner/metadata_phase.sh
+++ b/scaletest/templates/scaletest-runner/metadata_phase.sh
@@ -3,13 +3,4 @@
 # shellcheck disable=SC2153 source=scaletest/templates/scaletest-runner/scripts/lib.sh
 . "${SCRIPTS_DIR}/lib.sh"
 
-if [[ -f "${SCALETEST_PHASE_FILE}" ]]; then
-	phase_raw="$(tail -n1 "${SCALETEST_PHASE_FILE}")"
-	phase="$(echo "${phase_raw}" | cut -d' ' -f3-)"
-	if [[ ${phase_raw} == *"END:"* ]]; then
-		phase+=" (done)"
-	fi
-	echo "${phase}"
-else
-	echo "None"
-fi
+get_phase
diff --git a/scaletest/templates/scaletest-runner/metadata_status.sh b/scaletest/templates/scaletest-runner/metadata_status.sh
index 73209303d5845..6c294e1ea018c 100755
--- a/scaletest/templates/scaletest-runner/metadata_status.sh
+++ b/scaletest/templates/scaletest-runner/metadata_status.sh
@@ -1,14 +1,6 @@
 #!/bin/bash
 
-# Order of importance (reverse of creation).
-if [[ -f /tmp/.scaletest_failed ]]; then
-	echo "Failed"
-elif [[ -f /tmp/.scaletest_complete ]]; then
-	echo "Complete"
-elif [[ -f /tmp/.scaletest_running ]]; then
-	echo "Running"
-elif [[ -f /tmp/.scaletest_preparing ]]; then
-	echo "Preparing"
-else
-	echo "Not started"
-fi
+# shellcheck disable=SC2153 source=scaletest/templates/scaletest-runner/scripts/lib.sh
+. "${SCRIPTS_DIR}/lib.sh"
+
+get_status
diff --git a/scaletest/templates/scaletest-runner/scripts/lib.sh b/scaletest/templates/scaletest-runner/scripts/lib.sh
index 9df42ef364433..cfb31f7ff2e1b 100644
--- a/scaletest/templates/scaletest-runner/scripts/lib.sh
+++ b/scaletest/templates/scaletest-runner/scripts/lib.sh
@@ -12,7 +12,8 @@ fi
 . ~/coder/scripts/lib.sh
 
 # Environment variables shared between scripts.
-SCALETEST_PHASE_FILE=/tmp/.scaletest_phase
+SCALETEST_STATE_DIR="${SCALETEST_RUN_DIR}/state"
+SCALETEST_PHASE_FILE="${SCALETEST_STATE_DIR}/phase"
 
 coder() {
 	maybedryrun "$DRY_RUN" command coder "${@}"
@@ -22,6 +23,21 @@ show_json() {
 	maybedryrun "$DRY_RUN" jq 'del(.. | .logs?)' "${1}"
 }
 
+set_status() {
+	echo "$*" >"${SCALETEST_STATE_DIR}/status"
+}
+lock_status() {
+	chmod 0440 "${SCALETEST_STATE_DIR}/status"
+}
+get_status() {
+	# Order of importance (reverse of creation).
+	if [[ -f "${SCALETEST_STATE_DIR}/status" ]]; then
+		cat "${SCALETEST_STATE_DIR}/status"
+	else
+		echo "Not started"
+	fi
+}
+
 phase_num=0
 start_phase() {
 	((phase_num++))
@@ -37,6 +53,18 @@ end_phase() {
 	log "End phase ${phase_num}: ${phase}"
 	echo "$(date -Iseconds) END:${phase_num}: ${phase}" >>"${SCALETEST_PHASE_FILE}"
 }
+get_phase() {
+	if [[ -f "${SCALETEST_PHASE_FILE}" ]]; then
+		phase_raw="$(tail -n1 "${SCALETEST_PHASE_FILE}")"
+		phase="$(echo "${phase_raw}" | cut -d' ' -f3-)"
+		if [[ ${phase_raw} == *"END:"* ]]; then
+			phase+=" (done)"
+		fi
+		echo "${phase}"
+	else
+		echo "None"
+	fi
+}
 
 wait_baseline() {
 	s=${1:-2}
diff --git a/scaletest/templates/scaletest-runner/scripts/prepare.sh b/scaletest/templates/scaletest-runner/scripts/prepare.sh
index 31a224b52a45c..39a8caec0da06 100755
--- a/scaletest/templates/scaletest-runner/scripts/prepare.sh
+++ b/scaletest/templates/scaletest-runner/scripts/prepare.sh
@@ -3,10 +3,13 @@ set -euo pipefail
 
 [[ $VERBOSE == 1 ]] && set -x
 
-echo "Preparing scaletest workspace environment..."
-touch /tmp/.scaletest_preparing
+# shellcheck disable=SC2153 source=scaletest/templates/scaletest-runner/scripts/lib.sh
+. "${SCRIPTS_DIR}/lib.sh"
+
+mkdir -p "${SCALETEST_STATE_DIR}"
 
-mkdir -p "${SCALETEST_RUN_DIR}"
+echo "Preparing scaletest workspace environment..."
+set_status Preparing
 
 echo "Installing prerequisites (terraform, envsubst, gcloud, jq and kubectl)..."
 
diff --git a/scaletest/templates/scaletest-runner/scripts/run.sh b/scaletest/templates/scaletest-runner/scripts/run.sh
index cae4ff1e40851..694403ed18121 100755
--- a/scaletest/templates/scaletest-runner/scripts/run.sh
+++ b/scaletest/templates/scaletest-runner/scripts/run.sh
@@ -7,7 +7,7 @@ set -euo pipefail
 . "${SCRIPTS_DIR}/lib.sh"
 
 log "Running scaletest..."
-touch /tmp/.scaletest_running
+set_status Running
 
 start_phase "Creating workspaces"
 coder exp scaletest create-workspaces \
@@ -56,4 +56,4 @@ end_phase
 wait_baseline 5
 
 log "Scaletest complete!"
-touch /tmp/.scaletest_complete
+set_status Complete
diff --git a/scaletest/templates/scaletest-runner/startup.sh b/scaletest/templates/scaletest-runner/startup.sh
index 7e939f2f7cbb4..e30cd60028e8b 100755
--- a/scaletest/templates/scaletest-runner/startup.sh
+++ b/scaletest/templates/scaletest-runner/startup.sh
@@ -15,13 +15,11 @@ rm /tmp/scripts.zip
 # shellcheck disable=SC2153 source=scaletest/templates/scaletest-runner/scripts/lib.sh
 . "${SCRIPTS_DIR}/lib.sh"
 
-# Clean up any previous scaletest runs (in case /tmp persists).
-rm -rf /tmp/.scaletest_* || true
-
 # Show failure in the UI if script exits with error.
 failed() {
-	echo "Scaletest failed!"
-	touch /tmp/.scaletest_failed
+	log "Scaletest failed!"
+	set_status Failed
+	lock_status # Ensure we never rewrite the status after a failure.
 }
 trap failed ERR
 

From 6352ce1aea72fd4239f183cf55e7262d9abf07c2 Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Thu, 14 Sep 2023 13:35:31 +0000
Subject: [PATCH 14/36] move permission check higher up

---
 scaletest/templates/scaletest-runner/main.tf | 22 ++++++++++----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/scaletest/templates/scaletest-runner/main.tf b/scaletest/templates/scaletest-runner/main.tf
index 6ad92496d714f..8924a66376a26 100644
--- a/scaletest/templates/scaletest-runner/main.tf
+++ b/scaletest/templates/scaletest-runner/main.tf
@@ -20,17 +20,6 @@ resource "time_static" "start_time" {
   }
 }
 
-locals {
-  workspace_pod_name     = "coder-scaletest-runner-${lower(data.coder_workspace.me.owner)}-${lower(data.coder_workspace.me.name)}"
-  workspace_pod_instance = "coder-workspace-${lower(data.coder_workspace.me.owner)}-${lower(data.coder_workspace.me.name)}"
-  service_account_name   = "scaletest-sa"
-  cpu                    = 2
-  memory                 = 2
-  home_disk_size         = 10
-  scaletest_run_id       = "scaletest-${time_static.start_time.rfc3339}"
-  scaletest_run_dir      = "/home/coder/${local.scaletest_run_id}"
-}
-
 resource "null_resource" "permission_check" {
   count = data.coder_workspace.me.start_count
 
@@ -45,6 +34,17 @@ resource "null_resource" "permission_check" {
   }
 }
 
+locals {
+  workspace_pod_name     = "coder-scaletest-runner-${lower(data.coder_workspace.me.owner)}-${lower(data.coder_workspace.me.name)}"
+  workspace_pod_instance = "coder-workspace-${lower(data.coder_workspace.me.owner)}-${lower(data.coder_workspace.me.name)}"
+  service_account_name   = "scaletest-sa"
+  cpu                    = 2
+  memory                 = 2
+  home_disk_size         = 10
+  scaletest_run_id       = "scaletest-${time_static.start_time.rfc3339}"
+  scaletest_run_dir      = "/home/coder/${local.scaletest_run_id}"
+}
+
 data "coder_provisioner" "me" {
 }
 

From 67e54a6fa7fca93ca4dc21ce3cf701b8202d7b2e Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Thu, 14 Sep 2023 13:36:43 +0000
Subject: [PATCH 15/36] fixup! turn namespace into param

---
 scaletest/templates/scaletest-runner/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scaletest/templates/scaletest-runner/main.tf b/scaletest/templates/scaletest-runner/main.tf
index 8924a66376a26..eb80a511c6696 100644
--- a/scaletest/templates/scaletest-runner/main.tf
+++ b/scaletest/templates/scaletest-runner/main.tf
@@ -471,7 +471,7 @@ resource "kubernetes_manifest" "pod_monitor" {
     apiVersion = "monitoring.coreos.com/v1"
     kind       = "PodMonitor"
     metadata = {
-      namespace = local.namespace
+      namespace = data.coder_parameter.namespace.value
       name      = "podmonitor-${local.workspace_pod_name}"
     }
     spec = {

From 8f6950a8da4da0541643a16669ae19906f9ec14b Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Thu, 14 Sep 2023 13:37:48 +0000
Subject: [PATCH 16/36] fixup! turn namespace into param

---
 scaletest/templates/scaletest-runner/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scaletest/templates/scaletest-runner/main.tf b/scaletest/templates/scaletest-runner/main.tf
index eb80a511c6696..4990350d723a1 100644
--- a/scaletest/templates/scaletest-runner/main.tf
+++ b/scaletest/templates/scaletest-runner/main.tf
@@ -170,7 +170,7 @@ data "coder_parameter" "num_workspaces" {
 data "coder_parameter" "namespace" {
   order       = 999
   type        = "string"
-  name        = "namespace"
+  name        = "Namespace"
   default     = "coder-big"
   description = "The Kubernetes namespace to create the scaletest runner resources in."
 }

From 964778e17915f75ccf845055b7cbcf010a9defa6 Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Thu, 14 Sep 2023 13:56:03 +0000
Subject: [PATCH 17/36] add cleanup strategy and make minor tweaks

---
 scaletest/templates/scaletest-runner/main.tf  | 35 +++++++++++++++++--
 .../scaletest-runner/metadata_status.sh       |  2 +-
 .../templates/scaletest-runner/scripts/lib.sh |  2 +-
 .../templates/scaletest-runner/startup.sh     | 31 ++++++++++++++--
 4 files changed, 63 insertions(+), 7 deletions(-)

diff --git a/scaletest/templates/scaletest-runner/main.tf b/scaletest/templates/scaletest-runner/main.tf
index 4990350d723a1..ede24db4de2f7 100644
--- a/scaletest/templates/scaletest-runner/main.tf
+++ b/scaletest/templates/scaletest-runner/main.tf
@@ -65,7 +65,7 @@ data "coder_parameter" "dry_run" {
   order       = 2
   type        = "bool"
   name        = "Dry-run"
-  default     = false
+  default     = true
   description = "Perform a dry-run to see what would happen."
   mutable     = true
   ephemeral   = true
@@ -90,7 +90,7 @@ data "coder_parameter" "create_concurrency" {
 data "coder_parameter" "job_concurrency" {
   order       = 11
   type        = "number"
-  name        = "Scaletest job concurrency"
+  name        = "Job concurrency"
   default     = 10
   description = "The number of concurrent jobs (e.g. when producing workspace traffic)."
   mutable     = true
@@ -119,6 +119,36 @@ data "coder_parameter" "cleanup_concurrency" {
   }
 }
 
+data "coder_parameter" "cleanup_strategy" {
+  order       = 13
+  name        = "Cleanup strategy"
+  default     = "always"
+  description = "The strategy used to cleanup workspaces after the scaletest is complete."
+  mutable     = true
+  ephemeral   = true
+  option {
+    name        = "Always"
+    value       = "always"
+    description = "Automatically cleanup workspaces after the scaletest ends."
+  }
+  option {
+    name        = "On stop"
+    value       = "on_stop"
+    description = "Cleanup workspaces when the workspace is stopped."
+  }
+  option {
+    name        = "On success"
+    value       = "on_success"
+    description = "Automatically cleanup workspaces after the scaletest is complete if no error occurs."
+  }
+  option {
+    name        = "On error"
+    value       = "on_error"
+    description = "Automatically cleanup workspaces after the scaletest is complete if an error occurs."
+  }
+}
+
+
 data "coder_parameter" "workspace_template" {
   order        = 20
   name         = "workspace_template"
@@ -204,6 +234,7 @@ resource "coder_agent" "main" {
     SCALETEST_SKIP_CLEANUP : "1",
     SCALETEST_NUM_WORKSPACES : data.coder_parameter.num_workspaces.value,
     SCALETEST_CREATE_CONCURRENCY : "${data.coder_parameter.create_concurrency.value}",
+    SCALETEST_CLEANUP_STRATEGY : data.coder_parameter.cleanup_strategy.value,
 
     SCRIPTS_ZIP : filebase64(data.archive_file.scripts_zip.output_path),
     SCRIPTS_DIR : "/tmp/scripts",
diff --git a/scaletest/templates/scaletest-runner/metadata_status.sh b/scaletest/templates/scaletest-runner/metadata_status.sh
index 6c294e1ea018c..8ec45f0875c1d 100755
--- a/scaletest/templates/scaletest-runner/metadata_status.sh
+++ b/scaletest/templates/scaletest-runner/metadata_status.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
 # shellcheck disable=SC2153 source=scaletest/templates/scaletest-runner/scripts/lib.sh
-. "${SCRIPTS_DIR}/lib.sh"
+. "${SCRIPTS_DIR}/lib.sh" 2>/dev/null || return
 
 get_status
diff --git a/scaletest/templates/scaletest-runner/scripts/lib.sh b/scaletest/templates/scaletest-runner/scripts/lib.sh
index cfb31f7ff2e1b..556c17009fa55 100644
--- a/scaletest/templates/scaletest-runner/scripts/lib.sh
+++ b/scaletest/templates/scaletest-runner/scripts/lib.sh
@@ -3,7 +3,7 @@ set -euo pipefail
 
 # Only source this script once, this env comes from sourcing
 # scripts/lib.sh from coder/coder below.
-if [[ ${SCRIPTS_LIB_IS_SOURCED} == 1 ]]; then
+if [[ ${SCRIPTS_LIB_IS_SOURCED:-0} == 1 ]]; then
 	return 0
 fi
 
diff --git a/scaletest/templates/scaletest-runner/startup.sh b/scaletest/templates/scaletest-runner/startup.sh
index e30cd60028e8b..0d7c8fb144324 100755
--- a/scaletest/templates/scaletest-runner/startup.sh
+++ b/scaletest/templates/scaletest-runner/startup.sh
@@ -16,12 +16,37 @@ rm /tmp/scripts.zip
 . "${SCRIPTS_DIR}/lib.sh"
 
 # Show failure in the UI if script exits with error.
-failed() {
+failed_status=Failed
+on_exit() {
+	trap - ERR EXIT
+
+	case "${SCALETEST_CLEANUP_STRATEGY}" in
+	on_stop)
+		# Handled by shutdown script.
+		;;
+	on_success)
+		if [[ $(get_status) != "${failed_status}" ]]; then
+			"${SCRIPTS_DIR}/cleanup.sh" "${SCALETEST_CLEANUP_STRATEGY}"
+		fi
+		;;
+	on_error)
+		if [[ $(get_status) = "${failed_status}" ]]; then
+			"${SCRIPTS_DIR}/cleanup.sh" "${SCALETEST_CLEANUP_STRATEGY}"
+		fi
+		;;
+	*)
+		"${SCRIPTS_DIR}/cleanup.sh" "${SCALETEST_CLEANUP_STRATEGY}"
+		;;
+	esac
+}
+trap on_exit EXIT
+
+on_err() {
 	log "Scaletest failed!"
-	set_status Failed
+	set_status "${failed_status}"
 	lock_status # Ensure we never rewrite the status after a failure.
 }
-trap failed ERR
+trap on_err ERR
 
 "${SCRIPTS_DIR}/prepare.sh"
 "${SCRIPTS_DIR}/run.sh"

From 57fd74e2b2fd02a4755ca3589f65f9a437613b83 Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Thu, 14 Sep 2023 14:20:52 +0000
Subject: [PATCH 18/36] move prepare/install step to Dockerfile

---
 .../templates/scaletest-runner/Dockerfile     | 34 +++++++++++++++++++
 scaletest/templates/scaletest-runner/main.tf  |  2 +-
 .../scaletest-runner/scripts/prepare.sh       | 18 ----------
 3 files changed, 35 insertions(+), 19 deletions(-)
 create mode 100644 scaletest/templates/scaletest-runner/Dockerfile

diff --git a/scaletest/templates/scaletest-runner/Dockerfile b/scaletest/templates/scaletest-runner/Dockerfile
new file mode 100644
index 0000000000000..58ca0c87a47df
--- /dev/null
+++ b/scaletest/templates/scaletest-runner/Dockerfile
@@ -0,0 +1,34 @@
+# This image is used to run scaletest jobs and, although it is inside
+# the template directory, it is built separately and pushed to
+# gcr.io/coder-dev-1/scaletest-runner:latest.
+#
+# Future improvements will include versioning and including the version
+# in the template push.
+
+FROM codercom/enterprise-base:ubuntu
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+USER root
+
+RUN wget --quiet -O /tmp/terraform.zip https://releases.hashicorp.com/terraform/1.5.7/terraform_1.5.7_linux_amd64.zip \
+	&& unzip /tmp/terraform.zip -d /usr/local/bin \
+	&& rm /tmp/terraform.zip \
+	&& terraform --version
+
+RUN wget --quiet -O /tmp/envsubst "https://github.com/a8m/envsubst/releases/download/v1.2.0/envsubst-$(uname -s)-$(uname -m)" \
+	&& chmod +x /tmp/envsubst \
+	&& mv /tmp/envsubst /usr/local/bin
+
+RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list \
+	&& curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - \
+	&& apt-get update \
+	&& apt-get install --yes \
+	google-cloud-cli \
+	jq \
+	kubectl \
+	&& gcloud --version \
+	&& kubectl version --client \
+	&& rm -rf /var/lib/apt/lists/*
+
+USER coder
diff --git a/scaletest/templates/scaletest-runner/main.tf b/scaletest/templates/scaletest-runner/main.tf
index ede24db4de2f7..a2d4c60997275 100644
--- a/scaletest/templates/scaletest-runner/main.tf
+++ b/scaletest/templates/scaletest-runner/main.tf
@@ -417,7 +417,7 @@ resource "kubernetes_pod" "main" {
 
     container {
       name              = "dev"
-      image             = "codercom/enterprise-base:ubuntu"
+      image             = "gcr.io/coder-dev-1/scaletest-runner:latest"
       image_pull_policy = "Always"
       command           = ["sh", "-c", coder_agent.main.init_script]
       security_context {
diff --git a/scaletest/templates/scaletest-runner/scripts/prepare.sh b/scaletest/templates/scaletest-runner/scripts/prepare.sh
index 39a8caec0da06..6ac50d7436932 100755
--- a/scaletest/templates/scaletest-runner/scripts/prepare.sh
+++ b/scaletest/templates/scaletest-runner/scripts/prepare.sh
@@ -11,24 +11,6 @@ mkdir -p "${SCALETEST_STATE_DIR}"
 echo "Preparing scaletest workspace environment..."
 set_status Preparing
 
-echo "Installing prerequisites (terraform, envsubst, gcloud, jq and kubectl)..."
-
-wget --quiet -O /tmp/terraform.zip https://releases.hashicorp.com/terraform/1.5.7/terraform_1.5.7_linux_amd64.zip
-sudo unzip /tmp/terraform.zip -d /usr/local/bin
-terraform --version
-
-wget --quiet -O /tmp/envsubst "https://github.com/a8m/envsubst/releases/download/v1.2.0/envsubst-$(uname -s)-$(uname -m)"
-chmod +x /tmp/envsubst
-sudo mv /tmp/envsubst /usr/local/bin
-
-echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
-curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
-sudo apt-get update
-sudo apt-get install --yes \
-	google-cloud-cli \
-	jq \
-	kubectl
-
 echo "Cloning coder/coder repo..."
 
 if [[ ! -d coder ]]; then

From 4c22a733068e889367f4b953b0c56574771a206c Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Thu, 14 Sep 2023 14:40:10 +0000
Subject: [PATCH 19/36] fix

---
 scaletest/templates/scaletest-runner/scripts/cleanup.sh | 2 +-
 scaletest/templates/scaletest-runner/scripts/lib.sh     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/scaletest/templates/scaletest-runner/scripts/cleanup.sh b/scaletest/templates/scaletest-runner/scripts/cleanup.sh
index f4ede63040776..63bcd5d5d9fea 100755
--- a/scaletest/templates/scaletest-runner/scripts/cleanup.sh
+++ b/scaletest/templates/scaletest-runner/scripts/cleanup.sh
@@ -21,7 +21,7 @@ if [[ $event = manual ]]; then
 	fi
 fi
 
-phase_start "Cleanup (${event})"
+start_phase "Cleanup (${event})"
 coder exp scaletest cleanup \
 	--cleanup-concurrency "${SCALETEST_CLEANUP_CONCURRENCY}" \
 	--cleanup-job-timeout 15m \
diff --git a/scaletest/templates/scaletest-runner/scripts/lib.sh b/scaletest/templates/scaletest-runner/scripts/lib.sh
index 556c17009fa55..b7a304724acb6 100644
--- a/scaletest/templates/scaletest-runner/scripts/lib.sh
+++ b/scaletest/templates/scaletest-runner/scripts/lib.sh
@@ -40,7 +40,7 @@ get_status() {
 
 phase_num=0
 start_phase() {
-	((phase_num++))
+	phase_num=$((phase_num + 1))
 	log "Start phase ${phase_num}: ${*}"
 	echo "$(date -Iseconds) START:${phase_num}: ${*}" >>"${SCALETEST_PHASE_FILE}"
 }

From 35ac68f0d8b70707c27817646014ca91fa9fe413 Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Thu, 14 Sep 2023 15:01:49 +0000
Subject: [PATCH 20/36] add todo for cleanup

---
 scaletest/templates/scaletest-runner/Dockerfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scaletest/templates/scaletest-runner/Dockerfile b/scaletest/templates/scaletest-runner/Dockerfile
index 58ca0c87a47df..9351c3d498dcd 100644
--- a/scaletest/templates/scaletest-runner/Dockerfile
+++ b/scaletest/templates/scaletest-runner/Dockerfile
@@ -11,6 +11,7 @@ ARG DEBIAN_FRONTEND=noninteractive
 
 USER root
 
+# TODO(mafredri): Remove unneeded dependencies once we have a clear idea of what's needed.
 RUN wget --quiet -O /tmp/terraform.zip https://releases.hashicorp.com/terraform/1.5.7/terraform_1.5.7_linux_amd64.zip \
 	&& unzip /tmp/terraform.zip -d /usr/local/bin \
 	&& rm /tmp/terraform.zip \

From 7cffd308ba0f39306f83bd3e38b7b41a6af4610a Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Thu, 14 Sep 2023 15:02:50 +0000
Subject: [PATCH 21/36] store timestamps for all status messages

---
 scaletest/templates/scaletest-runner/scripts/lib.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scaletest/templates/scaletest-runner/scripts/lib.sh b/scaletest/templates/scaletest-runner/scripts/lib.sh
index b7a304724acb6..fb82f6c7d2a18 100644
--- a/scaletest/templates/scaletest-runner/scripts/lib.sh
+++ b/scaletest/templates/scaletest-runner/scripts/lib.sh
@@ -24,7 +24,7 @@ show_json() {
 }
 
 set_status() {
-	echo "$*" >"${SCALETEST_STATE_DIR}/status"
+	echo "$(date -Iseconds) $*" >>"${SCALETEST_STATE_DIR}/status"
 }
 lock_status() {
 	chmod 0440 "${SCALETEST_STATE_DIR}/status"
@@ -32,7 +32,7 @@ lock_status() {
 get_status() {
 	# Order of importance (reverse of creation).
 	if [[ -f "${SCALETEST_STATE_DIR}/status" ]]; then
-		cat "${SCALETEST_STATE_DIR}/status"
+		tail -n1 "${SCALETEST_STATE_DIR}/status" | cut -d' ' -f2-
 	else
 		echo "Not started"
 	fi

From 5cb0e9b82c1702cb52b5a00136d69b106849833b Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Thu, 14 Sep 2023 15:07:38 +0000
Subject: [PATCH 22/36] make fmt/shfmt

---
 scaletest/templates/scaletest-runner/scripts/cleanup.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scaletest/templates/scaletest-runner/scripts/cleanup.sh b/scaletest/templates/scaletest-runner/scripts/cleanup.sh
index 63bcd5d5d9fea..53505b3d9bd0e 100755
--- a/scaletest/templates/scaletest-runner/scripts/cleanup.sh
+++ b/scaletest/templates/scaletest-runner/scripts/cleanup.sh
@@ -25,8 +25,8 @@ start_phase "Cleanup (${event})"
 coder exp scaletest cleanup \
 	--cleanup-concurrency "${SCALETEST_CLEANUP_CONCURRENCY}" \
 	--cleanup-job-timeout 15m \
-	--cleanup-timeout 30m \
-	| tee "result-cleanup-${event}.txt"
+	--cleanup-timeout 30m |
+	tee "result-cleanup-${event}.txt"
 end_phase
 
 if [[ $event = manual ]]; then

From e037b6b3f116198ebe4914fd2b17aaa5d4d32a27 Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Thu, 14 Sep 2023 15:27:54 +0000
Subject: [PATCH 23/36] fix unbound var

---
 scaletest/templates/scaletest-runner/scripts/cleanup.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scaletest/templates/scaletest-runner/scripts/cleanup.sh b/scaletest/templates/scaletest-runner/scripts/cleanup.sh
index 53505b3d9bd0e..42f3e614f19cf 100755
--- a/scaletest/templates/scaletest-runner/scripts/cleanup.sh
+++ b/scaletest/templates/scaletest-runner/scripts/cleanup.sh
@@ -23,7 +23,6 @@ fi
 
 start_phase "Cleanup (${event})"
 coder exp scaletest cleanup \
-	--cleanup-concurrency "${SCALETEST_CLEANUP_CONCURRENCY}" \
 	--cleanup-job-timeout 15m \
 	--cleanup-timeout 30m |
 	tee "result-cleanup-${event}.txt"

From f336a0a81715b857a2fee15660a732a05b4340e6 Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Thu, 14 Sep 2023 15:56:28 +0000
Subject: [PATCH 24/36] clean up previous runs if needed

---
 scaletest/templates/scaletest-runner/scripts/lib.sh     | 2 +-
 scaletest/templates/scaletest-runner/scripts/prepare.sh | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/scaletest/templates/scaletest-runner/scripts/lib.sh b/scaletest/templates/scaletest-runner/scripts/lib.sh
index fb82f6c7d2a18..8aa3ea70490ee 100644
--- a/scaletest/templates/scaletest-runner/scripts/lib.sh
+++ b/scaletest/templates/scaletest-runner/scripts/lib.sh
@@ -58,7 +58,7 @@ get_phase() {
 		phase_raw="$(tail -n1 "${SCALETEST_PHASE_FILE}")"
 		phase="$(echo "${phase_raw}" | cut -d' ' -f3-)"
 		if [[ ${phase_raw} == *"END:"* ]]; then
-			phase+=" (done)"
+			phase+=" [done]"
 		fi
 		echo "${phase}"
 	else
diff --git a/scaletest/templates/scaletest-runner/scripts/prepare.sh b/scaletest/templates/scaletest-runner/scripts/prepare.sh
index 6ac50d7436932..65d93c02661a1 100755
--- a/scaletest/templates/scaletest-runner/scripts/prepare.sh
+++ b/scaletest/templates/scaletest-runner/scripts/prepare.sh
@@ -34,4 +34,7 @@ unset CODER_SESSION_TOKEN
 echo -n "${token}" >"${CODER_CONFIG_DIR}/session"
 [[ $VERBOSE == 1 ]] && set -x # Restore logging (if enabled).
 
+log "Cleaning up from previous runs (if applicable)..."
+"${SCRIPTS_DIR}/cleanup.sh" "prepare"
+
 echo "Preparation complete!"

From 047c7840e545530a97aae1cc08c85c73306db882 Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Thu, 14 Sep 2023 16:02:58 +0000
Subject: [PATCH 25/36] clean up results json files

---
 scaletest/templates/scaletest-runner/main.tf     |  1 +
 .../scaletest-runner/scripts/cleanup.sh          |  2 +-
 .../templates/scaletest-runner/scripts/lib.sh    |  1 +
 .../scaletest-runner/scripts/prepare.sh          |  1 +
 .../templates/scaletest-runner/scripts/run.sh    | 16 ++++++++--------
 5 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/scaletest/templates/scaletest-runner/main.tf b/scaletest/templates/scaletest-runner/main.tf
index a2d4c60997275..6519d08dcf51c 100644
--- a/scaletest/templates/scaletest-runner/main.tf
+++ b/scaletest/templates/scaletest-runner/main.tf
@@ -213,6 +213,7 @@ data "archive_file" "scripts_zip" {
 
 resource "coder_agent" "main" {
   arch = data.coder_provisioner.me.arch
+  dir  = local.scaletest_run_dir
   os   = "linux"
   env = {
     VERBOSE : data.coder_parameter.verbose.value ? "1" : "0",
diff --git a/scaletest/templates/scaletest-runner/scripts/cleanup.sh b/scaletest/templates/scaletest-runner/scripts/cleanup.sh
index 42f3e614f19cf..a6d29211a080b 100755
--- a/scaletest/templates/scaletest-runner/scripts/cleanup.sh
+++ b/scaletest/templates/scaletest-runner/scripts/cleanup.sh
@@ -25,7 +25,7 @@ start_phase "Cleanup (${event})"
 coder exp scaletest cleanup \
 	--cleanup-job-timeout 15m \
 	--cleanup-timeout 30m |
-	tee "result-cleanup-${event}.txt"
+	tee "${SCALETEST_RESULTS_DIR}/cleanup-${event}.txt"
 end_phase
 
 if [[ $event = manual ]]; then
diff --git a/scaletest/templates/scaletest-runner/scripts/lib.sh b/scaletest/templates/scaletest-runner/scripts/lib.sh
index 8aa3ea70490ee..6258077ad2869 100644
--- a/scaletest/templates/scaletest-runner/scripts/lib.sh
+++ b/scaletest/templates/scaletest-runner/scripts/lib.sh
@@ -14,6 +14,7 @@ fi
 # Environment variables shared between scripts.
 SCALETEST_STATE_DIR="${SCALETEST_RUN_DIR}/state"
 SCALETEST_PHASE_FILE="${SCALETEST_STATE_DIR}/phase"
+SCALETEST_RESULTS_DIR="${SCALETEST_RUN_DIR}/results"
 
 coder() {
 	maybedryrun "$DRY_RUN" command coder "${@}"
diff --git a/scaletest/templates/scaletest-runner/scripts/prepare.sh b/scaletest/templates/scaletest-runner/scripts/prepare.sh
index 65d93c02661a1..cc0e8e9d3729f 100755
--- a/scaletest/templates/scaletest-runner/scripts/prepare.sh
+++ b/scaletest/templates/scaletest-runner/scripts/prepare.sh
@@ -7,6 +7,7 @@ set -euo pipefail
 . "${SCRIPTS_DIR}/lib.sh"
 
 mkdir -p "${SCALETEST_STATE_DIR}"
+mkdir -p "${SCALETEST_RESULTS_DIR}"
 
 echo "Preparing scaletest workspace environment..."
 set_status Preparing
diff --git a/scaletest/templates/scaletest-runner/scripts/run.sh b/scaletest/templates/scaletest-runner/scripts/run.sh
index 694403ed18121..7ebf8c4310593 100755
--- a/scaletest/templates/scaletest-runner/scripts/run.sh
+++ b/scaletest/templates/scaletest-runner/scripts/run.sh
@@ -16,8 +16,8 @@ coder exp scaletest create-workspaces \
 	--concurrency "${SCALETEST_CREATE_CONCURRENCY}" \
 	--job-timeout 15m \
 	--no-cleanup \
-	--output json:"${SCALETEST_RUN_DIR}/result-create-workspaces.json"
-show_json "${SCALETEST_RUN_DIR}/result-create-workspaces.json"
+	--output json:"${SCALETEST_RESULTS_DIR}/create-workspaces.json"
+show_json "${SCALETEST_RESULTS_DIR}/create-workspaces.json"
 end_phase
 
 wait_baseline 5
@@ -28,8 +28,8 @@ coder exp scaletest workspace-traffic \
 	--bytes-per-tick 10240 \
 	--tick-interval 1s \
 	--timeout 5m \
-	--output json:"${SCALETEST_RUN_DIR}/result-ssh.json"
-show_json "${SCALETEST_RUN_DIR}/result-ssh.json"
+	--output json:"${SCALETEST_RESULTS_DIR}/traffic-ssh.json"
+show_json "${SCALETEST_RESULTS_DIR}/traffic-ssh.json"
 end_phase
 
 wait_baseline 5
@@ -39,8 +39,8 @@ coder exp scaletest workspace-traffic \
 	--bytes-per-tick 10240 \
 	--tick-interval 1s \
 	--timeout 5m \
-	--output json:"${SCALETEST_RUN_DIR}/result-reconnectingpty.json"
-show_json "${SCALETEST_RUN_DIR}/result-reconnectingpty.json"
+	--output json:"${SCALETEST_RESULTS_DIR}/traffic-reconnectingpty.json"
+show_json "${SCALETEST_RESULTS_DIR}/traffic-reconnectingpty.json"
 end_phase
 
 wait_baseline 5
@@ -49,8 +49,8 @@ start_phase "Dashboard traffic"
 coder exp scaletest dashboard \
 	--count "${SCALETEST_NUM_WORKSPACES}" \
 	--job-timeout 5m \
-	--output json:"${SCALETEST_RUN_DIR}/result-dashboard.json"
-show_json "${SCALETEST_RUN_DIR}/result-dashboard.json"
+	--output json:"${SCALETEST_RESULTS_DIR}/traffic-dashboard.json"
+show_json "${SCALETEST_RESULTS_DIR}/traffic-dashboard.json"
 end_phase
 
 wait_baseline 5

From 18875f4af3a424f46ff039989a5955f9c1ca6433 Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Thu, 14 Sep 2023 16:22:14 +0000
Subject: [PATCH 26/36] add previous phase

---
 scaletest/templates/scaletest-runner/main.tf              | 8 ++++++++
 .../templates/scaletest-runner/metadata_previous_phase.sh | 6 ++++++
 scaletest/templates/scaletest-runner/scripts/lib.sh       | 7 +++++++
 scaletest/templates/scaletest-runner/scripts/prepare.sh   | 2 +-
 4 files changed, 22 insertions(+), 1 deletion(-)
 create mode 100755 scaletest/templates/scaletest-runner/metadata_previous_phase.sh

diff --git a/scaletest/templates/scaletest-runner/main.tf b/scaletest/templates/scaletest-runner/main.tf
index 6519d08dcf51c..4802c9887793d 100644
--- a/scaletest/templates/scaletest-runner/main.tf
+++ b/scaletest/templates/scaletest-runner/main.tf
@@ -267,6 +267,14 @@ resource "coder_agent" "main" {
     timeout      = 1
   }
 
+  metadata {
+    display_name = "Scaletest phase (previous)"
+    key          = "02_scaletest_previous_phase"
+    script       = file("metadata_previous_phase.sh")
+    interval     = 1
+    timeout      = 1
+  }
+
   # Misc workspace metadata.
   metadata {
     display_name = "CPU Usage"
diff --git a/scaletest/templates/scaletest-runner/metadata_previous_phase.sh b/scaletest/templates/scaletest-runner/metadata_previous_phase.sh
new file mode 100755
index 0000000000000..c858687b72ad8
--- /dev/null
+++ b/scaletest/templates/scaletest-runner/metadata_previous_phase.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# shellcheck disable=SC2153 source=scaletest/templates/scaletest-runner/scripts/lib.sh
+. "${SCRIPTS_DIR}/lib.sh" 2>/dev/null || return
+
+get_previous_phase
diff --git a/scaletest/templates/scaletest-runner/scripts/lib.sh b/scaletest/templates/scaletest-runner/scripts/lib.sh
index 6258077ad2869..2ae414ddb80b6 100644
--- a/scaletest/templates/scaletest-runner/scripts/lib.sh
+++ b/scaletest/templates/scaletest-runner/scripts/lib.sh
@@ -66,6 +66,13 @@ get_phase() {
 		echo "None"
 	fi
 }
+get_previous_phase() {
+	if [[ -f "${SCALETEST_PHASE_FILE}" ]] && [[ $(wc -l "${SCALETEST_PHASE_FILE}") -gt 1 ]]; then
+		grep START: "${SCALETEST_PHASE_FILE}" | tail -n2 | head -n1 | cut -d' ' -f3-
+	else
+		echo "None"
+	fi
+}
 
 wait_baseline() {
 	s=${1:-2}
diff --git a/scaletest/templates/scaletest-runner/scripts/prepare.sh b/scaletest/templates/scaletest-runner/scripts/prepare.sh
index cc0e8e9d3729f..145f24f86704f 100755
--- a/scaletest/templates/scaletest-runner/scripts/prepare.sh
+++ b/scaletest/templates/scaletest-runner/scripts/prepare.sh
@@ -14,7 +14,7 @@ set_status Preparing
 
 echo "Cloning coder/coder repo..."
 
-if [[ ! -d coder ]]; then
+if [[ ! -d ~/coder ]]; then
 	git clone https://github.com/coder/coder.git ~/coder
 fi
 (cd ~/coder && git pull)

From 050759124a53e00cf55b6fc7a6e72755adc10f5b Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Thu, 14 Sep 2023 16:33:03 +0000
Subject: [PATCH 27/36] fix

---
 scaletest/templates/scaletest-runner/scripts/lib.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scaletest/templates/scaletest-runner/scripts/lib.sh b/scaletest/templates/scaletest-runner/scripts/lib.sh
index 2ae414ddb80b6..275d3cb50a358 100644
--- a/scaletest/templates/scaletest-runner/scripts/lib.sh
+++ b/scaletest/templates/scaletest-runner/scripts/lib.sh
@@ -14,6 +14,7 @@ fi
 # Environment variables shared between scripts.
 SCALETEST_STATE_DIR="${SCALETEST_RUN_DIR}/state"
 SCALETEST_PHASE_FILE="${SCALETEST_STATE_DIR}/phase"
+# shellcheck disable=SC2034
 SCALETEST_RESULTS_DIR="${SCALETEST_RUN_DIR}/results"
 
 coder() {
@@ -67,7 +68,7 @@ get_phase() {
 	fi
 }
 get_previous_phase() {
-	if [[ -f "${SCALETEST_PHASE_FILE}" ]] && [[ $(wc -l "${SCALETEST_PHASE_FILE}") -gt 1 ]]; then
+	if [[ -f "${SCALETEST_PHASE_FILE}" ]] && [[ $(wc -l "${SCALETEST_PHASE_FILE}" | cut -d' ' -f1) -gt 1 ]]; then
 		grep START: "${SCALETEST_PHASE_FILE}" | tail -n2 | head -n1 | cut -d' ' -f3-
 	else
 		echo "None"

From d83f480cf2269cc1c9999e357e96461ed60a9b60 Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Thu, 14 Sep 2023 18:36:24 +0000
Subject: [PATCH 28/36] grep -c <3

---
 scaletest/templates/scaletest-runner/scripts/lib.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scaletest/templates/scaletest-runner/scripts/lib.sh b/scaletest/templates/scaletest-runner/scripts/lib.sh
index 275d3cb50a358..9c2c6a19049ac 100644
--- a/scaletest/templates/scaletest-runner/scripts/lib.sh
+++ b/scaletest/templates/scaletest-runner/scripts/lib.sh
@@ -42,6 +42,10 @@ get_status() {
 
 phase_num=0
 start_phase() {
+	# This may be incremented from another script, so we read it every time.
+	if [[ -f "${SCALETEST_PHASE_FILE}" ]]; then
+		phase_num="$(grep -c START: "${SCALETEST_PHASE_FILE}")"
+	fi
 	phase_num=$((phase_num + 1))
 	log "Start phase ${phase_num}: ${*}"
 	echo "$(date -Iseconds) START:${phase_num}: ${*}" >>"${SCALETEST_PHASE_FILE}"
@@ -68,7 +72,7 @@ get_phase() {
 	fi
 }
 get_previous_phase() {
-	if [[ -f "${SCALETEST_PHASE_FILE}" ]] && [[ $(wc -l "${SCALETEST_PHASE_FILE}" | cut -d' ' -f1) -gt 1 ]]; then
+	if [[ -f "${SCALETEST_PHASE_FILE}" ]] && [[ $(grep -c START: "${SCALETEST_PHASE_FILE}") -gt 1 ]]; then
 		grep START: "${SCALETEST_PHASE_FILE}" | tail -n2 | head -n1 | cut -d' ' -f3-
 	else
 		echo "None"

From 179f5b488618cf7ba2f5bdbe6e0d8efaa62fb2ff Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Thu, 14 Sep 2023 18:41:45 +0000
Subject: [PATCH 29/36] ns

---
 scaletest/templates/scaletest-runner/scripts/lib.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scaletest/templates/scaletest-runner/scripts/lib.sh b/scaletest/templates/scaletest-runner/scripts/lib.sh
index 9c2c6a19049ac..bb4dd6032cb6c 100644
--- a/scaletest/templates/scaletest-runner/scripts/lib.sh
+++ b/scaletest/templates/scaletest-runner/scripts/lib.sh
@@ -26,7 +26,7 @@ show_json() {
 }
 
 set_status() {
-	echo "$(date -Iseconds) $*" >>"${SCALETEST_STATE_DIR}/status"
+	echo "$(date -Ins) $*" >>"${SCALETEST_STATE_DIR}/status"
 }
 lock_status() {
 	chmod 0440 "${SCALETEST_STATE_DIR}/status"
@@ -48,7 +48,7 @@ start_phase() {
 	fi
 	phase_num=$((phase_num + 1))
 	log "Start phase ${phase_num}: ${*}"
-	echo "$(date -Iseconds) START:${phase_num}: ${*}" >>"${SCALETEST_PHASE_FILE}"
+	echo "$(date -Ins) START:${phase_num}: ${*}" >>"${SCALETEST_PHASE_FILE}"
 }
 end_phase() {
 	phase="$(tail -n 1 "${SCALETEST_PHASE_FILE}" | grep "START:${phase_num}:" | cut -d' ' -f3-)"
@@ -57,7 +57,7 @@ end_phase() {
 		exit 1
 	fi
 	log "End phase ${phase_num}: ${phase}"
-	echo "$(date -Iseconds) END:${phase_num}: ${phase}" >>"${SCALETEST_PHASE_FILE}"
+	echo "$(date -Ins) END:${phase_num}: ${phase}" >>"${SCALETEST_PHASE_FILE}"
 }
 get_phase() {
 	if [[ -f "${SCALETEST_PHASE_FILE}" ]]; then

From 5f5d3b180a67c1a0c8d28da64a5af81038ae6cde Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Thu, 14 Sep 2023 19:03:06 +0000
Subject: [PATCH 30/36] add compression of old archives

---
 .../templates/scaletest-runner/Dockerfile     |  1 +
 .../templates/scaletest-runner/scripts/lib.sh |  2 +-
 .../scaletest-runner/scripts/prepare.sh       | 22 ++++++++++++++++---
 3 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/scaletest/templates/scaletest-runner/Dockerfile b/scaletest/templates/scaletest-runner/Dockerfile
index 9351c3d498dcd..9aa016b534a17 100644
--- a/scaletest/templates/scaletest-runner/Dockerfile
+++ b/scaletest/templates/scaletest-runner/Dockerfile
@@ -28,6 +28,7 @@ RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.
 	google-cloud-cli \
 	jq \
 	kubectl \
+	zstd \
 	&& gcloud --version \
 	&& kubectl version --client \
 	&& rm -rf /var/lib/apt/lists/*
diff --git a/scaletest/templates/scaletest-runner/scripts/lib.sh b/scaletest/templates/scaletest-runner/scripts/lib.sh
index bb4dd6032cb6c..58a6feae86cf3 100644
--- a/scaletest/templates/scaletest-runner/scripts/lib.sh
+++ b/scaletest/templates/scaletest-runner/scripts/lib.sh
@@ -9,7 +9,7 @@ fi
 
 # Source scripts/lib.sh from coder/coder for common functions.
 # shellcheck source=scripts/lib.sh
-. ~/coder/scripts/lib.sh
+. "${HOME}/coder/scripts/lib.sh"
 
 # Environment variables shared between scripts.
 SCALETEST_STATE_DIR="${SCALETEST_RUN_DIR}/state"
diff --git a/scaletest/templates/scaletest-runner/scripts/prepare.sh b/scaletest/templates/scaletest-runner/scripts/prepare.sh
index 145f24f86704f..ad89cc16c1391 100755
--- a/scaletest/templates/scaletest-runner/scripts/prepare.sh
+++ b/scaletest/templates/scaletest-runner/scripts/prepare.sh
@@ -12,12 +12,28 @@ mkdir -p "${SCALETEST_RESULTS_DIR}"
 echo "Preparing scaletest workspace environment..."
 set_status Preparing
 
+log "Compressing previous run logs (if applicable)..."
+mkdir -p "${HOME}/archive"
+for dir in "${HOME}/scaletest-"*; do
+	if [[ ${dir} = "${SCALETEST_RUN_DIR}" ]]; then
+		continue
+	fi
+	if [[ -d ${dir} ]]; then
+		name="$(basename "${dir}")"
+		(
+			cd "$(dirname "${dir}")"
+			tar --zstd -cf "${HOME}/archive/${name}.tar.zst" "${name}"
+		)
+		rm -rf "${dir}"
+	fi
+done
+
 echo "Cloning coder/coder repo..."
 
-if [[ ! -d ~/coder ]]; then
-	git clone https://github.com/coder/coder.git ~/coder
+if [[ ! -d "${HOME}/coder" ]]; then
+	git clone https://github.com/coder/coder.git "${HOME}/coder"
 fi
-(cd ~/coder && git pull)
+(cd "${HOME}/coder" && git pull)
 
 echo "Creating coder CLI token (needed for cleanup during shutdown)..."
 

From c9205ce07ddabce1560787bd21288ad507e73ba1 Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Thu, 14 Sep 2023 19:11:17 +0000
Subject: [PATCH 31/36] s/echo/log/

---
 scaletest/templates/scaletest-runner/scripts/prepare.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scaletest/templates/scaletest-runner/scripts/prepare.sh b/scaletest/templates/scaletest-runner/scripts/prepare.sh
index ad89cc16c1391..9dca5b61332f9 100755
--- a/scaletest/templates/scaletest-runner/scripts/prepare.sh
+++ b/scaletest/templates/scaletest-runner/scripts/prepare.sh
@@ -9,7 +9,7 @@ set -euo pipefail
 mkdir -p "${SCALETEST_STATE_DIR}"
 mkdir -p "${SCALETEST_RESULTS_DIR}"
 
-echo "Preparing scaletest workspace environment..."
+log "Preparing scaletest workspace environment..."
 set_status Preparing
 
 log "Compressing previous run logs (if applicable)..."
@@ -28,14 +28,14 @@ for dir in "${HOME}/scaletest-"*; do
 	fi
 done
 
-echo "Cloning coder/coder repo..."
+log "Cloning coder/coder repo..."
 
 if [[ ! -d "${HOME}/coder" ]]; then
 	git clone https://github.com/coder/coder.git "${HOME}/coder"
 fi
 (cd "${HOME}/coder" && git pull)
 
-echo "Creating coder CLI token (needed for cleanup during shutdown)..."
+log "Creating coder CLI token (needed for cleanup during shutdown)..."
 
 mkdir -p "${CODER_CONFIG_DIR}"
 echo -n "${CODER_URL}" >"${CODER_CONFIG_DIR}/url"
@@ -54,4 +54,4 @@ echo -n "${token}" >"${CODER_CONFIG_DIR}/session"
 log "Cleaning up from previous runs (if applicable)..."
 "${SCRIPTS_DIR}/cleanup.sh" "prepare"
 
-echo "Preparation complete!"
+log "Preparation complete!"

From f7b8c003c688e56e50ba24a75eb70c579387459d Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Fri, 15 Sep 2023 10:05:48 +0000
Subject: [PATCH 32/36] increase zstd compression level

---
 scaletest/templates/scaletest-runner/scripts/prepare.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scaletest/templates/scaletest-runner/scripts/prepare.sh b/scaletest/templates/scaletest-runner/scripts/prepare.sh
index 9dca5b61332f9..01b29abb2d3a5 100755
--- a/scaletest/templates/scaletest-runner/scripts/prepare.sh
+++ b/scaletest/templates/scaletest-runner/scripts/prepare.sh
@@ -22,7 +22,7 @@ for dir in "${HOME}/scaletest-"*; do
 		name="$(basename "${dir}")"
 		(
 			cd "$(dirname "${dir}")"
-			tar --zstd -cf "${HOME}/archive/${name}.tar.zst" "${name}"
+			ZSTD_CLEVEL=12 tar --zstd -cf "${HOME}/archive/${name}.tar.zst" "${name}"
 		)
 		rm -rf "${dir}"
 	fi

From 4688f591679552ce3c72e284c15762722a6895d7 Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Fri, 15 Sep 2023 10:18:28 +0000
Subject: [PATCH 33/36] make sleep a dry-run op

---
 scaletest/templates/scaletest-runner/scripts/lib.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scaletest/templates/scaletest-runner/scripts/lib.sh b/scaletest/templates/scaletest-runner/scripts/lib.sh
index 58a6feae86cf3..9714ff2cf0527 100644
--- a/scaletest/templates/scaletest-runner/scripts/lib.sh
+++ b/scaletest/templates/scaletest-runner/scripts/lib.sh
@@ -82,6 +82,6 @@ get_previous_phase() {
 wait_baseline() {
 	s=${1:-2}
 	start_phase "Waiting ${s}m to establish baseline"
-	sleep $((s * 60))
+	maybedryrun "$DRY_RUN" sleep $((s * 60))
 	end_phase
 }

From 3f4008c02da03c584bc392b71bf62f1b897a96db Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Fri, 15 Sep 2023 10:20:34 +0000
Subject: [PATCH 34/36] make compression a dry-run op

---
 scaletest/templates/scaletest-runner/scripts/prepare.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scaletest/templates/scaletest-runner/scripts/prepare.sh b/scaletest/templates/scaletest-runner/scripts/prepare.sh
index 01b29abb2d3a5..f6fbcb7dd3227 100755
--- a/scaletest/templates/scaletest-runner/scripts/prepare.sh
+++ b/scaletest/templates/scaletest-runner/scripts/prepare.sh
@@ -22,9 +22,9 @@ for dir in "${HOME}/scaletest-"*; do
 		name="$(basename "${dir}")"
 		(
 			cd "$(dirname "${dir}")"
-			ZSTD_CLEVEL=12 tar --zstd -cf "${HOME}/archive/${name}.tar.zst" "${name}"
+			ZSTD_CLEVEL=12 maybedryrun "$DRY_RUN" tar --zstd -cf "${HOME}/archive/${name}.tar.zst" "${name}"
 		)
-		rm -rf "${dir}"
+		maybedryrun "$DRY_RUN" rm -rf "${dir}"
 	fi
 done
 

From 0fce5130bbb0cb08490288fb00a8493fe8a8195b Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Fri, 15 Sep 2023 10:23:19 +0000
Subject: [PATCH 35/36] show dry-run in status

---
 scaletest/templates/scaletest-runner/scripts/lib.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scaletest/templates/scaletest-runner/scripts/lib.sh b/scaletest/templates/scaletest-runner/scripts/lib.sh
index 9714ff2cf0527..4f2c64efb720e 100644
--- a/scaletest/templates/scaletest-runner/scripts/lib.sh
+++ b/scaletest/templates/scaletest-runner/scripts/lib.sh
@@ -26,7 +26,11 @@ show_json() {
 }
 
 set_status() {
-	echo "$(date -Ins) $*" >>"${SCALETEST_STATE_DIR}/status"
+	dry_run=
+	if [[ $DRY_RUN == 1 ]]; then
+		dry_run=" (dry-ryn)"
+	fi
+	echo "$(date -Ins) ${*}${dry_run}" >>"${SCALETEST_STATE_DIR}/status"
 }
 lock_status() {
 	chmod 0440 "${SCALETEST_STATE_DIR}/status"

From 94b49e9f29c99e2e01910661bfca458ae068ddf4 Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Fri, 15 Sep 2023 13:16:39 +0000
Subject: [PATCH 36/36] make shellcheck happy

---
 scaletest/templates/scaletest-runner/scripts/lib.sh | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/scaletest/templates/scaletest-runner/scripts/lib.sh b/scaletest/templates/scaletest-runner/scripts/lib.sh
index 4f2c64efb720e..d392d09681f0a 100644
--- a/scaletest/templates/scaletest-runner/scripts/lib.sh
+++ b/scaletest/templates/scaletest-runner/scripts/lib.sh
@@ -11,6 +11,9 @@ fi
 # shellcheck source=scripts/lib.sh
 . "${HOME}/coder/scripts/lib.sh"
 
+# Make shellcheck happy.
+DRY_RUN=${DRY_RUN:-0}
+
 # Environment variables shared between scripts.
 SCALETEST_STATE_DIR="${SCALETEST_RUN_DIR}/state"
 SCALETEST_PHASE_FILE="${SCALETEST_STATE_DIR}/phase"
@@ -18,16 +21,16 @@ SCALETEST_PHASE_FILE="${SCALETEST_STATE_DIR}/phase"
 SCALETEST_RESULTS_DIR="${SCALETEST_RUN_DIR}/results"
 
 coder() {
-	maybedryrun "$DRY_RUN" command coder "${@}"
+	maybedryrun "${DRY_RUN}" command coder "${@}"
 }
 
 show_json() {
-	maybedryrun "$DRY_RUN" jq 'del(.. | .logs?)' "${1}"
+	maybedryrun "${DRY_RUN}" jq 'del(.. | .logs?)' "${1}"
 }
 
 set_status() {
 	dry_run=
-	if [[ $DRY_RUN == 1 ]]; then
+	if [[ ${DRY_RUN} == 1 ]]; then
 		dry_run=" (dry-ryn)"
 	fi
 	echo "$(date -Ins) ${*}${dry_run}" >>"${SCALETEST_STATE_DIR}/status"