feat(scaletest): add greedy agent test to runner

coder · mafredri · Dec 5, 2023 · Nov 7, 2023 · Nov 13, 2023 · Dec 1, 2023
commit b3f9cb33f736d9edb15706121f1e7765e17a8dd0
diff --git a/scaletest/templates/scaletest-runner/main.tf b/scaletest/templates/scaletest-runner/main.tf
@@ -335,6 +335,50 @@ data "coder_parameter" "load_scenario_baseline_duration" {
   }
 }
 
+data "coder_parameter" "greedy_agent" {
+  order       = 30
+  type        = "bool"
+  name        = "Greedy Agent"
+  default     = false
+  description = "If true, the agent will attempt to consume all available resources."
+  mutable     = true
+  ephemeral   = true
+}
+
+data "coder_parameter" "greedy_agent_template" {
+  order        = 31
+  name         = "Greedy Agent Template"
+  display_name = "Greedy Agent Template"
+  description  = "The template used for the greedy agent workspace (must not be same as workspace template)."
+  default      = "kubernetes-medium"
+  icon         = "/emojis/1f4dc.png" # Scroll.
+  mutable      = true
+  option {
+    name        = "Minimal"
+    value       = "kubernetes-minimal" # Feather.
+    icon        = "/emojis/1fab6.png"
+    description = "Sized to fit approx. 32 per t2d-standard-8 instance."
+  }
+  option {
+    name        = "Small"
+    value       = "kubernetes-small"
+    icon        = "/emojis/1f42d.png" # Mouse.
+    description = "Provisions a small-sized workspace with no persistent storage."
+  }
+  option {
+    name        = "Medium"
+    value       = "kubernetes-medium"
+    icon        = "/emojis/1f436.png" # Dog.
+    description = "Provisions a medium-sized workspace with no persistent storage."
+  }
+  option {
+    name        = "Large"
+    value       = "kubernetes-large"
+    icon        = "/emojis/1f434.png" # Horse.
+    description = "Provisions a large-sized workspace with no persistent storage."
+  }
+}
+
 data "coder_parameter" "namespace" {
   order       = 999
   type        = "string"
@@ -395,6 +439,8 @@ resource "coder_agent" "main" {
     SCALETEST_PARAM_LOAD_SCENARIO_WEB_TERMINAL_TRAFFIC_TICK_INTERVAL : "${data.coder_parameter.load_scenario_web_terminal_tick_interval.value}",
     SCALETEST_PARAM_LOAD_SCENARIO_DASHBOARD_TRAFFIC_DURATION : "${data.coder_parameter.load_scenario_dashboard_traffic_duration.value}",
     SCALETEST_PARAM_LOAD_SCENARIO_BASELINE_DURATION : "${data.coder_parameter.load_scenario_baseline_duration.value}",
+    SCALETEST_PARAM_GREEDY_AGENT : data.coder_parameter.greedy_agent.value ? "1" : "0",
+    SCALETEST_PARAM_GREEDY_AGENT_TEMPLATE : data.coder_parameter.greedy_agent_template.value,
 
     GRAFANA_URL : local.grafana_url,
 

diff --git a/scaletest/templates/scaletest-runner/scripts/run.sh b/scaletest/templates/scaletest-runner/scripts/run.sh
@@ -26,6 +26,53 @@ end_phase
 
 wait_baseline "${SCALETEST_PARAM_LOAD_SCENARIO_BASELINE_DURATION}"
 
+if [[ ${SCALETEST_PARAM_GREEDY_AGENT} != 1 ]]; then
+	greedy_agent() { :; }
+else
+	echo "WARNING: Greedy agent enabled, this may cause the load tests to fail." >&2
+
+	coder exp scaletest create-workspaces \
+		--count 1 \
+		--template "${SCALETEST_PARAM_GREEDY_AGENT_TEMPLATE}" \
+		--concurrency 1 \
+		--timeout 5h \
+		--job-timeout 5h \
+		--no-cleanup \
+		--output json:"${SCALETEST_RESULTS_DIR}/create-workspaces-greedy-agent.json"
+
+	greedy_agent() {
+		local timeout=${1} scenario=${2}
+		# Run the greedy test for ~1/3 of the timeout.
+		delay=$((timeout * 60 / 3))
+
+		local type=web-terminal
+		args=()
+		if [[ ${scenario} == "SSH Traffic" ]]; then
+			type=ssh
+			args+=(--ssh)
+		fi
+
+		sleep "${delay}"
+		annotate_grafana greedy_agent "${scenario}: Greedy agent"
+
+		# Produce load at about 1000MB/s.
+		set +e
+		coder exp scaletest workspace-traffic \
+			--template "${SCALETEST_PARAM_GREEDY_AGENT_TEMPLATE}" \
+			--timeout "$((delay))s" \
+			--job-timeout "$((delay))s" \
+			--output json:"${SCALETEST_RESULTS_DIR}/traffic-${type}-greedy-agent.json" \
+			--bytes-per-tick $((1024 * 1000)) \
+			--tick-interval 1ms \
+			"${args[@]}"
+		status=${?}
+
+		annotate_grafana_end greedy_agent "${scenario}: Greedy agent"
+
+		return ${status}
+	}
+fi
+
 declare -A failed=()
 for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do
 	start_phase "Load scenario: ${scenario}"
@@ -34,24 +81,38 @@ for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do
 	status=0
 	case "${scenario}" in
 	"SSH Traffic")
+		greedy_agent "${SCALETEST_PARAM_LOAD_SCENARIO_SSH_TRAFFIC_DURATION}" "${scenario}" &
 		coder exp scaletest workspace-traffic \
+			--template "${SCALETEST_PARAM_TEMPLATE}" \
 			--ssh \
 			--bytes-per-tick "${SCALETEST_PARAM_LOAD_SCENARIO_SSH_TRAFFIC_BYTES_PER_TICK}" \
 			--tick-interval "${SCALETEST_PARAM_LOAD_SCENARIO_SSH_TRAFFIC_TICK_INTERVAL}ms" \
 			--timeout "${SCALETEST_PARAM_LOAD_SCENARIO_SSH_TRAFFIC_DURATION}m" \
 			--job-timeout "${SCALETEST_PARAM_LOAD_SCENARIO_SSH_TRAFFIC_DURATION}m30s" \
 			--output json:"${SCALETEST_RESULTS_DIR}/traffic-ssh.json"
 		status=$?
+		wait
+		status2=$?
+		if [[ ${status} == 0 ]]; then
+			status=${status2}
+		fi
 		show_json "${SCALETEST_RESULTS_DIR}/traffic-ssh.json"
 		;;
 	"Web Terminal Traffic")
+		greedy_agent "${SCALETEST_PARAM_LOAD_SCENARIO_WEB_TERMINAL_TRAFFIC_DURATION}" "${scenario}" &
 		coder exp scaletest workspace-traffic \
+			--template "${SCALETEST_PARAM_TEMPLATE}" \
 			--bytes-per-tick "${SCALETEST_PARAM_LOAD_SCENARIO_WEB_TERMINAL_TRAFFIC_BYTES_PER_TICK}" \
 			--tick-interval "${SCALETEST_PARAM_LOAD_SCENARIO_WEB_TERMINAL_TRAFFIC_TICK_INTERVAL}ms" \
 			--timeout "${SCALETEST_PARAM_LOAD_SCENARIO_WEB_TERMINAL_TRAFFIC_DURATION}m" \
 			--job-timeout "${SCALETEST_PARAM_LOAD_SCENARIO_WEB_TERMINAL_TRAFFIC_DURATION}m30s" \
 			--output json:"${SCALETEST_RESULTS_DIR}/traffic-web-terminal.json"
 		status=$?
+		wait
+		status2=$?
+		if [[ ${status} == 0 ]]; then
+			status=${status2}
+		fi
 		show_json "${SCALETEST_RESULTS_DIR}/traffic-web-terminal.json"
 		;;
 	"Dashboard Traffic")
@@ -65,6 +126,10 @@ for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do
 		;;
 
 	# Debug scenarios, for testing the runner.
+	"debug:greedy_agent")
+		greedy_agent 10 "${scenario}"
+		status=$?
+		;;
 	"debug:success")
 		maybedryrun "$DRY_RUN" sleep 10
 		status=0

diff --git a/scaletest/templates/scaletest-runner/startup.sh b/scaletest/templates/scaletest-runner/startup.sh
@@ -3,6 +3,11 @@ set -euo pipefail
 
 [[ $VERBOSE == 1 ]] && set -x
 
+if [[ ${SCALETEST_PARAM_GREEDY_AGENT_TEMPLATE} == "${SCALETEST_PARAM_TEMPLATE}" ]]; then
+	echo "ERROR: Greedy agent template must be different from the scaletest template." >&2
+	exit 1
+fi
+
 # Unzip scripts and add to path.
 # shellcheck disable=SC2153
 echo "Extracting scaletest scripts into ${SCRIPTS_DIR}..."