From aa3945c214da13200a57a09972b90fa959940309 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 10 Oct 2023 13:13:54 +0000 Subject: [PATCH 1/5] chore(scaletest/templates/scaletest-runner): add --retries on kubectl cp --- scaletest/templates/scaletest-runner/scripts/lib.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scaletest/templates/scaletest-runner/scripts/lib.sh b/scaletest/templates/scaletest-runner/scripts/lib.sh index 57c96e091c4e6..da5b9fde5c46c 100644 --- a/scaletest/templates/scaletest-runner/scripts/lib.sh +++ b/scaletest/templates/scaletest-runner/scripts/lib.sh @@ -271,10 +271,12 @@ fetch_coder_full() { exit 1 fi log "Fetching full Coder binary from ${pod}" + # We need --retries due to https://github.com/kubernetes/kubernetes/issues/60140 :( maybedryrun "${DRY_RUN}" kubectl \ --namespace "${namespace}" \ cp \ --container coder \ + --retries 3 \ "${pod}:/opt/coder" "${SCALETEST_CODER_BINARY}" maybedryrun "${DRY_RUN}" chmod +x "${SCALETEST_CODER_BINARY}" log "Full Coder binary downloaded to ${SCALETEST_CODER_BINARY}" From 54830cc0b255f4a4b9066f26efdc802e87738455 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 10 Oct 2023 13:17:21 +0000 Subject: [PATCH 2/5] chore(scaletest/templates/scaletest-runner): remove --count parameter from scaletest dashboard command --- scaletest/templates/scaletest-runner/scripts/run.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scaletest/templates/scaletest-runner/scripts/run.sh b/scaletest/templates/scaletest-runner/scripts/run.sh index c9053e5a6b15f..2925fdb867ae5 100755 --- a/scaletest/templates/scaletest-runner/scripts/run.sh +++ b/scaletest/templates/scaletest-runner/scripts/run.sh @@ -49,7 +49,6 @@ for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do ;; "Dashboard Traffic") coder exp scaletest dashboard \ - --count "${SCALETEST_PARAM_NUM_WORKSPACES}" \ --timeout "${SCALETEST_PARAM_LOAD_SCENARIO_DASHBOARD_TRAFFIC_DURATION}m" \ --job-timeout "${SCALETEST_PARAM_LOAD_SCENARIO_DASHBOARD_TRAFFIC_DURATION}m30s" \ --output json:"${SCALETEST_RESULTS_DIR}/traffic-dashboard.json" \ From aaf16d864a2e7e19283e239654ddee49498e989e Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 10 Oct 2023 13:17:57 +0000 Subject: [PATCH 3/5] feat(scaletest/templates/scaletest-runner): scale provisioners up and down --- scaletest/templates/scaletest-runner/scripts/cleanup.sh | 6 ++++++ scaletest/templates/scaletest-runner/scripts/prepare.sh | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/scaletest/templates/scaletest-runner/scripts/cleanup.sh b/scaletest/templates/scaletest-runner/scripts/cleanup.sh index cf34da8297192..ecdd086d9a4e0 100755 --- a/scaletest/templates/scaletest-runner/scripts/cleanup.sh +++ b/scaletest/templates/scaletest-runner/scripts/cleanup.sh @@ -28,6 +28,12 @@ coder exp scaletest cleanup \ tee "${SCALETEST_RESULTS_DIR}/cleanup-${event}.txt" end_phase +if [[ $event != prepare ]]; then + start_phase "Scaling down provisioners..." + maybedryrun "$DRY_RUN" kubectl scale deployment/coder-provisioner --replicas 1 + maybedryrun "$DRY_RUN" kubectl rollout status deployment/coder-provisioner +fi + if [[ $event = manual ]]; then echo 'Press any key to continue...' read -s -r -n 1 diff --git a/scaletest/templates/scaletest-runner/scripts/prepare.sh b/scaletest/templates/scaletest-runner/scripts/prepare.sh index 055d0afc90b3b..882dd197e2fdb 100755 --- a/scaletest/templates/scaletest-runner/scripts/prepare.sh +++ b/scaletest/templates/scaletest-runner/scripts/prepare.sh @@ -51,3 +51,9 @@ log "Cleaning up from previous runs (if applicable)..." "${SCRIPTS_DIR}/cleanup.sh" "prepare" log "Preparation complete!" + +log "Scaling up provisioners to ${SCALETEST_PARAM_CREATE_CONCURRENCY}..." +maybedryrun "$DRY_RUN" kubectl scale deployment/coder-provisioner \ + --replicas "${SCALETEST_PARAM_CREATE_CONCURRENCY}" +log "Waiting for provisioners to scale up..." +maybedryrun "$DRY_RUN" kubectl rollout status deployment/coder-provisioner From 4a651c6576ec0256363757d5066a90eb347e1335 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 10 Oct 2023 15:06:04 +0100 Subject: [PATCH 4/5] go a bit nuttier Co-authored-by: Mathias Fredriksson --- scaletest/templates/scaletest-runner/scripts/lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scaletest/templates/scaletest-runner/scripts/lib.sh b/scaletest/templates/scaletest-runner/scripts/lib.sh index da5b9fde5c46c..884e0a3b91eff 100644 --- a/scaletest/templates/scaletest-runner/scripts/lib.sh +++ b/scaletest/templates/scaletest-runner/scripts/lib.sh @@ -276,7 +276,7 @@ fetch_coder_full() { --namespace "${namespace}" \ cp \ --container coder \ - --retries 3 \ + --retries 10 \ "${pod}:/opt/coder" "${SCALETEST_CODER_BINARY}" maybedryrun "${DRY_RUN}" chmod +x "${SCALETEST_CODER_BINARY}" log "Full Coder binary downloaded to ${SCALETEST_CODER_BINARY}" From db541c5784a5dd59bb81a003d0d82665e2766eac Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 10 Oct 2023 14:23:16 +0000 Subject: [PATCH 5/5] handle case where SCALETEST_PARAM_CREATE_CONCURRENCY=0 --- .../templates/scaletest-runner/scripts/prepare.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scaletest/templates/scaletest-runner/scripts/prepare.sh b/scaletest/templates/scaletest-runner/scripts/prepare.sh index 882dd197e2fdb..e7e6c4d2a292a 100755 --- a/scaletest/templates/scaletest-runner/scripts/prepare.sh +++ b/scaletest/templates/scaletest-runner/scripts/prepare.sh @@ -52,8 +52,14 @@ log "Cleaning up from previous runs (if applicable)..." log "Preparation complete!" -log "Scaling up provisioners to ${SCALETEST_PARAM_CREATE_CONCURRENCY}..." +PROVISIONER_REPLICA_COUNT="${SCALETEST_PARAM_CREATE_CONCURRENCY:-0}" +if [[ "${PROVISIONER_REPLICA_COUNT}" -eq 0 ]]; then + # TODO(Cian): what is a good default value here? + echo "Setting PROVISIONER_REPLICA_COUNT to 10 since SCALETEST_PARAM_CREATE_CONCURRENCY is 0" + PROVISIONER_REPLICA_COUNT=10 +fi +log "Scaling up provisioners to ${PROVISIONER_REPLICA_COUNT}..." maybedryrun "$DRY_RUN" kubectl scale deployment/coder-provisioner \ - --replicas "${SCALETEST_PARAM_CREATE_CONCURRENCY}" + --replicas "${PROVISIONER_REPLICA_COUNT}" log "Waiting for provisioners to scale up..." maybedryrun "$DRY_RUN" kubectl rollout status deployment/coder-provisioner