|
| 1 | +#!/usr/bin/env bash |
| 2 | + |
| 3 | +[[ -n ${VERBOSE:-} ]] && set -x |
| 4 | +set -euo pipefail |
| 5 | + |
| 6 | +PROJECT_ROOT="$(git rev-parse --show-toplevel)" |
| 7 | +# shellcheck source=scripts/lib.sh |
| 8 | +source "${PROJECT_ROOT}/scripts/lib.sh" |
| 9 | + |
| 10 | +DRY_RUN="${DRY_RUN:-0}" |
| 11 | +SCALETEST_NAME="${SCALETEST_NAME:-}" |
| 12 | +SCALETEST_NUM_WORKSPACES="${SCALETEST_NUM_WORKSPACES:-}" |
| 13 | +SCALETEST_SCENARIO="${SCALETEST_SCENARIO:-}" |
| 14 | +SCALETEST_PROJECT="${SCALETEST_PROJECT:-}" |
| 15 | +SCALETEST_PROMETHEUS_REMOTE_WRITE_USER="${SCALETEST_PROMETHEUS_REMOTE_WRITE_USER:-}" |
| 16 | +SCALETEST_PROMETHEUS_REMOTE_WRITE_PASSWORD="${SCALETEST_PROMETHEUS_REMOTE_WRITE_PASSWORD:-}" |
| 17 | +SCALETEST_SKIP_CLEANUP="${SCALETEST_SKIP_CLEANUP:-0}" |
| 18 | + |
| 19 | +script_name=$(basename "$0") |
| 20 | +args="$(getopt -o "" -l dry-run,help,name:,num-workspaces:,project:,scenario:,skip-cleanup -- "$@")" |
| 21 | +eval set -- "$args" |
| 22 | +while true; do |
| 23 | + case "$1" in |
| 24 | + --dry-run) |
| 25 | + DRY_RUN=1 |
| 26 | + shift |
| 27 | + ;; |
| 28 | + --help) |
| 29 | + echo "Usage: $script_name --name <name> --project <project> --num-workspaces <num-workspaces> --scenario <scenario> [--dry-run] [--skip-cleanup]" |
| 30 | + exit 1 |
| 31 | + ;; |
| 32 | + --name) |
| 33 | + SCALETEST_NAME="$2" |
| 34 | + shift 2 |
| 35 | + ;; |
| 36 | + --num-workspaces) |
| 37 | + SCALETEST_NUM_WORKSPACES="$2" |
| 38 | + shift 2 |
| 39 | + ;; |
| 40 | + --project) |
| 41 | + SCALETEST_PROJECT="$2" |
| 42 | + shift 2 |
| 43 | + ;; |
| 44 | + --scenario) |
| 45 | + SCALETEST_SCENARIO="$2" |
| 46 | + shift 2 |
| 47 | + ;; |
| 48 | + --skip-cleanup) |
| 49 | + SCALETEST_SKIP_CLEANUP=1 |
| 50 | + shift |
| 51 | + ;; |
| 52 | + --) |
| 53 | + shift |
| 54 | + break |
| 55 | + ;; |
| 56 | + *) |
| 57 | + error "Unrecognized option: $1" |
| 58 | + ;; |
| 59 | + esac |
| 60 | +done |
| 61 | + |
| 62 | +dependencies gcloud kubectl terraform |
| 63 | + |
| 64 | +if [[ -z "${SCALETEST_NAME}" ]]; then |
| 65 | + echo "Must specify --name" |
| 66 | + exit 1 |
| 67 | +fi |
| 68 | + |
| 69 | +if [[ -z "${SCALETEST_PROJECT}" ]]; then |
| 70 | + echo "Must specify --project" |
| 71 | + exit 1 |
| 72 | +fi |
| 73 | + |
| 74 | +if [[ -z "${SCALETEST_NUM_WORKSPACES}" ]]; then |
| 75 | + echo "Must specify --num-workspaces" |
| 76 | + exit 1 |
| 77 | +fi |
| 78 | + |
| 79 | +if [[ -z "${SCALETEST_SCENARIO}" ]]; then |
| 80 | + echo "Must specify --scenario" |
| 81 | + exit 1 |
| 82 | +fi |
| 83 | + |
| 84 | +if [[ -z "${SCALETEST_PROMETHEUS_REMOTE_WRITE_USER}" ]] || [[ -z "${SCALETEST_PROMETHEUS_REMOTE_WRITE_PASSWORD}" ]]; then |
| 85 | + echo "SCALETEST_PROMETHEUS_REMOTE_WRITE_USER or SCALETEST_PROMETHEUS_REMOTE_WRITE_PASSWORD not specified." |
| 86 | + echo "No prometheus metrics will be collected!" |
| 87 | + read -pr "Continue (y/N)? " choice |
| 88 | + case "$choice" in |
| 89 | + y | Y | yes | YES) ;; |
| 90 | + *) exit 1 ;; |
| 91 | + esac |
| 92 | +fi |
| 93 | + |
| 94 | +SCALETEST_SCENARIO_VARS="${PROJECT_ROOT}/scaletest/terraform/scenario-${SCALETEST_SCENARIO}.tfvars" |
| 95 | +if [[ ! -f "${SCALETEST_SCENARIO_VARS}" ]]; then |
| 96 | + echo "Scenario ${SCALETEST_SCENARIO_VARS} not found." |
| 97 | + echo "Please create it or choose another scenario:" |
| 98 | + find "${PROJECT_ROOT}/scaletest/terraform" -type f -name 'scenario-*.tfvars' |
| 99 | + exit 1 |
| 100 | +fi |
| 101 | + |
| 102 | +if [[ "${SCALETEST_SKIP_CLEANUP}" == 1 ]]; then |
| 103 | + log "WARNING: you told me to not clean up after myself, so this is now your job!" |
| 104 | +fi |
| 105 | + |
| 106 | +CONFIG_DIR="${PROJECT_ROOT}/scaletest/.coderv2" |
| 107 | +if [[ -d "${CONFIG_DIR}" ]] && files=$(ls -qAH -- "${CONFIG_DIR}") && [[ -z "$files" ]]; then |
| 108 | + echo "Cleaning previous configuration" |
| 109 | + maybedryrun "$DRY_RUN" rm -fv "${CONFIG_DIR}/*" |
| 110 | +fi |
| 111 | +maybedryrun "$DRY_RUN" mkdir -p "${CONFIG_DIR}" |
| 112 | + |
| 113 | +SCALETEST_SCENARIO_VARS="${PROJECT_ROOT}/scaletest/terraform/scenario-${SCALETEST_SCENARIO}.tfvars" |
| 114 | +SCALETEST_SECRETS="${PROJECT_ROOT}/scaletest/terraform/secrets.tfvars" |
| 115 | +SCALETEST_SECRETS_TEMPLATE="${PROJECT_ROOT}/scaletest/terraform/secrets.tfvars.tpl" |
| 116 | + |
| 117 | +log "Writing scaletest secrets to file." |
| 118 | +SCALETEST_NAME="${SCALETEST_NAME}" \ |
| 119 | + SCALETEST_PROJECT="${SCALETEST_PROJECT}" \ |
| 120 | + SCALETEST_PROMETHEUS_REMOTE_WRITE_USER="${SCALETEST_PROMETHEUS_REMOTE_WRITE_USER}" \ |
| 121 | + SCALETEST_PROMETHEUS_REMOTE_WRITE_PASSWORD="${SCALETEST_PROMETHEUS_REMOTE_WRITE_PASSWORD}" \ |
| 122 | + envsubst <"${SCALETEST_SECRETS_TEMPLATE}" >"${SCALETEST_SECRETS}" |
| 123 | + |
| 124 | +pushd "${PROJECT_ROOT}/scaletest/terraform" |
| 125 | + |
| 126 | +echo "Initializing terraform." |
| 127 | +maybedryrun "$DRY_RUN" terraform init |
| 128 | + |
| 129 | +echo "Setting up infrastructure." |
| 130 | +maybedryrun "$DRY_RUN" terraform apply --var-file="${SCALETEST_SCENARIO_VARS}" --var-file="${SCALETEST_SECRETS}" --auto-approve |
| 131 | + |
| 132 | +if [[ "${DRY_RUN}" != 1 ]]; then |
| 133 | + SCALETEST_CODER_URL=$(<"${CONFIG_DIR}/url") |
| 134 | +else |
| 135 | + SCALETEST_CODER_URL="http://coder.dryrun.local:3000" |
| 136 | +fi |
| 137 | +KUBECONFIG="${PROJECT_ROOT}/scaletest/.coderv2/${SCALETEST_NAME}-cluster.kubeconfig" |
| 138 | +echo "Waiting for Coder deployment at ${SCALETEST_CODER_URL} to become ready" |
| 139 | +maybedryrun "$DRY_RUN" kubectl --kubeconfig="${KUBECONFIG}" -n "coder-${SCALETEST_NAME}" rollout status deployment/coder |
| 140 | + |
| 141 | +echo "Initializing Coder deployment." |
| 142 | +DRY_RUN="$DRY_RUN" "${PROJECT_ROOT}/scaletest/lib/coder_init.sh" "${SCALETEST_CODER_URL}" |
| 143 | + |
| 144 | +echo "Creating ${SCALETEST_NUM_WORKSPACES} workspaces." |
| 145 | +DRY_RUN="$DRY_RUN" "${PROJECT_ROOT}/scaletest/lib/coder_shim.sh" scaletest create-workspaces \ |
| 146 | + --count "${SCALETEST_NUM_WORKSPACES}" \ |
| 147 | + --template=kubernetes \ |
| 148 | + --concurrency 10 \ |
| 149 | + --no-cleanup |
| 150 | + |
| 151 | +echo "Sleeping 10 minutes to establish a baseline measurement." |
| 152 | +maybedryrun "$DRY_RUN" sleep 600 |
| 153 | + |
| 154 | +echo "Sending traffic to workspaces" |
| 155 | +maybedryrun "$DRY_RUN" "${PROJECT_ROOT}/scaletest/lib/coder_workspacetraffic.sh" "${SCALETEST_NAME}" |
| 156 | +maybedryrun "$DRY_RUN" kubectl --kubeconfig="${KUBECONFIG}" -n "coder-${SCALETEST_NAME}" wait pods coder-scaletest-workspace-traffic --for condition=Ready |
| 157 | +maybedryrun "$DRY_RUN" kubectl --kubeconfig="${KUBECONFIG}" -n "coder-${SCALETEST_NAME}" logs -f pod/coder-scaletest-workspace-traffic |
| 158 | + |
| 159 | +echo "Starting pprof" |
| 160 | +maybedryrun "$DRY_RUN" kubectl -n "coder-${SCALETEST_NAME}" port-forward deployment/coder 6061:6060 & |
| 161 | +pfpid=$! |
| 162 | +maybedryrun "$DRY_RUN" trap "kill $pfpid" EXIT |
| 163 | + |
| 164 | +echo "Waiting for pprof endpoint to become available" |
| 165 | +pprof_attempt_counter=0 |
| 166 | +while ! maybedryrun "$DRY_RUN" timeout 1 bash -c "echo > /dev/tcp/localhost/6061"; do |
| 167 | + if [[ $pprof_attempt_counter -eq 10 ]]; then |
| 168 | + echo |
| 169 | + echo "pprof failed to become ready in time!" |
| 170 | + exit 1 |
| 171 | + fi |
| 172 | + maybedryrun "$DRY_RUN" sleep 3 |
| 173 | +done |
| 174 | + |
| 175 | +echo "Taking pprof snapshots" |
| 176 | +maybedryrun "$DRY_RUN" curl --silent --fail --output "${SCALETEST_NAME}-heap.pprof.gz" http://localhost:6061/debug/pprof/heap |
| 177 | +maybedryrun "$DRY_RUN" curl --silent --fail --output "${SCALETEST_NAME}-goroutine.pprof.gz" http://localhost:6061/debug/pprof/goroutine |
| 178 | +# No longer need to port-forward |
| 179 | +maybedryrun "$DRY_RUN" kill "$pfpid" |
| 180 | +maybedryrun "$DRY_RUN" trap - EXIT |
| 181 | + |
| 182 | +if [[ "${SCALETEST_SKIP_CLEANUP}" == 1 ]]; then |
| 183 | + echo "Leaving resources up for you to inspect." |
| 184 | + echo "Please don't forget to clean up afterwards:" |
| 185 | + echo "cd terraform && terraform destroy --var-file=${SCALETEST_SCENARIO_VARS} --var-file=${SCALETEST_SECRETS} --auto-approve" |
| 186 | + exit 0 |
| 187 | +fi |
| 188 | + |
| 189 | +echo "Cleaning up" |
| 190 | +maybedryrun "$DRY_RUN" terraform destroy --var-file="${SCALETEST_SCENARIO_VARS}" --var-file="${SCALETEST_SECRETS}" --auto-approve |
0 commit comments