From 0abd25b89ac4f838fb4f4122f44eb011cec4c438 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 29 Sep 2023 15:14:02 +0100 Subject: [PATCH 1/8] fix(cli): remove exp scaletest from slim binary --- cli/exp_scaletest.go | 2 ++ cli/exp_scaletest_slim.go | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 cli/exp_scaletest_slim.go diff --git a/cli/exp_scaletest.go b/cli/exp_scaletest.go index 8c7a3ae427e01..5b8da2713fa40 100644 --- a/cli/exp_scaletest.go +++ b/cli/exp_scaletest.go @@ -1,3 +1,5 @@ +//go:build !slim + package cli import ( diff --git a/cli/exp_scaletest_slim.go b/cli/exp_scaletest_slim.go new file mode 100644 index 0000000000000..e0cf9d1aaf3b0 --- /dev/null +++ b/cli/exp_scaletest_slim.go @@ -0,0 +1,18 @@ +//go:build slim + +package cli + +import "github.com/coder/coder/v2/cli/clibase" + +func (r *RootCmd) scaletestCmd() *clibase.Cmd { + cmd := &clibase.Cmd{ + Use: "scaletest", + Short: "Run a scale test against the Coder API", + Handler: func(inv *clibase.Invocation) error { + SlimUnsupported(inv.Stderr, "scaletest") + return nil + }, + } + + return cmd +} From ead423e87abc93d5aeea3d20ba1f5196e4b6ed01 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 3 Oct 2023 11:50:36 +0100 Subject: [PATCH 2/8] fetch full coder binary --- cli/exp_scaletest_slim.go | 2 +- scaletest/lib/coder_init.sh | 17 ++++++-- .../templates/scaletest-runner/scripts/lib.sh | 40 ++++++++++++++++++- 3 files changed, 54 insertions(+), 5 deletions(-) diff --git a/cli/exp_scaletest_slim.go b/cli/exp_scaletest_slim.go index e0cf9d1aaf3b0..d9ccd325e5ccd 100644 --- a/cli/exp_scaletest_slim.go +++ b/cli/exp_scaletest_slim.go @@ -9,7 +9,7 @@ func (r *RootCmd) scaletestCmd() *clibase.Cmd { Use: "scaletest", Short: "Run a scale test against the Coder API", Handler: func(inv *clibase.Invocation) error { - SlimUnsupported(inv.Stderr, "scaletest") + SlimUnsupported(inv.Stderr, "exp scaletest") return nil }, } diff --git a/scaletest/lib/coder_init.sh b/scaletest/lib/coder_init.sh index c3b322e6c47a0..f8c905958ece4 100755 --- a/scaletest/lib/coder_init.sh +++ b/scaletest/lib/coder_init.sh @@ -20,7 +20,6 @@ ARCH="$(arch)" if [[ "$ARCH" == "x86_64" ]]; then ARCH="amd64" fi -PLATFORM="$(uname | tr '[:upper:]' '[:lower:]')" if [[ -f "${CONFIG_DIR}/coder.env" ]]; then echo "Found existing coder.env in ${CONFIG_DIR}!" @@ -29,8 +28,20 @@ if [[ -f "${CONFIG_DIR}/coder.env" ]]; then fi maybedryrun "$DRY_RUN" mkdir -p "${CONFIG_DIR}" -echo "Fetching Coder CLI for first-time setup!" -maybedryrun "$DRY_RUN" curl -fsSLk "${CODER_URL}/bin/coder-${PLATFORM}-${ARCH}" -o "${CONFIG_DIR}/coder" +echo "Fetching Coder for first-time setup!" +pod=$(kubectl get pods \ + --namespace="${NAMESPACE}" \ + --selector="app.kubernetes.io/name=coder,app.kubernetes.io/part-of=coder" \ + --output="jsonpath='{.items[0].metadata.name}'") +if [[ -z ${pod} ]]; then + log "Could not find coder pod!" + exit 1 +fi +maybedryrun "$DRY_RUN" kubectl \ + --namespace="${NAMESPACE}" \ + cp \ + --container=coder \ + "${pod}:/opt/coder" "${CONFIG_DIR}/coder" maybedryrun "$DRY_RUN" chmod +x "${CONFIG_DIR}/coder" set +o pipefail diff --git a/scaletest/templates/scaletest-runner/scripts/lib.sh b/scaletest/templates/scaletest-runner/scripts/lib.sh index ec94e952fda3c..719b116164d99 100644 --- a/scaletest/templates/scaletest-runner/scripts/lib.sh +++ b/scaletest/templates/scaletest-runner/scripts/lib.sh @@ -24,7 +24,12 @@ SCALETEST_PPROF_DIR="${SCALETEST_RUN_DIR}/pprof" mkdir -p "${SCALETEST_STATE_DIR}" "${SCALETEST_RESULTS_DIR}" "${SCALETEST_PPROF_DIR}" coder() { - maybedryrun "${DRY_RUN}" command coder "${@}" + if [[ -z "${SCALETEST_CODER_BINARY}" ]]; then + echo "Fetching full coder binary..." + fetch_coder_full + echo "SCALETEST_CODER_BINARY=${SCALETEST_CODER_BINARY}" + fi + maybedryrun "${DRY_RUN}" "${SCALETEST_CODER_BINARY}" "${@}" } show_json() { @@ -240,3 +245,36 @@ set_appearance() { --data "${newjson}" \ "${CODER_URL}/api/v2/appearance" } + +# fetch_coder_full fetches the full (non-slim) coder binary from one of the coder pods +# running in the same namespace as the current pod. +fetch_coder_full() { + local mkdir_dry_run_arg="" + if [[ "${DRY_RUN}" == "1" ]]; then + mkdir_dry_run_arg="--dry-run" + fi + target_dir=$(mktemp ${mkdir_dry_run_arg} --directory -t scaletest-coder-full-XXXXXX) + target_path="${target_dir}/coder" + local pod + local namespace + namespace=<(/var/run/secrets/kubernetes.io/serviceaccount/namespace) + if [[ -z ${namespace} ]]; then + log "Could not determine namespace!" + exit 1 + fi + pod=$(kubectl get pods \ + --namespace "${namespace}" \ + --selector "app.kubernetes.io/name=coder,app.kubernetes.io/part-of=coder" \ + --output jsonpath='{.items[0].metadata.name}') + if [[ -z ${pod} ]]; then + log "Could not find coder pod!" + exit 1 + fi + maybedryrun "${DRY_RUN}" kubectl \ + --namespace "${namespace}" \ + cp \ + --container coder \ + "${pod}:/opt/coder" "${target_path}" + maybedryrun chmod +x "${target_path}" + export SCALETEST_CODER_BINARY="${target_path}" +} From dc5ee10b9ece7dd0f9490cb1e9e52d5061e5a0cd Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 3 Oct 2023 11:04:26 +0000 Subject: [PATCH 3/8] fixup! fetch full coder binary --- scaletest/templates/scaletest-runner/scripts/lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scaletest/templates/scaletest-runner/scripts/lib.sh b/scaletest/templates/scaletest-runner/scripts/lib.sh index 719b116164d99..de66e45216982 100644 --- a/scaletest/templates/scaletest-runner/scripts/lib.sh +++ b/scaletest/templates/scaletest-runner/scripts/lib.sh @@ -24,7 +24,7 @@ SCALETEST_PPROF_DIR="${SCALETEST_RUN_DIR}/pprof" mkdir -p "${SCALETEST_STATE_DIR}" "${SCALETEST_RESULTS_DIR}" "${SCALETEST_PPROF_DIR}" coder() { - if [[ -z "${SCALETEST_CODER_BINARY}" ]]; then + if [[ -z "${SCALETEST_CODER_BINARY:-}" ]]; then echo "Fetching full coder binary..." fetch_coder_full echo "SCALETEST_CODER_BINARY=${SCALETEST_CODER_BINARY}" From a31a2d37b94f8f3dcc05eef629209acbbb3a7381 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 3 Oct 2023 11:13:50 +0000 Subject: [PATCH 4/8] fix namespace read --- scaletest/templates/scaletest-runner/scripts/lib.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scaletest/templates/scaletest-runner/scripts/lib.sh b/scaletest/templates/scaletest-runner/scripts/lib.sh index de66e45216982..94e8029782080 100644 --- a/scaletest/templates/scaletest-runner/scripts/lib.sh +++ b/scaletest/templates/scaletest-runner/scripts/lib.sh @@ -257,8 +257,8 @@ fetch_coder_full() { target_path="${target_dir}/coder" local pod local namespace - namespace=<(/var/run/secrets/kubernetes.io/serviceaccount/namespace) - if [[ -z ${namespace} ]]; then + namespace=$( Date: Tue, 3 Oct 2023 11:21:41 +0000 Subject: [PATCH 5/8] fix maybedryrun invocation, add more logging --- scaletest/templates/scaletest-runner/scripts/lib.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scaletest/templates/scaletest-runner/scripts/lib.sh b/scaletest/templates/scaletest-runner/scripts/lib.sh index 94e8029782080..0cfc3e848bc08 100644 --- a/scaletest/templates/scaletest-runner/scripts/lib.sh +++ b/scaletest/templates/scaletest-runner/scripts/lib.sh @@ -25,9 +25,9 @@ mkdir -p "${SCALETEST_STATE_DIR}" "${SCALETEST_RESULTS_DIR}" "${SCALETEST_PPROF_ coder() { if [[ -z "${SCALETEST_CODER_BINARY:-}" ]]; then - echo "Fetching full coder binary..." + log "Fetching full coder binary..." fetch_coder_full - echo "SCALETEST_CODER_BINARY=${SCALETEST_CODER_BINARY}" + log "SCALETEST_CODER_BINARY=${SCALETEST_CODER_BINARY}" fi maybedryrun "${DRY_RUN}" "${SCALETEST_CODER_BINARY}" "${@}" } @@ -262,6 +262,7 @@ fetch_coder_full() { log "Could not determine namespace!" exit 1 fi + log "Namespace from serviceaccount token is ${namespace}" pod=$(kubectl get pods \ --namespace "${namespace}" \ --selector "app.kubernetes.io/name=coder,app.kubernetes.io/part-of=coder" \ @@ -270,11 +271,13 @@ fetch_coder_full() { log "Could not find coder pod!" exit 1 fi + log "Fetching full Coder binary from ${pod}" maybedryrun "${DRY_RUN}" kubectl \ --namespace "${namespace}" \ cp \ --container coder \ "${pod}:/opt/coder" "${target_path}" - maybedryrun chmod +x "${target_path}" + maybedryrun "${DRY_RUN}" chmod +x "${target_path}" export SCALETEST_CODER_BINARY="${target_path}" + log "Full Coder binary is at ${SCALETEST_CODER_BINARY}" } From 1095cc8921222a2648f80965d35d8d9957109125 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 3 Oct 2023 11:49:34 +0000 Subject: [PATCH 6/8] idempotency --- scaletest/templates/scaletest-runner/scripts/lib.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scaletest/templates/scaletest-runner/scripts/lib.sh b/scaletest/templates/scaletest-runner/scripts/lib.sh index 0cfc3e848bc08..10b05a1e62c02 100644 --- a/scaletest/templates/scaletest-runner/scripts/lib.sh +++ b/scaletest/templates/scaletest-runner/scripts/lib.sh @@ -250,6 +250,10 @@ set_appearance() { # running in the same namespace as the current pod. fetch_coder_full() { local mkdir_dry_run_arg="" + if [[ -n "${SCALETEST_CODER_BINARY:-}" ]]; then + log "Full Coder binary already exists at ${SCALETEST_CODER_BINARY}" + return + fi if [[ "${DRY_RUN}" == "1" ]]; then mkdir_dry_run_arg="--dry-run" fi From 0618cbc0171cab46d5e465004dde96424101b01c Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 3 Oct 2023 12:07:22 +0000 Subject: [PATCH 7/8] store binary under known unique stable path --- .../templates/scaletest-runner/scripts/lib.sh | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/scaletest/templates/scaletest-runner/scripts/lib.sh b/scaletest/templates/scaletest-runner/scripts/lib.sh index 10b05a1e62c02..59ac3bd6b71bc 100644 --- a/scaletest/templates/scaletest-runner/scripts/lib.sh +++ b/scaletest/templates/scaletest-runner/scripts/lib.sh @@ -20,14 +20,14 @@ SCALETEST_PHASE_FILE="${SCALETEST_STATE_DIR}/phase" # shellcheck disable=SC2034 SCALETEST_RESULTS_DIR="${SCALETEST_RUN_DIR}/results" SCALETEST_PPROF_DIR="${SCALETEST_RUN_DIR}/pprof" +SCALETEST_CODER_BINARY="/tmp/coder-full-${SCALETEST_RUN_ID}" mkdir -p "${SCALETEST_STATE_DIR}" "${SCALETEST_RESULTS_DIR}" "${SCALETEST_PPROF_DIR}" coder() { - if [[ -z "${SCALETEST_CODER_BINARY:-}" ]]; then + if [[ ! -x "${SCALETEST_CODER_BINARY}" ]]; then log "Fetching full coder binary..." fetch_coder_full - log "SCALETEST_CODER_BINARY=${SCALETEST_CODER_BINARY}" fi maybedryrun "${DRY_RUN}" "${SCALETEST_CODER_BINARY}" "${@}" } @@ -249,16 +249,10 @@ set_appearance() { # fetch_coder_full fetches the full (non-slim) coder binary from one of the coder pods # running in the same namespace as the current pod. fetch_coder_full() { - local mkdir_dry_run_arg="" - if [[ -n "${SCALETEST_CODER_BINARY:-}" ]]; then + if [[ -x "${SCALETEST_CODER_BINARY}" ]]; then log "Full Coder binary already exists at ${SCALETEST_CODER_BINARY}" return fi - if [[ "${DRY_RUN}" == "1" ]]; then - mkdir_dry_run_arg="--dry-run" - fi - target_dir=$(mktemp ${mkdir_dry_run_arg} --directory -t scaletest-coder-full-XXXXXX) - target_path="${target_dir}/coder" local pod local namespace namespace=$( Date: Tue, 3 Oct 2023 12:18:27 +0000 Subject: [PATCH 8/8] work around kubectl foible --- scaletest/templates/scaletest-runner/scripts/lib.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scaletest/templates/scaletest-runner/scripts/lib.sh b/scaletest/templates/scaletest-runner/scripts/lib.sh index 59ac3bd6b71bc..57c96e091c4e6 100644 --- a/scaletest/templates/scaletest-runner/scripts/lib.sh +++ b/scaletest/templates/scaletest-runner/scripts/lib.sh @@ -20,7 +20,8 @@ SCALETEST_PHASE_FILE="${SCALETEST_STATE_DIR}/phase" # shellcheck disable=SC2034 SCALETEST_RESULTS_DIR="${SCALETEST_RUN_DIR}/results" SCALETEST_PPROF_DIR="${SCALETEST_RUN_DIR}/pprof" -SCALETEST_CODER_BINARY="/tmp/coder-full-${SCALETEST_RUN_ID}" +# https://github.com/kubernetes/kubernetes/issues/72501 :-( +SCALETEST_CODER_BINARY="/tmp/coder-full-${SCALETEST_RUN_ID//:/-}" mkdir -p "${SCALETEST_STATE_DIR}" "${SCALETEST_RESULTS_DIR}" "${SCALETEST_PPROF_DIR}"