From 28d68794a646f97a75b79a4ad7b26ef81e2b776d Mon Sep 17 00:00:00 2001 From: Parkreiner Date: Fri, 22 Nov 2024 20:21:30 +0000 Subject: [PATCH 01/14] chore: add updates to force redeployment on Vercel --- .github/scripts/check.sh | 58 ++++++++++++++++++++++++++++++++++++ .github/workflows/check.yaml | 1 + 2 files changed, 59 insertions(+) diff --git a/.github/scripts/check.sh b/.github/scripts/check.sh index abb47907..df251672 100755 --- a/.github/scripts/check.sh +++ b/.github/scripts/check.sh @@ -7,6 +7,8 @@ required_vars=( "INSTATUS_API_KEY" "INSTATUS_PAGE_ID" "INSTATUS_COMPONENT_ID" + "VERCEL_API_KEY" + ) # Check if each required variable is set @@ -17,6 +19,11 @@ for var in "${required_vars[@]}"; do fi done +JUST_REDEPLOYED="$JUST_REDEPLOYED:-0" +if (JUST_REDEPLOYED); do + return 1 +fi + REGISTRY_BASE_URL="${REGISTRY_BASE_URL:-https://registry.coder.com}" status=0 @@ -74,6 +81,41 @@ create_incident() { echo "$incident_id" } +force_redeploy_registry () { + # These are not secret values; safe to just expose directly in script + local VERCEL_TEAM_SLUG="codercom" + local VERCEL_TEAM_ID="team_tGkWfhEGGelkkqUUm9nXq17r" + local VERCEL_APP="registry" + + local latest_res=$(curl "https://api.vercel.com/v6/deployments?app=$VERCEL_APP&limit=1&slug=$VERCEL_TEAM_SLUG&teamId=$VERCEL_TEAM_ID" \ + --fail \ + --silent \ + -H "Authorization: Bearer $VERCEL_API_KEY" \ + -H "Content-Type: application/json" + ) + + # If we have zero deployments, something is VERY wrong. Make the whole + # script exit with a non-zero status code + local latest_id=$(echo $latest_res | jq '.deployments[0].uid') + if (( latest_id == "null" )); do + echo "Unable to pull any previous deployments for redeployment" + return 1 + fi + + local redeploy_res=$(curl -X POST "https://api.vercel.com/v13/deployments?forceNew=1&skipAutoDetectionConfirmation=1&slug=$VERCEL_TEAM_SLUG&teamId=$VERCEL_TEAM_ID" \ + --fail \ + --silent \ + --output "/dev/null" \ + -H "Authorization: Bearer $VERCEL_API_KEY" \ + -H "Content-Type: application/json" \ + -d "{ + "deploymentId": $latest_id, + }" + ) + + echo $redeploy_res +} + # Check each module's accessibility for module in "${modules[@]}"; do # Trim leading/trailing whitespace from module name @@ -96,6 +138,8 @@ if (( status == 0 )); then echo "All modules are operational." # set to update_component_status "OPERATIONAL" + + echo "JUST_REDEPLOYED=0" >> $GITHUB_ENV else echo "The following modules have issues: ${failures[*]}" # check if all modules are down @@ -108,6 +152,20 @@ else # Create a new incident incident_id=$(create_incident) echo "Created incident with ID: $incident_id" + + # If a module is down, force a reployment to try getting things back online + # ASAP + status_code=$(force_redeploy_registry) + # shellcheck disable=SC2181 + if (( status_code == 200 )); then + echo "Reployment successful" + else + echo "Unable to redeploy automatically" + fi + + # Update environment variable so that if automatic re-deployment fails, we + # don't keep running the script over and over again + echo "JUST_REDEPLOYED=1" >> $GITHUB_ENV fi exit "${status}" diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index 4095073e..2e3561ef 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -20,3 +20,4 @@ jobs: INSTATUS_API_KEY: ${{ secrets.INSTATUS_API_KEY }} INSTATUS_PAGE_ID: ${{ secrets.INSTATUS_PAGE_ID }} INSTATUS_COMPONENT_ID: ${{ secrets.INSTATUS_COMPONENT_ID }} + VERCEL_API_KEY: ${{ secrets.VERCEL_API_KEY }} From 5827168382903c90e4d884745605cd0f041d183b Mon Sep 17 00:00:00 2001 From: Parkreiner Date: Fri, 22 Nov 2024 20:25:02 +0000 Subject: [PATCH 02/14] refactor: clean up --- .github/scripts/check.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/scripts/check.sh b/.github/scripts/check.sh index df251672..28981027 100755 --- a/.github/scripts/check.sh +++ b/.github/scripts/check.sh @@ -8,7 +8,6 @@ required_vars=( "INSTATUS_PAGE_ID" "INSTATUS_COMPONENT_ID" "VERCEL_API_KEY" - ) # Check if each required variable is set @@ -19,8 +18,9 @@ for var in "${required_vars[@]}"; do fi done -JUST_REDEPLOYED="$JUST_REDEPLOYED:-0" -if (JUST_REDEPLOYED); do +LATEST_REDEPLOY_FAILED="$LATEST_REDEPLOY_FAILED:-0" +if (LATEST_REDEPLOY_FAILED); do + echo "Trying to re-run job when previous re-deploy failed" return 1 fi @@ -139,7 +139,7 @@ if (( status == 0 )); then # set to update_component_status "OPERATIONAL" - echo "JUST_REDEPLOYED=0" >> $GITHUB_ENV + echo "LATEST_REDEPLOY_FAILED=0" >> $GITHUB_ENV else echo "The following modules have issues: ${failures[*]}" # check if all modules are down @@ -165,7 +165,7 @@ else # Update environment variable so that if automatic re-deployment fails, we # don't keep running the script over and over again - echo "JUST_REDEPLOYED=1" >> $GITHUB_ENV + echo "LATEST_REDEPLOY_FAILED=1" >> $GITHUB_ENV fi exit "${status}" From bf3e5efacaca2ac1455f55e2351d6afa574eacc8 Mon Sep 17 00:00:00 2001 From: Parkreiner Date: Fri, 22 Nov 2024 20:30:39 +0000 Subject: [PATCH 03/14] docs: update comment for clarity --- .github/scripts/check.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/scripts/check.sh b/.github/scripts/check.sh index 28981027..078bc75a 100755 --- a/.github/scripts/check.sh +++ b/.github/scripts/check.sh @@ -164,7 +164,9 @@ else fi # Update environment variable so that if automatic re-deployment fails, we - # don't keep running the script over and over again + # don't keep running the script over and over again. Note that even if a + # re-deployment succeeds, that doesn't necessarily mean that everything is + # fully operational echo "LATEST_REDEPLOY_FAILED=1" >> $GITHUB_ENV fi From 3956f45e62f2f2371a79d5a55249411bc9857f66 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 22 Nov 2024 21:52:06 +0000 Subject: [PATCH 04/14] address linter complaints --- .github/scripts/check.sh | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/.github/scripts/check.sh b/.github/scripts/check.sh index 078bc75a..483b1942 100755 --- a/.github/scripts/check.sh +++ b/.github/scripts/check.sh @@ -18,8 +18,8 @@ for var in "${required_vars[@]}"; do fi done -LATEST_REDEPLOY_FAILED="$LATEST_REDEPLOY_FAILED:-0" -if (LATEST_REDEPLOY_FAILED); do +LATEST_REDEPLOY_FAILED="${LATEST_REDEPLOY_FAILED:-0}" +if [[ "${LATEST_REDEPLOY_FAILED}" -ne "0" ]] ;then echo "Trying to re-run job when previous re-deploy failed" return 1 fi @@ -52,7 +52,7 @@ create_incident() { local incident_name="Testing Instatus" local message="The following modules are experiencing issues:\n" for i in "${!failures[@]}"; do - message+="$(($i + 1)). ${failures[$i]}\n" + message+="$((i + 1)). ${failures[$i]}\n" done component_status="PARTIALOUTAGE" @@ -87,7 +87,8 @@ force_redeploy_registry () { local VERCEL_TEAM_ID="team_tGkWfhEGGelkkqUUm9nXq17r" local VERCEL_APP="registry" - local latest_res=$(curl "https://api.vercel.com/v6/deployments?app=$VERCEL_APP&limit=1&slug=$VERCEL_TEAM_SLUG&teamId=$VERCEL_TEAM_ID" \ + local latest_res + latest_res=$(curl "https://api.vercel.com/v6/deployments?app=$VERCEL_APP&limit=1&slug=$VERCEL_TEAM_SLUG&teamId=$VERCEL_TEAM_ID" \ --fail \ --silent \ -H "Authorization: Bearer $VERCEL_API_KEY" \ @@ -96,24 +97,24 @@ force_redeploy_registry () { # If we have zero deployments, something is VERY wrong. Make the whole # script exit with a non-zero status code - local latest_id=$(echo $latest_res | jq '.deployments[0].uid') - if (( latest_id == "null" )); do - echo "Unable to pull any previous deployments for redeployment" + local latest_id + latest_id=$(echo "${latest_res}" | jq '.deployments[0].uid') + if [[ "${latest_id}" = "null" ]]; then + echo "Unable to pull any previous deployments for redeployment" return 1 fi - local redeploy_res=$(curl -X POST "https://api.vercel.com/v13/deployments?forceNew=1&skipAutoDetectionConfirmation=1&slug=$VERCEL_TEAM_SLUG&teamId=$VERCEL_TEAM_ID" \ + local redeploy_res + redeploy_res=$(curl -X POST "https://api.vercel.com/v13/deployments?forceNew=1&skipAutoDetectionConfirmation=1&slug=$VERCEL_TEAM_SLUG&teamId=$VERCEL_TEAM_ID" \ --fail \ --silent \ --output "/dev/null" \ -H "Authorization: Bearer $VERCEL_API_KEY" \ -H "Content-Type: application/json" \ - -d "{ - "deploymentId": $latest_id, - }" + -d "{ \"deploymentId\": \"${latest_id}\" }" ) - echo $redeploy_res + echo "${redeploy_res}" } # Check each module's accessibility @@ -136,13 +137,13 @@ done # Determine overall status and update Instatus component if (( status == 0 )); then echo "All modules are operational." - # set to + # set to update_component_status "OPERATIONAL" - echo "LATEST_REDEPLOY_FAILED=0" >> $GITHUB_ENV + echo "LATEST_REDEPLOY_FAILED=0" >> "${GITHUB_ENV}" else echo "The following modules have issues: ${failures[*]}" - # check if all modules are down + # check if all modules are down if (( ${#failures[@]} == ${#modules[@]} )); then update_component_status "MAJOROUTAGE" else @@ -152,7 +153,7 @@ else # Create a new incident incident_id=$(create_incident) echo "Created incident with ID: $incident_id" - + # If a module is down, force a reployment to try getting things back online # ASAP status_code=$(force_redeploy_registry) @@ -167,7 +168,7 @@ else # don't keep running the script over and over again. Note that even if a # re-deployment succeeds, that doesn't necessarily mean that everything is # fully operational - echo "LATEST_REDEPLOY_FAILED=1" >> $GITHUB_ENV + echo "LATEST_REDEPLOY_FAILED=1" >> "${GITHUB_ENV}" fi exit "${status}" From 0173eebd860ba5f9b75fc34319f736fb802ee06e Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 22 Nov 2024 21:53:33 +0000 Subject: [PATCH 05/14] allow verbosity --- .github/scripts/check.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/scripts/check.sh b/.github/scripts/check.sh index 483b1942..e11d3ae5 100755 --- a/.github/scripts/check.sh +++ b/.github/scripts/check.sh @@ -2,6 +2,11 @@ set -o pipefail set -u +VERBOSE="${VERBOSE:-0}" +if [[ "${VERBOSE}" -ne "0" ]]; then + set -x +fi + # List of required environment variables required_vars=( "INSTATUS_API_KEY" From e5ad19dcf87c7b7ecf3a457d3119c33f47f9b2dd Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 22 Nov 2024 21:53:43 +0000 Subject: [PATCH 06/14] fix find complaint about arg order --- .github/scripts/check.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/check.sh b/.github/scripts/check.sh index e11d3ae5..74625a2b 100755 --- a/.github/scripts/check.sh +++ b/.github/scripts/check.sh @@ -36,7 +36,7 @@ declare -a modules=() declare -a failures=() # Collect all module directories containing a main.tf file -for path in $(find . -not -path '*/.*' -type f -name main.tf -maxdepth 2 | cut -d '/' -f 2 | sort -u); do +for path in $(find . -maxdepth 2 -not -path '*/.*' -type f -name main.tf | cut -d '/' -f 2 | sort -u); do modules+=("${path}") done From 2870f4b0a5081cbdd7e7589401233a71ad7223fc Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 22 Nov 2024 22:38:51 +0000 Subject: [PATCH 07/14] do not automatically force deploy if previous redeploy was within the last 2 hours --- .github/scripts/check.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/scripts/check.sh b/.github/scripts/check.sh index 74625a2b..e1f14114 100755 --- a/.github/scripts/check.sh +++ b/.github/scripts/check.sh @@ -109,6 +109,16 @@ force_redeploy_registry () { return 1 fi + local latest_date_ts_seconds + latest_date_ts_seconds=$(echo "${latest_res}" | jq '.deployments[0].createdAt/1000|floor)') + local current_date_ts_seconds + current_date_ts_seconds="$(date +%s)" + local max_redeploy_interval_seconds=7200 # 2 hours + if (( current_date_ts_seconds - latest_date_ts_seconds < max_redeploy_interval_seconds )); then + echo "Last deployment was less than 2 hours ago. Skipping redeployment." + return 1 + fi + local redeploy_res redeploy_res=$(curl -X POST "https://api.vercel.com/v13/deployments?forceNew=1&skipAutoDetectionConfirmation=1&slug=$VERCEL_TEAM_SLUG&teamId=$VERCEL_TEAM_ID" \ --fail \ From c4ad39d58429a756475889624df02f0fdeff1ba5 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 22 Nov 2024 22:38:59 +0000 Subject: [PATCH 08/14] update cron to every 15 minutes --- .github/workflows/check.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index 2e3561ef..02422ff2 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -2,7 +2,7 @@ name: Health # Check modules health on registry.coder.com on: schedule: - - cron: "*/13 * * * *" # Runs every 13th minute + - cron: "0,15,30,45 * * * *" # Runs every 15 minutes workflow_dispatch: # Allows manual triggering of the workflow if needed jobs: From 4216c7d02cbabfa0d1fa58d9e65832bd34f31d1e Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 22 Nov 2024 23:16:03 +0000 Subject: [PATCH 09/14] fix bad jq --- .github/scripts/check.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/check.sh b/.github/scripts/check.sh index e1f14114..74095de8 100755 --- a/.github/scripts/check.sh +++ b/.github/scripts/check.sh @@ -110,7 +110,7 @@ force_redeploy_registry () { fi local latest_date_ts_seconds - latest_date_ts_seconds=$(echo "${latest_res}" | jq '.deployments[0].createdAt/1000|floor)') + latest_date_ts_seconds=$(echo "${latest_res}" | jq '.deployments[0].createdAt/1000|floor') local current_date_ts_seconds current_date_ts_seconds="$(date +%s)" local max_redeploy_interval_seconds=7200 # 2 hours From b14b7fc6652658a91dd14939f861980347e85097 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 22 Nov 2024 23:17:15 +0000 Subject: [PATCH 10/14] jq raw output --- .github/scripts/check.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/scripts/check.sh b/.github/scripts/check.sh index 74095de8..d2b8e7fb 100755 --- a/.github/scripts/check.sh +++ b/.github/scripts/check.sh @@ -103,14 +103,14 @@ force_redeploy_registry () { # If we have zero deployments, something is VERY wrong. Make the whole # script exit with a non-zero status code local latest_id - latest_id=$(echo "${latest_res}" | jq '.deployments[0].uid') + latest_id=$(echo "${latest_res}" | jq -r '.deployments[0].uid') if [[ "${latest_id}" = "null" ]]; then echo "Unable to pull any previous deployments for redeployment" return 1 fi local latest_date_ts_seconds - latest_date_ts_seconds=$(echo "${latest_res}" | jq '.deployments[0].createdAt/1000|floor') + latest_date_ts_seconds=$(echo "${latest_res}" | jq -r '.deployments[0].createdAt/1000|floor') local current_date_ts_seconds current_date_ts_seconds="$(date +%s)" local max_redeploy_interval_seconds=7200 # 2 hours @@ -139,7 +139,6 @@ for module in "${modules[@]}"; do url="${REGISTRY_BASE_URL}/modules/${module}" printf "=== Checking module %s at %s\n" "${module}" "${url}" status_code=$(curl --output /dev/null --head --silent --fail --location "${url}" --retry 3 --write-out "%{http_code}") - # shellcheck disable=SC2181 if (( status_code != 200 )); then printf "==> FAIL(%s)\n" "${status_code}" status=1 From 95fd65b875866271c7066a2b7ae3a5fc10f9f691 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 22 Nov 2024 23:19:40 +0000 Subject: [PATCH 11/14] ben says this is what we want --- .github/scripts/check.sh | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/.github/scripts/check.sh b/.github/scripts/check.sh index d2b8e7fb..7fef919c 100755 --- a/.github/scripts/check.sh +++ b/.github/scripts/check.sh @@ -93,11 +93,11 @@ force_redeploy_registry () { local VERCEL_APP="registry" local latest_res - latest_res=$(curl "https://api.vercel.com/v6/deployments?app=$VERCEL_APP&limit=1&slug=$VERCEL_TEAM_SLUG&teamId=$VERCEL_TEAM_ID" \ + latest_res=$(curl "https://api.vercel.com/v6/deployments?app=$VERCEL_APP&limit=1&slug=$VERCEL_TEAM_SLUG&teamId=$VERCEL_TEAM_ID&target=production&state=BUILDING,INITIALIZING,QUEUED,READY" \ --fail \ --silent \ - -H "Authorization: Bearer $VERCEL_API_KEY" \ - -H "Content-Type: application/json" + --header "Authorization: Bearer $VERCEL_API_KEY" \ + --header "Content-Type: application/json" ) # If we have zero deployments, something is VERY wrong. Make the whole @@ -119,17 +119,26 @@ force_redeploy_registry () { return 1 fi - local redeploy_res - redeploy_res=$(curl -X POST "https://api.vercel.com/v13/deployments?forceNew=1&skipAutoDetectionConfirmation=1&slug=$VERCEL_TEAM_SLUG&teamId=$VERCEL_TEAM_ID" \ - --fail \ - --silent \ - --output "/dev/null" \ - -H "Authorization: Bearer $VERCEL_API_KEY" \ - -H "Content-Type: application/json" \ - -d "{ \"deploymentId\": \"${latest_id}\" }" - ) + local latest_deployment_state + latest_deployment_state="$(echo "${latest_res}" | jq -r '.deployments[0].state')" + if [[ "${latest_deployment_state}" != "READY" ]]; then + echo "Last deployment was not in READY state. Skipping redeployment." + return 1 + fi + + echo "=============================================================" + echo "!!! Redeploying registry with deployment ID: ${latest_id} !!!" + echo "=============================================================" - echo "${redeploy_res}" + if curl -X POST "https://api.vercel.com/v13/deployments?forceNew=1&skipAutoDetectionConfirmation=1&slug=$VERCEL_TEAM_SLUG&teamId=$VERCEL_TEAM_ID&target=production" \ + --fail \ + --header "Authorization: Bearer $VERCEL_API_KEY" \ + --header "Content-Type: application/json" \ + --data-raw "{ \"deploymentId\": \"${latest_id}\", \"name\": \"${VERCEL_APP}\" }"; then + echo "0" + else + echo "1" + fi } # Check each module's accessibility @@ -170,9 +179,8 @@ else # If a module is down, force a reployment to try getting things back online # ASAP - status_code=$(force_redeploy_registry) - # shellcheck disable=SC2181 - if (( status_code == 200 )); then + redeploy_result=$(force_redeploy_registry) + if (( redeploy_result == 0 )); then echo "Reployment successful" else echo "Unable to redeploy automatically" From 5cfa5916e521a18e90873918fdb5e9459539ba27 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 22 Nov 2024 23:32:18 +0000 Subject: [PATCH 12/14] fix logic --- .github/scripts/check.sh | 35 +++++++++++++---------------------- 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/.github/scripts/check.sh b/.github/scripts/check.sh index 7fef919c..fcecc8de 100755 --- a/.github/scripts/check.sh +++ b/.github/scripts/check.sh @@ -106,7 +106,7 @@ force_redeploy_registry () { latest_id=$(echo "${latest_res}" | jq -r '.deployments[0].uid') if [[ "${latest_id}" = "null" ]]; then echo "Unable to pull any previous deployments for redeployment" - return 1 + exit 1 fi local latest_date_ts_seconds @@ -116,28 +116,32 @@ force_redeploy_registry () { local max_redeploy_interval_seconds=7200 # 2 hours if (( current_date_ts_seconds - latest_date_ts_seconds < max_redeploy_interval_seconds )); then echo "Last deployment was less than 2 hours ago. Skipping redeployment." - return 1 + echo "Please check the Vercel dashboard for more information." + echo "https://vercel.com/codercom/registry/deployments" + exit 1 fi local latest_deployment_state latest_deployment_state="$(echo "${latest_res}" | jq -r '.deployments[0].state')" if [[ "${latest_deployment_state}" != "READY" ]]; then echo "Last deployment was not in READY state. Skipping redeployment." - return 1 + echo "Please check the Vercel dashboard for more information." + echo "https://vercel.com/codercom/registry/deployments" + exit 1 fi echo "=============================================================" echo "!!! Redeploying registry with deployment ID: ${latest_id} !!!" echo "=============================================================" - if curl -X POST "https://api.vercel.com/v13/deployments?forceNew=1&skipAutoDetectionConfirmation=1&slug=$VERCEL_TEAM_SLUG&teamId=$VERCEL_TEAM_ID&target=production" \ + if ! curl -X POST "https://api.vercel.com/v13/deployments?forceNew=1&skipAutoDetectionConfirmation=1&slug=$VERCEL_TEAM_SLUG&teamId=$VERCEL_TEAM_ID" \ --fail \ --header "Authorization: Bearer $VERCEL_API_KEY" \ --header "Content-Type: application/json" \ - --data-raw "{ \"deploymentId\": \"${latest_id}\", \"name\": \"${VERCEL_APP}\" }"; then - echo "0" - else - echo "1" + --data-raw "{ \"deploymentId\": \"${latest_id}\", \"name\": \"${VERCEL_APP}\", \"target\": \"production\" }"; then + echo "DEPLOYMENT FAILED! Please check the Vercel dashboard for more information." + echo "https://vercel.com/codercom/registry/deployments" + exit 1 fi } @@ -162,8 +166,6 @@ if (( status == 0 )); then echo "All modules are operational." # set to update_component_status "OPERATIONAL" - - echo "LATEST_REDEPLOY_FAILED=0" >> "${GITHUB_ENV}" else echo "The following modules have issues: ${failures[*]}" # check if all modules are down @@ -179,18 +181,7 @@ else # If a module is down, force a reployment to try getting things back online # ASAP - redeploy_result=$(force_redeploy_registry) - if (( redeploy_result == 0 )); then - echo "Reployment successful" - else - echo "Unable to redeploy automatically" - fi - - # Update environment variable so that if automatic re-deployment fails, we - # don't keep running the script over and over again. Note that even if a - # re-deployment succeeds, that doesn't necessarily mean that everything is - # fully operational - echo "LATEST_REDEPLOY_FAILED=1" >> "${GITHUB_ENV}" + force_redeploy_registry fi exit "${status}" From 53968e41498b6c8d8adcb388aec5b280f4f42597 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 22 Nov 2024 23:40:14 +0000 Subject: [PATCH 13/14] removed --- .github/scripts/check.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/scripts/check.sh b/.github/scripts/check.sh index fcecc8de..14f6f2a2 100755 --- a/.github/scripts/check.sh +++ b/.github/scripts/check.sh @@ -23,12 +23,6 @@ for var in "${required_vars[@]}"; do fi done -LATEST_REDEPLOY_FAILED="${LATEST_REDEPLOY_FAILED:-0}" -if [[ "${LATEST_REDEPLOY_FAILED}" -ne "0" ]] ;then - echo "Trying to re-run job when previous re-deploy failed" - return 1 -fi - REGISTRY_BASE_URL="${REGISTRY_BASE_URL:-https://registry.coder.com}" status=0 From 91d40577e3e3dc4928c2a9296f8af822272dd68f Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 22 Nov 2024 23:43:35 +0000 Subject: [PATCH 14/14] wordsmithing --- .github/scripts/check.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/scripts/check.sh b/.github/scripts/check.sh index 14f6f2a2..5c1e83d4 100755 --- a/.github/scripts/check.sh +++ b/.github/scripts/check.sh @@ -100,6 +100,8 @@ force_redeploy_registry () { latest_id=$(echo "${latest_res}" | jq -r '.deployments[0].uid') if [[ "${latest_id}" = "null" ]]; then echo "Unable to pull any previous deployments for redeployment" + echo "Please redeploy the latest deployment manually in Vercel." + echo "https://vercel.com/codercom/registry/deployments" exit 1 fi @@ -109,7 +111,9 @@ force_redeploy_registry () { current_date_ts_seconds="$(date +%s)" local max_redeploy_interval_seconds=7200 # 2 hours if (( current_date_ts_seconds - latest_date_ts_seconds < max_redeploy_interval_seconds )); then - echo "Last deployment was less than 2 hours ago. Skipping redeployment." + echo "The registry was deployed less than 2 hours ago." + echo "Not automatically re-deploying the regitstry." + echo "A human reading this message should decide if a redeployment is necessary." echo "Please check the Vercel dashboard for more information." echo "https://vercel.com/codercom/registry/deployments" exit 1 @@ -119,6 +123,7 @@ force_redeploy_registry () { latest_deployment_state="$(echo "${latest_res}" | jq -r '.deployments[0].state')" if [[ "${latest_deployment_state}" != "READY" ]]; then echo "Last deployment was not in READY state. Skipping redeployment." + echo "A human reading this message should decide if a redeployment is necessary." echo "Please check the Vercel dashboard for more information." echo "https://vercel.com/codercom/registry/deployments" exit 1