From bc06a6f902b03b0d21e51f290257da8994661de4 Mon Sep 17 00:00:00 2001 From: deansheather Date: Tue, 7 Jun 2022 21:15:47 +0000 Subject: [PATCH 01/36] chore: write build_go.sh script and dependencies --- scripts/build_go.sh | 91 +++++++++++++++++++++++++++++++++++++++++++++ scripts/lib.sh | 33 ++++++++++++++++ scripts/version.sh | 31 +++++++++++++++ 3 files changed, 155 insertions(+) create mode 100755 scripts/build_go.sh create mode 100644 scripts/lib.sh create mode 100755 scripts/version.sh diff --git a/scripts/build_go.sh b/scripts/build_go.sh new file mode 100755 index 0000000000000..d0c95627db219 --- /dev/null +++ b/scripts/build_go.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash + +# This script builds a single Go binary of Coder with the given parameters. +# +# Usage: ./build_go.sh [--version v1.2.3+devel.abcdef] [--os linux] [--arch amd64] [--output path/to/output] [--slim] +# +# Defaults to linux:amd64 with slim disabled, but can be controlled with GOOS, +# GOARCH and CODER_SLIM_BUILD=1. If no version is specified, defaults to the +# version from ./version.sh. +# +# Unless overridden via --output, the built binary will be dropped in +# "$repo_root/dist/coder(-slim)?_$version_$os_$arch" (with a ".exe" suffix for +# windows builds) and the absolute path to the binary will be printed to stdout +# on completion. + +set -euo pipefail +# shellcheck source=lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +version="" +os="${GOOS:-linux}" +arch="${GOARCH:-amd64}" +slim="${CODER_SLIM_BUILD:-0}" +output_path="" + +args="$(getopt -o "" -l version:,os:,arch:,output:,slim -- "$@")" +eval set -- "$args" +while true; do + case "$1" in + --version) + version="$2" + shift 2 + ;; + --os) + os="$2" + shift 2 + ;; + --arch) + arch="$2" + shift 2 + ;; + --output) + output_path="$(realpath "$2")" + shift 2 + ;; + --slim) + slim=1 + shift + ;; + --) + shift + break + ;; + *) + error "Unrecognized option: $1" + ;; + esac +done + +if [[ "$version" == "" ]]; then + cdself + version="$(./version.sh)" +fi + +build_args=( + -ldflags "-s -w -X 'github.com/coder/coder/buildinfo.tag=$version'" +) +if [[ "$slim" == 0 ]]; then + build_args+=(-tags embed) +fi + +# cd to the root of the repo. +cdroot + +# Compute default output path. +if [[ "$output_path" == "" ]]; then + dist_dir="dist" + mkdir -p "$dist_dir" + output_path="${dist_dir}/coder_${version}_${os}_${arch}" + if [[ "$os" == "windows" ]]; then + output_path+=".exe" + fi + output_path="$(realpath "$output_path")" +fi +build_args+=(-o "$output_path") + +CGO_ENABLED=0 GOOS="$os" GOARCH="$arch" go build \ + "${build_args[@]}" \ + ./cmd/coder 1>&2 + +echo -n "$output_path" diff --git a/scripts/lib.sh b/scripts/lib.sh new file mode 100644 index 0000000000000..cfb9673654ed2 --- /dev/null +++ b/scripts/lib.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# This script is meant to be sourced by other scripts. To source this script: +# source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +set -euo pipefail + +SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") +PROJECT_ROOT=$(cd "$SCRIPT_DIR" && git rev-parse --show-toplevel) + +cdself() { + cd "$SCRIPT_DIR" || error "Could not change directory to '$SCRIPT_DIR'" +} + +cdroot() { + cd "$PROJECT_ROOT" || error "Could not change directory to '$PROJECT_ROOT'" +} + +# Taken from https://stackoverflow.com/a/3915420 (CC-BY-SA 4.0) +# Fails if the directory doesn't exist. +realpath() { + dir="$(dirname "$1")" + base="$(basename "$1")" + echo "$( + cd "$dir" || error "Could not change directory to '$dir'" + pwd -P + )"/"$base" +} + +error() { + echo "ERROR: $*" 1>&2 + exit 1 +} diff --git a/scripts/version.sh b/scripts/version.sh new file mode 100755 index 0000000000000..2ee0a2ee1442d --- /dev/null +++ b/scripts/version.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# This script generates the version string used by Coder, including for dev +# versions. Note: the version returned by this script will NOT include the "v" +# prefix that is included in the Git tag. + +set -euo pipefail +# shellcheck source=lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +cdroot + +# $current will equal $last_tag if we're currently checked out on the tag's +# commit. +last_tag="$(git describe --tags --abbrev=0)" +current="$(git describe --always)" + +version="$last_tag" + +# If the HEAD has extra commits since the last tag then we are in a dev version. +# +# Dev versions are denoted by the "+dev." suffix with a trailing commit short +# SHA. +if [[ "$last_tag" != "$current" ]]; then + if [[ "${CODER_NO_DEV_VERSION:-}" != "" ]]; then + error "version.sh attemped to generate a dev version string when CODER_NO_DEV_VERSION was set" + fi + version+="+dev.$(git rev-parse --short HEAD)" +fi + +# Remove the "v" prefix. +echo -n "${version#v}" From 850bc822babb368dc1360a9dd19b58cea914b7a9 Mon Sep 17 00:00:00 2001 From: deansheather Date: Wed, 8 Jun 2022 18:44:35 +0000 Subject: [PATCH 02/36] chore: write build_go_matrix.sh and archive.sh --- scripts/archive.sh | 104 ++++++++++++++++ scripts/build_go.sh | 33 +++-- scripts/build_go_matrix.sh | 145 ++++++++++++++++++++++ scripts/lib.sh | 34 ++++- scripts/{sign_macos.sh => sign_darwin.sh} | 0 5 files changed, 306 insertions(+), 10 deletions(-) create mode 100644 scripts/archive.sh create mode 100755 scripts/build_go_matrix.sh rename scripts/{sign_macos.sh => sign_darwin.sh} (100%) diff --git a/scripts/archive.sh b/scripts/archive.sh new file mode 100644 index 0000000000000..f18d50a4f06e9 --- /dev/null +++ b/scripts/archive.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash + +# This script creates an archive containing the given binary, as well as the +# README.md and LICENSE files. +# +# Usage: ./archive.sh --format tar.gz [--output path/to/output.tar.gz] [--sign-darwin] path/to/binary +# +# The --format parameter must be set, and must either be "zip" or "tar.gz". +# +# If the --output parameter is not set, the default output path is the binary +# path (minus any .exe suffix) plus the format extension ".zip" or ".tar.gz". +# +# If --sign-darwin is specified, the zip file is signed with the `codesign` +# utility and then notarized using the `gon` utility, which may take a while. +# $AC_APPLICATION_IDENTITY must be set and the signing certificate must be +# imported for this to work. Also, the input binary must already be signed with +# the `codesign` tool. + +set -euo pipefail +# shellcheck source=lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +format="" +output_path="" +sign_darwin=0 + +args="$(getopt -o "" -l version:,output:,slim,sign-darwin -- "$@")" +eval set -- "$args" +while true; do + case "$1" in + --format) + format="${2#.}" + if [[ "$format" != "zip" ]] && [[ "$format" != "tar.gz" ]]; then + error "Invalid --format parameter '$format', must be 'zip' or 'tar.gz'" + fi + shift 2 + ;; + --output) + # realpath fails if the dir doesn't exist. + mkdir -p "$(dirname "$2")" + output_path="$(realpath "$2")" + shift 2 + ;; + --sign-darwin) + if [[ "${AC_APPLICATION_IDENTITY:-}" == "" ]]; then + error "AC_APPLICATION_IDENTITY must be set when --sign-darwin is supplied" + fi + sign_darwin=1 + shift + ;; + --) + shift + break + ;; + *) + error "Unrecognized option: $1" + ;; + esac +done + +if [[ "$#" != 1 ]]; then + error "Exactly one argument must be provided to this script" +fi +if [[ ! -f "$1" ]]; then + error "File '$1' does not exist or is not a regular file" +fi +input_file="$(realpath "$1")" + +# Determine default output path. +if [[ "$output_path" == "" ]]; then + output_path="${input_file%.exe}" + output_path+=".$format" +fi + +# Determine the filename of the binary inside the archive. +output_file="coder" +if [[ "$input_file" == *".exe" ]]; then + output_file+=".exe" +fi + +# Make temporary dir where all source files intended to be in the archive will +# be symlinked from. +cdroot +temp_dir="$(mktemp -d)" +ln -s "$input_file" "$temp_dir/$output_path" +ln -s README.md "$temp_dir/" +ln -s LICENSE "$temp_dir/" + +# Ensure parent output dir. +mkdir -p "$(dirname "$output_path")" + +cd "$temp_dir" +if [[ "$format" == "zip" ]]; then + zip "$output_path" ./* +else + tar -czvf "$output_path" ./* +fi + +rm -rf "$temp_dir" + +if [[ "$sign_darwin" == 1 ]]; then + echo "Notarizing binary..." + execrelative ./sign_darwin.sh "$output_path" +fi diff --git a/scripts/build_go.sh b/scripts/build_go.sh index d0c95627db219..a263b656f296f 100755 --- a/scripts/build_go.sh +++ b/scripts/build_go.sh @@ -2,25 +2,31 @@ # This script builds a single Go binary of Coder with the given parameters. # -# Usage: ./build_go.sh [--version v1.2.3+devel.abcdef] [--os linux] [--arch amd64] [--output path/to/output] [--slim] +# Usage: ./build_go.sh [--version 1.2.3+devel.abcdef] [--os linux] [--arch amd64] [--output path/to/output] [--slim] # # Defaults to linux:amd64 with slim disabled, but can be controlled with GOOS, # GOARCH and CODER_SLIM_BUILD=1. If no version is specified, defaults to the # version from ./version.sh. # # Unless overridden via --output, the built binary will be dropped in -# "$repo_root/dist/coder(-slim)?_$version_$os_$arch" (with a ".exe" suffix for -# windows builds) and the absolute path to the binary will be printed to stdout -# on completion. +# "$repo_root/dist/coder_$version_$os_$arch" (with a ".exe" suffix for windows +# builds) and the absolute path to the binary will be printed to stdout on +# completion. +# +# If the --sign-darwin parameter is specified and the OS is darwin, binaries +# will be signed using the `codesign` utility. $AC_APPLICATION_IDENTITY must be +# set and the signing certificate must be imported for this to work. set -euo pipefail # shellcheck source=lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +cdroot version="" os="${GOOS:-linux}" arch="${GOARCH:-amd64}" slim="${CODER_SLIM_BUILD:-0}" +sign_darwin=0 output_path="" args="$(getopt -o "" -l version:,os:,arch:,output:,slim -- "$@")" @@ -47,6 +53,13 @@ while true; do slim=1 shift ;; + --sign-darwin) + if [[ "${AC_APPLICATION_IDENTITY:-}" == "" ]]; then + error "AC_APPLICATION_IDENTITY must be set when --sign-darwin is supplied" + fi + sign_darwin=1 + shift + ;; --) shift break @@ -57,9 +70,10 @@ while true; do esac done +# Remove the "v" prefix. +version="${version#v}" if [[ "$version" == "" ]]; then - cdself - version="$(./version.sh)" + version="$(execrelative ./version.sh)" fi build_args=( @@ -69,9 +83,6 @@ if [[ "$slim" == 0 ]]; then build_args+=(-tags embed) fi -# cd to the root of the repo. -cdroot - # Compute default output path. if [[ "$output_path" == "" ]]; then dist_dir="dist" @@ -88,4 +99,8 @@ CGO_ENABLED=0 GOOS="$os" GOARCH="$arch" go build \ "${build_args[@]}" \ ./cmd/coder 1>&2 +if [[ "$GOOS" == "darwin" ]] && [[ "$sign_darwin" == 1 ]]; then + codesign -s "$AC_APPLICATION_IDENTITY" -f -v --timestamp --options runtime "$output_path" +fi + echo -n "$output_path" diff --git a/scripts/build_go_matrix.sh b/scripts/build_go_matrix.sh new file mode 100755 index 0000000000000..741ca6d6382e1 --- /dev/null +++ b/scripts/build_go_matrix.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env bash + +# This script builds multiple Go binaries for Coder with the given OS and +# architecture combinations. +# +# Usage: ./build_go_matrix.sh [--version 1.2.3+devel.abcdef] [--output dist/] [--slim] [--sign-darwin] os1:arch1,arch2 os2:arch1 os1:arch3 +# +# If no OS:arch combinations are provided, nothing will happen and no error will +# be returned. Slim builds are disabled by default. If no version is specified, +# defaults to the version from ./version.sh +# +# The --output parameter must be a directory with a trailing slash where all +# files will be dropped with the default name scheme +# `coder_$version_$os_$arch(.exe)?`, or must contain the `{os}` and `{arch}` +# template variables. You may also use `{version}`. Note that for windows builds +# the `.exe` suffix will be appended automatically. +# +# Unless overridden via --output, the built binary will be dropped in +# "$repo_root/dist/coder_$version_$os_$arch" (with a ".exe" suffix for windows +# builds). +# +# If the --sign-darwin parameter is specified, all darwin binaries will be +# signed using the `codesign` utility. $AC_APPLICATION_IDENTITY must be set and +# the signing certificate must be imported for this to work. + +set -euo pipefail +# shellcheck source=lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +version="" +output_path="" +slim=0 +sign_darwin=0 + +args="$(getopt -o "" -l version:,output:,slim,sign-darwin -- "$@")" +eval set -- "$args" +while true; do + case "$1" in + --version) + version="$2" + shift 2 + ;; + --output) + # realpath fails if the dir doesn't exist. + mkdir -p "$(dirname "$2")" + output_path="$(realpath "$2")" + shift 2 + ;; + --slim) + slim=1 + shift + ;; + --sign-darwin) + if [[ "${AC_APPLICATION_IDENTITY:-}" == "" ]]; then + error "AC_APPLICATION_IDENTITY must be set when --sign-darwin is supplied" + fi + sign_darwin=1 + shift + ;; + --) + shift + break + ;; + *) + error "Unrecognized option: $1" + ;; + esac +done + +# Verify the output path template. +if [[ "$output_path" == "" ]]; then + # Input paths are relative, so we don't cdroot at the top, but for this case + # we want it to be relative to the root. + cdroot + output_path="dist/coder_{version}_{os}_{arch}" +elif [[ "$output_path" == */ ]]; then + output_path="${output_path}coder_{version_{os}_{arch}" +else + # Verify that it contains {os} and {arch} at least. + if [[ "$output_path" != *"{os}"* ]] || [[ "$output_path" != *"{arch}"* ]]; then + error "Templated output path '$output_path' must contain {os} and {arch}" + fi +fi + +# Remove the "v" prefix. +version="${version#v}" +if [[ "$version" == "" ]]; then + version="$(execrelative ./version.sh)" +fi + +# Parse the os:arch specs into an array. +specs=() +for spec in "$@"; do + spec_os="$(echo "$spec" | cut -d ":" -f 1)" + if [[ "$spec_os" == "" ]] || [[ "$spec_os" == *" "* ]]; then + error "Could not parse matrix build spec '$spec': invalid OS '$spec_os'" + fi + + # No quoting is important here. + for spec_arch in $(echo "$spec" | cut -d ":" -f 2 | tr "," "\n"); do + if [[ "$spec_arch" == "" ]] || [[ "$spec_os" == *" "* ]]; then + error "Could not parse matrix build spec '$spec': invalid architecture '$spec_arch'" + fi + + specs+=("$spec_os $spec_arch") + done +done + +build_args=() +if [[ "$slim" == 1 ]]; then + build_args+=(--slim) +fi +if [[ "$sign_darwin" == 1 ]]; then + build_args+=(--sign-darwin) +fi + +# Build each spec. +for spec in "${specs[@]}"; do + spec_os="$(echo "$spec" | cut -d " " -f 1)" + spec_arch="$(echo "$spec" | cut -d " " -f 2)" + + # Craft output path from the template. + spec_output="$output_path" + spec_output="${spec_output//\{os\}/"$spec_os"}" + spec_output="${spec_output//\{arch\}/"$spec_arch"}" + spec_output="${spec_output//\{version\}/"$version"}" + + spec_output_binary="$spec_output" + if [[ "$spec_os" == "windows" ]]; then + spec_output_binary+=".exe" + fi + + # Ensure parent dir. + mkdir -p "$(dirname "$spec_output")" + + echo "--- Building coder for $spec_os $spec_arch ($spec_output_binary)" + execrelative ./build_go.sh \ + --version "$version" \ + --os "$spec_os" \ + --arch "$spec_arch" \ + --output "$spec_output_binary" \ + "${build_args[@]}" + echo + echo +done diff --git a/scripts/lib.sh b/scripts/lib.sh index cfb9673654ed2..8a3c14d8979f8 100644 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -8,19 +8,51 @@ set -euo pipefail SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") PROJECT_ROOT=$(cd "$SCRIPT_DIR" && git rev-parse --show-toplevel) +# pushd is a silent alternative to the real pushd shell command. +pushd() { + command pushd "$@" >/dev/null +} + +# popd is a silent alternative to the real popd shell command. +# shellcheck disable=SC2120 +popd() { + command popd "$@" >/dev/null +} + +# cdself changes directory to the directory of the current script. This should +# not be used in scripts that may be sourced by other scripts. cdself() { cd "$SCRIPT_DIR" || error "Could not change directory to '$SCRIPT_DIR'" } +# cdroot changes directory to the root of the repository. cdroot() { cd "$PROJECT_ROOT" || error "Could not change directory to '$PROJECT_ROOT'" } +# execrelative can be used to execute scripts as if you were in the parent +# directory of the current script. This should not be used in scripts that may +# be sourced by other scripts. +execrelative() { + pushd "$SCRIPT_DIR" || error "Could not change directory to '$SCRIPT_DIR'" + "$@" + popd +} + +# realpath returns an absolute path to the given relative path. It will fail if +# the parent directory of the path does not exist. Make sure you are in the +# expected directory before running this to avoid errors. +# +# GNU realpath relies on coreutils, which are not installed or the default on +# Macs out of the box, so we have this mostly working bash alternative instead. +# # Taken from https://stackoverflow.com/a/3915420 (CC-BY-SA 4.0) -# Fails if the directory doesn't exist. realpath() { dir="$(dirname "$1")" base="$(basename "$1")" + if [[ ! -d "$dir" ]]; then + error "Could not change directory to '$dir': directory does not exist" + fi echo "$( cd "$dir" || error "Could not change directory to '$dir'" pwd -P diff --git a/scripts/sign_macos.sh b/scripts/sign_darwin.sh similarity index 100% rename from scripts/sign_macos.sh rename to scripts/sign_darwin.sh From e847230b6fcd12e9c720b59185c6a0e4331034d5 Mon Sep 17 00:00:00 2001 From: deansheather Date: Wed, 8 Jun 2022 18:45:17 +0000 Subject: [PATCH 03/36] fixup! chore: write build_go_matrix.sh and archive.sh --- .github/.goreleaser-release-darwin.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/.goreleaser-release-darwin.yaml b/.github/.goreleaser-release-darwin.yaml index 1c6ea3f5d20d3..13825ece8aeeb 100644 --- a/.github/.goreleaser-release-darwin.yaml +++ b/.github/.goreleaser-release-darwin.yaml @@ -47,7 +47,7 @@ env: signs: - ids: [coder-darwin] artifacts: archive - cmd: ./scripts/sign_macos.sh + cmd: ./scripts/sign_darwin.sh args: ["${artifact}"] output: true From 4edb6490d81b4e0bb62ac9268e9cf7e494c8586f Mon Sep 17 00:00:00 2001 From: deansheather Date: Wed, 8 Jun 2022 19:08:14 +0000 Subject: [PATCH 04/36] fixup! chore: write build_go_matrix.sh and archive.sh --- scripts/archive.sh | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) mode change 100644 => 100755 scripts/archive.sh diff --git a/scripts/archive.sh b/scripts/archive.sh old mode 100644 new mode 100755 index f18d50a4f06e9..cb235321453fa --- a/scripts/archive.sh +++ b/scripts/archive.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash -# This script creates an archive containing the given binary, as well as the -# README.md and LICENSE files. +# This script creates an archive containing the given binary renamed to +# `coder(.exe)?`, as well as the README.md and LICENSE files from the repo root. # # Usage: ./archive.sh --format tar.gz [--output path/to/output.tar.gz] [--sign-darwin] path/to/binary # @@ -24,7 +24,7 @@ format="" output_path="" sign_darwin=0 -args="$(getopt -o "" -l version:,output:,slim,sign-darwin -- "$@")" +args="$(getopt -o "" -l format:,output:,sign-darwin -- "$@")" eval set -- "$args" while true; do case "$1" in @@ -58,8 +58,12 @@ while true; do esac done +if [[ "$format" == "" ]]; then + error "--format is a required parameter" +fi + if [[ "$#" != 1 ]]; then - error "Exactly one argument must be provided to this script" + error "Exactly one argument must be provided to this script, $# were supplied" fi if [[ ! -f "$1" ]]; then error "File '$1' does not exist or is not a regular file" @@ -82,18 +86,21 @@ fi # be symlinked from. cdroot temp_dir="$(mktemp -d)" -ln -s "$input_file" "$temp_dir/$output_path" -ln -s README.md "$temp_dir/" -ln -s LICENSE "$temp_dir/" +ln -s "$input_file" "$temp_dir/$output_file" +ln -s "$(realpath README.md)" "$temp_dir/" +ln -s "$(realpath LICENSE)" "$temp_dir/" -# Ensure parent output dir. +# Ensure parent output dir and non-existent output file. mkdir -p "$(dirname "$output_path")" +if [[ -e "$output_path" ]]; then + error "Output path '$output_path' already exists!" +fi cd "$temp_dir" if [[ "$format" == "zip" ]]; then zip "$output_path" ./* else - tar -czvf "$output_path" ./* + tar --dereference -czvf "$output_path" ./* fi rm -rf "$temp_dir" From cef0221055699d7e5d0eb9030efaadd07537c106 Mon Sep 17 00:00:00 2001 From: deansheather Date: Wed, 8 Jun 2022 21:27:02 +0000 Subject: [PATCH 05/36] chore: add scripts for packages --- scripts/archive.sh | 13 +++++-- scripts/build_go.sh | 4 +- scripts/build_go_matrix.sh | 64 +++++++++++++++++++++++++++--- scripts/lib.sh | 4 +- scripts/nfpm.yaml | 25 ++++++++++++ scripts/package.sh | 80 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 176 insertions(+), 14 deletions(-) create mode 100644 scripts/nfpm.yaml create mode 100755 scripts/package.sh diff --git a/scripts/archive.sh b/scripts/archive.sh index cb235321453fa..71acc913593e0 100755 --- a/scripts/archive.sh +++ b/scripts/archive.sh @@ -15,6 +15,8 @@ # $AC_APPLICATION_IDENTITY must be set and the signing certificate must be # imported for this to work. Also, the input binary must already be signed with # the `codesign` tool. +# +# The absolute output path is printed on success. set -euo pipefail # shellcheck source=lib.sh @@ -93,19 +95,22 @@ ln -s "$(realpath LICENSE)" "$temp_dir/" # Ensure parent output dir and non-existent output file. mkdir -p "$(dirname "$output_path")" if [[ -e "$output_path" ]]; then - error "Output path '$output_path' already exists!" + rm "$output_path" fi cd "$temp_dir" if [[ "$format" == "zip" ]]; then - zip "$output_path" ./* + zip "$output_path" ./* 1>&2 else - tar --dereference -czvf "$output_path" ./* + tar --dereference -czvf "$output_path" ./* 1>&2 fi +cdroot rm -rf "$temp_dir" if [[ "$sign_darwin" == 1 ]]; then - echo "Notarizing binary..." + echo "Notarizing archive..." execrelative ./sign_darwin.sh "$output_path" fi + +echo -n "$output_path" diff --git a/scripts/build_go.sh b/scripts/build_go.sh index a263b656f296f..86b4cbe9f51dd 100755 --- a/scripts/build_go.sh +++ b/scripts/build_go.sh @@ -29,7 +29,7 @@ slim="${CODER_SLIM_BUILD:-0}" sign_darwin=0 output_path="" -args="$(getopt -o "" -l version:,os:,arch:,output:,slim -- "$@")" +args="$(getopt -o "" -l version:,os:,arch:,output:,slim,sign-darwin -- "$@")" eval set -- "$args" while true; do case "$1" in @@ -99,7 +99,7 @@ CGO_ENABLED=0 GOOS="$os" GOARCH="$arch" go build \ "${build_args[@]}" \ ./cmd/coder 1>&2 -if [[ "$GOOS" == "darwin" ]] && [[ "$sign_darwin" == 1 ]]; then +if [[ "$sign_darwin" == 1 ]] && [[ "$os" == "darwin" ]]; then codesign -s "$AC_APPLICATION_IDENTITY" -f -v --timestamp --options runtime "$output_path" fi diff --git a/scripts/build_go_matrix.sh b/scripts/build_go_matrix.sh index 741ca6d6382e1..42aee56c8d0e1 100755 --- a/scripts/build_go_matrix.sh +++ b/scripts/build_go_matrix.sh @@ -3,7 +3,7 @@ # This script builds multiple Go binaries for Coder with the given OS and # architecture combinations. # -# Usage: ./build_go_matrix.sh [--version 1.2.3+devel.abcdef] [--output dist/] [--slim] [--sign-darwin] os1:arch1,arch2 os2:arch1 os1:arch3 +# Usage: ./build_go_matrix.sh [--version 1.2.3+devel.abcdef] [--output dist/] [--slim] [--sign-darwin] [--archive] [--package-linux] os1:arch1,arch2 os2:arch1 os1:arch3 # # If no OS:arch combinations are provided, nothing will happen and no error will # be returned. Slim builds are disabled by default. If no version is specified, @@ -22,6 +22,14 @@ # If the --sign-darwin parameter is specified, all darwin binaries will be # signed using the `codesign` utility. $AC_APPLICATION_IDENTITY must be set and # the signing certificate must be imported for this to work. +# +# If the --archive parameter is specified, all binaries will be archived using +# ./archive.sh. The --sign-darwin parameter will be carried through, and all +# archive files will be dropped in the output directory with the same name as +# the binary and the .zip (for windows and darwin) or .tar.gz extension. +# +# If the --package-linux parameter is specified, all linux binaries will be +# packaged using ./package.sh. Requires the nfpm binary. set -euo pipefail # shellcheck source=lib.sh @@ -31,8 +39,10 @@ version="" output_path="" slim=0 sign_darwin=0 +archive=0 +package_linux=0 -args="$(getopt -o "" -l version:,output:,slim,sign-darwin -- "$@")" +args="$(getopt -o "" -l version:,output:,slim,sign-darwin,archive,package-linux -- "$@")" eval set -- "$args" while true; do case "$1" in @@ -57,6 +67,14 @@ while true; do sign_darwin=1 shift ;; + --archive) + archive=1 + shift + ;; + --package-linux) + package_linux=1 + shift + ;; --) shift break @@ -72,7 +90,8 @@ if [[ "$output_path" == "" ]]; then # Input paths are relative, so we don't cdroot at the top, but for this case # we want it to be relative to the root. cdroot - output_path="dist/coder_{version}_{os}_{arch}" + mkdir -p dist + output_path="$(realpath "dist/coder_{version}_{os}_{arch}")" elif [[ "$output_path" == */ ]]; then output_path="${output_path}coder_{version_{os}_{arch}" else @@ -102,10 +121,13 @@ for spec in "$@"; do error "Could not parse matrix build spec '$spec': invalid architecture '$spec_arch'" fi - specs+=("$spec_os $spec_arch") + specs+=("$spec_os:$spec_arch") done done +# Remove duplicate specs while maintaining the same order. +mapfile -t specs < <(echo "${specs[@]}" | tr " " "\n" | awk '!a[$0]++') + build_args=() if [[ "$slim" == 1 ]]; then build_args+=(--slim) @@ -116,8 +138,8 @@ fi # Build each spec. for spec in "${specs[@]}"; do - spec_os="$(echo "$spec" | cut -d " " -f 1)" - spec_arch="$(echo "$spec" | cut -d " " -f 2)" + spec_os="$(echo "$spec" | cut -d ":" -f 1)" + spec_arch="$(echo "$spec" | cut -d ":" -f 2)" # Craft output path from the template. spec_output="$output_path" @@ -142,4 +164,34 @@ for spec in "${specs[@]}"; do "${build_args[@]}" echo echo + + if [[ "$archive" == 1 ]]; then + spec_archive_format="tar.gz" + if [[ "$spec_os" == "windows" ]] || [[ "$spec_os" == "darwin" ]]; then + spec_archive_format="zip" + fi + spec_output_archive="$spec_output.$spec_archive_format" + + archive_args=() + if [[ "$sign_darwin" == 1 ]] && [[ "$spec_os" == "darwin" ]]; then + archive_args+=(--sign-darwin) + fi + + echo "--- Creating archive for $spec_os $spec_arch ($spec_output_archive)" + execrelative ./archive.sh \ + --format "$spec_archive_format" \ + --output "$spec_output_archive" \ + "${archive_args[@]}" \ + "$spec_output_binary" + echo + echo + fi + + if [[ "$package_linux" == 1 ]] && [[ "$spec_os" == "linux" ]]; then + execrelative ./package.sh \ + --arch "$spec_arch" \ + --version "$version" \ + "$spec_output_binary" + echo + fi done diff --git a/scripts/lib.sh b/scripts/lib.sh index 8a3c14d8979f8..558799dbbfdb2 100644 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -5,8 +5,8 @@ set -euo pipefail -SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") -PROJECT_ROOT=$(cd "$SCRIPT_DIR" && git rev-parse --show-toplevel) +SCRIPT_DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" +PROJECT_ROOT="$(cd "$SCRIPT_DIR" && realpath "$(git rev-parse --show-toplevel)")" # pushd is a silent alternative to the real pushd shell command. pushd() { diff --git a/scripts/nfpm.yaml b/scripts/nfpm.yaml new file mode 100644 index 0000000000000..1058384c2ddf0 --- /dev/null +++ b/scripts/nfpm.yaml @@ -0,0 +1,25 @@ +name: coder +platform: linux +arch: "${GOARCH}" +version: "${CODER_VERSION}" +version_schema: semver +release: 1 + +vendor: Coder +homepage: https://coder.com +maintainer: Coder +description: | + Provision development environments with infrastructure with code +license: AGPL-3.0 + +suggests: + - postgresql + +contents: + - src: coder + dst: /usr/bin/coder + - src: coder.env + dst: /etc/coder.d/coder.env + type: "config|noreplace" + - src: coder.service + dst: /usr/lib/systemd/system/coder.service diff --git a/scripts/package.sh b/scripts/package.sh new file mode 100755 index 0000000000000..974fecb747bf2 --- /dev/null +++ b/scripts/package.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +# This script creates Linux packages for the given binary. It will output a +# .rpm, .deb and .apk file in the same directory as the input file with the same +# filename (except the package format suffix). +# +# ./package.sh --arch amd64 [--version 1.2.3] path/to/coder +# +# The --arch parameter is required. If no version is specified, defaults to the +# version from ./version.sh. + +set -euo pipefail +# shellcheck source=lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +version="" +arch="" + +args="$(getopt -o "" -l arch:,version: -- "$@")" +eval set -- "$args" +while true; do + case "$1" in + --arch) + arch="$2" + shift 2 + ;; + --version) + version="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + error "Unrecognized option: $1" + ;; + esac +done + +if [[ "$arch" == "" ]]; then + error "--arch is a required parameter" +fi + +if [[ "$#" != 1 ]]; then + error "Exactly one argument must be provided to this script, $# were supplied" +fi +if [[ ! -f "$1" ]]; then + error "File '$1' does not exist or is not a regular file" +fi +input_file="$(realpath "$1")" + +# Remove the "v" prefix. +version="${version#v}" +if [[ "$version" == "" ]]; then + version="$(execrelative ./version.sh)" +fi + +# Make temporary dir where all source files intended to be in the package will +# be hardlinked from. +cdroot +temp_dir="$(TMPDIR="$(dirname "$input_file")" mktemp -d)" +ln -P "$input_file" "$temp_dir/coder" +ln -P "$(realpath coder.env)" "$temp_dir/" +ln -P "$(realpath coder.service)" "$temp_dir/" +ln -P "$(realpath scripts/nfpm.yaml)" "$temp_dir/" + +cd "$temp_dir" + +formats=(apk deb rpm) +for format in "${formats[@]}"; do + output_path="$input_file.$format" + echo "--- Building $format package ($output_path)" + nfpm package \ + -f nfpm.yaml \ + -p "$format" \ + -t "$output_path" +done + +rm -rf "$temp_dir" From 64f9648f5de2dc110c156bb301b35cb8abc1df09 Mon Sep 17 00:00:00 2001 From: deansheather Date: Wed, 8 Jun 2022 22:27:14 +0000 Subject: [PATCH 06/36] chore: add scripts for building a docker image --- Dockerfile | 2 +- docker-compose.yaml | 2 +- scripts/build_docker.sh | 131 ++++++++++++++++++++++++++++++++++++++++ scripts/build_go.sh | 1 + scripts/lib.sh | 1 + scripts/package.sh | 1 + 6 files changed, 136 insertions(+), 2 deletions(-) create mode 100755 scripts/build_docker.sh diff --git a/Dockerfile b/Dockerfile index 8bbcbe848837f..76f244a65bd3f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM alpine -# Generated by goreleaser on `goreleaser release` +# The coder binary is injected by scripts/build_docker.sh. ADD coder /opt/coder ENTRYPOINT [ "/opt/coder", "server" ] diff --git a/docker-compose.yaml b/docker-compose.yaml index 74c8916fd0d21..d943ff185d617 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,7 +1,7 @@ version: "3.9" services: coder: - image: ghcr.io/coder/coder:v${CODER_VERSION:-0.5.10}-${ARCH:-amd64} + image: ghcr.io/coder/coder:v${CODER_VERSION:-0.5.10} ports: - "7080:7080" environment: diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh new file mode 100755 index 0000000000000..1ba159ad6a52a --- /dev/null +++ b/scripts/build_docker.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash + +# This script builds a Docker image of Coder containing the given binary, for +# the given architecture. Only linux binaries are supported at this time. +# +# Usage: ./build_docker.sh --arch amd64 --tags v1.2.4-rc.1-arm64,v1.2.3+devel.abcdef [--version 1.2.3] [--push] +# +# The --arch parameter is required and accepts a Golang arch specification. It +# will be automatically mapped to a suitable architecture that Docker accepts. +# +# The image will be built and tagged against all supplied tags. At least one tag +# must be supplied. All tags will be sanitized to remove invalid characters like +# plus signs. +# +# If no version is specified, defaults to the version from ./version.sh. +# +# If the --push parameter is supplied, all supplied tags will be pushed. + +set -euo pipefail +# shellcheck source=lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +image="ghcr.io/coder/coder" +arch="" +tags_str="" +version="" +push=0 + +args="$(getopt -o "" -l arch:,tags:,version:,push -- "$@")" +eval set -- "$args" +while true; do + case "$1" in + --arch) + arch="$2" + shift 2 + ;; + --tags) + tags_str="$2" + shift 2 + ;; + --version) + version="$2" + shift 2 + ;; + --push) + push=1 + shift + ;; + --) + shift + break + ;; + *) + error "Unrecognized option: $1" + ;; + esac +done + +if [[ "$arch" == "" ]]; then + error "The --arch parameter is required" +fi + +tags=() +for tag in $(echo "$tags_str" | tr "," "\n"); do + # Docker images don't support plus signs, which devel versions may contain. + tag="${tag//+/-}" + tags+=("$tag") +done +if [[ "${#tags[@]}" == 0 ]]; then + error "At least one tag must be supplied through --tags" +fi + +if [[ "$#" != 1 ]]; then + error "Exactly one argument must be provided to this script, $# were supplied" +fi +if [[ ! -f "$1" ]]; then + error "File '$1' does not exist or is not a regular file" +fi +input_file="$(realpath "$1")" + +# Remove the "v" prefix. +version="${version#v}" +if [[ "$version" == "" ]]; then + version="$(execrelative ./version.sh)" +fi + +# Remap the arch from Golang to Docker. +declare -A arch_map=( + [amd64]="linux/amd64" + [arm64]="linux/arm64" + [arm]="linux/arm/v7" +) +if [[ "${arch_map[$arch]+exists}" != "" ]]; then + arch="${arch_map[$arch]}" +fi + +# Make temporary dir where all source files intended to be in the image will be +# hardlinked from. +cdroot +temp_dir="$(TMPDIR="$(dirname "$input_file")" mktemp -d)" +ln -P "$input_file" "$temp_dir/coder" +ln -P Dockerfile "$temp_dir/" + +cd "$temp_dir" + +build_args=( + "--platform=$arch" + "--label=org.opencontainers.image.title=Coder" + "--label=org.opencontainers.image.description=A tool for provisioning self-hosted development environments with Terraform." + "--label=org.opencontainers.image.url=https://github.com/coder/coder" + "--label=org.opencontainers.image.source=https://github.com/coder/coder" + "--label=org.opencontainers.image.version=$version" + "--label=org.opencontainers.image.licenses=AGPL-3.0" +) +for tag in "${tags[@]}"; do + build_args+=(--tag "$image:$tag") +done + +echo "--- Building Docker image for $arch" +docker buildx build "${build_args[@]}" . + +cdroot +rm -rf "$temp_dir" + +if [[ "$push" == 1 ]]; then + echo "--- Pushing Docker images for $arch" + for tag in "${tags[@]}"; do + echo "Pushing $image:$tag" + docker push "$image:$tag" + done +fi diff --git a/scripts/build_go.sh b/scripts/build_go.sh index 86b4cbe9f51dd..1c4c414d6cf21 100755 --- a/scripts/build_go.sh +++ b/scripts/build_go.sh @@ -95,6 +95,7 @@ if [[ "$output_path" == "" ]]; then fi build_args+=(-o "$output_path") +# TODO: GOARM CGO_ENABLED=0 GOOS="$os" GOARCH="$arch" go build \ "${build_args[@]}" \ ./cmd/coder 1>&2 diff --git a/scripts/lib.sh b/scripts/lib.sh index 558799dbbfdb2..35ead677adcea 100644 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -59,6 +59,7 @@ realpath() { )"/"$base" } +# error prints an error message and returns an error exit code. error() { echo "ERROR: $*" 1>&2 exit 1 diff --git a/scripts/package.sh b/scripts/package.sh index 974fecb747bf2..1bbb2e2b0a8f8 100755 --- a/scripts/package.sh +++ b/scripts/package.sh @@ -77,4 +77,5 @@ for format in "${formats[@]}"; do -t "$output_path" done +cdroot rm -rf "$temp_dir" From 998bc31bce9f44bf5ca66b9eff8fcb2b6acfaefe Mon Sep 17 00:00:00 2001 From: deansheather Date: Thu, 9 Jun 2022 20:35:36 +0000 Subject: [PATCH 07/36] chore: add docker multi-arch script and release script --- scripts/archive.sh | 2 +- scripts/build_docker.sh | 59 ++++++--------- scripts/build_docker_multiarch.sh | 73 ++++++++++++++++++ scripts/build_go_matrix.sh | 14 ++-- scripts/image_tag.sh | 58 +++++++++++++++ scripts/lib.sh | 26 ++++++- scripts/package.sh | 2 +- scripts/publish_release.sh | 119 ++++++++++++++++++++++++++++++ scripts/sign_darwin.sh | 43 +++++++---- scripts/version.sh | 3 +- 10 files changed, 337 insertions(+), 62 deletions(-) create mode 100755 scripts/build_docker_multiarch.sh create mode 100755 scripts/image_tag.sh create mode 100755 scripts/publish_release.sh diff --git a/scripts/archive.sh b/scripts/archive.sh index 71acc913593e0..095d102ffd62b 100755 --- a/scripts/archive.sh +++ b/scripts/archive.sh @@ -109,7 +109,7 @@ cdroot rm -rf "$temp_dir" if [[ "$sign_darwin" == 1 ]]; then - echo "Notarizing archive..." + log "Notarizing archive..." execrelative ./sign_darwin.sh "$output_path" fi diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh index 1ba159ad6a52a..f0e6f9e958d59 100755 --- a/scripts/build_docker.sh +++ b/scripts/build_docker.sh @@ -3,30 +3,30 @@ # This script builds a Docker image of Coder containing the given binary, for # the given architecture. Only linux binaries are supported at this time. # -# Usage: ./build_docker.sh --arch amd64 --tags v1.2.4-rc.1-arm64,v1.2.3+devel.abcdef [--version 1.2.3] [--push] +# Usage: ./build_docker.sh --arch amd64 [--version 1.2.3] [--push] # # The --arch parameter is required and accepts a Golang arch specification. It -# will be automatically mapped to a suitable architecture that Docker accepts. +# will be automatically mapped to a suitable architecture that Docker accepts +# before being passed to `docker buildx build`. # -# The image will be built and tagged against all supplied tags. At least one tag -# must be supplied. All tags will be sanitized to remove invalid characters like -# plus signs. +# The image will be built and tagged against the image tag returned by +# ./image_tag.sh. # # If no version is specified, defaults to the version from ./version.sh. # -# If the --push parameter is supplied, all supplied tags will be pushed. +# If the --push parameter is supplied, the image will be pushed. +# +# Prints the image tag on success. set -euo pipefail # shellcheck source=lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -image="ghcr.io/coder/coder" arch="" -tags_str="" version="" push=0 -args="$(getopt -o "" -l arch:,tags:,version:,push -- "$@")" +args="$(getopt -o "" -l arch:,version:,push -- "$@")" eval set -- "$args" while true; do case "$1" in @@ -34,10 +34,6 @@ while true; do arch="$2" shift 2 ;; - --tags) - tags_str="$2" - shift 2 - ;; --version) version="$2" shift 2 @@ -60,16 +56,14 @@ if [[ "$arch" == "" ]]; then error "The --arch parameter is required" fi -tags=() -for tag in $(echo "$tags_str" | tr "," "\n"); do - # Docker images don't support plus signs, which devel versions may contain. - tag="${tag//+/-}" - tags+=("$tag") -done -if [[ "${#tags[@]}" == 0 ]]; then - error "At least one tag must be supplied through --tags" +# Remove the "v" prefix. +version="${version#v}" +if [[ "$version" == "" ]]; then + version="$(execrelative ./version.sh)" fi +image_tag="$(execrelative ./image_tag.sh --arch "$arch" --version="$version")" + if [[ "$#" != 1 ]]; then error "Exactly one argument must be provided to this script, $# were supplied" fi @@ -78,12 +72,6 @@ if [[ ! -f "$1" ]]; then fi input_file="$(realpath "$1")" -# Remove the "v" prefix. -version="${version#v}" -if [[ "$version" == "" ]]; then - version="$(execrelative ./version.sh)" -fi - # Remap the arch from Golang to Docker. declare -A arch_map=( [amd64]="linux/amd64" @@ -111,21 +99,18 @@ build_args=( "--label=org.opencontainers.image.source=https://github.com/coder/coder" "--label=org.opencontainers.image.version=$version" "--label=org.opencontainers.image.licenses=AGPL-3.0" + "--tag=$image_tag" ) -for tag in "${tags[@]}"; do - build_args+=(--tag "$image:$tag") -done -echo "--- Building Docker image for $arch" -docker buildx build "${build_args[@]}" . +log "--- Building Docker image for $arch ($image_tag)" +docker buildx build "${build_args[@]}" . 1>&2 cdroot rm -rf "$temp_dir" if [[ "$push" == 1 ]]; then - echo "--- Pushing Docker images for $arch" - for tag in "${tags[@]}"; do - echo "Pushing $image:$tag" - docker push "$image:$tag" - done + log "--- Pushing Docker image for $arch ($image_tag)" + docker push "$image_tag" fi + +echo -n "$image_tag" diff --git a/scripts/build_docker_multiarch.sh b/scripts/build_docker_multiarch.sh new file mode 100755 index 0000000000000..9765e48c8a56d --- /dev/null +++ b/scripts/build_docker_multiarch.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash + +# This script merges Coder Docker images of different architectures together +# into the archless image tag returned by ./image_tag.sh. +# +# Usage: ./build_docker_multiarch.sh [--version 1.2.3] [--push] image1:tag1 image2:tag2 +# +# The supplied images must already be pushed to the registry or this will fail. +# Also, the source images cannot be in a different registry than the target +# image generated by ./image_tag.sh. +# +# If no version is specified, defaults to the version from ./version.sh. +# +# If the --push parameter is supplied, all supplied tags will be pushed. +# +# Returns the merged image tag. + +set -euo pipefail +# shellcheck source=lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +version="" +push=0 + +args="$(getopt -o "" -l version:,push -- "$@")" +eval set -- "$args" +while true; do + case "$1" in + --version) + version="$2" + shift 2 + ;; + --push) + push=1 + shift + ;; + --) + shift + break + ;; + *) + error "Unrecognized option: $1" + ;; + esac +done + +# Remove the "v" prefix. +version="${version#v}" +if [[ "$version" == "" ]]; then + version="$(execrelative ./version.sh)" +fi + +if [[ "$#" == 0 ]]; then + error "At least one argument must be provided to this script, $# were supplied" +fi + +create_args=() +for image_tag in "$@"; do + create_args+=(--amend "$image_tag") +done + +output_tag="$(execrelative ./image_tag.sh --version "$version")" +log "--- Creating multi-arch Docker image ($output_tag)" +docker manifest create \ + "$output_tag" \ + "${create_args[@]}" + +if [[ "$push" == 1 ]]; then + log "--- Pushing multi-arch Docker image ($output_tag)" + docker push "$output_tag" +fi + +echo -n "$output_tag" diff --git a/scripts/build_go_matrix.sh b/scripts/build_go_matrix.sh index 42aee56c8d0e1..e7346f0a1da64 100755 --- a/scripts/build_go_matrix.sh +++ b/scripts/build_go_matrix.sh @@ -155,15 +155,15 @@ for spec in "${specs[@]}"; do # Ensure parent dir. mkdir -p "$(dirname "$spec_output")" - echo "--- Building coder for $spec_os $spec_arch ($spec_output_binary)" + log "--- Building coder for $spec_os $spec_arch ($spec_output_binary)" execrelative ./build_go.sh \ --version "$version" \ --os "$spec_os" \ --arch "$spec_arch" \ --output "$spec_output_binary" \ "${build_args[@]}" - echo - echo + log + log if [[ "$archive" == 1 ]]; then spec_archive_format="tar.gz" @@ -177,14 +177,14 @@ for spec in "${specs[@]}"; do archive_args+=(--sign-darwin) fi - echo "--- Creating archive for $spec_os $spec_arch ($spec_output_archive)" + log "--- Creating archive for $spec_os $spec_arch ($spec_output_archive)" execrelative ./archive.sh \ --format "$spec_archive_format" \ --output "$spec_output_archive" \ "${archive_args[@]}" \ "$spec_output_binary" - echo - echo + log + log fi if [[ "$package_linux" == 1 ]] && [[ "$spec_os" == "linux" ]]; then @@ -192,6 +192,6 @@ for spec in "${specs[@]}"; do --arch "$spec_arch" \ --version "$version" \ "$spec_output_binary" - echo + log fi done diff --git a/scripts/image_tag.sh b/scripts/image_tag.sh new file mode 100755 index 0000000000000..bc2608bed72e2 --- /dev/null +++ b/scripts/image_tag.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +# This script prints the image tag to use for the given arch and version +# combination. +# +# Usage: ./image_tag.sh [--arch amd64] [--version 1.2.3] +# +# The --arch parameter accepts a Golang arch specification. If not specified, +# the image tag for the multi-arch image will be returned instead. +# +# If no version is specified, defaults to the version from ./version.sh. +# +# The returned tag will be sanitized to remove invalid characters like the plus +# sign. + +set -euo pipefail +# shellcheck source=lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +arch="" +version="" + +args="$(getopt -o "" -l arch:,version: -- "$@")" +eval set -- "$args" +while true; do + case "$1" in + --arch) + arch="$2" + shift 2 + ;; + --version) + version="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + error "Unrecognized option: $1" + ;; + esac +done + +# Remove the "v" prefix. +version="${version#v}" +if [[ "$version" == "" ]]; then + version="$(execrelative ./version.sh)" +fi + +image="${CODER_IMAGE_BASE:-ghcr.io/coder/coder}" +tag="v$version" +if [[ "$arch" != "" ]]; then + tag+="-$arch" +fi + +tag="${tag//+/-}" +echo -n "$image:$tag" diff --git a/scripts/lib.sh b/scripts/lib.sh index 35ead677adcea..14f3c6d871a6c 100644 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -59,8 +59,32 @@ realpath() { )"/"$base" } +# maybedryrun prints the given program and flags, and then, if the first +# argument is 0, executes it. The reason the first argument should be 0 is that +# it is expected that you have a dry_run variable in your script that is set to +# 0 by default (i.e. do not dry run) and set to 1 if the --dry-run flag is +# specified. +# +# Usage: maybedryrun 1 gh release create ... +# Usage: maybedryrun 0 docker push ghcr.io/coder/coder:latest +maybedryrun() { + if [[ "$1" == 1 ]]; then + shift + log "DRYRUN: $*" + else + shift + log $ "$@" + "$@" + fi +} + +# log prints a message to stderr. +log() { + echo "$*" 1>&2 +} + # error prints an error message and returns an error exit code. error() { - echo "ERROR: $*" 1>&2 + log "ERROR: $*" exit 1 } diff --git a/scripts/package.sh b/scripts/package.sh index 1bbb2e2b0a8f8..d97ab52e3fa6e 100755 --- a/scripts/package.sh +++ b/scripts/package.sh @@ -70,7 +70,7 @@ cd "$temp_dir" formats=(apk deb rpm) for format in "${formats[@]}"; do output_path="$input_file.$format" - echo "--- Building $format package ($output_path)" + log "--- Building $format package ($output_path)" nfpm package \ -f nfpm.yaml \ -p "$format" \ diff --git a/scripts/publish_release.sh b/scripts/publish_release.sh new file mode 100755 index 0000000000000..2a18557a35cb3 --- /dev/null +++ b/scripts/publish_release.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash + +# This script generates release notes and publishes all of the given assets to +# GitHub releases. Depends on GitHub CLI. +# +# Usage: ./publish_release.sh [--version 1.2.3] [--dry-run] path/to/asset1 path/to/asset2 ... +# +# The supplied images must already be pushed to the registry or this will fail. +# Also, the source images cannot be in a different registry than the target +# image generated by ./image_tag.sh. +# The supplied assets will be uploaded to the GitHub release as-is, as well as a +# file containing checksums. +# +# If no version is specified, defaults to the version from ./version.sh. The +# script will exit early if the branch is not tagged with the provided version +# (plus the "v" prefix) unless run with --dry-run. +# +# If the --dry-run parameter is supplied, the release will not be published to +# GitHub at all. +# +# Returns the link to the created GitHub release (unless --dry-run was +# specified). + +set -euo pipefail +# shellcheck source=lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +version="" +dry_run=0 + +args="$(getopt -o "" -l version:,dry-run -- "$@")" +eval set -- "$args" +while true; do + case "$1" in + --version) + version="$2" + shift 2 + ;; + --dry-run) + dry_run=1 + shift + ;; + --) + shift + break + ;; + *) + error "Unrecognized option: $1" + ;; + esac +done + +# Remove the "v" prefix. +version="${version#v}" +if [[ "$version" == "" ]]; then + version="$(execrelative ./version.sh)" +fi + +# Verify that we're currently checked out on the supplied tag. +new_tag="v$version" +if [[ "$(git describe --always)" != "$new_tag" ]]; then + if [[ "$dry_run" == 0 ]]; then + error "The provided version '$new_tag' does not match the current git describe output '$(git describe --always)'" + fi + + log "The provided version does not match the current git tag, but --dry-run was supplied so continuing..." +fi + +# This returns the tag before the current tag. +old_tag="$(git describe --abbrev=0 --tags "$(git rev-list --tags --skip=1 --max-count=1)")" + +# Craft the release notes. +release_notes=" +## Changelog + +$(git log --no-merges --pretty=format:"- %h %s" "$old_tag..$new_tag") + +## Container Image +- \`docker pull $(execrelative ./image_tag.sh --version "$version")\` + +" + +# Create temporary release folder so we can generate checksums. +temp_dir="$(mktemp -d)" +for f in "$@"; do + ln -s "$(realpath "$f")" "$temp_dir/" +done + +# Generate checksums file. sha256sum seems to play nicely with symlinks so this +# works well. +pushd "$temp_dir" +sha256sum ./* | sed -e 's/\.\///' - >"coder_${version}_checksums.txt" +popd + +log "--- Creating release $new_tag" +log +log "Description:" +echo "$release_notes" | sed -e 's/^/\t/' - 1>&2 +log +log "Contents:" +pushd "$temp_dir" +find ./* 2>&1 | sed -e 's/^/\t/;s/\.\///' - 1>&2 +popd +log +log + +# We echo the release notes in instead of writing to a file and referencing that +# to prevent GitHub CLI from becoming interactive. +# +# GitHub CLI seems to follow symlinks when uploading files. +echo "$release_notes" | + maybedryrun "$dry_run" gh release create \ + -R deansheather/x \ + --title "$new_tag" \ + --notes-file - \ + "$new_tag" \ + "$temp_dir"/* + +rm -rf "$temp_dir" diff --git a/scripts/sign_darwin.sh b/scripts/sign_darwin.sh index 523649d7c5cf4..ed2474055bc50 100755 --- a/scripts/sign_darwin.sh +++ b/scripts/sign_darwin.sh @@ -1,15 +1,27 @@ #!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") -PROJECT_ROOT=$(cd "$SCRIPT_DIR" && git rev-parse --show-toplevel) +# This script notarizes the provided zip file. +# +# Usage: ./publish_release.sh [--version 1.2.3] [--dry-run] path/to/asset1 path/to/asset2 ... +# +# The provided zip file must contain a coder binary that has already been signed +# using the codesign tool. +# +# On success, the input file will be successfully signed and notarized. +# +# Depends on codesign and gon utilities. Requires the $AC_APPLICATION_IDENTITY +# environment variable to be set. -cd "${PROJECT_ROOT}" +set -euo pipefail +# shellcheck source=lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -codesign -s "$AC_APPLICATION_IDENTITY" -f -v --timestamp --options runtime "$1" +if [[ "${AC_APPLICATION_IDENTITY:-}" == "" ]]; then + error "AC_APPLICATION_IDENTITY must be set for ./sign_darwin.sh" +fi -config=$(mktemp -d)/gon.json +# Create the gon config. +config="$(mktemp -d)/gon.json" jq -r --null-input --arg path "$(pwd)/$1" '{ "notarize": [ { @@ -19,21 +31,26 @@ jq -r --null-input --arg path "$(pwd)/$1" '{ ] }' >"$config" +# Sign the zip file with our certificate. +codesign -s "$AC_APPLICATION_IDENTITY" -f -v --timestamp --options runtime "$1" + +# Notarize the signed zip file. +# # The notarization process is very fragile and heavily dependent on Apple's # notarization server not returning server errors, so we retry this step 5 # times with a delay of 30 seconds between each attempt. rc=0 for i in $(seq 1 5); do gon "$config" && rc=0 && break || rc=$? - echo "gon exit code: $rc" + log "gon exit code: $rc" if [ "$i" -lt 5 ]; then - echo - echo "Retrying notarization in 30 seconds" - echo + log + log "Retrying notarization in 30 seconds" + log sleep 30 else - echo - echo "Giving up :(" + log + log "Giving up :(" fi done diff --git a/scripts/version.sh b/scripts/version.sh index 2ee0a2ee1442d..c714e7bc2e2b0 100755 --- a/scripts/version.sh +++ b/scripts/version.sh @@ -9,8 +9,7 @@ set -euo pipefail source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" cdroot -# $current will equal $last_tag if we're currently checked out on the tag's -# commit. +# $current will equal $last_tag if we currently have the tag checked out. last_tag="$(git describe --tags --abbrev=0)" current="$(git describe --always)" From 46b5d79ad8f0920f23f6d1dad5ccccda441a8942 Mon Sep 17 00:00:00 2001 From: deansheather Date: Thu, 9 Jun 2022 20:35:58 +0000 Subject: [PATCH 08/36] fixup! chore: add docker multi-arch script and release script --- scripts/publish_release.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/publish_release.sh b/scripts/publish_release.sh index 2a18557a35cb3..a67dbb5b9e7c4 100755 --- a/scripts/publish_release.sh +++ b/scripts/publish_release.sh @@ -110,7 +110,6 @@ log # GitHub CLI seems to follow symlinks when uploading files. echo "$release_notes" | maybedryrun "$dry_run" gh release create \ - -R deansheather/x \ --title "$new_tag" \ --notes-file - \ "$new_tag" \ From 9e279a9cdaf9874aa43149606c1bc394e4e4509d Mon Sep 17 00:00:00 2001 From: deansheather Date: Thu, 9 Jun 2022 20:46:55 +0000 Subject: [PATCH 09/36] chore: fix indenting --- .vscode/extensions.json | 3 +- scripts/archive.sh | 82 +++++----- scripts/build_docker.sh | 80 +++++----- scripts/build_docker_multiarch.sh | 50 +++--- scripts/build_go.sh | 100 ++++++------ scripts/build_go_matrix.sh | 242 +++++++++++++++--------------- scripts/image_tag.sh | 40 ++--- scripts/lib.sh | 54 +++---- scripts/package.sh | 56 +++---- scripts/publish_release.sh | 58 +++---- scripts/sign_darwin.sh | 2 +- scripts/version.sh | 10 +- 12 files changed, 389 insertions(+), 388 deletions(-) diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 4384381d74265..ddecd5626f957 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -8,6 +8,7 @@ "zxh404.vscode-proto3", "redhat.vscode-yaml", "streetsidesoftware.code-spell-checker", - "dbaeumer.vscode-eslint" + "dbaeumer.vscode-eslint", + "EditorConfig.EditorConfig" ] } diff --git a/scripts/archive.sh b/scripts/archive.sh index 095d102ffd62b..7498ff750f57b 100755 --- a/scripts/archive.sh +++ b/scripts/archive.sh @@ -19,7 +19,7 @@ # The absolute output path is printed on success. set -euo pipefail -# shellcheck source=lib.sh +# shellcheck source=scripts/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" format="" @@ -29,59 +29,59 @@ sign_darwin=0 args="$(getopt -o "" -l format:,output:,sign-darwin -- "$@")" eval set -- "$args" while true; do - case "$1" in - --format) - format="${2#.}" - if [[ "$format" != "zip" ]] && [[ "$format" != "tar.gz" ]]; then - error "Invalid --format parameter '$format', must be 'zip' or 'tar.gz'" - fi - shift 2 - ;; - --output) - # realpath fails if the dir doesn't exist. - mkdir -p "$(dirname "$2")" - output_path="$(realpath "$2")" - shift 2 - ;; - --sign-darwin) - if [[ "${AC_APPLICATION_IDENTITY:-}" == "" ]]; then - error "AC_APPLICATION_IDENTITY must be set when --sign-darwin is supplied" - fi - sign_darwin=1 - shift - ;; - --) - shift - break - ;; - *) - error "Unrecognized option: $1" - ;; - esac + case "$1" in + --format) + format="${2#.}" + if [[ "$format" != "zip" ]] && [[ "$format" != "tar.gz" ]]; then + error "Invalid --format parameter '$format', must be 'zip' or 'tar.gz'" + fi + shift 2 + ;; + --output) + # realpath fails if the dir doesn't exist. + mkdir -p "$(dirname "$2")" + output_path="$(realpath "$2")" + shift 2 + ;; + --sign-darwin) + if [[ "${AC_APPLICATION_IDENTITY:-}" == "" ]]; then + error "AC_APPLICATION_IDENTITY must be set when --sign-darwin is supplied" + fi + sign_darwin=1 + shift + ;; + --) + shift + break + ;; + *) + error "Unrecognized option: $1" + ;; + esac done if [[ "$format" == "" ]]; then - error "--format is a required parameter" + error "--format is a required parameter" fi if [[ "$#" != 1 ]]; then - error "Exactly one argument must be provided to this script, $# were supplied" + error "Exactly one argument must be provided to this script, $# were supplied" fi if [[ ! -f "$1" ]]; then - error "File '$1' does not exist or is not a regular file" + error "File '$1' does not exist or is not a regular file" fi input_file="$(realpath "$1")" # Determine default output path. if [[ "$output_path" == "" ]]; then - output_path="${input_file%.exe}" - output_path+=".$format" + output_path="${input_file%.exe}" + output_path+=".$format" fi # Determine the filename of the binary inside the archive. output_file="coder" if [[ "$input_file" == *".exe" ]]; then - output_file+=".exe" + output_file+=".exe" fi # Make temporary dir where all source files intended to be in the archive will @@ -95,22 +95,22 @@ ln -s "$(realpath LICENSE)" "$temp_dir/" # Ensure parent output dir and non-existent output file. mkdir -p "$(dirname "$output_path")" if [[ -e "$output_path" ]]; then - rm "$output_path" + rm "$output_path" fi cd "$temp_dir" if [[ "$format" == "zip" ]]; then - zip "$output_path" ./* 1>&2 + zip "$output_path" ./* 1>&2 else - tar --dereference -czvf "$output_path" ./* 1>&2 + tar --dereference -czvf "$output_path" ./* 1>&2 fi cdroot rm -rf "$temp_dir" if [[ "$sign_darwin" == 1 ]]; then - log "Notarizing archive..." - execrelative ./sign_darwin.sh "$output_path" + log "Notarizing archive..." + execrelative ./sign_darwin.sh "$output_path" fi echo -n "$output_path" diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh index f0e6f9e958d59..4f7a7e305e500 100755 --- a/scripts/build_docker.sh +++ b/scripts/build_docker.sh @@ -19,7 +19,7 @@ # Prints the image tag on success. set -euo pipefail -# shellcheck source=lib.sh +# shellcheck source=scripts/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" arch="" @@ -29,57 +29,57 @@ push=0 args="$(getopt -o "" -l arch:,version:,push -- "$@")" eval set -- "$args" while true; do - case "$1" in - --arch) - arch="$2" - shift 2 - ;; - --version) - version="$2" - shift 2 - ;; - --push) - push=1 - shift - ;; - --) - shift - break - ;; - *) - error "Unrecognized option: $1" - ;; - esac + case "$1" in + --arch) + arch="$2" + shift 2 + ;; + --version) + version="$2" + shift 2 + ;; + --push) + push=1 + shift + ;; + --) + shift + break + ;; + *) + error "Unrecognized option: $1" + ;; + esac done if [[ "$arch" == "" ]]; then - error "The --arch parameter is required" + error "The --arch parameter is required" fi # Remove the "v" prefix. version="${version#v}" if [[ "$version" == "" ]]; then - version="$(execrelative ./version.sh)" + version="$(execrelative ./version.sh)" fi image_tag="$(execrelative ./image_tag.sh --arch "$arch" --version="$version")" if [[ "$#" != 1 ]]; then - error "Exactly one argument must be provided to this script, $# were supplied" + error "Exactly one argument must be provided to this script, $# were supplied" fi if [[ ! -f "$1" ]]; then - error "File '$1' does not exist or is not a regular file" + error "File '$1' does not exist or is not a regular file" fi input_file="$(realpath "$1")" # Remap the arch from Golang to Docker. declare -A arch_map=( - [amd64]="linux/amd64" - [arm64]="linux/arm64" - [arm]="linux/arm/v7" + [amd64]="linux/amd64" + [arm64]="linux/arm64" + [arm]="linux/arm/v7" ) if [[ "${arch_map[$arch]+exists}" != "" ]]; then - arch="${arch_map[$arch]}" + arch="${arch_map[$arch]}" fi # Make temporary dir where all source files intended to be in the image will be @@ -92,14 +92,14 @@ ln -P Dockerfile "$temp_dir/" cd "$temp_dir" build_args=( - "--platform=$arch" - "--label=org.opencontainers.image.title=Coder" - "--label=org.opencontainers.image.description=A tool for provisioning self-hosted development environments with Terraform." - "--label=org.opencontainers.image.url=https://github.com/coder/coder" - "--label=org.opencontainers.image.source=https://github.com/coder/coder" - "--label=org.opencontainers.image.version=$version" - "--label=org.opencontainers.image.licenses=AGPL-3.0" - "--tag=$image_tag" + "--platform=$arch" + "--label=org.opencontainers.image.title=Coder" + "--label=org.opencontainers.image.description=A tool for provisioning self-hosted development environments with Terraform." + "--label=org.opencontainers.image.url=https://github.com/coder/coder" + "--label=org.opencontainers.image.source=https://github.com/coder/coder" + "--label=org.opencontainers.image.version=$version" + "--label=org.opencontainers.image.licenses=AGPL-3.0" + "--tag=$image_tag" ) log "--- Building Docker image for $arch ($image_tag)" @@ -109,8 +109,8 @@ cdroot rm -rf "$temp_dir" if [[ "$push" == 1 ]]; then - log "--- Pushing Docker image for $arch ($image_tag)" - docker push "$image_tag" + log "--- Pushing Docker image for $arch ($image_tag)" + docker push "$image_tag" fi echo -n "$image_tag" diff --git a/scripts/build_docker_multiarch.sh b/scripts/build_docker_multiarch.sh index 9765e48c8a56d..5ba5215e251dc 100755 --- a/scripts/build_docker_multiarch.sh +++ b/scripts/build_docker_multiarch.sh @@ -16,7 +16,7 @@ # Returns the merged image tag. set -euo pipefail -# shellcheck source=lib.sh +# shellcheck source=scripts/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" version="" @@ -25,49 +25,49 @@ push=0 args="$(getopt -o "" -l version:,push -- "$@")" eval set -- "$args" while true; do - case "$1" in - --version) - version="$2" - shift 2 - ;; - --push) - push=1 - shift - ;; - --) - shift - break - ;; - *) - error "Unrecognized option: $1" - ;; - esac + case "$1" in + --version) + version="$2" + shift 2 + ;; + --push) + push=1 + shift + ;; + --) + shift + break + ;; + *) + error "Unrecognized option: $1" + ;; + esac done # Remove the "v" prefix. version="${version#v}" if [[ "$version" == "" ]]; then - version="$(execrelative ./version.sh)" + version="$(execrelative ./version.sh)" fi if [[ "$#" == 0 ]]; then - error "At least one argument must be provided to this script, $# were supplied" + error "At least one argument must be provided to this script, $# were supplied" fi create_args=() for image_tag in "$@"; do - create_args+=(--amend "$image_tag") + create_args+=(--amend "$image_tag") done output_tag="$(execrelative ./image_tag.sh --version "$version")" log "--- Creating multi-arch Docker image ($output_tag)" docker manifest create \ - "$output_tag" \ - "${create_args[@]}" + "$output_tag" \ + "${create_args[@]}" if [[ "$push" == 1 ]]; then - log "--- Pushing multi-arch Docker image ($output_tag)" - docker push "$output_tag" + log "--- Pushing multi-arch Docker image ($output_tag)" + docker push "$output_tag" fi echo -n "$output_tag" diff --git a/scripts/build_go.sh b/scripts/build_go.sh index 1c4c414d6cf21..96ee4ea3bc817 100755 --- a/scripts/build_go.sh +++ b/scripts/build_go.sh @@ -18,7 +18,7 @@ # set and the signing certificate must be imported for this to work. set -euo pipefail -# shellcheck source=lib.sh +# shellcheck source=scripts/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" cdroot @@ -32,76 +32,76 @@ output_path="" args="$(getopt -o "" -l version:,os:,arch:,output:,slim,sign-darwin -- "$@")" eval set -- "$args" while true; do - case "$1" in - --version) - version="$2" - shift 2 - ;; - --os) - os="$2" - shift 2 - ;; - --arch) - arch="$2" - shift 2 - ;; - --output) - output_path="$(realpath "$2")" - shift 2 - ;; - --slim) - slim=1 - shift - ;; - --sign-darwin) - if [[ "${AC_APPLICATION_IDENTITY:-}" == "" ]]; then - error "AC_APPLICATION_IDENTITY must be set when --sign-darwin is supplied" - fi - sign_darwin=1 - shift - ;; - --) - shift - break - ;; - *) - error "Unrecognized option: $1" - ;; - esac + case "$1" in + --version) + version="$2" + shift 2 + ;; + --os) + os="$2" + shift 2 + ;; + --arch) + arch="$2" + shift 2 + ;; + --output) + output_path="$(realpath "$2")" + shift 2 + ;; + --slim) + slim=1 + shift + ;; + --sign-darwin) + if [[ "${AC_APPLICATION_IDENTITY:-}" == "" ]]; then + error "AC_APPLICATION_IDENTITY must be set when --sign-darwin is supplied" + fi + sign_darwin=1 + shift + ;; + --) + shift + break + ;; + *) + error "Unrecognized option: $1" + ;; + esac done # Remove the "v" prefix. version="${version#v}" if [[ "$version" == "" ]]; then - version="$(execrelative ./version.sh)" + version="$(execrelative ./version.sh)" fi build_args=( - -ldflags "-s -w -X 'github.com/coder/coder/buildinfo.tag=$version'" + -ldflags "-s -w -X 'github.com/coder/coder/buildinfo.tag=$version'" ) if [[ "$slim" == 0 ]]; then - build_args+=(-tags embed) + build_args+=(-tags embed) fi # Compute default output path. if [[ "$output_path" == "" ]]; then - dist_dir="dist" - mkdir -p "$dist_dir" - output_path="${dist_dir}/coder_${version}_${os}_${arch}" - if [[ "$os" == "windows" ]]; then - output_path+=".exe" - fi - output_path="$(realpath "$output_path")" + dist_dir="dist" + mkdir -p "$dist_dir" + output_path="${dist_dir}/coder_${version}_${os}_${arch}" + if [[ "$os" == "windows" ]]; then + output_path+=".exe" + fi + output_path="$(realpath "$output_path")" fi build_args+=(-o "$output_path") # TODO: GOARM CGO_ENABLED=0 GOOS="$os" GOARCH="$arch" go build \ - "${build_args[@]}" \ - ./cmd/coder 1>&2 + "${build_args[@]}" \ + ./cmd/coder 1>&2 if [[ "$sign_darwin" == 1 ]] && [[ "$os" == "darwin" ]]; then - codesign -s "$AC_APPLICATION_IDENTITY" -f -v --timestamp --options runtime "$output_path" + codesign -s "$AC_APPLICATION_IDENTITY" -f -v --timestamp --options runtime "$output_path" fi echo -n "$output_path" diff --git a/scripts/build_go_matrix.sh b/scripts/build_go_matrix.sh index e7346f0a1da64..5db03b41fe337 100755 --- a/scripts/build_go_matrix.sh +++ b/scripts/build_go_matrix.sh @@ -32,7 +32,7 @@ # packaged using ./package.sh. Requires the nfpm binary. set -euo pipefail -# shellcheck source=lib.sh +# shellcheck source=scripts/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" version="" @@ -45,84 +45,84 @@ package_linux=0 args="$(getopt -o "" -l version:,output:,slim,sign-darwin,archive,package-linux -- "$@")" eval set -- "$args" while true; do - case "$1" in - --version) - version="$2" - shift 2 - ;; - --output) - # realpath fails if the dir doesn't exist. - mkdir -p "$(dirname "$2")" - output_path="$(realpath "$2")" - shift 2 - ;; - --slim) - slim=1 - shift - ;; - --sign-darwin) - if [[ "${AC_APPLICATION_IDENTITY:-}" == "" ]]; then - error "AC_APPLICATION_IDENTITY must be set when --sign-darwin is supplied" - fi - sign_darwin=1 - shift - ;; - --archive) - archive=1 - shift - ;; - --package-linux) - package_linux=1 - shift - ;; - --) - shift - break - ;; - *) - error "Unrecognized option: $1" - ;; - esac + case "$1" in + --version) + version="$2" + shift 2 + ;; + --output) + # realpath fails if the dir doesn't exist. + mkdir -p "$(dirname "$2")" + output_path="$(realpath "$2")" + shift 2 + ;; + --slim) + slim=1 + shift + ;; + --sign-darwin) + if [[ "${AC_APPLICATION_IDENTITY:-}" == "" ]]; then + error "AC_APPLICATION_IDENTITY must be set when --sign-darwin is supplied" + fi + sign_darwin=1 + shift + ;; + --archive) + archive=1 + shift + ;; + --package-linux) + package_linux=1 + shift + ;; + --) + shift + break + ;; + *) + error "Unrecognized option: $1" + ;; + esac done # Verify the output path template. if [[ "$output_path" == "" ]]; then - # Input paths are relative, so we don't cdroot at the top, but for this case - # we want it to be relative to the root. - cdroot - mkdir -p dist - output_path="$(realpath "dist/coder_{version}_{os}_{arch}")" + # Input paths are relative, so we don't cdroot at the top, but for this case + # we want it to be relative to the root. + cdroot + mkdir -p dist + output_path="$(realpath "dist/coder_{version}_{os}_{arch}")" elif [[ "$output_path" == */ ]]; then - output_path="${output_path}coder_{version_{os}_{arch}" + output_path="${output_path}coder_{version_{os}_{arch}" else - # Verify that it contains {os} and {arch} at least. - if [[ "$output_path" != *"{os}"* ]] || [[ "$output_path" != *"{arch}"* ]]; then - error "Templated output path '$output_path' must contain {os} and {arch}" - fi + # Verify that it contains {os} and {arch} at least. + if [[ "$output_path" != *"{os}"* ]] || [[ "$output_path" != *"{arch}"* ]]; then + error "Templated output path '$output_path' must contain {os} and {arch}" + fi fi # Remove the "v" prefix. version="${version#v}" if [[ "$version" == "" ]]; then - version="$(execrelative ./version.sh)" + version="$(execrelative ./version.sh)" fi # Parse the os:arch specs into an array. specs=() for spec in "$@"; do - spec_os="$(echo "$spec" | cut -d ":" -f 1)" - if [[ "$spec_os" == "" ]] || [[ "$spec_os" == *" "* ]]; then - error "Could not parse matrix build spec '$spec': invalid OS '$spec_os'" - fi - - # No quoting is important here. - for spec_arch in $(echo "$spec" | cut -d ":" -f 2 | tr "," "\n"); do - if [[ "$spec_arch" == "" ]] || [[ "$spec_os" == *" "* ]]; then - error "Could not parse matrix build spec '$spec': invalid architecture '$spec_arch'" - fi - - specs+=("$spec_os:$spec_arch") - done + spec_os="$(echo "$spec" | cut -d ":" -f 1)" + if [[ "$spec_os" == "" ]] || [[ "$spec_os" == *" "* ]]; then + error "Could not parse matrix build spec '$spec': invalid OS '$spec_os'" + fi + + # No quoting is important here. + for spec_arch in $(echo "$spec" | cut -d ":" -f 2 | tr "," "\n"); do + if [[ "$spec_arch" == "" ]] || [[ "$spec_os" == *" "* ]]; then + error "Could not parse matrix build spec '$spec': invalid architecture '$spec_arch'" + fi + + specs+=("$spec_os:$spec_arch") + done done # Remove duplicate specs while maintaining the same order. @@ -130,68 +130,68 @@ mapfile -t specs < <(echo "${specs[@]}" | tr " " "\n" | awk '!a[$0]++') build_args=() if [[ "$slim" == 1 ]]; then - build_args+=(--slim) + build_args+=(--slim) fi if [[ "$sign_darwin" == 1 ]]; then - build_args+=(--sign-darwin) + build_args+=(--sign-darwin) fi # Build each spec. for spec in "${specs[@]}"; do - spec_os="$(echo "$spec" | cut -d ":" -f 1)" - spec_arch="$(echo "$spec" | cut -d ":" -f 2)" - - # Craft output path from the template. - spec_output="$output_path" - spec_output="${spec_output//\{os\}/"$spec_os"}" - spec_output="${spec_output//\{arch\}/"$spec_arch"}" - spec_output="${spec_output//\{version\}/"$version"}" - - spec_output_binary="$spec_output" - if [[ "$spec_os" == "windows" ]]; then - spec_output_binary+=".exe" - fi - - # Ensure parent dir. - mkdir -p "$(dirname "$spec_output")" - - log "--- Building coder for $spec_os $spec_arch ($spec_output_binary)" - execrelative ./build_go.sh \ - --version "$version" \ - --os "$spec_os" \ - --arch "$spec_arch" \ - --output "$spec_output_binary" \ - "${build_args[@]}" - log - log - - if [[ "$archive" == 1 ]]; then - spec_archive_format="tar.gz" - if [[ "$spec_os" == "windows" ]] || [[ "$spec_os" == "darwin" ]]; then - spec_archive_format="zip" - fi - spec_output_archive="$spec_output.$spec_archive_format" - - archive_args=() - if [[ "$sign_darwin" == 1 ]] && [[ "$spec_os" == "darwin" ]]; then - archive_args+=(--sign-darwin) - fi - - log "--- Creating archive for $spec_os $spec_arch ($spec_output_archive)" - execrelative ./archive.sh \ - --format "$spec_archive_format" \ - --output "$spec_output_archive" \ - "${archive_args[@]}" \ - "$spec_output_binary" - log - log - fi - - if [[ "$package_linux" == 1 ]] && [[ "$spec_os" == "linux" ]]; then - execrelative ./package.sh \ - --arch "$spec_arch" \ - --version "$version" \ - "$spec_output_binary" - log - fi + spec_os="$(echo "$spec" | cut -d ":" -f 1)" + spec_arch="$(echo "$spec" | cut -d ":" -f 2)" + + # Craft output path from the template. + spec_output="$output_path" + spec_output="${spec_output//\{os\}/"$spec_os"}" + spec_output="${spec_output//\{arch\}/"$spec_arch"}" + spec_output="${spec_output//\{version\}/"$version"}" + + spec_output_binary="$spec_output" + if [[ "$spec_os" == "windows" ]]; then + spec_output_binary+=".exe" + fi + + # Ensure parent dir. + mkdir -p "$(dirname "$spec_output")" + + log "--- Building coder for $spec_os $spec_arch ($spec_output_binary)" + execrelative ./build_go.sh \ + --version "$version" \ + --os "$spec_os" \ + --arch "$spec_arch" \ + --output "$spec_output_binary" \ + "${build_args[@]}" + log + log + + if [[ "$archive" == 1 ]]; then + spec_archive_format="tar.gz" + if [[ "$spec_os" == "windows" ]] || [[ "$spec_os" == "darwin" ]]; then + spec_archive_format="zip" + fi + spec_output_archive="$spec_output.$spec_archive_format" + + archive_args=() + if [[ "$sign_darwin" == 1 ]] && [[ "$spec_os" == "darwin" ]]; then + archive_args+=(--sign-darwin) + fi + + log "--- Creating archive for $spec_os $spec_arch ($spec_output_archive)" + execrelative ./archive.sh \ + --format "$spec_archive_format" \ + --output "$spec_output_archive" \ + "${archive_args[@]}" \ + "$spec_output_binary" + log + log + fi + + if [[ "$package_linux" == 1 ]] && [[ "$spec_os" == "linux" ]]; then + execrelative ./package.sh \ + --arch "$spec_arch" \ + --version "$version" \ + "$spec_output_binary" + log + fi done diff --git a/scripts/image_tag.sh b/scripts/image_tag.sh index bc2608bed72e2..43dc16b67e38a 100755 --- a/scripts/image_tag.sh +++ b/scripts/image_tag.sh @@ -14,7 +14,7 @@ # sign. set -euo pipefail -# shellcheck source=lib.sh +# shellcheck source=scripts/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" arch="" @@ -23,35 +23,35 @@ version="" args="$(getopt -o "" -l arch:,version: -- "$@")" eval set -- "$args" while true; do - case "$1" in - --arch) - arch="$2" - shift 2 - ;; - --version) - version="$2" - shift 2 - ;; - --) - shift - break - ;; - *) - error "Unrecognized option: $1" - ;; - esac + case "$1" in + --arch) + arch="$2" + shift 2 + ;; + --version) + version="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + error "Unrecognized option: $1" + ;; + esac done # Remove the "v" prefix. version="${version#v}" if [[ "$version" == "" ]]; then - version="$(execrelative ./version.sh)" + version="$(execrelative ./version.sh)" fi image="${CODER_IMAGE_BASE:-ghcr.io/coder/coder}" tag="v$version" if [[ "$arch" != "" ]]; then - tag+="-$arch" + tag+="-$arch" fi tag="${tag//+/-}" diff --git a/scripts/lib.sh b/scripts/lib.sh index 14f3c6d871a6c..50f19a6f2f08c 100644 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -10,33 +10,33 @@ PROJECT_ROOT="$(cd "$SCRIPT_DIR" && realpath "$(git rev-parse --show-toplevel)") # pushd is a silent alternative to the real pushd shell command. pushd() { - command pushd "$@" >/dev/null + command pushd "$@" >/dev/null } # popd is a silent alternative to the real popd shell command. # shellcheck disable=SC2120 popd() { - command popd "$@" >/dev/null + command popd "$@" >/dev/null } # cdself changes directory to the directory of the current script. This should # not be used in scripts that may be sourced by other scripts. cdself() { - cd "$SCRIPT_DIR" || error "Could not change directory to '$SCRIPT_DIR'" + cd "$SCRIPT_DIR" || error "Could not change directory to '$SCRIPT_DIR'" } # cdroot changes directory to the root of the repository. cdroot() { - cd "$PROJECT_ROOT" || error "Could not change directory to '$PROJECT_ROOT'" + cd "$PROJECT_ROOT" || error "Could not change directory to '$PROJECT_ROOT'" } # execrelative can be used to execute scripts as if you were in the parent # directory of the current script. This should not be used in scripts that may # be sourced by other scripts. execrelative() { - pushd "$SCRIPT_DIR" || error "Could not change directory to '$SCRIPT_DIR'" - "$@" - popd + pushd "$SCRIPT_DIR" || error "Could not change directory to '$SCRIPT_DIR'" + "$@" + popd } # realpath returns an absolute path to the given relative path. It will fail if @@ -48,15 +48,15 @@ execrelative() { # # Taken from https://stackoverflow.com/a/3915420 (CC-BY-SA 4.0) realpath() { - dir="$(dirname "$1")" - base="$(basename "$1")" - if [[ ! -d "$dir" ]]; then - error "Could not change directory to '$dir': directory does not exist" - fi - echo "$( - cd "$dir" || error "Could not change directory to '$dir'" - pwd -P - )"/"$base" + dir="$(dirname "$1")" + base="$(basename "$1")" + if [[ ! -d "$dir" ]]; then + error "Could not change directory to '$dir': directory does not exist" + fi + echo "$( + cd "$dir" || error "Could not change directory to '$dir'" + pwd -P + )"/"$base" } # maybedryrun prints the given program and flags, and then, if the first @@ -68,23 +68,23 @@ realpath() { # Usage: maybedryrun 1 gh release create ... # Usage: maybedryrun 0 docker push ghcr.io/coder/coder:latest maybedryrun() { - if [[ "$1" == 1 ]]; then - shift - log "DRYRUN: $*" - else - shift - log $ "$@" - "$@" - fi + if [[ "$1" == 1 ]]; then + shift + log "DRYRUN: $*" + else + shift + log $ "$@" + "$@" + fi } # log prints a message to stderr. log() { - echo "$*" 1>&2 + echo "$*" 1>&2 } # error prints an error message and returns an error exit code. error() { - log "ERROR: $*" - exit 1 + log "ERROR: $*" + exit 1 } diff --git a/scripts/package.sh b/scripts/package.sh index d97ab52e3fa6e..1dc54a41dad59 100755 --- a/scripts/package.sh +++ b/scripts/package.sh @@ -10,7 +10,7 @@ # version from ./version.sh. set -euo pipefail -# shellcheck source=lib.sh +# shellcheck source=scripts/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" version="" @@ -19,41 +19,41 @@ arch="" args="$(getopt -o "" -l arch:,version: -- "$@")" eval set -- "$args" while true; do - case "$1" in - --arch) - arch="$2" - shift 2 - ;; - --version) - version="$2" - shift 2 - ;; - --) - shift - break - ;; - *) - error "Unrecognized option: $1" - ;; - esac + case "$1" in + --arch) + arch="$2" + shift 2 + ;; + --version) + version="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + error "Unrecognized option: $1" + ;; + esac done if [[ "$arch" == "" ]]; then - error "--arch is a required parameter" + error "--arch is a required parameter" fi if [[ "$#" != 1 ]]; then - error "Exactly one argument must be provided to this script, $# were supplied" + error "Exactly one argument must be provided to this script, $# were supplied" fi if [[ ! -f "$1" ]]; then - error "File '$1' does not exist or is not a regular file" + error "File '$1' does not exist or is not a regular file" fi input_file="$(realpath "$1")" # Remove the "v" prefix. version="${version#v}" if [[ "$version" == "" ]]; then - version="$(execrelative ./version.sh)" + version="$(execrelative ./version.sh)" fi # Make temporary dir where all source files intended to be in the package will @@ -69,12 +69,12 @@ cd "$temp_dir" formats=(apk deb rpm) for format in "${formats[@]}"; do - output_path="$input_file.$format" - log "--- Building $format package ($output_path)" - nfpm package \ - -f nfpm.yaml \ - -p "$format" \ - -t "$output_path" + output_path="$input_file.$format" + log "--- Building $format package ($output_path)" + nfpm package \ + -f nfpm.yaml \ + -p "$format" \ + -t "$output_path" done cdroot diff --git a/scripts/publish_release.sh b/scripts/publish_release.sh index a67dbb5b9e7c4..4d2032220ff7a 100755 --- a/scripts/publish_release.sh +++ b/scripts/publish_release.sh @@ -22,7 +22,7 @@ # specified). set -euo pipefail -# shellcheck source=lib.sh +# shellcheck source=scripts/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" version="" @@ -31,39 +31,39 @@ dry_run=0 args="$(getopt -o "" -l version:,dry-run -- "$@")" eval set -- "$args" while true; do - case "$1" in - --version) - version="$2" - shift 2 - ;; - --dry-run) - dry_run=1 - shift - ;; - --) - shift - break - ;; - *) - error "Unrecognized option: $1" - ;; - esac + case "$1" in + --version) + version="$2" + shift 2 + ;; + --dry-run) + dry_run=1 + shift + ;; + --) + shift + break + ;; + *) + error "Unrecognized option: $1" + ;; + esac done # Remove the "v" prefix. version="${version#v}" if [[ "$version" == "" ]]; then - version="$(execrelative ./version.sh)" + version="$(execrelative ./version.sh)" fi # Verify that we're currently checked out on the supplied tag. new_tag="v$version" if [[ "$(git describe --always)" != "$new_tag" ]]; then - if [[ "$dry_run" == 0 ]]; then - error "The provided version '$new_tag' does not match the current git describe output '$(git describe --always)'" - fi + if [[ "$dry_run" == 0 ]]; then + error "The provided version '$new_tag' does not match the current git describe output '$(git describe --always)'" + fi - log "The provided version does not match the current git tag, but --dry-run was supplied so continuing..." + log "The provided version does not match the current git tag, but --dry-run was supplied so continuing..." fi # This returns the tag before the current tag. @@ -83,7 +83,7 @@ $(git log --no-merges --pretty=format:"- %h %s" "$old_tag..$new_tag") # Create temporary release folder so we can generate checksums. temp_dir="$(mktemp -d)" for f in "$@"; do - ln -s "$(realpath "$f")" "$temp_dir/" + ln -s "$(realpath "$f")" "$temp_dir/" done # Generate checksums file. sha256sum seems to play nicely with symlinks so this @@ -109,10 +109,10 @@ log # # GitHub CLI seems to follow symlinks when uploading files. echo "$release_notes" | - maybedryrun "$dry_run" gh release create \ - --title "$new_tag" \ - --notes-file - \ - "$new_tag" \ - "$temp_dir"/* + maybedryrun "$dry_run" gh release create \ + --title "$new_tag" \ + --notes-file - \ + "$new_tag" \ + "$temp_dir"/* rm -rf "$temp_dir" diff --git a/scripts/sign_darwin.sh b/scripts/sign_darwin.sh index ed2474055bc50..84ec406e17fee 100755 --- a/scripts/sign_darwin.sh +++ b/scripts/sign_darwin.sh @@ -13,7 +13,7 @@ # environment variable to be set. set -euo pipefail -# shellcheck source=lib.sh +# shellcheck source=scripts/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" if [[ "${AC_APPLICATION_IDENTITY:-}" == "" ]]; then diff --git a/scripts/version.sh b/scripts/version.sh index c714e7bc2e2b0..3bf23ce7d44ec 100755 --- a/scripts/version.sh +++ b/scripts/version.sh @@ -5,7 +5,7 @@ # prefix that is included in the Git tag. set -euo pipefail -# shellcheck source=lib.sh +# shellcheck source=scripts/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" cdroot @@ -20,10 +20,10 @@ version="$last_tag" # Dev versions are denoted by the "+dev." suffix with a trailing commit short # SHA. if [[ "$last_tag" != "$current" ]]; then - if [[ "${CODER_NO_DEV_VERSION:-}" != "" ]]; then - error "version.sh attemped to generate a dev version string when CODER_NO_DEV_VERSION was set" - fi - version+="+dev.$(git rev-parse --short HEAD)" + if [[ "${CODER_NO_DEV_VERSION:-}" != "" ]]; then + error "version.sh attemped to generate a dev version string when CODER_NO_DEV_VERSION was set" + fi + version+="+dev.$(git rev-parse --short HEAD)" fi # Remove the "v" prefix. From 4885076fc57b7df7576cff53ca388d3c436d1aec Mon Sep 17 00:00:00 2001 From: deansheather Date: Thu, 9 Jun 2022 22:56:17 +0000 Subject: [PATCH 10/36] chore: update makefile to use new build scripts --- Makefile | 81 +++++++++++++++++++++++++++++------- scripts/build_go.sh | 17 +++++++- scripts/build_go_matrix.sh | 18 ++++---- scripts/build_go_slim.sh | 85 ++++++++++++++++++++++++++++++++++++++ scripts/lib.sh | 1 + 5 files changed, 175 insertions(+), 27 deletions(-) create mode 100755 scripts/build_go_slim.sh diff --git a/Makefile b/Makefile index a75a9cb2884c9..5dc227684262e 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,53 @@ .DEFAULT_GOAL := build +# use a single bash shell for each job, and immediately exit on failure +SHELL := bash +.SHELLFLAGS = -ceu +.ONESHELL: + +# This doesn't work on directories. +# See https://stackoverflow.com/questions/25752543/make-delete-on-error-for-directory-targets +.DELETE_ON_ERROR: + INSTALL_DIR=$(shell go env GOPATH)/bin GOOS=$(shell go env GOOS) GOARCH=$(shell go env GOARCH) +VERSION=$(shell ./scripts/version.sh) bin: $(shell find . -not -path './vendor/*' -type f -name '*.go') go.mod go.sum $(shell find ./examples/templates) - @echo "== This builds binaries for command-line usage." + @echo "== This builds slim binaries for command-line usage." @echo "== Use \"make build\" to embed the site." - goreleaser build --snapshot --rm-dist --single-target -build: dist/artifacts.json + rm -rf ./dist/coder-slim_* + ./scripts/build_go_slim.sh \ + --version "$(VERSION)" \ + --output ./dist/ \ + linux:amd64,armv7,arm64 \ + windows:amd64,arm64 \ + darwin:amd64,arm64 +.PHONY: bin + +build: site/out/index.html $(shell find . -not -path './vendor/*' -type f -name '*.go') go.mod go.sum $(shell find ./examples/templates) + rm -rf ./dist + mkdir -p ./dist + + # build slim artifacts and copy them to the site output directory + ./scripts/build_go_slim.sh \ + --version "$(VERSION)" \ + --output ./dist/ \ + linux:amd64,armv7,arm64 \ + windows:amd64,arm64 \ + darwin:amd64,arm64 + + # build not-so-slim artifacts with the default name format + ./scripts/build_go_matrix.sh \ + --version "$(VERSION)" \ + --output ./dist/ \ + --archive \ + --package-linux \ + linux:amd64,armv7,arm64 \ + windows:amd64,arm64 \ + darwin:amd64,arm64 .PHONY: build # Runs migrations to output a dump of the database. @@ -24,9 +62,6 @@ dev: ./scripts/develop.sh .PHONY: dev -dist/artifacts.json: site/out/index.html $(shell find . -not -path './vendor/*' -type f -name '*.go') go.mod go.sum $(shell find ./examples/templates) - goreleaser release --snapshot --rm-dist --skip-sign - fmt/prettier: @echo "--- prettier" # Avoid writing files in CI to reduce file write activity @@ -49,20 +84,34 @@ ifdef CI else shfmt -w $(shell shfmt -f .) endif +.PHONY: fmt/shfmt fmt: fmt/prettier fmt/terraform fmt/shfmt .PHONY: fmt gen: coderd/database/querier.go peerbroker/proto/peerbroker.pb.go provisionersdk/proto/provisioner.pb.go provisionerd/proto/provisionerd.pb.go site/src/api/typesGenerated.ts +.PHONY: gen + +install: site/out/index.html $(shell find . -not -path './vendor/*' -type f -name '*.go') go.mod go.sum $(shell find ./examples/templates) + @output_file="$(INSTALL_DIR)/coder" -install: build - mkdir -p $(INSTALL_DIR) - @echo "--- Copying from bin to $(INSTALL_DIR)" - cp -r ./dist/coder-$(GOOS)_$(GOOS)_$(GOARCH)*/* $(INSTALL_DIR) - @echo "-- CLI available at $(shell ls $(INSTALL_DIR)/coder*)" + @if [[ "$(GOOS)" == "windows" ]]; then + @output_file="$${output_file}.exe" + @fi + + @echo "-- Building CLI for $(GOOS) $(GOARCH) at $$output_file" + + ./scripts/build_go.sh \ + --version "$(VERSION)" \ + --output "$$output_file" \ + --os "$(GOOS)" \ + --arch "$(GOARCH)" + + @echo .PHONY: install lint: lint/shellcheck lint/go +.PHONY: lint lint/go: golangci-lint run @@ -72,6 +121,7 @@ lint/go: lint/shellcheck: $(shell shfmt -f .) @echo "--- shellcheck" shellcheck --external-sources $(shell shfmt -f .) +.PHONY: lint/shellcheck peerbroker/proto/peerbroker.pb.go: peerbroker/proto/peerbroker.proto protoc \ @@ -108,19 +158,17 @@ site/src/api/typesGenerated.ts: scripts/apitypings/main.go $(shell find codersdk go run scripts/apitypings/main.go > site/src/api/typesGenerated.ts cd site && yarn run format:types -.PHONY: test test: test-clean gotestsum -- -v -short ./... +.PHONY: test -.PHONY: test-postgres test-postgres: test-clean DB=ci gotestsum --junitfile="gotests.xml" --packages="./..." -- \ -covermode=atomic -coverprofile="gotests.coverage" -timeout=5m \ -coverpkg=./...,github.com/coder/coder/codersdk \ -count=1 -parallel=1 -race -failfast +.PHONY: test-postgres - -.PHONY: test-postgres-docker test-postgres-docker: docker run \ --env POSTGRES_PASSWORD=postgres \ @@ -134,7 +182,8 @@ test-postgres-docker: postgres:11 \ -c shared_buffers=1GB \ -c max_connections=1000 +.PHONY: test-postgres-docker -.PHONY: test-clean test-clean: go clean -testcache +.PHONY: test-clean diff --git a/scripts/build_go.sh b/scripts/build_go.sh index 96ee4ea3bc817..9fb680cbb7dcd 100755 --- a/scripts/build_go.sh +++ b/scripts/build_go.sh @@ -8,6 +8,9 @@ # GOARCH and CODER_SLIM_BUILD=1. If no version is specified, defaults to the # version from ./version.sh. # +# GOARM can be controlled by suffixing any arm architecture (i.e. arm or arm64) +# with "vX" (e.g. "v7", "v8"). +# # Unless overridden via --output, the built binary will be dropped in # "$repo_root/dist/coder_$version_$os_$arch" (with a ".exe" suffix for windows # builds) and the absolute path to the binary will be printed to stdout on @@ -95,8 +98,18 @@ if [[ "$output_path" == "" ]]; then fi build_args+=(-o "$output_path") -# TODO: GOARM -CGO_ENABLED=0 GOOS="$os" GOARCH="$arch" go build \ +# Determine GOARM. +arm_version="" +if [[ "$arch" == "arm" ]]; then + arm_version="7" +elif [[ "$arch" == "armv"* ]] || [[ "$arch" == "arm64v"* ]]; then + arm_version="${arch//*v/}" + + # Remove the v* suffix. + arch="${arch//v*/}" +fi + +CGO_ENABLED=0 GOOS="$os" GOARCH="$arch" GOARM="$arm_version" go build \ "${build_args[@]}" \ ./cmd/coder 1>&2 diff --git a/scripts/build_go_matrix.sh b/scripts/build_go_matrix.sh index 5db03b41fe337..569865e1a32dc 100755 --- a/scripts/build_go_matrix.sh +++ b/scripts/build_go_matrix.sh @@ -51,9 +51,7 @@ while true; do shift 2 ;; --output) - # realpath fails if the dir doesn't exist. - mkdir -p "$(dirname "$2")" - output_path="$(realpath "$2")" + output_path="$2" shift 2 ;; --slim) @@ -93,14 +91,16 @@ if [[ "$output_path" == "" ]]; then mkdir -p dist output_path="$(realpath "dist/coder_{version}_{os}_{arch}")" elif [[ "$output_path" == */ ]]; then - output_path="${output_path}coder_{version_{os}_{arch}" -else - # Verify that it contains {os} and {arch} at least. - if [[ "$output_path" != *"{os}"* ]] || [[ "$output_path" != *"{arch}"* ]]; then - error "Templated output path '$output_path' must contain {os} and {arch}" - fi + output_path="${output_path}coder_{version}_{os}_{arch}" +elif [[ "$output_path" != *"{os}"* ]] || [[ "$output_path" != *"{arch}"* ]]; then + # If the output path isn't a directory (ends with /) then it must have + # template variables. + error "Templated output path '$output_path' must contain {os} and {arch}" fi +mkdir -p "$(dirname "$output_path")" +output_path="$(realpath "$output_path")" + # Remove the "v" prefix. version="${version#v}" if [[ "$version" == "" ]]; then diff --git a/scripts/build_go_slim.sh b/scripts/build_go_slim.sh new file mode 100755 index 0000000000000..c5a6d6d00d966 --- /dev/null +++ b/scripts/build_go_slim.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash + +# This script builds multiple "slim" Go binaries for Coder with the given OS and +# architecture combinations. This wraps ./build_go_matrix.sh. +# +# Usage: ./build_go_slim.sh [--version 1.2.3+devel.abcdef] [--output dist/] os1:arch1,arch2 os2:arch1 os1:arch3 +# +# If no OS:arch combinations are provided, nothing will happen and no error will +# be returned. If no version is specified, defaults to the version from +# ./version.sh +# +# The --output parameter differs from ./build_go_matrix.sh, in that it does not +# accept variables such as `{os}` and `{arch}` and only accepts a directory +# ending with `/`. +# +# The built binaries are additionally copied to the site output directory so +# they can be packaged into non-slim binaries correctly. + +set -euo pipefail +# shellcheck source=scripts/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +version="" +output_path="" + +args="$(getopt -o "" -l version:,output: -- "$@")" +eval set -- "$args" +while true; do + case "$1" in + --version) + version="$2" + shift 2 + ;; + --output) + output_path="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + error "Unrecognized option: $1" + ;; + esac +done + +# Remove the "v" prefix. +version="${version#v}" +if [[ "$version" == "" ]]; then + version="$(execrelative ./version.sh)" +fi + +# Verify the output path. +if [[ "$output_path" == "" ]]; then + # Input paths are relative, so we don't cdroot at the top, but for this case + # we want it to be relative to the root. + cdroot + mkdir -p dist + output_path="$(realpath "dist/coder-slim_{version}_{os}_{arch}")" +elif [[ "$output_path" != */ ]] || [[ "$output_path" == *"{"* ]]; then + error "The output path '$output_path' cannot contain variables and must end with a slash" +else + mkdir -p "$output_path" + output_path="$(realpath "${output_path}coder-slim_{version}_{os}_{arch}")" +fi + +./scripts/build_go_matrix.sh \ + --version "$version" \ + --output "$output_path" \ + --slim \ + "$@" + +cdroot +dest_dir="./site/out/bin" +mkdir -p "$dest_dir" +dest_dir="$(realpath "$dest_dir")" + +# Copy the binaries to the site directory. +cd "$(dirname "$output_path")" +for f in ./coder-slim_*; do + f="${f#./}" + dest="$dest_dir/${f//-slim_$version/}" + cp "$f" "$dest" +done diff --git a/scripts/lib.sh b/scripts/lib.sh index 50f19a6f2f08c..64123c0069172 100644 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # This script is meant to be sourced by other scripts. To source this script: +# # shellcheck source=scripts/lib.sh # source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" set -euo pipefail From 044780d6bf6f6f88606665c9f33e44b7614407e1 Mon Sep 17 00:00:00 2001 From: deansheather Date: Fri, 10 Jun 2022 20:43:14 +0000 Subject: [PATCH 11/36] chore: update release workflow to use new build scripts --- .github/workflows/release.yaml | 178 +++++++++++++++++++++++++-------- Makefile | 3 +- 2 files changed, 140 insertions(+), 41 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 1736a45ce1d9d..6d36c6925554f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,39 +1,95 @@ +# GitHub release workflow. +# +# This workflow is a bit complicated because we have to build darwin binaries on +# a mac runner, but the mac runners are extremely slow. So instead of running +# the entire release on a mac (which will take an hour to run), we run only the +# mac build on a mac, and the rest on a linux runner. The final release is then +# published using a final linux runner. name: release on: push: tags: - "v*" workflow_dispatch: + inputs: + snapshot: + description: Allow a +dev version to be generated, implies dry_run. + type: boolean + required: true + dry_run: + description: Perform a dry-run release. + type: boolean + required: true + +# In a non-snapshot we want the ./scripts/version.sh script to never generate a +# +dev version. +env: + CODER_NO_DEV_VERSION: ${{ github.event.inputs.snapshot || 'true' }} jobs: - goreleaser: - runs-on: macos-latest - env: - # Necessary for Docker manifest - DOCKER_CLI_EXPERIMENTAL: "enabled" + linux-windows: + runs-on: ubuntu-latest steps: - # Docker is not included on macos-latest - - uses: docker-practice/actions-setup-docker@1.0.10 - - uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + - uses: actions/setup-go@v3 + with: + go-version: "~1.18" - - name: Docker Login - uses: docker/login-action@v2 + - name: Cache Node + id: cache-node + uses: actions/cache@v3 with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} + path: | + **/node_modules + .eslintcache + key: js-${{ runner.os }}-test-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + js-${{ runner.os }}- + + - name: Build Site + run: make site/out/index.html + + - name: Build Slim Binaries + run: make bin + + - name: Build Linux and Windows Binaries + run: | + ./scripts/build_go_matrix.sh \ + --output ./dist/ \ + --archive \ + --package-linux \ + linux:amd64,armv7,arm64 \ + windows:amd64,arm64 + + - name: Upload binary artifacts + uses: actions/upload-artifact@v3 + with: + name: linux + path: | + dist/*.zip + dist/*.tar.gz + dist/*.apk + dist/*.deb + dist/*.rpm + + # The mac binaries get built on mac runners because they need to be signed, + # and the signing tool only runs on mac. This darwin job only builds the Mac + # binaries and uploads them as job artifacts used by the publish step. + darwin: + runs-on: macos-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 - uses: actions/setup-go@v3 with: go-version: "~1.18" - - name: Install Gon + - name: Install gon run: | brew tap mitchellh/gon brew install mitchellh/gon/gon @@ -44,24 +100,6 @@ jobs: p12-file-base64: ${{ secrets.AC_CERTIFICATE_P12_BASE64 }} p12-password: ${{ secrets.AC_CERTIFICATE_PASSWORD }} - - name: Echo Go Cache Paths - id: go-cache-paths - run: | - echo "::set-output name=go-build::$(go env GOCACHE)" - echo "::set-output name=go-mod::$(go env GOMODCACHE)" - - - name: Go Build Cache - uses: actions/cache@v3 - with: - path: ${{ steps.go-cache-paths.outputs.go-build }} - key: ${{ runner.os }}-release-go-build-${{ hashFiles('**/go.sum') }} - - - name: Go Mod Cache - uses: actions/cache@v3 - with: - path: ${{ steps.go-cache-paths.outputs.go-mod }} - key: ${{ runner.os }}-release-go-mod-${{ hashFiles('**/go.sum') }} - - name: Cache Node id: cache-node uses: actions/cache@v3 @@ -79,12 +117,72 @@ jobs: - name: Build Site run: make site/out/index.html - - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v3 - with: - version: latest - args: release --rm-dist --timeout 60m + - name: Build Slim Binaries + run: make bin + + - name: Build darwin Binaries (with signatures) + run: | + ./scripts/build_go_matrix.sh \ + --output ./dist/ \ + --archive \ + --sign-darwin \ + darwin:amd64,arm64 env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} AC_USERNAME: ${{ secrets.AC_USERNAME }} AC_PASSWORD: ${{ secrets.AC_PASSWORD }} + + - name: Upload Binary Artifacts + uses: actions/upload-artifact@v3 + with: + name: darwin + path: ./dist/coder_*.zip + + publish: + runs-on: ubuntu-latest + needs: + - linux-windows + - darwin + env: + # Necessary for Docker manifest + DOCKER_CLI_EXPERIMENTAL: "enabled" + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Docker Login + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: mkdir artifacts + run: mkdir artifacts + + - name: Download darwin Artifacts + uses: actions/download-artifact@v3 + with: + name: darwin + path: artifacts + + - name: Download Linux and Windows Artifacts + uses: actions/download-artifact@v3 + with: + name: linux + path: artifacts + + - name: ls artifacts + run: ls artifacts + + - name: Publish Release + run: | + ./scripts/publish_release.sh \ + ${{ (github.event.inputs.dry_run || github.event.inputs.snapshot) && '--dry-run' }} \ + ./artifacts/*.zip \ + ./artifacts/*.tar.gz \ + ./artifacts/*.apk \ + ./artifacts/*.deb \ + ./artifacts/*.rpm + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/Makefile b/Makefile index 5dc227684262e..644aeb99c1627 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .DEFAULT_GOAL := build -# use a single bash shell for each job, and immediately exit on failure +# Use a single bash shell for each job, and immediately exit on failure SHELL := bash .SHELLFLAGS = -ceu .ONESHELL: @@ -18,6 +18,7 @@ bin: $(shell find . -not -path './vendor/*' -type f -name '*.go') go.mod go.sum @echo "== This builds slim binaries for command-line usage." @echo "== Use \"make build\" to embed the site." + mkdir -p ./dist rm -rf ./dist/coder-slim_* ./scripts/build_go_slim.sh \ --version "$(VERSION)" \ From 59cd2c70f69c86706563ddd88a47b4c6e3555173 Mon Sep 17 00:00:00 2001 From: deansheather Date: Sun, 12 Jun 2022 22:03:44 +0000 Subject: [PATCH 12/36] fixup! chore: update release workflow to use new build scripts --- .github/workflows/release.yaml | 2 +- Dockerfile | 11 +++++++++ scripts/build_docker.sh | 11 +++------ scripts/build_docker_multiarch.sh | 1 + scripts/lib.sh | 41 ++++++++++++++++--------------- 5 files changed, 37 insertions(+), 29 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 6d36c6925554f..a0d92339e8339 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -24,7 +24,7 @@ on: # In a non-snapshot we want the ./scripts/version.sh script to never generate a # +dev version. env: - CODER_NO_DEV_VERSION: ${{ github.event.inputs.snapshot || 'true' }} + CODER_NO_DEV_VERSION: ${{ github.event.inputs.snapshot && '' || 'true' }} jobs: linux-windows: diff --git a/Dockerfile b/Dockerfile index 76f244a65bd3f..489c7266485ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,16 @@ FROM alpine +# LABEL doesn't add any real layers so it's fine (and easier) to do it here than +# in the build script. +ARG CODER_VERSION +LABEL \ + org.opencontainers.image.title="Coder" \ + org.opencontainers.image.description="A tool for provisioning self-hosted development environments with Terraform." \ + org.opencontainers.image.url="https://github.com/coder/coder" \ + org.opencontainers.image.source="https://github.com/coder/coder" \ + org.opencontainers.image.version="$CODER_VERSION" \ + org.opencontainers.image.licenses="AGPL-3.0" + # The coder binary is injected by scripts/build_docker.sh. ADD coder /opt/coder diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh index 4f7a7e305e500..b528b44ad78db 100755 --- a/scripts/build_docker.sh +++ b/scripts/build_docker.sh @@ -92,14 +92,9 @@ ln -P Dockerfile "$temp_dir/" cd "$temp_dir" build_args=( - "--platform=$arch" - "--label=org.opencontainers.image.title=Coder" - "--label=org.opencontainers.image.description=A tool for provisioning self-hosted development environments with Terraform." - "--label=org.opencontainers.image.url=https://github.com/coder/coder" - "--label=org.opencontainers.image.source=https://github.com/coder/coder" - "--label=org.opencontainers.image.version=$version" - "--label=org.opencontainers.image.licenses=AGPL-3.0" - "--tag=$image_tag" + --platform "$arch" + --build-arg "CODER_VERSION=$version" + --tag "$image_tag" ) log "--- Building Docker image for $arch ($image_tag)" diff --git a/scripts/build_docker_multiarch.sh b/scripts/build_docker_multiarch.sh index 5ba5215e251dc..7b2648b688904 100755 --- a/scripts/build_docker_multiarch.sh +++ b/scripts/build_docker_multiarch.sh @@ -59,6 +59,7 @@ for image_tag in "$@"; do create_args+=(--amend "$image_tag") done +# Sadly, multi-arch images don't seem to support labels. output_tag="$(execrelative ./image_tag.sh --version "$version")" log "--- Creating multi-arch Docker image ($output_tag)" docker manifest create \ diff --git a/scripts/lib.sh b/scripts/lib.sh index 64123c0069172..69173d6276f2b 100644 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -6,6 +6,27 @@ set -euo pipefail +# realpath returns an absolute path to the given relative path. It will fail if +# the parent directory of the path does not exist. Make sure you are in the +# expected directory before running this to avoid errors. +# +# GNU realpath relies on coreutils, which are not installed or the default on +# Macs out of the box, so we have this mostly working bash alternative instead. +# +# Taken from https://stackoverflow.com/a/3915420 (CC-BY-SA 4.0) +realpath() { + dir="$(dirname "$1")" + base="$(basename "$1")" + if [[ ! -d "$dir" ]]; then + error "Could not change directory to '$dir': directory does not exist" + fi + echo "$( + cd "$dir" || error "Could not change directory to '$dir'" + pwd -P + )"/"$base" +} + +# We have to define realpath before these otherwise it fails on Mac's bash. SCRIPT_DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" PROJECT_ROOT="$(cd "$SCRIPT_DIR" && realpath "$(git rev-parse --show-toplevel)")" @@ -40,26 +61,6 @@ execrelative() { popd } -# realpath returns an absolute path to the given relative path. It will fail if -# the parent directory of the path does not exist. Make sure you are in the -# expected directory before running this to avoid errors. -# -# GNU realpath relies on coreutils, which are not installed or the default on -# Macs out of the box, so we have this mostly working bash alternative instead. -# -# Taken from https://stackoverflow.com/a/3915420 (CC-BY-SA 4.0) -realpath() { - dir="$(dirname "$1")" - base="$(basename "$1")" - if [[ ! -d "$dir" ]]; then - error "Could not change directory to '$dir': directory does not exist" - fi - echo "$( - cd "$dir" || error "Could not change directory to '$dir'" - pwd -P - )"/"$base" -} - # maybedryrun prints the given program and flags, and then, if the first # argument is 0, executes it. The reason the first argument should be 0 is that # it is expected that you have a dry_run variable in your script that is set to From d1edee559330ad43a7ec5c74461e80b219cc3c07 Mon Sep 17 00:00:00 2001 From: deansheather Date: Sun, 12 Jun 2022 22:22:11 +0000 Subject: [PATCH 13/36] fixup! chore: update release workflow to use new build scripts --- .github/workflows/release.yaml | 4 +++- Makefile | 15 +++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index a0d92339e8339..862aa2a62e01e 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -112,7 +112,9 @@ jobs: js-${{ runner.os }}- - name: Install make - run: brew install make + run: | + brew install make + echo "$(brew --prefix)/opt/make/libexec/gnubin" >> $GITHUB_PATH - name: Build Site run: make site/out/index.html diff --git a/Makefile b/Makefile index 644aeb99c1627..cbd2bbe5663fe 100644 --- a/Makefile +++ b/Makefile @@ -65,11 +65,12 @@ dev: fmt/prettier: @echo "--- prettier" + cd site # Avoid writing files in CI to reduce file write activity ifdef CI - cd site && yarn run format:check + yarn run format:check else - cd site && yarn run format:write + yarn run format:write endif .PHONY: fmt/prettier @@ -150,14 +151,16 @@ provisionersdk/proto/provisioner.pb.go: provisionersdk/proto/provisioner.proto site/out/index.html: $(shell find ./site -not -path './site/node_modules/*' -type f -name '*.tsx') $(shell find ./site -not -path './site/node_modules/*' -type f -name '*.ts') site/package.json ./scripts/yarn_install.sh - cd site && yarn typegen - cd site && yarn build + cd site + yarn typegen + yarn build # Restores GITKEEP files! - git checkout HEAD site/out + git checkout HEAD out site/src/api/typesGenerated.ts: scripts/apitypings/main.go $(shell find codersdk -type f -name '*.go') go run scripts/apitypings/main.go > site/src/api/typesGenerated.ts - cd site && yarn run format:types + cd site + yarn run format:types test: test-clean gotestsum -- -v -short ./... From 4f03e370beee3ed3962eb41b6710fdb112eb1184 Mon Sep 17 00:00:00 2001 From: deansheather Date: Sun, 12 Jun 2022 22:37:17 +0000 Subject: [PATCH 14/36] fixup! chore: update release workflow to use new build scripts --- .github/workflows/release.yaml | 2 +- scripts/lib.sh | 4 +++- scripts/version.sh | 9 ++++++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 862aa2a62e01e..916dd886f3643 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -24,7 +24,7 @@ on: # In a non-snapshot we want the ./scripts/version.sh script to never generate a # +dev version. env: - CODER_NO_DEV_VERSION: ${{ github.event.inputs.snapshot && '' || 'true' }} + CODER_NO_DEV_VERSION: ${{ github.event.inputs.snapshot && 'false' || 'true' }} jobs: linux-windows: diff --git a/scripts/lib.sh b/scripts/lib.sh index 69173d6276f2b..7bddeacbc333a 100644 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -57,8 +57,10 @@ cdroot() { # be sourced by other scripts. execrelative() { pushd "$SCRIPT_DIR" || error "Could not change directory to '$SCRIPT_DIR'" - "$@" + rc=0 + "$@" || rc=$? popd + return $rc } # maybedryrun prints the given program and flags, and then, if the first diff --git a/scripts/version.sh b/scripts/version.sh index 3bf23ce7d44ec..3e49fe1840e93 100755 --- a/scripts/version.sh +++ b/scripts/version.sh @@ -20,7 +20,14 @@ version="$last_tag" # Dev versions are denoted by the "+dev." suffix with a trailing commit short # SHA. if [[ "$last_tag" != "$current" ]]; then - if [[ "${CODER_NO_DEV_VERSION:-}" != "" ]]; then + if [[ "${CODER_NO_DEV_VERSION:-}" == *t* ]]; then + # make won't exit on $(shell cmd) failures :( + if [[ "$(ps -o comm= "$PPID" || true)" == *make* ]]; then + log "ERROR: version.sh attemped to generate a dev version string when CODER_NO_DEV_VERSION was set" + kill "$PPID" + exit 1 + fi + error "version.sh attemped to generate a dev version string when CODER_NO_DEV_VERSION was set" fi version+="+dev.$(git rev-parse --short HEAD)" From 5c9dd9e02e5b7872dfe17271cca2bb2be9db6c44 Mon Sep 17 00:00:00 2001 From: deansheather Date: Sun, 12 Jun 2022 23:07:05 +0000 Subject: [PATCH 15/36] fixup! chore: update release workflow to use new build scripts --- .github/workflows/release.yaml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 916dd886f3643..e48fcfcda2364 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -53,7 +53,14 @@ jobs: run: make site/out/index.html - name: Build Slim Binaries - run: make bin + run: | + mkdir -p ./dist + ./scripts/build_go_slim.sh \ + --version "$(VERSION)" \ + --output ./dist/ \ + linux:amd64,armv7,arm64 \ + windows:amd64,arm64 \ + darwin:amd64,arm64 - name: Build Linux and Windows Binaries run: | @@ -120,7 +127,14 @@ jobs: run: make site/out/index.html - name: Build Slim Binaries - run: make bin + run: | + mkdir -p ./dist + ./scripts/build_go_slim.sh \ + --version "$(VERSION)" \ + --output ./dist/ \ + linux:amd64,armv7,arm64 \ + windows:amd64,arm64 \ + darwin:amd64,arm64 - name: Build darwin Binaries (with signatures) run: | From e09edfdaefa70dc0f1e866c0cded00178396b5a6 Mon Sep 17 00:00:00 2001 From: deansheather Date: Mon, 13 Jun 2022 15:51:01 +0000 Subject: [PATCH 16/36] fixup! chore: update release workflow to use new build scripts --- .github/.goreleaser-release-darwin.yaml | 58 ------------------------- .github/workflows/release.yaml | 12 ++--- scripts/build_go_matrix.sh | 6 ++- 3 files changed, 11 insertions(+), 65 deletions(-) delete mode 100644 .github/.goreleaser-release-darwin.yaml diff --git a/.github/.goreleaser-release-darwin.yaml b/.github/.goreleaser-release-darwin.yaml deleted file mode 100644 index 13825ece8aeeb..0000000000000 --- a/.github/.goreleaser-release-darwin.yaml +++ /dev/null @@ -1,58 +0,0 @@ -before: - hooks: - - go mod tidy - - rm -f site/out/bin/coder* - -archives: - - id: coder-darwin - builds: [coder-darwin] - format: zip - -builds: - - id: coder-slim - dir: cmd/coder - ldflags: ["-s -w -X github.com/coder/coder/buildinfo.tag={{ .Version }}"] - env: [CGO_ENABLED=0] - goos: [darwin, linux, windows] - goarch: [amd64, arm, arm64] - goarm: ["7"] - # Only build arm 7 for Linux - ignore: - - goos: windows - goarm: "7" - - goos: darwin - goarm: "7" - hooks: - # The "trimprefix" appends ".exe" on Windows. - post: | - cp {{.Path}} site/out/bin/coder-{{ .Os }}-{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}{{ trimprefix .Name "coder" }} - - - id: coder-darwin - dir: cmd/coder - flags: [-tags=embed] - ldflags: ["-s -w -X github.com/coder/coder/buildinfo.tag={{ .Version }}"] - env: [CGO_ENABLED=0] - goos: [darwin] - goarch: [amd64, arm64] - hooks: - # This signs the binary that will be located inside the zip. MacOS - # requires the binary to be signed for notarization. - post: | - sh -c 'codesign -s {{.Env.AC_APPLICATION_IDENTITY}} -f -v --timestamp --options runtime {{.Path}}' - -env: - # Apple identity for signing! - - AC_APPLICATION_IDENTITY=BDB050EB749EDD6A80C6F119BF1382ECA119CCCC - -signs: - - ids: [coder-darwin] - artifacts: archive - cmd: ./scripts/sign_darwin.sh - args: ["${artifact}"] - output: true - -release: - ids: [coder-darwin] - -snapshot: - name_template: "{{ .Version }}-devel+{{ .ShortCommit }}" diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e48fcfcda2364..18b2f4bbd0b27 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -52,9 +52,10 @@ jobs: - name: Build Site run: make site/out/index.html - - name: Build Slim Binaries + - name: Build Linux and Windows Binaries run: | mkdir -p ./dist + # build slim binaries ./scripts/build_go_slim.sh \ --version "$(VERSION)" \ --output ./dist/ \ @@ -62,8 +63,7 @@ jobs: windows:amd64,arm64 \ darwin:amd64,arm64 - - name: Build Linux and Windows Binaries - run: | + # build linux and windows binaries ./scripts/build_go_matrix.sh \ --output ./dist/ \ --archive \ @@ -126,9 +126,10 @@ jobs: - name: Build Site run: make site/out/index.html - - name: Build Slim Binaries + - name: Build darwin Binaries (with signatures) run: | mkdir -p ./dist + # build slim binaries ./scripts/build_go_slim.sh \ --version "$(VERSION)" \ --output ./dist/ \ @@ -136,8 +137,7 @@ jobs: windows:amd64,arm64 \ darwin:amd64,arm64 - - name: Build darwin Binaries (with signatures) - run: | + # build darwin binaries ./scripts/build_go_matrix.sh \ --output ./dist/ \ --archive \ diff --git a/scripts/build_go_matrix.sh b/scripts/build_go_matrix.sh index 569865e1a32dc..0720376e6e82b 100755 --- a/scripts/build_go_matrix.sh +++ b/scripts/build_go_matrix.sh @@ -126,7 +126,11 @@ for spec in "$@"; do done # Remove duplicate specs while maintaining the same order. -mapfile -t specs < <(echo "${specs[@]}" | tr " " "\n" | awk '!a[$0]++') +specs_str="${specs[*]}" +specs=() +for s in $(echo "$specs_str" | tr " " "\n" | awk '!a[$0]++'); do + specs+=("$s") +done build_args=() if [[ "$slim" == 1 ]]; then From 4dd6f4df0f937dce917cfeba8001db9a1d4bc00e Mon Sep 17 00:00:00 2001 From: deansheather Date: Mon, 13 Jun 2022 17:15:53 +0000 Subject: [PATCH 17/36] chore: add dependency checks to release scripts --- .github/workflows/release.yaml | 3 +++ scripts/archive.sh | 19 ++++++++++++++++ scripts/build_docker.sh | 5 +++++ scripts/build_docker_multiarch.sh | 13 +++++++---- scripts/build_go.sh | 16 ++++++++++++++ scripts/build_go_matrix.sh | 36 +++++++++++++++++++++++++++++++ scripts/build_go_slim.sh | 5 +++++ scripts/package.sh | 5 +++++ scripts/publish_release.sh | 5 +++++ scripts/sign_darwin.sh | 11 ++++++++++ 10 files changed, 114 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 18b2f4bbd0b27..85aecddd4f60f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -49,6 +49,9 @@ jobs: restore-keys: | js-${{ runner.os }}- + - name: Install nfpm + run: go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.16.0 + - name: Build Site run: make site/out/index.html diff --git a/scripts/archive.sh b/scripts/archive.sh index 7498ff750f57b..e2155c44b79a9 100755 --- a/scripts/archive.sh +++ b/scripts/archive.sh @@ -72,6 +72,25 @@ if [[ ! -f "$1" ]]; then fi input_file="$(realpath "$1")" +# Check dependencies +if [[ "$format" == "zip" ]] && ! command -v zip; then + error "The 'zip' binary is required." +fi +if [[ "$format" == "tar.gz" ]] && ! command -v tar; then + error "The 'tar' binary is required." +fi +if [[ "$sign_darwin" == 1 ]]; then + if ! command -v jq; then + error "The 'jq' binary is required." + fi + if ! command -v codesign; then + error "The 'codesign' binary is required." + fi + if ! command -v gon; then + error "The 'gon' binary is required." + fi +fi + # Determine default output path. if [[ "$output_path" == "" ]]; then output_path="${input_file%.exe}" diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh index b528b44ad78db..22093d4a71a48 100755 --- a/scripts/build_docker.sh +++ b/scripts/build_docker.sh @@ -56,6 +56,11 @@ if [[ "$arch" == "" ]]; then error "The --arch parameter is required" fi +# Check dependencies +if ! command -v docker; then + error "The 'docker' binary is required." +fi + # Remove the "v" prefix. version="${version#v}" if [[ "$version" == "" ]]; then diff --git a/scripts/build_docker_multiarch.sh b/scripts/build_docker_multiarch.sh index 7b2648b688904..0ba66094912b3 100755 --- a/scripts/build_docker_multiarch.sh +++ b/scripts/build_docker_multiarch.sh @@ -44,16 +44,21 @@ while true; do esac done +if [[ "$#" == 0 ]]; then + error "At least one argument must be provided to this script, $# were supplied" +fi + +# Check dependencies +if ! command -v docker; then + error "The 'docker' binary is required." +fi + # Remove the "v" prefix. version="${version#v}" if [[ "$version" == "" ]]; then version="$(execrelative ./version.sh)" fi -if [[ "$#" == 0 ]]; then - error "At least one argument must be provided to this script, $# were supplied" -fi - create_args=() for image_tag in "$@"; do create_args+=(--amend "$image_tag") diff --git a/scripts/build_go.sh b/scripts/build_go.sh index 9fb680cbb7dcd..6d9696c7957e1 100755 --- a/scripts/build_go.sh +++ b/scripts/build_go.sh @@ -79,6 +79,22 @@ if [[ "$version" == "" ]]; then version="$(execrelative ./version.sh)" fi +# Check dependencies +if ! command -v go; then + error "The 'go' binary is required." +fi +if [[ "$sign_darwin" == 1 ]]; then + if ! command -v jq; then + error "The 'jq' binary is required." + fi + if ! command -v codesign; then + error "The 'codesign' binary is required." + fi + if ! command -v gon; then + error "The 'gon' binary is required." + fi +fi + build_args=( -ldflags "-s -w -X 'github.com/coder/coder/buildinfo.tag=$version'" ) diff --git a/scripts/build_go_matrix.sh b/scripts/build_go_matrix.sh index 0720376e6e82b..218ff5a2e14b0 100755 --- a/scripts/build_go_matrix.sh +++ b/scripts/build_go_matrix.sh @@ -109,12 +109,21 @@ fi # Parse the os:arch specs into an array. specs=() +may_zip=0 +may_tar=0 for spec in "$@"; do spec_os="$(echo "$spec" | cut -d ":" -f 1)" if [[ "$spec_os" == "" ]] || [[ "$spec_os" == *" "* ]]; then error "Could not parse matrix build spec '$spec': invalid OS '$spec_os'" fi + # Determine which dependencies we need. + if [[ "$spec_os" == "windows" ]] || [[ "$spec_os" == "darwin" ]]; then + may_zip=1 + else + may_tar=1 + fi + # No quoting is important here. for spec_arch in $(echo "$spec" | cut -d ":" -f 2 | tr "," "\n"); do if [[ "$spec_arch" == "" ]] || [[ "$spec_os" == *" "* ]]; then @@ -132,6 +141,33 @@ for s in $(echo "$specs_str" | tr " " "\n" | awk '!a[$0]++'); do specs+=("$s") done +# Check dependencies +if ! command -v go; then + error "The 'go' binary is required." +fi +if [[ "$sign_darwin" == 1 ]]; then + if ! command -v jq; then + error "The 'jq' binary is required." + fi + if ! command -v codesign; then + error "The 'codesign' binary is required." + fi + if ! command -v gon; then + error "The 'gon' binary is required." + fi +fi +if [[ "$archive" == 1 ]]; then + if [[ "$may_zip" == 1 ]] && ! command -v zip; then + error "The 'zip' binary is required." + fi + if [[ "$may_tar" == 1 ]] && ! command -v tar; then + error "The 'zip' binary is required." + fi +fi +if [[ "$package_linux" == 1 ]] && ! command -v nfpm; then + error "The 'nfpm' binary is required." +fi + build_args=() if [[ "$slim" == 1 ]]; then build_args+=(--slim) diff --git a/scripts/build_go_slim.sh b/scripts/build_go_slim.sh index c5a6d6d00d966..4dbb82fad7167 100755 --- a/scripts/build_go_slim.sh +++ b/scripts/build_go_slim.sh @@ -45,6 +45,11 @@ while true; do esac done +# Check dependencies +if ! command -v go; then + error "The 'go' binary is required." +fi + # Remove the "v" prefix. version="${version#v}" if [[ "$version" == "" ]]; then diff --git a/scripts/package.sh b/scripts/package.sh index 1dc54a41dad59..99de7062142ea 100755 --- a/scripts/package.sh +++ b/scripts/package.sh @@ -50,6 +50,11 @@ if [[ ! -f "$1" ]]; then fi input_file="$(realpath "$1")" +# Check dependencies +if ! command -v nfpm; then + error "The 'nfpm' binary is required." +fi + # Remove the "v" prefix. version="${version#v}" if [[ "$version" == "" ]]; then diff --git a/scripts/publish_release.sh b/scripts/publish_release.sh index 4d2032220ff7a..c8db386682149 100755 --- a/scripts/publish_release.sh +++ b/scripts/publish_release.sh @@ -50,6 +50,11 @@ while true; do esac done +# Check dependencies +if ! command -v gh; then + error "The 'gh' binary is required." +fi + # Remove the "v" prefix. version="${version#v}" if [[ "$version" == "" ]]; then diff --git a/scripts/sign_darwin.sh b/scripts/sign_darwin.sh index 84ec406e17fee..ce79ba47c7ea2 100755 --- a/scripts/sign_darwin.sh +++ b/scripts/sign_darwin.sh @@ -20,6 +20,17 @@ if [[ "${AC_APPLICATION_IDENTITY:-}" == "" ]]; then error "AC_APPLICATION_IDENTITY must be set for ./sign_darwin.sh" fi +# Check dependencies +if ! command -v jq; then + error "The 'jq' binary is required." +fi +if ! command -v codesign; then + error "The 'codesign' binary is required." +fi +if ! command -v gon; then + error "The 'gon' binary is required." +fi + # Create the gon config. config="$(mktemp -d)/gon.json" jq -r --null-input --arg path "$(pwd)/$1" '{ From 72d8b506f25fd7af33096a8e00cdebc7bd39b990 Mon Sep 17 00:00:00 2001 From: deansheather Date: Mon, 13 Jun 2022 17:37:05 +0000 Subject: [PATCH 18/36] fixup! chore: add dependency checks to release scripts --- .github/workflows/release.yaml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 85aecddd4f60f..3beb8c354b185 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -99,11 +99,6 @@ jobs: with: go-version: "~1.18" - - name: Install gon - run: | - brew tap mitchellh/gon - brew install mitchellh/gon/gon - - name: Import Signing Certificates uses: Apple-Actions/import-codesign-certs@v1 with: @@ -121,11 +116,21 @@ jobs: restore-keys: | js-${{ runner.os }}- - - name: Install make + - name: Install dependencies run: | + brew install bash + brew install make echo "$(brew --prefix)/opt/make/libexec/gnubin" >> $GITHUB_PATH + brew tap mitchellh/gon + brew install mitchellh/gon/gon + + - name: which bash + run: | + which bash + echo "$0" + - name: Build Site run: make site/out/index.html From 17cda1fcd7038f2c1d8b0246cbaeb7200f2f192c Mon Sep 17 00:00:00 2001 From: deansheather Date: Mon, 13 Jun 2022 17:38:22 +0000 Subject: [PATCH 19/36] fixup! chore: add dependency checks to release scripts --- .github/workflows/release.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 3beb8c354b185..ebe14800c853a 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -116,6 +116,11 @@ jobs: restore-keys: | js-${{ runner.os }}- + - name: which bash + run: | + which bash + echo "$0" + - name: Install dependencies run: | brew install bash From 6a4bc44fdba24f70c972a6728987d55eb72df83d Mon Sep 17 00:00:00 2001 From: deansheather Date: Mon, 13 Jun 2022 18:28:47 +0000 Subject: [PATCH 20/36] fixup! chore: add dependency checks to release scripts --- .github/workflows/release.yaml | 3 +++ scripts/build_go_slim.sh | 1 + 2 files changed, 4 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ebe14800c853a..1c046c6929ead 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -128,6 +128,9 @@ jobs: brew install make echo "$(brew --prefix)/opt/make/libexec/gnubin" >> $GITHUB_PATH + brew install gnu-getopt + echo "$(brew --prefix)/opt/gnu-getopt/bin" >> $GITHUB_PATH + brew tap mitchellh/gon brew install mitchellh/gon/gon diff --git a/scripts/build_go_slim.sh b/scripts/build_go_slim.sh index 4dbb82fad7167..735ffb19045e3 100755 --- a/scripts/build_go_slim.sh +++ b/scripts/build_go_slim.sh @@ -17,6 +17,7 @@ # they can be packaged into non-slim binaries correctly. set -euo pipefail +shopt -s nullglob # shellcheck source=scripts/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" From e150438bc683ede71fa630bd491436f13464cec2 Mon Sep 17 00:00:00 2001 From: deansheather Date: Mon, 13 Jun 2022 18:51:27 +0000 Subject: [PATCH 21/36] fixup! chore: add dependency checks to release scripts --- .github/workflows/release.yaml | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 1c046c6929ead..9d8ca35dc96b7 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -116,29 +116,23 @@ jobs: restore-keys: | js-${{ runner.os }}- - - name: which bash - run: | - which bash - echo "$0" - - name: Install dependencies run: | + # The version of bash that MacOS ships with is too old brew install bash + # The version of make that MacOS ships with is too old brew install make echo "$(brew --prefix)/opt/make/libexec/gnubin" >> $GITHUB_PATH + # BSD getopt is incompatible with the build scripts brew install gnu-getopt echo "$(brew --prefix)/opt/gnu-getopt/bin" >> $GITHUB_PATH + # Used for notarizing the binaries brew tap mitchellh/gon brew install mitchellh/gon/gon - - name: which bash - run: | - which bash - echo "$0" - - name: Build Site run: make site/out/index.html @@ -162,6 +156,7 @@ jobs: env: AC_USERNAME: ${{ secrets.AC_USERNAME }} AC_PASSWORD: ${{ secrets.AC_PASSWORD }} + AC_APPLICATION_IDENTITY: BDB050EB749EDD6A80C6F119BF1382ECA119CCCC - name: Upload Binary Artifacts uses: actions/upload-artifact@v3 From 449a1a1e8db7cd7fcfc41ac6f3ddb969294ad78f Mon Sep 17 00:00:00 2001 From: deansheather Date: Tue, 14 Jun 2022 18:05:09 +0000 Subject: [PATCH 22/36] chore: make dependency checks nicer --- scripts/archive.sh | 18 +++++------------- scripts/build_docker.sh | 4 +--- scripts/build_docker_multiarch.sh | 4 +--- scripts/build_go.sh | 14 ++------------ scripts/build_go_matrix.sh | 30 +++++++++++------------------- scripts/build_go_slim.sh | 4 +--- scripts/lib.sh | 20 +++++++++++++++++++- scripts/package.sh | 4 +--- scripts/publish_release.sh | 4 +--- scripts/sign_darwin.sh | 16 +++++----------- scripts/version.sh | 2 +- 11 files changed, 48 insertions(+), 72 deletions(-) diff --git a/scripts/archive.sh b/scripts/archive.sh index e2155c44b79a9..c8722c0c0c4eb 100755 --- a/scripts/archive.sh +++ b/scripts/archive.sh @@ -73,22 +73,14 @@ fi input_file="$(realpath "$1")" # Check dependencies -if [[ "$format" == "zip" ]] && ! command -v zip; then - error "The 'zip' binary is required." +if [[ "$format" == "zip" ]]; then + dependencies zip fi -if [[ "$format" == "tar.gz" ]] && ! command -v tar; then - error "The 'tar' binary is required." +if [[ "$format" == "tar.gz" ]]; then + dependencies tar fi if [[ "$sign_darwin" == 1 ]]; then - if ! command -v jq; then - error "The 'jq' binary is required." - fi - if ! command -v codesign; then - error "The 'codesign' binary is required." - fi - if ! command -v gon; then - error "The 'gon' binary is required." - fi + dependencies jq codesign gon fi # Determine default output path. diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh index 22093d4a71a48..cd5749f02a14b 100755 --- a/scripts/build_docker.sh +++ b/scripts/build_docker.sh @@ -57,9 +57,7 @@ if [[ "$arch" == "" ]]; then fi # Check dependencies -if ! command -v docker; then - error "The 'docker' binary is required." -fi +dependencies docker # Remove the "v" prefix. version="${version#v}" diff --git a/scripts/build_docker_multiarch.sh b/scripts/build_docker_multiarch.sh index 0ba66094912b3..40255cf92529f 100755 --- a/scripts/build_docker_multiarch.sh +++ b/scripts/build_docker_multiarch.sh @@ -49,9 +49,7 @@ if [[ "$#" == 0 ]]; then fi # Check dependencies -if ! command -v docker; then - error "The 'docker' binary is required." -fi +dependencies docker # Remove the "v" prefix. version="${version#v}" diff --git a/scripts/build_go.sh b/scripts/build_go.sh index 6d9696c7957e1..df1448064d702 100755 --- a/scripts/build_go.sh +++ b/scripts/build_go.sh @@ -80,19 +80,9 @@ if [[ "$version" == "" ]]; then fi # Check dependencies -if ! command -v go; then - error "The 'go' binary is required." -fi +dependencies go if [[ "$sign_darwin" == 1 ]]; then - if ! command -v jq; then - error "The 'jq' binary is required." - fi - if ! command -v codesign; then - error "The 'codesign' binary is required." - fi - if ! command -v gon; then - error "The 'gon' binary is required." - fi + dependencies codesign fi build_args=( diff --git a/scripts/build_go_matrix.sh b/scripts/build_go_matrix.sh index 218ff5a2e14b0..b3815597c6f97 100755 --- a/scripts/build_go_matrix.sh +++ b/scripts/build_go_matrix.sh @@ -142,34 +142,26 @@ for s in $(echo "$specs_str" | tr " " "\n" | awk '!a[$0]++'); do done # Check dependencies -if ! command -v go; then - error "The 'go' binary is required." -fi +dependencies go if [[ "$sign_darwin" == 1 ]]; then - if ! command -v jq; then - error "The 'jq' binary is required." - fi - if ! command -v codesign; then - error "The 'codesign' binary is required." - fi - if ! command -v gon; then - error "The 'gon' binary is required." - fi + dependencies jq codesign gon fi if [[ "$archive" == 1 ]]; then - if [[ "$may_zip" == 1 ]] && ! command -v zip; then - error "The 'zip' binary is required." + if [[ "$may_zip" == 1 ]]; then + dependencies zip fi - if [[ "$may_tar" == 1 ]] && ! command -v tar; then - error "The 'zip' binary is required." + if [[ "$may_tar" == 1 ]]; then + dependencies tar fi fi -if [[ "$package_linux" == 1 ]] && ! command -v nfpm; then - error "The 'nfpm' binary is required." +if [[ "$package_linux" == 1 ]]; then + dependencies nfpm fi +bin_name="coder" build_args=() if [[ "$slim" == 1 ]]; then + bin_name+="-slim" build_args+=(--slim) fi if [[ "$sign_darwin" == 1 ]]; then @@ -195,7 +187,7 @@ for spec in "${specs[@]}"; do # Ensure parent dir. mkdir -p "$(dirname "$spec_output")" - log "--- Building coder for $spec_os $spec_arch ($spec_output_binary)" + log "--- Building $bin_name for $spec_os $spec_arch ($spec_output_binary)" execrelative ./build_go.sh \ --version "$version" \ --os "$spec_os" \ diff --git a/scripts/build_go_slim.sh b/scripts/build_go_slim.sh index 735ffb19045e3..e060c399c3fc5 100755 --- a/scripts/build_go_slim.sh +++ b/scripts/build_go_slim.sh @@ -47,9 +47,7 @@ while true; do done # Check dependencies -if ! command -v go; then - error "The 'go' binary is required." -fi +dependencies go # Remove the "v" prefix. version="${version#v}" diff --git a/scripts/lib.sh b/scripts/lib.sh index 7bddeacbc333a..b0b42a01cb02f 100644 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -15,8 +15,11 @@ set -euo pipefail # # Taken from https://stackoverflow.com/a/3915420 (CC-BY-SA 4.0) realpath() { + local dir + local base dir="$(dirname "$1")" base="$(basename "$1")" + if [[ ! -d "$dir" ]]; then error "Could not change directory to '$dir': directory does not exist" fi @@ -57,12 +60,27 @@ cdroot() { # be sourced by other scripts. execrelative() { pushd "$SCRIPT_DIR" || error "Could not change directory to '$SCRIPT_DIR'" - rc=0 + local rc=0 "$@" || rc=$? popd return $rc } +dependencies() { + local fail=0 + for dep in "$@"; do + if ! command -v "$dep" >/dev/null; then + log "ERROR: The '$dep' dependency is required, but is not available." + fail=1 + fi + done + + if [[ "$fail" == 1 ]]; then + log + error "One or more dependencies are not available, check above log output for more details." + fi +} + # maybedryrun prints the given program and flags, and then, if the first # argument is 0, executes it. The reason the first argument should be 0 is that # it is expected that you have a dry_run variable in your script that is set to diff --git a/scripts/package.sh b/scripts/package.sh index 99de7062142ea..9c0c0af3f618b 100755 --- a/scripts/package.sh +++ b/scripts/package.sh @@ -51,9 +51,7 @@ fi input_file="$(realpath "$1")" # Check dependencies -if ! command -v nfpm; then - error "The 'nfpm' binary is required." -fi +dependencies nfpm # Remove the "v" prefix. version="${version#v}" diff --git a/scripts/publish_release.sh b/scripts/publish_release.sh index c8db386682149..f13e0fe7db27a 100755 --- a/scripts/publish_release.sh +++ b/scripts/publish_release.sh @@ -51,9 +51,7 @@ while true; do done # Check dependencies -if ! command -v gh; then - error "The 'gh' binary is required." -fi +dependencies gh # Remove the "v" prefix. version="${version#v}" diff --git a/scripts/sign_darwin.sh b/scripts/sign_darwin.sh index ce79ba47c7ea2..41f178048c4e6 100755 --- a/scripts/sign_darwin.sh +++ b/scripts/sign_darwin.sh @@ -21,19 +21,13 @@ if [[ "${AC_APPLICATION_IDENTITY:-}" == "" ]]; then fi # Check dependencies -if ! command -v jq; then - error "The 'jq' binary is required." -fi -if ! command -v codesign; then - error "The 'codesign' binary is required." -fi -if ! command -v gon; then - error "The 'gon' binary is required." -fi +dependencies jq codesign gon + +output_path="$1" # Create the gon config. config="$(mktemp -d)/gon.json" -jq -r --null-input --arg path "$(pwd)/$1" '{ +jq -r --null-input --arg path "$output_path" '{ "notarize": [ { "path": $path, @@ -43,7 +37,7 @@ jq -r --null-input --arg path "$(pwd)/$1" '{ }' >"$config" # Sign the zip file with our certificate. -codesign -s "$AC_APPLICATION_IDENTITY" -f -v --timestamp --options runtime "$1" +codesign -s "$AC_APPLICATION_IDENTITY" -f -v --timestamp --options runtime "$output_path" # Notarize the signed zip file. # diff --git a/scripts/version.sh b/scripts/version.sh index 3e49fe1840e93..fa1d8963991d0 100755 --- a/scripts/version.sh +++ b/scripts/version.sh @@ -24,7 +24,7 @@ if [[ "$last_tag" != "$current" ]]; then # make won't exit on $(shell cmd) failures :( if [[ "$(ps -o comm= "$PPID" || true)" == *make* ]]; then log "ERROR: version.sh attemped to generate a dev version string when CODER_NO_DEV_VERSION was set" - kill "$PPID" + kill "$PPID" || true exit 1 fi From ba780768f4425483a283308dc3d422d8c8a8ea17 Mon Sep 17 00:00:00 2001 From: deansheather Date: Tue, 14 Jun 2022 18:44:29 +0000 Subject: [PATCH 23/36] fixup! chore: make dependency checks nicer --- .github/workflows/release.yaml | 6 ++++-- scripts/archive.sh | 2 +- scripts/build_docker.sh | 2 +- scripts/build_docker_multiarch.sh | 2 +- scripts/build_go.sh | 2 +- scripts/image_tag.sh | 2 +- scripts/version.sh | 2 +- 7 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 9d8ca35dc96b7..a98493d18d83a 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -57,10 +57,11 @@ jobs: - name: Build Linux and Windows Binaries run: | + go mod download + mkdir -p ./dist # build slim binaries ./scripts/build_go_slim.sh \ - --version "$(VERSION)" \ --output ./dist/ \ linux:amd64,armv7,arm64 \ windows:amd64,arm64 \ @@ -138,10 +139,11 @@ jobs: - name: Build darwin Binaries (with signatures) run: | + go mod download + mkdir -p ./dist # build slim binaries ./scripts/build_go_slim.sh \ - --version "$(VERSION)" \ --output ./dist/ \ linux:amd64,armv7,arm64 \ windows:amd64,arm64 \ diff --git a/scripts/archive.sh b/scripts/archive.sh index c8722c0c0c4eb..9190b142a0c50 100755 --- a/scripts/archive.sh +++ b/scripts/archive.sh @@ -124,4 +124,4 @@ if [[ "$sign_darwin" == 1 ]]; then execrelative ./sign_darwin.sh "$output_path" fi -echo -n "$output_path" +echo "$output_path" diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh index cd5749f02a14b..8508e95771e7d 100755 --- a/scripts/build_docker.sh +++ b/scripts/build_docker.sh @@ -111,4 +111,4 @@ if [[ "$push" == 1 ]]; then docker push "$image_tag" fi -echo -n "$image_tag" +echo "$image_tag" diff --git a/scripts/build_docker_multiarch.sh b/scripts/build_docker_multiarch.sh index 40255cf92529f..37dd4ddf2885b 100755 --- a/scripts/build_docker_multiarch.sh +++ b/scripts/build_docker_multiarch.sh @@ -74,4 +74,4 @@ if [[ "$push" == 1 ]]; then docker push "$output_tag" fi -echo -n "$output_tag" +echo "$output_tag" diff --git a/scripts/build_go.sh b/scripts/build_go.sh index df1448064d702..739b5def6bbbc 100755 --- a/scripts/build_go.sh +++ b/scripts/build_go.sh @@ -123,4 +123,4 @@ if [[ "$sign_darwin" == 1 ]] && [[ "$os" == "darwin" ]]; then codesign -s "$AC_APPLICATION_IDENTITY" -f -v --timestamp --options runtime "$output_path" fi -echo -n "$output_path" +echo "$output_path" diff --git a/scripts/image_tag.sh b/scripts/image_tag.sh index 43dc16b67e38a..674c3193b4686 100755 --- a/scripts/image_tag.sh +++ b/scripts/image_tag.sh @@ -55,4 +55,4 @@ if [[ "$arch" != "" ]]; then fi tag="${tag//+/-}" -echo -n "$image:$tag" +echo "$image:$tag" diff --git a/scripts/version.sh b/scripts/version.sh index fa1d8963991d0..d6a884501cb3a 100755 --- a/scripts/version.sh +++ b/scripts/version.sh @@ -34,4 +34,4 @@ if [[ "$last_tag" != "$current" ]]; then fi # Remove the "v" prefix. -echo -n "${version#v}" +echo "${version#v}" From a19ef36d871d0a9e82f7307e495028d25e633466 Mon Sep 17 00:00:00 2001 From: deansheather Date: Tue, 14 Jun 2022 18:51:39 +0000 Subject: [PATCH 24/36] fixup! chore: make dependency checks nicer --- scripts/publish_release.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/publish_release.sh b/scripts/publish_release.sh index f13e0fe7db27a..52dc3f96b1306 100755 --- a/scripts/publish_release.sh +++ b/scripts/publish_release.sh @@ -70,13 +70,20 @@ if [[ "$(git describe --always)" != "$new_tag" ]]; then fi # This returns the tag before the current tag. -old_tag="$(git describe --abbrev=0 --tags "$(git rev-list --tags --skip=1 --max-count=1)")" +old_tag="$(git describe --abbrev=0 HEAD^1)" + +# For dry-run builds we want to use the SHA instead of the tag, because the new +# tag probably doesn't exist. +changelog_range="$old_tag..$new_tag" +if [[ "$dry_run" == 1 ]]; then + changelog_range="$old_tag..$(git rev-parse --short HEAD)" +fi # Craft the release notes. release_notes=" ## Changelog -$(git log --no-merges --pretty=format:"- %h %s" "$old_tag..$new_tag") +$(git log --no-merges --pretty=format:"- %h %s" "$changelog_range") ## Container Image - \`docker pull $(execrelative ./image_tag.sh --version "$version")\` From 94a6fcd1a798146cace6ab59f6b8ec8afa323bda Mon Sep 17 00:00:00 2001 From: deansheather Date: Tue, 14 Jun 2022 19:09:35 +0000 Subject: [PATCH 25/36] fixup! chore: make dependency checks nicer --- scripts/publish_release.sh | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/scripts/publish_release.sh b/scripts/publish_release.sh index 52dc3f96b1306..86457773c6ee5 100755 --- a/scripts/publish_release.sh +++ b/scripts/publish_release.sh @@ -59,6 +59,21 @@ if [[ "$version" == "" ]]; then version="$(execrelative ./version.sh)" fi +# realpath-ify all input files so we can cdroot below. +files=() +for f in "$@"; do + if [[ ! -e "$f" ]]; then + error "File not found: $f" + fi + files+=("$(realpath "$f")") +done +if [[ "${#files[@]}" == 0 ]]; then + error "No files supplied" +fi + +# The git commands need to be executed from within the repository. +cdroot + # Verify that we're currently checked out on the supplied tag. new_tag="v$version" if [[ "$(git describe --always)" != "$new_tag" ]]; then @@ -80,24 +95,26 @@ if [[ "$dry_run" == 1 ]]; then fi # Craft the release notes. +changelog="$(git log --no-merges --pretty=format:"- %h %s" "$changelog_range")" +image_tag="$(execrelative ./image_tag.sh --version "$version")" release_notes=" ## Changelog -$(git log --no-merges --pretty=format:"- %h %s" "$changelog_range") +$changelog ## Container Image -- \`docker pull $(execrelative ./image_tag.sh --version "$version")\` +- \`docker pull $image_tag\` " -# Create temporary release folder so we can generate checksums. +# Create temporary release folder so we can generate checksums. Both the +# sha256sum and gh binaries support symlinks as input files so this works well. temp_dir="$(mktemp -d)" -for f in "$@"; do - ln -s "$(realpath "$f")" "$temp_dir/" +for f in "${files[@]}"; do + ln -s "$f" "$temp_dir/" done -# Generate checksums file. sha256sum seems to play nicely with symlinks so this -# works well. +# Generate checksums file which will be uploaded to the GitHub release. pushd "$temp_dir" sha256sum ./* | sed -e 's/\.\///' - >"coder_${version}_checksums.txt" popd @@ -116,8 +133,6 @@ log # We echo the release notes in instead of writing to a file and referencing that # to prevent GitHub CLI from becoming interactive. -# -# GitHub CLI seems to follow symlinks when uploading files. echo "$release_notes" | maybedryrun "$dry_run" gh release create \ --title "$new_tag" \ From c5245072394b5f8e3eea9728ec31ce0d7215000b Mon Sep 17 00:00:00 2001 From: deansheather Date: Tue, 14 Jun 2022 19:26:08 +0000 Subject: [PATCH 26/36] fixup! chore: make dependency checks nicer --- .github/workflows/release.yaml | 3 ++- scripts/build_go.sh | 2 +- scripts/build_go_matrix.sh | 2 +- scripts/build_go_slim.sh | 2 +- scripts/image_tag.sh | 1 + scripts/version.sh | 12 +++++++++--- 6 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index a98493d18d83a..5c7570afe08ed 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -13,7 +13,7 @@ on: workflow_dispatch: inputs: snapshot: - description: Allow a +dev version to be generated, implies dry_run. + description: Force a dev version to be generated, implies dry_run. type: boolean required: true dry_run: @@ -24,6 +24,7 @@ on: # In a non-snapshot we want the ./scripts/version.sh script to never generate a # +dev version. env: + CODER_FORCE_DEV_VERSION: ${{ github.event.inputs.snapshot && 'true' || 'false' }} CODER_NO_DEV_VERSION: ${{ github.event.inputs.snapshot && 'false' || 'true' }} jobs: diff --git a/scripts/build_go.sh b/scripts/build_go.sh index 739b5def6bbbc..ca29cd04e2beb 100755 --- a/scripts/build_go.sh +++ b/scripts/build_go.sh @@ -2,7 +2,7 @@ # This script builds a single Go binary of Coder with the given parameters. # -# Usage: ./build_go.sh [--version 1.2.3+devel.abcdef] [--os linux] [--arch amd64] [--output path/to/output] [--slim] +# Usage: ./build_go.sh [--version 1.2.3-devel+abcdef] [--os linux] [--arch amd64] [--output path/to/output] [--slim] # # Defaults to linux:amd64 with slim disabled, but can be controlled with GOOS, # GOARCH and CODER_SLIM_BUILD=1. If no version is specified, defaults to the diff --git a/scripts/build_go_matrix.sh b/scripts/build_go_matrix.sh index b3815597c6f97..3082393d523e2 100755 --- a/scripts/build_go_matrix.sh +++ b/scripts/build_go_matrix.sh @@ -3,7 +3,7 @@ # This script builds multiple Go binaries for Coder with the given OS and # architecture combinations. # -# Usage: ./build_go_matrix.sh [--version 1.2.3+devel.abcdef] [--output dist/] [--slim] [--sign-darwin] [--archive] [--package-linux] os1:arch1,arch2 os2:arch1 os1:arch3 +# Usage: ./build_go_matrix.sh [--version 1.2.3-devel+abcdef] [--output dist/] [--slim] [--sign-darwin] [--archive] [--package-linux] os1:arch1,arch2 os2:arch1 os1:arch3 # # If no OS:arch combinations are provided, nothing will happen and no error will # be returned. Slim builds are disabled by default. If no version is specified, diff --git a/scripts/build_go_slim.sh b/scripts/build_go_slim.sh index e060c399c3fc5..6b4e041402beb 100755 --- a/scripts/build_go_slim.sh +++ b/scripts/build_go_slim.sh @@ -3,7 +3,7 @@ # This script builds multiple "slim" Go binaries for Coder with the given OS and # architecture combinations. This wraps ./build_go_matrix.sh. # -# Usage: ./build_go_slim.sh [--version 1.2.3+devel.abcdef] [--output dist/] os1:arch1,arch2 os2:arch1 os1:arch3 +# Usage: ./build_go_slim.sh [--version 1.2.3-devel+abcdef] [--output dist/] os1:arch1,arch2 os2:arch1 os1:arch3 # # If no OS:arch combinations are provided, nothing will happen and no error will # be returned. If no version is specified, defaults to the version from diff --git a/scripts/image_tag.sh b/scripts/image_tag.sh index 674c3193b4686..480496d1918c9 100755 --- a/scripts/image_tag.sh +++ b/scripts/image_tag.sh @@ -54,5 +54,6 @@ if [[ "$arch" != "" ]]; then tag+="-$arch" fi +# Dev versions contain plus signs which are illegal characters in Docker tags. tag="${tag//+/-}" echo "$image:$tag" diff --git a/scripts/version.sh b/scripts/version.sh index d6a884501cb3a..8a8e1367975f1 100755 --- a/scripts/version.sh +++ b/scripts/version.sh @@ -3,6 +3,12 @@ # This script generates the version string used by Coder, including for dev # versions. Note: the version returned by this script will NOT include the "v" # prefix that is included in the Git tag. +# +# If $CODER_FORCE_DEV_VERSION is set to "true", the returned version will be a +# dev version even if the current commit is tagged. +# +# If $CODER_NO_DEV_VERSION is set to "true", the script will fail if the current +# commit is not tagged. set -euo pipefail # shellcheck source=scripts/lib.sh @@ -17,9 +23,9 @@ version="$last_tag" # If the HEAD has extra commits since the last tag then we are in a dev version. # -# Dev versions are denoted by the "+dev." suffix with a trailing commit short +# Dev versions are denoted by the "-devel+" suffix with a trailing commit short # SHA. -if [[ "$last_tag" != "$current" ]]; then +if [[ "${CODER_FORCE_DEV_VERSION:-}" == *t* ]] || [[ "$last_tag" != "$current" ]]; then if [[ "${CODER_NO_DEV_VERSION:-}" == *t* ]]; then # make won't exit on $(shell cmd) failures :( if [[ "$(ps -o comm= "$PPID" || true)" == *make* ]]; then @@ -30,7 +36,7 @@ if [[ "$last_tag" != "$current" ]]; then error "version.sh attemped to generate a dev version string when CODER_NO_DEV_VERSION was set" fi - version+="+dev.$(git rev-parse --short HEAD)" + version+="-devel+$(git rev-parse --short HEAD)" fi # Remove the "v" prefix. From 48eaa18813ba301f271c044ee6b2785652016ee2 Mon Sep 17 00:00:00 2001 From: deansheather Date: Tue, 14 Jun 2022 19:29:19 +0000 Subject: [PATCH 27/36] fixup! chore: make dependency checks nicer --- .github/workflows/coder.yaml | 23 ++--- .goreleaser.yaml | 160 ----------------------------------- 2 files changed, 6 insertions(+), 177 deletions(-) delete mode 100644 .goreleaser.yaml diff --git a/.github/workflows/coder.yaml b/.github/workflows/coder.yaml index 39a20bba18348..b52e563e067ef 100644 --- a/.github/workflows/coder.yaml +++ b/.github/workflows/coder.yaml @@ -188,7 +188,7 @@ jobs: path: ${{ steps.go-cache-paths.outputs.go-mod }} key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }} - - name: Install goreleaser + - name: Install gotestsum uses: jaxxstorm/action-install-gh-release@v1.7.1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -226,7 +226,7 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} files: ./gotests.coverage flags: unittest-go-${{ matrix.os }} - # this flakes and sometimes fails the build + # this flakes and sometimes fails the build fail_ci_if_error: false test-go-postgres: @@ -258,7 +258,7 @@ jobs: path: ${{ steps.go-cache-paths.outputs.go-mod }} key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }} - - name: Install goreleaser + - name: Install gotestsum uses: jaxxstorm/action-install-gh-release@v1.7.1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -310,7 +310,7 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} files: ./gotests.coverage flags: unittest-go-postgres-${{ matrix.os }} - # this flakes and sometimes fails the build + # this flakes and sometimes fails the build fail_ci_if_error: false deploy: @@ -366,18 +366,11 @@ jobs: restore-keys: | js-${{ runner.os }}- - - uses: goreleaser/goreleaser-action@v3 - with: - install-only: true - - name: Build site run: make -B site/out/index.html - name: Build Release - uses: goreleaser/goreleaser-action@v3 - with: - version: latest - args: release --snapshot --rm-dist --skip-sign + run: make build - name: Install Release run: | @@ -437,7 +430,7 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} files: ./site/coverage/lcov.info flags: unittest-js - # this flakes and sometimes fails the build + # this flakes and sometimes fails the build fail_ci_if_error: false - name: Upload DataDog Trace @@ -484,10 +477,6 @@ jobs: with: node-version: "14" - - uses: goreleaser/goreleaser-action@v3 - with: - install-only: true - - name: Echo Go Cache Paths id: go-cache-paths run: | diff --git a/.goreleaser.yaml b/.goreleaser.yaml deleted file mode 100644 index 7bcedf5087db5..0000000000000 --- a/.goreleaser.yaml +++ /dev/null @@ -1,160 +0,0 @@ -archives: - - id: coder-linux - builds: [coder-linux] - format: tar.gz - - - id: coder-darwin - builds: [coder-darwin] - format: zip - - - id: coder-windows - builds: [coder-windows] - format: zip - -before: - hooks: - - go mod tidy - - rm -f site/out/bin/coder* - -builds: - - id: coder-slim - dir: cmd/coder - ldflags: ["-s -w -X github.com/coder/coder/buildinfo.tag={{ .Version }}"] - env: [CGO_ENABLED=0] - goos: [darwin, linux, windows] - goarch: [amd64, arm, arm64] - goarm: ["7"] - # Only build arm 7 for Linux - ignore: - - goos: windows - goarm: "7" - - goos: darwin - goarm: "7" - hooks: - # The "trimprefix" appends ".exe" on Windows. - post: | - cp {{.Path}} site/out/bin/coder-{{ .Os }}-{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}{{ trimprefix .Name "coder" }} - - - id: coder-linux - dir: cmd/coder - flags: [-tags=embed] - ldflags: ["-s -w -X github.com/coder/coder/buildinfo.tag={{ .Version }}"] - env: [CGO_ENABLED=0] - goos: [linux] - goarch: [amd64, arm, arm64] - goarm: ["7"] - - - id: coder-windows - dir: cmd/coder - flags: [-tags=embed] - ldflags: ["-s -w -X github.com/coder/coder/buildinfo.tag={{ .Version }}"] - env: [CGO_ENABLED=0] - goos: [windows] - goarch: [amd64, arm64] - - - id: coder-darwin - dir: cmd/coder - flags: [-tags=embed] - ldflags: ["-s -w -X github.com/coder/coder/buildinfo.tag={{ .Version }}"] - env: [CGO_ENABLED=0] - goos: [darwin] - goarch: [amd64, arm64] - hooks: - # This signs the binary that will be located inside the zip. - # MacOS requires the binary to be signed for notarization. - # - # If it doesn't successfully sign, the zip sign step will error. - post: | - sh -c 'codesign -s {{.Env.AC_APPLICATION_IDENTITY}} -f -v --timestamp --options runtime {{.Path}} || true' - -env: - # Apple identity for signing! - - AC_APPLICATION_IDENTITY=BDB050EB749EDD6A80C6F119BF1382ECA119CCCC - -nfpms: - - id: packages - vendor: Coder - homepage: https://coder.com - maintainer: Coder - description: | - Provision development environments with infrastructure with code - formats: - - apk - - deb - - rpm - suggests: - - postgresql - builds: - - coder-linux - bindir: /usr/bin - contents: - - src: coder.env - dst: /etc/coder.d/coder.env - type: "config|noreplace" - - src: coder.service - dst: /usr/lib/systemd/system/coder.service - -# Image templates are empty on snapshots to avoid lengthy builds for development. -dockers: - - image_templates: ["{{ if not .IsSnapshot }}ghcr.io/coder/coder:{{ .Tag }}-amd64{{ end }}"] - id: coder-linux - dockerfile: Dockerfile - use: buildx - build_flag_templates: - - --platform=linux/amd64 - - --label=org.opencontainers.image.title=Coder - - --label=org.opencontainers.image.description=A tool for provisioning self-hosted development environments with Terraform. - - --label=org.opencontainers.image.url=https://github.com/coder/coder - - --label=org.opencontainers.image.source=https://github.com/coder/coder - - --label=org.opencontainers.image.version={{ .Version }} - - --label=org.opencontainers.image.revision={{ .FullCommit }} - - --label=org.opencontainers.image.licenses=AGPL-3.0 - - image_templates: ["{{ if not .IsSnapshot }}ghcr.io/coder/coder:{{ .Tag }}-arm64{{ end }}"] - goarch: arm64 - dockerfile: Dockerfile - use: buildx - build_flag_templates: - - --platform=linux/arm64/v8 - - --label=org.opencontainers.image.title=coder - - --label=org.opencontainers.image.description=A tool for provisioning self-hosted development environments with Terraform. - - --label=org.opencontainers.image.url=https://github.com/coder/coder - - --label=org.opencontainers.image.source=https://github.com/coder/coder - - --label=org.opencontainers.image.version={{ .Tag }} - - --label=org.opencontainers.image.revision={{ .FullCommit }} - - --label=org.opencontainers.image.licenses=AGPL-3.0 - - image_templates: ["{{ if not .IsSnapshot }}ghcr.io/coder/coder:{{ .Tag }}-armv7{{ end }}"] - goarch: arm - goarm: "7" - dockerfile: Dockerfile - use: buildx - build_flag_templates: - - --platform=linux/arm/v7 - - --label=org.opencontainers.image.title=Coder - - --label=org.opencontainers.image.description=A tool for provisioning self-hosted development environments with Terraform. - - --label=org.opencontainers.image.url=https://github.com/coder/coder - - --label=org.opencontainers.image.source=https://github.com/coder/coder - - --label=org.opencontainers.image.version={{ .Tag }} - - --label=org.opencontainers.image.revision={{ .FullCommit }} - - --label=org.opencontainers.image.licenses=AGPL-3.0 -docker_manifests: - - name_template: ghcr.io/coder/coder:{{ .Tag }} - image_templates: - - ghcr.io/coder/coder:{{ .Tag }}-amd64 - - ghcr.io/coder/coder:{{ .Tag }}-arm64 - - ghcr.io/coder/coder:{{ .Tag }}-armv7 - -release: - ids: [coder-linux, coder-darwin, coder-windows, packages] - footer: | - ## Container Image - - `docker pull ghcr.io/coder/coder:{{ .Tag }}` - -signs: - - ids: [coder-darwin] - artifacts: archive - cmd: ./scripts/sign_macos.sh - args: ["${artifact}"] - output: true - -snapshot: - name_template: "{{ .Version }}-devel+{{ .ShortCommit }}" From ce3f4f500a5a219613cd1920930274ccf866cad6 Mon Sep 17 00:00:00 2001 From: deansheather Date: Tue, 14 Jun 2022 19:31:40 +0000 Subject: [PATCH 28/36] fixup! chore: make dependency checks nicer --- .github/workflows/coder.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/coder.yaml b/.github/workflows/coder.yaml index b52e563e067ef..f65b14bcc28f3 100644 --- a/.github/workflows/coder.yaml +++ b/.github/workflows/coder.yaml @@ -111,9 +111,8 @@ jobs: - uses: actions/setup-go@v3 with: go-version: "~1.18" - - run: curl -sSL - https://github.com/kyleconroy/sqlc/releases/download/v1.13.0/sqlc_1.13.0_linux_amd64.tar.gz - | sudo tar -C /usr/bin -xz sqlc + - run: | + curl -sSL https://github.com/kyleconroy/sqlc/releases/download/v1.13.0/sqlc_1.13.0_linux_amd64.tar.gz | sudo tar -C /usr/bin -xz sqlc - run: go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26 - run: go install storj.io/drpc/cmd/protoc-gen-go-drpc@v0.0.26 From 4886542149ddd709b17b69569e7b1118fd668baa Mon Sep 17 00:00:00 2001 From: deansheather Date: Tue, 14 Jun 2022 20:15:10 +0000 Subject: [PATCH 29/36] fixup! chore: make dependency checks nicer --- scripts/publish_release.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/publish_release.sh b/scripts/publish_release.sh index 86457773c6ee5..16ec8c7a2c795 100755 --- a/scripts/publish_release.sh +++ b/scripts/publish_release.sh @@ -107,6 +107,9 @@ $changelog " +release_notes_file="$(mktemp)" +echo "$release_notes" >"$release_notes_file" + # Create temporary release folder so we can generate checksums. Both the # sha256sum and gh binaries support symlinks as input files so this works well. temp_dir="$(mktemp -d)" @@ -131,13 +134,13 @@ popd log log -# We echo the release notes in instead of writing to a file and referencing that -# to prevent GitHub CLI from becoming interactive. -echo "$release_notes" | +# We pipe `true` into `gh` so that it never tries to be interactive. +true | maybedryrun "$dry_run" gh release create \ --title "$new_tag" \ - --notes-file - \ + --notes-file "$release_notes_file" \ "$new_tag" \ "$temp_dir"/* rm -rf "$temp_dir" +rm -rf "$release_notes_file" From d4631ada6032d95e4cbf0662296e50a00a1c7e54 Mon Sep 17 00:00:00 2001 From: deansheather Date: Tue, 14 Jun 2022 20:20:47 +0000 Subject: [PATCH 30/36] fixup! chore: make dependency checks nicer --- .github/workflows/coder.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/coder.yaml b/.github/workflows/coder.yaml index f65b14bcc28f3..5b7499c771a3f 100644 --- a/.github/workflows/coder.yaml +++ b/.github/workflows/coder.yaml @@ -365,6 +365,9 @@ jobs: restore-keys: | js-${{ runner.os }}- + - name: Install nfpm + run: go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.16.0 + - name: Build site run: make -B site/out/index.html @@ -386,8 +389,11 @@ jobs: with: name: coder path: | - ./dist/coder_*_linux_amd64.tar.gz - ./dist/coder_*_windows_amd64.zip + ./dist/*.zip + ./dist/*.tar.gz + ./dist/*.apk + ./dist/*.deb + ./dist/*.rpm retention-days: 7 test-js: From a0c9a964981e8b4ee26820a44fb192b917ca2836 Mon Sep 17 00:00:00 2001 From: deansheather Date: Wed, 15 Jun 2022 17:55:36 +0000 Subject: [PATCH 31/36] chore: integrate docker into pipeline --- .github/workflows/release.yaml | 34 +++++++++++++++++++++++++++++++ docker-compose.yaml | 2 +- scripts/build_docker.sh | 3 ++- scripts/build_docker_multiarch.sh | 32 ++++++++++++++++++++--------- scripts/image_tag.sh | 8 ++++++-- 5 files changed, 65 insertions(+), 14 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 5c7570afe08ed..5919fda346b5c 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -58,6 +58,7 @@ jobs: - name: Build Linux and Windows Binaries run: | + set -euo pipefail go mod download mkdir -p ./dist @@ -76,6 +77,37 @@ jobs: linux:amd64,armv7,arm64 \ windows:amd64,arm64 + - name: Build Linux Docker images + run: | + set -euo pipefail + + # build and push Docker images for each architecture + images=() + for arch in amd64 armv7 arm64; do + img="$( + ./scripts/build_docker.sh \ + ${{ !github.event.inputs.dry_run && !github.event.inputs.snapshot && '--push' }} \ + --arch "$arch" \ + ./dist/coder_*_linux_"$arch" + )" + images+=("$img") + done + + # build and push multi-arch manifest + ./scripts/build_docker_multiarch.sh \ + ${{ !github.event.inputs.dry_run && !github.event.inputs.snapshot && '--push' }} \ + "${images[@]}" + + # if the current version is equal to the highest (according to semver) + # version in the repo, also create a multi-arch image as ":latest" and + # push it + if [[ "$(git tag | grep '^v' | grep -vE '(rc|dev|-|\+|\/)' | sort -r --version-sort | head -n1)" == "v$(./scripts/version.sh)" ]]; then + ./scripts/build_docker_multiarch.sh \ + --target "$(./scripts/image_tag.sh --version latest)" \ + ${{ !github.event.inputs.dry_run && !github.event.inputs.snapshot && '--push' }} \ + "${images[@]}" + fi + - name: Upload binary artifacts uses: actions/upload-artifact@v3 with: @@ -120,6 +152,7 @@ jobs: - name: Install dependencies run: | + set -euo pipefail # The version of bash that MacOS ships with is too old brew install bash @@ -140,6 +173,7 @@ jobs: - name: Build darwin Binaries (with signatures) run: | + set -euo pipefail go mod download mkdir -p ./dist diff --git a/docker-compose.yaml b/docker-compose.yaml index d943ff185d617..8e9a3a505a690 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,7 +1,7 @@ version: "3.9" services: coder: - image: ghcr.io/coder/coder:v${CODER_VERSION:-0.5.10} + image: ghcr.io/coder/coder:v${CODER_VERSION:-latest} ports: - "7080:7080" environment: diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh index 8508e95771e7d..64b3fef3f7703 100755 --- a/scripts/build_docker.sh +++ b/scripts/build_docker.sh @@ -3,7 +3,7 @@ # This script builds a Docker image of Coder containing the given binary, for # the given architecture. Only linux binaries are supported at this time. # -# Usage: ./build_docker.sh --arch amd64 [--version 1.2.3] [--push] +# Usage: ./build_docker.sh --arch amd64 [--version 1.2.3] [--push] path/to/coder # # The --arch parameter is required and accepts a Golang arch specification. It # will be automatically mapped to a suitable architecture that Docker accepts @@ -80,6 +80,7 @@ declare -A arch_map=( [amd64]="linux/amd64" [arm64]="linux/arm64" [arm]="linux/arm/v7" + [armv7]="linux/arm/v7" ) if [[ "${arch_map[$arch]+exists}" != "" ]]; then arch="${arch_map[$arch]}" diff --git a/scripts/build_docker_multiarch.sh b/scripts/build_docker_multiarch.sh index 37dd4ddf2885b..375815d119f1a 100755 --- a/scripts/build_docker_multiarch.sh +++ b/scripts/build_docker_multiarch.sh @@ -1,16 +1,20 @@ #!/usr/bin/env bash # This script merges Coder Docker images of different architectures together -# into the archless image tag returned by ./image_tag.sh. +# into the specified target image+tag, or the arch-less image tag returned by +# ./image_tag.sh. # -# Usage: ./build_docker_multiarch.sh [--version 1.2.3] [--push] image1:tag1 image2:tag2 +# Usage: ./build_docker_multiarch.sh [--version 1.2.3] [--target image:tag] [--push] image1:tag1 image2:tag2 # # The supplied images must already be pushed to the registry or this will fail. # Also, the source images cannot be in a different registry than the target -# image generated by ./image_tag.sh. +# image. # # If no version is specified, defaults to the version from ./version.sh. # +# If no target tag is supplied, the arch-less image tag returned by +# ./image_tag.sh will be used. +# # If the --push parameter is supplied, all supplied tags will be pushed. # # Returns the merged image tag. @@ -20,9 +24,10 @@ set -euo pipefail source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" version="" +target="" push=0 -args="$(getopt -o "" -l version:,push -- "$@")" +args="$(getopt -o "" -l version:,target:,push -- "$@")" eval set -- "$args" while true; do case "$1" in @@ -30,6 +35,10 @@ while true; do version="$2" shift 2 ;; + --target) + target="$2" + shift 2 + ;; --push) push=1 shift @@ -57,21 +66,24 @@ if [[ "$version" == "" ]]; then version="$(execrelative ./version.sh)" fi +if [[ "$target" == "" ]]; then + target="$(execrelative ./image_tag.sh --version "$version")" +fi + create_args=() for image_tag in "$@"; do create_args+=(--amend "$image_tag") done # Sadly, multi-arch images don't seem to support labels. -output_tag="$(execrelative ./image_tag.sh --version "$version")" -log "--- Creating multi-arch Docker image ($output_tag)" +log "--- Creating multi-arch Docker image ($target)" docker manifest create \ - "$output_tag" \ + "$target" \ "${create_args[@]}" if [[ "$push" == 1 ]]; then - log "--- Pushing multi-arch Docker image ($output_tag)" - docker push "$output_tag" + log "--- Pushing multi-arch Docker image ($target)" + docker manifest push "$target" fi -echo "$output_tag" +echo "$target" diff --git a/scripts/image_tag.sh b/scripts/image_tag.sh index 480496d1918c9..8b405c48e304f 100755 --- a/scripts/image_tag.sh +++ b/scripts/image_tag.sh @@ -8,7 +8,8 @@ # The --arch parameter accepts a Golang arch specification. If not specified, # the image tag for the multi-arch image will be returned instead. # -# If no version is specified, defaults to the version from ./version.sh. +# If no version is specified, defaults to the version from ./version.sh. If the +# supplied version is "latest", no `v` prefix will be added to the tag. # # The returned tag will be sanitized to remove invalid characters like the plus # sign. @@ -42,7 +43,7 @@ while true; do esac done -# Remove the "v" prefix. +# Remove the "v" prefix because we don't want to add it twice. version="${version#v}" if [[ "$version" == "" ]]; then version="$(execrelative ./version.sh)" @@ -50,6 +51,9 @@ fi image="${CODER_IMAGE_BASE:-ghcr.io/coder/coder}" tag="v$version" +if [[ "$version" == "latest" ]]; then + tag="latest" +fi if [[ "$arch" != "" ]]; then tag+="-$arch" fi From 0907c37f92d37a69da9c6a0dcb9904087ca66314 Mon Sep 17 00:00:00 2001 From: deansheather Date: Wed, 15 Jun 2022 18:42:34 +0000 Subject: [PATCH 32/36] fixup! chore: integrate docker into pipeline --- .github/workflows/release.yaml | 13 +++++-------- scripts/version.sh | 27 +++++++++++++-------------- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 5919fda346b5c..fcbb408d4a1cd 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -21,11 +21,8 @@ on: type: boolean required: true -# In a non-snapshot we want the ./scripts/version.sh script to never generate a -# +dev version. env: - CODER_FORCE_DEV_VERSION: ${{ github.event.inputs.snapshot && 'true' || 'false' }} - CODER_NO_DEV_VERSION: ${{ github.event.inputs.snapshot && 'false' || 'true' }} + CODER_RELEASE: ${{ github.event.inputs.snapshot && 'false' || 'true' }} jobs: linux-windows: @@ -79,14 +76,14 @@ jobs: - name: Build Linux Docker images run: | - set -euo pipefail + set -euxo pipefail # build and push Docker images for each architecture images=() for arch in amd64 armv7 arm64; do img="$( ./scripts/build_docker.sh \ - ${{ !github.event.inputs.dry_run && !github.event.inputs.snapshot && '--push' }} \ + ${{ (!github.event.inputs.dry_run && !github.event.inputs.snapshot) && '--push' || '' }} \ --arch "$arch" \ ./dist/coder_*_linux_"$arch" )" @@ -95,7 +92,7 @@ jobs: # build and push multi-arch manifest ./scripts/build_docker_multiarch.sh \ - ${{ !github.event.inputs.dry_run && !github.event.inputs.snapshot && '--push' }} \ + ${{ (!github.event.inputs.dry_run && !github.event.inputs.snapshot) && '--push' || '' }} \ "${images[@]}" # if the current version is equal to the highest (according to semver) @@ -104,7 +101,7 @@ jobs: if [[ "$(git tag | grep '^v' | grep -vE '(rc|dev|-|\+|\/)' | sort -r --version-sort | head -n1)" == "v$(./scripts/version.sh)" ]]; then ./scripts/build_docker_multiarch.sh \ --target "$(./scripts/image_tag.sh --version latest)" \ - ${{ !github.event.inputs.dry_run && !github.event.inputs.snapshot && '--push' }} \ + ${{ (!github.event.inputs.dry_run && !github.event.inputs.snapshot) && '--push' || '' }} \ "${images[@]}" fi diff --git a/scripts/version.sh b/scripts/version.sh index 8a8e1367975f1..c3bccd617e3d2 100755 --- a/scripts/version.sh +++ b/scripts/version.sh @@ -4,38 +4,37 @@ # versions. Note: the version returned by this script will NOT include the "v" # prefix that is included in the Git tag. # -# If $CODER_FORCE_DEV_VERSION is set to "true", the returned version will be a -# dev version even if the current commit is tagged. +# If $CODER_RELEASE is set to "true", the returned version will equal the +# current git tag. If the current commit is not tagged, this will fail. # -# If $CODER_NO_DEV_VERSION is set to "true", the script will fail if the current -# commit is not tagged. +# If $CODER_RELEASE is not set, the returned version will always be a dev +# version. set -euo pipefail # shellcheck source=scripts/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" cdroot -# $current will equal $last_tag if we currently have the tag checked out. -last_tag="$(git describe --tags --abbrev=0)" -current="$(git describe --always)" - -version="$last_tag" +version="$(git describe --tags --abbrev=0)" # If the HEAD has extra commits since the last tag then we are in a dev version. # # Dev versions are denoted by the "-devel+" suffix with a trailing commit short # SHA. -if [[ "${CODER_FORCE_DEV_VERSION:-}" == *t* ]] || [[ "$last_tag" != "$current" ]]; then - if [[ "${CODER_NO_DEV_VERSION:-}" == *t* ]]; then - # make won't exit on $(shell cmd) failures :( +if [[ "${CODER_RELEASE:-}" == *t* ]]; then + # $version will equal `git describe --always` if we currently have the tag + # checked out. + if [[ "$version" != "$(git describe --always)" ]]; then + # make won't exit on $(shell cmd) failures, so we have to kill it :( if [[ "$(ps -o comm= "$PPID" || true)" == *make* ]]; then - log "ERROR: version.sh attemped to generate a dev version string when CODER_NO_DEV_VERSION was set" + log "ERROR: version.sh attemped to generate a dev version string when CODER_RELEASE was set" kill "$PPID" || true exit 1 fi - error "version.sh attemped to generate a dev version string when CODER_NO_DEV_VERSION was set" + error "version.sh attemped to generate a dev version string when CODER_RELEASE was set" fi +else version+="-devel+$(git rev-parse --short HEAD)" fi From 4a94e25c612a983a956c52332edf5da069c24940 Mon Sep 17 00:00:00 2001 From: deansheather Date: Thu, 16 Jun 2022 16:08:29 +0000 Subject: [PATCH 33/36] fixup! chore: integrate docker into pipeline --- .github/workflows/release.yaml | 13 ++++++++++--- scripts/publish_release.sh | 4 ++++ scripts/version.sh | 7 ++++--- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index fcbb408d4a1cd..9f79c4512dd1d 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -78,7 +78,7 @@ jobs: run: | set -euxo pipefail - # build and push Docker images for each architecture + # build and (maybe) push Docker images for each architecture images=() for arch in amd64 armv7 arm64; do img="$( @@ -90,9 +90,16 @@ jobs: images+=("$img") done + # we can't build multi-arch if the images aren't pushed, so quit now + # if dry-running + if [[ "$CODER_RELEASE" != *t* ]]; then + echo Skipping multi-arch docker builds due to dry-run. + exit 0 + fi + # build and push multi-arch manifest ./scripts/build_docker_multiarch.sh \ - ${{ (!github.event.inputs.dry_run && !github.event.inputs.snapshot) && '--push' || '' }} \ + --push \ "${images[@]}" # if the current version is equal to the highest (according to semver) @@ -100,8 +107,8 @@ jobs: # push it if [[ "$(git tag | grep '^v' | grep -vE '(rc|dev|-|\+|\/)' | sort -r --version-sort | head -n1)" == "v$(./scripts/version.sh)" ]]; then ./scripts/build_docker_multiarch.sh \ + --push \ --target "$(./scripts/image_tag.sh --version latest)" \ - ${{ (!github.event.inputs.dry_run && !github.event.inputs.snapshot) && '--push' || '' }} \ "${images[@]}" fi diff --git a/scripts/publish_release.sh b/scripts/publish_release.sh index 16ec8c7a2c795..338ffbb99ff7b 100755 --- a/scripts/publish_release.sh +++ b/scripts/publish_release.sh @@ -71,6 +71,10 @@ if [[ "${#files[@]}" == 0 ]]; then error "No files supplied" fi +if [[ "$dry_run" == 0 ]] && [[ "$version" == *dev* ]]; then + error "Cannot publish a dev version to GitHub" +fi + # The git commands need to be executed from within the repository. cdroot diff --git a/scripts/version.sh b/scripts/version.sh index c3bccd617e3d2..d924117fe9640 100755 --- a/scripts/version.sh +++ b/scripts/version.sh @@ -15,16 +15,17 @@ set -euo pipefail source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" cdroot -version="$(git describe --tags --abbrev=0)" +last_tag="$(git describe --tags --abbrev=0)" +version="$last_tag" # If the HEAD has extra commits since the last tag then we are in a dev version. # # Dev versions are denoted by the "-devel+" suffix with a trailing commit short # SHA. if [[ "${CODER_RELEASE:-}" == *t* ]]; then - # $version will equal `git describe --always` if we currently have the tag + # $last_tag will equal `git describe --always` if we currently have the tag # checked out. - if [[ "$version" != "$(git describe --always)" ]]; then + if [[ "$last_tag" != "$(git describe --always)" ]]; then # make won't exit on $(shell cmd) failures, so we have to kill it :( if [[ "$(ps -o comm= "$PPID" || true)" == *make* ]]; then log "ERROR: version.sh attemped to generate a dev version string when CODER_RELEASE was set" From 173031f0b19cf2146097013b5048d018ad84b7c1 Mon Sep 17 00:00:00 2001 From: deansheather Date: Thu, 16 Jun 2022 16:50:07 +0000 Subject: [PATCH 34/36] chore: add version checks to lib.sh --- scripts/lib.sh | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/scripts/lib.sh b/scripts/lib.sh index b0b42a01cb02f..fbec603eae5ea 100644 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -110,3 +110,52 @@ error() { log "ERROR: $*" exit 1 } + +libsh_bad_dependencies=0 + +if ((BASH_VERSINFO[0] < 4)); then + libsh_bad_dependencies=1 + log "ERROR: You need at least bash 4.0 to run the scripts in the Coder repo." + if [[ "${OSTYPE:-darwin}" == "darwin" ]]; then + log "On darwin:" + log "- brew install bash" + log "- Restart your terminal" + fi + log +fi + +# BSD getopt (which is installed by default on Macs) is not supported. +if [[ "$(getopt --version)" == *--* ]]; then + libsh_bad_dependencies=1 + log "ERROR: You need GNU getopt to run the scripts in the Coder repo." + if [[ "${OSTYPE:-darwin}" == "darwin" ]]; then + log "On darwin:" + log "- brew install gnu-getopt" + # shellcheck disable=SC2016 + log '- Add "$(brew --prefix)/opt/gnu-getopt/bin" to your PATH' + log "- Restart your terminal" + fi + log +fi + +# The bash scripts don't call Make directly, but we want to make (ha ha) sure +# that make supports the features the repo uses. Notably, Macs have an old +# version of Make installed out of the box that doesn't support new features +# like ONESHELL. +make_version="$(make --version 2>/dev/null | head -n1 | grep -oE '([[:digit:]]+\.){1,2}[[:digit:]]+')" +if [ "${make_version//.*/}" -lt 4 ]; then + libsh_bad_dependencies=1 + log "ERROR: You need at least make 4.0 to run the scripts in the Coder repo." + if [[ "${OSTYPE:-darwin}" == "darwin" ]]; then + log "On darwin:" + log "- brew install make" + # shellcheck disable=SC2016 + log '- Add "$(brew --prefix)/opt/make/libexec/gnubin" to your PATH (you should Google this first)' + log "- Restart your terminal" + fi + log +fi + +if [[ "$libsh_bad_dependencies" == 1 ]]; then + error "Invalid dependencies, see above for more details." +fi From ae48f20b01873e4619a9f6e75a46e412b1cbdf34 Mon Sep 17 00:00:00 2001 From: deansheather Date: Thu, 16 Jun 2022 16:52:58 +0000 Subject: [PATCH 35/36] fixup! chore: add version checks to lib.sh --- scripts/lib.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/lib.sh b/scripts/lib.sh index fbec603eae5ea..98876155b2477 100644 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -111,12 +111,17 @@ error() { exit 1 } +# isdarwin returns an error if the current platform is not darwin. +isdarwin() { + [[ "${OSTYPE:-darwin}" == *darwin* ]] +} + libsh_bad_dependencies=0 if ((BASH_VERSINFO[0] < 4)); then libsh_bad_dependencies=1 log "ERROR: You need at least bash 4.0 to run the scripts in the Coder repo." - if [[ "${OSTYPE:-darwin}" == "darwin" ]]; then + if isdarwin; then log "On darwin:" log "- brew install bash" log "- Restart your terminal" @@ -128,7 +133,7 @@ fi if [[ "$(getopt --version)" == *--* ]]; then libsh_bad_dependencies=1 log "ERROR: You need GNU getopt to run the scripts in the Coder repo." - if [[ "${OSTYPE:-darwin}" == "darwin" ]]; then + if isdarwin; then log "On darwin:" log "- brew install gnu-getopt" # shellcheck disable=SC2016 @@ -146,7 +151,7 @@ make_version="$(make --version 2>/dev/null | head -n1 | grep -oE '([[:digit:]]+\ if [ "${make_version//.*/}" -lt 4 ]; then libsh_bad_dependencies=1 log "ERROR: You need at least make 4.0 to run the scripts in the Coder repo." - if [[ "${OSTYPE:-darwin}" == "darwin" ]]; then + if isdarwin; then log "On darwin:" log "- brew install make" # shellcheck disable=SC2016 From d8a624cddbeaaf33f86430a10a001a8679f5c281 Mon Sep 17 00:00:00 2001 From: deansheather Date: Thu, 16 Jun 2022 17:25:05 +0000 Subject: [PATCH 36/36] fixup! chore: add version checks to lib.sh --- scripts/sign_darwin.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/sign_darwin.sh b/scripts/sign_darwin.sh index 41f178048c4e6..f1bfef735767b 100755 --- a/scripts/sign_darwin.sh +++ b/scripts/sign_darwin.sh @@ -42,17 +42,17 @@ codesign -s "$AC_APPLICATION_IDENTITY" -f -v --timestamp --options runtime "$out # Notarize the signed zip file. # # The notarization process is very fragile and heavily dependent on Apple's -# notarization server not returning server errors, so we retry this step 5 -# times with a delay of 30 seconds between each attempt. +# notarization server not returning server errors, so we retry this step twice +# with a delay of a minute between attempts. rc=0 -for i in $(seq 1 5); do +for i in $(seq 1 2); do gon "$config" && rc=0 && break || rc=$? log "gon exit code: $rc" if [ "$i" -lt 5 ]; then log - log "Retrying notarization in 30 seconds" + log "Retrying notarization in 60 seconds" log - sleep 30 + sleep 60 else log log "Giving up :("