From 73e4db46ada03f7f173c0f21b6b4e372c5fa53e2 Mon Sep 17 00:00:00 2001 From: Dean Sheather Date: Fri, 26 Aug 2022 00:28:31 +0000 Subject: [PATCH 01/12] chore: buff makefile pt. 1 --- .gitignore | 1 + Makefile | 184 +++++++++++++++++++++++++++++++++----------- scripts/archive.sh | 4 +- scripts/build_go.sh | 3 +- scripts/package.sh | 44 ++++++----- 5 files changed, 172 insertions(+), 64 deletions(-) diff --git a/.gitignore b/.gitignore index d3deb0c72f550..3bda1a5df28d2 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ site/**/*.typegen.ts site/build-storybook.log # Build +build/ dist/ site/out/ diff --git a/Makefile b/Makefile index f1b55cfaba274..649cf6a400a8e 100644 --- a/Makefile +++ b/Makefile @@ -9,51 +9,149 @@ SHELL := bash # See https://stackoverflow.com/questions/25752543/make-delete-on-error-for-directory-targets .DELETE_ON_ERROR: -INSTALL_DIR=$(shell go env GOPATH)/bin -GOOS=$(shell go env GOOS) -GOARCH=$(shell go env GOARCH) -VERSION=$(shell ./scripts/version.sh) - -bin: $(shell find . -not -path './vendor/*' -type f -name '*.go') go.mod go.sum $(shell find ./examples/templates) - @echo "== This builds slim binaries for command-line usage." - @echo "== Use \"make build\" to embed the site." - - mkdir -p ./dist - rm -rf ./dist/coder-slim_* - rm -f ./site/out/bin/coder* - ./scripts/build_go_slim.sh \ - --compress 6 \ - --version "$(VERSION)" \ - --output ./dist/ \ - linux:amd64,armv7,arm64 \ - windows:amd64,arm64 \ - darwin:amd64,arm64 -.PHONY: bin - -build: site/out/index.html $(shell find . -not -path './vendor/*' -type f -name '*.go') go.mod go.sum $(shell find ./examples/templates) - rm -rf ./dist - mkdir -p ./dist - rm -f ./site/out/bin/coder* - - # build slim artifacts and copy them to the site output directory - ./scripts/build_go_slim.sh \ +INSTALL_DIR = $(shell go env GOPATH)/bin +GOOS = $(shell go env GOOS) +GOARCH = $(shell go env GOARCH) +VERSION = $(shell ./scripts/version.sh) + +# All ${OS}_${ARCH} combos we build for. Windows binaries have the .exe suffix. +OS_ARCHES = \ + linux_amd64 linux_arm64 linux_armv7 \ + darwin_amd64 darwin_arm64 \ + windows_amd64.exe windows_arm64.exe + +# Archive formats and their corresponding ${OS}_${ARCH} combos. +ARCHIVE_TAR_GZ = linux_amd64 linux_arm64 linux_armv7 +ARCHIVE_ZIP = \ + darwin darwin_arm64 \ + windows_amd64 windows_arm64 + +# All ${OS}_${ARCH} combos we build packages for. +PACKAGE_OS_ARCHES = linux_amd64 linux_armv7 linux_arm64 + +# All package formats we build. +PACKAGE_FORMATS = apk deb rpm + +bin build-slim: $(addprefix build/coder-slim_$(VERSION)_,$(OS_ARCHES)) +.PHONY: bin build-slim + +build build-full: $(addprefix build/coder_$(VERSION)_,$(OS_ARCHES)) +.PHONY: build build-full + +# Redirect from version-less targets to the versioned ones. This is kinda gross +# since it's make shelling out to make, but it's the easiest way less we write +# out every target manually. +$(addprefix build/coder_,$(OS_ARCHES)): site/out/index.html build-slim + @target="coder_$(VERSION)_$(@:build/coder_%=%)" + @$(MAKE) \ + --no-print-directory \ + --assume-old site/out/index.html \ + --assume-old build-slim \ + "build/$$target" + @rm -f "$@" + @ln -s "$$target" "$@" +.PHONY: $(addprefix build/coder_,$(OS_ARCHES)) + +$(addprefix build/coder-slim_,$(OS_ARCHES)): + @target="coder-slim_$(VERSION)_$(@:build/coder-slim_%=%)" + @echo $(MAKE) \ + --no-print-directory \ + "build/$$target" + @rm -f "$@" + @ln -s "$$target" "$@" +.PHONY: $(addprefix build/coder-slim_,$(OS_ARCHES)) + +# "full" binaries always depend on all "slim" binaries. +$(addprefix build/coder_$(VERSION)_,$(OS_ARCHES)): site/out/index.html build-slim + +# This task handles all builds, for both "full" and "slim" binaries. It parses +# the target name to get the metadata for the build, so it must be specified in +# this format: +# build/coder(-slim)?_$version_$os_$arch(.exe)? +$(addprefix build/coder_$(VERSION)_,$(OS_ARCHES)) $(addprefix build/coder-slim_$(VERSION)_,$(OS_ARCHES)): \ + go.mod go.sum \ + $(shell find . -not -path './vendor/*' -type f -name '*.go') \ + $(shell find ./examples/templates) + + @mkdir -p build + @mode=$$([[ "$@" = build/coder-slim* ]] && echo "slim" || echo "full") + @os=$$(echo $@ | cut -d_ -f3) + @arch=$$(echo $@ | cut -d_ -f4) + @if [ "$$mode" != "full" ] && [ "$$mode" != "slim" ]; then + @echo "Invalid build mode: $$mode" + @exit 1 + @fi + @if [[ "$$os" == "windows" ]] && [[ "$$arch" == *.exe ]]; then + @arch=$${arch%.exe} + @fi + + @build_args=( \ + --os "$$os" \ + --arch "$$arch" \ --version "$(VERSION)" \ - --compress 6 \ - --output ./dist/ \ - linux:amd64,armv7,arm64 \ - windows:amd64,arm64 \ - darwin:amd64,arm64 - - # build not-so-slim artifacts with the default name format - ./scripts/build_go_matrix.sh \ + --output "$@" \ + ) + @if [ "$$mode" == "slim" ]; then + @build_args+=(--slim) + @fi + + ./scripts/build_go.sh "$${build_args[@]}" + +# This task builds all archives. It parses the target name to get the metadata +# for the build, so it must be specified in this format: +# build/coder_${version}_${os}_${arch}.${format} +# +# The following OS/arch/format combinations are supported: +# .tar.gz: linux_amd64, linux_arm64, linux_armv7 +# .zip: darwin_amd64, darwin_arm64, windows_amd64, windows_arm64 +# +# This depends on build-full because it's difficult to do dynamic dependencies. +$(foreach os_arch, $(ARCHIVE_TAR_GZ), build/coder_$(VERSION)_$(os_arch).tar.gz) \ +$(foreach os_arch, $(ARCHIVE_ZIP), build/coder_$(VERSION)_$(os_arch).zip): \ + build-full + + @mkdir -p build + @os=$$(echo $@ | cut -d_ -f3) + @arch_format=$$(echo $@ | cut -d_ -f4) + @arch=$$(echo $$arch_format | cut -d. -f1) + @format=$${arch_format#*.} + + @bin_ext="" + @if [[ "$$os" == "windows" ]]; then + @bin_ext=".exe" + @fi + + ./script/archive.sh \ + --format "$$format" \ + --output "$@" \ + "build/coder_$(VERSION)_$${os}_$${arch}$${bin_ext}" + +# This task builds all packages. It parses the target name to get the metadata +# for the build, so it must be specified in this format: +# build/coder_${version}_${os}_${arch}.${format} +# +# Supports apk, deb, rpm for all linux targets. +# +# This depends on build-full because it's difficult to do dynamic dependencies. +$(foreach os_arch, $(PACKAGE_OS_ARCHES), $(addprefix build/coder_$(VERSION)_$(os_arch).,$(PACKAGE_FORMATS))): \ + build-full + + @mkdir -p build + @os=$$(echo $@ | cut -d_ -f3) + @arch_format=$$(echo $@ | cut -d_ -f4) + @arch=$$(echo $$arch_format | cut -d. -f1) + @format=$${arch_format#*.} + + ./scripts/package.sh \ + --arch "$$arch" \ + --format "$$format" \ --version "$(VERSION)" \ - --output ./dist/ \ - --archive \ - --package-linux \ - linux:amd64,armv7,arm64 \ - windows:amd64,arm64 \ - darwin:amd64,arm64 -.PHONY: build + --output "$@" \ + "build/coder_$(VERSION)_$${os}_$${arch}" + +clean: + rm -rf build +.PHONY: clean # Runs migrations to output a dump of the database. coderd/database/dump.sql: coderd/database/dump/main.go $(wildcard coderd/database/migrations/*.sql) diff --git a/scripts/archive.sh b/scripts/archive.sh index 4f75e20c4a74a..267c67f6d2226 100755 --- a/scripts/archive.sh +++ b/scripts/archive.sh @@ -14,7 +14,7 @@ # utility and then notarized using the `gon` utility, which may take a while. # $AC_APPLICATION_IDENTITY must be set and the signing certificate must be # imported for this to work. Also, the input binary must already be signed with -# the `codesign` tool.= +# the `codesign` tool. # # If the --agpl parameter is specified, only includes AGPL license. # @@ -26,7 +26,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" format="" output_path="" -sign_darwin=0 +sign_darwin="${CODER_SIGN_DARWIN:-0}" agpl="${CODER_BUILD_AGPL:-0}" args="$(getopt -o "" -l format:,output:,sign-darwin,agpl -- "$@")" diff --git a/scripts/build_go.sh b/scripts/build_go.sh index fc5825e82725b..f1f3e34c7fa32 100755 --- a/scripts/build_go.sh +++ b/scripts/build_go.sh @@ -32,7 +32,7 @@ version="" os="${GOOS:-linux}" arch="${GOARCH:-amd64}" slim="${CODER_SLIM_BUILD:-0}" -sign_darwin=0 +sign_darwin="${CODER_SIGN_DARWIN:-0}" output_path="" agpl="${CODER_BUILD_AGPL:-0}" @@ -53,6 +53,7 @@ while true; do shift 2 ;; --output) + mkdir -p "$(dirname "$2")" output_path="$(realpath "$2")" shift 2 ;; diff --git a/scripts/package.sh b/scripts/package.sh index a9135f732fa3e..752478808d766 100755 --- a/scripts/package.sh +++ b/scripts/package.sh @@ -1,22 +1,21 @@ #!/usr/bin/env bash -# This script creates Linux packages for the given binary. It will output a -# .rpm, .deb and .apk file in the same directory as the input file with the same -# filename (except the package format suffix). +# This script creates a Linux package for the given binary. # -# ./package.sh --arch amd64 [--version 1.2.3] path/to/coder +# ./package.sh --arch amd64 --format "(apk|deb|rpm)" --output "path/to/coder.apk" [--version 1.2.3] path/to/coder # -# The --arch parameter is required. If no version is specified, defaults to the -# version from ./version.sh. +# If no version is specified, defaults to the version from ./version.sh. set -euo pipefail # shellcheck source=scripts/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -version="" arch="" +format="" +output_path="" +version="" -args="$(getopt -o "" -l arch:,version: -- "$@")" +args="$(getopt -o "" -l arch:,format:,output:,version: -- "$@")" eval set -- "$args" while true; do case "$1" in @@ -24,6 +23,15 @@ while true; do arch="$2" shift 2 ;; + --format) + format="$2" + shift 2 + ;; + --output) + mkdir -p "$(dirname "$2")" + output_path="$(realpath "$2")" + shift 2 + ;; --version) version="$2" shift 2 @@ -41,6 +49,12 @@ done if [[ "$arch" == "" ]]; then error "--arch is a required parameter" fi +if [[ "$format" != "apk" ]] && [[ "$format" != "deb" ]] && [[ "$format" != "rpm" ]]; then + error "--format is a required parameter and must be one of 'apk', 'deb', or 'rpm'" +fi +if [[ "$output_path" == "" ]]; then + error "--output is a required parameter" +fi if [[ "$#" != 1 ]]; then error "Exactly one argument must be provided to this script, $# were supplied" @@ -74,18 +88,12 @@ ln -P "$(realpath coder.service)" "$temp_dir/" ln -P "$(realpath preinstall.sh)" "$temp_dir/" ln -P "$(realpath scripts/nfpm.yaml)" "$temp_dir/" -cd "$temp_dir" - -formats=(apk deb rpm) -for format in "${formats[@]}"; do - output_path="$input_file.$format" - log "--- Building $format package ($output_path)" - +pushd "$temp_dir" GOARCH="$arch" CODER_VERSION="$version" nfpm package \ -f nfpm.yaml \ -p "$format" \ - -t "$output_path" -done + -t "$output_path" \ + 1>&2 +popd -cdroot rm -rf "$temp_dir" From 8765bc4eed38140a72bc0c1ce6cbb71fe4d3c5b4 Mon Sep 17 00:00:00 2001 From: Dean Sheather Date: Fri, 26 Aug 2022 17:34:01 +0000 Subject: [PATCH 02/12] chore: buff makefile pt. 2 --- .github/workflows/coder.yaml | 33 ++--- .github/workflows/release.yaml | 58 +++----- Makefile | 253 +++++++++++++++++++++------------ scripts/build_go.sh | 8 +- scripts/build_go_matrix.sh | 243 ------------------------------- scripts/build_go_slim.sh | 146 ------------------- scripts/publish_release.sh | 8 ++ 7 files changed, 199 insertions(+), 550 deletions(-) delete mode 100755 scripts/build_go_matrix.sh delete mode 100755 scripts/build_go_slim.sh diff --git a/.github/workflows/coder.yaml b/.github/workflows/coder.yaml index f15e3be0659c4..c98d026f8e9da 100644 --- a/.github/workflows/coder.yaml +++ b/.github/workflows/coder.yaml @@ -490,35 +490,21 @@ jobs: - name: Install zstd run: sudo apt-get install -y zstd - - name: Build site - run: make -B site/out/index.html - - name: Build Release run: | set -euo pipefail go mod download - mkdir -p ./dist - # build slim binaries - ./scripts/build_go_slim.sh \ - --output ./dist/ \ - --compress 22 \ - linux:amd64,armv7,arm64 \ - windows:amd64,arm64 \ - darwin:amd64,arm64 - - # build linux amd64 packages - ./scripts/build_go_matrix.sh \ - --output ./dist/ \ - --package-linux \ - linux:amd64 \ - windows:amd64 + version="$(./script/version.sh)" + make -j \ + build/coder_"$version"_windows_amd64.zip \ + build/coder_"$version"_linux_amd64.{tar.gz,deb} - name: Install Release run: | gcloud config set project coder-dogfood gcloud config set compute/zone us-central1-a - gcloud compute scp ./dist/coder_*_linux_amd64.deb coder:/tmp/coder.deb + gcloud compute scp ./build/coder_*_linux_amd64.deb coder:/tmp/coder.deb gcloud compute ssh coder -- sudo dpkg -i --force-confdef /tmp/coder.deb gcloud compute ssh coder -- sudo systemctl daemon-reload @@ -529,12 +515,9 @@ jobs: with: name: coder path: | - ./dist/*.zip - ./dist/*.exe - ./dist/*.tar.gz - ./dist/*.apk - ./dist/*.deb - ./dist/*.rpm + ./build/*.zip + ./build/*.tar.gz + ./build/*.deb retention-days: 7 test-js: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ba57b22024be8..a668024f61392 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -71,30 +71,15 @@ jobs: - name: Install zstd run: sudo apt-get install -y zstd - - name: Build Site - run: make site/out/index.html - - name: Build Linux and Windows Binaries run: | set -euo pipefail go mod download - mkdir -p ./dist - # build slim binaries - ./scripts/build_go_slim.sh \ - --output ./dist/ \ - --compress 22 \ - linux:amd64,armv7,arm64 \ - windows:amd64,arm64 \ - darwin:amd64,arm64 - - # build linux and windows binaries - ./scripts/build_go_matrix.sh \ - --output ./dist/ \ - --archive \ - --package-linux \ - linux:amd64,armv7,arm64 \ - windows:amd64,arm64 + version="$(./script/version.sh)" + make -j \ + build/coder_"$version"_linux_{amd64,armv7,arm64}.{tar.gz,apk,deb,rpm} \ + build/coder_"$version"_windows_{amd64,arm64}.zip \ - name: Build Linux Docker images run: | @@ -107,7 +92,7 @@ jobs: ./scripts/build_docker.sh \ ${{ (!github.event.inputs.dry_run && !github.event.inputs.snapshot) && '--push' || '' }} \ --arch "$arch" \ - ./dist/coder_*_linux_"$arch" + ./build/coder_*_linux_"$arch" )" images+=("$img") done @@ -139,11 +124,11 @@ jobs: with: name: linux path: | - dist/*.zip - dist/*.tar.gz - dist/*.apk - dist/*.deb - dist/*.rpm + ./build/*.zip + ./build/*.tar.gz + ./build/*.apk + ./build/*.deb + ./build/*.rpm # The mac binaries get built on mac runners because they need to be signed, # and the signing tool only runs on mac. This darwin job only builds the Mac @@ -213,22 +198,11 @@ jobs: set -euo pipefail go mod download - mkdir -p ./dist - # build slim binaries - ./scripts/build_go_slim.sh \ - --output ./dist/ \ - --compress 22 \ - linux:amd64,armv7,arm64 \ - windows:amd64,arm64 \ - darwin:amd64,arm64 - - # build darwin binaries - ./scripts/build_go_matrix.sh \ - --output ./dist/ \ - --archive \ - --sign-darwin \ - darwin:amd64,arm64 + version="$(./script/version.sh)" + make -j \ + build/coder_"$version"_darwin_{amd64,arm64}.zip env: + CODER_SIGN_DARWIN: "true" AC_USERNAME: ${{ secrets.AC_USERNAME }} AC_PASSWORD: ${{ secrets.AC_PASSWORD }} AC_APPLICATION_IDENTITY: BDB050EB749EDD6A80C6F119BF1382ECA119CCCC @@ -237,7 +211,7 @@ jobs: uses: actions/upload-artifact@v3 with: name: darwin - path: ./dist/coder_*.zip + path: ./build/*.zip publish: runs-on: ubuntu-latest @@ -279,7 +253,7 @@ jobs: run: | set -euxo pipefail ./scripts/helm.sh --push - mv ./dist/*.tgz ./artifacts/ + mv ./build/*.tgz ./artifacts/ - name: Publish Release run: | diff --git a/Makefile b/Makefile index 649cf6a400a8e..28d27cf038f26 100644 --- a/Makefile +++ b/Makefile @@ -1,102 +1,185 @@ -.DEFAULT_GOAL := build +.DEFAULT_GOAL := build-fat # Use a single bash shell for each job, and immediately exit on failure SHELL := bash -.SHELLFLAGS = -ceu +.SHELLFLAGS := -ceu .ONESHELL: # This doesn't work on directories. # See https://stackoverflow.com/questions/25752543/make-delete-on-error-for-directory-targets .DELETE_ON_ERROR: -INSTALL_DIR = $(shell go env GOPATH)/bin -GOOS = $(shell go env GOOS) -GOARCH = $(shell go env GOARCH) -VERSION = $(shell ./scripts/version.sh) +# Don't print the commands in the file unless you specify VERBOSE. This is +# essentially the same as putting "@" at the start of each line. +ifndef VERBOSE +.SILENT: +endif + +# Create the build directory if it does not exist. +$(shell mkdir -p build) + +INSTALL_DIR := $(shell go env GOPATH)/bin +GOOS := $(shell go env GOOS) +GOARCH := $(shell go env GOARCH) +VERSION := $(shell ./scripts/version.sh) # All ${OS}_${ARCH} combos we build for. Windows binaries have the .exe suffix. -OS_ARCHES = \ +OS_ARCHES := \ linux_amd64 linux_arm64 linux_armv7 \ darwin_amd64 darwin_arm64 \ windows_amd64.exe windows_arm64.exe # Archive formats and their corresponding ${OS}_${ARCH} combos. -ARCHIVE_TAR_GZ = linux_amd64 linux_arm64 linux_armv7 -ARCHIVE_ZIP = \ +ARCHIVE_TAR_GZ := linux_amd64 linux_arm64 linux_armv7 +ARCHIVE_ZIP := \ darwin darwin_arm64 \ windows_amd64 windows_arm64 -# All ${OS}_${ARCH} combos we build packages for. -PACKAGE_OS_ARCHES = linux_amd64 linux_armv7 linux_arm64 +# All package formats we build and the ${OS}_${ARCH} combos we build them for. +PACKAGE_FORMATS := apk deb rpm +PACKAGE_OS_ARCHES := linux_amd64 linux_armv7 linux_arm64 + +# Computed variables based on the above. +CODER_SLIM_BINARIES := $(addprefix build/coder-slim_$(VERSION)_,$(OS_ARCHES)) +CODER_FAT_BINARIES := $(addprefix build/coder_$(VERSION)_,$(OS_ARCHES)) +CODER_ALL_BINARIES := $(CODER_SLIM_BINARIES) $(CODER_FAT_BINARIES) +CODER_SLIM_NOVERSION_BINARIES := $(addprefix build/coder-slim_,$(OS_ARCHES)) +CODER_FAT_NOVERSION_BINARIES := $(addprefix build/coder_,$(OS_ARCHES)) +CODER_TAR_GZ_ARCHIVES := $(foreach os_arch, $(ARCHIVE_TAR_GZ), build/coder_$(VERSION)_$(os_arch).tar.gz) +CODER_ZIP_ARCHIVES := $(foreach os_arch, $(ARCHIVE_ZIP), build/coder_$(VERSION)_$(os_arch).zip) +CODER_ALL_ARCHIVES := $(CODER_TAR_GZ_ARCHIVES) $(CODER_ZIP_ARCHIVES) +CODER_ALL_PACKAGES := $(foreach os_arch, $(PACKAGE_OS_ARCHES), $(addprefix build/coder_$(VERSION)_$(os_arch).,$(PACKAGE_FORMATS))) -# All package formats we build. -PACKAGE_FORMATS = apk deb rpm +clean: + rm -rf build + mkdir -p build +.PHONY: clean -bin build-slim: $(addprefix build/coder-slim_$(VERSION)_,$(OS_ARCHES)) -.PHONY: bin build-slim +build-slim bin: $(CODER_SLIM_BINARIES) +.PHONY: build-slim bin -build build-full: $(addprefix build/coder_$(VERSION)_,$(OS_ARCHES)) -.PHONY: build build-full +build-fat build-full build: $(CODER_FAT_BINARIES) +.PHONY: build-fat build-full build + +build/coder-slim_$(VERSION)_checksums.sha1: $(CODER_SLIM_BINARIES) + pushd ./build + openssl dgst -r -sha1 coder-slim_"$(VERSION)"_* | tee "$(@F)" + popd + +build/coder-slim_$(VERSION).tar: build/coder-slim_$(VERSION)_checksums.sha1 $(CODER_SLIM_BINARIES) + pushd ./build + tar cf "$(@F)" coder-slim_"$(VERSION)"_* + popd + +build/coder-slim_$(VERSION).tar.zst site/out/coder.tar.zst: build/coder-slim_$(VERSION).tar + zstd -6 \ + --force \ + --ultra \ + --long \ + --no-progress \ + -o "build/coder-slim_$(VERSION).tar.zst" \ + "build/coder-slim_$(VERSION).tar" + + cp "build/coder-slim_$(VERSION).tar.zst" "site/out/coder.tar.zst" # Redirect from version-less targets to the versioned ones. This is kinda gross # since it's make shelling out to make, but it's the easiest way less we write # out every target manually. -$(addprefix build/coder_,$(OS_ARCHES)): site/out/index.html build-slim - @target="coder_$(VERSION)_$(@:build/coder_%=%)" - @$(MAKE) \ +# +# There is a similar target for slim binaries below. +# +# Called like this: +# make build/coder_linux_amd64 +# make build/coder_windows_amd64.exe +$(CODER_FAT_NOVERSION_BINARIES): site/out/index.html site/out/coder.tar.zst + target="coder_$(VERSION)_$(@:build/coder_%=%)" + $(MAKE) \ --no-print-directory \ --assume-old site/out/index.html \ - --assume-old build-slim \ + --assume-old site/out/coder.tar.zst \ "build/$$target" - @rm -f "$@" - @ln -s "$$target" "$@" -.PHONY: $(addprefix build/coder_,$(OS_ARCHES)) + rm -f "$@" + ln -s "$$target" "$@" +.PHONY: $(CODER_FAT_NONVERSION_BINARIES) -$(addprefix build/coder-slim_,$(OS_ARCHES)): - @target="coder-slim_$(VERSION)_$(@:build/coder-slim_%=%)" - @echo $(MAKE) \ +# Same as above, but for slim binaries. +# +# Called like this: +# make build/coder-slim_linux_amd64 +# make build/coder-slim_windows_amd64.exe +$(CODER_SLIM_NOVERSION_BINARIES): + target="coder-slim_$(VERSION)_$(@:build/coder-slim_%=%)" + $(MAKE) \ --no-print-directory \ "build/$$target" - @rm -f "$@" - @ln -s "$$target" "$@" -.PHONY: $(addprefix build/coder-slim_,$(OS_ARCHES)) + rm -f "$@" + ln -s "$$target" "$@" +.PHONY: $(CODER_SLIM_NOVERSION_BINARIES) -# "full" binaries always depend on all "slim" binaries. -$(addprefix build/coder_$(VERSION)_,$(OS_ARCHES)): site/out/index.html build-slim +# "fat" binaries always depend on the site and the compressed slim binaries. +$(CODER_FAT_BINARIES): site/out/index.html site/out/coder.tar.zst -# This task handles all builds, for both "full" and "slim" binaries. It parses +# This is a handy block that parses the target to determine whether it's "slim" +# or "fat", which OS was specified and which architecture was specified. +# +# It populates the following variables: mode, os, arch_ext, arch, ext (without +# dot). +define get-mode-os-arch-ext = + mode="$$([[ "$@" = build/coder-slim* ]] && echo "slim" || echo "fat")" + os="$$(echo $@ | cut -d_ -f3)" + arch_ext="$$(echo $@ | cut -d_ -f4)" + if [[ "$$arch_ext" == *.* ]]; then + arch="$$(echo $$arch_ext | cut -d. -f1)" + ext="$${arch_ext#*.}" + else + arch="$$arch_ext" + ext="" + fi +endef + +# This task handles all builds, for both "fat" and "slim" binaries. It parses # the target name to get the metadata for the build, so it must be specified in # this format: -# build/coder(-slim)?_$version_$os_$arch(.exe)? -$(addprefix build/coder_$(VERSION)_,$(OS_ARCHES)) $(addprefix build/coder-slim_$(VERSION)_,$(OS_ARCHES)): \ - go.mod go.sum \ +# build/coder(-slim)?_${version}_${os}_${arch}(.exe)? +# +# You should probably use the non-version targets above instead if you're +# calling this manually. +$(CODER_ALL_BINARIES): go.mod go.sum \ $(shell find . -not -path './vendor/*' -type f -name '*.go') \ $(shell find ./examples/templates) - @mkdir -p build - @mode=$$([[ "$@" = build/coder-slim* ]] && echo "slim" || echo "full") - @os=$$(echo $@ | cut -d_ -f3) - @arch=$$(echo $@ | cut -d_ -f4) - @if [ "$$mode" != "full" ] && [ "$$mode" != "slim" ]; then - @echo "Invalid build mode: $$mode" - @exit 1 - @fi - @if [[ "$$os" == "windows" ]] && [[ "$$arch" == *.exe ]]; then - @arch=$${arch%.exe} - @fi - - @build_args=( \ + $(get-mode-os-arch-ext) + if [[ "$$os" != "windows" ]] && [[ "$$ext" != "" ]]; then + echo "ERROR: Invalid build binary extension" 1>&2 + exit 1 + fi + if [[ "$$os" == "windows" ]] && [[ "$$ext" != exe ]]; then + echo "ERROR: Windows binaries must have an .exe extension." 1>&2 + exit 1 + fi + + build_args=( \ --os "$$os" \ --arch "$$arch" \ --version "$(VERSION)" \ --output "$@" \ ) - @if [ "$$mode" == "slim" ]; then - @build_args+=(--slim) - @fi + if [ "$$mode" == "slim" ]; then + build_args+=(--slim) + fi ./scripts/build_go.sh "$${build_args[@]}" + if [[ "$$mode" == "slim" ]]; then + dot_ext="" + if [[ "$$ext" != "" ]]; then + dot_ext=".$$ext" + fi + + mkdir -p ./site/out + cp "$@" "./site/out/coder-$$os-$$arch$$dot_ext" + fi + # This task builds all archives. It parses the target name to get the metadata # for the build, so it must be specified in this format: # build/coder_${version}_${os}_${arch}.${format} @@ -105,24 +188,17 @@ $(addprefix build/coder_$(VERSION)_,$(OS_ARCHES)) $(addprefix build/coder-slim_$ # .tar.gz: linux_amd64, linux_arm64, linux_armv7 # .zip: darwin_amd64, darwin_arm64, windows_amd64, windows_arm64 # -# This depends on build-full because it's difficult to do dynamic dependencies. -$(foreach os_arch, $(ARCHIVE_TAR_GZ), build/coder_$(VERSION)_$(os_arch).tar.gz) \ -$(foreach os_arch, $(ARCHIVE_ZIP), build/coder_$(VERSION)_$(os_arch).zip): \ - build-full - - @mkdir -p build - @os=$$(echo $@ | cut -d_ -f3) - @arch_format=$$(echo $@ | cut -d_ -f4) - @arch=$$(echo $$arch_format | cut -d. -f1) - @format=$${arch_format#*.} - - @bin_ext="" - @if [[ "$$os" == "windows" ]]; then - @bin_ext=".exe" - @fi - - ./script/archive.sh \ - --format "$$format" \ +# This depends on all fat binaries because it's difficult to do dynamic +# dependencies. These targets are typically only used during release anyways. +$(CODER_ALL_ARCHIVES): $(CODER_FAT_BINARIES) + $(get-mode-os-arch-ext) + bin_ext="" + if [[ "$$os" == "windows" ]]; then + bin_ext=".exe" + fi + + ./scripts/archive.sh \ + --format "$$ext" \ --output "$@" \ "build/coder_$(VERSION)_$${os}_$${arch}$${bin_ext}" @@ -132,27 +208,18 @@ $(foreach os_arch, $(ARCHIVE_ZIP), build/coder_$(VERSION)_$(os_arch).zip): \ # # Supports apk, deb, rpm for all linux targets. # -# This depends on build-full because it's difficult to do dynamic dependencies. -$(foreach os_arch, $(PACKAGE_OS_ARCHES), $(addprefix build/coder_$(VERSION)_$(os_arch).,$(PACKAGE_FORMATS))): \ - build-full - - @mkdir -p build - @os=$$(echo $@ | cut -d_ -f3) - @arch_format=$$(echo $@ | cut -d_ -f4) - @arch=$$(echo $$arch_format | cut -d. -f1) - @format=$${arch_format#*.} +# This depends on all fat binaries because it's difficult to do dynamic +# dependencies. These targets are typically only used during release anyways. +$(CODER_ALL_PACKAGES): $(CODER_FAT_BINARIES) + $(get-mode-os-arch-ext) ./scripts/package.sh \ --arch "$$arch" \ - --format "$$format" \ + --format "$$ext" \ --version "$(VERSION)" \ --output "$@" \ "build/coder_$(VERSION)_$${os}_$${arch}" -clean: - rm -rf build -.PHONY: clean - # Runs migrations to output a dump of the database. coderd/database/dump.sql: coderd/database/dump/main.go $(wildcard coderd/database/migrations/*.sql) go run coderd/database/dump/main.go @@ -162,7 +229,7 @@ coderd/database/querier.go: coderd/database/sqlc.yaml coderd/database/dump.sql $ coderd/database/generate.sh fmt/prettier: - @echo "--- prettier" + echo "--- prettier" cd site # Avoid writing files in CI to reduce file write activity ifdef CI @@ -177,7 +244,7 @@ fmt/terraform: $(wildcard *.tf) .PHONY: fmt/terraform fmt/shfmt: $(shell shfmt -f .) - @echo "--- shfmt" + echo "--- shfmt" # Only do diff check in CI, errors on diff. ifdef CI shfmt -d $(shell shfmt -f .) @@ -193,13 +260,13 @@ gen: coderd/database/querier.go peerbroker/proto/peerbroker.pb.go provisionersdk .PHONY: gen install: site/out/index.html $(shell find . -not -path './vendor/*' -type f -name '*.go') go.mod go.sum $(shell find ./examples/templates) - @output_file="$(INSTALL_DIR)/coder" + output_file="$(INSTALL_DIR)/coder" - @if [[ "$(GOOS)" == "windows" ]]; then - @output_file="$${output_file}.exe" - @fi + if [[ "$(GOOS)" == "windows" ]]; then + output_file="$${output_file}.exe" + fi - @echo "-- Building CLI for $(GOOS) $(GOARCH) at $$output_file" + echo "-- Building CLI for $(GOOS) $(GOARCH) at $$output_file" ./scripts/build_go.sh \ --version "$(VERSION)" \ @@ -207,7 +274,7 @@ install: site/out/index.html $(shell find . -not -path './vendor/*' -type f -nam --os "$(GOOS)" \ --arch "$(GOARCH)" - @echo + echo .PHONY: install lint: lint/shellcheck lint/go @@ -220,7 +287,7 @@ lint/go: # Use shfmt to determine the shell files, takes editorconfig into consideration. lint/shellcheck: $(shell shfmt -f .) - @echo "--- shellcheck" + echo "--- shellcheck" shellcheck --external-sources $(shell shfmt -f .) .PHONY: lint/shellcheck diff --git a/scripts/build_go.sh b/scripts/build_go.sh index f1f3e34c7fa32..990c0d79ec249 100755 --- a/scripts/build_go.sh +++ b/scripts/build_go.sh @@ -133,7 +133,13 @@ CGO_ENABLED=0 GOOS="$os" GOARCH="$arch" GOARM="$arm_version" go build \ "$cmd_path" 1>&2 if [[ "$sign_darwin" == 1 ]] && [[ "$os" == "darwin" ]]; then - codesign -s "$AC_APPLICATION_IDENTITY" -f -v --timestamp --options runtime "$output_path" + codesign \ + -f -v \ + -s "$AC_APPLICATION_IDENTITY" \ + --timestamp \ + --options runtime \ + "$output_path" \ + 1>&2 fi echo "$output_path" diff --git a/scripts/build_go_matrix.sh b/scripts/build_go_matrix.sh deleted file mode 100755 index cb6b2bedec203..0000000000000 --- a/scripts/build_go_matrix.sh +++ /dev/null @@ -1,243 +0,0 @@ -#!/usr/bin/env bash - -# This script builds multiple Go binaries for Coder with the given OS and -# architecture combinations. -# -# Usage: ./build_go_matrix.sh [--version 1.2.3-devel+abcdef] [--output dist/] [--slim] [--sign-darwin] [--archive] [--package-linux] [--agpl] os1:arch1,arch2 os2:arch1 os1:arch3 -# -# If no OS:arch combinations are provided, nothing will happen and no error will -# be returned. Slim builds are disabled by default. If no version is specified, -# defaults to the version from ./version.sh -# -# The --output parameter must be a directory with a trailing slash where all -# files will be dropped with the default name scheme -# `coder_$version_$os_$arch(.exe)?`, or must contain the `{os}` and `{arch}` -# template variables. You may also use `{version}`. Note that for windows builds -# the `.exe` suffix will be appended automatically. -# -# Unless overridden via --output, the built binary will be dropped in -# "$repo_root/dist/coder_$version_$os_$arch" (with a ".exe" suffix for windows -# builds). -# -# If the --sign-darwin parameter is specified, all darwin binaries will be -# signed using the `codesign` utility. $AC_APPLICATION_IDENTITY must be set and -# the signing certificate must be imported for this to work. -# -# If the --archive parameter is specified, all binaries will be archived using -# ./archive.sh. The --sign-darwin parameter will be carried through, and all -# archive files will be dropped in the output directory with the same name as -# the binary and the .zip (for windows and darwin) or .tar.gz extension. -# -# If the --package-linux parameter is specified, all linux binaries will be -# packaged using ./package.sh. Requires the nfpm binary. -# -# If the --agpl parameter is specified, builds only the AGPL-licensed code (no -# Coder enterprise features). - -set -euo pipefail -# shellcheck source=scripts/lib.sh -source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" - -version="" -output_path="" -slim=0 -sign_darwin=0 -archive=0 -package_linux=0 -agpl=0 - -args="$(getopt -o "" -l version:,output:,slim,sign-darwin,archive,package-linux,agpl -- "$@")" -eval set -- "$args" -while true; do - case "$1" in - --version) - version="$2" - shift 2 - ;; - --output) - output_path="$2" - shift 2 - ;; - --slim) - slim=1 - shift - ;; - --sign-darwin) - if [[ "${AC_APPLICATION_IDENTITY:-}" == "" ]]; then - error "AC_APPLICATION_IDENTITY must be set when --sign-darwin is supplied" - fi - sign_darwin=1 - shift - ;; - --archive) - archive=1 - shift - ;; - --package-linux) - package_linux=1 - shift - ;; - --agpl) - agpl=1 - shift - ;; - --) - shift - break - ;; - *) - error "Unrecognized option: $1" - ;; - esac -done - -# Verify the output path template. -if [[ "$output_path" == "" ]]; then - # Input paths are relative, so we don't cdroot at the top, but for this case - # we want it to be relative to the root. - cdroot - mkdir -p dist - output_path="$(realpath "dist/coder_{version}_{os}_{arch}")" -elif [[ "$output_path" == */ ]]; then - output_path="${output_path}coder_{version}_{os}_{arch}" -elif [[ "$output_path" != *"{os}"* ]] || [[ "$output_path" != *"{arch}"* ]]; then - # If the output path isn't a directory (ends with /) then it must have - # template variables. - error "Templated output path '$output_path' must contain {os} and {arch}" -fi - -mkdir -p "$(dirname "$output_path")" -output_path="$(realpath "$output_path")" - -# Remove the "v" prefix. -version="${version#v}" -if [[ "$version" == "" ]]; then - version="$(execrelative ./version.sh)" -fi - -# Parse the os:arch specs into an array. -specs=() -may_zip=0 -may_tar=0 -for spec in "$@"; do - spec_os="$(echo "$spec" | cut -d ":" -f 1)" - if [[ "$spec_os" == "" ]] || [[ "$spec_os" == *" "* ]]; then - error "Could not parse matrix build spec '$spec': invalid OS '$spec_os'" - fi - - # Determine which dependencies we need. - if [[ "$spec_os" == "windows" ]] || [[ "$spec_os" == "darwin" ]]; then - may_zip=1 - else - may_tar=1 - fi - - # No quoting is important here. - for spec_arch in $(echo "$spec" | cut -d ":" -f 2 | tr "," "\n"); do - if [[ "$spec_arch" == "" ]] || [[ "$spec_os" == *" "* ]]; then - error "Could not parse matrix build spec '$spec': invalid architecture '$spec_arch'" - fi - - specs+=("$spec_os:$spec_arch") - done -done - -# Remove duplicate specs while maintaining the same order. -specs_str="${specs[*]}" -specs=() -for s in $(echo "$specs_str" | tr " " "\n" | awk '!a[$0]++'); do - specs+=("$s") -done - -# Check dependencies -dependencies go -if [[ "$sign_darwin" == 1 ]]; then - dependencies jq codesign gon -fi -if [[ "$archive" == 1 ]]; then - if [[ "$may_zip" == 1 ]]; then - dependencies zip - fi - if [[ "$may_tar" == 1 ]]; then - dependencies tar - fi -fi -if [[ "$package_linux" == 1 ]]; then - dependencies nfpm -fi - -bin_name="coder" -build_args=() -if [[ "$slim" == 1 ]]; then - bin_name+="-slim" - build_args+=(--slim) -fi -if [[ "$sign_darwin" == 1 ]]; then - build_args+=(--sign-darwin) -fi -if [[ "$agpl" == 1 ]]; then - build_args+=(--agpl) -fi - -# Build each spec. -for spec in "${specs[@]}"; do - spec_os="$(echo "$spec" | cut -d ":" -f 1)" - spec_arch="$(echo "$spec" | cut -d ":" -f 2)" - - # Craft output path from the template. - spec_output="$output_path" - spec_output="${spec_output//\{os\}/"$spec_os"}" - spec_output="${spec_output//\{arch\}/"$spec_arch"}" - spec_output="${spec_output//\{version\}/"$version"}" - - spec_output_binary="$spec_output" - if [[ "$spec_os" == "windows" ]]; then - spec_output_binary+=".exe" - fi - - # Ensure parent dir. - mkdir -p "$(dirname "$spec_output")" - - log "--- Building $bin_name for $spec_os $spec_arch ($spec_output_binary)" - execrelative ./build_go.sh \ - --version "$version" \ - --os "$spec_os" \ - --arch "$spec_arch" \ - --output "$spec_output_binary" \ - "${build_args[@]}" - log - log - - if [[ "$archive" == 1 ]]; then - spec_archive_format="tar.gz" - if [[ "$spec_os" == "windows" ]] || [[ "$spec_os" == "darwin" ]]; then - spec_archive_format="zip" - fi - spec_output_archive="$spec_output.$spec_archive_format" - - archive_args=() - if [[ "$sign_darwin" == 1 ]] && [[ "$spec_os" == "darwin" ]]; then - archive_args+=(--sign-darwin) - fi - if [[ "$agpl" == 1 ]]; then - archive_args+=(--agpl) - fi - - log "--- Creating archive for $spec_os $spec_arch ($spec_output_archive)" - execrelative ./archive.sh \ - --format "$spec_archive_format" \ - --output "$spec_output_archive" \ - "${archive_args[@]}" \ - "$spec_output_binary" - log - log - fi - - if [[ "$package_linux" == 1 ]] && [[ "$spec_os" == "linux" ]]; then - execrelative ./package.sh \ - --arch "$spec_arch" \ - --version "$version" \ - "$spec_output_binary" - log - fi -done diff --git a/scripts/build_go_slim.sh b/scripts/build_go_slim.sh deleted file mode 100755 index 69320a8a3c8e7..0000000000000 --- a/scripts/build_go_slim.sh +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/env bash - -# This script builds multiple "slim" Go binaries for Coder with the given OS and -# architecture combinations. This wraps ./build_go_matrix.sh. -# -# Usage: ./build_go_slim.sh [--version 1.2.3-devel+abcdef] [--output dist/] [--compress 22] [--agpl] os1:arch1,arch2 os2:arch1 os1:arch3 -# -# If no OS:arch combinations are provided, nothing will happen and no error will -# be returned. If no version is specified, defaults to the version from -# ./version.sh -# -# The --output parameter differs from ./build_go_matrix.sh, in that it does not -# accept variables such as `{os}` and `{arch}` and only accepts a directory -# ending with `/`. -# -# The built binaries are additionally copied to the site output directory so -# they can be packaged into non-slim binaries correctly. -# -# When the --compress parameter is provided, the binaries in site/bin -# will be compressed using zstd into site/bin/coder.tar.zst, this helps reduce -# final binary size significantly. -# -# If the --agpl parameter is specified, builds only the AGPL-licensed code (no -# Coder enterprise features). - -set -euo pipefail -shopt -s nullglob -# shellcheck source=scripts/lib.sh -source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" - -version="" -output_path="" -compress=0 -agpl=0 - -args="$(getopt -o "" -l version:,output:,compress:,agpl -- "$@")" -eval set -- "$args" -while true; do - case "$1" in - --version) - version="$2" - shift 2 - ;; - --output) - output_path="$2" - shift 2 - ;; - --compress) - compress="$2" - shift 2 - ;; - --agpl) - agpl=1 - shift - ;; - --) - shift - break - ;; - *) - error "Unrecognized option: $1" - ;; - esac -done - -# Check dependencies -dependencies go -if [[ $compress != 0 ]]; then - dependencies openssl tar zstd - - if [[ $compress != [0-9]* ]] || [[ $compress -gt 22 ]] || [[ $compress -lt 1 ]]; then - error "Invalid value for compress, must in in the range of [1, 22]" - fi -fi - -# Remove the "v" prefix. -version="${version#v}" -if [[ "$version" == "" ]]; then - version="$(execrelative ./version.sh)" -fi - -# Verify the output path. -if [[ "$output_path" == "" ]]; then - # Input paths are relative, so we don't cdroot at the top, but for this case - # we want it to be relative to the root. - cdroot - mkdir -p dist - output_path="$(realpath "dist/coder-slim_{version}_{os}_{arch}")" -elif [[ "$output_path" != */ ]] || [[ "$output_path" == *"{"* ]]; then - error "The output path '$output_path' cannot contain variables and must end with a slash" -else - mkdir -p "$output_path" - output_path="$(realpath "${output_path}coder-slim_{version}_{os}_{arch}")" -fi - -build_args=(--slim) -if [[ "$agpl" == 1 ]]; then - build_args+=(--agpl) -fi - -./scripts/build_go_matrix.sh \ - --version "$version" \ - --output "$output_path" \ - "${build_args[@]}" \ - "$@" - -cdroot -dest_dir="./site/out/bin" -mkdir -p "$dest_dir" -dest_dir="$(realpath "$dest_dir")" - -# Copy the binaries to the site directory. -cd "$(dirname "$output_path")" -for f in ./coder-slim_*; do - # Remove ./ prefix - name="${f#./}" - # Remove "-slim_$version" - truncated="${name//-slim_$version/}" - # Replace underscores with hyphens - hyphenated="${truncated//_/-}" - dest="$dest_dir/$hyphenated" - cp "$f" "$dest" -done - -if [[ $compress != 0 ]]; then - pushd "$dest_dir" - sha_file=coder.sha1 - sha_dest="$dest_dir/$sha_file" - log "--- Generating SHA1 for coder-slim binaries ($sha_dest)" - openssl dgst -r -sha1 coder-* | tee $sha_file - echo "$sha_dest" - log - log - - tar_name=coder.tar.zst - tar_dest="$dest_dir/$tar_name" - log "--- Compressing coder-slim binaries using zstd level $compress ($tar_dest)" - tar cf coder.tar $sha_file coder-* - rm coder-* - zstd --force --ultra --long -"${compress}" --rm --no-progress coder.tar -o $tar_name - echo "$tar_dest" - log - log - - popd -fi diff --git a/scripts/publish_release.sh b/scripts/publish_release.sh index 338ffbb99ff7b..80725cca9a931 100755 --- a/scripts/publish_release.sh +++ b/scripts/publish_release.sh @@ -3,6 +3,10 @@ # This script generates release notes and publishes all of the given assets to # GitHub releases. Depends on GitHub CLI. # +# THIS IS NOT INTENDED TO BE CALLED BY DEVELOPERS! This is called by the release +# pipeline to do the final publish step. If you want to create a release use: +# git tag -a -m "$ver" "$ver" && git push origin "$ver" +# # Usage: ./publish_release.sh [--version 1.2.3] [--dry-run] path/to/asset1 path/to/asset2 ... # # The supplied images must already be pushed to the registry or this will fail. @@ -25,6 +29,10 @@ set -euo pipefail # shellcheck source=scripts/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +if [[ "${CI:-}" == "" ]]; then + error "This script must be run in CI" +fi + version="" dry_run=0 From e208509fdbb94dc07c87107f953f4358613f3347 Mon Sep 17 00:00:00 2001 From: Dean Sheather Date: Fri, 26 Aug 2022 19:30:30 +0000 Subject: [PATCH 03/12] chore: buff makefile pt. 3 docker --- Makefile | 87 +++++++++++++++++++++++++++++++++++------ scripts/build_docker.sh | 15 +++++-- 2 files changed, 85 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 28d27cf038f26..c7802f5a6ffa5 100644 --- a/Makefile +++ b/Makefile @@ -31,24 +31,36 @@ OS_ARCHES := \ # Archive formats and their corresponding ${OS}_${ARCH} combos. ARCHIVE_TAR_GZ := linux_amd64 linux_arm64 linux_armv7 -ARCHIVE_ZIP := \ +ARCHIVE_ZIP := \ darwin darwin_arm64 \ windows_amd64 windows_arm64 # All package formats we build and the ${OS}_${ARCH} combos we build them for. -PACKAGE_FORMATS := apk deb rpm +PACKAGE_FORMATS := apk deb rpm PACKAGE_OS_ARCHES := linux_amd64 linux_armv7 linux_arm64 +# All architectures we build Docker images for (Linux only). +DOCKER_ARCHES := amd64 arm64 armv7 + # Computed variables based on the above. -CODER_SLIM_BINARIES := $(addprefix build/coder-slim_$(VERSION)_,$(OS_ARCHES)) -CODER_FAT_BINARIES := $(addprefix build/coder_$(VERSION)_,$(OS_ARCHES)) -CODER_ALL_BINARIES := $(CODER_SLIM_BINARIES) $(CODER_FAT_BINARIES) -CODER_SLIM_NOVERSION_BINARIES := $(addprefix build/coder-slim_,$(OS_ARCHES)) -CODER_FAT_NOVERSION_BINARIES := $(addprefix build/coder_,$(OS_ARCHES)) -CODER_TAR_GZ_ARCHIVES := $(foreach os_arch, $(ARCHIVE_TAR_GZ), build/coder_$(VERSION)_$(os_arch).tar.gz) -CODER_ZIP_ARCHIVES := $(foreach os_arch, $(ARCHIVE_ZIP), build/coder_$(VERSION)_$(os_arch).zip) -CODER_ALL_ARCHIVES := $(CODER_TAR_GZ_ARCHIVES) $(CODER_ZIP_ARCHIVES) -CODER_ALL_PACKAGES := $(foreach os_arch, $(PACKAGE_OS_ARCHES), $(addprefix build/coder_$(VERSION)_$(os_arch).,$(PACKAGE_FORMATS))) +CODER_SLIM_BINARIES := $(addprefix build/coder-slim_$(VERSION)_,$(OS_ARCHES)) +CODER_FAT_BINARIES := $(addprefix build/coder_$(VERSION)_,$(OS_ARCHES)) +CODER_ALL_BINARIES := $(CODER_SLIM_BINARIES) $(CODER_FAT_BINARIES) +CODER_TAR_GZ_ARCHIVES := $(foreach os_arch, $(ARCHIVE_TAR_GZ), build/coder_$(VERSION)_$(os_arch).tar.gz) +CODER_ZIP_ARCHIVES := $(foreach os_arch, $(ARCHIVE_ZIP), build/coder_$(VERSION)_$(os_arch).zip) +CODER_ALL_ARCHIVES := $(CODER_TAR_GZ_ARCHIVES) $(CODER_ZIP_ARCHIVES) +CODER_ALL_PACKAGES := $(foreach os_arch, $(PACKAGE_OS_ARCHES), $(addprefix build/coder_$(VERSION)_$(os_arch).,$(PACKAGE_FORMATS))) +CODER_ALL_ARCH_IMAGES := $(foreach arch, $(DOCKER_ARCHES), build/coder_$(VERSION)_linux_$(arch).tag) +CODER_ALL_ARCH_IMAGES_PUSHED := $(addprefix push/, $(CODER_ALL_ARCH_IMAGES)) +CODER_MAIN_IMAGE := build/coder_$(VERSION)_linux.tag +CODER_ALL_IMAGES := $(CODER_ALL_ARCH_IMAGES) $(CODER_MAIN_IMAGE) +CODER_ALL_IMAGES_PUSHED := $(addprefix push/, $(CODER_ALL_IMAGES)) + +CODER_SLIM_NOVERSION_BINARIES := $(addprefix build/coder-slim_,$(OS_ARCHES)) +CODER_FAT_NOVERSION_BINARIES := $(addprefix build/coder_,$(OS_ARCHES)) +CODER_ALL_NOVERSION_IMAGES := $(foreach arch, $(DOCKER_ARCHES), build/coder_linux_$(arch).tag) build/coder_linux.tag +CODER_ALL_NOVERSION_IMAGES_PUSHED := $(addprefix push/, $(CODER_ALL_NOVERSION_IMAGES)) + clean: rm -rf build @@ -189,7 +201,8 @@ $(CODER_ALL_BINARIES): go.mod go.sum \ # .zip: darwin_amd64, darwin_arm64, windows_amd64, windows_arm64 # # This depends on all fat binaries because it's difficult to do dynamic -# dependencies. These targets are typically only used during release anyways. +# dependencies due to the .exe requirement on Windows. These targets are +# typically only used during release anyways. $(CODER_ALL_ARCHIVES): $(CODER_FAT_BINARIES) $(get-mode-os-arch-ext) bin_ext="" @@ -209,7 +222,8 @@ $(CODER_ALL_ARCHIVES): $(CODER_FAT_BINARIES) # Supports apk, deb, rpm for all linux targets. # # This depends on all fat binaries because it's difficult to do dynamic -# dependencies. These targets are typically only used during release anyways. +# dependencies due to the .exe requirement on Windows. These targets are +# typically only used during release anyways. $(CODER_ALL_PACKAGES): $(CODER_FAT_BINARIES) $(get-mode-os-arch-ext) @@ -220,6 +234,53 @@ $(CODER_ALL_PACKAGES): $(CODER_FAT_BINARIES) --output "$@" \ "build/coder_$(VERSION)_$${os}_$${arch}" +# Redirect from version-less Docker image targets to the versioned ones. +# +# Called like this: +# make build/coder_linux_amd64.tag +$(CODER_ALL_NOVERSION_IMAGES): build/coder_%: build/coder_$(VERSION)_% +.PHONY: $(CODER_ALL_NOVERSION_IMAGES) + +# Redirect from version-less push Docker image targets to the versioned ones. +# +# Called like this: +# make push/build/coder_linux_amd64.tag +$(CODER_ALL_NOVERSION_IMAGES_PUSHED): push/build/coder_%: push/build/coder_$(VERSION)_% +.PHONY: $(CODER_ALL_NOVERSION_IMAGES_PUSHED) + +# This task builds all Docker images. It parses the target name to get the +# metadata for the build, so it must be specified in this format: +# build/coder_${version}_${os}_${arch}.tag +# +# Supports linux_amd64, linux_arm64, linux_armv7. +$(CODER_ALL_ARCH_IMAGES): build/coder_$(VERSION)_%.tag: build/coder_$(VERSION)_% + $(get-mode-os-arch-ext) + + image_tag="$$(./scripts/image_tag.sh --arch "$$arch" --version "$(VERSION)")" + ./scripts/build_docker.sh \ + --arch "$$arch" \ + --target "$$image_tag" \ + --version "$(VERSION)" \ + "build/coder_$(VERSION)_$${os}_$${arch}" + + echo "$$image_tag" > "$@" + +# Multi-arch Docker image. This requires all architecture-specific images to be +# built AND pushed. +build/coder_$(VERSION)_linux.tag: $(CODER_ALL_ARCH_IMAGES_PUSHED) + image_tag="$$(./scripts/image_tag.sh --version "$(VERSION)")" + ./scripts/build_docker_multiarch.sh \ + --target "$$image_tag" \ + --version "$(VERSION)" \ + $(foreach img, $^, "$$(cat "$(img)")") + + echo "$$image_tag" > "$@" + +$(CODER_ALL_IMAGES_PUSHED): push/%: % + image_tag="$$(cat "$<")" + docker push "$$image_tag" +.PHONY: $(CODER_ALL_IMAGES_PUSHED) + # Runs migrations to output a dump of the database. coderd/database/dump.sql: coderd/database/dump/main.go $(wildcard coderd/database/migrations/*.sql) go run coderd/database/dump/main.go diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh index c84913e658ab7..a47400f7b228b 100755 --- a/scripts/build_docker.sh +++ b/scripts/build_docker.sh @@ -3,14 +3,14 @@ # This script builds a Docker image of Coder containing the given binary, for # the given architecture. Only linux binaries are supported at this time. # -# Usage: ./build_docker.sh --arch amd64 [--version 1.2.3] [--push] path/to/coder +# Usage: ./build_docker.sh --arch amd64 [--version 1.2.3] [--target image_tag] [--push] path/to/coder # # The --arch parameter is required and accepts a Golang arch specification. It # will be automatically mapped to a suitable architecture that Docker accepts # before being passed to `docker buildx build`. # # The image will be built and tagged against the image tag returned by -# ./image_tag.sh. +# ./image_tag.sh unless a --target parameter is supplied. # # If no version is specified, defaults to the version from ./version.sh. # @@ -23,10 +23,11 @@ set -euo pipefail source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" arch="" +image_tag="" version="" push=0 -args="$(getopt -o "" -l arch:,version:,push -- "$@")" +args="$(getopt -o "" -l arch:,target:,version:,push -- "$@")" eval set -- "$args" while true; do case "$1" in @@ -34,6 +35,10 @@ while true; do arch="$2" shift 2 ;; + --target) + image_tag="$2" + shift 2 + ;; --version) version="$2" shift 2 @@ -65,7 +70,9 @@ if [[ "$version" == "" ]]; then version="$(execrelative ./version.sh)" fi -image_tag="$(execrelative ./image_tag.sh --arch "$arch" --version="$version")" +if [[ "$image_tag" == "" ]]; then + image_tag="$(execrelative ./image_tag.sh --arch "$arch" --version="$version")" +fi if [[ "$#" != 1 ]]; then error "Exactly one argument must be provided to this script, $# were supplied" From 5a5cd1d835e570838564a7a0cb51a0188d744a3c Mon Sep 17 00:00:00 2001 From: Dean Sheather Date: Fri, 26 Aug 2022 19:32:55 +0000 Subject: [PATCH 04/12] fixup! chore: buff makefile pt. 3 docker --- scripts/package.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/package.sh b/scripts/package.sh index 752478808d766..918f7bfe21354 100755 --- a/scripts/package.sh +++ b/scripts/package.sh @@ -89,11 +89,11 @@ ln -P "$(realpath preinstall.sh)" "$temp_dir/" ln -P "$(realpath scripts/nfpm.yaml)" "$temp_dir/" pushd "$temp_dir" - GOARCH="$arch" CODER_VERSION="$version" nfpm package \ - -f nfpm.yaml \ - -p "$format" \ - -t "$output_path" \ - 1>&2 +GOARCH="$arch" CODER_VERSION="$version" nfpm package \ + -f nfpm.yaml \ + -p "$format" \ + -t "$output_path" \ + 1>&2 popd rm -rf "$temp_dir" From 29de6ced14e27f4e08ebe754695676ea850cea98 Mon Sep 17 00:00:00 2001 From: Dean Sheather Date: Fri, 26 Aug 2022 20:06:46 +0000 Subject: [PATCH 05/12] chore: buff makefile pt. 4 make install --- Makefile | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index c7802f5a6ffa5..b83cec64eab18 100644 --- a/Makefile +++ b/Makefile @@ -15,12 +15,12 @@ ifndef VERBOSE .SILENT: endif -# Create the build directory if it does not exist. -$(shell mkdir -p build) +# Create the output directories if they do not exist. +$(shell mkdir -p build site/out/bin) -INSTALL_DIR := $(shell go env GOPATH)/bin GOOS := $(shell go env GOOS) GOARCH := $(shell go env GOARCH) +GOOS_BIN_EXT := $(if $(filter windows, $(GOOS)),.exe,) VERSION := $(shell ./scripts/version.sh) # All ${OS}_${ARCH} combos we build for. Windows binaries have the .exe suffix. @@ -63,8 +63,9 @@ CODER_ALL_NOVERSION_IMAGES_PUSHED := $(addprefix push/, $(CODER_ALL_NOVERSION_IM clean: - rm -rf build - mkdir -p build + rm -rf build site/out + mkdir -p build site/out/bin + git restore site/out .PHONY: clean build-slim bin: $(CODER_SLIM_BINARIES) @@ -320,22 +321,12 @@ fmt: fmt/prettier fmt/terraform fmt/shfmt gen: coderd/database/querier.go peerbroker/proto/peerbroker.pb.go provisionersdk/proto/provisioner.pb.go provisionerd/proto/provisionerd.pb.go site/src/api/typesGenerated.ts .PHONY: gen -install: site/out/index.html $(shell find . -not -path './vendor/*' -type f -name '*.go') go.mod go.sum $(shell find ./examples/templates) - output_file="$(INSTALL_DIR)/coder" +install: build/coder_$(VERSION)_$(GOOS)_$(GOARCH)$(GOOS_BIN_EXT) + install_dir="$$(go env GOPATH)/bin" + output_file="$${install_dir}/coder$(GOOS_BIN_EXT)" - if [[ "$(GOOS)" == "windows" ]]; then - output_file="$${output_file}.exe" - fi - - echo "-- Building CLI for $(GOOS) $(GOARCH) at $$output_file" - - ./scripts/build_go.sh \ - --version "$(VERSION)" \ - --output "$$output_file" \ - --os "$(GOOS)" \ - --arch "$(GOARCH)" - - echo + mkdir -p "$$install_dir" + cp "$<" "$$output_file" .PHONY: install lint: lint/shellcheck lint/go From 7a0f045fdcee7c392b6e4aeb170906cd475dcdb1 Mon Sep 17 00:00:00 2001 From: Dean Sheather Date: Sun, 28 Aug 2022 19:20:16 +0000 Subject: [PATCH 06/12] chore: buff makefile pt. 5 helm stuff --- Makefile | 39 +++++++------- scripts/archive.sh | 3 +- scripts/build_go.sh | 10 ++-- scripts/check_unstaged.sh | 44 ++++++++-------- scripts/coder-dev.sh | 15 +++--- scripts/develop.sh | 106 ++++++++++++++++---------------------- scripts/helm.sh | 6 +-- scripts/yarn_install.sh | 56 ++++++++++---------- 8 files changed, 129 insertions(+), 150 deletions(-) diff --git a/Makefile b/Makefile index b83cec64eab18..76b9bb7fcf022 100644 --- a/Makefile +++ b/Makefile @@ -95,39 +95,24 @@ build/coder-slim_$(VERSION).tar.zst site/out/coder.tar.zst: build/coder-slim_$(V cp "build/coder-slim_$(VERSION).tar.zst" "site/out/coder.tar.zst" -# Redirect from version-less targets to the versioned ones. This is kinda gross -# since it's make shelling out to make, but it's the easiest way less we write -# out every target manually. -# -# There is a similar target for slim binaries below. +# Redirect from version-less targets to the versioned ones. There is a similar +# target for slim binaries below. # # Called like this: # make build/coder_linux_amd64 # make build/coder_windows_amd64.exe -$(CODER_FAT_NOVERSION_BINARIES): site/out/index.html site/out/coder.tar.zst - target="coder_$(VERSION)_$(@:build/coder_%=%)" - $(MAKE) \ - --no-print-directory \ - --assume-old site/out/index.html \ - --assume-old site/out/coder.tar.zst \ - "build/$$target" +$(CODER_FAT_NOVERSION_BINARIES): build/coder_%: build/coder_$(VERSION)_% rm -f "$@" - ln -s "$$target" "$@" -.PHONY: $(CODER_FAT_NONVERSION_BINARIES) + ln -s "$$(basename "$<")" "$@" # Same as above, but for slim binaries. # # Called like this: # make build/coder-slim_linux_amd64 # make build/coder-slim_windows_amd64.exe -$(CODER_SLIM_NOVERSION_BINARIES): - target="coder-slim_$(VERSION)_$(@:build/coder-slim_%=%)" - $(MAKE) \ - --no-print-directory \ - "build/$$target" +$(CODER_SLIM_NOVERSION_BINARIES): build/coder-slim_%: build/coder-slim_$(VERSION)_% rm -f "$@" - ln -s "$$target" "$@" -.PHONY: $(CODER_SLIM_NOVERSION_BINARIES) + ln -s "$$(basename "$<")" "$@" # "fat" binaries always depend on the site and the compressed slim binaries. $(CODER_FAT_BINARIES): site/out/index.html site/out/coder.tar.zst @@ -277,11 +262,23 @@ build/coder_$(VERSION)_linux.tag: $(CODER_ALL_ARCH_IMAGES_PUSHED) echo "$$image_tag" > "$@" +# Push a Docker image. $(CODER_ALL_IMAGES_PUSHED): push/%: % image_tag="$$(cat "$<")" docker push "$$image_tag" .PHONY: $(CODER_ALL_IMAGES_PUSHED) +# Shortcut for Helm chart package. +build/coder_helm.tgz: build/coder_helm_$(VERSION).tgz + rm -f "$@" + ln -s "$$(basename "$<")" "$@" + +# Helm chart package. +build/coder_helm_$(VERSION).tgz: + ./scripts/helm.sh \ + --version "$(VERSION)" \ + --output "$@" + # Runs migrations to output a dump of the database. coderd/database/dump.sql: coderd/database/dump/main.go $(wildcard coderd/database/migrations/*.sql) go run coderd/database/dump/main.go diff --git a/scripts/archive.sh b/scripts/archive.sh index 267c67f6d2226..cc44f19521b28 100755 --- a/scripts/archive.sh +++ b/scripts/archive.sh @@ -16,7 +16,8 @@ # imported for this to work. Also, the input binary must already be signed with # the `codesign` tool. # -# If the --agpl parameter is specified, only includes AGPL license. +# If the --agpl parameter is specified, only the AGPL license is included in the +# outputted archive. # # The absolute output path is printed on success. diff --git a/scripts/build_go.sh b/scripts/build_go.sh index 990c0d79ec249..3057c3e8cbf5b 100755 --- a/scripts/build_go.sh +++ b/scripts/build_go.sh @@ -12,7 +12,7 @@ # with "vX" (e.g. "v7", "v8"). # # Unless overridden via --output, the built binary will be dropped in -# "$repo_root/dist/coder_$version_$os_$arch" (with a ".exe" suffix for windows +# "$repo_root/build/coder_$version_$os_$arch" (with a ".exe" suffix for windows # builds) and the absolute path to the binary will be printed to stdout on # completion. # @@ -26,7 +26,6 @@ set -euo pipefail # shellcheck source=scripts/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -cdroot version="" os="${GOOS:-linux}" @@ -82,6 +81,8 @@ while true; do esac done +cdroot + # Remove the "v" prefix. version="${version#v}" if [[ "$version" == "" ]]; then @@ -103,9 +104,8 @@ fi # Compute default output path. if [[ "$output_path" == "" ]]; then - dist_dir="dist" - mkdir -p "$dist_dir" - output_path="${dist_dir}/coder_${version}_${os}_${arch}" + mkdir -p "build" + output_path="build/coder_${version}_${os}_${arch}" if [[ "$os" == "windows" ]]; then output_path+=".exe" fi diff --git a/scripts/check_unstaged.sh b/scripts/check_unstaged.sh index 47bdd265ce407..a6de5f0204ef8 100755 --- a/scripts/check_unstaged.sh +++ b/scripts/check_unstaged.sh @@ -1,30 +1,28 @@ #!/bin/bash set -euo pipefail +# shellcheck source=scripts/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +cdroot -SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") -PROJECT_ROOT=$(cd "$SCRIPT_DIR" && git rev-parse --show-toplevel) +FILES="$(git ls-files --other --modified --exclude-standard)" +if [[ "$FILES" != "" ]]; then + mapfile -t files <<<"$FILES" -( - cd "${PROJECT_ROOT}" + log + log "The following files contain unstaged changes:" + log + for file in "${files[@]}"; do + log " - $file" + done - FILES="$(git ls-files --other --modified --exclude-standard)" - if [[ "$FILES" != "" ]]; then - mapfile -t files <<<"$FILES" + log + log "These are the changes:" + log + for file in "${files[@]}"; do + git --no-pager diff "$file" 1>&2 + done - echo "The following files contain unstaged changes:" - echo - for file in "${files[@]}"; do - echo " - $file" - done - echo - - echo "These are the changes:" - echo - for file in "${files[@]}"; do - git --no-pager diff "$file" - done - exit 1 - fi -) -exit 0 + log + error "Unstaged changes, see above for details." +fi diff --git a/scripts/coder-dev.sh b/scripts/coder-dev.sh index f824b4b5a821b..5f9b2d02fba3e 100755 --- a/scripts/coder-dev.sh +++ b/scripts/coder-dev.sh @@ -7,17 +7,18 @@ set -euo pipefail SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") # shellcheck disable=SC1091,SC1090 source "${SCRIPT_DIR}/lib.sh" -PROJECT_ROOT=$(cd "$SCRIPT_DIR" && git rev-parse --show-toplevel) -CODER_DEV_DIR="$PROJECT_ROOT/.coderv2/" -CODER_DEV_BIN="${CODER_DEV_DIR}/coder" -if [[ ! -d "${CODER_DEV_DIR}" ]]; then - mkdir -p "${CODER_DEV_DIR}" -fi +GOOS="$(go env GOOS)" +GOARCH="$(go env GOARCH)" +CODER_DEV_BIN="build/coder_${GOOS}_${GOARCH}" + +cdroot +mkdir -p ./.coderv2 +CODER_DEV_DIR="$(realpath ./.coderv2)" if [[ ! -x "${CODER_DEV_BIN}" ]]; then echo "Run this command first:" - echo "go build -o ${CODER_DEV_BIN} ${PROJECT_ROOT}/enterprise/cmd/coder" + echo " make $CODER_DEV_BIN" exit 1 fi diff --git a/scripts/develop.sh b/scripts/develop.sh index df0cafd2bae65..1c05bcabf990e 100755 --- a/scripts/develop.sh +++ b/scripts/develop.sh @@ -5,19 +5,28 @@ # If the --agpl parameter is specified, builds only the AGPL-licensed code (no # Coder enterprise features). +SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") +# shellcheck disable=SC1091,SC1090 +source "${SCRIPT_DIR}/lib.sh" + # Allow toggling verbose output -[[ -n ${VERBOSE:-""} ]] && set -x +[[ -n ${VERBOSE:-} ]] && set -x set -euo pipefail -agpl="${CODER_BUILD_AGPL:-0}" -args="$(getopt -o "" -l agpl -- "$@")" +password="${CODER_DEV_ADMIN_PASSWORD:-password}" + +args="$(getopt -o "" -l agpl,password: -- "$@")" eval set -- "$args" while true; do case "$1" in --agpl) - agpl=1 + export CODER_BUILD_AGPL=1 shift ;; + --password) + password="$2" + shift 2 + ;; --) shift break @@ -28,43 +37,20 @@ while true; do esac done -SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") -# shellcheck disable=SC1091,SC1090 -source "${SCRIPT_DIR}/lib.sh" -PROJECT_ROOT=$(cd "$SCRIPT_DIR" && git rev-parse --show-toplevel) -CODER_DEV_BIN="${PROJECT_ROOT}/.coderv2/coder" -set +u -CODER_DEV_ADMIN_PASSWORD="${CODER_DEV_ADMIN_PASSWORD:-password}" -set -u - # Preflight checks: ensure we have our required dependencies, and make sure nothing is listening on port 3000 or 8080 dependencies curl git go make yarn curl --fail http://127.0.0.1:3000 >/dev/null 2>&1 && echo '== ERROR: something is listening on port 3000. Kill it and re-run this script.' && exit 1 curl --fail http://127.0.0.1:8080 >/dev/null 2>&1 && echo '== ERROR: something is listening on port 8080. Kill it and re-run this script.' && exit 1 -if [[ ! -e ./site/out/bin/coder.sha1 && ! -e ./site/out/bin/coder.tar.zst ]]; then - log - log "=======================================================================" - log "== Run 'make bin' before running this command to build binaries. ==" - log "== Without these binaries, workspaces will fail to start! ==" - log "=======================================================================" - log - exit 1 -fi +# Compile the CLI binary. This should also compile the frontend and refresh +# node_modules if necessary. +GOOS="$(go env GOOS)" +GOARCH="$(go env GOARCH)" +make "build/coder_${GOOS}_${GOARCH}" -cmd_path="enterprise/cmd/coder" -if [[ "$agpl" == 1 ]]; then - cmd_path="cmd/coder" -fi - -# Compile the CLI binary once just so we don't waste time compiling things multiple times -go build -tags embed -o "${CODER_DEV_BIN}" "${PROJECT_ROOT}/${cmd_path}" # Use the coder dev shim so we don't overwrite the user's existing Coder config. CODER_DEV_SHIM="${PROJECT_ROOT}/scripts/coder-dev.sh" -# Run yarn install, to make sure node_modules are ready to go -"$PROJECT_ROOT/scripts/yarn_install.sh" - # This is a way to run multiple processes in parallel, and have Ctrl-C work correctly # to kill both at the same time. For more details, see: # https://stackoverflow.com/questions/3004811/how-do-you-run-multiple-programs-in-parallel-from-a-bash-script @@ -73,51 +59,49 @@ CODER_DEV_SHIM="${PROJECT_ROOT}/scripts/coder-dev.sh" # rather than leaving things in an inconsistent state. trap 'kill -TERM -$$' ERR cdroot - "${CODER_DEV_SHIM}" server --address 127.0.0.1:3000 --in-memory --tunnel || kill -INT -$$ & + "${CODER_DEV_SHIM}" server --address 127.0.0.1:3000 --tunnel || kill -INT -$$ & echo '== Waiting for Coder to become ready' timeout 60s bash -c 'until curl -s --fail http://localhost:3000 > /dev/null 2>&1; do sleep 0.5; done' - # create the first user, the admin - "${CODER_DEV_SHIM}" login http://127.0.0.1:3000 --username=admin --email=admin@coder.com --password="${CODER_DEV_ADMIN_PASSWORD}" || + # Try to create the initial admin user. + "${CODER_DEV_SHIM}" login http://127.0.0.1:3000 --username=admin --email=admin@coder.com --password="${password}" || echo 'Failed to create admin user. To troubleshoot, try running this command manually.' - # || true to always exit code 0. If this fails, whelp. - "${CODER_DEV_SHIM}" users create --email=member@coder.com --username=member --password="${CODER_DEV_ADMIN_PASSWORD}" || + # Try to create a regular user. + "${CODER_DEV_SHIM}" users create --email=member@coder.com --username=member --password="${password}" || echo 'Failed to create regular user. To troubleshoot, try running this command manually.' - # If we have docker available, then let's try to create a template! - template_name="" - if docker info >/dev/null 2>&1; then - temp_template_dir=$(mktemp -d) - echo code-server | "${CODER_DEV_SHIM}" templates init "${temp_template_dir}" - # shellcheck disable=SC1090 - source <(go env | grep GOARCH) - DOCKER_HOST=$(docker context inspect --format '{{.Endpoints.docker.Host}}') - printf 'docker_arch: "%s"\ndocker_host: "%s"\n' "${GOARCH}" "${DOCKER_HOST}" | tee "${temp_template_dir}/params.yaml" - template_name="docker-${GOARCH}" + # If we have docker available and the "docker" template doesn't already + # exist, then let's try to create a template! + example_template="code-server" + template_name="docker" + if docker info >/dev/null 2>&1 && ! "${CODER_DEV_SHIM}" templates versions list "${template_name}"; then + temp_template_dir="$(mktemp -d)" + echo "${example_template}" | "${CODER_DEV_SHIM}" templates init "${temp_template_dir}" + + DOCKER_HOST="$(docker context inspect --format '{{ .Endpoints.docker.Host }}')" + printf 'docker_arch: "%s"\ndocker_host: "%s"\n' "${GOARCH}" "${DOCKER_HOST}" > "${temp_template_dir}/params.yaml" ( - "${CODER_DEV_SHIM}" templates create "${template_name}" --directory "${temp_template_dir}" --parameter-file "${temp_template_dir}/params.yaml" --yes && - rm -rfv "${temp_template_dir}" # Only delete template dir if template creation succeeds + "${CODER_DEV_SHIM}" templates create "${template_name}" --directory "${temp_template_dir}" --parameter-file "${temp_template_dir}/params.yaml" --yes + rm -rfv "${temp_template_dir}" # Only delete template dir if template creation succeeds ) || echo "Failed to create a template. The template files are in ${temp_template_dir}" fi # Start the frontend once we have a template up and running CODER_HOST=http://127.0.0.1:3000 INSPECT_XSTATE=true yarn --cwd=./site dev || kill -INT -$$ & + log - log "=======================================================================" - log "== ==" - log "== Coder is now running in development mode. ==" - log "== API : http://localhost:3000 ==" - log "== Web UI: http://localhost:8080 ==" - if [[ -n "${template_name}" ]]; then - log "== ==" - log "== Docker template ${template_name} is ready to use! ==" - log "== Use ./scripts/coder-dev.sh to talk to this instance! ==" - log "== ==" - fi - log "=======================================================================" + log "====================================================================" + log "== ==" + log "== Coder is now running in development mode. ==" + log "== API: http://localhost:3000 ==" + log "== Web UI: http://localhost:8080 ==" + log "== ==" + log "== Use ./scripts/coder-dev.sh to talk to this instance! ==" + log "====================================================================" log + # Wait for both frontend and backend to exit. wait ) diff --git a/scripts/helm.sh b/scripts/helm.sh index 5978a5f373937..33b556f0100a1 100755 --- a/scripts/helm.sh +++ b/scripts/helm.sh @@ -9,7 +9,7 @@ # If no version is specified, defaults to the version from ./version.sh. # # If no output path is specified, defaults to -# "$repo_root/dist/coder_helm_$version.tgz". +# "$repo_root/build/coder_helm_$version.tgz". # # If the --push parameter is specified, the resulting artifact will be published # to the Coder OSS repo. This requires `gsutil` to be installed and configured. @@ -56,8 +56,8 @@ fi if [[ "$output_path" == "" ]]; then cdroot - mkdir -p dist - output_path="$(realpath "dist/coder_helm_$version.tgz")" + mkdir -p build + output_path="$(realpath "build/coder_helm_$version.tgz")" fi # Check dependencies diff --git a/scripts/yarn_install.sh b/scripts/yarn_install.sh index 45a7dbf7e559e..ed235ba3dacd1 100755 --- a/scripts/yarn_install.sh +++ b/scripts/yarn_install.sh @@ -5,39 +5,37 @@ # # Usage: yarn_install.sh [optional extra flags] -set -eo pipefail +set -euo pipefail +# shellcheck source=scripts/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") -PROJECT_ROOT=$(cd "$SCRIPT_DIR" && git rev-parse --show-toplevel) +cdroot +cd site -( - cd "$PROJECT_ROOT/site" +yarn_flags=( + # Do not execute install scripts + # TODO: check if build works properly with this enabled + # --ignore-scripts - yarn_flags=( - # Do not execute install scripts - # TODO: check if build works properly with this enabled - # --ignore-scripts + # Check if existing node_modules are valid + # TODO: determine if this is necessary + # --check-files +) - # Check if existing node_modules are valid - # TODO: determine if this is necessary - # --check-files +if [[ -n ${CI:-} ]]; then + yarn_flags+=( + # Install dependencies from lockfile, ensuring builds are fully + # reproducible + --frozen-lockfile + # Suppress progress information + --silent + # Disable interactive prompts for build + --non-interactive ) +fi - if [[ -n ${CI:-} ]]; then - yarn_flags+=( - # Install dependencies from lockfile, ensuring builds are fully - # reproducible - --frozen-lockfile - # Suppress progress information - --silent - # Disable interactive prompts for build - --non-interactive - ) - fi - - # Append whatever is specified on the command line - yarn_flags+=("$@") +# Append whatever is specified on the command line +yarn_flags+=("$@") - echo "+ yarn install ${yarn_flags[*]}" - yarn install "${yarn_flags[@]}" -) +echo "+ yarn install ${yarn_flags[*]}" +yarn install "${yarn_flags[@]}" From f866e1fa64a865cc96e2d096620f79490ef6d081 Mon Sep 17 00:00:00 2001 From: Dean Sheather Date: Sun, 28 Aug 2022 19:29:34 +0000 Subject: [PATCH 07/12] fixup! chore: buff makefile pt. 5 helm stuff --- .github/workflows/release.yaml | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index a668024f61392..d15e2bda8d014 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -86,16 +86,13 @@ jobs: set -euxo pipefail # build and (maybe) push Docker images for each architecture - images=() - for arch in amd64 armv7 arm64; do - img="$( - ./scripts/build_docker.sh \ - ${{ (!github.event.inputs.dry_run && !github.event.inputs.snapshot) && '--push' || '' }} \ - --arch "$arch" \ - ./build/coder_*_linux_"$arch" - )" - images+=("$img") - done + version="$(./script/version.sh)" + push="" + if [[ "$CODER_RELEASE" == *t* ]]; then + push="push/" + fi + make -j \ + "$push"build/coder_"$version"_linux_{amd64,arm64,armv7}.tag # we can't build multi-arch if the images aren't pushed, so quit now # if dry-running @@ -105,14 +102,13 @@ jobs: fi # build and push multi-arch manifest - ./scripts/build_docker_multiarch.sh \ - --push \ - "${images[@]}" + make -j push/build/coder_"$version"_linux.tag # if the current version is equal to the highest (according to semver) # version in the repo, also create a multi-arch image as ":latest" and # push it if [[ "$(git tag | grep '^v' | grep -vE '(rc|dev|-|\+|\/)' | sort -r --version-sort | head -n1)" == "v$(./scripts/version.sh)" ]]; then + make -j push/build/coder_"$version"_linux.tag ./scripts/build_docker_multiarch.sh \ --push \ --target "$(./scripts/image_tag.sh --version latest)" \ @@ -252,7 +248,10 @@ jobs: - name: Publish Helm run: | set -euxo pipefail - ./scripts/helm.sh --push + + version="$(./script/version.sh)" + make -j \ + build/coder_helm_"$version".tgz mv ./build/*.tgz ./artifacts/ - name: Publish Release From b81712b123e4c63135c26529cef195f6fe89e180 Mon Sep 17 00:00:00 2001 From: Dean Sheather Date: Mon, 29 Aug 2022 16:13:36 +0000 Subject: [PATCH 08/12] fixup! chore: buff makefile pt. 5 helm stuff --- scripts/develop.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/develop.sh b/scripts/develop.sh index 1c05bcabf990e..0020b724b0718 100755 --- a/scripts/develop.sh +++ b/scripts/develop.sh @@ -81,7 +81,7 @@ CODER_DEV_SHIM="${PROJECT_ROOT}/scripts/coder-dev.sh" echo "${example_template}" | "${CODER_DEV_SHIM}" templates init "${temp_template_dir}" DOCKER_HOST="$(docker context inspect --format '{{ .Endpoints.docker.Host }}')" - printf 'docker_arch: "%s"\ndocker_host: "%s"\n' "${GOARCH}" "${DOCKER_HOST}" > "${temp_template_dir}/params.yaml" + printf 'docker_arch: "%s"\ndocker_host: "%s"\n' "${GOARCH}" "${DOCKER_HOST}" >"${temp_template_dir}/params.yaml" ( "${CODER_DEV_SHIM}" templates create "${template_name}" --directory "${temp_template_dir}" --parameter-file "${temp_template_dir}/params.yaml" --yes rm -rfv "${temp_template_dir}" # Only delete template dir if template creation succeeds From c7317f4dc3cbc3da87d6a5d647487e5da12112a0 Mon Sep 17 00:00:00 2001 From: Dean Sheather Date: Mon, 29 Aug 2022 16:25:04 +0000 Subject: [PATCH 09/12] chore: add makefile docs --- Makefile | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 76b9bb7fcf022..f118ed0b62087 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,16 @@ +# This is the Coder Makefile. The build directory for most tasks is `build/`. +# +# These are the targets you're probably looking for: +# - clean +# - build-fat: builds all "fat" binaries for all architectures +# - build-slim: builds all "slim" binaries (no frontend or slim binaries +# embedded) for all architectures +# - build/coder(-slim)?_${os}_${arch}(.exe)?: build a single fat binary +# - build/coder_${os}_${arch}.(zip|tar.gz): build a release archive +# - build/coder_linux_${arch}.(apk|deb|rpm): build a release Linux package +# - build/coder_${version}_linux_${arch}.tag: build a release Linux Docker image +# - build/coder_helm.tgz: build a release Helm chart + .DEFAULT_GOAL := build-fat # Use a single bash shell for each job, and immediately exit on failure @@ -208,7 +221,7 @@ $(CODER_ALL_ARCHIVES): $(CODER_FAT_BINARIES) # Supports apk, deb, rpm for all linux targets. # # This depends on all fat binaries because it's difficult to do dynamic -# dependencies due to the .exe requirement on Windows. These targets are +# dependencies due to the extensions in the filenames. These targets are # typically only used during release anyways. $(CODER_ALL_PACKAGES): $(CODER_FAT_BINARIES) $(get-mode-os-arch-ext) From 317e80dd5fd928e5fc85786f5bad8e333e71a0ab Mon Sep 17 00:00:00 2001 From: Dean Sheather Date: Mon, 29 Aug 2022 19:25:22 +0000 Subject: [PATCH 10/12] chore: fix release makefile stuff --- .github/workflows/release.yaml | 2 +- Makefile | 31 ++++++++++++++++++++++++------- scripts/archive.sh | 17 ++++++++++++----- 3 files changed, 37 insertions(+), 13 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index d15e2bda8d014..90be119323b60 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -198,7 +198,7 @@ jobs: make -j \ build/coder_"$version"_darwin_{amd64,arm64}.zip env: - CODER_SIGN_DARWIN: "true" + CODER_SIGN_DARWIN: "1" AC_USERNAME: ${{ secrets.AC_USERNAME }} AC_PASSWORD: ${{ secrets.AC_PASSWORD }} AC_APPLICATION_IDENTITY: BDB050EB749EDD6A80C6F119BF1382ECA119CCCC diff --git a/Makefile b/Makefile index f118ed0b62087..7c55decfeeadd 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,7 @@ # - build-fat: builds all "fat" binaries for all architectures # - build-slim: builds all "slim" binaries (no frontend or slim binaries # embedded) for all architectures +# - release: simulate a release (mostly, does not push images) # - build/coder(-slim)?_${os}_${arch}(.exe)?: build a single fat binary # - build/coder_${os}_${arch}.(zip|tar.gz): build a release archive # - build/coder_linux_${arch}.(apk|deb|rpm): build a release Linux package @@ -45,7 +46,7 @@ OS_ARCHES := \ # Archive formats and their corresponding ${OS}_${ARCH} combos. ARCHIVE_TAR_GZ := linux_amd64 linux_arm64 linux_armv7 ARCHIVE_ZIP := \ - darwin darwin_arm64 \ + darwin_amd64 darwin_arm64 \ windows_amd64 windows_arm64 # All package formats we build and the ${OS}_${ARCH} combos we build them for. @@ -87,6 +88,9 @@ build-slim bin: $(CODER_SLIM_BINARIES) build-fat build-full build: $(CODER_FAT_BINARIES) .PHONY: build-fat build-full build +release: $(CODER_FAT_BINARIES) $(CODER_ALL_ARCHIVES) $(CODER_ALL_PACKAGES) $(CODER_ALL_ARCH_IMAGES) build/coder_helm_$(VERSION).tgz +.PHONY: release + build/coder-slim_$(VERSION)_checksums.sha1: $(CODER_SLIM_BINARIES) pushd ./build openssl dgst -r -sha1 coder-slim_"$(VERSION)"_* | tee "$(@F)" @@ -216,14 +220,18 @@ $(CODER_ALL_ARCHIVES): $(CODER_FAT_BINARIES) # This task builds all packages. It parses the target name to get the metadata # for the build, so it must be specified in this format: -# build/coder_${version}_${os}_${arch}.${format} +# build/coder_${version}_linux_${arch}.${format} # # Supports apk, deb, rpm for all linux targets. # -# This depends on all fat binaries because it's difficult to do dynamic -# dependencies due to the extensions in the filenames. These targets are -# typically only used during release anyways. -$(CODER_ALL_PACKAGES): $(CODER_FAT_BINARIES) +# This depends on all Linux fat binaries and archives because it's difficult to +# do dynamic dependencies due to the extensions in the filenames. These targets +# are typically only used during release anyways. +# +# Packages need to run after the archives are built, otherwise they cause tar +# errors like "file changed as we read it". +CODER_PACKAGE_DEPS := $(foreach os_arch, $(PACKAGE_OS_ARCHES), build/coder_$(VERSION)_$(os_arch) build/coder_$(VERSION)_$(os_arch).tar.gz) +$(CODER_ALL_PACKAGES): $(CODER_PACKAGE_DEPS) $(get-mode-os-arch-ext) ./scripts/package.sh \ @@ -252,7 +260,16 @@ $(CODER_ALL_NOVERSION_IMAGES_PUSHED): push/build/coder_%: push/build/coder_$(VER # build/coder_${version}_${os}_${arch}.tag # # Supports linux_amd64, linux_arm64, linux_armv7. -$(CODER_ALL_ARCH_IMAGES): build/coder_$(VERSION)_%.tag: build/coder_$(VERSION)_% +# +# Images need to run after the archives and packages are built, otherwise they +# cause errors like "file changed as we read it". +$(CODER_ALL_ARCH_IMAGES): build/coder_$(VERSION)_%.tag: \ + build/coder_$(VERSION)_% \ + build/coder_$(VERSION)_%.apk \ + build/coder_$(VERSION)_%.deb \ + build/coder_$(VERSION)_%.rpm \ + build/coder_$(VERSION)_%.tar.gz + $(get-mode-os-arch-ext) image_tag="$$(./scripts/image_tag.sh --arch "$$arch" --version "$(VERSION)")" diff --git a/scripts/archive.sh b/scripts/archive.sh index cc44f19521b28..647ef9a714125 100755 --- a/scripts/archive.sh +++ b/scripts/archive.sh @@ -3,7 +3,7 @@ # This script creates an archive containing the given binary renamed to # `coder(.exe)?`, as well as the README.md and LICENSE files from the repo root. # -# Usage: ./archive.sh --format tar.gz [--output path/to/output.tar.gz] [--sign-darwin] [--agpl] path/to/binary +# Usage: ./archive.sh --format tar.gz --os linux/darwin/windows [--output path/to/output.tar.gz] [--sign-darwin] [--agpl] path/to/binary # # The --format parameter must be set, and must either be "zip" or "tar.gz". # @@ -28,9 +28,10 @@ source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" format="" output_path="" sign_darwin="${CODER_SIGN_DARWIN:-0}" +os="" agpl="${CODER_BUILD_AGPL:-0}" -args="$(getopt -o "" -l format:,output:,sign-darwin,agpl -- "$@")" +args="$(getopt -o "" -l format:,output:,sign-darwin,os:,agpl -- "$@")" eval set -- "$args" while true; do case "$1" in @@ -47,10 +48,11 @@ while true; do output_path="$(realpath "$2")" shift 2 ;; + --os) + os="$2" + shift 2 + ;; --sign-darwin) - if [[ "${AC_APPLICATION_IDENTITY:-}" == "" ]]; then - error "AC_APPLICATION_IDENTITY must be set when --sign-darwin is supplied" - fi sign_darwin=1 shift ;; @@ -80,6 +82,11 @@ if [[ ! -f "$1" ]]; then fi input_file="$(realpath "$1")" +sign_darwin="$([[ "$sign_darwin" == 1 ]] && [[ "$os" == "darwin" ]] && echo 1 || echo 0)" +if [[ "$sign_darwin" == 1 ]] && [[ "${AC_APPLICATION_IDENTITY:-}" == "" ]]; then + error "AC_APPLICATION_IDENTITY must be set when --sign-darwin or CODER_SIGN_DARWIN=1 is supplied" +fi + # Check dependencies if [[ "$format" == "zip" ]]; then dependencies zip From ec1c10ae9f35715ce73ffba85cb31bceb6f1b2e2 Mon Sep 17 00:00:00 2001 From: Dean Sheather Date: Mon, 29 Aug 2022 20:01:50 +0000 Subject: [PATCH 11/12] fixup! chore: fix release makefile stuff --- Makefile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 7c55decfeeadd..2e71d77cd4b9c 100644 --- a/Makefile +++ b/Makefile @@ -111,6 +111,8 @@ build/coder-slim_$(VERSION).tar.zst site/out/coder.tar.zst: build/coder-slim_$(V "build/coder-slim_$(VERSION).tar" cp "build/coder-slim_$(VERSION).tar.zst" "site/out/coder.tar.zst" + # delete the uncompressed binaries from the embedded dir + rm site/out/coder-* # Redirect from version-less targets to the versioned ones. There is a similar # target for slim binaries below. @@ -120,7 +122,7 @@ build/coder-slim_$(VERSION).tar.zst site/out/coder.tar.zst: build/coder-slim_$(V # make build/coder_windows_amd64.exe $(CODER_FAT_NOVERSION_BINARIES): build/coder_%: build/coder_$(VERSION)_% rm -f "$@" - ln -s "$$(basename "$<")" "$@" + ln "$<" "$@" # Same as above, but for slim binaries. # @@ -129,7 +131,7 @@ $(CODER_FAT_NOVERSION_BINARIES): build/coder_%: build/coder_$(VERSION)_% # make build/coder-slim_windows_amd64.exe $(CODER_SLIM_NOVERSION_BINARIES): build/coder-slim_%: build/coder-slim_$(VERSION)_% rm -f "$@" - ln -s "$$(basename "$<")" "$@" + ln "$<" "$@" # "fat" binaries always depend on the site and the compressed slim binaries. $(CODER_FAT_BINARIES): site/out/index.html site/out/coder.tar.zst @@ -215,6 +217,7 @@ $(CODER_ALL_ARCHIVES): $(CODER_FAT_BINARIES) ./scripts/archive.sh \ --format "$$ext" \ + --os "$$os" \ --output "$@" \ "build/coder_$(VERSION)_$${os}_$${arch}$${bin_ext}" @@ -301,7 +304,7 @@ $(CODER_ALL_IMAGES_PUSHED): push/%: % # Shortcut for Helm chart package. build/coder_helm.tgz: build/coder_helm_$(VERSION).tgz rm -f "$@" - ln -s "$$(basename "$<")" "$@" + ln "$<" "$@" # Helm chart package. build/coder_helm_$(VERSION).tgz: From a6f1b4c5eac0e5ab64e3fc0e3e6b07a3e8fc90c1 Mon Sep 17 00:00:00 2001 From: Dean Sheather Date: Thu, 1 Sep 2022 12:19:35 +0000 Subject: [PATCH 12/12] chore: remove make bin target --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 2e71d77cd4b9c..7a399740635de 100644 --- a/Makefile +++ b/Makefile @@ -82,8 +82,8 @@ clean: git restore site/out .PHONY: clean -build-slim bin: $(CODER_SLIM_BINARIES) -.PHONY: build-slim bin +build-slim: $(CODER_SLIM_BINARIES) +.PHONY: build-slim build-fat build-full build: $(CODER_FAT_BINARIES) .PHONY: build-fat build-full build