From cf323e7dfa258e78c9f9072255fd20f6ec1c13ec Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Fri, 2 Sep 2022 14:38:16 -0500 Subject: [PATCH] revert: Makefile buff-ification (#3700) This caused the following issues: - Slim binaries weren't being updated. - The coder.tar.ztd was misplaced. - There is no coder.sha1 file with proper filenames. This should be reintroduced in a future change with those fixes. --- .github/workflows/coder.yaml | 33 ++- .github/workflows/release.yaml | 83 ++++--- .gitignore | 1 - Makefile | 389 +++++++-------------------------- scripts/archive.sh | 24 +- scripts/build_docker.sh | 15 +- scripts/build_go.sh | 21 +- scripts/build_go_matrix.sh | 243 ++++++++++++++++++++ scripts/build_go_slim.sh | 146 +++++++++++++ scripts/check_unstaged.sh | 44 ++-- scripts/coder-dev.sh | 15 +- scripts/develop.sh | 106 +++++---- scripts/helm.sh | 6 +- scripts/package.sh | 50 ++--- scripts/publish_release.sh | 8 - scripts/yarn_install.sh | 56 ++--- 16 files changed, 716 insertions(+), 524 deletions(-) create mode 100755 scripts/build_go_matrix.sh create mode 100755 scripts/build_go_slim.sh diff --git a/.github/workflows/coder.yaml b/.github/workflows/coder.yaml index 16488ed31f6d4..9f9e6cd8554a0 100644 --- a/.github/workflows/coder.yaml +++ b/.github/workflows/coder.yaml @@ -498,21 +498,35 @@ jobs: - name: Install zstd run: sudo apt-get install -y zstd + - name: Build site + run: make -B site/out/index.html + - name: Build Release run: | set -euo pipefail go mod download - version="$(./scripts/version.sh)" - make -j \ - build/coder_"$version"_windows_amd64.zip \ - build/coder_"$version"_linux_amd64.{tar.gz,deb} + mkdir -p ./dist + # build slim binaries + ./scripts/build_go_slim.sh \ + --output ./dist/ \ + --compress 22 \ + linux:amd64,armv7,arm64 \ + windows:amd64,arm64 \ + darwin:amd64,arm64 + + # build linux amd64 packages + ./scripts/build_go_matrix.sh \ + --output ./dist/ \ + --package-linux \ + linux:amd64 \ + windows:amd64 - name: Install Release run: | gcloud config set project coder-dogfood gcloud config set compute/zone us-central1-a - gcloud compute scp ./build/coder_*_linux_amd64.deb coder:/tmp/coder.deb + gcloud compute scp ./dist/coder_*_linux_amd64.deb coder:/tmp/coder.deb gcloud compute ssh coder -- sudo dpkg -i --force-confdef /tmp/coder.deb gcloud compute ssh coder -- sudo systemctl daemon-reload @@ -523,9 +537,12 @@ jobs: with: name: coder path: | - ./build/*.zip - ./build/*.tar.gz - ./build/*.deb + ./dist/*.zip + ./dist/*.exe + ./dist/*.tar.gz + ./dist/*.apk + ./dist/*.deb + ./dist/*.rpm retention-days: 7 test-js: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e5665baabb5a7..fd89c988c3954 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -71,28 +71,46 @@ jobs: - name: Install zstd run: sudo apt-get install -y zstd + - name: Build Site + run: make site/out/index.html + - name: Build Linux and Windows Binaries run: | set -euo pipefail go mod download - version="$(./scripts/version.sh)" - make -j \ - build/coder_"$version"_linux_{amd64,armv7,arm64}.{tar.gz,apk,deb,rpm} \ - build/coder_"$version"_windows_{amd64,arm64}.zip \ + mkdir -p ./dist + # build slim binaries + ./scripts/build_go_slim.sh \ + --output ./dist/ \ + --compress 22 \ + linux:amd64,armv7,arm64 \ + windows:amd64,arm64 \ + darwin:amd64,arm64 + + # build linux and windows binaries + ./scripts/build_go_matrix.sh \ + --output ./dist/ \ + --archive \ + --package-linux \ + linux:amd64,armv7,arm64 \ + windows:amd64,arm64 - name: Build Linux Docker images run: | set -euxo pipefail # build and (maybe) push Docker images for each architecture - version="$(./scripts/version.sh)" - push="" - if [[ "$CODER_RELEASE" == *t* ]]; then - push="push/" - fi - make -j \ - "$push"build/coder_"$version"_linux_{amd64,arm64,armv7}.tag + images=() + for arch in amd64 armv7 arm64; do + img="$( + ./scripts/build_docker.sh \ + ${{ (!github.event.inputs.dry_run && !github.event.inputs.snapshot) && '--push' || '' }} \ + --arch "$arch" \ + ./dist/coder_*_linux_"$arch" + )" + images+=("$img") + done # we can't build multi-arch if the images aren't pushed, so quit now # if dry-running @@ -102,13 +120,14 @@ jobs: fi # build and push multi-arch manifest - make -j push/build/coder_"$version"_linux.tag + ./scripts/build_docker_multiarch.sh \ + --push \ + "${images[@]}" # if the current version is equal to the highest (according to semver) # version in the repo, also create a multi-arch image as ":latest" and # push it if [[ "$(git tag | grep '^v' | grep -vE '(rc|dev|-|\+|\/)' | sort -r --version-sort | head -n1)" == "v$(./scripts/version.sh)" ]]; then - make -j push/build/coder_"$version"_linux.tag ./scripts/build_docker_multiarch.sh \ --push \ --target "$(./scripts/image_tag.sh --version latest)" \ @@ -120,11 +139,11 @@ jobs: with: name: linux path: | - ./build/*.zip - ./build/*.tar.gz - ./build/*.apk - ./build/*.deb - ./build/*.rpm + dist/*.zip + dist/*.tar.gz + dist/*.apk + dist/*.deb + dist/*.rpm # The mac binaries get built on mac runners because they need to be signed, # and the signing tool only runs on mac. This darwin job only builds the Mac @@ -194,11 +213,22 @@ jobs: set -euo pipefail go mod download - version="$(./scripts/version.sh)" - make -j \ - build/coder_"$version"_darwin_{amd64,arm64}.zip + mkdir -p ./dist + # build slim binaries + ./scripts/build_go_slim.sh \ + --output ./dist/ \ + --compress 22 \ + linux:amd64,armv7,arm64 \ + windows:amd64,arm64 \ + darwin:amd64,arm64 + + # build darwin binaries + ./scripts/build_go_matrix.sh \ + --output ./dist/ \ + --archive \ + --sign-darwin \ + darwin:amd64,arm64 env: - CODER_SIGN_DARWIN: "1" AC_USERNAME: ${{ secrets.AC_USERNAME }} AC_PASSWORD: ${{ secrets.AC_PASSWORD }} AC_APPLICATION_IDENTITY: BDB050EB749EDD6A80C6F119BF1382ECA119CCCC @@ -207,7 +237,7 @@ jobs: uses: actions/upload-artifact@v3 with: name: darwin - path: ./build/*.zip + path: ./dist/coder_*.zip publish: runs-on: ubuntu-latest @@ -248,11 +278,8 @@ jobs: - name: Publish Helm run: | set -euxo pipefail - - version="$(./scripts/version.sh)" - make -j \ - build/coder_helm_"$version".tgz - mv ./build/*.tgz ./artifacts/ + ./scripts/helm.sh --push + mv ./dist/*.tgz ./artifacts/ - name: Publish Release run: | diff --git a/.gitignore b/.gitignore index b1a821d914c4b..ae2c40f0a576c 100644 --- a/.gitignore +++ b/.gitignore @@ -30,7 +30,6 @@ site/**/*.typegen.ts site/build-storybook.log # Build -build/ dist/ site/out/ diff --git a/Makefile b/Makefile index 0c1d77b4a78e9..3c1a1d74aa038 100644 --- a/Makefile +++ b/Makefile @@ -1,316 +1,85 @@ -# This is the Coder Makefile. The build directory for most tasks is `build/`. -# -# These are the targets you're probably looking for: -# - clean -# - build-fat: builds all "fat" binaries for all architectures -# - build-slim: builds all "slim" binaries (no frontend or slim binaries -# embedded) for all architectures -# - release: simulate a release (mostly, does not push images) -# - build/coder(-slim)?_${os}_${arch}(.exe)?: build a single fat binary -# - build/coder_${os}_${arch}.(zip|tar.gz): build a release archive -# - build/coder_linux_${arch}.(apk|deb|rpm): build a release Linux package -# - build/coder_${version}_linux_${arch}.tag: build a release Linux Docker image -# - build/coder_helm.tgz: build a release Helm chart - -.DEFAULT_GOAL := build-fat +.DEFAULT_GOAL := build # Use a single bash shell for each job, and immediately exit on failure SHELL := bash -.SHELLFLAGS := -ceu +.SHELLFLAGS = -ceu .ONESHELL: # This doesn't work on directories. # See https://stackoverflow.com/questions/25752543/make-delete-on-error-for-directory-targets .DELETE_ON_ERROR: -# Don't print the commands in the file unless you specify VERBOSE. This is -# essentially the same as putting "@" at the start of each line. -ifndef VERBOSE -.SILENT: -endif +INSTALL_DIR=$(shell go env GOPATH)/bin +GOOS=$(shell go env GOOS) +GOARCH=$(shell go env GOARCH) +VERSION=$(shell ./scripts/version.sh) -# Create the output directories if they do not exist. -$(shell mkdir -p build site/out/bin) - -GOOS := $(shell go env GOOS) -GOARCH := $(shell go env GOARCH) -GOOS_BIN_EXT := $(if $(filter windows, $(GOOS)),.exe,) -VERSION := $(shell ./scripts/version.sh) - -# All ${OS}_${ARCH} combos we build for. Windows binaries have the .exe suffix. -OS_ARCHES := \ - linux_amd64 linux_arm64 linux_armv7 \ - darwin_amd64 darwin_arm64 \ - windows_amd64.exe windows_arm64.exe - -# Archive formats and their corresponding ${OS}_${ARCH} combos. -ARCHIVE_TAR_GZ := linux_amd64 linux_arm64 linux_armv7 -ARCHIVE_ZIP := \ - darwin_amd64 darwin_arm64 \ - windows_amd64 windows_arm64 - -# All package formats we build and the ${OS}_${ARCH} combos we build them for. -PACKAGE_FORMATS := apk deb rpm -PACKAGE_OS_ARCHES := linux_amd64 linux_armv7 linux_arm64 - -# All architectures we build Docker images for (Linux only). -DOCKER_ARCHES := amd64 arm64 armv7 - -# Computed variables based on the above. -CODER_SLIM_BINARIES := $(addprefix build/coder-slim_$(VERSION)_,$(OS_ARCHES)) -CODER_FAT_BINARIES := $(addprefix build/coder_$(VERSION)_,$(OS_ARCHES)) -CODER_ALL_BINARIES := $(CODER_SLIM_BINARIES) $(CODER_FAT_BINARIES) -CODER_TAR_GZ_ARCHIVES := $(foreach os_arch, $(ARCHIVE_TAR_GZ), build/coder_$(VERSION)_$(os_arch).tar.gz) -CODER_ZIP_ARCHIVES := $(foreach os_arch, $(ARCHIVE_ZIP), build/coder_$(VERSION)_$(os_arch).zip) -CODER_ALL_ARCHIVES := $(CODER_TAR_GZ_ARCHIVES) $(CODER_ZIP_ARCHIVES) -CODER_ALL_PACKAGES := $(foreach os_arch, $(PACKAGE_OS_ARCHES), $(addprefix build/coder_$(VERSION)_$(os_arch).,$(PACKAGE_FORMATS))) -CODER_ALL_ARCH_IMAGES := $(foreach arch, $(DOCKER_ARCHES), build/coder_$(VERSION)_linux_$(arch).tag) -CODER_ALL_ARCH_IMAGES_PUSHED := $(addprefix push/, $(CODER_ALL_ARCH_IMAGES)) -CODER_MAIN_IMAGE := build/coder_$(VERSION)_linux.tag -CODER_ALL_IMAGES := $(CODER_ALL_ARCH_IMAGES) $(CODER_MAIN_IMAGE) -CODER_ALL_IMAGES_PUSHED := $(addprefix push/, $(CODER_ALL_IMAGES)) - -CODER_SLIM_NOVERSION_BINARIES := $(addprefix build/coder-slim_,$(OS_ARCHES)) -CODER_FAT_NOVERSION_BINARIES := $(addprefix build/coder_,$(OS_ARCHES)) -CODER_ALL_NOVERSION_IMAGES := $(foreach arch, $(DOCKER_ARCHES), build/coder_linux_$(arch).tag) build/coder_linux.tag -CODER_ALL_NOVERSION_IMAGES_PUSHED := $(addprefix push/, $(CODER_ALL_NOVERSION_IMAGES)) - - -clean: - rm -rf build site/out - mkdir -p build site/out/bin - git restore site/out -.PHONY: clean - -build-slim: $(CODER_SLIM_BINARIES) -.PHONY: build-slim - -build-fat build-full build: $(CODER_FAT_BINARIES) -.PHONY: build-fat build-full build - -release: $(CODER_FAT_BINARIES) $(CODER_ALL_ARCHIVES) $(CODER_ALL_PACKAGES) $(CODER_ALL_ARCH_IMAGES) build/coder_helm_$(VERSION).tgz -.PHONY: release - -build/coder-slim_$(VERSION)_checksums.sha1: $(CODER_SLIM_BINARIES) - pushd ./build - openssl dgst -r -sha1 coder-slim_"$(VERSION)"_* | tee "$(@F)" - popd - -build/coder-slim_$(VERSION).tar: build/coder-slim_$(VERSION)_checksums.sha1 $(CODER_SLIM_BINARIES) - pushd ./build - tar cf "$(@F)" coder-slim_"$(VERSION)"_* - popd - -build/coder-slim_$(VERSION).tar.zst site/out/coder.tar.zst: build/coder-slim_$(VERSION).tar - zstd -6 \ - --force \ - --ultra \ - --long \ - --no-progress \ - -o "build/coder-slim_$(VERSION).tar.zst" \ - "build/coder-slim_$(VERSION).tar" - - cp "build/coder-slim_$(VERSION).tar.zst" "site/out/coder.tar.zst" - # delete the uncompressed binaries from the embedded dir - rm site/out/coder-* - -# Redirect from version-less targets to the versioned ones. There is a similar -# target for slim binaries below. -# -# Called like this: -# make build/coder_linux_amd64 -# make build/coder_windows_amd64.exe -$(CODER_FAT_NOVERSION_BINARIES): build/coder_%: build/coder_$(VERSION)_% - rm -f "$@" - ln "$<" "$@" - -# Same as above, but for slim binaries. -# -# Called like this: -# make build/coder-slim_linux_amd64 -# make build/coder-slim_windows_amd64.exe -$(CODER_SLIM_NOVERSION_BINARIES): build/coder-slim_%: build/coder-slim_$(VERSION)_% - rm -f "$@" - ln "$<" "$@" - -# "fat" binaries always depend on the site and the compressed slim binaries. -$(CODER_FAT_BINARIES): site/out/index.html site/out/coder.tar.zst - -# This is a handy block that parses the target to determine whether it's "slim" -# or "fat", which OS was specified and which architecture was specified. -# -# It populates the following variables: mode, os, arch_ext, arch, ext (without -# dot). -define get-mode-os-arch-ext = - mode="$$([[ "$@" = build/coder-slim* ]] && echo "slim" || echo "fat")" - os="$$(echo $@ | cut -d_ -f3)" - arch_ext="$$(echo $@ | cut -d_ -f4)" - if [[ "$$arch_ext" == *.* ]]; then - arch="$$(echo $$arch_ext | cut -d. -f1)" - ext="$${arch_ext#*.}" - else - arch="$$arch_ext" - ext="" - fi -endef - -# This task handles all builds, for both "fat" and "slim" binaries. It parses -# the target name to get the metadata for the build, so it must be specified in -# this format: -# build/coder(-slim)?_${version}_${os}_${arch}(.exe)? -# -# You should probably use the non-version targets above instead if you're -# calling this manually. -$(CODER_ALL_BINARIES): go.mod go.sum \ - $(shell find . -not -path './vendor/*' -type f -name '*.go') \ - $(shell find ./examples/templates) - - $(get-mode-os-arch-ext) - if [[ "$$os" != "windows" ]] && [[ "$$ext" != "" ]]; then - echo "ERROR: Invalid build binary extension" 1>&2 - exit 1 - fi - if [[ "$$os" == "windows" ]] && [[ "$$ext" != exe ]]; then - echo "ERROR: Windows binaries must have an .exe extension." 1>&2 - exit 1 - fi - - build_args=( \ - --os "$$os" \ - --arch "$$arch" \ - --version "$(VERSION)" \ - --output "$@" \ - ) - if [ "$$mode" == "slim" ]; then - build_args+=(--slim) - fi - - ./scripts/build_go.sh "$${build_args[@]}" - - if [[ "$$mode" == "slim" ]]; then - dot_ext="" - if [[ "$$ext" != "" ]]; then - dot_ext=".$$ext" - fi - - mkdir -p ./site/out - cp "$@" "./site/out/coder-$$os-$$arch$$dot_ext" - fi - -# This task builds all archives. It parses the target name to get the metadata -# for the build, so it must be specified in this format: -# build/coder_${version}_${os}_${arch}.${format} -# -# The following OS/arch/format combinations are supported: -# .tar.gz: linux_amd64, linux_arm64, linux_armv7 -# .zip: darwin_amd64, darwin_arm64, windows_amd64, windows_arm64 -# -# This depends on all fat binaries because it's difficult to do dynamic -# dependencies due to the .exe requirement on Windows. These targets are -# typically only used during release anyways. -$(CODER_ALL_ARCHIVES): $(CODER_FAT_BINARIES) - $(get-mode-os-arch-ext) - bin_ext="" - if [[ "$$os" == "windows" ]]; then - bin_ext=".exe" - fi - - ./scripts/archive.sh \ - --format "$$ext" \ - --os "$$os" \ - --output "$@" \ - "build/coder_$(VERSION)_$${os}_$${arch}$${bin_ext}" - -# This task builds all packages. It parses the target name to get the metadata -# for the build, so it must be specified in this format: -# build/coder_${version}_linux_${arch}.${format} -# -# Supports apk, deb, rpm for all linux targets. -# -# This depends on all Linux fat binaries and archives because it's difficult to -# do dynamic dependencies due to the extensions in the filenames. These targets -# are typically only used during release anyways. -# -# Packages need to run after the archives are built, otherwise they cause tar -# errors like "file changed as we read it". -CODER_PACKAGE_DEPS := $(foreach os_arch, $(PACKAGE_OS_ARCHES), build/coder_$(VERSION)_$(os_arch) build/coder_$(VERSION)_$(os_arch).tar.gz) -$(CODER_ALL_PACKAGES): $(CODER_PACKAGE_DEPS) - $(get-mode-os-arch-ext) - - ./scripts/package.sh \ - --arch "$$arch" \ - --format "$$ext" \ - --version "$(VERSION)" \ - --output "$@" \ - "build/coder_$(VERSION)_$${os}_$${arch}" - -# Redirect from version-less Docker image targets to the versioned ones. -# -# Called like this: -# make build/coder_linux_amd64.tag -$(CODER_ALL_NOVERSION_IMAGES): build/coder_%: build/coder_$(VERSION)_% -.PHONY: $(CODER_ALL_NOVERSION_IMAGES) - -# Redirect from version-less push Docker image targets to the versioned ones. -# -# Called like this: -# make push/build/coder_linux_amd64.tag -$(CODER_ALL_NOVERSION_IMAGES_PUSHED): push/build/coder_%: push/build/coder_$(VERSION)_% -.PHONY: $(CODER_ALL_NOVERSION_IMAGES_PUSHED) - -# This task builds all Docker images. It parses the target name to get the -# metadata for the build, so it must be specified in this format: -# build/coder_${version}_${os}_${arch}.tag -# -# Supports linux_amd64, linux_arm64, linux_armv7. -# -# Images need to run after the archives and packages are built, otherwise they -# cause errors like "file changed as we read it". -$(CODER_ALL_ARCH_IMAGES): build/coder_$(VERSION)_%.tag: \ - build/coder_$(VERSION)_% \ - build/coder_$(VERSION)_%.apk \ - build/coder_$(VERSION)_%.deb \ - build/coder_$(VERSION)_%.rpm \ - build/coder_$(VERSION)_%.tar.gz - - $(get-mode-os-arch-ext) - - image_tag="$$(./scripts/image_tag.sh --arch "$$arch" --version "$(VERSION)")" - ./scripts/build_docker.sh \ - --arch "$$arch" \ - --target "$$image_tag" \ - --version "$(VERSION)" \ - "build/coder_$(VERSION)_$${os}_$${arch}" +bin: $(shell find . -not -path './vendor/*' -type f -name '*.go') go.mod go.sum $(shell find ./examples/templates) + @echo "== This builds slim binaries for command-line usage." + @echo "== Use \"make build\" to embed the site." - echo "$$image_tag" > "$@" - -# Multi-arch Docker image. This requires all architecture-specific images to be -# built AND pushed. -build/coder_$(VERSION)_linux.tag: $(CODER_ALL_ARCH_IMAGES_PUSHED) - image_tag="$$(./scripts/image_tag.sh --version "$(VERSION)")" - ./scripts/build_docker_multiarch.sh \ - --target "$$image_tag" \ + mkdir -p ./dist + rm -rf ./dist/coder-slim_* + rm -f ./site/out/bin/coder* + ./scripts/build_go_slim.sh \ + --compress 6 \ --version "$(VERSION)" \ - $(foreach img, $^, "$$(cat "$(img)")") - - echo "$$image_tag" > "$@" + --output ./dist/ \ + linux:amd64,armv7,arm64 \ + windows:amd64,arm64 \ + darwin:amd64,arm64 +.PHONY: bin -# Push a Docker image. -$(CODER_ALL_IMAGES_PUSHED): push/%: % - image_tag="$$(cat "$<")" - docker push "$$image_tag" -.PHONY: $(CODER_ALL_IMAGES_PUSHED) +GO_FILES=$(shell find . -not -path './vendor/*' -type f -name '*.go') go.mod go.sum $(shell find ./examples/templates) -# Shortcut for Helm chart package. -build/coder_helm.tgz: build/coder_helm_$(VERSION).tgz - rm -f "$@" - ln "$<" "$@" +build: site/out/index.html $(GO_FILES) + rm -rf ./dist + mkdir -p ./dist + rm -f ./site/out/bin/coder* -# Helm chart package. -build/coder_helm_$(VERSION).tgz: - ./scripts/helm.sh \ + # build slim artifacts and copy them to the site output directory + ./scripts/build_go_slim.sh \ + --version "$(VERSION)" \ + --compress 6 \ + --output ./dist/ \ + linux:amd64,armv7,arm64 \ + windows:amd64,arm64 \ + darwin:amd64,arm64 + + # build not-so-slim artifacts with the default name format + ./scripts/build_go_matrix.sh \ --version "$(VERSION)" \ - --output "$@" + --output ./dist/ \ + --archive \ + --package-linux \ + linux:amd64,armv7,arm64 \ + windows:amd64,arm64 \ + darwin:amd64,arm64 +.PHONY: build + +# Builds a test binary for just Linux +build-linux-test: site/out/index.html $(GO_FILES) + rm -rf ./dist + mkdir -p ./dist + rm -f ./site/out/bin/coder* + + # build slim artifacts and copy them to the site output directory + ./scripts/build_go_slim.sh \ + --version "$(VERSION)" \ + --compress 6 \ + --output ./dist/ \ + linux:amd64,armv7,arm64 \ + windows:amd64,arm64 \ + darwin:amd64,arm64 + + # build not-so-slim artifacts with the default name format + ./scripts/build_go_matrix.sh \ + --version "$(VERSION)" \ + --output ./dist/ \ + --archive \ + --package-linux \ + linux:amd64 +.PHONY: build-linux-test # Runs migrations to output a dump of the database. coderd/database/dump.sql: coderd/database/gen/dump/main.go $(wildcard coderd/database/migrations/*.sql) @@ -321,7 +90,7 @@ coderd/database/querier.go: coderd/database/sqlc.yaml coderd/database/dump.sql $ coderd/database/generate.sh fmt/prettier: - echo "--- prettier" + @echo "--- prettier" cd site # Avoid writing files in CI to reduce file write activity ifdef CI @@ -336,7 +105,7 @@ fmt/terraform: $(wildcard *.tf) .PHONY: fmt/terraform fmt/shfmt: $(shell shfmt -f .) - echo "--- shfmt" + @echo "--- shfmt" # Only do diff check in CI, errors on diff. ifdef CI shfmt -d $(shell shfmt -f .) @@ -351,12 +120,22 @@ fmt: fmt/prettier fmt/terraform fmt/shfmt gen: coderd/database/querier.go peerbroker/proto/peerbroker.pb.go provisionersdk/proto/provisioner.pb.go provisionerd/proto/provisionerd.pb.go site/src/api/typesGenerated.ts .PHONY: gen -install: build/coder_$(VERSION)_$(GOOS)_$(GOARCH)$(GOOS_BIN_EXT) - install_dir="$$(go env GOPATH)/bin" - output_file="$${install_dir}/coder$(GOOS_BIN_EXT)" +install: site/out/index.html $(shell find . -not -path './vendor/*' -type f -name '*.go') go.mod go.sum $(shell find ./examples/templates) + @output_file="$(INSTALL_DIR)/coder" + + @if [[ "$(GOOS)" == "windows" ]]; then + @output_file="$${output_file}.exe" + @fi + + @echo "-- Building CLI for $(GOOS) $(GOARCH) at $$output_file" + + ./scripts/build_go.sh \ + --version "$(VERSION)" \ + --output "$$output_file" \ + --os "$(GOOS)" \ + --arch "$(GOARCH)" - mkdir -p "$$install_dir" - cp "$<" "$$output_file" + @echo .PHONY: install lint: lint/shellcheck lint/go @@ -369,7 +148,7 @@ lint/go: # Use shfmt to determine the shell files, takes editorconfig into consideration. lint/shellcheck: $(shell shfmt -f .) - echo "--- shellcheck" + @echo "--- shellcheck" shellcheck --external-sources $(shell shfmt -f .) .PHONY: lint/shellcheck diff --git a/scripts/archive.sh b/scripts/archive.sh index 647ef9a714125..4f75e20c4a74a 100755 --- a/scripts/archive.sh +++ b/scripts/archive.sh @@ -3,7 +3,7 @@ # This script creates an archive containing the given binary renamed to # `coder(.exe)?`, as well as the README.md and LICENSE files from the repo root. # -# Usage: ./archive.sh --format tar.gz --os linux/darwin/windows [--output path/to/output.tar.gz] [--sign-darwin] [--agpl] path/to/binary +# Usage: ./archive.sh --format tar.gz [--output path/to/output.tar.gz] [--sign-darwin] [--agpl] path/to/binary # # The --format parameter must be set, and must either be "zip" or "tar.gz". # @@ -14,10 +14,9 @@ # utility and then notarized using the `gon` utility, which may take a while. # $AC_APPLICATION_IDENTITY must be set and the signing certificate must be # imported for this to work. Also, the input binary must already be signed with -# the `codesign` tool. +# the `codesign` tool.= # -# If the --agpl parameter is specified, only the AGPL license is included in the -# outputted archive. +# If the --agpl parameter is specified, only includes AGPL license. # # The absolute output path is printed on success. @@ -27,11 +26,10 @@ source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" format="" output_path="" -sign_darwin="${CODER_SIGN_DARWIN:-0}" -os="" +sign_darwin=0 agpl="${CODER_BUILD_AGPL:-0}" -args="$(getopt -o "" -l format:,output:,sign-darwin,os:,agpl -- "$@")" +args="$(getopt -o "" -l format:,output:,sign-darwin,agpl -- "$@")" eval set -- "$args" while true; do case "$1" in @@ -48,11 +46,10 @@ while true; do output_path="$(realpath "$2")" shift 2 ;; - --os) - os="$2" - shift 2 - ;; --sign-darwin) + if [[ "${AC_APPLICATION_IDENTITY:-}" == "" ]]; then + error "AC_APPLICATION_IDENTITY must be set when --sign-darwin is supplied" + fi sign_darwin=1 shift ;; @@ -82,11 +79,6 @@ if [[ ! -f "$1" ]]; then fi input_file="$(realpath "$1")" -sign_darwin="$([[ "$sign_darwin" == 1 ]] && [[ "$os" == "darwin" ]] && echo 1 || echo 0)" -if [[ "$sign_darwin" == 1 ]] && [[ "${AC_APPLICATION_IDENTITY:-}" == "" ]]; then - error "AC_APPLICATION_IDENTITY must be set when --sign-darwin or CODER_SIGN_DARWIN=1 is supplied" -fi - # Check dependencies if [[ "$format" == "zip" ]]; then dependencies zip diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh index 06b1b2fbe6bc7..1c23668e028c7 100755 --- a/scripts/build_docker.sh +++ b/scripts/build_docker.sh @@ -3,14 +3,14 @@ # This script builds a Docker image of Coder containing the given binary, for # the given architecture. Only linux binaries are supported at this time. # -# Usage: ./build_docker.sh --arch amd64 [--version 1.2.3] [--target image_tag] [--push] path/to/coder +# Usage: ./build_docker.sh --arch amd64 [--version 1.2.3] [--push] path/to/coder # # The --arch parameter is required and accepts a Golang arch specification. It # will be automatically mapped to a suitable architecture that Docker accepts # before being passed to `docker buildx build`. # # The image will be built and tagged against the image tag returned by -# ./image_tag.sh unless a --target parameter is supplied. +# ./image_tag.sh. # # If no version is specified, defaults to the version from ./version.sh. # @@ -23,11 +23,10 @@ set -euo pipefail source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" arch="" -image_tag="" version="" push=0 -args="$(getopt -o "" -l arch:,target:,version:,push -- "$@")" +args="$(getopt -o "" -l arch:,version:,push -- "$@")" eval set -- "$args" while true; do case "$1" in @@ -35,10 +34,6 @@ while true; do arch="$2" shift 2 ;; - --target) - image_tag="$2" - shift 2 - ;; --version) version="$2" shift 2 @@ -70,9 +65,7 @@ if [[ "$version" == "" ]]; then version="$(execrelative ./version.sh)" fi -if [[ "$image_tag" == "" ]]; then - image_tag="$(execrelative ./image_tag.sh --arch "$arch" --version="$version")" -fi +image_tag="$(execrelative ./image_tag.sh --arch "$arch" --version="$version")" if [[ "$#" != 1 ]]; then error "Exactly one argument must be provided to this script, $# were supplied" diff --git a/scripts/build_go.sh b/scripts/build_go.sh index 3057c3e8cbf5b..fc5825e82725b 100755 --- a/scripts/build_go.sh +++ b/scripts/build_go.sh @@ -12,7 +12,7 @@ # with "vX" (e.g. "v7", "v8"). # # Unless overridden via --output, the built binary will be dropped in -# "$repo_root/build/coder_$version_$os_$arch" (with a ".exe" suffix for windows +# "$repo_root/dist/coder_$version_$os_$arch" (with a ".exe" suffix for windows # builds) and the absolute path to the binary will be printed to stdout on # completion. # @@ -26,12 +26,13 @@ set -euo pipefail # shellcheck source=scripts/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +cdroot version="" os="${GOOS:-linux}" arch="${GOARCH:-amd64}" slim="${CODER_SLIM_BUILD:-0}" -sign_darwin="${CODER_SIGN_DARWIN:-0}" +sign_darwin=0 output_path="" agpl="${CODER_BUILD_AGPL:-0}" @@ -52,7 +53,6 @@ while true; do shift 2 ;; --output) - mkdir -p "$(dirname "$2")" output_path="$(realpath "$2")" shift 2 ;; @@ -81,8 +81,6 @@ while true; do esac done -cdroot - # Remove the "v" prefix. version="${version#v}" if [[ "$version" == "" ]]; then @@ -104,8 +102,9 @@ fi # Compute default output path. if [[ "$output_path" == "" ]]; then - mkdir -p "build" - output_path="build/coder_${version}_${os}_${arch}" + dist_dir="dist" + mkdir -p "$dist_dir" + output_path="${dist_dir}/coder_${version}_${os}_${arch}" if [[ "$os" == "windows" ]]; then output_path+=".exe" fi @@ -133,13 +132,7 @@ CGO_ENABLED=0 GOOS="$os" GOARCH="$arch" GOARM="$arm_version" go build \ "$cmd_path" 1>&2 if [[ "$sign_darwin" == 1 ]] && [[ "$os" == "darwin" ]]; then - codesign \ - -f -v \ - -s "$AC_APPLICATION_IDENTITY" \ - --timestamp \ - --options runtime \ - "$output_path" \ - 1>&2 + codesign -s "$AC_APPLICATION_IDENTITY" -f -v --timestamp --options runtime "$output_path" fi echo "$output_path" diff --git a/scripts/build_go_matrix.sh b/scripts/build_go_matrix.sh new file mode 100755 index 0000000000000..cb6b2bedec203 --- /dev/null +++ b/scripts/build_go_matrix.sh @@ -0,0 +1,243 @@ +#!/usr/bin/env bash + +# This script builds multiple Go binaries for Coder with the given OS and +# architecture combinations. +# +# Usage: ./build_go_matrix.sh [--version 1.2.3-devel+abcdef] [--output dist/] [--slim] [--sign-darwin] [--archive] [--package-linux] [--agpl] os1:arch1,arch2 os2:arch1 os1:arch3 +# +# If no OS:arch combinations are provided, nothing will happen and no error will +# be returned. Slim builds are disabled by default. If no version is specified, +# defaults to the version from ./version.sh +# +# The --output parameter must be a directory with a trailing slash where all +# files will be dropped with the default name scheme +# `coder_$version_$os_$arch(.exe)?`, or must contain the `{os}` and `{arch}` +# template variables. You may also use `{version}`. Note that for windows builds +# the `.exe` suffix will be appended automatically. +# +# Unless overridden via --output, the built binary will be dropped in +# "$repo_root/dist/coder_$version_$os_$arch" (with a ".exe" suffix for windows +# builds). +# +# If the --sign-darwin parameter is specified, all darwin binaries will be +# signed using the `codesign` utility. $AC_APPLICATION_IDENTITY must be set and +# the signing certificate must be imported for this to work. +# +# If the --archive parameter is specified, all binaries will be archived using +# ./archive.sh. The --sign-darwin parameter will be carried through, and all +# archive files will be dropped in the output directory with the same name as +# the binary and the .zip (for windows and darwin) or .tar.gz extension. +# +# If the --package-linux parameter is specified, all linux binaries will be +# packaged using ./package.sh. Requires the nfpm binary. +# +# If the --agpl parameter is specified, builds only the AGPL-licensed code (no +# Coder enterprise features). + +set -euo pipefail +# shellcheck source=scripts/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +version="" +output_path="" +slim=0 +sign_darwin=0 +archive=0 +package_linux=0 +agpl=0 + +args="$(getopt -o "" -l version:,output:,slim,sign-darwin,archive,package-linux,agpl -- "$@")" +eval set -- "$args" +while true; do + case "$1" in + --version) + version="$2" + shift 2 + ;; + --output) + output_path="$2" + shift 2 + ;; + --slim) + slim=1 + shift + ;; + --sign-darwin) + if [[ "${AC_APPLICATION_IDENTITY:-}" == "" ]]; then + error "AC_APPLICATION_IDENTITY must be set when --sign-darwin is supplied" + fi + sign_darwin=1 + shift + ;; + --archive) + archive=1 + shift + ;; + --package-linux) + package_linux=1 + shift + ;; + --agpl) + agpl=1 + shift + ;; + --) + shift + break + ;; + *) + error "Unrecognized option: $1" + ;; + esac +done + +# Verify the output path template. +if [[ "$output_path" == "" ]]; then + # Input paths are relative, so we don't cdroot at the top, but for this case + # we want it to be relative to the root. + cdroot + mkdir -p dist + output_path="$(realpath "dist/coder_{version}_{os}_{arch}")" +elif [[ "$output_path" == */ ]]; then + output_path="${output_path}coder_{version}_{os}_{arch}" +elif [[ "$output_path" != *"{os}"* ]] || [[ "$output_path" != *"{arch}"* ]]; then + # If the output path isn't a directory (ends with /) then it must have + # template variables. + error "Templated output path '$output_path' must contain {os} and {arch}" +fi + +mkdir -p "$(dirname "$output_path")" +output_path="$(realpath "$output_path")" + +# Remove the "v" prefix. +version="${version#v}" +if [[ "$version" == "" ]]; then + version="$(execrelative ./version.sh)" +fi + +# Parse the os:arch specs into an array. +specs=() +may_zip=0 +may_tar=0 +for spec in "$@"; do + spec_os="$(echo "$spec" | cut -d ":" -f 1)" + if [[ "$spec_os" == "" ]] || [[ "$spec_os" == *" "* ]]; then + error "Could not parse matrix build spec '$spec': invalid OS '$spec_os'" + fi + + # Determine which dependencies we need. + if [[ "$spec_os" == "windows" ]] || [[ "$spec_os" == "darwin" ]]; then + may_zip=1 + else + may_tar=1 + fi + + # No quoting is important here. + for spec_arch in $(echo "$spec" | cut -d ":" -f 2 | tr "," "\n"); do + if [[ "$spec_arch" == "" ]] || [[ "$spec_os" == *" "* ]]; then + error "Could not parse matrix build spec '$spec': invalid architecture '$spec_arch'" + fi + + specs+=("$spec_os:$spec_arch") + done +done + +# Remove duplicate specs while maintaining the same order. +specs_str="${specs[*]}" +specs=() +for s in $(echo "$specs_str" | tr " " "\n" | awk '!a[$0]++'); do + specs+=("$s") +done + +# Check dependencies +dependencies go +if [[ "$sign_darwin" == 1 ]]; then + dependencies jq codesign gon +fi +if [[ "$archive" == 1 ]]; then + if [[ "$may_zip" == 1 ]]; then + dependencies zip + fi + if [[ "$may_tar" == 1 ]]; then + dependencies tar + fi +fi +if [[ "$package_linux" == 1 ]]; then + dependencies nfpm +fi + +bin_name="coder" +build_args=() +if [[ "$slim" == 1 ]]; then + bin_name+="-slim" + build_args+=(--slim) +fi +if [[ "$sign_darwin" == 1 ]]; then + build_args+=(--sign-darwin) +fi +if [[ "$agpl" == 1 ]]; then + build_args+=(--agpl) +fi + +# Build each spec. +for spec in "${specs[@]}"; do + spec_os="$(echo "$spec" | cut -d ":" -f 1)" + spec_arch="$(echo "$spec" | cut -d ":" -f 2)" + + # Craft output path from the template. + spec_output="$output_path" + spec_output="${spec_output//\{os\}/"$spec_os"}" + spec_output="${spec_output//\{arch\}/"$spec_arch"}" + spec_output="${spec_output//\{version\}/"$version"}" + + spec_output_binary="$spec_output" + if [[ "$spec_os" == "windows" ]]; then + spec_output_binary+=".exe" + fi + + # Ensure parent dir. + mkdir -p "$(dirname "$spec_output")" + + log "--- Building $bin_name for $spec_os $spec_arch ($spec_output_binary)" + execrelative ./build_go.sh \ + --version "$version" \ + --os "$spec_os" \ + --arch "$spec_arch" \ + --output "$spec_output_binary" \ + "${build_args[@]}" + log + log + + if [[ "$archive" == 1 ]]; then + spec_archive_format="tar.gz" + if [[ "$spec_os" == "windows" ]] || [[ "$spec_os" == "darwin" ]]; then + spec_archive_format="zip" + fi + spec_output_archive="$spec_output.$spec_archive_format" + + archive_args=() + if [[ "$sign_darwin" == 1 ]] && [[ "$spec_os" == "darwin" ]]; then + archive_args+=(--sign-darwin) + fi + if [[ "$agpl" == 1 ]]; then + archive_args+=(--agpl) + fi + + log "--- Creating archive for $spec_os $spec_arch ($spec_output_archive)" + execrelative ./archive.sh \ + --format "$spec_archive_format" \ + --output "$spec_output_archive" \ + "${archive_args[@]}" \ + "$spec_output_binary" + log + log + fi + + if [[ "$package_linux" == 1 ]] && [[ "$spec_os" == "linux" ]]; then + execrelative ./package.sh \ + --arch "$spec_arch" \ + --version "$version" \ + "$spec_output_binary" + log + fi +done diff --git a/scripts/build_go_slim.sh b/scripts/build_go_slim.sh new file mode 100755 index 0000000000000..69320a8a3c8e7 --- /dev/null +++ b/scripts/build_go_slim.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash + +# This script builds multiple "slim" Go binaries for Coder with the given OS and +# architecture combinations. This wraps ./build_go_matrix.sh. +# +# Usage: ./build_go_slim.sh [--version 1.2.3-devel+abcdef] [--output dist/] [--compress 22] [--agpl] os1:arch1,arch2 os2:arch1 os1:arch3 +# +# If no OS:arch combinations are provided, nothing will happen and no error will +# be returned. If no version is specified, defaults to the version from +# ./version.sh +# +# The --output parameter differs from ./build_go_matrix.sh, in that it does not +# accept variables such as `{os}` and `{arch}` and only accepts a directory +# ending with `/`. +# +# The built binaries are additionally copied to the site output directory so +# they can be packaged into non-slim binaries correctly. +# +# When the --compress parameter is provided, the binaries in site/bin +# will be compressed using zstd into site/bin/coder.tar.zst, this helps reduce +# final binary size significantly. +# +# If the --agpl parameter is specified, builds only the AGPL-licensed code (no +# Coder enterprise features). + +set -euo pipefail +shopt -s nullglob +# shellcheck source=scripts/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +version="" +output_path="" +compress=0 +agpl=0 + +args="$(getopt -o "" -l version:,output:,compress:,agpl -- "$@")" +eval set -- "$args" +while true; do + case "$1" in + --version) + version="$2" + shift 2 + ;; + --output) + output_path="$2" + shift 2 + ;; + --compress) + compress="$2" + shift 2 + ;; + --agpl) + agpl=1 + shift + ;; + --) + shift + break + ;; + *) + error "Unrecognized option: $1" + ;; + esac +done + +# Check dependencies +dependencies go +if [[ $compress != 0 ]]; then + dependencies openssl tar zstd + + if [[ $compress != [0-9]* ]] || [[ $compress -gt 22 ]] || [[ $compress -lt 1 ]]; then + error "Invalid value for compress, must in in the range of [1, 22]" + fi +fi + +# Remove the "v" prefix. +version="${version#v}" +if [[ "$version" == "" ]]; then + version="$(execrelative ./version.sh)" +fi + +# Verify the output path. +if [[ "$output_path" == "" ]]; then + # Input paths are relative, so we don't cdroot at the top, but for this case + # we want it to be relative to the root. + cdroot + mkdir -p dist + output_path="$(realpath "dist/coder-slim_{version}_{os}_{arch}")" +elif [[ "$output_path" != */ ]] || [[ "$output_path" == *"{"* ]]; then + error "The output path '$output_path' cannot contain variables and must end with a slash" +else + mkdir -p "$output_path" + output_path="$(realpath "${output_path}coder-slim_{version}_{os}_{arch}")" +fi + +build_args=(--slim) +if [[ "$agpl" == 1 ]]; then + build_args+=(--agpl) +fi + +./scripts/build_go_matrix.sh \ + --version "$version" \ + --output "$output_path" \ + "${build_args[@]}" \ + "$@" + +cdroot +dest_dir="./site/out/bin" +mkdir -p "$dest_dir" +dest_dir="$(realpath "$dest_dir")" + +# Copy the binaries to the site directory. +cd "$(dirname "$output_path")" +for f in ./coder-slim_*; do + # Remove ./ prefix + name="${f#./}" + # Remove "-slim_$version" + truncated="${name//-slim_$version/}" + # Replace underscores with hyphens + hyphenated="${truncated//_/-}" + dest="$dest_dir/$hyphenated" + cp "$f" "$dest" +done + +if [[ $compress != 0 ]]; then + pushd "$dest_dir" + sha_file=coder.sha1 + sha_dest="$dest_dir/$sha_file" + log "--- Generating SHA1 for coder-slim binaries ($sha_dest)" + openssl dgst -r -sha1 coder-* | tee $sha_file + echo "$sha_dest" + log + log + + tar_name=coder.tar.zst + tar_dest="$dest_dir/$tar_name" + log "--- Compressing coder-slim binaries using zstd level $compress ($tar_dest)" + tar cf coder.tar $sha_file coder-* + rm coder-* + zstd --force --ultra --long -"${compress}" --rm --no-progress coder.tar -o $tar_name + echo "$tar_dest" + log + log + + popd +fi diff --git a/scripts/check_unstaged.sh b/scripts/check_unstaged.sh index a6de5f0204ef8..47bdd265ce407 100755 --- a/scripts/check_unstaged.sh +++ b/scripts/check_unstaged.sh @@ -1,28 +1,30 @@ #!/bin/bash set -euo pipefail -# shellcheck source=scripts/lib.sh -source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -cdroot -FILES="$(git ls-files --other --modified --exclude-standard)" -if [[ "$FILES" != "" ]]; then - mapfile -t files <<<"$FILES" +SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") +PROJECT_ROOT=$(cd "$SCRIPT_DIR" && git rev-parse --show-toplevel) - log - log "The following files contain unstaged changes:" - log - for file in "${files[@]}"; do - log " - $file" - done +( + cd "${PROJECT_ROOT}" - log - log "These are the changes:" - log - for file in "${files[@]}"; do - git --no-pager diff "$file" 1>&2 - done + FILES="$(git ls-files --other --modified --exclude-standard)" + if [[ "$FILES" != "" ]]; then + mapfile -t files <<<"$FILES" - log - error "Unstaged changes, see above for details." -fi + echo "The following files contain unstaged changes:" + echo + for file in "${files[@]}"; do + echo " - $file" + done + echo + + echo "These are the changes:" + echo + for file in "${files[@]}"; do + git --no-pager diff "$file" + done + exit 1 + fi +) +exit 0 diff --git a/scripts/coder-dev.sh b/scripts/coder-dev.sh index 5f9b2d02fba3e..f824b4b5a821b 100755 --- a/scripts/coder-dev.sh +++ b/scripts/coder-dev.sh @@ -7,18 +7,17 @@ set -euo pipefail SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") # shellcheck disable=SC1091,SC1090 source "${SCRIPT_DIR}/lib.sh" +PROJECT_ROOT=$(cd "$SCRIPT_DIR" && git rev-parse --show-toplevel) -GOOS="$(go env GOOS)" -GOARCH="$(go env GOARCH)" -CODER_DEV_BIN="build/coder_${GOOS}_${GOARCH}" - -cdroot -mkdir -p ./.coderv2 -CODER_DEV_DIR="$(realpath ./.coderv2)" +CODER_DEV_DIR="$PROJECT_ROOT/.coderv2/" +CODER_DEV_BIN="${CODER_DEV_DIR}/coder" +if [[ ! -d "${CODER_DEV_DIR}" ]]; then + mkdir -p "${CODER_DEV_DIR}" +fi if [[ ! -x "${CODER_DEV_BIN}" ]]; then echo "Run this command first:" - echo " make $CODER_DEV_BIN" + echo "go build -o ${CODER_DEV_BIN} ${PROJECT_ROOT}/enterprise/cmd/coder" exit 1 fi diff --git a/scripts/develop.sh b/scripts/develop.sh index 0020b724b0718..df0cafd2bae65 100755 --- a/scripts/develop.sh +++ b/scripts/develop.sh @@ -5,28 +5,19 @@ # If the --agpl parameter is specified, builds only the AGPL-licensed code (no # Coder enterprise features). -SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") -# shellcheck disable=SC1091,SC1090 -source "${SCRIPT_DIR}/lib.sh" - # Allow toggling verbose output -[[ -n ${VERBOSE:-} ]] && set -x +[[ -n ${VERBOSE:-""} ]] && set -x set -euo pipefail -password="${CODER_DEV_ADMIN_PASSWORD:-password}" - -args="$(getopt -o "" -l agpl,password: -- "$@")" +agpl="${CODER_BUILD_AGPL:-0}" +args="$(getopt -o "" -l agpl -- "$@")" eval set -- "$args" while true; do case "$1" in --agpl) - export CODER_BUILD_AGPL=1 + agpl=1 shift ;; - --password) - password="$2" - shift 2 - ;; --) shift break @@ -37,20 +28,43 @@ while true; do esac done +SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") +# shellcheck disable=SC1091,SC1090 +source "${SCRIPT_DIR}/lib.sh" +PROJECT_ROOT=$(cd "$SCRIPT_DIR" && git rev-parse --show-toplevel) +CODER_DEV_BIN="${PROJECT_ROOT}/.coderv2/coder" +set +u +CODER_DEV_ADMIN_PASSWORD="${CODER_DEV_ADMIN_PASSWORD:-password}" +set -u + # Preflight checks: ensure we have our required dependencies, and make sure nothing is listening on port 3000 or 8080 dependencies curl git go make yarn curl --fail http://127.0.0.1:3000 >/dev/null 2>&1 && echo '== ERROR: something is listening on port 3000. Kill it and re-run this script.' && exit 1 curl --fail http://127.0.0.1:8080 >/dev/null 2>&1 && echo '== ERROR: something is listening on port 8080. Kill it and re-run this script.' && exit 1 -# Compile the CLI binary. This should also compile the frontend and refresh -# node_modules if necessary. -GOOS="$(go env GOOS)" -GOARCH="$(go env GOARCH)" -make "build/coder_${GOOS}_${GOARCH}" +if [[ ! -e ./site/out/bin/coder.sha1 && ! -e ./site/out/bin/coder.tar.zst ]]; then + log + log "=======================================================================" + log "== Run 'make bin' before running this command to build binaries. ==" + log "== Without these binaries, workspaces will fail to start! ==" + log "=======================================================================" + log + exit 1 +fi +cmd_path="enterprise/cmd/coder" +if [[ "$agpl" == 1 ]]; then + cmd_path="cmd/coder" +fi + +# Compile the CLI binary once just so we don't waste time compiling things multiple times +go build -tags embed -o "${CODER_DEV_BIN}" "${PROJECT_ROOT}/${cmd_path}" # Use the coder dev shim so we don't overwrite the user's existing Coder config. CODER_DEV_SHIM="${PROJECT_ROOT}/scripts/coder-dev.sh" +# Run yarn install, to make sure node_modules are ready to go +"$PROJECT_ROOT/scripts/yarn_install.sh" + # This is a way to run multiple processes in parallel, and have Ctrl-C work correctly # to kill both at the same time. For more details, see: # https://stackoverflow.com/questions/3004811/how-do-you-run-multiple-programs-in-parallel-from-a-bash-script @@ -59,49 +73,51 @@ CODER_DEV_SHIM="${PROJECT_ROOT}/scripts/coder-dev.sh" # rather than leaving things in an inconsistent state. trap 'kill -TERM -$$' ERR cdroot - "${CODER_DEV_SHIM}" server --address 127.0.0.1:3000 --tunnel || kill -INT -$$ & + "${CODER_DEV_SHIM}" server --address 127.0.0.1:3000 --in-memory --tunnel || kill -INT -$$ & echo '== Waiting for Coder to become ready' timeout 60s bash -c 'until curl -s --fail http://localhost:3000 > /dev/null 2>&1; do sleep 0.5; done' - # Try to create the initial admin user. - "${CODER_DEV_SHIM}" login http://127.0.0.1:3000 --username=admin --email=admin@coder.com --password="${password}" || + # create the first user, the admin + "${CODER_DEV_SHIM}" login http://127.0.0.1:3000 --username=admin --email=admin@coder.com --password="${CODER_DEV_ADMIN_PASSWORD}" || echo 'Failed to create admin user. To troubleshoot, try running this command manually.' - # Try to create a regular user. - "${CODER_DEV_SHIM}" users create --email=member@coder.com --username=member --password="${password}" || + # || true to always exit code 0. If this fails, whelp. + "${CODER_DEV_SHIM}" users create --email=member@coder.com --username=member --password="${CODER_DEV_ADMIN_PASSWORD}" || echo 'Failed to create regular user. To troubleshoot, try running this command manually.' - # If we have docker available and the "docker" template doesn't already - # exist, then let's try to create a template! - example_template="code-server" - template_name="docker" - if docker info >/dev/null 2>&1 && ! "${CODER_DEV_SHIM}" templates versions list "${template_name}"; then - temp_template_dir="$(mktemp -d)" - echo "${example_template}" | "${CODER_DEV_SHIM}" templates init "${temp_template_dir}" - - DOCKER_HOST="$(docker context inspect --format '{{ .Endpoints.docker.Host }}')" - printf 'docker_arch: "%s"\ndocker_host: "%s"\n' "${GOARCH}" "${DOCKER_HOST}" >"${temp_template_dir}/params.yaml" + # If we have docker available, then let's try to create a template! + template_name="" + if docker info >/dev/null 2>&1; then + temp_template_dir=$(mktemp -d) + echo code-server | "${CODER_DEV_SHIM}" templates init "${temp_template_dir}" + # shellcheck disable=SC1090 + source <(go env | grep GOARCH) + DOCKER_HOST=$(docker context inspect --format '{{.Endpoints.docker.Host}}') + printf 'docker_arch: "%s"\ndocker_host: "%s"\n' "${GOARCH}" "${DOCKER_HOST}" | tee "${temp_template_dir}/params.yaml" + template_name="docker-${GOARCH}" ( - "${CODER_DEV_SHIM}" templates create "${template_name}" --directory "${temp_template_dir}" --parameter-file "${temp_template_dir}/params.yaml" --yes - rm -rfv "${temp_template_dir}" # Only delete template dir if template creation succeeds + "${CODER_DEV_SHIM}" templates create "${template_name}" --directory "${temp_template_dir}" --parameter-file "${temp_template_dir}/params.yaml" --yes && + rm -rfv "${temp_template_dir}" # Only delete template dir if template creation succeeds ) || echo "Failed to create a template. The template files are in ${temp_template_dir}" fi # Start the frontend once we have a template up and running CODER_HOST=http://127.0.0.1:3000 INSPECT_XSTATE=true yarn --cwd=./site dev || kill -INT -$$ & - log - log "====================================================================" - log "== ==" - log "== Coder is now running in development mode. ==" - log "== API: http://localhost:3000 ==" - log "== Web UI: http://localhost:8080 ==" - log "== ==" - log "== Use ./scripts/coder-dev.sh to talk to this instance! ==" - log "====================================================================" + log "=======================================================================" + log "== ==" + log "== Coder is now running in development mode. ==" + log "== API : http://localhost:3000 ==" + log "== Web UI: http://localhost:8080 ==" + if [[ -n "${template_name}" ]]; then + log "== ==" + log "== Docker template ${template_name} is ready to use! ==" + log "== Use ./scripts/coder-dev.sh to talk to this instance! ==" + log "== ==" + fi + log "=======================================================================" log - # Wait for both frontend and backend to exit. wait ) diff --git a/scripts/helm.sh b/scripts/helm.sh index 33b556f0100a1..5978a5f373937 100755 --- a/scripts/helm.sh +++ b/scripts/helm.sh @@ -9,7 +9,7 @@ # If no version is specified, defaults to the version from ./version.sh. # # If no output path is specified, defaults to -# "$repo_root/build/coder_helm_$version.tgz". +# "$repo_root/dist/coder_helm_$version.tgz". # # If the --push parameter is specified, the resulting artifact will be published # to the Coder OSS repo. This requires `gsutil` to be installed and configured. @@ -56,8 +56,8 @@ fi if [[ "$output_path" == "" ]]; then cdroot - mkdir -p build - output_path="$(realpath "build/coder_helm_$version.tgz")" + mkdir -p dist + output_path="$(realpath "dist/coder_helm_$version.tgz")" fi # Check dependencies diff --git a/scripts/package.sh b/scripts/package.sh index 50db188aba9bd..1b07e8fd563fa 100755 --- a/scripts/package.sh +++ b/scripts/package.sh @@ -1,21 +1,22 @@ #!/usr/bin/env bash -# This script creates a Linux package for the given binary. +# This script creates Linux packages for the given binary. It will output a +# .rpm, .deb and .apk file in the same directory as the input file with the same +# filename (except the package format suffix). # -# ./package.sh --arch amd64 --format "(apk|deb|rpm)" --output "path/to/coder.apk" [--version 1.2.3] path/to/coder +# ./package.sh --arch amd64 [--version 1.2.3] path/to/coder # -# If no version is specified, defaults to the version from ./version.sh. +# The --arch parameter is required. If no version is specified, defaults to the +# version from ./version.sh. set -euo pipefail # shellcheck source=scripts/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -arch="" -format="" -output_path="" version="" +arch="" -args="$(getopt -o "" -l arch:,format:,output:,version: -- "$@")" +args="$(getopt -o "" -l arch:,version: -- "$@")" eval set -- "$args" while true; do case "$1" in @@ -23,15 +24,6 @@ while true; do arch="$2" shift 2 ;; - --format) - format="$2" - shift 2 - ;; - --output) - mkdir -p "$(dirname "$2")" - output_path="$(realpath "$2")" - shift 2 - ;; --version) version="$2" shift 2 @@ -49,12 +41,6 @@ done if [[ "$arch" == "" ]]; then error "--arch is a required parameter" fi -if [[ "$format" != "apk" ]] && [[ "$format" != "deb" ]] && [[ "$format" != "rpm" ]]; then - error "--format is a required parameter and must be one of 'apk', 'deb', or 'rpm'" -fi -if [[ "$output_path" == "" ]]; then - error "--output is a required parameter" -fi if [[ "$#" != 1 ]]; then error "Exactly one argument must be provided to this script, $# were supplied" @@ -88,12 +74,18 @@ ln "$(realpath coder.service)" "$temp_dir/" ln "$(realpath preinstall.sh)" "$temp_dir/" ln "$(realpath scripts/nfpm.yaml)" "$temp_dir/" -pushd "$temp_dir" -GOARCH="$arch" CODER_VERSION="$version" nfpm package \ - -f nfpm.yaml \ - -p "$format" \ - -t "$output_path" \ - 1>&2 -popd +cd "$temp_dir" + +formats=(apk deb rpm) +for format in "${formats[@]}"; do + output_path="$input_file.$format" + log "--- Building $format package ($output_path)" + GOARCH="$arch" CODER_VERSION="$version" nfpm package \ + -f nfpm.yaml \ + -p "$format" \ + -t "$output_path" +done + +cdroot rm -rf "$temp_dir" diff --git a/scripts/publish_release.sh b/scripts/publish_release.sh index 80725cca9a931..338ffbb99ff7b 100755 --- a/scripts/publish_release.sh +++ b/scripts/publish_release.sh @@ -3,10 +3,6 @@ # This script generates release notes and publishes all of the given assets to # GitHub releases. Depends on GitHub CLI. # -# THIS IS NOT INTENDED TO BE CALLED BY DEVELOPERS! This is called by the release -# pipeline to do the final publish step. If you want to create a release use: -# git tag -a -m "$ver" "$ver" && git push origin "$ver" -# # Usage: ./publish_release.sh [--version 1.2.3] [--dry-run] path/to/asset1 path/to/asset2 ... # # The supplied images must already be pushed to the registry or this will fail. @@ -29,10 +25,6 @@ set -euo pipefail # shellcheck source=scripts/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -if [[ "${CI:-}" == "" ]]; then - error "This script must be run in CI" -fi - version="" dry_run=0 diff --git a/scripts/yarn_install.sh b/scripts/yarn_install.sh index ed235ba3dacd1..45a7dbf7e559e 100755 --- a/scripts/yarn_install.sh +++ b/scripts/yarn_install.sh @@ -5,37 +5,39 @@ # # Usage: yarn_install.sh [optional extra flags] -set -euo pipefail -# shellcheck source=scripts/lib.sh -source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +set -eo pipefail -cdroot -cd site +SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") +PROJECT_ROOT=$(cd "$SCRIPT_DIR" && git rev-parse --show-toplevel) -yarn_flags=( - # Do not execute install scripts - # TODO: check if build works properly with this enabled - # --ignore-scripts +( + cd "$PROJECT_ROOT/site" - # Check if existing node_modules are valid - # TODO: determine if this is necessary - # --check-files -) + yarn_flags=( + # Do not execute install scripts + # TODO: check if build works properly with this enabled + # --ignore-scripts -if [[ -n ${CI:-} ]]; then - yarn_flags+=( - # Install dependencies from lockfile, ensuring builds are fully - # reproducible - --frozen-lockfile - # Suppress progress information - --silent - # Disable interactive prompts for build - --non-interactive + # Check if existing node_modules are valid + # TODO: determine if this is necessary + # --check-files ) -fi -# Append whatever is specified on the command line -yarn_flags+=("$@") + if [[ -n ${CI:-} ]]; then + yarn_flags+=( + # Install dependencies from lockfile, ensuring builds are fully + # reproducible + --frozen-lockfile + # Suppress progress information + --silent + # Disable interactive prompts for build + --non-interactive + ) + fi + + # Append whatever is specified on the command line + yarn_flags+=("$@") -echo "+ yarn install ${yarn_flags[*]}" -yarn install "${yarn_flags[@]}" + echo "+ yarn install ${yarn_flags[*]}" + yarn install "${yarn_flags[@]}" +)