diff --git a/.github/workflows/coder.yaml b/.github/workflows/coder.yaml index bd6d6f5753939..382e9adb4bc13 100644 --- a/.github/workflows/coder.yaml +++ b/.github/workflows/coder.yaml @@ -111,9 +111,8 @@ jobs: - uses: actions/setup-go@v3 with: go-version: "~1.18" - - run: curl -sSL - https://github.com/kyleconroy/sqlc/releases/download/v1.13.0/sqlc_1.13.0_linux_amd64.tar.gz - | sudo tar -C /usr/bin -xz sqlc + - run: | + curl -sSL https://github.com/kyleconroy/sqlc/releases/download/v1.13.0/sqlc_1.13.0_linux_amd64.tar.gz | sudo tar -C /usr/bin -xz sqlc - run: go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26 - run: go install storj.io/drpc/cmd/protoc-gen-go-drpc@v0.0.26 @@ -188,7 +187,7 @@ jobs: path: ${{ steps.go-cache-paths.outputs.go-mod }} key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }} - - name: Install goreleaser + - name: Install gotestsum uses: jaxxstorm/action-install-gh-release@v1.7.1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -226,7 +225,7 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} files: ./gotests.coverage flags: unittest-go-${{ matrix.os }} - # this flakes and sometimes fails the build + # this flakes and sometimes fails the build fail_ci_if_error: false test-go-postgres: @@ -258,7 +257,7 @@ jobs: path: ${{ steps.go-cache-paths.outputs.go-mod }} key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }} - - name: Install goreleaser + - name: Install gotestsum uses: jaxxstorm/action-install-gh-release@v1.7.1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -310,7 +309,7 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} files: ./gotests.coverage flags: unittest-go-postgres-${{ matrix.os }} - # this flakes and sometimes fails the build + # this flakes and sometimes fails the build fail_ci_if_error: false deploy: @@ -366,18 +365,14 @@ jobs: restore-keys: | js-${{ runner.os }}- - - uses: goreleaser/goreleaser-action@v3 - with: - install-only: true + - name: Install nfpm + run: go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.16.0 - name: Build site run: make -B site/out/index.html - name: Build Release - uses: goreleaser/goreleaser-action@v3 - with: - version: latest - args: release --snapshot --rm-dist --skip-sign + run: make build - name: Install Release run: | @@ -394,8 +389,11 @@ jobs: with: name: coder path: | - ./dist/coder_*_linux_amd64.tar.gz - ./dist/coder_*_windows_amd64.zip + ./dist/*.zip + ./dist/*.tar.gz + ./dist/*.apk + ./dist/*.deb + ./dist/*.rpm retention-days: 7 test-js: @@ -437,7 +435,7 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} files: ./site/coverage/lcov.info flags: unittest-js - # this flakes and sometimes fails the build + # this flakes and sometimes fails the build fail_ci_if_error: false - name: Upload DataDog Trace @@ -484,10 +482,6 @@ jobs: with: node-version: "14" - - uses: goreleaser/goreleaser-action@v3 - with: - install-only: true - - name: Echo Go Cache Paths id: go-cache-paths run: | diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 1736a45ce1d9d..9f79c4512dd1d 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,66 +1,147 @@ +# GitHub release workflow. +# +# This workflow is a bit complicated because we have to build darwin binaries on +# a mac runner, but the mac runners are extremely slow. So instead of running +# the entire release on a mac (which will take an hour to run), we run only the +# mac build on a mac, and the rest on a linux runner. The final release is then +# published using a final linux runner. name: release on: push: tags: - "v*" workflow_dispatch: + inputs: + snapshot: + description: Force a dev version to be generated, implies dry_run. + type: boolean + required: true + dry_run: + description: Perform a dry-run release. + type: boolean + required: true + +env: + CODER_RELEASE: ${{ github.event.inputs.snapshot && 'false' || 'true' }} jobs: - goreleaser: - runs-on: macos-latest - env: - # Necessary for Docker manifest - DOCKER_CLI_EXPERIMENTAL: "enabled" + linux-windows: + runs-on: ubuntu-latest steps: - # Docker is not included on macos-latest - - uses: docker-practice/actions-setup-docker@1.0.10 - - uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Docker Login - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - uses: actions/setup-go@v3 with: go-version: "~1.18" - - name: Install Gon + - name: Cache Node + id: cache-node + uses: actions/cache@v3 + with: + path: | + **/node_modules + .eslintcache + key: js-${{ runner.os }}-test-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + js-${{ runner.os }}- + + - name: Install nfpm + run: go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.16.0 + + - name: Build Site + run: make site/out/index.html + + - name: Build Linux and Windows Binaries run: | - brew tap mitchellh/gon - brew install mitchellh/gon/gon + set -euo pipefail + go mod download - - name: Import Signing Certificates - uses: Apple-Actions/import-codesign-certs@v1 - with: - p12-file-base64: ${{ secrets.AC_CERTIFICATE_P12_BASE64 }} - p12-password: ${{ secrets.AC_CERTIFICATE_PASSWORD }} + mkdir -p ./dist + # build slim binaries + ./scripts/build_go_slim.sh \ + --output ./dist/ \ + linux:amd64,armv7,arm64 \ + windows:amd64,arm64 \ + darwin:amd64,arm64 - - name: Echo Go Cache Paths - id: go-cache-paths + # build linux and windows binaries + ./scripts/build_go_matrix.sh \ + --output ./dist/ \ + --archive \ + --package-linux \ + linux:amd64,armv7,arm64 \ + windows:amd64,arm64 + + - name: Build Linux Docker images run: | - echo "::set-output name=go-build::$(go env GOCACHE)" - echo "::set-output name=go-mod::$(go env GOMODCACHE)" + set -euxo pipefail - - name: Go Build Cache - uses: actions/cache@v3 + # build and (maybe) push Docker images for each architecture + images=() + for arch in amd64 armv7 arm64; do + img="$( + ./scripts/build_docker.sh \ + ${{ (!github.event.inputs.dry_run && !github.event.inputs.snapshot) && '--push' || '' }} \ + --arch "$arch" \ + ./dist/coder_*_linux_"$arch" + )" + images+=("$img") + done + + # we can't build multi-arch if the images aren't pushed, so quit now + # if dry-running + if [[ "$CODER_RELEASE" != *t* ]]; then + echo Skipping multi-arch docker builds due to dry-run. + exit 0 + fi + + # build and push multi-arch manifest + ./scripts/build_docker_multiarch.sh \ + --push \ + "${images[@]}" + + # if the current version is equal to the highest (according to semver) + # version in the repo, also create a multi-arch image as ":latest" and + # push it + if [[ "$(git tag | grep '^v' | grep -vE '(rc|dev|-|\+|\/)' | sort -r --version-sort | head -n1)" == "v$(./scripts/version.sh)" ]]; then + ./scripts/build_docker_multiarch.sh \ + --push \ + --target "$(./scripts/image_tag.sh --version latest)" \ + "${images[@]}" + fi + + - name: Upload binary artifacts + uses: actions/upload-artifact@v3 with: - path: ${{ steps.go-cache-paths.outputs.go-build }} - key: ${{ runner.os }}-release-go-build-${{ hashFiles('**/go.sum') }} + name: linux + path: | + dist/*.zip + dist/*.tar.gz + dist/*.apk + dist/*.deb + dist/*.rpm - - name: Go Mod Cache - uses: actions/cache@v3 + # The mac binaries get built on mac runners because they need to be signed, + # and the signing tool only runs on mac. This darwin job only builds the Mac + # binaries and uploads them as job artifacts used by the publish step. + darwin: + runs-on: macos-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - uses: actions/setup-go@v3 + with: + go-version: "~1.18" + + - name: Import Signing Certificates + uses: Apple-Actions/import-codesign-certs@v1 with: - path: ${{ steps.go-cache-paths.outputs.go-mod }} - key: ${{ runner.os }}-release-go-mod-${{ hashFiles('**/go.sum') }} + p12-file-base64: ${{ secrets.AC_CERTIFICATE_P12_BASE64 }} + p12-password: ${{ secrets.AC_CERTIFICATE_PASSWORD }} - name: Cache Node id: cache-node @@ -73,18 +154,103 @@ jobs: restore-keys: | js-${{ runner.os }}- - - name: Install make - run: brew install make + - name: Install dependencies + run: | + set -euo pipefail + # The version of bash that MacOS ships with is too old + brew install bash + + # The version of make that MacOS ships with is too old + brew install make + echo "$(brew --prefix)/opt/make/libexec/gnubin" >> $GITHUB_PATH + + # BSD getopt is incompatible with the build scripts + brew install gnu-getopt + echo "$(brew --prefix)/opt/gnu-getopt/bin" >> $GITHUB_PATH + + # Used for notarizing the binaries + brew tap mitchellh/gon + brew install mitchellh/gon/gon - name: Build Site run: make site/out/index.html - - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v3 - with: - version: latest - args: release --rm-dist --timeout 60m + - name: Build darwin Binaries (with signatures) + run: | + set -euo pipefail + go mod download + + mkdir -p ./dist + # build slim binaries + ./scripts/build_go_slim.sh \ + --output ./dist/ \ + linux:amd64,armv7,arm64 \ + windows:amd64,arm64 \ + darwin:amd64,arm64 + + # build darwin binaries + ./scripts/build_go_matrix.sh \ + --output ./dist/ \ + --archive \ + --sign-darwin \ + darwin:amd64,arm64 env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} AC_USERNAME: ${{ secrets.AC_USERNAME }} AC_PASSWORD: ${{ secrets.AC_PASSWORD }} + AC_APPLICATION_IDENTITY: BDB050EB749EDD6A80C6F119BF1382ECA119CCCC + + - name: Upload Binary Artifacts + uses: actions/upload-artifact@v3 + with: + name: darwin + path: ./dist/coder_*.zip + + publish: + runs-on: ubuntu-latest + needs: + - linux-windows + - darwin + env: + # Necessary for Docker manifest + DOCKER_CLI_EXPERIMENTAL: "enabled" + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Docker Login + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: mkdir artifacts + run: mkdir artifacts + + - name: Download darwin Artifacts + uses: actions/download-artifact@v3 + with: + name: darwin + path: artifacts + + - name: Download Linux and Windows Artifacts + uses: actions/download-artifact@v3 + with: + name: linux + path: artifacts + + - name: ls artifacts + run: ls artifacts + + - name: Publish Release + run: | + ./scripts/publish_release.sh \ + ${{ (github.event.inputs.dry_run || github.event.inputs.snapshot) && '--dry-run' }} \ + ./artifacts/*.zip \ + ./artifacts/*.tar.gz \ + ./artifacts/*.apk \ + ./artifacts/*.deb \ + ./artifacts/*.rpm + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.goreleaser.yaml b/.goreleaser.yaml deleted file mode 100644 index 4d6ef72bc4c87..0000000000000 --- a/.goreleaser.yaml +++ /dev/null @@ -1,162 +0,0 @@ -archives: - - id: coder-linux - builds: [coder-linux] - format: tar.gz - - - id: coder-darwin - builds: [coder-darwin] - format: zip - - - id: coder-windows - builds: [coder-windows] - format: zip - -before: - hooks: - - go mod tidy - - rm -f site/out/bin/coder* - -builds: - - id: coder-slim - dir: cmd/coder - ldflags: ["-s -w -X github.com/coder/coder/buildinfo.tag={{ .Version }}"] - env: [CGO_ENABLED=0] - goos: [darwin, linux, windows] - goarch: [amd64, arm, arm64] - goarm: ["7"] - # Only build arm 7 for Linux - ignore: - - goos: windows - goarm: "7" - - goos: darwin - goarm: "7" - hooks: - # The "trimprefix" appends ".exe" on Windows. - post: | - cp {{.Path}} site/out/bin/coder-{{ .Os }}-{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}{{ trimprefix .Name "coder" }} - - - id: coder-linux - dir: cmd/coder - flags: [-tags=embed] - ldflags: ["-s -w -X github.com/coder/coder/buildinfo.tag={{ .Version }}"] - env: [CGO_ENABLED=0] - goos: [linux] - goarch: [amd64, arm, arm64] - goarm: ["7"] - - - id: coder-windows - dir: cmd/coder - flags: [-tags=embed] - ldflags: ["-s -w -X github.com/coder/coder/buildinfo.tag={{ .Version }}"] - env: [CGO_ENABLED=0] - goos: [windows] - goarch: [amd64, arm64] - - - id: coder-darwin - dir: cmd/coder - flags: [-tags=embed] - ldflags: ["-s -w -X github.com/coder/coder/buildinfo.tag={{ .Version }}"] - env: [CGO_ENABLED=0] - goos: [darwin] - goarch: [amd64, arm64] - hooks: - # This signs the binary that will be located inside the zip. - # MacOS requires the binary to be signed for notarization. - # - # If it doesn't successfully sign, the zip sign step will error. - post: | - sh -c 'codesign -s {{.Env.AC_APPLICATION_IDENTITY}} -f -v --timestamp --options runtime {{.Path}} || true' - -env: - # Apple identity for signing! - - AC_APPLICATION_IDENTITY=BDB050EB749EDD6A80C6F119BF1382ECA119CCCC - -nfpms: - - id: packages - vendor: Coder - homepage: https://coder.com - maintainer: Coder - description: | - Provision development environments with infrastructure with code - formats: - - apk - - deb - - rpm - suggests: - - postgresql - builds: - - coder-linux - bindir: /usr/bin - contents: - - src: coder.env - dst: /etc/coder.d/coder.env - type: "config|noreplace" - - src: coder.service - dst: /usr/lib/systemd/system/coder.service - scripts: - preinstall: preinstall.sh - -# Image templates are empty on snapshots to avoid lengthy builds for development. -dockers: - - image_templates: ["{{ if not .IsSnapshot }}ghcr.io/coder/coder:{{ .Tag }}-amd64{{ end }}"] - id: coder-linux - dockerfile: Dockerfile - use: buildx - build_flag_templates: - - --platform=linux/amd64 - - --label=org.opencontainers.image.title=Coder - - --label=org.opencontainers.image.description=A tool for provisioning self-hosted development environments with Terraform. - - --label=org.opencontainers.image.url=https://github.com/coder/coder - - --label=org.opencontainers.image.source=https://github.com/coder/coder - - --label=org.opencontainers.image.version={{ .Version }} - - --label=org.opencontainers.image.revision={{ .FullCommit }} - - --label=org.opencontainers.image.licenses=AGPL-3.0 - - image_templates: ["{{ if not .IsSnapshot }}ghcr.io/coder/coder:{{ .Tag }}-arm64{{ end }}"] - goarch: arm64 - dockerfile: Dockerfile - use: buildx - build_flag_templates: - - --platform=linux/arm64/v8 - - --label=org.opencontainers.image.title=coder - - --label=org.opencontainers.image.description=A tool for provisioning self-hosted development environments with Terraform. - - --label=org.opencontainers.image.url=https://github.com/coder/coder - - --label=org.opencontainers.image.source=https://github.com/coder/coder - - --label=org.opencontainers.image.version={{ .Tag }} - - --label=org.opencontainers.image.revision={{ .FullCommit }} - - --label=org.opencontainers.image.licenses=AGPL-3.0 - - image_templates: ["{{ if not .IsSnapshot }}ghcr.io/coder/coder:{{ .Tag }}-armv7{{ end }}"] - goarch: arm - goarm: "7" - dockerfile: Dockerfile - use: buildx - build_flag_templates: - - --platform=linux/arm/v7 - - --label=org.opencontainers.image.title=Coder - - --label=org.opencontainers.image.description=A tool for provisioning self-hosted development environments with Terraform. - - --label=org.opencontainers.image.url=https://github.com/coder/coder - - --label=org.opencontainers.image.source=https://github.com/coder/coder - - --label=org.opencontainers.image.version={{ .Tag }} - - --label=org.opencontainers.image.revision={{ .FullCommit }} - - --label=org.opencontainers.image.licenses=AGPL-3.0 -docker_manifests: - - name_template: ghcr.io/coder/coder:{{ .Tag }} - image_templates: - - ghcr.io/coder/coder:{{ .Tag }}-amd64 - - ghcr.io/coder/coder:{{ .Tag }}-arm64 - - ghcr.io/coder/coder:{{ .Tag }}-armv7 - -release: - ids: [coder-linux, coder-darwin, coder-windows, packages] - footer: | - ## Container Image - - `docker pull ghcr.io/coder/coder:{{ .Tag }}` - -signs: - - ids: [coder-darwin] - artifacts: archive - cmd: ./scripts/sign_macos.sh - args: ["${artifact}"] - output: true - -snapshot: - name_template: "{{ .Version }}-devel+{{ .ShortCommit }}" diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 4384381d74265..ddecd5626f957 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -8,6 +8,7 @@ "zxh404.vscode-proto3", "redhat.vscode-yaml", "streetsidesoftware.code-spell-checker", - "dbaeumer.vscode-eslint" + "dbaeumer.vscode-eslint", + "EditorConfig.EditorConfig" ] } diff --git a/Dockerfile b/Dockerfile index 8bbcbe848837f..489c7266485ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,17 @@ FROM alpine -# Generated by goreleaser on `goreleaser release` +# LABEL doesn't add any real layers so it's fine (and easier) to do it here than +# in the build script. +ARG CODER_VERSION +LABEL \ + org.opencontainers.image.title="Coder" \ + org.opencontainers.image.description="A tool for provisioning self-hosted development environments with Terraform." \ + org.opencontainers.image.url="https://github.com/coder/coder" \ + org.opencontainers.image.source="https://github.com/coder/coder" \ + org.opencontainers.image.version="$CODER_VERSION" \ + org.opencontainers.image.licenses="AGPL-3.0" + +# The coder binary is injected by scripts/build_docker.sh. ADD coder /opt/coder ENTRYPOINT [ "/opt/coder", "server" ] diff --git a/Makefile b/Makefile index 111d56a11d7c9..e7b5fbe4668fd 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,54 @@ .DEFAULT_GOAL := build +# Use a single bash shell for each job, and immediately exit on failure +SHELL := bash +.SHELLFLAGS = -ceu +.ONESHELL: + +# This doesn't work on directories. +# See https://stackoverflow.com/questions/25752543/make-delete-on-error-for-directory-targets +.DELETE_ON_ERROR: + INSTALL_DIR=$(shell go env GOPATH)/bin GOOS=$(shell go env GOOS) GOARCH=$(shell go env GOARCH) +VERSION=$(shell ./scripts/version.sh) bin: $(shell find . -not -path './vendor/*' -type f -name '*.go') go.mod go.sum $(shell find ./examples/templates) - @echo "== This builds binaries for command-line usage." + @echo "== This builds slim binaries for command-line usage." @echo "== Use \"make build\" to embed the site." - goreleaser build --snapshot --rm-dist --single-target -build: dist/artifacts.json + mkdir -p ./dist + rm -rf ./dist/coder-slim_* + ./scripts/build_go_slim.sh \ + --version "$(VERSION)" \ + --output ./dist/ \ + linux:amd64,armv7,arm64 \ + windows:amd64,arm64 \ + darwin:amd64,arm64 +.PHONY: bin + +build: site/out/index.html $(shell find . -not -path './vendor/*' -type f -name '*.go') go.mod go.sum $(shell find ./examples/templates) + rm -rf ./dist + mkdir -p ./dist + + # build slim artifacts and copy them to the site output directory + ./scripts/build_go_slim.sh \ + --version "$(VERSION)" \ + --output ./dist/ \ + linux:amd64,armv7,arm64 \ + windows:amd64,arm64 \ + darwin:amd64,arm64 + + # build not-so-slim artifacts with the default name format + ./scripts/build_go_matrix.sh \ + --version "$(VERSION)" \ + --output ./dist/ \ + --archive \ + --package-linux \ + linux:amd64,armv7,arm64 \ + windows:amd64,arm64 \ + darwin:amd64,arm64 .PHONY: build # Runs migrations to output a dump of the database. @@ -24,16 +63,14 @@ dev: ./scripts/develop.sh .PHONY: dev -dist/artifacts.json: site/out/index.html $(shell find . -not -path './vendor/*' -type f -name '*.go') go.mod go.sum $(shell find ./examples/templates) - goreleaser release --snapshot --rm-dist --skip-sign - fmt/prettier: @echo "--- prettier" + cd site # Avoid writing files in CI to reduce file write activity ifdef CI - cd site && yarn run format:check + yarn run format:check else - cd site && yarn run format:write + yarn run format:write endif .PHONY: fmt/prettier @@ -49,20 +86,34 @@ ifdef CI else shfmt -w $(shell shfmt -f .) endif +.PHONY: fmt/shfmt fmt: fmt/prettier fmt/terraform fmt/shfmt .PHONY: fmt gen: coderd/database/querier.go peerbroker/proto/peerbroker.pb.go provisionersdk/proto/provisioner.pb.go provisionerd/proto/provisionerd.pb.go site/src/api/typesGenerated.ts +.PHONY: gen + +install: site/out/index.html $(shell find . -not -path './vendor/*' -type f -name '*.go') go.mod go.sum $(shell find ./examples/templates) + @output_file="$(INSTALL_DIR)/coder" -install: build - mkdir -p $(INSTALL_DIR) - @echo "--- Copying from bin to $(INSTALL_DIR)" - cp -r ./dist/coder-$(GOOS)_$(GOOS)_$(GOARCH)*/* $(INSTALL_DIR) - @echo "-- CLI available at $(shell ls $(INSTALL_DIR)/coder*)" + @if [[ "$(GOOS)" == "windows" ]]; then + @output_file="$${output_file}.exe" + @fi + + @echo "-- Building CLI for $(GOOS) $(GOARCH) at $$output_file" + + ./scripts/build_go.sh \ + --version "$(VERSION)" \ + --output "$$output_file" \ + --os "$(GOOS)" \ + --arch "$(GOARCH)" + + @echo .PHONY: install lint: lint/shellcheck lint/go +.PHONY: lint lint/go: golangci-lint run @@ -72,6 +123,7 @@ lint/go: lint/shellcheck: $(shell shfmt -f .) @echo "--- shellcheck" shellcheck --external-sources $(shell shfmt -f .) +.PHONY: lint/shellcheck peerbroker/proto/peerbroker.pb.go: peerbroker/proto/peerbroker.proto protoc \ @@ -99,28 +151,28 @@ provisionersdk/proto/provisioner.pb.go: provisionersdk/proto/provisioner.proto site/out/index.html: $(shell find ./site -not -path './site/node_modules/*' -type f -name '*.tsx') $(shell find ./site -not -path './site/node_modules/*' -type f -name '*.ts') site/package.json ./scripts/yarn_install.sh - cd site && yarn typegen - cd site && yarn build + cd site + yarn typegen + yarn build # Restores GITKEEP files! - git checkout HEAD site/out + git checkout HEAD out site/src/api/typesGenerated.ts: scripts/apitypings/main.go $(shell find codersdk -type f -name '*.go') go run scripts/apitypings/main.go > site/src/api/typesGenerated.ts - cd site && yarn run format:types + cd site + yarn run format:types -.PHONY: test test: test-clean gotestsum -- -v -short ./... +.PHONY: test -.PHONY: test-postgres test-postgres: test-clean DB=ci gotestsum --junitfile="gotests.xml" --packages="./..." -- \ -covermode=atomic -coverprofile="gotests.coverage" -timeout=30m \ -coverpkg=./...,github.com/coder/coder/codersdk \ -count=1 -race -failfast +.PHONY: test-postgres - -.PHONY: test-postgres-docker test-postgres-docker: docker run \ --env POSTGRES_PASSWORD=postgres \ @@ -138,7 +190,8 @@ test-postgres-docker: -c fsync=off \ -c synchronous_commit=off \ -c full_page_writes=off +.PHONY: test-postgres-docker -.PHONY: test-clean test-clean: go clean -testcache +.PHONY: test-clean diff --git a/docker-compose.yaml b/docker-compose.yaml index 74c8916fd0d21..8e9a3a505a690 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,7 +1,7 @@ version: "3.9" services: coder: - image: ghcr.io/coder/coder:v${CODER_VERSION:-0.5.10}-${ARCH:-amd64} + image: ghcr.io/coder/coder:v${CODER_VERSION:-latest} ports: - "7080:7080" environment: diff --git a/scripts/archive.sh b/scripts/archive.sh new file mode 100755 index 0000000000000..9190b142a0c50 --- /dev/null +++ b/scripts/archive.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash + +# This script creates an archive containing the given binary renamed to +# `coder(.exe)?`, as well as the README.md and LICENSE files from the repo root. +# +# Usage: ./archive.sh --format tar.gz [--output path/to/output.tar.gz] [--sign-darwin] path/to/binary +# +# The --format parameter must be set, and must either be "zip" or "tar.gz". +# +# If the --output parameter is not set, the default output path is the binary +# path (minus any .exe suffix) plus the format extension ".zip" or ".tar.gz". +# +# If --sign-darwin is specified, the zip file is signed with the `codesign` +# utility and then notarized using the `gon` utility, which may take a while. +# $AC_APPLICATION_IDENTITY must be set and the signing certificate must be +# imported for this to work. Also, the input binary must already be signed with +# the `codesign` tool. +# +# The absolute output path is printed on success. + +set -euo pipefail +# shellcheck source=scripts/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +format="" +output_path="" +sign_darwin=0 + +args="$(getopt -o "" -l format:,output:,sign-darwin -- "$@")" +eval set -- "$args" +while true; do + case "$1" in + --format) + format="${2#.}" + if [[ "$format" != "zip" ]] && [[ "$format" != "tar.gz" ]]; then + error "Invalid --format parameter '$format', must be 'zip' or 'tar.gz'" + fi + shift 2 + ;; + --output) + # realpath fails if the dir doesn't exist. + mkdir -p "$(dirname "$2")" + output_path="$(realpath "$2")" + shift 2 + ;; + --sign-darwin) + if [[ "${AC_APPLICATION_IDENTITY:-}" == "" ]]; then + error "AC_APPLICATION_IDENTITY must be set when --sign-darwin is supplied" + fi + sign_darwin=1 + shift + ;; + --) + shift + break + ;; + *) + error "Unrecognized option: $1" + ;; + esac +done + +if [[ "$format" == "" ]]; then + error "--format is a required parameter" +fi + +if [[ "$#" != 1 ]]; then + error "Exactly one argument must be provided to this script, $# were supplied" +fi +if [[ ! -f "$1" ]]; then + error "File '$1' does not exist or is not a regular file" +fi +input_file="$(realpath "$1")" + +# Check dependencies +if [[ "$format" == "zip" ]]; then + dependencies zip +fi +if [[ "$format" == "tar.gz" ]]; then + dependencies tar +fi +if [[ "$sign_darwin" == 1 ]]; then + dependencies jq codesign gon +fi + +# Determine default output path. +if [[ "$output_path" == "" ]]; then + output_path="${input_file%.exe}" + output_path+=".$format" +fi + +# Determine the filename of the binary inside the archive. +output_file="coder" +if [[ "$input_file" == *".exe" ]]; then + output_file+=".exe" +fi + +# Make temporary dir where all source files intended to be in the archive will +# be symlinked from. +cdroot +temp_dir="$(mktemp -d)" +ln -s "$input_file" "$temp_dir/$output_file" +ln -s "$(realpath README.md)" "$temp_dir/" +ln -s "$(realpath LICENSE)" "$temp_dir/" + +# Ensure parent output dir and non-existent output file. +mkdir -p "$(dirname "$output_path")" +if [[ -e "$output_path" ]]; then + rm "$output_path" +fi + +cd "$temp_dir" +if [[ "$format" == "zip" ]]; then + zip "$output_path" ./* 1>&2 +else + tar --dereference -czvf "$output_path" ./* 1>&2 +fi + +cdroot +rm -rf "$temp_dir" + +if [[ "$sign_darwin" == 1 ]]; then + log "Notarizing archive..." + execrelative ./sign_darwin.sh "$output_path" +fi + +echo "$output_path" diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh new file mode 100755 index 0000000000000..64b3fef3f7703 --- /dev/null +++ b/scripts/build_docker.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash + +# This script builds a Docker image of Coder containing the given binary, for +# the given architecture. Only linux binaries are supported at this time. +# +# Usage: ./build_docker.sh --arch amd64 [--version 1.2.3] [--push] path/to/coder +# +# The --arch parameter is required and accepts a Golang arch specification. It +# will be automatically mapped to a suitable architecture that Docker accepts +# before being passed to `docker buildx build`. +# +# The image will be built and tagged against the image tag returned by +# ./image_tag.sh. +# +# If no version is specified, defaults to the version from ./version.sh. +# +# If the --push parameter is supplied, the image will be pushed. +# +# Prints the image tag on success. + +set -euo pipefail +# shellcheck source=scripts/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +arch="" +version="" +push=0 + +args="$(getopt -o "" -l arch:,version:,push -- "$@")" +eval set -- "$args" +while true; do + case "$1" in + --arch) + arch="$2" + shift 2 + ;; + --version) + version="$2" + shift 2 + ;; + --push) + push=1 + shift + ;; + --) + shift + break + ;; + *) + error "Unrecognized option: $1" + ;; + esac +done + +if [[ "$arch" == "" ]]; then + error "The --arch parameter is required" +fi + +# Check dependencies +dependencies docker + +# Remove the "v" prefix. +version="${version#v}" +if [[ "$version" == "" ]]; then + version="$(execrelative ./version.sh)" +fi + +image_tag="$(execrelative ./image_tag.sh --arch "$arch" --version="$version")" + +if [[ "$#" != 1 ]]; then + error "Exactly one argument must be provided to this script, $# were supplied" +fi +if [[ ! -f "$1" ]]; then + error "File '$1' does not exist or is not a regular file" +fi +input_file="$(realpath "$1")" + +# Remap the arch from Golang to Docker. +declare -A arch_map=( + [amd64]="linux/amd64" + [arm64]="linux/arm64" + [arm]="linux/arm/v7" + [armv7]="linux/arm/v7" +) +if [[ "${arch_map[$arch]+exists}" != "" ]]; then + arch="${arch_map[$arch]}" +fi + +# Make temporary dir where all source files intended to be in the image will be +# hardlinked from. +cdroot +temp_dir="$(TMPDIR="$(dirname "$input_file")" mktemp -d)" +ln -P "$input_file" "$temp_dir/coder" +ln -P Dockerfile "$temp_dir/" + +cd "$temp_dir" + +build_args=( + --platform "$arch" + --build-arg "CODER_VERSION=$version" + --tag "$image_tag" +) + +log "--- Building Docker image for $arch ($image_tag)" +docker buildx build "${build_args[@]}" . 1>&2 + +cdroot +rm -rf "$temp_dir" + +if [[ "$push" == 1 ]]; then + log "--- Pushing Docker image for $arch ($image_tag)" + docker push "$image_tag" +fi + +echo "$image_tag" diff --git a/scripts/build_docker_multiarch.sh b/scripts/build_docker_multiarch.sh new file mode 100755 index 0000000000000..375815d119f1a --- /dev/null +++ b/scripts/build_docker_multiarch.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash + +# This script merges Coder Docker images of different architectures together +# into the specified target image+tag, or the arch-less image tag returned by +# ./image_tag.sh. +# +# Usage: ./build_docker_multiarch.sh [--version 1.2.3] [--target image:tag] [--push] image1:tag1 image2:tag2 +# +# The supplied images must already be pushed to the registry or this will fail. +# Also, the source images cannot be in a different registry than the target +# image. +# +# If no version is specified, defaults to the version from ./version.sh. +# +# If no target tag is supplied, the arch-less image tag returned by +# ./image_tag.sh will be used. +# +# If the --push parameter is supplied, all supplied tags will be pushed. +# +# Returns the merged image tag. + +set -euo pipefail +# shellcheck source=scripts/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +version="" +target="" +push=0 + +args="$(getopt -o "" -l version:,target:,push -- "$@")" +eval set -- "$args" +while true; do + case "$1" in + --version) + version="$2" + shift 2 + ;; + --target) + target="$2" + shift 2 + ;; + --push) + push=1 + shift + ;; + --) + shift + break + ;; + *) + error "Unrecognized option: $1" + ;; + esac +done + +if [[ "$#" == 0 ]]; then + error "At least one argument must be provided to this script, $# were supplied" +fi + +# Check dependencies +dependencies docker + +# Remove the "v" prefix. +version="${version#v}" +if [[ "$version" == "" ]]; then + version="$(execrelative ./version.sh)" +fi + +if [[ "$target" == "" ]]; then + target="$(execrelative ./image_tag.sh --version "$version")" +fi + +create_args=() +for image_tag in "$@"; do + create_args+=(--amend "$image_tag") +done + +# Sadly, multi-arch images don't seem to support labels. +log "--- Creating multi-arch Docker image ($target)" +docker manifest create \ + "$target" \ + "${create_args[@]}" + +if [[ "$push" == 1 ]]; then + log "--- Pushing multi-arch Docker image ($target)" + docker manifest push "$target" +fi + +echo "$target" diff --git a/scripts/build_go.sh b/scripts/build_go.sh new file mode 100755 index 0000000000000..ca29cd04e2beb --- /dev/null +++ b/scripts/build_go.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash + +# This script builds a single Go binary of Coder with the given parameters. +# +# Usage: ./build_go.sh [--version 1.2.3-devel+abcdef] [--os linux] [--arch amd64] [--output path/to/output] [--slim] +# +# Defaults to linux:amd64 with slim disabled, but can be controlled with GOOS, +# GOARCH and CODER_SLIM_BUILD=1. If no version is specified, defaults to the +# version from ./version.sh. +# +# GOARM can be controlled by suffixing any arm architecture (i.e. arm or arm64) +# with "vX" (e.g. "v7", "v8"). +# +# Unless overridden via --output, the built binary will be dropped in +# "$repo_root/dist/coder_$version_$os_$arch" (with a ".exe" suffix for windows +# builds) and the absolute path to the binary will be printed to stdout on +# completion. +# +# If the --sign-darwin parameter is specified and the OS is darwin, binaries +# will be signed using the `codesign` utility. $AC_APPLICATION_IDENTITY must be +# set and the signing certificate must be imported for this to work. + +set -euo pipefail +# shellcheck source=scripts/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +cdroot + +version="" +os="${GOOS:-linux}" +arch="${GOARCH:-amd64}" +slim="${CODER_SLIM_BUILD:-0}" +sign_darwin=0 +output_path="" + +args="$(getopt -o "" -l version:,os:,arch:,output:,slim,sign-darwin -- "$@")" +eval set -- "$args" +while true; do + case "$1" in + --version) + version="$2" + shift 2 + ;; + --os) + os="$2" + shift 2 + ;; + --arch) + arch="$2" + shift 2 + ;; + --output) + output_path="$(realpath "$2")" + shift 2 + ;; + --slim) + slim=1 + shift + ;; + --sign-darwin) + if [[ "${AC_APPLICATION_IDENTITY:-}" == "" ]]; then + error "AC_APPLICATION_IDENTITY must be set when --sign-darwin is supplied" + fi + sign_darwin=1 + shift + ;; + --) + shift + break + ;; + *) + error "Unrecognized option: $1" + ;; + esac +done + +# Remove the "v" prefix. +version="${version#v}" +if [[ "$version" == "" ]]; then + version="$(execrelative ./version.sh)" +fi + +# Check dependencies +dependencies go +if [[ "$sign_darwin" == 1 ]]; then + dependencies codesign +fi + +build_args=( + -ldflags "-s -w -X 'github.com/coder/coder/buildinfo.tag=$version'" +) +if [[ "$slim" == 0 ]]; then + build_args+=(-tags embed) +fi + +# Compute default output path. +if [[ "$output_path" == "" ]]; then + dist_dir="dist" + mkdir -p "$dist_dir" + output_path="${dist_dir}/coder_${version}_${os}_${arch}" + if [[ "$os" == "windows" ]]; then + output_path+=".exe" + fi + output_path="$(realpath "$output_path")" +fi +build_args+=(-o "$output_path") + +# Determine GOARM. +arm_version="" +if [[ "$arch" == "arm" ]]; then + arm_version="7" +elif [[ "$arch" == "armv"* ]] || [[ "$arch" == "arm64v"* ]]; then + arm_version="${arch//*v/}" + + # Remove the v* suffix. + arch="${arch//v*/}" +fi + +CGO_ENABLED=0 GOOS="$os" GOARCH="$arch" GOARM="$arm_version" go build \ + "${build_args[@]}" \ + ./cmd/coder 1>&2 + +if [[ "$sign_darwin" == 1 ]] && [[ "$os" == "darwin" ]]; then + codesign -s "$AC_APPLICATION_IDENTITY" -f -v --timestamp --options runtime "$output_path" +fi + +echo "$output_path" diff --git a/scripts/build_go_matrix.sh b/scripts/build_go_matrix.sh new file mode 100755 index 0000000000000..3082393d523e2 --- /dev/null +++ b/scripts/build_go_matrix.sh @@ -0,0 +1,229 @@ +#!/usr/bin/env bash + +# This script builds multiple Go binaries for Coder with the given OS and +# architecture combinations. +# +# Usage: ./build_go_matrix.sh [--version 1.2.3-devel+abcdef] [--output dist/] [--slim] [--sign-darwin] [--archive] [--package-linux] os1:arch1,arch2 os2:arch1 os1:arch3 +# +# If no OS:arch combinations are provided, nothing will happen and no error will +# be returned. Slim builds are disabled by default. If no version is specified, +# defaults to the version from ./version.sh +# +# The --output parameter must be a directory with a trailing slash where all +# files will be dropped with the default name scheme +# `coder_$version_$os_$arch(.exe)?`, or must contain the `{os}` and `{arch}` +# template variables. You may also use `{version}`. Note that for windows builds +# the `.exe` suffix will be appended automatically. +# +# Unless overridden via --output, the built binary will be dropped in +# "$repo_root/dist/coder_$version_$os_$arch" (with a ".exe" suffix for windows +# builds). +# +# If the --sign-darwin parameter is specified, all darwin binaries will be +# signed using the `codesign` utility. $AC_APPLICATION_IDENTITY must be set and +# the signing certificate must be imported for this to work. +# +# If the --archive parameter is specified, all binaries will be archived using +# ./archive.sh. The --sign-darwin parameter will be carried through, and all +# archive files will be dropped in the output directory with the same name as +# the binary and the .zip (for windows and darwin) or .tar.gz extension. +# +# If the --package-linux parameter is specified, all linux binaries will be +# packaged using ./package.sh. Requires the nfpm binary. + +set -euo pipefail +# shellcheck source=scripts/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +version="" +output_path="" +slim=0 +sign_darwin=0 +archive=0 +package_linux=0 + +args="$(getopt -o "" -l version:,output:,slim,sign-darwin,archive,package-linux -- "$@")" +eval set -- "$args" +while true; do + case "$1" in + --version) + version="$2" + shift 2 + ;; + --output) + output_path="$2" + shift 2 + ;; + --slim) + slim=1 + shift + ;; + --sign-darwin) + if [[ "${AC_APPLICATION_IDENTITY:-}" == "" ]]; then + error "AC_APPLICATION_IDENTITY must be set when --sign-darwin is supplied" + fi + sign_darwin=1 + shift + ;; + --archive) + archive=1 + shift + ;; + --package-linux) + package_linux=1 + shift + ;; + --) + shift + break + ;; + *) + error "Unrecognized option: $1" + ;; + esac +done + +# Verify the output path template. +if [[ "$output_path" == "" ]]; then + # Input paths are relative, so we don't cdroot at the top, but for this case + # we want it to be relative to the root. + cdroot + mkdir -p dist + output_path="$(realpath "dist/coder_{version}_{os}_{arch}")" +elif [[ "$output_path" == */ ]]; then + output_path="${output_path}coder_{version}_{os}_{arch}" +elif [[ "$output_path" != *"{os}"* ]] || [[ "$output_path" != *"{arch}"* ]]; then + # If the output path isn't a directory (ends with /) then it must have + # template variables. + error "Templated output path '$output_path' must contain {os} and {arch}" +fi + +mkdir -p "$(dirname "$output_path")" +output_path="$(realpath "$output_path")" + +# Remove the "v" prefix. +version="${version#v}" +if [[ "$version" == "" ]]; then + version="$(execrelative ./version.sh)" +fi + +# Parse the os:arch specs into an array. +specs=() +may_zip=0 +may_tar=0 +for spec in "$@"; do + spec_os="$(echo "$spec" | cut -d ":" -f 1)" + if [[ "$spec_os" == "" ]] || [[ "$spec_os" == *" "* ]]; then + error "Could not parse matrix build spec '$spec': invalid OS '$spec_os'" + fi + + # Determine which dependencies we need. + if [[ "$spec_os" == "windows" ]] || [[ "$spec_os" == "darwin" ]]; then + may_zip=1 + else + may_tar=1 + fi + + # No quoting is important here. + for spec_arch in $(echo "$spec" | cut -d ":" -f 2 | tr "," "\n"); do + if [[ "$spec_arch" == "" ]] || [[ "$spec_os" == *" "* ]]; then + error "Could not parse matrix build spec '$spec': invalid architecture '$spec_arch'" + fi + + specs+=("$spec_os:$spec_arch") + done +done + +# Remove duplicate specs while maintaining the same order. +specs_str="${specs[*]}" +specs=() +for s in $(echo "$specs_str" | tr " " "\n" | awk '!a[$0]++'); do + specs+=("$s") +done + +# Check dependencies +dependencies go +if [[ "$sign_darwin" == 1 ]]; then + dependencies jq codesign gon +fi +if [[ "$archive" == 1 ]]; then + if [[ "$may_zip" == 1 ]]; then + dependencies zip + fi + if [[ "$may_tar" == 1 ]]; then + dependencies tar + fi +fi +if [[ "$package_linux" == 1 ]]; then + dependencies nfpm +fi + +bin_name="coder" +build_args=() +if [[ "$slim" == 1 ]]; then + bin_name+="-slim" + build_args+=(--slim) +fi +if [[ "$sign_darwin" == 1 ]]; then + build_args+=(--sign-darwin) +fi + +# Build each spec. +for spec in "${specs[@]}"; do + spec_os="$(echo "$spec" | cut -d ":" -f 1)" + spec_arch="$(echo "$spec" | cut -d ":" -f 2)" + + # Craft output path from the template. + spec_output="$output_path" + spec_output="${spec_output//\{os\}/"$spec_os"}" + spec_output="${spec_output//\{arch\}/"$spec_arch"}" + spec_output="${spec_output//\{version\}/"$version"}" + + spec_output_binary="$spec_output" + if [[ "$spec_os" == "windows" ]]; then + spec_output_binary+=".exe" + fi + + # Ensure parent dir. + mkdir -p "$(dirname "$spec_output")" + + log "--- Building $bin_name for $spec_os $spec_arch ($spec_output_binary)" + execrelative ./build_go.sh \ + --version "$version" \ + --os "$spec_os" \ + --arch "$spec_arch" \ + --output "$spec_output_binary" \ + "${build_args[@]}" + log + log + + if [[ "$archive" == 1 ]]; then + spec_archive_format="tar.gz" + if [[ "$spec_os" == "windows" ]] || [[ "$spec_os" == "darwin" ]]; then + spec_archive_format="zip" + fi + spec_output_archive="$spec_output.$spec_archive_format" + + archive_args=() + if [[ "$sign_darwin" == 1 ]] && [[ "$spec_os" == "darwin" ]]; then + archive_args+=(--sign-darwin) + fi + + log "--- Creating archive for $spec_os $spec_arch ($spec_output_archive)" + execrelative ./archive.sh \ + --format "$spec_archive_format" \ + --output "$spec_output_archive" \ + "${archive_args[@]}" \ + "$spec_output_binary" + log + log + fi + + if [[ "$package_linux" == 1 ]] && [[ "$spec_os" == "linux" ]]; then + execrelative ./package.sh \ + --arch "$spec_arch" \ + --version "$version" \ + "$spec_output_binary" + log + fi +done diff --git a/scripts/build_go_slim.sh b/scripts/build_go_slim.sh new file mode 100755 index 0000000000000..6b4e041402beb --- /dev/null +++ b/scripts/build_go_slim.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash + +# This script builds multiple "slim" Go binaries for Coder with the given OS and +# architecture combinations. This wraps ./build_go_matrix.sh. +# +# Usage: ./build_go_slim.sh [--version 1.2.3-devel+abcdef] [--output dist/] os1:arch1,arch2 os2:arch1 os1:arch3 +# +# If no OS:arch combinations are provided, nothing will happen and no error will +# be returned. If no version is specified, defaults to the version from +# ./version.sh +# +# The --output parameter differs from ./build_go_matrix.sh, in that it does not +# accept variables such as `{os}` and `{arch}` and only accepts a directory +# ending with `/`. +# +# The built binaries are additionally copied to the site output directory so +# they can be packaged into non-slim binaries correctly. + +set -euo pipefail +shopt -s nullglob +# shellcheck source=scripts/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +version="" +output_path="" + +args="$(getopt -o "" -l version:,output: -- "$@")" +eval set -- "$args" +while true; do + case "$1" in + --version) + version="$2" + shift 2 + ;; + --output) + output_path="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + error "Unrecognized option: $1" + ;; + esac +done + +# Check dependencies +dependencies go + +# Remove the "v" prefix. +version="${version#v}" +if [[ "$version" == "" ]]; then + version="$(execrelative ./version.sh)" +fi + +# Verify the output path. +if [[ "$output_path" == "" ]]; then + # Input paths are relative, so we don't cdroot at the top, but for this case + # we want it to be relative to the root. + cdroot + mkdir -p dist + output_path="$(realpath "dist/coder-slim_{version}_{os}_{arch}")" +elif [[ "$output_path" != */ ]] || [[ "$output_path" == *"{"* ]]; then + error "The output path '$output_path' cannot contain variables and must end with a slash" +else + mkdir -p "$output_path" + output_path="$(realpath "${output_path}coder-slim_{version}_{os}_{arch}")" +fi + +./scripts/build_go_matrix.sh \ + --version "$version" \ + --output "$output_path" \ + --slim \ + "$@" + +cdroot +dest_dir="./site/out/bin" +mkdir -p "$dest_dir" +dest_dir="$(realpath "$dest_dir")" + +# Copy the binaries to the site directory. +cd "$(dirname "$output_path")" +for f in ./coder-slim_*; do + f="${f#./}" + dest="$dest_dir/${f//-slim_$version/}" + cp "$f" "$dest" +done diff --git a/scripts/image_tag.sh b/scripts/image_tag.sh new file mode 100755 index 0000000000000..8b405c48e304f --- /dev/null +++ b/scripts/image_tag.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +# This script prints the image tag to use for the given arch and version +# combination. +# +# Usage: ./image_tag.sh [--arch amd64] [--version 1.2.3] +# +# The --arch parameter accepts a Golang arch specification. If not specified, +# the image tag for the multi-arch image will be returned instead. +# +# If no version is specified, defaults to the version from ./version.sh. If the +# supplied version is "latest", no `v` prefix will be added to the tag. +# +# The returned tag will be sanitized to remove invalid characters like the plus +# sign. + +set -euo pipefail +# shellcheck source=scripts/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +arch="" +version="" + +args="$(getopt -o "" -l arch:,version: -- "$@")" +eval set -- "$args" +while true; do + case "$1" in + --arch) + arch="$2" + shift 2 + ;; + --version) + version="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + error "Unrecognized option: $1" + ;; + esac +done + +# Remove the "v" prefix because we don't want to add it twice. +version="${version#v}" +if [[ "$version" == "" ]]; then + version="$(execrelative ./version.sh)" +fi + +image="${CODER_IMAGE_BASE:-ghcr.io/coder/coder}" +tag="v$version" +if [[ "$version" == "latest" ]]; then + tag="latest" +fi +if [[ "$arch" != "" ]]; then + tag+="-$arch" +fi + +# Dev versions contain plus signs which are illegal characters in Docker tags. +tag="${tag//+/-}" +echo "$image:$tag" diff --git a/scripts/lib.sh b/scripts/lib.sh new file mode 100644 index 0000000000000..98876155b2477 --- /dev/null +++ b/scripts/lib.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash + +# This script is meant to be sourced by other scripts. To source this script: +# # shellcheck source=scripts/lib.sh +# source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +set -euo pipefail + +# realpath returns an absolute path to the given relative path. It will fail if +# the parent directory of the path does not exist. Make sure you are in the +# expected directory before running this to avoid errors. +# +# GNU realpath relies on coreutils, which are not installed or the default on +# Macs out of the box, so we have this mostly working bash alternative instead. +# +# Taken from https://stackoverflow.com/a/3915420 (CC-BY-SA 4.0) +realpath() { + local dir + local base + dir="$(dirname "$1")" + base="$(basename "$1")" + + if [[ ! -d "$dir" ]]; then + error "Could not change directory to '$dir': directory does not exist" + fi + echo "$( + cd "$dir" || error "Could not change directory to '$dir'" + pwd -P + )"/"$base" +} + +# We have to define realpath before these otherwise it fails on Mac's bash. +SCRIPT_DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" +PROJECT_ROOT="$(cd "$SCRIPT_DIR" && realpath "$(git rev-parse --show-toplevel)")" + +# pushd is a silent alternative to the real pushd shell command. +pushd() { + command pushd "$@" >/dev/null +} + +# popd is a silent alternative to the real popd shell command. +# shellcheck disable=SC2120 +popd() { + command popd "$@" >/dev/null +} + +# cdself changes directory to the directory of the current script. This should +# not be used in scripts that may be sourced by other scripts. +cdself() { + cd "$SCRIPT_DIR" || error "Could not change directory to '$SCRIPT_DIR'" +} + +# cdroot changes directory to the root of the repository. +cdroot() { + cd "$PROJECT_ROOT" || error "Could not change directory to '$PROJECT_ROOT'" +} + +# execrelative can be used to execute scripts as if you were in the parent +# directory of the current script. This should not be used in scripts that may +# be sourced by other scripts. +execrelative() { + pushd "$SCRIPT_DIR" || error "Could not change directory to '$SCRIPT_DIR'" + local rc=0 + "$@" || rc=$? + popd + return $rc +} + +dependencies() { + local fail=0 + for dep in "$@"; do + if ! command -v "$dep" >/dev/null; then + log "ERROR: The '$dep' dependency is required, but is not available." + fail=1 + fi + done + + if [[ "$fail" == 1 ]]; then + log + error "One or more dependencies are not available, check above log output for more details." + fi +} + +# maybedryrun prints the given program and flags, and then, if the first +# argument is 0, executes it. The reason the first argument should be 0 is that +# it is expected that you have a dry_run variable in your script that is set to +# 0 by default (i.e. do not dry run) and set to 1 if the --dry-run flag is +# specified. +# +# Usage: maybedryrun 1 gh release create ... +# Usage: maybedryrun 0 docker push ghcr.io/coder/coder:latest +maybedryrun() { + if [[ "$1" == 1 ]]; then + shift + log "DRYRUN: $*" + else + shift + log $ "$@" + "$@" + fi +} + +# log prints a message to stderr. +log() { + echo "$*" 1>&2 +} + +# error prints an error message and returns an error exit code. +error() { + log "ERROR: $*" + exit 1 +} + +# isdarwin returns an error if the current platform is not darwin. +isdarwin() { + [[ "${OSTYPE:-darwin}" == *darwin* ]] +} + +libsh_bad_dependencies=0 + +if ((BASH_VERSINFO[0] < 4)); then + libsh_bad_dependencies=1 + log "ERROR: You need at least bash 4.0 to run the scripts in the Coder repo." + if isdarwin; then + log "On darwin:" + log "- brew install bash" + log "- Restart your terminal" + fi + log +fi + +# BSD getopt (which is installed by default on Macs) is not supported. +if [[ "$(getopt --version)" == *--* ]]; then + libsh_bad_dependencies=1 + log "ERROR: You need GNU getopt to run the scripts in the Coder repo." + if isdarwin; then + log "On darwin:" + log "- brew install gnu-getopt" + # shellcheck disable=SC2016 + log '- Add "$(brew --prefix)/opt/gnu-getopt/bin" to your PATH' + log "- Restart your terminal" + fi + log +fi + +# The bash scripts don't call Make directly, but we want to make (ha ha) sure +# that make supports the features the repo uses. Notably, Macs have an old +# version of Make installed out of the box that doesn't support new features +# like ONESHELL. +make_version="$(make --version 2>/dev/null | head -n1 | grep -oE '([[:digit:]]+\.){1,2}[[:digit:]]+')" +if [ "${make_version//.*/}" -lt 4 ]; then + libsh_bad_dependencies=1 + log "ERROR: You need at least make 4.0 to run the scripts in the Coder repo." + if isdarwin; then + log "On darwin:" + log "- brew install make" + # shellcheck disable=SC2016 + log '- Add "$(brew --prefix)/opt/make/libexec/gnubin" to your PATH (you should Google this first)' + log "- Restart your terminal" + fi + log +fi + +if [[ "$libsh_bad_dependencies" == 1 ]]; then + error "Invalid dependencies, see above for more details." +fi diff --git a/scripts/nfpm.yaml b/scripts/nfpm.yaml new file mode 100644 index 0000000000000..dc11e0aa3ca6c --- /dev/null +++ b/scripts/nfpm.yaml @@ -0,0 +1,28 @@ +name: coder +platform: linux +arch: "${GOARCH}" +version: "${CODER_VERSION}" +version_schema: semver +release: 1 + +vendor: Coder +homepage: https://coder.com +maintainer: Coder +description: | + Provision development environments with infrastructure with code +license: AGPL-3.0 + +suggests: + - postgresql + +scripts: + preinstall: preinstall.sh + +contents: + - src: coder + dst: /usr/bin/coder + - src: coder.env + dst: /etc/coder.d/coder.env + type: "config|noreplace" + - src: coder.service + dst: /usr/lib/systemd/system/coder.service diff --git a/scripts/package.sh b/scripts/package.sh new file mode 100755 index 0000000000000..ee1f3b3fe2f3f --- /dev/null +++ b/scripts/package.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash + +# This script creates Linux packages for the given binary. It will output a +# .rpm, .deb and .apk file in the same directory as the input file with the same +# filename (except the package format suffix). +# +# ./package.sh --arch amd64 [--version 1.2.3] path/to/coder +# +# The --arch parameter is required. If no version is specified, defaults to the +# version from ./version.sh. + +set -euo pipefail +# shellcheck source=scripts/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +version="" +arch="" + +args="$(getopt -o "" -l arch:,version: -- "$@")" +eval set -- "$args" +while true; do + case "$1" in + --arch) + arch="$2" + shift 2 + ;; + --version) + version="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + error "Unrecognized option: $1" + ;; + esac +done + +if [[ "$arch" == "" ]]; then + error "--arch is a required parameter" +fi + +if [[ "$#" != 1 ]]; then + error "Exactly one argument must be provided to this script, $# were supplied" +fi +if [[ ! -f "$1" ]]; then + error "File '$1' does not exist or is not a regular file" +fi +input_file="$(realpath "$1")" + +# Check dependencies +dependencies nfpm + +# Remove the "v" prefix. +version="${version#v}" +if [[ "$version" == "" ]]; then + version="$(execrelative ./version.sh)" +fi + +# Make temporary dir where all source files intended to be in the package will +# be hardlinked from. +cdroot +temp_dir="$(TMPDIR="$(dirname "$input_file")" mktemp -d)" +ln -P "$input_file" "$temp_dir/coder" +ln -P "$(realpath coder.env)" "$temp_dir/" +ln -P "$(realpath coder.service)" "$temp_dir/" +ln -P "$(realpath preinstall.sh)" "$temp_dir/" +ln -P "$(realpath scripts/nfpm.yaml)" "$temp_dir/" + +cd "$temp_dir" + +formats=(apk deb rpm) +for format in "${formats[@]}"; do + output_path="$input_file.$format" + log "--- Building $format package ($output_path)" + nfpm package \ + -f nfpm.yaml \ + -p "$format" \ + -t "$output_path" +done + +cdroot +rm -rf "$temp_dir" diff --git a/scripts/publish_release.sh b/scripts/publish_release.sh new file mode 100755 index 0000000000000..338ffbb99ff7b --- /dev/null +++ b/scripts/publish_release.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash + +# This script generates release notes and publishes all of the given assets to +# GitHub releases. Depends on GitHub CLI. +# +# Usage: ./publish_release.sh [--version 1.2.3] [--dry-run] path/to/asset1 path/to/asset2 ... +# +# The supplied images must already be pushed to the registry or this will fail. +# Also, the source images cannot be in a different registry than the target +# image generated by ./image_tag.sh. +# The supplied assets will be uploaded to the GitHub release as-is, as well as a +# file containing checksums. +# +# If no version is specified, defaults to the version from ./version.sh. The +# script will exit early if the branch is not tagged with the provided version +# (plus the "v" prefix) unless run with --dry-run. +# +# If the --dry-run parameter is supplied, the release will not be published to +# GitHub at all. +# +# Returns the link to the created GitHub release (unless --dry-run was +# specified). + +set -euo pipefail +# shellcheck source=scripts/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +version="" +dry_run=0 + +args="$(getopt -o "" -l version:,dry-run -- "$@")" +eval set -- "$args" +while true; do + case "$1" in + --version) + version="$2" + shift 2 + ;; + --dry-run) + dry_run=1 + shift + ;; + --) + shift + break + ;; + *) + error "Unrecognized option: $1" + ;; + esac +done + +# Check dependencies +dependencies gh + +# Remove the "v" prefix. +version="${version#v}" +if [[ "$version" == "" ]]; then + version="$(execrelative ./version.sh)" +fi + +# realpath-ify all input files so we can cdroot below. +files=() +for f in "$@"; do + if [[ ! -e "$f" ]]; then + error "File not found: $f" + fi + files+=("$(realpath "$f")") +done +if [[ "${#files[@]}" == 0 ]]; then + error "No files supplied" +fi + +if [[ "$dry_run" == 0 ]] && [[ "$version" == *dev* ]]; then + error "Cannot publish a dev version to GitHub" +fi + +# The git commands need to be executed from within the repository. +cdroot + +# Verify that we're currently checked out on the supplied tag. +new_tag="v$version" +if [[ "$(git describe --always)" != "$new_tag" ]]; then + if [[ "$dry_run" == 0 ]]; then + error "The provided version '$new_tag' does not match the current git describe output '$(git describe --always)'" + fi + + log "The provided version does not match the current git tag, but --dry-run was supplied so continuing..." +fi + +# This returns the tag before the current tag. +old_tag="$(git describe --abbrev=0 HEAD^1)" + +# For dry-run builds we want to use the SHA instead of the tag, because the new +# tag probably doesn't exist. +changelog_range="$old_tag..$new_tag" +if [[ "$dry_run" == 1 ]]; then + changelog_range="$old_tag..$(git rev-parse --short HEAD)" +fi + +# Craft the release notes. +changelog="$(git log --no-merges --pretty=format:"- %h %s" "$changelog_range")" +image_tag="$(execrelative ./image_tag.sh --version "$version")" +release_notes=" +## Changelog + +$changelog + +## Container Image +- \`docker pull $image_tag\` + +" + +release_notes_file="$(mktemp)" +echo "$release_notes" >"$release_notes_file" + +# Create temporary release folder so we can generate checksums. Both the +# sha256sum and gh binaries support symlinks as input files so this works well. +temp_dir="$(mktemp -d)" +for f in "${files[@]}"; do + ln -s "$f" "$temp_dir/" +done + +# Generate checksums file which will be uploaded to the GitHub release. +pushd "$temp_dir" +sha256sum ./* | sed -e 's/\.\///' - >"coder_${version}_checksums.txt" +popd + +log "--- Creating release $new_tag" +log +log "Description:" +echo "$release_notes" | sed -e 's/^/\t/' - 1>&2 +log +log "Contents:" +pushd "$temp_dir" +find ./* 2>&1 | sed -e 's/^/\t/;s/\.\///' - 1>&2 +popd +log +log + +# We pipe `true` into `gh` so that it never tries to be interactive. +true | + maybedryrun "$dry_run" gh release create \ + --title "$new_tag" \ + --notes-file "$release_notes_file" \ + "$new_tag" \ + "$temp_dir"/* + +rm -rf "$temp_dir" +rm -rf "$release_notes_file" diff --git a/scripts/sign_darwin.sh b/scripts/sign_darwin.sh new file mode 100755 index 0000000000000..f1bfef735767b --- /dev/null +++ b/scripts/sign_darwin.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +# This script notarizes the provided zip file. +# +# Usage: ./publish_release.sh [--version 1.2.3] [--dry-run] path/to/asset1 path/to/asset2 ... +# +# The provided zip file must contain a coder binary that has already been signed +# using the codesign tool. +# +# On success, the input file will be successfully signed and notarized. +# +# Depends on codesign and gon utilities. Requires the $AC_APPLICATION_IDENTITY +# environment variable to be set. + +set -euo pipefail +# shellcheck source=scripts/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +if [[ "${AC_APPLICATION_IDENTITY:-}" == "" ]]; then + error "AC_APPLICATION_IDENTITY must be set for ./sign_darwin.sh" +fi + +# Check dependencies +dependencies jq codesign gon + +output_path="$1" + +# Create the gon config. +config="$(mktemp -d)/gon.json" +jq -r --null-input --arg path "$output_path" '{ + "notarize": [ + { + "path": $path, + "bundle_id": "com.coder.cli" + } + ] +}' >"$config" + +# Sign the zip file with our certificate. +codesign -s "$AC_APPLICATION_IDENTITY" -f -v --timestamp --options runtime "$output_path" + +# Notarize the signed zip file. +# +# The notarization process is very fragile and heavily dependent on Apple's +# notarization server not returning server errors, so we retry this step twice +# with a delay of a minute between attempts. +rc=0 +for i in $(seq 1 2); do + gon "$config" && rc=0 && break || rc=$? + log "gon exit code: $rc" + if [ "$i" -lt 5 ]; then + log + log "Retrying notarization in 60 seconds" + log + sleep 60 + else + log + log "Giving up :(" + fi +done + +exit $rc diff --git a/scripts/sign_macos.sh b/scripts/sign_macos.sh deleted file mode 100755 index 26d3fd181bbcb..0000000000000 --- a/scripts/sign_macos.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") -PROJECT_ROOT=$(cd "$SCRIPT_DIR" && git rev-parse --show-toplevel) - -( - cd "${PROJECT_ROOT}" - - codesign -s "$AC_APPLICATION_IDENTITY" -f -v --timestamp --options runtime "$1" - - config=$(mktemp -d)/gon.json - jq -r --null-input --arg path "$(pwd)/$1" '{ - "notarize": [ - { - "path": $path, - "bundle_id": "com.coder.cli" - } - ] - }' >"$config" - gon "$config" -) diff --git a/scripts/version.sh b/scripts/version.sh new file mode 100755 index 0000000000000..d924117fe9640 --- /dev/null +++ b/scripts/version.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +# This script generates the version string used by Coder, including for dev +# versions. Note: the version returned by this script will NOT include the "v" +# prefix that is included in the Git tag. +# +# If $CODER_RELEASE is set to "true", the returned version will equal the +# current git tag. If the current commit is not tagged, this will fail. +# +# If $CODER_RELEASE is not set, the returned version will always be a dev +# version. + +set -euo pipefail +# shellcheck source=scripts/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +cdroot + +last_tag="$(git describe --tags --abbrev=0)" +version="$last_tag" + +# If the HEAD has extra commits since the last tag then we are in a dev version. +# +# Dev versions are denoted by the "-devel+" suffix with a trailing commit short +# SHA. +if [[ "${CODER_RELEASE:-}" == *t* ]]; then + # $last_tag will equal `git describe --always` if we currently have the tag + # checked out. + if [[ "$last_tag" != "$(git describe --always)" ]]; then + # make won't exit on $(shell cmd) failures, so we have to kill it :( + if [[ "$(ps -o comm= "$PPID" || true)" == *make* ]]; then + log "ERROR: version.sh attemped to generate a dev version string when CODER_RELEASE was set" + kill "$PPID" || true + exit 1 + fi + + error "version.sh attemped to generate a dev version string when CODER_RELEASE was set" + fi +else + version+="-devel+$(git rev-parse --short HEAD)" +fi + +# Remove the "v" prefix. +echo "${version#v}"