From cfe0ddccfd7dab759f628101dcd9e07d95cb6342 Mon Sep 17 00:00:00 2001 From: Muhammad Atif Ali Date: Wed, 10 Jan 2024 12:15:12 +0300 Subject: [PATCH 01/26] wip builds a multi-arch image and closes #11127 --- .github/workflows/ci.yaml | 47 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a5f2f60d9b88d..b7494f862a607 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -655,7 +655,7 @@ jobs: # to main branch. We are only building this for amd64 platform. (>95% pulls # are for amd64) needs: changes - if: github.ref == 'refs/heads/main' && needs.changes.outputs.docs-only == 'false' + if: needs.changes.outputs.docs-only == 'false' runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }} env: DOCKER_CLI_EXPERIMENTAL: "enabled" @@ -692,12 +692,57 @@ jobs: go mod download version="$(./scripts/version.sh)" + tag="main-$(echo "$version" | sed 's/+/-/g')" + echo "tag=$tag" >> $GITHUB_OUTPUT + make gen/mark-fresh make -j \ build/coder_linux_amd64 \ build/coder_"$version"_windows_amd64.zip \ build/coder_"$version"_linux_amd64.{tar.gz,deb} + - name: Install depot.dev CLI + uses: depot/setup-action@v1 + + - name: Create an empty base-build-context directory + run: mkdir base-build-context + + - name: Determine base image tag + id: image-base-tag + run: | + set -euo pipefail + version="$(./scripts/version.sh)" + echo "base_tag=$(CODER_IMAGE_BASE=coder-base ./scripts/image_tag.sh --version "$version")" >> $GITHUB_OUTPUT + + # This uses OIDC authentication, so no auth variables are required. + - name: Build base Docker image via depot.dev + uses: depot/build-push-action@v1 + with: + project: wl5hnrrkns + context: base-build-context + file: scripts/Dockerfile.base + platforms: linux/amd64,linux/arm64,linux/arm/v7 + tags: ${{ steps.image-base-tag.outputs.base_tag }} + + - name: Build Linux Docker images + run: | + set -euxo pipefail + + # build Docker images for each architecture + version="$(./scripts/version.sh)" + make -j build/coder_"$version"_linux_{amd64,arm64,armv7}.tag + + # build and push multi-arch manifest, this depends on the other images + # being pushed so will automatically push them. + + make -j push/build/coder_"$version"_linux.tag + ./scripts/build_docker_multiarch.sh \ + --push \ + --target "$(./scripts/image_tag.sh --version latest)" \ + $(cat build/coder_"$version"_linux_{amd64,arm64,armv7}.tag) + env: + CODER_BASE_IMAGE_TAG: ${{ steps.image-base-tag.outputs.base_tag }} + - name: Build and Push Linux amd64 Docker Image id: build-docker run: | From b13efd7b5a5dbce625ed00ceb3a1b991730dbb1e Mon Sep 17 00:00:00 2001 From: Muhammad Atif Ali Date: Wed, 10 Jan 2024 13:03:43 +0000 Subject: [PATCH 02/26] `fmt` --- .github/workflows/ci.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b7494f862a607..765ad05021831 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -713,7 +713,7 @@ jobs: set -euo pipefail version="$(./scripts/version.sh)" echo "base_tag=$(CODER_IMAGE_BASE=coder-base ./scripts/image_tag.sh --version "$version")" >> $GITHUB_OUTPUT - + # This uses OIDC authentication, so no auth variables are required. - name: Build base Docker image via depot.dev uses: depot/build-push-action@v1 @@ -734,7 +734,7 @@ jobs: # build and push multi-arch manifest, this depends on the other images # being pushed so will automatically push them. - + make -j push/build/coder_"$version"_linux.tag ./scripts/build_docker_multiarch.sh \ --push \ @@ -742,7 +742,7 @@ jobs: $(cat build/coder_"$version"_linux_{amd64,arm64,armv7}.tag) env: CODER_BASE_IMAGE_TAG: ${{ steps.image-base-tag.outputs.base_tag }} - + - name: Build and Push Linux amd64 Docker Image id: build-docker run: | From 131a227103cb89935e258b177d6f7248aeb2bcb1 Mon Sep 17 00:00:00 2001 From: Muhammad Atif Ali Date: Wed, 10 Jan 2024 13:04:58 +0000 Subject: [PATCH 03/26] wip --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 765ad05021831..eee31a94c597d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,6 +19,7 @@ permissions: repository-projects: none security-events: none statuses: none + id-token: write # This is required for the build to authenticate to depot.dev # Cancel in-progress runs for pull requests when developers push # additional changes From 206a11ce407102f6322cb4f4672cf62cffc70b4b Mon Sep 17 00:00:00 2001 From: Muhammad Atif Ali Date: Wed, 10 Jan 2024 13:08:32 +0000 Subject: [PATCH 04/26] wip --- .github/workflows/ci.yaml | 42 +++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index eee31a94c597d..63d9f7324f9af 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -744,27 +744,27 @@ jobs: env: CODER_BASE_IMAGE_TAG: ${{ steps.image-base-tag.outputs.base_tag }} - - name: Build and Push Linux amd64 Docker Image - id: build-docker - run: | - set -euxo pipefail - version="$(./scripts/version.sh)" - tag="main-$(echo "$version" | sed 's/+/-/g')" - - export CODER_IMAGE_BUILD_BASE_TAG="$(CODER_IMAGE_BASE=coder-base ./scripts/image_tag.sh --version "$version")" - ./scripts/build_docker.sh \ - --arch amd64 \ - --target "ghcr.io/coder/coder-preview:$tag" \ - --version $version \ - --push \ - build/coder_linux_amd64 - - # Tag as main - docker tag "ghcr.io/coder/coder-preview:$tag" ghcr.io/coder/coder-preview:main - docker push ghcr.io/coder/coder-preview:main - - # Store the tag in an output variable so we can use it in other jobs - echo "tag=$tag" >> $GITHUB_OUTPUT + # - name: Build and Push Linux amd64 Docker Image + # id: build-docker + # run: | + # set -euxo pipefail + # version="$(./scripts/version.sh)" + # tag="main-$(echo "$version" | sed 's/+/-/g')" + + # export CODER_IMAGE_BUILD_BASE_TAG="$(CODER_IMAGE_BASE=coder-base ./scripts/image_tag.sh --version "$version")" + # ./scripts/build_docker.sh \ + # --arch amd64 \ + # --target "ghcr.io/coder/coder-preview:$tag" \ + # --version $version \ + # --push \ + # build/coder_linux_amd64 + + # # Tag as main + # docker tag "ghcr.io/coder/coder-preview:$tag" ghcr.io/coder/coder-preview:main + # docker push ghcr.io/coder/coder-preview:main + + # # Store the tag in an output variable so we can use it in other jobs + # echo "tag=$tag" >> $GITHUB_OUTPUT - name: Prune old images uses: vlaurin/action-ghcr-prune@v0.5.0 From cef4eb726caf47a21f9a5d1322433d4e890be75b Mon Sep 17 00:00:00 2001 From: Muhammad Atif Ali Date: Wed, 10 Jan 2024 13:28:31 +0000 Subject: [PATCH 05/26] add condition to not push --- .github/workflows/ci.yaml | 42 ++++++++++++++++----------------------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 63d9f7324f9af..d92684fceffe6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -736,36 +736,28 @@ jobs: # build and push multi-arch manifest, this depends on the other images # being pushed so will automatically push them. - make -j push/build/coder_"$version"_linux.tag - ./scripts/build_docker_multiarch.sh \ + # only push if we are on main branch + if [ "${{ github.ref }}" == "refs/heads/main" ]; then + make -j push/build/coder_"$version"_linux.tag + else + make -j build/coder_"$version"_linux.tag + fi + + # only push if we are on main branch + if [ "${{ github.ref }}" == "refs/heads/main" ]; then + ./scripts/build_docker_multiarch.sh \ --push \ - --target "$(./scripts/image_tag.sh --version latest)" \ + --target "$(./scripts/image_tag.sh --version main)" \ + $(cat build/coder_"$version"_linux_{amd64,arm64,armv7}.tag) + else + ./scripts/build_docker_multiarch.sh \ + --target "$(./scripts/image_tag.sh --version main)" \ $(cat build/coder_"$version"_linux_{amd64,arm64,armv7}.tag) + fi + env: CODER_BASE_IMAGE_TAG: ${{ steps.image-base-tag.outputs.base_tag }} - # - name: Build and Push Linux amd64 Docker Image - # id: build-docker - # run: | - # set -euxo pipefail - # version="$(./scripts/version.sh)" - # tag="main-$(echo "$version" | sed 's/+/-/g')" - - # export CODER_IMAGE_BUILD_BASE_TAG="$(CODER_IMAGE_BASE=coder-base ./scripts/image_tag.sh --version "$version")" - # ./scripts/build_docker.sh \ - # --arch amd64 \ - # --target "ghcr.io/coder/coder-preview:$tag" \ - # --version $version \ - # --push \ - # build/coder_linux_amd64 - - # # Tag as main - # docker tag "ghcr.io/coder/coder-preview:$tag" ghcr.io/coder/coder-preview:main - # docker push ghcr.io/coder/coder-preview:main - - # # Store the tag in an output variable so we can use it in other jobs - # echo "tag=$tag" >> $GITHUB_OUTPUT - - name: Prune old images uses: vlaurin/action-ghcr-prune@v0.5.0 with: From bcbafce64e979af86ad76cfa70d885be4b90feae Mon Sep 17 00:00:00 2001 From: Muhammad Atif Ali Date: Wed, 10 Jan 2024 13:44:45 +0000 Subject: [PATCH 06/26] enable push and pull for base image --- .github/workflows/ci.yaml | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d92684fceffe6..fec7b5a24bc64 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -722,39 +722,45 @@ jobs: project: wl5hnrrkns context: base-build-context file: scripts/Dockerfile.base + pull: true + push: true platforms: linux/amd64,linux/arm64,linux/arm/v7 tags: ${{ steps.image-base-tag.outputs.base_tag }} - name: Build Linux Docker images + id: build-docker run: | set -euxo pipefail # build Docker images for each architecture version="$(./scripts/version.sh)" + tag="main-$(echo "$version" | sed 's/+/-/g')" + make -j build/coder_"$version"_linux_{amd64,arm64,armv7}.tag # build and push multi-arch manifest, this depends on the other images - # being pushed so will automatically push them. - - # only push if we are on main branch - if [ "${{ github.ref }}" == "refs/heads/main" ]; then - make -j push/build/coder_"$version"_linux.tag - else - make -j build/coder_"$version"_linux.tag - fi + # being pushed so will automatically push them # only push if we are on main branch if [ "${{ github.ref }}" == "refs/heads/main" ]; then ./scripts/build_docker_multiarch.sh \ --push \ - --target "$(./scripts/image_tag.sh --version main)" \ + --target "ghcr.io/coder/coder-preview:$tag" \ + --version $version \ $(cat build/coder_"$version"_linux_{amd64,arm64,armv7}.tag) else ./scripts/build_docker_multiarch.sh \ - --target "$(./scripts/image_tag.sh --version main)" \ + --target "ghcr.io/coder/coder-preview:$tag" \ + --version $version \ $(cat build/coder_"$version"_linux_{amd64,arm64,armv7}.tag) fi + # Tag as main + docker tag "ghcr.io/coder/coder-preview:$tag" ghcr.io/coder/coder-preview:main + docker push ghcr.io/coder/coder-preview:main + + echo "tag=$tag" >> $GITHUB_OUTPUT + env: CODER_BASE_IMAGE_TAG: ${{ steps.image-base-tag.outputs.base_tag }} From 1332c94a342328fe32dfcd690952b6c5ff7fe919 Mon Sep 17 00:00:00 2001 From: Muhammad Atif Ali Date: Wed, 10 Jan 2024 13:55:25 +0000 Subject: [PATCH 07/26] update project --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fec7b5a24bc64..7751675baa17d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -719,7 +719,7 @@ jobs: - name: Build base Docker image via depot.dev uses: depot/build-push-action@v1 with: - project: wl5hnrrkns + project: d66vz2cd27 context: base-build-context file: scripts/Dockerfile.base pull: true From 7894508ade16e942fe0fd0820bce697723c33b1a Mon Sep 17 00:00:00 2001 From: Muhammad Atif Ali Date: Wed, 10 Jan 2024 13:57:47 +0000 Subject: [PATCH 08/26] Tagged Docker image as main --- .github/workflows/ci.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7751675baa17d..5b402a6cdf3ae 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -748,6 +748,10 @@ jobs: --target "ghcr.io/coder/coder-preview:$tag" \ --version $version \ $(cat build/coder_"$version"_linux_{amd64,arm64,armv7}.tag) + + # Tag as main + docker tag "ghcr.io/coder/coder-preview:$tag" ghcr.io/coder/coder-preview:main + docker push ghcr.io/coder/coder-preview:main else ./scripts/build_docker_multiarch.sh \ --target "ghcr.io/coder/coder-preview:$tag" \ @@ -755,10 +759,6 @@ jobs: $(cat build/coder_"$version"_linux_{amd64,arm64,armv7}.tag) fi - # Tag as main - docker tag "ghcr.io/coder/coder-preview:$tag" ghcr.io/coder/coder-preview:main - docker push ghcr.io/coder/coder-preview:main - echo "tag=$tag" >> $GITHUB_OUTPUT env: From e9d24217ce7d0aa6c6771f65d526768049a17d93 Mon Sep 17 00:00:00 2001 From: Muhammad Atif Ali Date: Wed, 10 Jan 2024 14:00:18 +0000 Subject: [PATCH 09/26] use free runner --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5b402a6cdf3ae..7a1cbb908cd11 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -657,7 +657,7 @@ jobs: # are for amd64) needs: changes if: needs.changes.outputs.docs-only == 'false' - runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }} + runs-on: "ubuntu-latest" env: DOCKER_CLI_EXPERIMENTAL: "enabled" outputs: From 568560cd8259f92bbfe09dec710e6b88036a56f9 Mon Sep 17 00:00:00 2001 From: Muhammad Atif Ali Date: Wed, 10 Jan 2024 14:08:52 +0000 Subject: [PATCH 10/26] do not push --- .github/workflows/ci.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7a1cbb908cd11..bcd7385c261c2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -713,7 +713,7 @@ jobs: run: | set -euo pipefail version="$(./scripts/version.sh)" - echo "base_tag=$(CODER_IMAGE_BASE=coder-base ./scripts/image_tag.sh --version "$version")" >> $GITHUB_OUTPUT + echo "base_tag=$(CODER_IMAGE_BASE=ghcr.io/coder/coder-base ./scripts/image_tag.sh --version "$version")" >> $GITHUB_OUTPUT # This uses OIDC authentication, so no auth variables are required. - name: Build base Docker image via depot.dev @@ -723,7 +723,6 @@ jobs: context: base-build-context file: scripts/Dockerfile.base pull: true - push: true platforms: linux/amd64,linux/arm64,linux/arm/v7 tags: ${{ steps.image-base-tag.outputs.base_tag }} From 30e9470792bfbc2839b59ef792acc90cd2e0a8ff Mon Sep 17 00:00:00 2001 From: Muhammad Atif Ali Date: Wed, 10 Jan 2024 14:21:24 +0000 Subject: [PATCH 11/26] use larger runner --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index bcd7385c261c2..061cc7d6d7dac 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -657,7 +657,7 @@ jobs: # are for amd64) needs: changes if: needs.changes.outputs.docs-only == 'false' - runs-on: "ubuntu-latest" + runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }} env: DOCKER_CLI_EXPERIMENTAL: "enabled" outputs: From 8dd78cd54a33b0fb4dfa3e0755045f1de03f0b56 Mon Sep 17 00:00:00 2001 From: Muhammad Atif Ali Date: Wed, 10 Jan 2024 14:33:55 +0000 Subject: [PATCH 12/26] wip --- .github/workflows/ci.yaml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 061cc7d6d7dac..e2dcef1ccf764 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -726,6 +726,35 @@ jobs: platforms: linux/amd64,linux/arm64,linux/arm/v7 tags: ${{ steps.image-base-tag.outputs.base_tag }} + - name: Verify that images are pushed properly + run: | + # retry 10 times with a 5 second delay as the images may not be + # available immediately + for i in {1..10}; do + rc=0 + raw_manifests=$(docker buildx imagetools inspect --raw "${{ steps.image-base-tag.outputs.tag }}") || rc=$? + if [[ "$rc" -eq 0 ]]; then + break + fi + if [[ "$i" -eq 10 ]]; then + echo "Failed to pull manifests after 10 retries" + exit 1 + fi + echo "Failed to pull manifests, retrying in 5 seconds" + sleep 5 + done + + manifests=$( + echo "$raw_manifests" | \ + jq -r '.manifests[].platform | .os + "/" + .architecture + (if .variant then "/" + .variant else "" end)' + ) + + # Verify all 3 platforms are present. + set -euxo pipefail + echo "$manifests" | grep -q linux/amd64 + echo "$manifests" | grep -q linux/arm64 + echo "$manifests" | grep -q linux/arm/v7 + - name: Build Linux Docker images id: build-docker run: | From 29e43e9580681f370dea02e42edeaa391b542eb9 Mon Sep 17 00:00:00 2001 From: Muhammad Atif Ali Date: Wed, 10 Jan 2024 14:49:23 +0000 Subject: [PATCH 13/26] do not build base --- .github/workflows/ci.yaml | 52 ++------------------------------------- 1 file changed, 2 insertions(+), 50 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e2dcef1ccf764..089c890a311e0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -705,56 +705,6 @@ jobs: - name: Install depot.dev CLI uses: depot/setup-action@v1 - - name: Create an empty base-build-context directory - run: mkdir base-build-context - - - name: Determine base image tag - id: image-base-tag - run: | - set -euo pipefail - version="$(./scripts/version.sh)" - echo "base_tag=$(CODER_IMAGE_BASE=ghcr.io/coder/coder-base ./scripts/image_tag.sh --version "$version")" >> $GITHUB_OUTPUT - - # This uses OIDC authentication, so no auth variables are required. - - name: Build base Docker image via depot.dev - uses: depot/build-push-action@v1 - with: - project: d66vz2cd27 - context: base-build-context - file: scripts/Dockerfile.base - pull: true - platforms: linux/amd64,linux/arm64,linux/arm/v7 - tags: ${{ steps.image-base-tag.outputs.base_tag }} - - - name: Verify that images are pushed properly - run: | - # retry 10 times with a 5 second delay as the images may not be - # available immediately - for i in {1..10}; do - rc=0 - raw_manifests=$(docker buildx imagetools inspect --raw "${{ steps.image-base-tag.outputs.tag }}") || rc=$? - if [[ "$rc" -eq 0 ]]; then - break - fi - if [[ "$i" -eq 10 ]]; then - echo "Failed to pull manifests after 10 retries" - exit 1 - fi - echo "Failed to pull manifests, retrying in 5 seconds" - sleep 5 - done - - manifests=$( - echo "$raw_manifests" | \ - jq -r '.manifests[].platform | .os + "/" + .architecture + (if .variant then "/" + .variant else "" end)' - ) - - # Verify all 3 platforms are present. - set -euxo pipefail - echo "$manifests" | grep -q linux/amd64 - echo "$manifests" | grep -q linux/arm64 - echo "$manifests" | grep -q linux/arm/v7 - - name: Build Linux Docker images id: build-docker run: | @@ -764,6 +714,8 @@ jobs: version="$(./scripts/version.sh)" tag="main-$(echo "$version" | sed 's/+/-/g')" + export CODER_IMAGE_BUILD_BASE_TAG="ghcr.io/coder/coder-base:latest" + make -j build/coder_"$version"_linux_{amd64,arm64,armv7}.tag # build and push multi-arch manifest, this depends on the other images From 3d285969de6fc7159db03cad20978bee46b2a83f Mon Sep 17 00:00:00 2001 From: Muhammad Atif Ali Date: Wed, 10 Jan 2024 14:50:02 +0000 Subject: [PATCH 14/26] remove excess permission --- .github/workflows/ci.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 089c890a311e0..84566d81c5af2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,7 +19,6 @@ permissions: repository-projects: none security-events: none statuses: none - id-token: write # This is required for the build to authenticate to depot.dev # Cancel in-progress runs for pull requests when developers push # additional changes From 0d1233bd6f7649bd05ffdfce348a6ca21af8cbc8 Mon Sep 17 00:00:00 2001 From: Muhammad Atif Ali Date: Wed, 10 Jan 2024 14:51:18 +0000 Subject: [PATCH 15/26] cleanup --- .github/workflows/ci.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 84566d81c5af2..349ab8c9cc2aa 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -701,9 +701,6 @@ jobs: build/coder_"$version"_windows_amd64.zip \ build/coder_"$version"_linux_amd64.{tar.gz,deb} - - name: Install depot.dev CLI - uses: depot/setup-action@v1 - - name: Build Linux Docker images id: build-docker run: | @@ -740,9 +737,6 @@ jobs: echo "tag=$tag" >> $GITHUB_OUTPUT - env: - CODER_BASE_IMAGE_TAG: ${{ steps.image-base-tag.outputs.base_tag }} - - name: Prune old images uses: vlaurin/action-ghcr-prune@v0.5.0 with: From 5caf911b16ede93f1406d33ef4de4cf3e22bd205 Mon Sep 17 00:00:00 2001 From: Muhammad Atif Ali Date: Wed, 10 Jan 2024 16:19:18 +0000 Subject: [PATCH 16/26] wip --- .github/workflows/ci.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 349ab8c9cc2aa..8d8508d697efe 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -712,6 +712,10 @@ jobs: export CODER_IMAGE_BUILD_BASE_TAG="ghcr.io/coder/coder-base:latest" + echo "ghcr.io/coder/coder-preview:$tag-amd64" > build/coder_"$version"_linux_amd64.tag + echo "ghcr.io/coder/coder-preview:$tag-arm64" > build/coder_"$version"_linux_arm64.tag + echo "ghcr.io/coder/coder-preview:$tag-armv7" > build/coder_"$version"_linux_armv7.tag + make -j build/coder_"$version"_linux_{amd64,arm64,armv7}.tag # build and push multi-arch manifest, this depends on the other images From 71e81458b4cbf5f94dfe8947efb80f64dca5ffdc Mon Sep 17 00:00:00 2001 From: Muhammad Atif Ali Date: Wed, 10 Jan 2024 22:40:33 +0000 Subject: [PATCH 17/26] test --- .github/workflows/ci.yaml | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8d8508d697efe..e6d5764d09e7e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -697,32 +697,31 @@ jobs: make gen/mark-fresh make -j \ - build/coder_linux_amd64 \ + build/coder_linux_{amd64,arm64,armv7} \ build/coder_"$version"_windows_amd64.zip \ build/coder_"$version"_linux_amd64.{tar.gz,deb} - name: Build Linux Docker images id: build-docker + env: + CODER_IMAGE_BASE: ghcr.io/coder/coder-preview run: | set -euxo pipefail # build Docker images for each architecture version="$(./scripts/version.sh)" tag="main-$(echo "$version" | sed 's/+/-/g')" - - export CODER_IMAGE_BUILD_BASE_TAG="ghcr.io/coder/coder-base:latest" - - echo "ghcr.io/coder/coder-preview:$tag-amd64" > build/coder_"$version"_linux_amd64.tag - echo "ghcr.io/coder/coder-preview:$tag-arm64" > build/coder_"$version"_linux_arm64.tag - echo "ghcr.io/coder/coder-preview:$tag-armv7" > build/coder_"$version"_linux_armv7.tag + echo "tag=$tag" >> $GITHUB_OUTPUT make -j build/coder_"$version"_linux_{amd64,arm64,armv7}.tag - # build and push multi-arch manifest, this depends on the other images - # being pushed so will automatically push them - # only push if we are on main branch if [ "${{ github.ref }}" == "refs/heads/main" ]; then + export DOCKER_CLI_EXPERIMENTAL=enabled + # build and push multi-arch manifest, this depends on the other images + # being pushed so will automatically push them + make -j push/build/coder_"$version"_linux_{amd64,arm64,armv7}.tag + ./scripts/build_docker_multiarch.sh \ --push \ --target "ghcr.io/coder/coder-preview:$tag" \ @@ -732,15 +731,8 @@ jobs: # Tag as main docker tag "ghcr.io/coder/coder-preview:$tag" ghcr.io/coder/coder-preview:main docker push ghcr.io/coder/coder-preview:main - else - ./scripts/build_docker_multiarch.sh \ - --target "ghcr.io/coder/coder-preview:$tag" \ - --version $version \ - $(cat build/coder_"$version"_linux_{amd64,arm64,armv7}.tag) fi - echo "tag=$tag" >> $GITHUB_OUTPUT - - name: Prune old images uses: vlaurin/action-ghcr-prune@v0.5.0 with: From 65678a71942bd89a65a7b696ae44e94e15ffa6e3 Mon Sep 17 00:00:00 2001 From: Muhammad Atif Ali Date: Wed, 10 Jan 2024 22:53:36 +0000 Subject: [PATCH 18/26] test --- .github/workflows/ci.yaml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e6d5764d09e7e..24b0a3591887c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -716,22 +716,22 @@ jobs: make -j build/coder_"$version"_linux_{amd64,arm64,armv7}.tag # only push if we are on main branch - if [ "${{ github.ref }}" == "refs/heads/main" ]; then - export DOCKER_CLI_EXPERIMENTAL=enabled - # build and push multi-arch manifest, this depends on the other images - # being pushed so will automatically push them - make -j push/build/coder_"$version"_linux_{amd64,arm64,armv7}.tag - - ./scripts/build_docker_multiarch.sh \ - --push \ - --target "ghcr.io/coder/coder-preview:$tag" \ - --version $version \ - $(cat build/coder_"$version"_linux_{amd64,arm64,armv7}.tag) - - # Tag as main - docker tag "ghcr.io/coder/coder-preview:$tag" ghcr.io/coder/coder-preview:main - docker push ghcr.io/coder/coder-preview:main - fi + # if [ "${{ github.ref }}" == "refs/heads/main" ]; then + export DOCKER_CLI_EXPERIMENTAL=enabled + # build and push multi-arch manifest, this depends on the other images + # being pushed so will automatically push them + make -j push/build/coder_"$version"_linux_{amd64,arm64,armv7}.tag + + ./scripts/build_docker_multiarch.sh \ + --push \ + --target "ghcr.io/coder/coder-preview:$tag" \ + --version $version \ + $(cat build/coder_"$version"_linux_{amd64,arm64,armv7}.tag) + + # Tag as main + docker tag "ghcr.io/coder/coder-preview:$tag" ghcr.io/coder/coder-preview:main + docker push ghcr.io/coder/coder-preview:main + # fi - name: Prune old images uses: vlaurin/action-ghcr-prune@v0.5.0 From d3ee8318ff63a8727861eb82cbacf64874c8837b Mon Sep 17 00:00:00 2001 From: Muhammad Atif Ali Date: Wed, 10 Jan 2024 22:59:56 +0000 Subject: [PATCH 19/26] test --- .github/workflows/ci.yaml | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 24b0a3591887c..596e5c0834c40 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -705,6 +705,7 @@ jobs: id: build-docker env: CODER_IMAGE_BASE: ghcr.io/coder/coder-preview + DOCKER_CLI_EXPERIMENTAL: "enabled" run: | set -euxo pipefail @@ -716,22 +717,27 @@ jobs: make -j build/coder_"$version"_linux_{amd64,arm64,armv7}.tag # only push if we are on main branch - # if [ "${{ github.ref }}" == "refs/heads/main" ]; then - export DOCKER_CLI_EXPERIMENTAL=enabled - # build and push multi-arch manifest, this depends on the other images - # being pushed so will automatically push them - make -j push/build/coder_"$version"_linux_{amd64,arm64,armv7}.tag - - ./scripts/build_docker_multiarch.sh \ - --push \ - --target "ghcr.io/coder/coder-preview:$tag" \ - --version $version \ - $(cat build/coder_"$version"_linux_{amd64,arm64,armv7}.tag) - - # Tag as main - docker tag "ghcr.io/coder/coder-preview:$tag" ghcr.io/coder/coder-preview:main - docker push ghcr.io/coder/coder-preview:main - # fi + # modify the condition to be always: true to push on all branches temporarily for testing + #if [ "${{ github.ref }}" == "refs/heads/main" ]; then + if [ "${{ github.ref }}" == "refs/heads/main" || true ]; then + + # build and push multi-arch manifest, this depends on the other images + # being pushed so will automatically push them + make -j push/build/coder_"$version"_linux_{amd64,arm64,armv7}.tag + + ./scripts/build_docker_multiarch.sh \ + --push \ + --target "ghcr.io/coder/coder-preview:$tag" \ + --version $version \ + $(cat build/coder_"$version"_linux_{amd64,arm64,armv7}.tag) + + # also push the main tag + ./scripts/build_docker_multiarch.sh \ + --push \ + --target "ghcr.io/coder/coder-preview:main" \ + --version $version \ + $(cat build/coder_"$version"_linux_{amd64,arm64,armv7}.tag) + fi - name: Prune old images uses: vlaurin/action-ghcr-prune@v0.5.0 From 6816e104fdb698ef041e457ae3e9f70d90326547 Mon Sep 17 00:00:00 2001 From: Muhammad Atif Ali Date: Wed, 10 Jan 2024 23:18:49 +0000 Subject: [PATCH 20/26] Update Docker image tags and retention policy --- .github/workflows/ci.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 596e5c0834c40..b557244e8c614 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -718,8 +718,7 @@ jobs: # only push if we are on main branch # modify the condition to be always: true to push on all branches temporarily for testing - #if [ "${{ github.ref }}" == "refs/heads/main" ]; then - if [ "${{ github.ref }}" == "refs/heads/main" || true ]; then + if [ "${{ github.ref }}" == "refs/heads/main" ]; then # build and push multi-arch manifest, this depends on the other images # being pushed so will automatically push them @@ -731,10 +730,10 @@ jobs: --version $version \ $(cat build/coder_"$version"_linux_{amd64,arm64,armv7}.tag) - # also push the main tag + # also push the latest tag ./scripts/build_docker_multiarch.sh \ --push \ - --target "ghcr.io/coder/coder-preview:main" \ + --target "ghcr.io/coder/coder-preview:latest" \ --version $version \ $(cat build/coder_"$version"_linux_{amd64,arm64,armv7}.tag) fi @@ -746,8 +745,8 @@ jobs: organization: coder container: coder-preview keep-younger-than: 7 # days + keep-tags: latest keep-tags-regexes: ^pr - prune-tags-regexes: ^main- prune-untagged: true - name: Upload build artifacts From 4116e940ea6150947e32aea51f50ea1d18706956 Mon Sep 17 00:00:00 2001 From: Muhammad Atif Ali Date: Wed, 10 Jan 2024 23:21:51 +0000 Subject: [PATCH 21/26] cleanup --- .github/workflows/ci.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b557244e8c614..3f0877ace4dfb 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -714,23 +714,23 @@ jobs: tag="main-$(echo "$version" | sed 's/+/-/g')" echo "tag=$tag" >> $GITHUB_OUTPUT + # build images for each architecture make -j build/coder_"$version"_linux_{amd64,arm64,armv7}.tag # only push if we are on main branch - # modify the condition to be always: true to push on all branches temporarily for testing if [ "${{ github.ref }}" == "refs/heads/main" ]; then - # build and push multi-arch manifest, this depends on the other images # being pushed so will automatically push them make -j push/build/coder_"$version"_linux_{amd64,arm64,armv7}.tag + # Create and push a multi-arch manifest ./scripts/build_docker_multiarch.sh \ --push \ --target "ghcr.io/coder/coder-preview:$tag" \ --version $version \ $(cat build/coder_"$version"_linux_{amd64,arm64,armv7}.tag) - # also push the latest tag + # Create and push a multi-arch manifest for the latest tag ./scripts/build_docker_multiarch.sh \ --push \ --target "ghcr.io/coder/coder-preview:latest" \ From 8b39c8b24b8c1844d57a981d6de1ee5f35201c25 Mon Sep 17 00:00:00 2001 From: Muhammad Atif Ali Date: Wed, 10 Jan 2024 23:25:09 +0000 Subject: [PATCH 22/26] Add prune-tags-regexes to CI workflow --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3f0877ace4dfb..364dac42455e7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -747,6 +747,7 @@ jobs: keep-younger-than: 7 # days keep-tags: latest keep-tags-regexes: ^pr + prune-tags-regexes: ^main- prune-untagged: true - name: Upload build artifacts From 6fbfcb61dfb4a653e0571b7e20d95c6384ddce43 Mon Sep 17 00:00:00 2001 From: Muhammad Atif Ali Date: Thu, 11 Jan 2024 03:56:10 +0300 Subject: [PATCH 23/26] prune intermediate images for each arch. --- .github/workflows/ci.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 364dac42455e7..b097b738e8e69 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -747,7 +747,9 @@ jobs: keep-younger-than: 7 # days keep-tags: latest keep-tags-regexes: ^pr - prune-tags-regexes: ^main- + prune-tags-regexes: | + ^main- + ^v prune-untagged: true - name: Upload build artifacts From 7c91a253935afbb95b11163da16731b80fda74cf Mon Sep 17 00:00:00 2001 From: Muhammad Atif Ali Date: Thu, 11 Jan 2024 15:23:24 +0300 Subject: [PATCH 24/26] prevent tags cleanup and artifact upload on PR branches --- .github/workflows/ci.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b097b738e8e69..5beddce646a99 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -655,7 +655,7 @@ jobs: # to main branch. We are only building this for amd64 platform. (>95% pulls # are for amd64) needs: changes - if: needs.changes.outputs.docs-only == 'false' + if: needs.changes.outputs.docs-only == 'false' && !github.event.pull_request.head.repo.fork runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }} env: DOCKER_CLI_EXPERIMENTAL: "enabled" @@ -739,6 +739,7 @@ jobs: fi - name: Prune old images + if: github.ref == 'refs/heads/main' uses: vlaurin/action-ghcr-prune@v0.5.0 with: token: ${{ secrets.GITHUB_TOKEN }} @@ -753,6 +754,7 @@ jobs: prune-untagged: true - name: Upload build artifacts + if: github.ref == 'refs/heads/main' uses: actions/upload-artifact@v4 with: name: coder From e48d0c9b2a2fe8eb09589aa80dabf4aa33ca1758 Mon Sep 17 00:00:00 2001 From: Muhammad Atif Ali Date: Wed, 17 Jan 2024 13:21:13 +0300 Subject: [PATCH 25/26] pr suggestions --- .github/workflows/ci.yaml | 8 ++++++++ scripts/image_tag.sh | 8 +++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index cabf9a45ed701..c7c29dd1bfbde 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -705,6 +705,7 @@ jobs: id: build-docker env: CODER_IMAGE_BASE: ghcr.io/coder/coder-preview + CODER_IMAGE_TAG_PREFIX: main DOCKER_CLI_EXPERIMENTAL: "enabled" run: | set -euxo pipefail @@ -730,6 +731,13 @@ jobs: --version $version \ $(cat build/coder_"$version"_linux_{amd64,arm64,armv7}.tag) + # Create and push a multi-arch manifest for the main tag + ./scripts/build_docker_multiarch.sh \ + --push \ + --target "ghcr.io/coder/coder-preview:main" \ + --version $version \ + $(cat build/coder_"$version"_linux_{amd64,arm64,armv7}.tag) + # Create and push a multi-arch manifest for the latest tag ./scripts/build_docker_multiarch.sh \ --push \ diff --git a/scripts/image_tag.sh b/scripts/image_tag.sh index 8b405c48e304f..68dfbcebf99cb 100755 --- a/scripts/image_tag.sh +++ b/scripts/image_tag.sh @@ -50,10 +50,16 @@ if [[ "$version" == "" ]]; then fi image="${CODER_IMAGE_BASE:-ghcr.io/coder/coder}" -tag="v$version" + +# use CODER_IMAGE_TAG_PREFIX if set as a prefix for the tag +tag_prefix="${CODER_IMAGE_TAG_PREFIX:-}" + +tag="${tag_prefix:+$tag_prefix-}v$version" + if [[ "$version" == "latest" ]]; then tag="latest" fi + if [[ "$arch" != "" ]]; then tag+="-$arch" fi From 319ffa1da2f4b4ded53d3b292451861a705baa42 Mon Sep 17 00:00:00 2001 From: Muhammad Atif Ali Date: Thu, 18 Jan 2024 13:27:10 +0300 Subject: [PATCH 26/26] PR commnets --- .github/workflows/ci.yaml | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c7c29dd1bfbde..6b628671fe511 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -724,26 +724,19 @@ jobs: # being pushed so will automatically push them make -j push/build/coder_"$version"_linux_{amd64,arm64,armv7}.tag - # Create and push a multi-arch manifest - ./scripts/build_docker_multiarch.sh \ - --push \ - --target "ghcr.io/coder/coder-preview:$tag" \ - --version $version \ - $(cat build/coder_"$version"_linux_{amd64,arm64,armv7}.tag) - - # Create and push a multi-arch manifest for the main tag - ./scripts/build_docker_multiarch.sh \ - --push \ - --target "ghcr.io/coder/coder-preview:main" \ - --version $version \ - $(cat build/coder_"$version"_linux_{amd64,arm64,armv7}.tag) - - # Create and push a multi-arch manifest for the latest tag - ./scripts/build_docker_multiarch.sh \ - --push \ - --target "ghcr.io/coder/coder-preview:latest" \ - --version $version \ - $(cat build/coder_"$version"_linux_{amd64,arm64,armv7}.tag) + # Define specific tags + tags=("$tag" "main" "latest") + + # Create and push a multi-arch manifest for each tag + # we are adding `latest` tag and keeping `main` for backward + # compatibality + for t in "${tags[@]}"; do + ./scripts/build_docker_multiarch.sh \ + --push \ + --target "ghcr.io/coder/coder-preview:$t" \ + --version $version \ + $(cat build/coder_"$version"_linux_{amd64,arm64,armv7}.tag) + done fi - name: Prune old images