From 18c974be55e537fc9e58a1e53bac41ea087fb032 Mon Sep 17 00:00:00 2001 From: vsoch Date: Tue, 3 Aug 2021 20:44:25 -0600 Subject: [PATCH 01/17] testing new recipe Signed-off-by: vsoch --- .github/workflows/native-install.yml | 100 ++++++++++++++++----------- README.md | 27 ++++++-- Singularity | 2 +- 3 files changed, 80 insertions(+), 49 deletions(-) diff --git a/.github/workflows/native-install.yml b/.github/workflows/native-install.yml index f3a035f..f60e700 100644 --- a/.github/workflows/native-install.yml +++ b/.github/workflows/native-install.yml @@ -1,59 +1,75 @@ name: Singularity Build (native) -on: [push] +on: + push: + + # Edit the branches here if you want to change deploy behavior + branches: + - main + + # Do the builds on all pull requests (to test them) + pull_request: [] + jobs: + changes: + name: "Changed Singularity Recipes" + runs-on: ubuntu-latest + outputs: + changed_file: ${{ steps.files.outputs.added_modified }} + steps: + - id: files + uses: jitterbit/get-changed-files@b17fbb00bdc0c0f63fcf166580804b4d2cdc2a42 + with: + format: 'json' + + build-test-containers: + needs: + - changes + runs-on: ubuntu-latest + strategy: + # Keep going on other deployments if anything bloops + fail-fast: false + matrix: + changed_file: ${{ fromJson(needs.changes.outputs.changed_file) }} - build: - name: Build - runs-on: ubuntu-18.04 + name: Check ${{ matrix.changed_file }} + if: ${{ env.keepgoing == 'true' }} steps: + - name: Continue if Singularity Recipe + run: | + # Continue if we have a changed Singularity recipe + if [[ "${{ matrix.changed_file }}" = *Singularity* ]]; then + echo "keepgoing=true" >> $GITHUB_ENV + fi - - name: Set up Go 1.13 - uses: actions/setup-go@v1 + - uses: eWaterCycle/setup-singularity@v6 with: - go-version: 1.13 - id: go + singularity-version: 3.8.1 - - name: Install Dependencies - run: | - sudo apt-get update && sudo apt-get install -y \ - build-essential \ - libssl-dev \ - uuid-dev \ - libgpgme11-dev \ - squashfs-tools \ - libseccomp-dev \ - pkg-config - - - name: Install Singularity - env: - SINGULARITY_VERSION: 3.8.1 - GOPATH: /tmp/go - run: | - mkdir -p $GOPATH - sudo mkdir -p /usr/local/var/singularity/mnt && \ - mkdir -p $GOPATH/src/github.com/sylabs && \ - cd $GOPATH/src/github.com/sylabs && \ - - wget -qO- https://github.com/sylabs/singularity/releases/download/v${SINGULARITY_VERSION}/singularity-ce-${SINGULARITY_VERSION}.tar.gz | \ - tar xzv && \ - cd singularity-ce-${SINGULARITY_VERSION} && \ - ./mconfig -p /usr/local && \ - make -C builddir && \ - sudo make -C builddir install - - name: Check out code for the container build - uses: actions/checkout@v1 + uses: actions/checkout@v2 - name: Build Container env: - SINGULARITY_RECIPE: Singularity - OUTPUT_CONTAINER: container.sif + recipe: ${{ matrix.changed_file }} run: | ls - if [ -f "${SINGULARITY_RECIPE}" ]; then - sudo -E singularity build ${OUTPUT_CONTAINER} ${SINGULARITY_RECIPE} + if [ -f "${{ matrix.changed_file }}" ]; then + sudo -E singularity build container.sif ${{ matrix.changed_file }} + tag=$(echo "${recipe/Singularity\./}") + if [ "$tag" == "Singularity" ]; then + tag=latest + fi + # Build the container and name by tag + echo "Tag is $tag." + echo "tag=$tag" >> $GITHUB_ENV else - echo "${SINGULARITY_RECIPE} is not found." + echo "${{ matrix.changed_file }} is not found." echo "Present working directory: $PWD" ls fi + + - name: Login and Deploy Container + if: (github.event_name != 'pull_request') + run: | + echo ${{ secrets.GITHUB_TOKEN }} | singularity remote login ghcr.io -u ${GITHUB_USERNAME} --password-stdin + singularity push container.sif oras://${GITHUB_REPOSITORY}:${tag} diff --git a/README.md b/README.md index d10b1f1..3a0b4fd 100644 --- a/README.md +++ b/README.md @@ -5,13 +5,16 @@ This is a simple example of how you can achieve: - version control of your recipes - - versioning to include image hash *and* commit id - build of associated container and - - (optional) push to a storage endpoint + - push to a storage endpoint -for a reproducible build workflow. +for a reproducible build workflow! By default, we will build on all pull requests and deploy +on push to main. The containers will go to an enabled GitHub package registry thanks to +the Singularity oras endpoint. -There are two workflows configured on master that build a container: +**updated** August 2021, we can now push containers to the GitHub package registry! Woohoo! + +There are two workflows configured on master that build a Singularity container: 1. [native install](.github/workflows/native-install.yml) builds Singularity 3.x (with GoLang). 2. [docker image](.github/workfolws/container.yml) builds in a [docker image](https://quay.io/repository/singularity/singularity). @@ -39,6 +42,11 @@ that writes the configuration. ## Quick Start +### 0. Enable Packages + +If you want to use the [GitHub package registry](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry) +you'll need to follow the instructions there to enable packages for your organization, specifically "public" and "internal" packages should be allowed to be created. + ### 1. Add Your Recipes Add your Singularity recipes to this repository, and edit the [build workflow](.github/workflows/native-install.yml) @@ -86,9 +94,16 @@ one way to eat a reeses: singularity run toasty.sif ``` -### 3. Push to a registry +### 3. Check Triggers + +The workflow files each have a section at the top that indicates when the workflow will +trigger. By default, we will do builds on pull requests, and deploys on pushes to a main +branch. If you want to change this logic, edit the top of the recipe files. + +### 4. Push to a registry -You might be done there. But if not, you can install [Singularity Registry Client](http://singularityhub.github.io/sregistry-cli) and push to your cloud storage of choice! You will want to add python and python-dev to the dependency +If you are good with GitHub packages, then you are good to go! Otherwise, +if you want to push to other kinds of storage, you can install the [Singularity Registry Client](http://singularityhub.github.io/sregistry-cli) and push to your cloud storage of choice! You will want to add python and python-dev to the dependency install: ```yaml diff --git a/Singularity b/Singularity index e610074..140be58 100644 --- a/Singularity +++ b/Singularity @@ -2,4 +2,4 @@ Bootstrap: docker From: busybox:latest %runscript -echo "Hold me closer... tiny container :D" +echo "Hold me closer... tiny container :) :D" From 97625a4f1b3ece23ca09ea9048dda3309a8dbd2d Mon Sep 17 00:00:00 2001 From: vsoch Date: Tue, 3 Aug 2021 20:46:50 -0600 Subject: [PATCH 02/17] invalid syntax Signed-off-by: vsoch --- .github/workflows/container.yml | 13 +++++++++-- .github/workflows/native-install.yml | 32 ++++++++++++++-------------- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index fc62354..6c08dc9 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -1,5 +1,14 @@ name: Singularity Build (docker) -on: [push] +on: + push: + + # Edit the branches here if you want to change deploy behavior + branches: + - main + + # Do the builds on all pull requests (to test them) + pull_request: [] + jobs: build: @@ -8,7 +17,7 @@ jobs: strategy: matrix: singularity_version: - - '3.5.3' + - '3.8.1' container: image: quay.io/singularity/singularity:v${{ matrix.singularity_version }} options: --privileged diff --git a/.github/workflows/native-install.yml b/.github/workflows/native-install.yml index f60e700..102ebbe 100644 --- a/.github/workflows/native-install.yml +++ b/.github/workflows/native-install.yml @@ -41,19 +41,19 @@ jobs: echo "keepgoing=true" >> $GITHUB_ENV fi - - uses: eWaterCycle/setup-singularity@v6 - with: - singularity-version: 3.8.1 + - uses: eWaterCycle/setup-singularity@v6 + with: + singularity-version: 3.8.1 - - name: Check out code for the container build - uses: actions/checkout@v2 + - name: Check out code for the container build + uses: actions/checkout@v2 - - name: Build Container - env: - recipe: ${{ matrix.changed_file }} - run: | - ls - if [ -f "${{ matrix.changed_file }}" ]; then + - name: Build Container + env: + recipe: ${{ matrix.changed_file }} + run: | + ls + if [ -f "${{ matrix.changed_file }}" ]; then sudo -E singularity build container.sif ${{ matrix.changed_file }} tag=$(echo "${recipe/Singularity\./}") if [ "$tag" == "Singularity" ]; then @@ -62,14 +62,14 @@ jobs: # Build the container and name by tag echo "Tag is $tag." echo "tag=$tag" >> $GITHUB_ENV - else + else echo "${{ matrix.changed_file }} is not found." echo "Present working directory: $PWD" ls - fi + fi - - name: Login and Deploy Container - if: (github.event_name != 'pull_request') - run: | + - name: Login and Deploy Container + if: (github.event_name != 'pull_request') + run: | echo ${{ secrets.GITHUB_TOKEN }} | singularity remote login ghcr.io -u ${GITHUB_USERNAME} --password-stdin singularity push container.sif oras://${GITHUB_REPOSITORY}:${tag} From 45e0cd425d239ff913bf958eb4f0283cf2cf14b2 Mon Sep 17 00:00:00 2001 From: vsoch Date: Tue, 3 Aug 2021 20:47:21 -0600 Subject: [PATCH 03/17] invalid syntax Signed-off-by: vsoch --- .github/workflows/native-install.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/native-install.yml b/.github/workflows/native-install.yml index 102ebbe..df8d883 100644 --- a/.github/workflows/native-install.yml +++ b/.github/workflows/native-install.yml @@ -68,8 +68,8 @@ jobs: ls fi - - name: Login and Deploy Container - if: (github.event_name != 'pull_request') - run: | + - name: Login and Deploy Container + if: (github.event_name != 'pull_request') + run: | echo ${{ secrets.GITHUB_TOKEN }} | singularity remote login ghcr.io -u ${GITHUB_USERNAME} --password-stdin singularity push container.sif oras://${GITHUB_REPOSITORY}:${tag} From c49b17dbca04919383f24f4efa281aea1e0cf040 Mon Sep 17 00:00:00 2001 From: vsoch Date: Tue, 3 Aug 2021 20:48:08 -0600 Subject: [PATCH 04/17] invalid syntax Signed-off-by: vsoch --- .github/workflows/container.yml | 4 ++-- .github/workflows/native-install.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 6c08dc9..a7a9222 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -6,8 +6,8 @@ on: branches: - main - # Do the builds on all pull requests (to test them) - pull_request: [] + # Do the builds on all pull requests (to test them) + pull_request: [] jobs: diff --git a/.github/workflows/native-install.yml b/.github/workflows/native-install.yml index df8d883..39e7898 100644 --- a/.github/workflows/native-install.yml +++ b/.github/workflows/native-install.yml @@ -6,8 +6,8 @@ on: branches: - main - # Do the builds on all pull requests (to test them) - pull_request: [] + # Do the builds on all pull requests (to test them) + pull_request: [] jobs: changes: From ebfa81fd7e518ec21435f0a5590477b106220b5a Mon Sep 17 00:00:00 2001 From: vsoch Date: Tue, 3 Aug 2021 20:49:44 -0600 Subject: [PATCH 05/17] invalid syntax Signed-off-by: vsoch --- .github/workflows/native-install.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/native-install.yml b/.github/workflows/native-install.yml index 39e7898..e816f10 100644 --- a/.github/workflows/native-install.yml +++ b/.github/workflows/native-install.yml @@ -32,7 +32,6 @@ jobs: changed_file: ${{ fromJson(needs.changes.outputs.changed_file) }} name: Check ${{ matrix.changed_file }} - if: ${{ env.keepgoing == 'true' }} steps: - name: Continue if Singularity Recipe run: | @@ -42,13 +41,16 @@ jobs: fi - uses: eWaterCycle/setup-singularity@v6 + if: ${{ env.keepgoing == 'true' }} with: singularity-version: 3.8.1 - name: Check out code for the container build + if: ${{ env.keepgoing == 'true' }} uses: actions/checkout@v2 - name: Build Container + if: ${{ env.keepgoing == 'true' }} env: recipe: ${{ matrix.changed_file }} run: | @@ -69,7 +71,7 @@ jobs: fi - name: Login and Deploy Container - if: (github.event_name != 'pull_request') + if: (github.event_name != 'pull_request' && ${{ env.keepgoing == 'true' }}) run: | echo ${{ secrets.GITHUB_TOKEN }} | singularity remote login ghcr.io -u ${GITHUB_USERNAME} --password-stdin singularity push container.sif oras://${GITHUB_REPOSITORY}:${tag} From 829f7e7c487b9f251f103cc8493d6bccdd4736f4 Mon Sep 17 00:00:00 2001 From: vsoch Date: Tue, 3 Aug 2021 20:52:18 -0600 Subject: [PATCH 06/17] invalid syntax Signed-off-by: vsoch --- .github/workflows/native-install.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/native-install.yml b/.github/workflows/native-install.yml index e816f10..c93a62f 100644 --- a/.github/workflows/native-install.yml +++ b/.github/workflows/native-install.yml @@ -43,7 +43,7 @@ jobs: - uses: eWaterCycle/setup-singularity@v6 if: ${{ env.keepgoing == 'true' }} with: - singularity-version: 3.8.1 + singularity-version: 3.8.0 - name: Check out code for the container build if: ${{ env.keepgoing == 'true' }} @@ -71,7 +71,11 @@ jobs: fi - name: Login and Deploy Container - if: (github.event_name != 'pull_request' && ${{ env.keepgoing == 'true' }}) + if: (github.event_name != 'pull_request') + env: + keepgoing: ${{ env.keepgoing }} run: | - echo ${{ secrets.GITHUB_TOKEN }} | singularity remote login ghcr.io -u ${GITHUB_USERNAME} --password-stdin - singularity push container.sif oras://${GITHUB_REPOSITORY}:${tag} + if [[ "${keepgoing}" == "true" ]]; then + echo ${{ secrets.GITHUB_TOKEN }} | singularity remote login ghcr.io -u ${GITHUB_USERNAME} --password-stdin + singularity push container.sif oras://${GITHUB_REPOSITORY}:${tag} + fi From 33bd733f4db20a685d4822be87b87aa138f252e6 Mon Sep 17 00:00:00 2001 From: vsoch Date: Tue, 3 Aug 2021 20:55:51 -0600 Subject: [PATCH 07/17] invalid syntax Signed-off-by: vsoch --- .github/workflows/native-install.yml | 36 ++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/.github/workflows/native-install.yml b/.github/workflows/native-install.yml index c93a62f..d51bdcc 100644 --- a/.github/workflows/native-install.yml +++ b/.github/workflows/native-install.yml @@ -40,10 +40,42 @@ jobs: echo "keepgoing=true" >> $GITHUB_ENV fi - - uses: eWaterCycle/setup-singularity@v6 + - name: Set up Go 1.13 if: ${{ env.keepgoing == 'true' }} + uses: actions/setup-go@v1 with: - singularity-version: 3.8.0 + go-version: 1.13 + id: go + + - name: Install Dependencies + if: ${{ env.keepgoing == 'true' }} + run: | + sudo apt-get update && sudo apt-get install -y \ + build-essential \ + libssl-dev \ + uuid-dev \ + libgpgme11-dev \ + squashfs-tools \ + libseccomp-dev \ + pkg-config + + - name: Install Singularity + if: ${{ env.keepgoing == 'true' }} + env: + SINGULARITY_VERSION: 3.8.1 + GOPATH: /tmp/go + + run: | + mkdir -p $GOPATH + sudo mkdir -p /usr/local/var/singularity/mnt && \ + mkdir -p $GOPATH/src/github.com/sylabs && \ + cd $GOPATH/src/github.com/sylabs && \ + wget -qO- https://github.com/sylabs/singularity/releases/download/v${SINGULARITY_VERSION}/singularity-ce-${SINGULARITY_VERSION}.tar.gz | \ + tar xzv && \ + cd singularity-ce-${SINGULARITY_VERSION} && \ + ./mconfig -p /usr/local && \ + make -C builddir && \ + sudo make -C builddir install - name: Check out code for the container build if: ${{ env.keepgoing == 'true' }} From 2105709ca40754c06acca9790adf88bf24c89ded Mon Sep 17 00:00:00 2001 From: vsoch Date: Tue, 3 Aug 2021 21:01:17 -0600 Subject: [PATCH 08/17] also try container-based build Signed-off-by: vsoch --- .github/workflows/container.yml | 78 +++++++++++++++++++++++++-------- 1 file changed, 60 insertions(+), 18 deletions(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index a7a9222..0ca05b2 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -10,31 +10,73 @@ on: pull_request: [] jobs: - - build: - name: Build - runs-on: ubuntu-18.04 + changes: + name: "Changed Singularity Recipes" + runs-on: ubuntu-latest + outputs: + changed_file: ${{ steps.files.outputs.added_modified }} + steps: + - id: files + uses: jitterbit/get-changed-files@b17fbb00bdc0c0f63fcf166580804b4d2cdc2a42 + with: + format: 'json' + + build-test-containers: + needs: + - changes + runs-on: ubuntu-latest strategy: + # Keep going on other deployments if anything bloops + fail-fast: false matrix: + changed_file: ${{ fromJson(needs.changes.outputs.changed_file) }} singularity_version: - '3.8.1' + container: image: quay.io/singularity/singularity:v${{ matrix.singularity_version }} options: --privileged + + name: Check ${{ matrix.changed_file }} steps: - - name: Check out code for the container build - uses: actions/checkout@v1 - - - name: Build Container - env: - SINGULARITY_RECIPE: Singularity - OUTPUT_CONTAINER: container.sif - run: | - ls - if [ -f "${SINGULARITY_RECIPE}" ]; then - singularity build ${OUTPUT_CONTAINER} ${SINGULARITY_RECIPE} - else - echo "${SINGULARITY_RECIPE} is not found." + + - name: Check out code for the container builds + uses: actions/checkout@v2 + + - name: Continue if Singularity Recipe + run: | + # Continue if we have a changed Singularity recipe + if [[ "${{ matrix.changed_file }}" = *Singularity* ]]; then + echo "keepgoing=true" >> $GITHUB_ENV + fi + + - name: Build Container + if: ${{ env.keepgoing == 'true' }} + env: + recipe: ${{ matrix.changed_file }} + run: | + ls + if [ -f "${{ matrix.changed_file }}" ]; then + sudo -E singularity build container.sif ${{ matrix.changed_file }} + tag=$(echo "${recipe/Singularity\./}") + if [ "$tag" == "Singularity" ]; then + tag=latest + fi + # Build the container and name by tag + echo "Tag is $tag." + echo "tag=$tag" >> $GITHUB_ENV + else + echo "${{ matrix.changed_file }} is not found." echo "Present working directory: $PWD" ls - fi + fi + + - name: Login and Deploy Container + if: (github.event_name != 'pull_request') + env: + keepgoing: ${{ env.keepgoing }} + run: | + if [[ "${keepgoing}" == "true" ]]; then + echo ${{ secrets.GITHUB_TOKEN }} | singularity remote login ghcr.io -u ${GITHUB_USERNAME} --password-stdin + singularity push container.sif oras://${GITHUB_REPOSITORY}:${tag} + fi From 3db8badac4c0d0f51d0fca7c920b73f4e928cc57 Mon Sep 17 00:00:00 2001 From: vsoch Date: Tue, 3 Aug 2021 21:07:08 -0600 Subject: [PATCH 09/17] try doing a deploy Signed-off-by: vsoch --- README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3a0b4fd..d3f2094 100644 --- a/README.md +++ b/README.md @@ -14,13 +14,15 @@ the Singularity oras endpoint. **updated** August 2021, we can now push containers to the GitHub package registry! Woohoo! -There are two workflows configured on master that build a Singularity container: +There are three workflows configured as examples to build and deploy Singularity containers: -1. [native install](.github/workflows/native-install.yml) builds Singularity 3.x (with GoLang). -2. [docker image](.github/workfolws/container.yml) builds in a [docker image](https://quay.io/repository/singularity/singularity). +1. [native install](.github/workflows/native-install.yml) discovers Singularity* changed files, and builds Singularity 3.x (with GoLang) natively, deploys to GitHub packages. +2. [docker image](.github/workfolws/container.yml) discovers Singularity* changed files, and builds in a [docker image](https://quay.io/repository/singularity/singularity), also deploys to GitHub packages. +3. [manual deploy](.github/workfolws/manual-deploy.yml) takes a list of manually specified Singularity recipes (that aren't required to be changed), builds Singularity 3.x natively, and deploys to GitHub packages. -While the second option is faster to complete and a more simple workflow, it should be noted that docker runs with -`--privileged` which may lead to issues with the resulting container in a non privileged situation. +While the "build in a container" option is faster to complete and a more simple workflow, it should be noted that docker runs with +`--privileged` which may lead to issues with the resulting container in a non privileged situation. Also note that you +are free to mix and match the above recipes to your liking, or [open an issue](https://github.com/singularityhub/github-ci/issues) if you want to ask for help! **Why should this be managed via Github?** From f300493680041b2dc919c6bfe945583527c92192 Mon Sep 17 00:00:00 2001 From: vsoch Date: Tue, 3 Aug 2021 21:07:13 -0600 Subject: [PATCH 10/17] try doing a deploy Signed-off-by: vsoch --- .github/workflows/manual-deploy.yml | 99 +++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 .github/workflows/manual-deploy.yml diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml new file mode 100644 index 0000000..24fef29 --- /dev/null +++ b/.github/workflows/manual-deploy.yml @@ -0,0 +1,99 @@ +name: Singularity Build (manual) +on: + push: + + # This recipe shows how to manually define a matrix of singularity recipes (paths) to build + # Edit the branches here if you want to change deploy behavior + branches: + - main + + # Do the builds on all pull requests (to test them) + pull_request: [] + +jobs: + build-test-containers: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + recipe: ["Singularity"] + + name: Check ${{ matrix.recipe }} + steps: + - name: Continue if Singularity Recipe Exists + run: | + if [[ -f "${{ matrix.recipe }}" ]]; then + echo "keepgoing=true" >> $GITHUB_ENV + fi + + - name: Set up Go 1.13 + if: ${{ env.keepgoing == 'true' }} + uses: actions/setup-go@v1 + with: + go-version: 1.13 + id: go + + - name: Install Dependencies + if: ${{ env.keepgoing == 'true' }} + run: | + sudo apt-get update && sudo apt-get install -y \ + build-essential \ + libssl-dev \ + uuid-dev \ + libgpgme11-dev \ + squashfs-tools \ + libseccomp-dev \ + pkg-config + + - name: Install Singularity + if: ${{ env.keepgoing == 'true' }} + env: + SINGULARITY_VERSION: 3.8.1 + GOPATH: /tmp/go + + run: | + mkdir -p $GOPATH + sudo mkdir -p /usr/local/var/singularity/mnt && \ + mkdir -p $GOPATH/src/github.com/sylabs && \ + cd $GOPATH/src/github.com/sylabs && \ + wget -qO- https://github.com/sylabs/singularity/releases/download/v${SINGULARITY_VERSION}/singularity-ce-${SINGULARITY_VERSION}.tar.gz | \ + tar xzv && \ + cd singularity-ce-${SINGULARITY_VERSION} && \ + ./mconfig -p /usr/local && \ + make -C builddir && \ + sudo make -C builddir install + + - name: Check out code for the container build + if: ${{ env.keepgoing == 'true' }} + uses: actions/checkout@v2 + + - name: Build Container + if: ${{ env.keepgoing == 'true' }} + env: + recipe: ${{ matrix.recipe }} + run: | + ls + if [ -f "${{ matrix.recipe }}" ]; then + sudo -E singularity build container.sif ${{ matrix.recipe }} + tag=$(echo "${recipe/Singularity\./}") + if [ "$tag" == "Singularity" ]; then + tag=latest + fi + # Build the container and name by tag + echo "Tag is $tag." + echo "tag=$tag" >> $GITHUB_ENV + else + echo "${{ matrix.recipe }} is not found." + echo "Present working directory: $PWD" + ls + fi + + - name: Login and Deploy Container + #if: (github.event_name != 'pull_request') + env: + keepgoing: ${{ env.keepgoing }} + run: | + if [[ "${keepgoing}" == "true" ]]; then + echo ${{ secrets.GITHUB_TOKEN }} | singularity remote login ghcr.io -u ${GITHUB_USERNAME} --password-stdin + singularity push container.sif oras://${GITHUB_REPOSITORY}:${tag} + fi From 2de774191433caead91fa9bdecae7306112b9118 Mon Sep 17 00:00:00 2001 From: vsoch Date: Tue, 3 Aug 2021 21:09:43 -0600 Subject: [PATCH 11/17] keep going in wrong place" Signed-off-by: vsoch --- .github/workflows/manual-deploy.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml index 24fef29..35e6994 100644 --- a/.github/workflows/manual-deploy.yml +++ b/.github/workflows/manual-deploy.yml @@ -20,6 +20,10 @@ jobs: name: Check ${{ matrix.recipe }} steps: + + - name: Check out code for the container build + uses: actions/checkout@v2 + - name: Continue if Singularity Recipe Exists run: | if [[ -f "${{ matrix.recipe }}" ]]; then @@ -63,10 +67,6 @@ jobs: make -C builddir && \ sudo make -C builddir install - - name: Check out code for the container build - if: ${{ env.keepgoing == 'true' }} - uses: actions/checkout@v2 - - name: Build Container if: ${{ env.keepgoing == 'true' }} env: From 765350f6cdfb34d4b15882972f2f5886b1022bd9 Mon Sep 17 00:00:00 2001 From: vsoch Date: Tue, 3 Aug 2021 21:15:34 -0600 Subject: [PATCH 12/17] try faster deploy Signed-off-by: vsoch --- .github/workflows/container.yml | 4 ++-- .github/workflows/manual-deploy.yml | 2 +- README.md | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 0ca05b2..0d8a35a 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -72,11 +72,11 @@ jobs: fi - name: Login and Deploy Container - if: (github.event_name != 'pull_request') + #if: (github.event_name != 'pull_request') env: keepgoing: ${{ env.keepgoing }} run: | if [[ "${keepgoing}" == "true" ]]; then - echo ${{ secrets.GITHUB_TOKEN }} | singularity remote login ghcr.io -u ${GITHUB_USERNAME} --password-stdin + echo ${{ secrets.GITHUB_TOKEN }} | singularity remote login ghcr.io -u ${{ secrets.GHCR_USERNAME }} --password-stdin singularity push container.sif oras://${GITHUB_REPOSITORY}:${tag} fi diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml index 35e6994..2d803f9 100644 --- a/.github/workflows/manual-deploy.yml +++ b/.github/workflows/manual-deploy.yml @@ -89,7 +89,7 @@ jobs: fi - name: Login and Deploy Container - #if: (github.event_name != 'pull_request') + if: (github.event_name != 'pull_request') env: keepgoing: ${{ env.keepgoing }} run: | diff --git a/README.md b/README.md index d3f2094..eca72bb 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,7 @@ that writes the configuration. If you want to use the [GitHub package registry](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry) you'll need to follow the instructions there to enable packages for your organization, specifically "public" and "internal" packages should be allowed to be created. +You'll also want to add a username associated with your GitHub organization to the repository secret `GHCR_USERNAME` ### 1. Add Your Recipes From 0aa998485084b3a1fd90abd60e46ae6abb8c249f Mon Sep 17 00:00:00 2001 From: vsoch Date: Tue, 3 Aug 2021 21:17:35 -0600 Subject: [PATCH 13/17] try faster deploy Signed-off-by: vsoch --- .github/workflows/container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 0d8a35a..4819782 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -77,6 +77,6 @@ jobs: keepgoing: ${{ env.keepgoing }} run: | if [[ "${keepgoing}" == "true" ]]; then - echo ${{ secrets.GITHUB_TOKEN }} | singularity remote login ghcr.io -u ${{ secrets.GHCR_USERNAME }} --password-stdin + echo ${{ secrets.GITHUB_TOKEN }} | singularity remote login ghcr.io -u ${{ secrets.GHCR_USERNAME }} --password-stdin oras://ghrc.io singularity push container.sif oras://${GITHUB_REPOSITORY}:${tag} fi From 2a581db9c7b30bbe4facdfa8e585824d60ed7b90 Mon Sep 17 00:00:00 2001 From: vsoch Date: Tue, 3 Aug 2021 21:19:23 -0600 Subject: [PATCH 14/17] try faster deploy Signed-off-by: vsoch --- .github/workflows/container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 4819782..5c8827e 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -77,6 +77,6 @@ jobs: keepgoing: ${{ env.keepgoing }} run: | if [[ "${keepgoing}" == "true" ]]; then - echo ${{ secrets.GITHUB_TOKEN }} | singularity remote login ghcr.io -u ${{ secrets.GHCR_USERNAME }} --password-stdin oras://ghrc.io + echo ${{ secrets.GITHUB_TOKEN }} | singularity remote login -u ${{ secrets.GHCR_USERNAME }} --password-stdin oras://ghrc.io singularity push container.sif oras://${GITHUB_REPOSITORY}:${tag} fi From 936b4c1064d07146ff05290ca36f1dac15d79727 Mon Sep 17 00:00:00 2001 From: vsoch Date: Tue, 3 Aug 2021 21:21:30 -0600 Subject: [PATCH 15/17] try faster deploy Signed-off-by: vsoch --- .github/workflows/container.yml | 2 +- .github/workflows/manual-deploy.yml | 2 +- .github/workflows/native-install.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 5c8827e..b23a917 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -77,6 +77,6 @@ jobs: keepgoing: ${{ env.keepgoing }} run: | if [[ "${keepgoing}" == "true" ]]; then - echo ${{ secrets.GITHUB_TOKEN }} | singularity remote login -u ${{ secrets.GHCR_USERNAME }} --password-stdin oras://ghrc.io + echo ${{ secrets.GITHUB_TOKEN }} | singularity remote login -u ${{ secrets.GHCR_USERNAME }} --password-stdin oras://ghcr.io singularity push container.sif oras://${GITHUB_REPOSITORY}:${tag} fi diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml index 2d803f9..90c2f22 100644 --- a/.github/workflows/manual-deploy.yml +++ b/.github/workflows/manual-deploy.yml @@ -94,6 +94,6 @@ jobs: keepgoing: ${{ env.keepgoing }} run: | if [[ "${keepgoing}" == "true" ]]; then - echo ${{ secrets.GITHUB_TOKEN }} | singularity remote login ghcr.io -u ${GITHUB_USERNAME} --password-stdin + echo ${{ secrets.GITHUB_TOKEN }} | singularity remote login -u ${{ secrets.GHCR_USERNAME }} --password-stdin oras://ghcr.io singularity push container.sif oras://${GITHUB_REPOSITORY}:${tag} fi diff --git a/.github/workflows/native-install.yml b/.github/workflows/native-install.yml index d51bdcc..d380657 100644 --- a/.github/workflows/native-install.yml +++ b/.github/workflows/native-install.yml @@ -108,6 +108,6 @@ jobs: keepgoing: ${{ env.keepgoing }} run: | if [[ "${keepgoing}" == "true" ]]; then - echo ${{ secrets.GITHUB_TOKEN }} | singularity remote login ghcr.io -u ${GITHUB_USERNAME} --password-stdin + echo ${{ secrets.GITHUB_TOKEN }} | singularity remote login -u ${{ secrets.GHCR_USERNAME }} --password-stdin oras://ghcr.io singularity push container.sif oras://${GITHUB_REPOSITORY}:${tag} fi From 0271fba19f1301c62f87a3df5fb729875a8fabc2 Mon Sep 17 00:00:00 2001 From: vsoch Date: Tue, 3 Aug 2021 21:23:57 -0600 Subject: [PATCH 16/17] one moretry Signed-off-by: vsoch --- .github/workflows/container.yml | 2 +- .github/workflows/manual-deploy.yml | 2 +- .github/workflows/native-install.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index b23a917..b8a0093 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -78,5 +78,5 @@ jobs: run: | if [[ "${keepgoing}" == "true" ]]; then echo ${{ secrets.GITHUB_TOKEN }} | singularity remote login -u ${{ secrets.GHCR_USERNAME }} --password-stdin oras://ghcr.io - singularity push container.sif oras://${GITHUB_REPOSITORY}:${tag} + singularity push container.sif oras://ghcr.io/${GITHUB_REPOSITORY}:${tag} fi diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml index 90c2f22..77dd72a 100644 --- a/.github/workflows/manual-deploy.yml +++ b/.github/workflows/manual-deploy.yml @@ -95,5 +95,5 @@ jobs: run: | if [[ "${keepgoing}" == "true" ]]; then echo ${{ secrets.GITHUB_TOKEN }} | singularity remote login -u ${{ secrets.GHCR_USERNAME }} --password-stdin oras://ghcr.io - singularity push container.sif oras://${GITHUB_REPOSITORY}:${tag} + singularity push container.sif oras://ghcr.io/${GITHUB_REPOSITORY}:${tag} fi diff --git a/.github/workflows/native-install.yml b/.github/workflows/native-install.yml index d380657..8ef86d7 100644 --- a/.github/workflows/native-install.yml +++ b/.github/workflows/native-install.yml @@ -109,5 +109,5 @@ jobs: run: | if [[ "${keepgoing}" == "true" ]]; then echo ${{ secrets.GITHUB_TOKEN }} | singularity remote login -u ${{ secrets.GHCR_USERNAME }} --password-stdin oras://ghcr.io - singularity push container.sif oras://${GITHUB_REPOSITORY}:${tag} + singularity push container.sif oras://ghcr.io/${GITHUB_REPOSITORY}:${tag} fi From 90aae322d19add3086f515448b7379aaa83d4494 Mon Sep 17 00:00:00 2001 From: vsoch Date: Tue, 3 Aug 2021 21:34:10 -0600 Subject: [PATCH 17/17] final tweak to readme Signed-off-by: vsoch --- .github/workflows/container.yml | 2 +- README.md | 77 ++++++++++++++------------------- 2 files changed, 33 insertions(+), 46 deletions(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index b8a0093..f5aa174 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -72,7 +72,7 @@ jobs: fi - name: Login and Deploy Container - #if: (github.event_name != 'pull_request') + if: (github.event_name != 'pull_request') env: keepgoing: ${{ env.keepgoing }} run: | diff --git a/README.md b/README.md index eca72bb..a20c1a0 100644 --- a/README.md +++ b/README.md @@ -29,10 +29,8 @@ are free to mix and match the above recipes to your liking, or [open an issue](h Github, by way of easy integration with **native** continuous integration, is an easy way to have a workflow set up where multiple people can collaborate on a container recipe, the recipe can be tested (with whatever testing you need), discussed in pull requests, -and tested on merge to master. If you add additional steps in the [build workflow](.github/workflows/native-install.yml) -you can also use [Singularity Registry Client](http://singularityhub.github.io/sregistry-cli) to push your container to a -[Singularity Registry Server](https://singularityhub.github.io/sregistry) or other -cloud storage. +and tested on merge to master. Further, now with GitHub packages we can push our containers +directly to the GitHub package registry! **Why should I use this instead of a service?** @@ -44,48 +42,23 @@ that writes the configuration. ## Quick Start -### 0. Enable Packages +### 1. Enable Packages If you want to use the [GitHub package registry](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry) you'll need to follow the instructions there to enable packages for your organization, specifically "public" and "internal" packages should be allowed to be created. You'll also want to add a username associated with your GitHub organization to the repository secret `GHCR_USERNAME` -### 1. Add Your Recipes +### 2. Add Your Recipes -Add your Singularity recipes to this repository, and edit the [build workflow](.github/workflows/native-install.yml) -section where the container is built. The default will look for a recipe file called -"Singularity" in the base of the respository, [as we have here](Singularity). -For example, here is the default: +Add your Singularity recipes to this repository, which should be named `Singularity.` +or just `Singularity` to follow the previously published convention. You can then choose your file in [.github/workflows](.github/workflows). +If you choose the `manual-deploy.yml` you can manually specify recipes in the matrix variable "recipe." +If you choose either of the other two workflows, changed files that start with Singularity.* will +be automatically detected and built. -```yaml - - name: Build Container - env: - SINGULARITY_RECIPE: Singularity - OUTPUT_CONTAINER: container.sif - run: | - ls - if [ -f "${SINGULARITY_RECIPE}" ]; then - sudo -E singularity build ${OUTPUT_CONTAINER} ${SINGULARITY_RECIPE} - else - echo "${SINGULARITY_RECIPE} is not found." - echo "Present working directory: $PWD" - ls - fi -``` - -And I could easily change that to build as many recipes as I like, and -even disregard the environment variable. - -```yaml - - name: Build Container - run: | - sudo -E singularity build smokey.sif Singularity.smokey - sudo -E singularity build toasty.sif marshmallow/Singularity.toasty -``` - -### 2. Test your Container +### 3. Test your Container -Importantly, then you should test your container! Whether that's running it, +Importantly, we suggest that you add some steps to test your container! Whether that's running it, exec'ing a custom command, or invoking the test command, there is more than one way to eat a reeses: @@ -97,13 +70,15 @@ one way to eat a reeses: singularity run toasty.sif ``` -### 3. Check Triggers +This step is not provided in the workflows, but it's recommended that you think about it and add if necessary. + +### 4. Check Triggers The workflow files each have a section at the top that indicates when the workflow will trigger. By default, we will do builds on pull requests, and deploys on pushes to a main branch. If you want to change this logic, edit the top of the recipe files. -### 4. Push to a registry +### 5. Push to a registry If you are good with GitHub packages, then you are good to go! Otherwise, if you want to push to other kinds of storage, you can install the [Singularity Registry Client](http://singularityhub.github.io/sregistry-cli) and push to your cloud storage of choice! You will want to add python and python-dev to the dependency @@ -136,19 +111,31 @@ And then install and use sregistry client. Here are many examples: ``` See the [clients page](https://singularityhub.github.io/sregistry-cli/clients) for all the options. -Remember that the example workflow is intended to run on push to master, so you might want to have -a similar one (without deployment) that runs on pull_request, or other events. -See [here](https://help.github.com/en/articles/about-github-actions#core-concepts-for-github-actions) +If you want to change the recipe triggers, see [here](https://help.github.com/en/articles/about-github-actions#core-concepts-for-github-actions) for getting started with GitHub actions, and [please open an issue](https://www.github.com/singularityhub/github-ci/issues) if you need any help. +### 6. Pull Your Container! + +The example container here is published to [singularithub/github-ci](https://github.com/singularityhub/github-ci/pkgs/container/github-ci) +and can be pulled as follows: + +```bash +$ singularity pull oras://ghcr.io/singularityhub/github-ci:latest +INFO: Downloading oras image +$ ls +github-ci_latest.sif img README.md Singularity + +$ ./github-ci_latest.sif +Hold me closer... tiny container :) :D +``` ## Other Options You can customize this base recipe in so many ways! For example: - - If you are building a Docker container, you can start with the docker base, build the container, and then pull it down into Singularity and test it. Successful builds can be pushed to Docker Hub, and then you know they will pull okay to a Singularity container. - - The action can be configured with a Matrix to run builds on multiple platforms. + - If you want to build a Docker container and pull down to Singularity, that's a good approach too! We have a [container-builder-template](github.com/autamus/container-builder-template) to help you authenticate with several popular registries. + - The action matrix can be extended to run builds on multiple platforms. - You can also do the same, but test multiple versions of Singularity. Have fun!