diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index fc62354..f5aa174 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -1,31 +1,82 @@ name: Singularity Build (docker) -on: [push] -jobs: +on: + push: + + # Edit the branches here if you want to change deploy behavior + branches: + - main + + # Do the builds on all pull requests (to test them) + pull_request: [] - build: - name: Build - runs-on: ubuntu-18.04 +jobs: + changes: + name: "Changed Singularity Recipes" + runs-on: ubuntu-latest + outputs: + changed_file: ${{ steps.files.outputs.added_modified }} + steps: + - id: files + uses: jitterbit/get-changed-files@b17fbb00bdc0c0f63fcf166580804b4d2cdc2a42 + with: + format: 'json' + + build-test-containers: + needs: + - changes + runs-on: ubuntu-latest strategy: + # Keep going on other deployments if anything bloops + fail-fast: false matrix: + changed_file: ${{ fromJson(needs.changes.outputs.changed_file) }} singularity_version: - - '3.5.3' + - '3.8.1' + container: image: quay.io/singularity/singularity:v${{ matrix.singularity_version }} options: --privileged + + name: Check ${{ matrix.changed_file }} steps: - - name: Check out code for the container build - uses: actions/checkout@v1 - - - name: Build Container - env: - SINGULARITY_RECIPE: Singularity - OUTPUT_CONTAINER: container.sif - run: | - ls - if [ -f "${SINGULARITY_RECIPE}" ]; then - singularity build ${OUTPUT_CONTAINER} ${SINGULARITY_RECIPE} - else - echo "${SINGULARITY_RECIPE} is not found." + + - name: Check out code for the container builds + uses: actions/checkout@v2 + + - name: Continue if Singularity Recipe + run: | + # Continue if we have a changed Singularity recipe + if [[ "${{ matrix.changed_file }}" = *Singularity* ]]; then + echo "keepgoing=true" >> $GITHUB_ENV + fi + + - name: Build Container + if: ${{ env.keepgoing == 'true' }} + env: + recipe: ${{ matrix.changed_file }} + run: | + ls + if [ -f "${{ matrix.changed_file }}" ]; then + sudo -E singularity build container.sif ${{ matrix.changed_file }} + tag=$(echo "${recipe/Singularity\./}") + if [ "$tag" == "Singularity" ]; then + tag=latest + fi + # Build the container and name by tag + echo "Tag is $tag." + echo "tag=$tag" >> $GITHUB_ENV + else + echo "${{ matrix.changed_file }} is not found." echo "Present working directory: $PWD" ls - fi + fi + + - name: Login and Deploy Container + if: (github.event_name != 'pull_request') + env: + keepgoing: ${{ env.keepgoing }} + run: | + if [[ "${keepgoing}" == "true" ]]; then + echo ${{ secrets.GITHUB_TOKEN }} | singularity remote login -u ${{ secrets.GHCR_USERNAME }} --password-stdin oras://ghcr.io + singularity push container.sif oras://ghcr.io/${GITHUB_REPOSITORY}:${tag} + fi diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml new file mode 100644 index 0000000..77dd72a --- /dev/null +++ b/.github/workflows/manual-deploy.yml @@ -0,0 +1,99 @@ +name: Singularity Build (manual) +on: + push: + + # This recipe shows how to manually define a matrix of singularity recipes (paths) to build + # Edit the branches here if you want to change deploy behavior + branches: + - main + + # Do the builds on all pull requests (to test them) + pull_request: [] + +jobs: + build-test-containers: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + recipe: ["Singularity"] + + name: Check ${{ matrix.recipe }} + steps: + + - name: Check out code for the container build + uses: actions/checkout@v2 + + - name: Continue if Singularity Recipe Exists + run: | + if [[ -f "${{ matrix.recipe }}" ]]; then + echo "keepgoing=true" >> $GITHUB_ENV + fi + + - name: Set up Go 1.13 + if: ${{ env.keepgoing == 'true' }} + uses: actions/setup-go@v1 + with: + go-version: 1.13 + id: go + + - name: Install Dependencies + if: ${{ env.keepgoing == 'true' }} + run: | + sudo apt-get update && sudo apt-get install -y \ + build-essential \ + libssl-dev \ + uuid-dev \ + libgpgme11-dev \ + squashfs-tools \ + libseccomp-dev \ + pkg-config + + - name: Install Singularity + if: ${{ env.keepgoing == 'true' }} + env: + SINGULARITY_VERSION: 3.8.1 + GOPATH: /tmp/go + + run: | + mkdir -p $GOPATH + sudo mkdir -p /usr/local/var/singularity/mnt && \ + mkdir -p $GOPATH/src/github.com/sylabs && \ + cd $GOPATH/src/github.com/sylabs && \ + wget -qO- https://github.com/sylabs/singularity/releases/download/v${SINGULARITY_VERSION}/singularity-ce-${SINGULARITY_VERSION}.tar.gz | \ + tar xzv && \ + cd singularity-ce-${SINGULARITY_VERSION} && \ + ./mconfig -p /usr/local && \ + make -C builddir && \ + sudo make -C builddir install + + - name: Build Container + if: ${{ env.keepgoing == 'true' }} + env: + recipe: ${{ matrix.recipe }} + run: | + ls + if [ -f "${{ matrix.recipe }}" ]; then + sudo -E singularity build container.sif ${{ matrix.recipe }} + tag=$(echo "${recipe/Singularity\./}") + if [ "$tag" == "Singularity" ]; then + tag=latest + fi + # Build the container and name by tag + echo "Tag is $tag." + echo "tag=$tag" >> $GITHUB_ENV + else + echo "${{ matrix.recipe }} is not found." + echo "Present working directory: $PWD" + ls + fi + + - name: Login and Deploy Container + if: (github.event_name != 'pull_request') + env: + keepgoing: ${{ env.keepgoing }} + run: | + if [[ "${keepgoing}" == "true" ]]; then + echo ${{ secrets.GITHUB_TOKEN }} | singularity remote login -u ${{ secrets.GHCR_USERNAME }} --password-stdin oras://ghcr.io + singularity push container.sif oras://ghcr.io/${GITHUB_REPOSITORY}:${tag} + fi diff --git a/.github/workflows/native-install.yml b/.github/workflows/native-install.yml index f3a035f..8ef86d7 100644 --- a/.github/workflows/native-install.yml +++ b/.github/workflows/native-install.yml @@ -1,59 +1,113 @@ name: Singularity Build (native) -on: [push] +on: + push: + + # Edit the branches here if you want to change deploy behavior + branches: + - main + + # Do the builds on all pull requests (to test them) + pull_request: [] + jobs: + changes: + name: "Changed Singularity Recipes" + runs-on: ubuntu-latest + outputs: + changed_file: ${{ steps.files.outputs.added_modified }} + steps: + - id: files + uses: jitterbit/get-changed-files@b17fbb00bdc0c0f63fcf166580804b4d2cdc2a42 + with: + format: 'json' + + build-test-containers: + needs: + - changes + runs-on: ubuntu-latest + strategy: + # Keep going on other deployments if anything bloops + fail-fast: false + matrix: + changed_file: ${{ fromJson(needs.changes.outputs.changed_file) }} - build: - name: Build - runs-on: ubuntu-18.04 + name: Check ${{ matrix.changed_file }} steps: + - name: Continue if Singularity Recipe + run: | + # Continue if we have a changed Singularity recipe + if [[ "${{ matrix.changed_file }}" = *Singularity* ]]; then + echo "keepgoing=true" >> $GITHUB_ENV + fi + + - name: Set up Go 1.13 + if: ${{ env.keepgoing == 'true' }} + uses: actions/setup-go@v1 + with: + go-version: 1.13 + id: go - - name: Set up Go 1.13 - uses: actions/setup-go@v1 - with: - go-version: 1.13 - id: go - - - name: Install Dependencies - run: | - sudo apt-get update && sudo apt-get install -y \ - build-essential \ - libssl-dev \ - uuid-dev \ - libgpgme11-dev \ - squashfs-tools \ - libseccomp-dev \ - pkg-config - - - name: Install Singularity - env: - SINGULARITY_VERSION: 3.8.1 - GOPATH: /tmp/go - run: | - mkdir -p $GOPATH - sudo mkdir -p /usr/local/var/singularity/mnt && \ - mkdir -p $GOPATH/src/github.com/sylabs && \ - cd $GOPATH/src/github.com/sylabs && \ - - wget -qO- https://github.com/sylabs/singularity/releases/download/v${SINGULARITY_VERSION}/singularity-ce-${SINGULARITY_VERSION}.tar.gz | \ - tar xzv && \ - cd singularity-ce-${SINGULARITY_VERSION} && \ - ./mconfig -p /usr/local && \ - make -C builddir && \ - sudo make -C builddir install - - - name: Check out code for the container build - uses: actions/checkout@v1 - - - name: Build Container - env: - SINGULARITY_RECIPE: Singularity - OUTPUT_CONTAINER: container.sif - run: | - ls - if [ -f "${SINGULARITY_RECIPE}" ]; then - sudo -E singularity build ${OUTPUT_CONTAINER} ${SINGULARITY_RECIPE} - else - echo "${SINGULARITY_RECIPE} is not found." + - name: Install Dependencies + if: ${{ env.keepgoing == 'true' }} + run: | + sudo apt-get update && sudo apt-get install -y \ + build-essential \ + libssl-dev \ + uuid-dev \ + libgpgme11-dev \ + squashfs-tools \ + libseccomp-dev \ + pkg-config + + - name: Install Singularity + if: ${{ env.keepgoing == 'true' }} + env: + SINGULARITY_VERSION: 3.8.1 + GOPATH: /tmp/go + + run: | + mkdir -p $GOPATH + sudo mkdir -p /usr/local/var/singularity/mnt && \ + mkdir -p $GOPATH/src/github.com/sylabs && \ + cd $GOPATH/src/github.com/sylabs && \ + wget -qO- https://github.com/sylabs/singularity/releases/download/v${SINGULARITY_VERSION}/singularity-ce-${SINGULARITY_VERSION}.tar.gz | \ + tar xzv && \ + cd singularity-ce-${SINGULARITY_VERSION} && \ + ./mconfig -p /usr/local && \ + make -C builddir && \ + sudo make -C builddir install + + - name: Check out code for the container build + if: ${{ env.keepgoing == 'true' }} + uses: actions/checkout@v2 + + - name: Build Container + if: ${{ env.keepgoing == 'true' }} + env: + recipe: ${{ matrix.changed_file }} + run: | + ls + if [ -f "${{ matrix.changed_file }}" ]; then + sudo -E singularity build container.sif ${{ matrix.changed_file }} + tag=$(echo "${recipe/Singularity\./}") + if [ "$tag" == "Singularity" ]; then + tag=latest + fi + # Build the container and name by tag + echo "Tag is $tag." + echo "tag=$tag" >> $GITHUB_ENV + else + echo "${{ matrix.changed_file }} is not found." echo "Present working directory: $PWD" ls - fi + fi + + - name: Login and Deploy Container + if: (github.event_name != 'pull_request') + env: + keepgoing: ${{ env.keepgoing }} + run: | + if [[ "${keepgoing}" == "true" ]]; then + echo ${{ secrets.GITHUB_TOKEN }} | singularity remote login -u ${{ secrets.GHCR_USERNAME }} --password-stdin oras://ghcr.io + singularity push container.sif oras://ghcr.io/${GITHUB_REPOSITORY}:${tag} + fi diff --git a/README.md b/README.md index d10b1f1..a20c1a0 100644 --- a/README.md +++ b/README.md @@ -5,29 +5,32 @@ This is a simple example of how you can achieve: - version control of your recipes - - versioning to include image hash *and* commit id - build of associated container and - - (optional) push to a storage endpoint + - push to a storage endpoint -for a reproducible build workflow. +for a reproducible build workflow! By default, we will build on all pull requests and deploy +on push to main. The containers will go to an enabled GitHub package registry thanks to +the Singularity oras endpoint. -There are two workflows configured on master that build a container: +**updated** August 2021, we can now push containers to the GitHub package registry! Woohoo! -1. [native install](.github/workflows/native-install.yml) builds Singularity 3.x (with GoLang). -2. [docker image](.github/workfolws/container.yml) builds in a [docker image](https://quay.io/repository/singularity/singularity). +There are three workflows configured as examples to build and deploy Singularity containers: -While the second option is faster to complete and a more simple workflow, it should be noted that docker runs with -`--privileged` which may lead to issues with the resulting container in a non privileged situation. +1. [native install](.github/workflows/native-install.yml) discovers Singularity* changed files, and builds Singularity 3.x (with GoLang) natively, deploys to GitHub packages. +2. [docker image](.github/workfolws/container.yml) discovers Singularity* changed files, and builds in a [docker image](https://quay.io/repository/singularity/singularity), also deploys to GitHub packages. +3. [manual deploy](.github/workfolws/manual-deploy.yml) takes a list of manually specified Singularity recipes (that aren't required to be changed), builds Singularity 3.x natively, and deploys to GitHub packages. + +While the "build in a container" option is faster to complete and a more simple workflow, it should be noted that docker runs with +`--privileged` which may lead to issues with the resulting container in a non privileged situation. Also note that you +are free to mix and match the above recipes to your liking, or [open an issue](https://github.com/singularityhub/github-ci/issues) if you want to ask for help! **Why should this be managed via Github?** Github, by way of easy integration with **native** continuous integration, is an easy way to have a workflow set up where multiple people can collaborate on a container recipe, the recipe can be tested (with whatever testing you need), discussed in pull requests, -and tested on merge to master. If you add additional steps in the [build workflow](.github/workflows/native-install.yml) -you can also use [Singularity Registry Client](http://singularityhub.github.io/sregistry-cli) to push your container to a -[Singularity Registry Server](https://singularityhub.github.io/sregistry) or other -cloud storage. +and tested on merge to master. Further, now with GitHub packages we can push our containers +directly to the GitHub package registry! **Why should I use this instead of a service?** @@ -39,42 +42,23 @@ that writes the configuration. ## Quick Start -### 1. Add Your Recipes - -Add your Singularity recipes to this repository, and edit the [build workflow](.github/workflows/native-install.yml) -section where the container is built. The default will look for a recipe file called -"Singularity" in the base of the respository, [as we have here](Singularity). -For example, here is the default: +### 1. Enable Packages -```yaml - - name: Build Container - env: - SINGULARITY_RECIPE: Singularity - OUTPUT_CONTAINER: container.sif - run: | - ls - if [ -f "${SINGULARITY_RECIPE}" ]; then - sudo -E singularity build ${OUTPUT_CONTAINER} ${SINGULARITY_RECIPE} - else - echo "${SINGULARITY_RECIPE} is not found." - echo "Present working directory: $PWD" - ls - fi -``` +If you want to use the [GitHub package registry](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry) +you'll need to follow the instructions there to enable packages for your organization, specifically "public" and "internal" packages should be allowed to be created. +You'll also want to add a username associated with your GitHub organization to the repository secret `GHCR_USERNAME` -And I could easily change that to build as many recipes as I like, and -even disregard the environment variable. +### 2. Add Your Recipes -```yaml - - name: Build Container - run: | - sudo -E singularity build smokey.sif Singularity.smokey - sudo -E singularity build toasty.sif marshmallow/Singularity.toasty -``` +Add your Singularity recipes to this repository, which should be named `Singularity.` +or just `Singularity` to follow the previously published convention. You can then choose your file in [.github/workflows](.github/workflows). +If you choose the `manual-deploy.yml` you can manually specify recipes in the matrix variable "recipe." +If you choose either of the other two workflows, changed files that start with Singularity.* will +be automatically detected and built. -### 2. Test your Container +### 3. Test your Container -Importantly, then you should test your container! Whether that's running it, +Importantly, we suggest that you add some steps to test your container! Whether that's running it, exec'ing a custom command, or invoking the test command, there is more than one way to eat a reeses: @@ -86,9 +70,18 @@ one way to eat a reeses: singularity run toasty.sif ``` -### 3. Push to a registry +This step is not provided in the workflows, but it's recommended that you think about it and add if necessary. -You might be done there. But if not, you can install [Singularity Registry Client](http://singularityhub.github.io/sregistry-cli) and push to your cloud storage of choice! You will want to add python and python-dev to the dependency +### 4. Check Triggers + +The workflow files each have a section at the top that indicates when the workflow will +trigger. By default, we will do builds on pull requests, and deploys on pushes to a main +branch. If you want to change this logic, edit the top of the recipe files. + +### 5. Push to a registry + +If you are good with GitHub packages, then you are good to go! Otherwise, +if you want to push to other kinds of storage, you can install the [Singularity Registry Client](http://singularityhub.github.io/sregistry-cli) and push to your cloud storage of choice! You will want to add python and python-dev to the dependency install: ```yaml @@ -118,19 +111,31 @@ And then install and use sregistry client. Here are many examples: ``` See the [clients page](https://singularityhub.github.io/sregistry-cli/clients) for all the options. -Remember that the example workflow is intended to run on push to master, so you might want to have -a similar one (without deployment) that runs on pull_request, or other events. -See [here](https://help.github.com/en/articles/about-github-actions#core-concepts-for-github-actions) +If you want to change the recipe triggers, see [here](https://help.github.com/en/articles/about-github-actions#core-concepts-for-github-actions) for getting started with GitHub actions, and [please open an issue](https://www.github.com/singularityhub/github-ci/issues) if you need any help. +### 6. Pull Your Container! + +The example container here is published to [singularithub/github-ci](https://github.com/singularityhub/github-ci/pkgs/container/github-ci) +and can be pulled as follows: + +```bash +$ singularity pull oras://ghcr.io/singularityhub/github-ci:latest +INFO: Downloading oras image +$ ls +github-ci_latest.sif img README.md Singularity + +$ ./github-ci_latest.sif +Hold me closer... tiny container :) :D +``` ## Other Options You can customize this base recipe in so many ways! For example: - - If you are building a Docker container, you can start with the docker base, build the container, and then pull it down into Singularity and test it. Successful builds can be pushed to Docker Hub, and then you know they will pull okay to a Singularity container. - - The action can be configured with a Matrix to run builds on multiple platforms. + - If you want to build a Docker container and pull down to Singularity, that's a good approach too! We have a [container-builder-template](github.com/autamus/container-builder-template) to help you authenticate with several popular registries. + - The action matrix can be extended to run builds on multiple platforms. - You can also do the same, but test multiple versions of Singularity. Have fun! diff --git a/Singularity b/Singularity index e610074..140be58 100644 --- a/Singularity +++ b/Singularity @@ -2,4 +2,4 @@ Bootstrap: docker From: busybox:latest %runscript -echo "Hold me closer... tiny container :D" +echo "Hold me closer... tiny container :) :D"