diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index fd9c3dc..286c0b2 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -1,7 +1,7 @@ -# This workflow builds and deploys a GitHub Actions self hosted runner to Google Kubernetes Engine. +# This workflow builds and deploys a GitHub Actions self hosted runner to Anthos GKE. # # REQUIREMENTS: -# - "Google Kubernetes Engine" setup steps in README, including adding appropriate secrets to repository +# - Setup steps in README, including adding appropriate secrets to repository name: Self Hosted Runner CI/CD on: @@ -11,51 +11,36 @@ on: pull_request: env: + GITHUB_REPO: ${{ secrets.REPO }} # Should be a private repository, see https://help.github.com/en/actions/hosting-your-own-runners/adding-self-hosted-runners + TOKEN: ${{ secrets.TOKEN }} # Personal Access Token used to register and deregister runners since GITHUB_TOKEN is only valid for one hour. GCP_PROJECT: ${{ secrets.GCP_PROJECT }} GKE_CLUSTER: self-hosted-runner-test-cluster GKE_SECRETS: self-hosted-runner-creds - GCP_REGION: us-central1 + GCP_ZONE: us-west1-a IMAGE: self-hosted-runner - GITHUB_REPO: owner/repo # should be a private repository, see https://help.github.com/en/actions/hosting-your-own-runners/adding-self-hosted-runners - TOKEN: ${{ secrets.TOKEN }} # Personal Access Token used to register and deregister runners. GITHUB_TOKEN isn't good for most use cases because it is only valid for one hour. jobs: - # Build and push image to GCR - publish: + # Test and build + test: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v2 - # Configure Google Cloud credentials - - name: Configure Google Cloud credentials - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master # until 0.2.0 release is available - with: - version: 275.0.0 - service_account_email: ${{ secrets.GCP_EMAIL }} - service_account_key: ${{ secrets.GCP_KEY }} - - # Use gcloud CLI to configure docker authentication for subsequent push - - run: | - gcloud auth configure-docker - - # Build Docker image - - name: Build image - run: docker build . -t gcr.io/"$GCP_PROJECT"/"$IMAGE":latest + # Insert other testing and linting steps here, eg. container analysis (https://cloud.google.com/container-registry/docs/container-analysis) - # Push the Docker image to Google Container Registry - - name: Publish - run: | - docker push gcr.io/$GCP_PROJECT/$IMAGE:latest + # Ensure Docker image can be built + - name: Build image + run: docker build . -t gcr.io/"$GCP_PROJECT"/"$IMAGE":"$GITHUB_SHA" - # Apply Kubernetes manifest to deploy image to cluster + # Build and publish image, apply Kubernetes manifest to deploy image to cluster deploy: - needs: publish + needs: test runs-on: ubuntu-latest # Only on push to master (a merged PR) - if: github.event_name == 'push' + if: github.ref == 'refs/heads/master' && github.event_name == 'push' steps: - name: Checkout @@ -63,19 +48,36 @@ jobs: # Configure Google Cloud credentials - name: Configure Google Cloud credentials - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master # until 0.2.0 release is available + uses: google-github-actions/setup-gcloud@v0.2.1 with: - version: 275.0.0 - service_account_email: ${{ secrets.GCP_EMAIL }} service_account_key: ${{ secrets.GCP_KEY }} + project_id: ${{ secrets.GCP_PROJECT }} + + # Use gcloud CLI to configure docker authentication for subsequent push + - run: | + gcloud auth configure-docker + + # Build Docker image + - name: Build image + run: docker build . -t gcr.io/"$GCP_PROJECT"/"$IMAGE":"$GITHUB_SHA" # Configure Kubernetes - name: Configure Kubernetes run: | - gcloud container clusters get-credentials $GKE_CLUSTER --region $GCP_REGION --project $GCP_PROJECT + gcloud container clusters get-credentials $GKE_CLUSTER --zone $GCP_ZONE + + # Push the Docker image to Google Container Registry + - name: Publish + run: | + docker push gcr.io/"$GCP_PROJECT"/"$IMAGE":"$GITHUB_SHA" + + # Set up kustomize + - name: Set up Kustomize + run: |- + curl -sfLo kustomize https://github.com/kubernetes-sigs/kustomize/releases/download/v3.1.0/kustomize_3.1.0_linux_amd64 + chmod u+x ./kustomize # Optional: Update secrets in Google Kubernetes Engine (GKE) cluster (to change repo the runner is available to or authentication token) - # Note that GITHUB_TOKEN is only valid for one hour. - name: Update secrets run: | kubectl get secrets $GKE_SECRETS -o json | @@ -85,5 +87,7 @@ jobs: # Deploy to Google Kubernetes Engine (GKE) cluster - name: Deploy - run: | - kubectl apply -f deployment.yml + run: |- + ./kustomize edit set image gcr.io/PROJECT_ID/IMAGE:TAG=gcr.io/$GCP_PROJECT/$IMAGE:$GITHUB_SHA + ./kustomize build . | kubectl apply -f - + diff --git a/Dockerfile b/Dockerfile index 85ea7fa..e29fb45 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ -FROM node:12 +FROM ubuntu:18.04 # Update and download dependencies RUN apt-get update RUN apt-get install -y libssl-dev curl iputils-ping jq wget -# Docker in docker for container builds on Kubernetes. Otherwise, follow this guidance: https://jpetazzo.github.io/2015/09/03/do-not-use-docker-in-docker-for-ci/. +# Download Docker for container builds on Kubernetes ENV DOCKER_CHANNEL stable ENV DOCKER_VERSION 18.09.1 RUN wget -O docker.tgz "https://download.docker.com/linux/static/${DOCKER_CHANNEL}/x86_64/docker-${DOCKER_VERSION}.tgz" && \ @@ -15,8 +15,19 @@ RUN wget -O docker.tgz "https://download.docker.com/linux/static/${DOCKER_CHANNE RUN mkdir ./actions-runner WORKDIR /home/actions-runner -COPY startup.sh . +# Download Actions runner +# https://github.com/terraform-google-modules/terraform-google-github-actions-runners/blob/598a38a72b7bbaf56be431c07de04752c521fd60/examples/gh-runner-gke-dind/Dockerfile#L28-L31 +ARG GH_RUNNER_VERSION="2.267.1" +RUN curl -o actions.tar.gz --location "https://github.com/actions/runner/releases/download/v${GH_RUNNER_VERSION}/actions-runner-linux-x64-${GH_RUNNER_VERSION}.tar.gz" && \ + tar -zxf actions.tar.gz && \ + rm -f actions.tar.gz + +# Install dependencies +RUN ./bin/installdependencies.sh -EXPOSE 8080 +# Allow runner to run as root +ENV RUNNER_ALLOW_RUNASROOT=1 + +COPY startup.sh . ENTRYPOINT ["./startup.sh"] diff --git a/README.md b/README.md index 5372703..826ef91 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,23 @@ # GitHub Actions Self Hosted Runners on Anthos -This project shows an _example_ configuration and usage of GitHub Actions self hosted runners on Anthos, using the [self hosted runners API](https://developer.github.com/v3/actions/self_hosted_runners/). Under active development 🧪. +> Build and deploy GitHub Actions [self hosted runners](https://help.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners) to Google Cloud [Anthos GKE](https://cloud.google.com/anthos/gke), making them available to a given GitHub repository. -A Continuous Integration [job](https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobs) builds the image and publishes it to Google Container Registry, and a Continuous Deployment job deploys it to Google Kubernetes Engine (GKE). The self hosted runners in this cluster are made available to the GitHub repository configured via the `GITHUB_REPO` environment variable below. +[![awesome-runners](https://img.shields.io/badge/listed%20on-awesome--runners-blue.svg)](https://github.com/jonico/awesome-runners)![Build status](https://github.com/github-developer/self-hosted-runners-anthos/workflows/Self%20Hosted%20Runner%20CI/CD/badge.svg) + +## About -## Usage +This project accompanies the "GitHub Actions self-hosted runners on Google Cloud" [blog post](https://github.blog/2020-08-04-github-actions-self-hosted-runners-on-google-cloud/). -### Local +![image](https://github.blog/wp-content/uploads/2020/08/hybrid-runners-with-anthos.png?resize=1024%2C654?w=1384) -#### Setup +A Continuous Integration [job](https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobs) builds the image and publishes it to Google Container Registry, and a Continuous Deployment job deploys it to Google Kubernetes Engine (GKE). The self hosted runners in this cluster are made available to the GitHub repository configured via the `GITHUB_REPO` environment variable below. -Set these in an `.env` file at the top level. Inject these into the Docker container at runtime; do _not_ check them in to Git in plaintext. -* `GITHUB_REPO` - repository to allow to use the self hosted runner (eg. `octocat/spoon-knife`) -* `TOKEN`: [Personal Access Token](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line) or [OAuth app token](https://developer.github.com/apps/building-oauth-apps/authorizing-oauth-apps/) with `administration` permission, which is necessary for interacting with the [Self Hosted Runner API](https://developer.github.com/v3/actions/self_hosted_runners/). [`GITHUB_TOKEN`](https://help.github.com/en/actions/configuring-and-managing-workflows/authenticating-with-the-github_token) does not have `administration` permission. +Because a Docker-in-Docker sidecar pod has been used in this project, these self-hosted runners can also run container builds. Though this approach offers build flexibility, it requires a [`privileged` security context](https://github.com/github-developer/self-hosted-runners-anthos/blob/cb2ee160def13ec3fff256ea43804cafe9fb7e20/deployment.yml#L55) and therefore extends the trust boundary to the whole cluster. Extra caution is recommended with this approach or [removing the sidecar](https://github.com/github-developer/self-hosted-runners-anthos/blob/cb2ee160def13ec3fff256ea43804cafe9fb7e20/deployment.yml#L45) if your application doesn’t require container builds. -#### Run Docker container -* `docker build -t self-hosted-runner .` -* `docker run --env-file=.env -v /var/run/docker.sock:/var/run/docker.sock self-hosted-runner` (Docker-in-Docker not recommended for production) +⚠️ Note that this use case is considered experimental and _not officially supported by GitHub at this time_. Additionally [it’s recommended](https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners) not to use self-hosted runners on public repositories for a number of security reasons. -### Google Kubernetes Engine +## Setup -#### Setup * Create a new Google Cloud Platform project ([docs](https://cloud.google.com/sdk/gcloud/reference/projects/create)) ``` @@ -48,28 +45,39 @@ gcloud projects add-iam-policy-binding self-hosted-runner-test \ gcloud services enable \ stackdriver.googleapis.com \ compute.googleapis.com \ - stackdriver.googleapis.com \ - container.googleapis.com + container.googleapis.com \ + anthos.googleapis.com ``` * Create GKE cluster ([docs](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster)) ``` -gcloud container clusters create self-hosted-runner-test-cluster \ - --zone us-central1 +gcloud container clusters create self-hosted-runner-test-cluster ``` -* Instead of setting these values in a local `.env` file as above, create [Kubernetes secrets](https://kubernetes.io/docs/concepts/configuration/secret/) available to your pods at runtime. +* Register cluster to the environ [docs](https://cloud.google.com/anthos/docs/setup/cloud#gcloud) +``` +gcloud container hub memberships register self-hosted-anthos-membership \ + --project=self-hosted-runner-test-myid \ + --gke-uri=https://container.googleapis.com/v1/projects/self-hosted-runner-test-myid/locations/us-west1/clusters/self-hosted-runner-test-cluster \ + --service-account-key-file=/path-to/service-account-key.json +``` + +* Get the credentails for this cluster +``` +gcloud container clusters get-credentials self-hosted-runner-test-cluster --region us-west1 +``` + +* Use [Kubernetes secrets](https://kubernetes.io/docs/concepts/configuration/secret/) to provide a Personal Access Token (`TOKEN`) and repository/organization (`GITHUB_REPO`) as environment variables available to your pods. ``` kubectl create secret generic self-hosted-runner-creds \ - --from-literal=GITHUB_REPO='https://github.com//' \ - --from-literal=GITHUB_TOKEN='token' + --from-literal=GITHUB_REPO='/' \ + --from-literal=TOKEN='token' ``` * Set these as secrets in your GitHub repository: - * `GCP_PROJECT`: Name of your Google Cloud Platform project, eg. `self-hosted-runner-test` - * `GCP_EMAIL`: Service Account email, eg. `runner-admin@self-hosted-runner-test.iam.gserviceaccount.com` + * `GCP_PROJECT`: ID of your Google Cloud Platform project, eg. `self-hosted-runner-test-897234` * `GCP_KEY`: Download your [Service Account JSON credentials](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) and Base64 encode them, eg. output of `cat ~/path/to/my/credentials.json | base64` * `TOKEN`: Personal Access Token. From the [documentation](https://developer.github.com/v3/actions/self_hosted_runners/), "Access tokens require `repo scope` for private repos and `public_repo scope` for public repos". @@ -80,8 +88,6 @@ kubectl create secret generic self-hosted-runner-creds \ * `IMAGE`: Name of your image used in [`ci.yml`](.github/workflows/ci.yml) and [`deployment.yml`](.github/workflows/deployment.yml) * `GITHUB_REPO`: `owner/repo` of the repository that will use the self hosted runner, eg. `octocat/sandbox` -* Update values in `deployment.yml` to reflect your image name and desired configuration - #### Automation * Upon push of any image-related code to any branch, [`ci.yml`](.github/workflows/ci.yml) will kick off to build and push the Docker image. * Upon push of any code to master branch, [`cd.yml`](.github/workflows/cd.yml) will kick off to deploy to Google Cloud. diff --git a/deployment.yml b/deployment.yml index 04fa3ae..49312a0 100644 --- a/deployment.yml +++ b/deployment.yml @@ -11,8 +11,7 @@ spec: spec: containers: - name: self-hosted-runner - # Update this image location - image: gcr.io/self-hosted-runner-test/self-hosted-runner:latest + image: gcr.io/PROJECT_ID/IMAGE:TAG imagePullPolicy: Always env: - name: GITHUB_REPO @@ -29,12 +28,31 @@ spec: value: 127.0.0.1 - name: DOCKER_BUILDKIT value: "1" + lifecycle: + preStop: + exec: + command: + [ + '/bin/bash', + '-c', + 'RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token $(curl -sS --data "" -H "Authorization: Bearer $TOKEN" https://api.github.com/repos/$GITHUB_REPO/actions/runners/remove-token | jq -r .token)' + ] resources: limits: - memory: "512Mi" - cpu: "250m" + memory: "256Mi" + cpu: "100m" # Docker-in-Docker not recommended for production - name: dind image: docker:18.09-dind + resources: + limits: + memory: "256Mi" + cpu: "100m" securityContext: - privileged: true \ No newline at end of file + privileged: true + volumeMounts: + - name: dind-storage + mountPath: /var/lib/docker + volumes: + - name: dind-storage + emptyDir: {} diff --git a/kustomization.yml b/kustomization.yml new file mode 100644 index 0000000..f3c9207 --- /dev/null +++ b/kustomization.yml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- deployment.yml \ No newline at end of file diff --git a/script/setup.sh b/script/setup.sh deleted file mode 100644 index 59ae13c..0000000 --- a/script/setup.sh +++ /dev/null @@ -1 +0,0 @@ -# Setup scripts go here \ No newline at end of file diff --git a/startup.sh b/startup.sh index 3c8efc3..575c496 100755 --- a/startup.sh +++ b/startup.sh @@ -1,36 +1,10 @@ #!/bin/bash -# Remove runner upon receiving an EXIT signal -function remove_runner { - echo "\nCaught EXIT signal. Removing runner and exiting.\n" - REMOVE_TOKEN=$(curl --data "" -H "Authorization: Bearer $TOKEN" https://api.github.com/repos/$GITHUB_REPO/actions/runners/remove-token | jq -r '.token') - ./config.sh remove --token $REMOVE_TOKEN - exit $? -} - -# Watch for EXIT signal to be able to shut down gracefully -trap remove_runner EXIT - -# Get latest binary version for Linux x64 -BINARY_URL=$(curl \ - --url https://api.github.com/repos/$GITHUB_REPO/actions/runners/downloads \ - --header "authorization: Bearer $TOKEN" | \ - jq -r '.[] | select(.os=="linux") | select(.architecture=="x64") | .download_url') - -# Follow any redirects to download and unpack the binary -curl -L $BINARY_URL | tar xz - -# Generate +# Generate CONFIG_TOKEN=$(curl --data "" --header "Authorization: Bearer $TOKEN" https://api.github.com/repos/$GITHUB_REPO/actions/runners/registration-token | jq -r '.token') -# Install dependencies -./bin/installdependencies.sh - -# Allow runner to run as root -export RUNNER_ALLOW_RUNASROOT=1 - # Create the runner and configure it -./config.sh --url https://github.com/$GITHUB_REPO --token $CONFIG_TOKEN --unattended +./config.sh --url https://github.com/$GITHUB_REPO --token $CONFIG_TOKEN --unattended --replace # Run it -./run.sh \ No newline at end of file +./bin/runsvc.sh