AWS / Build, Test, Push #127
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: AWS / Build, Test, Push | |
| on: | |
| schedule: | |
| - cron: 0 2 * * MON-FRI | |
| push: | |
| paths: | |
| - '**' | |
| - '.github/actions/**' | |
| - '.github/workflows/aws-main.yml' | |
| - '.github/workflows/aws-tests.yml' | |
| - '!CODEOWNERS' | |
| - '!README.md' | |
| - '!.gitignore' | |
| - '!.git-blame-ignore-revs' | |
| - '!.github/**' | |
| - '!docs/**' | |
| branches: | |
| - master | |
| pull_request: | |
| paths: | |
| - '**' | |
| - '.github/actions/**' | |
| - '.github/workflows/aws-main.yml' | |
| - '.github/workflows/aws-tests.yml' | |
| - '!CODEOWNERS' | |
| - '!README.md' | |
| - '!.gitignore' | |
| - '!.git-blame-ignore-revs' | |
| - '!.github/**' | |
| - '!docs/**' | |
| workflow_dispatch: | |
| inputs: | |
| onlyAcceptanceTests: | |
| description: 'Only run acceptance tests' | |
| required: false | |
| type: boolean | |
| default: false | |
| enableTestSelection: | |
| description: 'Enable Test Selection' | |
| required: false | |
| type: boolean | |
| default: false | |
| disableCaching: | |
| description: 'Disable Caching' | |
| required: false | |
| type: boolean | |
| default: false | |
| PYTEST_LOGLEVEL: | |
| type: choice | |
| description: Loglevel for PyTest | |
| options: | |
| - DEBUG | |
| - INFO | |
| - WARNING | |
| - ERROR | |
| - CRITICAL | |
| default: WARNING | |
| env: | |
| # Docker Image name and default tag used by docker-helper.sh | |
| IMAGE_NAME: "localstack/localstack" | |
| DEFAULT_TAG: "latest" | |
| PLATFORM_NAME_AMD64: "amd64" | |
| PLATFORM_NAME_ARM64: "arm64" | |
| jobs: | |
| test: | |
| name: "Run integration tests" | |
| uses: ./.github/workflows/aws-tests.yml | |
| with: | |
| # onlyAcceptance test is either explicitly set, or it's a push event. | |
| # otherwise it's false (schedule event, workflow_dispatch event without setting it to true) | |
| onlyAcceptanceTests: ${{ inputs.onlyAcceptanceTests == true || github.event_name == 'push' }} | |
| # default "disableCaching" to `false` if it's a push or schedule event | |
| disableCaching: ${{ inputs.disableCaching == true }} | |
| # default "disableTestSelection" to `true` if it's a push or schedule event | |
| disableTestSelection: ${{ inputs.enableTestSelection != true }} | |
| PYTEST_LOGLEVEL: ${{ inputs.PYTEST_LOGLEVEL }} | |
| secrets: | |
| DOCKERHUB_PULL_USERNAME: ${{ secrets.DOCKERHUB_PULL_USERNAME }} | |
| DOCKERHUB_PULL_TOKEN: ${{ secrets.DOCKERHUB_PULL_TOKEN }} | |
| TINYBIRD_CI_TOKEN: ${{ secrets.TINYBIRD_CI_TOKEN }} | |
| report: | |
| name: "Publish coverage and parity metrics" | |
| runs-on: ubuntu-latest | |
| needs: | |
| - test | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version-file: '.python-version' | |
| cache: 'pip' | |
| cache-dependency-path: 'requirements-dev.txt' | |
| - name: Install Community Dependencies | |
| shell: bash | |
| run: make install-dev | |
| - name: Load all test results | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: test-results-* | |
| path: target/coverage/ | |
| merge-multiple: true | |
| - name: Combine coverage results from acceptance tests | |
| run: | | |
| source .venv/bin/activate | |
| mkdir target/coverage/acceptance | |
| cp target/coverage/.coverage.acceptance* target/coverage/acceptance | |
| cd target/coverage/acceptance | |
| coverage combine | |
| mv .coverage ../../../.coverage.acceptance | |
| - name: Combine all coverage results | |
| run: | | |
| source .venv/bin/activate | |
| cd target/coverage | |
| ls -la | |
| coverage combine | |
| mv .coverage ../../ | |
| - name: Report coverage statistics | |
| env: | |
| COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} | |
| run: | | |
| source .venv/bin/activate | |
| coverage report || true | |
| coverage html || true | |
| # TO-DO: enable job after workflow in CircleCI is disabled | |
| # coveralls || true | |
| - name: Create Coverage Diff (Code Coverage) | |
| # pycobertura diff will return with exit code 0-3 -> we currently expect 2 (2: the changes worsened the overall coverage), | |
| # but we still want cirecleci to continue with the tasks, so we return 0. | |
| # From the docs: | |
| # Upon exit, the diff command may return various exit codes: | |
| # 0: all changes are covered, no new uncovered statements have been introduced | |
| # 1: some exception occurred (likely due to inappropriate usage or a bug in pycobertura) | |
| # 2: the changes worsened the overall coverage | |
| # 3: the changes introduced uncovered statements but the overall coverage is still better than before | |
| run: | | |
| source .venv/bin/activate | |
| pip install pycobertura | |
| coverage xml --data-file=.coverage -o all.coverage.report.xml --include="localstack-core/localstack/services/*/**" --omit="*/**/__init__.py" | |
| coverage xml --data-file=.coverage.acceptance -o acceptance.coverage.report.xml --include="localstack-core/localstack/services/*/**" --omit="*/**/__init__.py" | |
| pycobertura show --format html acceptance.coverage.report.xml -o coverage-acceptance.html | |
| bash -c "pycobertura diff --format html all.coverage.report.xml acceptance.coverage.report.xml -o coverage-diff.html; if [[ \$? -eq 1 ]] ; then exit 1 ; else exit 0 ; fi" | |
| - name: Create Metric Coverage Diff (API Coverage) | |
| env: | |
| COVERAGE_DIR_ALL: "parity_metrics" | |
| COVERAGE_DIR_ACCEPTANCE: "acceptance_parity_metrics" | |
| OUTPUT_DIR: "api-coverage" | |
| run: | | |
| source .venv/bin/activate | |
| mkdir $OUTPUT_DIR | |
| python -m scripts.metrics_coverage.diff_metrics_coverage | |
| - name: Archive coverage and parity metrics | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-and-parity-metrics | |
| path: | | |
| .coverage | |
| api-coverage/ | |
| coverage-acceptance.html | |
| coverage-diff.html | |
| parity_metrics/ | |
| acceptance_parity_metrics/ | |
| scripts/implementation_coverage_aggregated.csv | |
| scripts/implementation_coverage_full.csv | |
| retention-days: 7 | |
| push: | |
| name: "Push images" | |
| runs-on: ubuntu-latest | |
| # push image on master, target branch not set, and the dependent steps were either successful or skipped | |
| # TO-DO: enable job after workflow in CircleCI is disabled | |
| if: false | |
| # if: github.ref == 'refs/heads/master' && !failure() && !cancelled() && github.repository == 'localstack/localstack' | |
| needs: | |
| # all tests need to be successful for the image to be pushed | |
| - test | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| # setuptools_scm requires the git history (at least until the last tag) to determine the version | |
| fetch-depth: 0 | |
| - name: Load Localstack ${{ env.PLATFORM_NAME_AMD64 }} Docker Image | |
| uses: ./.github/actions/load-localstack-docker-from-artifacts | |
| with: | |
| platform: ${{ env.PLATFORM_NAME_AMD64 }} | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-east-1 | |
| - name: Login to Amazon ECR | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@v2 | |
| with: | |
| registry-type: public | |
| - name: Push ${{ env.PLATFORM_NAME_AMD64 }} Docker Image | |
| env: | |
| DOCKER_USERNAME: ${{ secrets.DOCKERHUB_PUSH_USERNAME }} | |
| DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_PUSH_TOKEN }} | |
| PLATFORM: ${{ env.PLATFORM_NAME_AMD64 }} | |
| run: | | |
| # Push to Docker Hub | |
| ./bin/docker-helper.sh push | |
| # Push to Amazon Public ECR | |
| TARGET_IMAGE_NAME="public.ecr.aws/localstack/localstack" ./bin/docker-helper.sh push | |
| - name: Load Localstack ${{ env.PLATFORM_NAME_ARM64 }} Docker Image | |
| uses: ./.github/actions/load-localstack-docker-from-artifacts | |
| with: | |
| platform: ${{ env.PLATFORM_NAME_ARM64 }} | |
| - name: Push ${{ env.PLATFORM_NAME_ARM64 }} Docker Image | |
| env: | |
| DOCKER_USERNAME: ${{ secrets.DOCKERHUB_PUSH_USERNAME }} | |
| DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_PUSH_TOKEN }} | |
| PLATFORM: ${{ env.PLATFORM_NAME_ARM64 }} | |
| run: | | |
| # Push to Docker Hub | |
| ./bin/docker-helper.sh push | |
| # Push to Amazon Public ECR | |
| TARGET_IMAGE_NAME="public.ecr.aws/localstack/localstack" ./bin/docker-helper.sh push | |
| - name: Push Multi-Arch Manifest | |
| env: | |
| DOCKER_USERNAME: ${{ secrets.DOCKERHUB_PUSH_USERNAME }} | |
| DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_PUSH_TOKEN }} | |
| run: | | |
| # Push to Docker Hub | |
| ./bin/docker-helper.sh push-manifests | |
| # Push to Amazon Public ECR | |
| IMAGE_NAME="public.ecr.aws/localstack/localstack" ./bin/docker-helper.sh push-manifests | |
| - name: Publish dev release | |
| env: | |
| DOCKER_USERNAME: ${{ secrets.DOCKERHUB_PUSH_USERNAME }} | |
| DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_PUSH_TOKEN }} | |
| run: | | |
| if git describe --exact-match --tags >/dev/null 2>&1; then | |
| echo "not publishing a dev release as this is a tagged commit" | |
| else | |
| source .venv/bin/activate | |
| make publish || echo "dev release failed (maybe it is already published)" | |
| fi | |
| push-to-tinybird: | |
| name: Push Workflow Status to Tinybird | |
| if: always() && github.ref == 'refs/heads/master' && github.repository == 'localstack/localstack' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - test | |
| steps: | |
| - name: Push to Tinybird | |
| uses: localstack/tinybird-workflow-push@v3 | |
| with: | |
| # differentiate between "acceptance only" and "proper / full" runs | |
| workflow_id: ${{ (inputs.onlyAcceptanceTests == true || github.event_name == 'push') && 'tests_acceptance' || 'tests_full' }} | |
| tinybird_token: ${{ secrets.TINYBIRD_CI_TOKEN }} | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| tinybird_datasource: "ci_workflows" | |
| # determine the output only for the jobs that are direct dependencies of this job (to avoid issues with workflow_call embeddings) | |
| outcome: ${{ ((contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) && 'failure') || 'success' }} | |
| cleanup: | |
| name: "Cleanup" | |
| runs-on: ubuntu-latest | |
| # only remove the image artifacts if the build was successful | |
| # (this allows a re-build of failed jobs until for the time of the retention period) | |
| if: success() | |
| needs: push | |
| steps: | |
| - uses: geekyeggo/delete-artifact@v5 | |
| with: | |
| # delete the docker images shared within the jobs (storage on GitHub is expensive) | |
| name: | | |
| localstack-docker-image-* | |
| lambda-common-* | |
| failOnError: false | |
| token: ${{ secrets.GITHUB_TOKEN }} |