From 437b60d825a253ae0b9869f0410b7f5b4ec76331 Mon Sep 17 00:00:00 2001 From: Julien Benhaim Date: Wed, 16 Jul 2025 15:07:40 +0200 Subject: [PATCH] Big refactoring --- .../.evergreen-functions.yml | 0 .../.evergreen-mco.yml | 0 .../.evergreen-periodic-builds.yaml | 0 .../.evergreen-tasks.yml | 0 .evergreen-legacy/evergreen.yml.original | 1972 ++++++++++++++++ .evergreen.yml | 1974 +---------------- .evergreen/functions/build.yml | 128 ++ .evergreen/functions/deploy.yml | 68 + .evergreen/functions/setup.yml | 259 +++ .evergreen/functions/test.yml | 162 ++ .evergreen/main.yml | 31 + .evergreen/periodic/scheduled-builds.yml | 112 + .evergreen/tasks/e2e-tests.yml | 117 + .evergreen/tasks/image-builds.yml | 248 +++ .evergreen/tasks/mco-tests.yml | 173 ++ .evergreen/tasks/preflight.yml | 163 ++ .evergreen/tasks/unit-tests.yml | 46 + .evergreen/variables/common.yml | 150 ++ .evergreen/variables/dependencies.yml | 162 ++ .evergreen/variables/versions.yml | 16 + .evergreen/variants/e2e-community.yml | 80 + .evergreen/variants/e2e-enterprise.yml | 134 ++ .evergreen/variants/image-builds.yml | 151 ++ .evergreen/variants/releases.yml | 70 + .evergreen/variants/unit-tests.yml | 13 + CI_MIGRATION_PLAN.md | 322 +++ 26 files changed, 4590 insertions(+), 1961 deletions(-) rename .evergreen-functions.yml => .evergreen-legacy/.evergreen-functions.yml (100%) rename .evergreen-mco.yml => .evergreen-legacy/.evergreen-mco.yml (100%) rename .evergreen-periodic-builds.yaml => .evergreen-legacy/.evergreen-periodic-builds.yaml (100%) rename .evergreen-tasks.yml => .evergreen-legacy/.evergreen-tasks.yml (100%) create mode 100644 .evergreen-legacy/evergreen.yml.original create mode 100644 .evergreen/functions/build.yml create mode 100644 .evergreen/functions/deploy.yml create mode 100644 .evergreen/functions/setup.yml create mode 100644 .evergreen/functions/test.yml create mode 100644 .evergreen/main.yml create mode 100644 .evergreen/periodic/scheduled-builds.yml create mode 100644 .evergreen/tasks/e2e-tests.yml create mode 100644 .evergreen/tasks/image-builds.yml create mode 100644 .evergreen/tasks/mco-tests.yml create mode 100644 .evergreen/tasks/preflight.yml create mode 100644 .evergreen/tasks/unit-tests.yml create mode 100644 .evergreen/variables/common.yml create mode 100644 .evergreen/variables/dependencies.yml create mode 100644 .evergreen/variables/versions.yml create mode 100644 .evergreen/variants/e2e-community.yml create mode 100644 .evergreen/variants/e2e-enterprise.yml create mode 100644 .evergreen/variants/image-builds.yml create mode 100644 .evergreen/variants/releases.yml create mode 100644 .evergreen/variants/unit-tests.yml create mode 100644 CI_MIGRATION_PLAN.md diff --git a/.evergreen-functions.yml b/.evergreen-legacy/.evergreen-functions.yml similarity index 100% rename from .evergreen-functions.yml rename to .evergreen-legacy/.evergreen-functions.yml diff --git a/.evergreen-mco.yml b/.evergreen-legacy/.evergreen-mco.yml similarity index 100% rename from .evergreen-mco.yml rename to .evergreen-legacy/.evergreen-mco.yml diff --git a/.evergreen-periodic-builds.yaml b/.evergreen-legacy/.evergreen-periodic-builds.yaml similarity index 100% rename from .evergreen-periodic-builds.yaml rename to .evergreen-legacy/.evergreen-periodic-builds.yaml diff --git a/.evergreen-tasks.yml b/.evergreen-legacy/.evergreen-tasks.yml similarity index 100% rename from .evergreen-tasks.yml rename to .evergreen-legacy/.evergreen-tasks.yml diff --git a/.evergreen-legacy/evergreen.yml.original b/.evergreen-legacy/evergreen.yml.original new file mode 100644 index 000000000..3b48ce8ad --- /dev/null +++ b/.evergreen-legacy/evergreen.yml.original @@ -0,0 +1,1972 @@ +# MongoDB Kubernetes Operator CI Configuration +# This file now uses a modular structure for better maintainability +# All configuration has been organized into logical, focused files + +include: + - filename: .evergreen/main.yml + +variables: + - &ops_manager_60_latest 6.0.27 # The order/index is important, since these are anchors. Please do not change + + - &ops_manager_70_latest 7.0.15 # The order/index is important, since these are anchors. Please do not change + + - &ops_manager_80_latest 8.0.7 # The order/index is important, since these are anchors. Please do not change + + # The dependency unification between static and non-static is intentional here. + # Even though some images are exclusive, in EVG they all are built once and in parallel. + # It is not worth the effort of splitting them out. + # Once Static Containers are default, this piece can be cleaned up. + - &base_om6_dependency + depends_on: + - name: build_om_images + variant: build_om60_images + - name: build_operator_ubi + variant: init_test_run + - name: build_init_database_image_ubi + variant: init_test_run + - name: build_database_image_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + - name: build_init_appdb_images_ubi + variant: init_test_run + - name: build_init_om_images_ubi + variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run + + - &base_no_om_image_dependency + depends_on: + - name: build_om_images + variant: build_om70_images + - name: build_operator_ubi + variant: init_test_run + - name: build_init_database_image_ubi + variant: init_test_run + - name: build_database_image_ubi + variant: init_test_run + - name: build_init_om_images_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + - name: build_init_appdb_images_ubi + variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run + + - &community_dependency + depends_on: + - name: build_operator_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run + - name: build_readiness_probe_image + variant: init_test_run + - name: build_upgrade_hook_image + variant: init_test_run + - name: build_mco_test_image + variant: init_test_run + + - &setup_group + setup_group_can_fail_task: true + setup_group: + - func: clone + - func: download_kube_tools + - func: setup_building_host + + - &setup_group_multi_cluster + setup_group_can_fail_task: true + setup_group: + - func: clone + - func: download_kube_tools + - func: setup_building_host + - func: build_multi_cluster_binary + + - &setup_and_teardown_group_gke_code_snippets + setup_task_can_fail_task: true + setup_group: + - func: clone + - func: setup_gcloud_cli + - func: setup_mongosh + - func: download_kube_tools + - func: build_multi_cluster_binary + teardown_group: + - func: upload_code_snippets_logs + + - &setup_and_teardown_group_kind_code_snippets + setup_task_can_fail_task: true + setup_group: + - func: clone + - func: cleanup_exec_environment + - func: download_kube_tools + - func: configure_docker_auth + - func: setup_kubernetes_environment + teardown_task: + - func: upload_e2e_logs + - func: upload_code_snippets_logs + + - &setup_and_teardown_task_cloudqa + setup_task_can_fail_task: true + setup_task: + - func: cleanup_exec_environment + - func: configure_docker_auth + - func: setup_kubernetes_environment + - func: setup_cloud_qa + teardown_task_can_fail_task: true + teardown_task: + - func: upload_e2e_logs + - func: teardown_kubernetes_environment + - func: teardown_cloud_qa + + - &setup_and_teardown_task + setup_task_can_fail_task: true + setup_task: + - func: cleanup_exec_environment + - func: configure_docker_auth + - func: setup_kubernetes_environment + teardown_task_can_fail_task: true + teardown_task: + - func: upload_e2e_logs + - func: teardown_kubernetes_environment + + - &teardown_group + teardown_group: + - func: prune_docker_resources + - func: run_retry_script + + - &base_om7_dependency + depends_on: + - name: build_om_images + variant: build_om70_images + - name: build_operator_ubi + variant: init_test_run + - name: build_init_database_image_ubi + variant: init_test_run + - name: build_database_image_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + - name: build_init_appdb_images_ubi + variant: init_test_run + - name: build_init_om_images_ubi + variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run + + - &base_om8_dependency + depends_on: + - name: build_om_images + variant: build_om80_images + - name: build_operator_ubi + variant: init_test_run + - name: build_init_database_image_ubi + variant: init_test_run + - name: build_database_image_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + - name: build_init_appdb_images_ubi + variant: init_test_run + - name: build_init_om_images_ubi + variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run + +parameters: + - key: evergreen_retry + value: "true" + description: set this to false to suppress retries on failure + + - key: pin_tag_at + value: 10:00 + description: Pin tags at this time of the day. Midnight by default for periodic and 10 for releases. + + - key: OVERRIDE_VERSION_ID + value: "" + description: "Patch id to reuse images from other Evergreen build" + + - key: code_snippets_teardown + value: "true" + description: set this to false if you would like to keep the clusters created during code snippets tests + + - key: code_snippets_reset + value: "false" + description: set this to true if you would like to delete the resources created in the code snippet tests, but keep the clusters + + - key: code_snippets_commit_output + value: "false" + description: set this to true if you would like the pipeline to automatically push a branch with updated snippets outputs + + +# Triggered manually or by PCT. +patch_aliases: + - alias: "periodic_builds" + variant_tags: [ "periodic_build" ] + task: ".*" + - alias: "periodic_teardowns" + variant_tags: [ "periodic_teardown" ] + task: ".*" + - alias: "release_agent" + variant_tags: [ "release_agent" ] + task: ".*" + - alias: "release_all_agents_manually" + variant_tags: [ "release_all_agents_manually" ] + task: ".*" + - alias: "release" + variant_tags: [ "release", "e2e_smoke_release_test_suite" ] + task_tags: [ "image_release", "image_preflight", "openshift_bundles", "code_snippets", "patch-run" ] + - alias: "smoke_test_release" + variant_tags: [ "e2e_smoke_release_test_suite" ] + task_tags: [ "patch-run" ] + - alias: "patch-run-cloudqa" + variant_tags: [ "cloudqa" ] + task: ".*" + +# Triggered whenever the GitHub PR is created +github_pr_aliases: + - variant_tags: [ "unit_tests" ] + task_tags: [ "unit_tests" ] + - variant_tags: [ "e2e_test_suite" ] + task_tags: [ "patch-run" ] + - variant_tags: [ "e2e_mco_test_suite" ] + task: ".*" + - variant_tags: [ "auto_bump" ] + task_tags: [ "patch-run" ] + - variant_tags: [ "release_agents_on_ecr" ] + task: ".*" + +# Allows to see evergreen checks in GitHub commits +# https://github.com/mongodb/mongodb-kubernetes/commits/master/ +github_checks_aliases: + - variant: ".*" + task: ".*" + +# Triggered on git tag +git_tag_aliases: + - git_tag: "^(\\d+\\.)?(\\d+\\.)?(\\d+)$" + variant_tags: [ "release", "e2e_smoke_release_test_suite" ] + task_tags: [ "image_release", "image_preflight", "openshift_bundles", "code_snippets", "patch-run" ] + +tasks: + - name: unit_tests_golang + tags: [ "unit_tests" ] + commands: + - func: "test_golang_unit" + + - name: unit_tests_python + tags: [ "unit_tests" ] + commands: + - func: "test_python_unit" + + - name: sbom_tests + tags: [ "unit_tests" ] + # The SBOM tests run only on commit builds. Running this on patches might cause false-positive failures + # because certain images might not be there yet. Such situation happens for OM image upgrades for example. + # See https://docs.devprod.prod.corp.mongodb.com/evergreen/Project-Configuration/Project-Configuration-Files#limiting-when-a-task-or-variant-will-run + patchable: false + commands: + - func: "test_sboms" + + - name: lint_repo + tags: [ "unit_tests" ] + commands: + - func: lint_repo + + - name: release_operator + tags: [ "image_release" ] + allowed_requesters: [ "patch", "github_tag" ] + commands: + - func: clone + - func: setup_building_host + - func: quay_login + - func: setup_docker_sbom + - func: pipeline + vars: + image_name: operator + include_tags: release + + # Releases init images to Quay + - name: release_init_appdb + tags: [ "image_release" ] + allowed_requesters: [ "patch", "github_tag" ] + commands: + - func: clone + - func: setup_building_host + - func: quay_login + - func: setup_docker_sbom + - func: pipeline + vars: + image_name: init-appdb + include_tags: release + + - name: release_init_database + tags: [ "image_release" ] + allowed_requesters: [ "patch", "github_tag" ] + commands: + - func: clone + - func: setup_building_host + - func: quay_login + - func: setup_docker_sbom + - func: pipeline + vars: + image_name: init-database + include_tags: release + + - name: release_init_ops_manager + tags: [ "image_release" ] + allowed_requesters: [ "patch", "github_tag" ] + commands: + - func: clone + - func: setup_building_host + - func: quay_login + - func: setup_docker_sbom + - func: pipeline + vars: + image_name: init-ops-manager + include_tags: release + + - name: release_agent_operator_release + tags: [ "image_release" ] + allowed_requesters: [ "patch", "github_tag" ] + commands: + - func: clone + - func: setup_building_host + - func: quay_login + - func: setup_docker_sbom + - func: pipeline + vars: + image_name: agent + include_tags: release + + # pct only triggers this variant once a new agent image is out + - name: release_agent + # this enables us to run this variant either manually (patch) which pct does or during an OM bump (github_pr) + allowed_requesters: [ "patch", "github_pr" ] + commands: + - func: clone + - func: setup_building_host + - func: quay_login + - func: setup_docker_sbom + - func: pipeline + vars: + image_name: agent-pct + include_tags: release + + - name: run_precommit_and_push + tags: ["patch-run"] + commands: + - func: clone + - func: python_venv + - func: download_kube_tools + - func: setup_shellcheck + - command: github.generate_token + params: + expansion_name: GH_TOKEN + - command: subprocess.exec + type: setup + params: + include_expansions_in_env: + - GH_TOKEN + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/precommit_bump.sh + + # Pct only triggers this variant once a new agent image is out + # these releases the agent with the operator suffix (not patch id) on ecr to allow for digest pinning to pass. + # For this to work, we rely on skip_tags which is used to determine whether + # we want to release on quay or not, in this case - ecr instead. + # We rely on the init_database from ecr for the agent x operator images. + # This runs on agent releases that are not concurrent with operator releases. + - name: release_agents_on_ecr_conditional + commands: + - func: clone + - func: run_task_conditionally + vars: + condition_script: scripts/evergreen/should_release_agents_on_ecr.sh + variant: init_release_agents_on_ecr + task: release_agents_on_ecr + + - name: release_agents_on_ecr + # this enables us to run this variant either manually (patch) which pct does or during an OM bump (github_pr) + allowed_requesters: [ "patch", "github_pr" ] + priority: 70 + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + image_name: agent-pct + skip_tags: release + + - name: release_all_agents_on_ecr + # this enables us to run this manually (patch) and release all agent versions to ECR + # it's needed during operator new version release process - e2e tests (especially olm tests) + # will look for agent with new operator version suffix, but during PR checks we only build + # agent versions for most recent major OM versions and the tests will fail. Before running the PR + # we have to manually release all agents to ECR by triggering this patch + allowed_requesters: [ "patch" ] + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + image_name: agent-pct + skip_tags: release + all_agents: true + + - name: build_test_image + commands: + - func: clone + - func: setup_building_host + - func: build_multi_cluster_binary + - func: pipeline + vars: + image_name: test + + - name: build_mco_test_image + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + image_name: mco-test + + - name: build_operator_ubi + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + skip_tags: ubuntu,release + distro: ubi + image_name: operator + + - name: build_init_om_images_ubi + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + image_name: init-ops-manager + skip_tags: ubuntu,release + + - name: build_init_appdb_images_ubi + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + image_name: init-appdb + skip_tags: ubuntu,release + + - name: build_agent_images_ubi + depends_on: + - name: build_init_database_image_ubi + variant: init_test_run + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + image_name: agent + skip_tags: ubuntu,release + + - name: build_init_database_image_ubi + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + image_name: init-database + skip_tags: ubuntu,release + + - name: build_database_image_ubi + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + image_name: database + skip_tags: ubuntu,release + + - name: build_readiness_probe_image + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + image_name: readiness-probe + skip_tags: ubuntu,release + + - name: build_upgrade_hook_image + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + image_name: upgrade-hook + skip_tags: ubuntu,release + + - name: prepare_aws + priority: 59 + commands: + - func: clone + - func: setup_jq + - func: setup_aws + - func: prepare_aws + + - name: run_retry_script + tags: [ "patch-run" ] + commands: + - func: run_retry_script + + - name: generate_perf_tasks_one_thread + commands: + - func: clone + - func: generate_perf_tests_tasks + vars: + variant: e2e_operator_perf_one_thread + size: small + + - name: generate_perf_tasks_10_thread + commands: + - func: clone + - func: generate_perf_tests_tasks + vars: + variant: e2e_operator_perf + size: small + + - name: generate_perf_tasks_30_thread + commands: + - func: clone + - func: generate_perf_tests_tasks + vars: + variant: e2e_operator_perf_thirty + size: small + + - name: release_database + tags: [ "image_release" ] + allowed_requesters: [ "patch", "github_tag" ] + commands: + - func: clone + - func: setup_building_host + - func: quay_login + - func: setup_docker_sbom + - func: pipeline + vars: + image_name: database + + - name: build_om_images + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + image_name: ops-manager + skip_tags: release + + - name: publish_ops_manager + commands: + - func: clone + - func: setup_building_host + - func: quay_login + - func: setup_docker_sbom + - func: pipeline + vars: + image_name: ops-manager + include_tags: release + + - name: prepare_and_upload_openshift_bundles_for_e2e + commands: + - func: clone + - func: setup_building_host + - func: download_kube_tools + - func: setup_prepare_openshift_bundles + - func: prepare_openshift_bundles_for_e2e + + - name: prepare_and_upload_openshift_bundles + tags: [ "openshift_bundles" ] + commands: + - func: clone + - func: setup_aws + - func: configure_docker_auth + - func: setup_prepare_openshift_bundles + - func: prepare_openshift_bundles + - func: update_evergreen_expansions + - func: upload_openshift_bundle + vars: + # mongoDbOperator expansion is added in update_evergreen_expansions func from release.json + bundle_file_name: "mck-operator-certified-${mongodbOperator}.tgz" + + - name: run_conditionally_prepare_and_upload_openshift_bundles + tags: [ "openshift_bundles" ] + commands: + - func: clone + - func: run_task_conditionally + vars: + condition_script: scripts/evergreen/should_prepare_openshift_bundles.sh + variant: prepare_openshift_bundles + task: prepare_and_upload_openshift_bundles + + - name: backup_csv_images_dry_run + commands: + - func: clone + - func: quay_login + - command: subprocess.exec + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: python3 + args: + - scripts/dev/release/backup_csv_images.py + - scripts/dev/release/1.2.0.clusterserviceversion.yaml + - --dry-run + - --verbose + - name: backup_csv_images_limit_3 + commands: + - func: clone + - func: quay_login + - command: subprocess.exec + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: python3 + args: + - scripts/dev/release/backup_csv_images.py + - scripts/dev/release/1.2.0.clusterserviceversion.yaml + - --limit + - "3" + - --verbose + - name: backup_csv_images_all + commands: + - func: clone + - func: quay_login + - command: subprocess.exec + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: python3 + args: + - scripts/dev/release/backup_csv_images.py + - scripts/dev/release/1.2.0.clusterserviceversion.yaml + - --verbose + +task_groups: + - name: unit_task_group + max_hosts: -1 + setup_group_can_fail_task: true + setup_group: + - func: clone + - func: python_venv + - func: download_kube_tools + - func: setup_shellcheck + tasks: + - lint_repo + - unit_tests_golang + - unit_tests_python + - sbom_tests + + - name: gke_code_snippets_task_group + <<: *setup_and_teardown_group_gke_code_snippets + max_hosts: -1 + tasks: + - task_gke_multi_cluster_snippets + - task_gke_multi_cluster_no_mesh_snippets + + - name: kind_code_snippets_task_group + <<: *setup_and_teardown_group_kind_code_snippets + max_hosts: -1 + tasks: + - task_kind_community_search_snippets + + # Task group for deploying mongodbcommunity resources and testing the (former) MCO + - name: e2e_mdb_community_task_group + max_hosts: -1 + <<: *setup_group + <<: *setup_and_teardown_task + tasks: + - e2e_community_replicaset_scale + - e2e_search_community_basic + + # This is the task group that contains all the tests run in the e2e_mdb_kind_ubuntu_cloudqa build variant + - name: e2e_mdb_kind_cloudqa_task_group + max_hosts: -1 + <<: *setup_group + <<: *setup_and_teardown_task_cloudqa + tasks: + # e2e_kube_only_task_group + - e2e_crd_validation + - e2e_replica_set_config_map + - e2e_replica_set_exposed_externally + - e2e_replica_set_pv + - e2e_replica_set_pv_multiple + - e2e_replica_set_schema_validation + - e2e_replica_set_statefulset_status + - e2e_sharded_cluster_pv + - e2e_sharded_cluster_recovery + - e2e_sharded_cluster_schema_validation + - e2e_sharded_cluster_statefulset_status + - e2e_standalone_config_map + - e2e_standalone_recovery + - e2e_standalone_schema_validation + - e2e_users_schema_validation + - e2e_replica_set_tls_default + - e2e_replica_set_tls_override + - e2e_replica_set_ignore_unknown_users + # e2e_core_task_group + - e2e_all_mongodb_resources_parallel + - e2e_multiple_cluster_failures + - e2e_replica_set_custom_podspec + - e2e_replica_set_custom_sa + - e2e_replica_set_report_pending_pods + - e2e_replica_set_recovery + - e2e_replica_set_upgrade_downgrade + - e2e_replica_set_agent_flags_and_readinessProbe + - e2e_sharded_cluster + - e2e_sharded_cluster_secret + - e2e_sharded_cluster_upgrade_downgrade + - e2e_sharded_cluster_custom_podspec + - e2e_sharded_cluster_agent_flags + - e2e_standalone_upgrade_downgrade + - e2e_standalone_custom_podspec + - e2e_standalone_agent_flags + - e2e_replica_set_process_hostnames + - e2e_replica_set_member_options + # e2e_tls_task_group + - e2e_replica_set_tls_allow + - e2e_replica_set_tls_prefer + - e2e_replica_set_tls_require_upgrade + - e2e_tls_rs_additional_certs + - e2e_tls_sc_additional_certs + - e2e_tls_rs_intermediate_ca + - e2e_tls_sharded_cluster_certs_prefix + - e2e_sharded_cluster_migration + - e2e_standalone_no_tls_no_status_is_set + - e2e_feature_controls_authentication + - e2e_replica_set + - e2e_replica_set_liveness_probe + - e2e_replica_set_mongod_options + - e2e_replica_set_readiness_probe + - e2e_replica_set_update_delete_parallel + - e2e_sharded_cluster_scale_shards + - e2e_sharded_cluster_shard_overrides + - e2e_sharded_cluster_mongod_options_and_log_rotation + - e2e_tls_rs_external_access + - e2e_replica_set_tls_require + - e2e_replica_set_tls_certs_secret_prefix + - e2e_disable_tls_scale_up + - e2e_replica_set_tls_require_and_disable + # e2e_x509_task_group + - e2e_configure_tls_and_x509_simultaneously_st + - e2e_configure_tls_and_x509_simultaneously_rs + - e2e_configure_tls_and_x509_simultaneously_sc + - e2e_tls_x509_rs + - e2e_tls_x509_sc + - e2e_tls_x509_configure_all_options_rs + - e2e_tls_x509_configure_all_options_sc + - e2e_tls_x509_user_connectivity + - e2e_tls_x509_users_addition_removal + - e2e_replica_set_tls_process_hostnames + # e2e_ldap_task_group + - e2e_replica_set_ldap + - e2e_sharded_cluster_ldap + - e2e_replica_set_ldap_tls + - e2e_replica_set_ldap_user_to_dn_mapping + # e2e_replica_set_ldap_agent_auth + - e2e_replica_set_ldap_agent_client_certs + - e2e_replica_set_ldap_custom_roles + - e2e_replica_set_update_roles_no_privileges + - e2e_replica_set_ldap_group_dn + - e2e_replica_set_ldap_group_dn_with_x509_agent + # e2e_scram_sha_task_group_with_manually_generated_certs + - e2e_replica_set_scram_sha_256_user_connectivity + - e2e_replica_set_scram_sha_256_user_first + - e2e_replica_set_scram_sha_1_user_connectivity + - e2e_replica_set_scram_sha_1_upgrade + - e2e_replica_set_scram_x509_ic_manual_certs + - e2e_sharded_cluster_scram_sha_1_upgrade + - e2e_sharded_cluster_scram_sha_256_user_connectivity + - e2e_sharded_cluster_scram_sha_1_user_connectivity + - e2e_sharded_cluster_scram_x509_ic_manual_certs + - e2e_sharded_cluster_external_access + # e2e_auth_transitions_task_group + - e2e_replica_set_scram_sha_and_x509 + - e2e_replica_set_x509_to_scram_transition + - e2e_sharded_cluster_scram_sha_and_x509 + - e2e_sharded_cluster_x509_to_scram_transition + - e2e_sharded_cluster_internal_cluster_transition + # e2e_webhook_validation_task_group + - e2e_mongodb_validation_webhook + - e2e_mongodb_roles_validation_webhook + - e2e_vault_setup + - e2e_vault_setup_tls + # this test will either migrate from static to non-static or the other way around + - e2e_replica_set_migration + - e2e_replica_set_pv_resize + - e2e_sharded_cluster_pv_resize + - e2e_community_and_meko_replicaset_scale + # OIDC test group + - e2e_replica_set_oidc_m2m_group + - e2e_replica_set_oidc_m2m_user + - e2e_replica_set_oidc_workforce + - e2e_sharded_cluster_oidc_m2m_group + - e2e_sharded_cluster_oidc_m2m_user + <<: *teardown_group + + # this task group contains just a one task, which is smoke testing whether the operator + # works correctly when we run it without installing webhook cluster roles + - name: e2e_mdb_kind_no_webhook_roles_cloudqa_task_group + max_hosts: -1 + <<: *setup_group + <<: *setup_and_teardown_task_cloudqa + tasks: + - e2e_replica_set_agent_flags_and_readinessProbe + <<: *teardown_group + + # This task group mostly duplicates tests running in e2e_mdb_kind_ubi_cloudqa + # This is mostly a smoke test, so we execute only a few essential ones. + - name: e2e_mdb_openshift_ubi_cloudqa_task_group + # OM tests are also run on the same Openshift cluster, so we use 1 max host to not + # allow the helm installations to interfere with each other during the setup of the tests. + max_hosts: 1 + <<: *setup_group + <<: *setup_and_teardown_task_cloudqa + tasks: + - e2e_crd_validation + - e2e_replica_set_scram_sha_256_user_connectivity + - e2e_sharded_cluster_scram_sha_256_user_connectivity + - e2e_replica_set_pv + - e2e_sharded_cluster_pv + + - name: e2e_custom_domain_task_group + max_hosts: -1 + <<: *setup_group + <<: *setup_and_teardown_task_cloudqa + tasks: + - e2e_replica_set + + # e2e_operator_task_group includes the tests for the specific Operator configuration/behavior. They may deal with + # cluster-wide resources so should be run in isolated K8s clusters only (Kind or Minikube). + - name: e2e_operator_task_group + max_hosts: -1 + <<: *setup_group + <<: *setup_and_teardown_task_cloudqa + tasks: + - e2e_operator_upgrade_replica_set + - e2e_sharded_cluster_operator_upgrade_v1_27_to_mck + - e2e_operator_upgrade_ops_manager + - e2e_operator_proxy + - e2e_operator_partial_crd + - e2e_operator_clusterwide + - e2e_operator_multi_namespaces + - e2e_appdb_tls_operator_upgrade_v1_32_to_mck + - e2e_meko_mck_upgrade + <<: *teardown_group + + # e2e_operator_race_with_telemetry_task_group includes the tests for testing the operator with race detector enabled + # additionally, it sends telemetry to cloud-dev; more here: https://wiki.corp.mongodb.com/display/MMS/Telemetry + - name: e2e_operator_race_with_telemetry_task_group + max_hosts: -1 + <<: *setup_group_multi_cluster + <<: *setup_and_teardown_task + tasks: + - e2e_om_reconcile_race_with_telemetry + <<: *teardown_group + + # e2e_operator_task_group includes the tests for the specific Operator configuration/behavior. They may deal with + # cluster-wide resources so should be run in isolated K8s clusters only (Kind or Minikube). + - name: e2e_static_operator_task_group + max_hosts: -1 + <<: *setup_group + <<: *setup_and_teardown_task_cloudqa + tasks: + - e2e_operator_upgrade_replica_set + - e2e_sharded_cluster_operator_upgrade_v1_27_to_mck + - e2e_operator_upgrade_ops_manager + - e2e_operator_proxy + - e2e_operator_partial_crd + - e2e_operator_clusterwide + - e2e_operator_multi_namespaces + - e2e_appdb_tls_operator_upgrade_v1_32_to_mck + - e2e_meko_mck_upgrade + <<: *teardown_group + + - name: e2e_multi_cluster_kind_task_group + max_hosts: -1 + <<: *setup_group + <<: *setup_and_teardown_task_cloudqa + tasks: + - e2e_multi_cluster_replica_set + - e2e_multi_cluster_replica_set_migration + - e2e_multi_cluster_replica_set_member_options + - e2e_multi_cluster_recover + - e2e_multi_cluster_recover_clusterwide + - e2e_multi_cluster_specific_namespaces + - e2e_multi_cluster_scram + - e2e_multi_cluster_tls_with_x509 + - e2e_multi_cluster_tls_no_mesh + - e2e_multi_cluster_enable_tls + # e2e_multi_cluster_with_ldap + # e2e_multi_cluster_with_ldap_custom_roles + - e2e_multi_cluster_mtls_test + - e2e_multi_cluster_replica_set_deletion + - e2e_multi_cluster_replica_set_scale_up + - e2e_multi_cluster_scale_up_cluster + - e2e_multi_cluster_scale_up_cluster_new_cluster + - e2e_multi_cluster_replica_set_scale_down + - e2e_multi_cluster_scale_down_cluster + - e2e_multi_sts_override + - e2e_multi_cluster_tls_with_scram + - e2e_multi_cluster_upgrade_downgrade + - e2e_multi_cluster_backup_restore + - e2e_multi_cluster_backup_restore_no_mesh + - e2e_multi_cluster_disaster_recovery + - e2e_multi_cluster_multi_disaster_recovery + - e2e_multi_cluster_recover_network_partition + - e2e_multi_cluster_validation + - e2e_multi_cluster_agent_flags + - e2e_multi_cluster_replica_set_ignore_unknown_users + - e2e_multi_cluster_pvc_resize + - e2e_multi_cluster_sharded_geo_sharding + - e2e_multi_cluster_sharded_scaling + - e2e_multi_cluster_sharded_scaling_all_shard_overrides + - e2e_multi_cluster_sharded_simplest + - e2e_multi_cluster_sharded_snippets + - e2e_multi_cluster_sharded_simplest_no_mesh + - e2e_multi_cluster_sharded_external_access_no_ext_domain + - e2e_multi_cluster_sharded_tls_no_mesh + - e2e_multi_cluster_sharded_tls + - e2e_multi_cluster_sharded_disaster_recovery + - e2e_sharded_cluster + - e2e_sharded_cluster_agent_flags + - e2e_sharded_cluster_custom_podspec + - e2e_sharded_cluster_mongod_options_and_log_rotation + - e2e_sharded_cluster_migration + - e2e_sharded_cluster_pv + - e2e_sharded_cluster_pv_resize + - e2e_sharded_cluster_recovery + - e2e_sharded_cluster_scale_shards + - e2e_sharded_cluster_shard_overrides + - e2e_sharded_cluster_secret + - e2e_sharded_cluster_statefulset_status + - e2e_sharded_cluster_upgrade_downgrade + - e2e_configure_tls_and_x509_simultaneously_sc + - e2e_sharded_cluster_tls_require_custom_ca + - e2e_tls_sharded_cluster_certs_prefix + - e2e_tls_sc_additional_certs + - e2e_tls_x509_configure_all_options_sc + - e2e_tls_x509_sc + - e2e_meko_mck_upgrade + - e2e_mongodb_custom_roles + - e2e_sharded_cluster_oidc_m2m_group + - e2e_sharded_cluster_oidc_m2m_user + - e2e_multi_cluster_oidc_m2m_group + - e2e_multi_cluster_oidc_m2m_user + + <<: *teardown_group + + - name: e2e_multi_cluster_2_clusters_task_group + max_hosts: -1 + <<: *setup_group + <<: *setup_and_teardown_task_cloudqa + tasks: + - e2e_multi_cluster_2_clusters_replica_set + - e2e_multi_cluster_2_clusters_clusterwide + <<: *teardown_group + + - name: e2e_multi_cluster_om_appdb_task_group + max_hosts: -1 + <<: *setup_group_multi_cluster + <<: *setup_and_teardown_task + tasks: + # Dedicated AppDB Multi-Cluster tests + - e2e_multi_cluster_appdb_validation + - e2e_multi_cluster_om_validation + - e2e_multi_cluster_appdb + - e2e_multi_cluster_appdb_cleanup + - e2e_multi_cluster_appdb_s3_based_backup_restore + - e2e_multi_cluster_appdb_disaster_recovery + - e2e_multi_cluster_appdb_disaster_recovery_force_reconfigure + - e2e_multi_cluster_om_networking_clusterwide + - e2e_multi_cluster_om_appdb_no_mesh + # Reused OM tests with AppDB Multi-Cluster topology + - e2e_appdb_tls_operator_upgrade_v1_32_to_mck + - e2e_om_appdb_flags_and_config + - e2e_om_appdb_upgrade + - e2e_om_appdb_monitoring_tls + - e2e_om_ops_manager_backup + - e2e_om_ops_manager_backup_light + - e2e_om_ops_manager_backup_liveness_probe + - e2e_om_ops_manager_pod_spec + - e2e_om_ops_manager_secure_config + - e2e_om_appdb_validation + - e2e_om_appdb_scram + - e2e_om_external_connectivity + - e2e_om_jvm_params + - e2e_om_migration + - e2e_om_localmode + - e2e_om_ops_manager_enable_local_mode_running_om + - e2e_om_localmode_multiple_pv + - e2e_om_weak_password + - e2e_om_ops_manager_backup_delete_sts_and_log_rotation + - e2e_om_ops_manager_backup_tls + - e2e_om_ops_manager_backup_s3_tls + - e2e_om_ops_manager_backup_tls_custom_ca + - e2e_om_ops_manager_backup_sharded_cluster + - e2e_om_ops_manager_backup_kmip + - e2e_om_ops_manager_https_enabled + - e2e_om_ops_manager_https_enabled_hybrid + - e2e_om_ops_manager_https_enabled_internet_mode + - e2e_om_ops_manager_https_enabled_prefix + - e2e_om_ops_manager_scale + - e2e_om_ops_manager_upgrade + - e2e_om_update_before_reconciliation + - e2e_om_feature_controls + - e2e_multi_cluster_appdb_upgrade_downgrade_v1_27_to_mck + - e2e_multi_cluster_sharded_disaster_recovery + # disabled tests: + # - e2e_om_multiple # multi-cluster failures in EVG + # - e2e_om_appdb_scale_up_down # test not "reused" for multi-cluster appdb + <<: *teardown_group + + - name: e2e_static_multi_cluster_om_appdb_task_group + max_hosts: -1 + <<: *setup_group_multi_cluster + <<: *setup_and_teardown_task + tasks: + # Dedicated AppDB Multi-Cluster tests + - e2e_multi_cluster_appdb_validation + - e2e_multi_cluster_om_validation + - e2e_multi_cluster_appdb + - e2e_multi_cluster_appdb_cleanup + - e2e_multi_cluster_appdb_s3_based_backup_restore + - e2e_multi_cluster_appdb_disaster_recovery + - e2e_multi_cluster_appdb_disaster_recovery_force_reconfigure + - e2e_multi_cluster_om_networking_clusterwide + - e2e_multi_cluster_om_appdb_no_mesh + # Reused OM tests with AppDB Multi-Cluster topology + - e2e_om_appdb_flags_and_config + - e2e_om_appdb_upgrade + - e2e_om_appdb_monitoring_tls + - e2e_om_ops_manager_backup + - e2e_om_ops_manager_backup_light + - e2e_om_ops_manager_backup_liveness_probe + - e2e_om_ops_manager_pod_spec + - e2e_om_ops_manager_secure_config + - e2e_om_appdb_validation + - e2e_om_appdb_scram + - e2e_om_external_connectivity + - e2e_om_jvm_params + - e2e_om_migration + # Deactivated tests + # - e2e_om_localmode + # - e2e_om_localmode_multiple_pv + # - e2e_om_ops_manager_https_enabled + # - e2e_om_ops_manager_https_enabled_hybrid + # - e2e_om_ops_manager_https_enabled_internet_mode + - e2e_om_ops_manager_https_enabled_prefix + - e2e_om_ops_manager_enable_local_mode_running_om + - e2e_om_weak_password + - e2e_om_ops_manager_backup_delete_sts_and_log_rotation + - e2e_om_ops_manager_backup_tls + - e2e_om_ops_manager_backup_s3_tls + - e2e_om_ops_manager_backup_tls_custom_ca + - e2e_om_ops_manager_backup_sharded_cluster + - e2e_om_ops_manager_backup_kmip + - e2e_om_ops_manager_scale + - e2e_om_ops_manager_upgrade + - e2e_multi_cluster_appdb_upgrade_downgrade_v1_27_to_mck + - e2e_om_update_before_reconciliation + - e2e_om_feature_controls + - e2e_multi_cluster_sharded_disaster_recovery + <<: *teardown_group + + # Dedicated task group for deploying OM Multi-Cluster when the operator is in the central cluster + # that is not in the mesh + - name: e2e_multi_cluster_om_operator_not_in_mesh_task_group + max_hosts: -1 + <<: *setup_group_multi_cluster + <<: *setup_and_teardown_task + tasks: + - e2e_multi_cluster_om_clusterwide_operator_not_in_mesh_networking + <<: *teardown_group + + + # This task group runs on Kind clusters. In theory ALL Ops Manager tests should be added here + # including the cluster-wide resources changing. Uses Cloud-qa. TODO why Cloud-qa? those are OM tests + - name: e2e_ops_manager_kind_only_task_group + max_hosts: -1 + <<: *setup_group + <<: *setup_and_teardown_task + tasks: + - e2e_om_appdb_external_connectivity + - e2e_om_appdb_flags_and_config + - e2e_om_appdb_monitoring_tls + - e2e_om_appdb_multi_change + - e2e_om_appdb_scale_up_down + - e2e_om_appdb_upgrade + - e2e_om_appdb_validation + - e2e_om_appdb_scram + - e2e_om_external_connectivity + - e2e_om_jvm_params + - e2e_om_migration + - e2e_om_localmode + - e2e_om_ops_manager_enable_local_mode_running_om + - e2e_om_localmode_multiple_pv + - e2e_om_weak_password + - e2e_om_ops_manager_backup_delete_sts_and_log_rotation + - e2e_om_ops_manager_backup_light + - e2e_om_ops_manager_backup_tls + - e2e_om_ops_manager_backup_s3_tls + - e2e_om_ops_manager_backup_restore_minio + - e2e_om_ops_manager_backup_tls_custom_ca + - e2e_om_ops_manager_backup_liveness_probe + - e2e_om_ops_manager_backup_sharded_cluster + - e2e_om_ops_manager_backup_kmip + - e2e_om_ops_manager_pod_spec + - e2e_om_ops_manager_https_enabled + - e2e_om_ops_manager_https_enabled_hybrid + - e2e_om_ops_manager_https_enabled_internet_mode + - e2e_om_ops_manager_https_enabled_prefix + - e2e_om_ops_manager_scale + - e2e_om_validation_webhook + - e2e_om_ops_manager_secure_config + - e2e_om_update_before_reconciliation + - e2e_om_multiple + - e2e_om_feature_controls + - e2e_vault_setup_om + - e2e_vault_setup_om_backup + <<: *teardown_group + + - name: e2e_static_ops_manager_kind_only_task_group + max_hosts: -1 + <<: *setup_group + <<: *setup_and_teardown_task + tasks: + - e2e_om_appdb_external_connectivity + - e2e_om_appdb_flags_and_config + - e2e_om_appdb_monitoring_tls + - e2e_om_appdb_multi_change + - e2e_om_appdb_scale_up_down + - e2e_om_appdb_upgrade + - e2e_om_appdb_validation + - e2e_om_appdb_scram + - e2e_om_external_connectivity + - e2e_om_jvm_params + - e2e_om_weak_password + - e2e_om_ops_manager_backup_delete_sts_and_log_rotation + - e2e_om_ops_manager_backup_light + - e2e_om_ops_manager_backup_tls + - e2e_om_ops_manager_backup_s3_tls + - e2e_om_ops_manager_backup_restore_minio + - e2e_om_ops_manager_backup_tls_custom_ca + - e2e_om_ops_manager_backup_liveness_probe + - e2e_om_ops_manager_backup_sharded_cluster + - e2e_om_ops_manager_backup_kmip + - e2e_om_ops_manager_pod_spec + - e2e_om_ops_manager_scale + - e2e_om_validation_webhook + - e2e_om_ops_manager_secure_config + - e2e_om_update_before_reconciliation + - e2e_om_multiple + - e2e_om_feature_controls + - e2e_vault_setup_om + - e2e_vault_setup_om_backup + # Deactivated tests + # - e2e_om_localmode_multiple_pv + # - e2e_om_localmode + # - e2e_om_ops_manager_https_enabled_hybrid + # - e2e_om_ops_manager_https_enabled_internet_mode + # - e2e_om_ops_manager_https_enabled + # - e2e_om_ops_manager_https_enabled_prefix + <<: *teardown_group + + - name: e2e_smoke_task_group + max_hosts: -1 + <<: *setup_group + <<: *setup_and_teardown_task + tasks: + - e2e_om_ops_manager_backup + <<: *teardown_group + + - name: e2e_ops_manager_kind_5_0_only_task_group + max_hosts: -1 + <<: *setup_group + <<: *setup_and_teardown_task + tasks: + - e2e_om_remotemode + - e2e_om_ops_manager_backup_restore + - e2e_om_ops_manager_queryable_backup + - e2e_om_ops_manager_backup + <<: *teardown_group + + # Tests features only supported on OM60 + - name: e2e_ops_manager_kind_6_0_only_task_group + max_hosts: -1 + <<: *setup_group + <<: *setup_and_teardown_task + tasks: + - e2e_om_ops_manager_prometheus + <<: *teardown_group + + # Tests features only supported on OM70 and OM80, its only upgrade test as we test upgrading from 6 to 7 or 7 to 8 + - name: e2e_ops_manager_upgrade_only_task_group + max_hosts: -1 + <<: *setup_group + <<: *setup_and_teardown_task + tasks: + - e2e_om_ops_manager_upgrade + <<: *teardown_group + + # Tests features only supported on OM60 + - name: e2e_static_ops_manager_kind_6_0_only_task_group + max_hosts: -1 + <<: *setup_group + <<: *setup_and_teardown_task + tasks: + - e2e_om_ops_manager_prometheus + <<: *teardown_group + + - name: e2e_kind_olm_group + max_hosts: -1 + <<: *setup_group + <<: *setup_and_teardown_task + setup_task_can_fail_task: true + setup_task: + # Even if the two first tasks are already part of setup_and_teardown_task, + # we need to repeat them because we are overriding the setup_task variable of the YAML anchor + - func: cleanup_exec_environment + - func: configure_docker_auth + - func: setup_kubernetes_environment + - func: setup_prepare_openshift_bundles + - func: install_olm + tasks: + - e2e_olm_operator_upgrade + - e2e_olm_operator_upgrade_with_resources + - e2e_olm_meko_operator_upgrade_with_resources + <<: *teardown_group + +buildvariants: + ## Unit tests + lint build variant + + - name: unit_tests + display_name: "unit_tests" + tags: [ "unit_tests" ] + run_on: + - ubuntu2204-small + tasks: + - name: "unit_task_group" + + ## Build variants for E2E tests + + # The pattern for naming build variants for E2E tests: + # e2e___[_] + # where is any of mdb|om|operator + # where is any of kind|openshift + # where is any of ubuntu|ubi + # where denotes the OM version tested (e.g. om50, om60, cloudqa) - used only for MDB tests + + # MongoDBCommunity build variant + - name: e2e_mdb_community + display_name: e2e_mdb_community + tags: [ "e2e_test_suite"] + run_on: + - ubuntu2204-large + <<: *community_dependency + tasks: + - name: e2e_mdb_community_task_group + + ## MongoDB build variants + - name: e2e_mdb_kind_ubi_cloudqa + display_name: e2e_mdb_kind_ubi_cloudqa + tags: [ "e2e_test_suite", "cloudqa" ] + run_on: + - ubuntu2204-large + <<: *base_no_om_image_dependency + tasks: + - name: e2e_mdb_kind_cloudqa_task_group + + - name: e2e_custom_domain_mdb_kind_ubi_cloudqa + display_name: e2e_custom_domain_mdb_kind_ubi_cloudqa + tags: [ "e2e_test_suite", "cloudqa" ] + run_on: + - ubuntu2204-large + <<: *base_no_om_image_dependency + tasks: + - name: e2e_custom_domain_task_group + + - name: e2e_static_mdb_kind_ubi_cloudqa + display_name: e2e_static_mdb_kind_ubi_cloudqa + tags: [ "e2e_test_suite", "cloudqa" ] + run_on: + - ubuntu2204-large + <<: *base_no_om_image_dependency + tasks: + - name: e2e_mdb_kind_cloudqa_task_group + + - name: e2e_static_custom_domain_mdb_kind_ubi_cloudqa + display_name: e2e_static_custom_domain_mdb_kind_ubi_cloudqa + tags: [ "e2e_test_suite", "cloudqa" ] + run_on: + - ubuntu2204-large + depends_on: + - name: build_operator_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + - name: build_init_appdb_images_ubi + variant: init_test_run + - name: build_init_om_images_ubi + variant: init_test_run + - name: build_init_database_image_ubi + variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run + tasks: + - name: e2e_custom_domain_task_group + + - name: e2e_mdb_openshift_ubi_cloudqa + display_name: e2e_mdb_openshift_ubi_cloudqa + tags: [ "e2e_openshift_test_suite", "cloudqa" ] + depends_on: + - name: build_operator_ubi + variant: init_test_run + - name: build_init_database_image_ubi + variant: init_test_run + - name: build_database_image_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + run_on: + - ubuntu2204-small + tasks: + - name: e2e_mdb_openshift_ubi_cloudqa_task_group + + # This name is on purpose reversed from e2e_static_openshift to e2e_openshift_static. + # That is because we run a regex + # in evergreen for all variants matching e2e_static-*, but we do not want to run openshift variants on every pr. + - name: e2e_openshift_static_mdb_ubi_cloudqa + display_name: e2e_openshift_static_mdb_ubi_cloudqa + tags: [ "e2e_openshift_test_suite", "cloudqa" ] + depends_on: + - name: build_operator_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + - name: build_init_database_image_ubi + variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run + run_on: + - ubuntu2204-small + tasks: + - name: e2e_mdb_openshift_ubi_cloudqa_task_group + + ## Ops Manager build variants + + # Isolated Ops Manager Tests for 6.0 version + - name: e2e_om60_kind_ubi + display_name: e2e_om60_kind_ubi + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om6_dependency + tasks: + - name: e2e_ops_manager_kind_only_task_group + - name: e2e_ops_manager_kind_5_0_only_task_group + - name: e2e_ops_manager_kind_6_0_only_task_group + + # Isolated Ops Manager Tests for 6.0 version + - name: e2e_static_om60_kind_ubi + display_name: e2e_static_om60_kind_ubi + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om6_dependency + tasks: + - name: e2e_static_ops_manager_kind_only_task_group + - name: e2e_static_ops_manager_kind_6_0_only_task_group + + - name: e2e_om70_kind_ubi + display_name: e2e_om70_kind_ubi + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om7_dependency + tasks: + - name: e2e_ops_manager_kind_only_task_group + - name: e2e_ops_manager_kind_5_0_only_task_group + - name: e2e_ops_manager_kind_6_0_only_task_group + - name: e2e_ops_manager_upgrade_only_task_group + + - name: e2e_static_om70_kind_ubi + display_name: e2e_static_om70_kind_ubi + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om7_dependency + tasks: + - name: e2e_static_ops_manager_kind_only_task_group + - name: e2e_static_ops_manager_kind_6_0_only_task_group + - name: e2e_ops_manager_upgrade_only_task_group + + - name: e2e_om80_kind_ubi + display_name: e2e_om80_kind_ubi + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om8_dependency + tasks: + - name: e2e_ops_manager_kind_only_task_group + - name: e2e_ops_manager_kind_5_0_only_task_group + - name: e2e_ops_manager_kind_6_0_only_task_group + - name: e2e_ops_manager_upgrade_only_task_group + + - name: e2e_static_om80_kind_ubi + display_name: e2e_static_om80_kind_ubi + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om8_dependency + tasks: + - name: e2e_static_ops_manager_kind_only_task_group + - name: e2e_static_ops_manager_kind_6_0_only_task_group + - name: e2e_ops_manager_upgrade_only_task_group + + - name: e2e_operator_race_ubi_with_telemetry + display_name: e2e_operator_race_ubi_with_telemetry + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu1804-xlarge + <<: *base_om7_dependency + tasks: + - name: e2e_operator_race_with_telemetry_task_group + + - name: e2e_smoke + display_name: e2e_smoke + tags: [ "e2e_test_suite", "e2e_smoke_release_test_suite" ] + run_on: + - ubuntu2204-large + allowed_requesters: [ "patch", "github_tag" ] + depends_on: + - name: build_test_image + variant: init_test_run + tasks: + - name: e2e_smoke_task_group + + - name: e2e_static_smoke + display_name: e2e_static_smoke + tags: [ "e2e_test_suite", "e2e_smoke_release_test_suite" ] + run_on: + - ubuntu2204-large + allowed_requesters: [ "patch", "github_tag" ] + depends_on: + - name: build_test_image + variant: init_test_run + tasks: + - name: e2e_smoke_task_group + + - name: e2e_multi_cluster_kind + display_name: e2e_multi_cluster_kind + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om6_dependency + tasks: + - name: e2e_multi_cluster_kind_task_group + + - name: e2e_static_multi_cluster_kind + display_name: e2e_static_multi_cluster_kind + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om6_dependency + tasks: + - name: e2e_multi_cluster_kind_task_group + + - name: e2e_multi_cluster_2_clusters + display_name: e2e_multi_cluster_2_clusters + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om6_dependency + tasks: + - name: e2e_multi_cluster_2_clusters_task_group + + - name: e2e_static_multi_cluster_2_clusters + display_name: e2e_static_multi_cluster_2_clusters + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om6_dependency + tasks: + - name: e2e_multi_cluster_2_clusters_task_group + + - name: e2e_multi_cluster_om_appdb + display_name: e2e_multi_cluster_om_appdb + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om6_dependency + tasks: + - name: e2e_multi_cluster_om_appdb_task_group + + - name: e2e_static_multi_cluster_om_appdb + display_name: e2e_static_multi_cluster_om_appdb + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om6_dependency + tasks: + - name: e2e_static_multi_cluster_om_appdb_task_group + + - name: e2e_multi_cluster_om_operator_not_in_mesh + display_name: e2e_multi_cluster_om_operator_not_in_mesh + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om7_dependency + tasks: + - name: e2e_multi_cluster_om_operator_not_in_mesh_task_group + + ## Operator tests build variants + + - name: e2e_operator_kind_ubi_cloudqa + display_name: e2e_operator_kind_ubi_cloudqa + tags: [ "e2e_test_suite", "cloudqa" ] + run_on: + - ubuntu2204-large + <<: *base_no_om_image_dependency + tasks: + - name: e2e_operator_task_group + + - name: e2e_static_operator_kind_ubi_cloudqa + display_name: e2e_static_operator_kind_ubi_cloudqa + tags: [ "e2e_test_suite", "cloudqa" ] + run_on: + - ubuntu2204-large + <<: *base_no_om_image_dependency + tasks: + - name: e2e_static_operator_task_group + + - name: e2e_operator_no_webhook_roles_cloudqa + display_name: e2e_operator_no_webhook_roles_cloudqa + tags: [ "e2e_test_suite", "cloudqa" ] + run_on: + - ubuntu2204-large + <<: *base_no_om_image_dependency + tasks: + - name: e2e_mdb_kind_no_webhook_roles_cloudqa_task_group + + - name: e2e_kind_olm_ubi + display_name: e2e_kind_olm_ubi + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + depends_on: + - name: build_om_images + variant: build_om60_images + - name: build_operator_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + - name: build_init_om_images_ubi + variant: init_test_run + - name: build_init_database_image_ubi + variant: init_test_run + - name: build_database_image_ubi + variant: init_test_run + - name: build_init_appdb_images_ubi + variant: init_test_run + - name: prepare_and_upload_openshift_bundles_for_e2e + variant: init_tests_with_olm + tasks: + - name: e2e_kind_olm_group + + - name: e2e_static_kind_olm_ubi + display_name: e2e_static_kind_olm_ubi + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + depends_on: + - name: build_om_images + variant: build_om60_images + - name: build_operator_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + - name: build_init_om_images_ubi + variant: init_test_run + - name: build_init_appdb_images_ubi + variant: init_test_run + - name: prepare_and_upload_openshift_bundles_for_e2e + variant: init_tests_with_olm + - name: build_init_database_image_ubi + variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run + tasks: + - name: e2e_kind_olm_group + + # This variants runs the tests from MCO with the MEKO operator binary + - name: e2e_mco_tests + display_name: "e2e_mco_tests" + tags: [ "e2e_mco_test_suite"] + <<: *community_dependency + run_on: + - ubuntu2004-large + + ## Manual (patch) E2E tests not run for every PR and commit + + - name: e2e_operator_perf + display_name: e2e_operator_perf + tags: [ "e2e_perf_test_suite" ] + allowed_requesters: [ "patch" ] + run_on: + - ubuntu1804-xlarge + <<: *base_om7_dependency + tasks: + - name: generate_perf_tasks_10_thread + + - name: e2e_operator_perf_one_thread + display_name: e2e_operator_perf_one_thread + tags: [ "e2e_perf_test_suite" ] + allowed_requesters: [ "patch" ] + run_on: + - ubuntu1804-xlarge + <<: *base_om7_dependency + tasks: + - name: generate_perf_tasks_one_thread + + - name: e2e_operator_perf_thirty + display_name: e2e_operator_perf_thirty + tags: [ "e2e_perf_test_suite" ] + allowed_requesters: [ "patch" ] + run_on: + - ubuntu1804-xlarge + <<: *base_om7_dependency + tasks: + - name: generate_perf_tasks_30_thread + + ### Prerequisites for E2E test suite + + - name: init_test_run + display_name: init_test_run + max_hosts: -1 + run_on: + - ubuntu2204-small + tasks: + - name: build_operator_ubi + - name: build_test_image + - name: build_mco_test_image + - name: build_init_appdb_images_ubi + - name: build_init_om_images_ubi + - name: build_init_database_image_ubi + - name: build_database_image_ubi + - name: build_agent_images_ubi + - name: build_readiness_probe_image + - name: build_upgrade_hook_image + - name: prepare_aws + + - name: init_release_agents_on_ecr + display_name: init_release_agents_on_ecr + # this enables us to run this variant either manually (patch) which pct does or during an OM bump (github_pr) + allowed_requesters: [ "patch", "github_pr" ] + tags: [ "release_agents_on_ecr" ] + # We want that to run first and finish asap. Digest pinning depends on this to succeed. + priority: 70 + run_on: + - ubuntu2204-large + tasks: + - name: release_agents_on_ecr_conditional + + - name: run_pre_commit + priority: 70 + display_name: run_pre_commit + allowed_requesters: [ "patch", "github_pr" ] + tags: [ "auto_bump" ] + run_on: + - ubuntu2204-small + tasks: + - name: run_precommit_and_push + + - name: init_tests_with_olm + display_name: init_tests_with_olm + depends_on: + - name: build_om_images + variant: build_om60_images + - name: build_operator_ubi + variant: init_test_run + - name: build_init_database_image_ubi + variant: init_test_run + - name: build_database_image_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + - name: build_init_appdb_images_ubi + variant: init_test_run + - name: build_init_om_images_ubi + variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run + run_on: + - ubuntu2204-small + tasks: + - name: prepare_and_upload_openshift_bundles_for_e2e + + ### End of build variants for E2E + + ### Variants that run on each commit + + - name: preflight_release_images_check_only + display_name: preflight_release_images_check_only + run_on: + - rhel90-large + tasks: + - name: preflight_images_task_group + + - name: build_om60_images + display_name: build_om60_images + run_on: + - ubuntu2204-small + tasks: + - name: build_om_images + + - name: preflight_om60_images + display_name: preflight_om60_images + run_on: + - rhel90-large + tasks: + - name: preflight_om_image + + - name: build_om70_images + display_name: build_om70_images + run_on: + - ubuntu2204-small + tasks: + - name: build_om_images + + - name: preflight_om70_images + display_name: preflight_om70_images + run_on: + - rhel90-large + tasks: + - name: preflight_om_image + + - name: build_om80_images + display_name: build_om80_images + run_on: + - ubuntu2204-small + tasks: + - name: build_om_images + + - name: preflight_om80_images + display_name: preflight_om80_images + run_on: + - rhel90-large + tasks: + - name: preflight_om_image + + ### Release build variants + + ## Adds versions as supported in the supported versions Database. + - name: release_images + display_name: release_images + tags: [ "release" ] + allowed_requesters: [ "patch", "github_tag" ] + max_hosts: -1 + run_on: + - release-ubuntu2204-large # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780 + depends_on: + - name: build_operator_ubi + variant: init_test_run + - name: build_init_om_images_ubi + variant: init_test_run + - name: build_init_appdb_images_ubi + variant: init_test_run + - name: build_init_database_image_ubi + variant: init_test_run + - name: build_database_image_ubi + variant: init_test_run + tasks: + - name: release_operator + - name: release_init_appdb + - name: release_init_database + - name: release_init_ops_manager + - name: release_database + # Once we release the operator, we will also release the init databases, we require them to be out first + # such that we can reference them and retrieve those binaries. + # Since we immediately run daily rebuild after creating the image, we can ensure that the init_database is out + # such that the agent image build can use it. + - name: release_agent_operator_release + depends_on: + - name: release_init_database + + - name: preflight_release_images + display_name: preflight_release_images + tags: [ "release" ] + allowed_requesters: [ "patch", "github_tag" ] + depends_on: + - name: "*" + variant: release_images + run_on: + - rhel90-large + expansions: + preflight_submit: true + tasks: + - name: preflight_images_task_group + + - name: prepare_openshift_bundles + display_name: prepare_openshift_bundles + tags: [ "release" ] + allowed_requesters: [ "patch", "github_tag" ] + depends_on: + - name: "*" + variant: release_images + - name: "*" + variant: preflight_release_images + run_on: + - ubuntu2204-large + tasks: + - name: run_conditionally_prepare_and_upload_openshift_bundles + + # It will be called by pct while bumping the agent cloud manager image + - name: release_agent + display_name: (Static Containers) Release Agent matrix + tags: [ "release_agent" ] + run_on: + - release-ubuntu2204-large # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780 + depends_on: + - variant: init_release_agents_on_ecr + name: '*' + - variant: e2e_multi_cluster_kind + name: '*' + - variant: e2e_static_multi_cluster_2_clusters + name: '*' + - variant: e2e_mdb_kind_ubi_cloudqa + name: '*' + - variant: e2e_static_mdb_kind_ubi_cloudqa + name: '*' + tasks: + - name: release_agent + + # Only called manually, It's used for testing the task release_agents_on_ecr in case the release.json + # has not changed, and you still want to push the images to ecr. + - name: manual_ecr_release_agent + display_name: Manual Agent Release for all versions + tags: [ "release_all_agents_manually" ] + run_on: + - ubuntu2204-large + tasks: + - name: release_all_agents_on_ecr + + # These variants are used to test the code snippets and each one can be used in patches + # Prerelease is especially used when the repo is tagged + # More details in the TD: https://docs.google.com/document/d/1fuTxfRtP8QPtn7sKYxQM_AGcD6xycTZH8svngGxyKhc/edit?tab=t.0#bookmark=id.e8uva0393mbe + - name: public_gke_code_snippets + display_name: public_gke_code_snippets + allowed_requesters: ["patch"] + run_on: + - ubuntu2204-small + tasks: + - name: gke_code_snippets_task_group + + - name: prerelease_gke_code_snippets + display_name: prerelease_gke_code_snippets + tags: [ "release" ] + allowed_requesters: ["patch", "github_tag"] + depends_on: + - variant: release_images + name: '*' + patch_optional: true + run_on: + - ubuntu2204-small + tasks: + - name: gke_code_snippets_task_group + + - name: private_gke_code_snippets + display_name: private_gke_code_snippets + allowed_requesters: ["patch"] + run_on: + - ubuntu2204-small + <<: *base_om8_dependency + tasks: + - name: gke_code_snippets_task_group + + - name: private_kind_code_snippets + display_name: private_kind_code_snippets + tags: [ "e2e_test_suite" ] + allowed_requesters: ["patch", "github_pr"] + run_on: + - ubuntu2204-large + <<: *base_om8_dependency + tasks: + - name: kind_code_snippets_task_group + + - name: prerelease_kind_code_snippets + display_name: prerelease_kind_code_snippets + allowed_requesters: ["patch"] + run_on: + - ubuntu2204-large + tasks: + - name: kind_code_snippets_task_group + + - name: public_kind_code_snippets + display_name: public_kind_code_snippets + allowed_requesters: ["patch"] + run_on: + - ubuntu2204-large + tasks: + - name: kind_code_snippets_task_group + + ### Build variants for manual patch only + + - name: backup_csv_images + display_name: "Backup CSV Images" + allowed_requesters: [ "patch" ] + run_on: + - ubuntu2204-small + tasks: + - name: backup_csv_images_dry_run + - name: backup_csv_images_limit_3 + - name: backup_csv_images_all + + - name: publish_om60_images + display_name: publish_om60_images + allowed_requesters: [ "patch", "github_pr" ] + run_on: + - release-ubuntu2204-large # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780 + depends_on: + - variant: e2e_om60_kind_ubi + name: '*' + - variant: e2e_static_om60_kind_ubi + name: '*' + tasks: + - name: publish_ops_manager + - name: release_agent + + - name: publish_om70_images + display_name: publish_om70_images + allowed_requesters: [ "patch", "github_pr" ] + run_on: + - release-ubuntu2204-large # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780 + depends_on: + - variant: e2e_om70_kind_ubi + name: '*' + - variant: e2e_static_om70_kind_ubi + name: '*' + tasks: + - name: publish_ops_manager + - name: release_agent + + - name: publish_om80_images + display_name: publish_om80_images + allowed_requesters: [ "patch", "github_pr" ] + run_on: + - release-ubuntu2204-large # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780 + depends_on: + - variant: e2e_om80_kind_ubi + name: '*' + - variant: e2e_static_om80_kind_ubi + name: '*' + tasks: + - name: publish_ops_manager + - name: release_agent diff --git a/.evergreen.yml b/.evergreen.yml index 058c0ea1f..86f5c4beb 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -1,1973 +1,25 @@ -# 2h timeout for all the tasks -exec_timeout_secs: 7200 +# MongoDB Kubernetes Operator CI Configuration +# This file now uses a modular structure for better maintainability +# All configuration has been organized into logical, focused files include: - - filename: .evergreen-functions.yml - - filename: .evergreen-tasks.yml - - filename: .evergreen-mco.yml + - filename: .evergreen/main.yml -variables: - - &ops_manager_60_latest 6.0.27 # The order/index is important, since these are anchors. Please do not change - - - &ops_manager_70_latest 7.0.15 # The order/index is important, since these are anchors. Please do not change - - - &ops_manager_80_latest 8.0.7 # The order/index is important, since these are anchors. Please do not change - - # The dependency unification between static and non-static is intentional here. - # Even though some images are exclusive, in EVG they all are built once and in parallel. - # It is not worth the effort of splitting them out. - # Once Static Containers are default, this piece can be cleaned up. - - &base_om6_dependency - depends_on: - - name: build_om_images - variant: build_om60_images - - name: build_operator_ubi - variant: init_test_run - - name: build_init_database_image_ubi - variant: init_test_run - - name: build_database_image_ubi - variant: init_test_run - - name: build_test_image - variant: init_test_run - - name: build_init_appdb_images_ubi - variant: init_test_run - - name: build_init_om_images_ubi - variant: init_test_run - - name: build_agent_images_ubi - variant: init_test_run - - - &base_no_om_image_dependency - depends_on: - - name: build_om_images - variant: build_om70_images - - name: build_operator_ubi - variant: init_test_run - - name: build_init_database_image_ubi - variant: init_test_run - - name: build_database_image_ubi - variant: init_test_run - - name: build_init_om_images_ubi - variant: init_test_run - - name: build_test_image - variant: init_test_run - - name: build_init_appdb_images_ubi - variant: init_test_run - - name: build_agent_images_ubi - variant: init_test_run - - - &community_dependency - depends_on: - - name: build_operator_ubi - variant: init_test_run - - name: build_test_image - variant: init_test_run - - name: build_agent_images_ubi - variant: init_test_run - - name: build_readiness_probe_image - variant: init_test_run - - name: build_upgrade_hook_image - variant: init_test_run - - name: build_mco_test_image - variant: init_test_run - - - &setup_group - setup_group_can_fail_task: true - setup_group: - - func: clone - - func: download_kube_tools - - func: setup_building_host - - - &setup_group_multi_cluster - setup_group_can_fail_task: true - setup_group: - - func: clone - - func: download_kube_tools - - func: setup_building_host - - func: build_multi_cluster_binary - - - &setup_and_teardown_group_gke_code_snippets - setup_task_can_fail_task: true - setup_group: - - func: clone - - func: setup_gcloud_cli - - func: setup_mongosh - - func: download_kube_tools - - func: build_multi_cluster_binary - teardown_group: - - func: upload_code_snippets_logs - - - &setup_and_teardown_group_kind_code_snippets - setup_task_can_fail_task: true - setup_group: - - func: clone - - func: cleanup_exec_environment - - func: download_kube_tools - - func: configure_docker_auth - - func: setup_kubernetes_environment - teardown_task: - - func: upload_e2e_logs - - func: upload_code_snippets_logs - - - &setup_and_teardown_task_cloudqa - setup_task_can_fail_task: true - setup_task: - - func: cleanup_exec_environment - - func: configure_docker_auth - - func: setup_kubernetes_environment - - func: setup_cloud_qa - teardown_task_can_fail_task: true - teardown_task: - - func: upload_e2e_logs - - func: teardown_kubernetes_environment - - func: teardown_cloud_qa - - - &setup_and_teardown_task - setup_task_can_fail_task: true - setup_task: - - func: cleanup_exec_environment - - func: configure_docker_auth - - func: setup_kubernetes_environment - teardown_task_can_fail_task: true - teardown_task: - - func: upload_e2e_logs - - func: teardown_kubernetes_environment - - - &teardown_group - teardown_group: - - func: prune_docker_resources - - func: run_retry_script - - - &base_om7_dependency - depends_on: - - name: build_om_images - variant: build_om70_images - - name: build_operator_ubi - variant: init_test_run - - name: build_init_database_image_ubi - variant: init_test_run - - name: build_database_image_ubi - variant: init_test_run - - name: build_test_image - variant: init_test_run - - name: build_init_appdb_images_ubi - variant: init_test_run - - name: build_init_om_images_ubi - variant: init_test_run - - name: build_agent_images_ubi - variant: init_test_run - - - &base_om8_dependency - depends_on: - - name: build_om_images - variant: build_om80_images - - name: build_operator_ubi - variant: init_test_run - - name: build_init_database_image_ubi - variant: init_test_run - - name: build_database_image_ubi - variant: init_test_run - - name: build_test_image - variant: init_test_run - - name: build_init_appdb_images_ubi - variant: init_test_run - - name: build_init_om_images_ubi - variant: init_test_run - - name: build_agent_images_ubi - variant: init_test_run - -parameters: - - key: evergreen_retry - value: "true" - description: set this to false to suppress retries on failure - - - key: pin_tag_at - value: 10:00 - description: Pin tags at this time of the day. Midnight by default for periodic and 10 for releases. - - - key: OVERRIDE_VERSION_ID - value: "" - description: "Patch id to reuse images from other Evergreen build" - - - key: code_snippets_teardown - value: "true" - description: set this to false if you would like to keep the clusters created during code snippets tests - - - key: code_snippets_reset - value: "false" - description: set this to true if you would like to delete the resources created in the code snippet tests, but keep the clusters - - - key: code_snippets_commit_output - value: "false" - description: set this to true if you would like the pipeline to automatically push a branch with updated snippets outputs - - -# Triggered manually or by PCT. -patch_aliases: - - alias: "periodic_builds" - variant_tags: [ "periodic_build" ] - task: ".*" - - alias: "periodic_teardowns" - variant_tags: [ "periodic_teardown" ] - task: ".*" - - alias: "release_agent" - variant_tags: [ "release_agent" ] - task: ".*" - - alias: "release_all_agents_manually" - variant_tags: [ "release_all_agents_manually" ] - task: ".*" - - alias: "release" - variant_tags: [ "release", "e2e_smoke_release_test_suite" ] - task_tags: [ "image_release", "image_preflight", "openshift_bundles", "code_snippets", "patch-run" ] - - alias: "smoke_test_release" - variant_tags: [ "e2e_smoke_release_test_suite" ] - task_tags: [ "patch-run" ] - - alias: "patch-run-cloudqa" - variant_tags: [ "cloudqa" ] - task: ".*" - -# Triggered whenever the GitHub PR is created -github_pr_aliases: - - variant_tags: [ "unit_tests" ] - task_tags: [ "unit_tests" ] - - variant_tags: [ "e2e_test_suite" ] - task_tags: [ "patch-run" ] - - variant_tags: [ "e2e_mco_test_suite" ] - task: ".*" - - variant_tags: [ "auto_bump" ] - task_tags: [ "patch-run" ] - - variant_tags: [ "release_agents_on_ecr" ] - task: ".*" - -# Allows to see evergreen checks in GitHub commits -# https://github.com/mongodb/mongodb-kubernetes/commits/master/ -github_checks_aliases: - - variant: ".*" - task: ".*" - -# Triggered on git tag -git_tag_aliases: - - git_tag: "^(\\d+\\.)?(\\d+\\.)?(\\d+)$" - variant_tags: [ "release", "e2e_smoke_release_test_suite" ] - task_tags: [ "image_release", "image_preflight", "openshift_bundles", "code_snippets", "patch-run" ] tasks: - - name: unit_tests_golang - tags: [ "unit_tests" ] - commands: - - func: "test_golang_unit" - - - name: unit_tests_python - tags: [ "unit_tests" ] - commands: - - func: "test_python_unit" - - - name: sbom_tests - tags: [ "unit_tests" ] - # The SBOM tests run only on commit builds. Running this on patches might cause false-positive failures - # because certain images might not be there yet. Such situation happens for OM image upgrades for example. - # See https://docs.devprod.prod.corp.mongodb.com/evergreen/Project-Configuration/Project-Configuration-Files#limiting-when-a-task-or-variant-will-run - patchable: false - commands: - - func: "test_sboms" - - - name: lint_repo - tags: [ "unit_tests" ] - commands: - - func: lint_repo - - - name: release_operator - tags: [ "image_release" ] - allowed_requesters: [ "patch", "github_tag" ] - commands: - - func: clone - - func: setup_building_host - - func: quay_login - - func: setup_docker_sbom - - func: pipeline - vars: - image_name: operator - include_tags: release - - # Releases init images to Quay - - name: release_init_appdb - tags: [ "image_release" ] - allowed_requesters: [ "patch", "github_tag" ] - commands: - - func: clone - - func: setup_building_host - - func: quay_login - - func: setup_docker_sbom - - func: pipeline - vars: - image_name: init-appdb - include_tags: release - - - name: release_init_database - tags: [ "image_release" ] - allowed_requesters: [ "patch", "github_tag" ] - commands: - - func: clone - - func: setup_building_host - - func: quay_login - - func: setup_docker_sbom - - func: pipeline - vars: - image_name: init-database - include_tags: release - - - name: release_init_ops_manager - tags: [ "image_release" ] - allowed_requesters: [ "patch", "github_tag" ] - commands: - - func: clone - - func: setup_building_host - - func: quay_login - - func: setup_docker_sbom - - func: pipeline - vars: - image_name: init-ops-manager - include_tags: release - - - name: release_agent_operator_release - tags: [ "image_release" ] - allowed_requesters: [ "patch", "github_tag" ] - commands: - - func: clone - - func: setup_building_host - - func: quay_login - - func: setup_docker_sbom - - func: pipeline - vars: - image_name: agent - include_tags: release - - # pct only triggers this variant once a new agent image is out - - name: release_agent - # this enables us to run this variant either manually (patch) which pct does or during an OM bump (github_pr) - allowed_requesters: [ "patch", "github_pr" ] - commands: - - func: clone - - func: setup_building_host - - func: quay_login - - func: setup_docker_sbom - - func: pipeline - vars: - image_name: agent-pct - include_tags: release - - - name: run_precommit_and_push - tags: ["patch-run"] - commands: - - func: clone - - func: python_venv - - func: download_kube_tools - - func: setup_shellcheck - - command: github.generate_token - params: - expansion_name: GH_TOKEN - - command: subprocess.exec - type: setup - params: - include_expansions_in_env: - - GH_TOKEN - working_dir: src/github.com/mongodb/mongodb-kubernetes - binary: scripts/evergreen/precommit_bump.sh - - # Pct only triggers this variant once a new agent image is out - # these releases the agent with the operator suffix (not patch id) on ecr to allow for digest pinning to pass. - # For this to work, we rely on skip_tags which is used to determine whether - # we want to release on quay or not, in this case - ecr instead. - # We rely on the init_database from ecr for the agent x operator images. - # This runs on agent releases that are not concurrent with operator releases. - - name: release_agents_on_ecr_conditional - commands: - - func: clone - - func: run_task_conditionally - vars: - condition_script: scripts/evergreen/should_release_agents_on_ecr.sh - variant: init_release_agents_on_ecr - task: release_agents_on_ecr - - - name: release_agents_on_ecr - # this enables us to run this variant either manually (patch) which pct does or during an OM bump (github_pr) - allowed_requesters: [ "patch", "github_pr" ] - priority: 70 - commands: - - func: clone - - func: setup_building_host - - func: pipeline - vars: - image_name: agent-pct - skip_tags: release - - - name: release_all_agents_on_ecr - # this enables us to run this manually (patch) and release all agent versions to ECR - # it's needed during operator new version release process - e2e tests (especially olm tests) - # will look for agent with new operator version suffix, but during PR checks we only build - # agent versions for most recent major OM versions and the tests will fail. Before running the PR - # we have to manually release all agents to ECR by triggering this patch - allowed_requesters: [ "patch" ] - commands: - - func: clone - - func: setup_building_host - - func: pipeline - vars: - image_name: agent-pct - skip_tags: release - all_agents: true - - - name: build_test_image - commands: - - func: clone - - func: setup_building_host - - func: build_multi_cluster_binary - - func: pipeline - vars: - image_name: test - - - name: build_mco_test_image - commands: - - func: clone - - func: setup_building_host - - func: pipeline - vars: - image_name: mco-test - - - name: build_operator_ubi - commands: - - func: clone - - func: setup_building_host - - func: pipeline - vars: - skip_tags: ubuntu,release - distro: ubi - image_name: operator - - - name: build_init_om_images_ubi - commands: - - func: clone - - func: setup_building_host - - func: pipeline - vars: - image_name: init-ops-manager - skip_tags: ubuntu,release - - - name: build_init_appdb_images_ubi - commands: - - func: clone - - func: setup_building_host - - func: pipeline - vars: - image_name: init-appdb - skip_tags: ubuntu,release - - - name: build_agent_images_ubi - depends_on: - - name: build_init_database_image_ubi - variant: init_test_run - commands: - - func: clone - - func: setup_building_host - - func: pipeline - vars: - image_name: agent - skip_tags: ubuntu,release - - - name: build_init_database_image_ubi - commands: - - func: clone - - func: setup_building_host - - func: pipeline - vars: - image_name: init-database - skip_tags: ubuntu,release - - - name: build_database_image_ubi - commands: - - func: clone - - func: setup_building_host - - func: pipeline - vars: - image_name: database - skip_tags: ubuntu,release - - - name: build_readiness_probe_image - commands: - - func: clone - - func: setup_building_host - - func: pipeline - vars: - image_name: readiness-probe - skip_tags: ubuntu,release - - - name: build_upgrade_hook_image - commands: - - func: clone - - func: setup_building_host - - func: pipeline - vars: - image_name: upgrade-hook - skip_tags: ubuntu,release - - - name: prepare_aws - priority: 59 - commands: - - func: clone - - func: setup_jq - - func: setup_aws - - func: prepare_aws - - - name: run_retry_script - tags: [ "patch-run" ] - commands: - - func: run_retry_script - - - name: generate_perf_tasks_one_thread - commands: - - func: clone - - func: generate_perf_tests_tasks - vars: - variant: e2e_operator_perf_one_thread - size: small - - - name: generate_perf_tasks_10_thread - commands: - - func: clone - - func: generate_perf_tests_tasks - vars: - variant: e2e_operator_perf - size: small - - - name: generate_perf_tasks_30_thread - commands: - - func: clone - - func: generate_perf_tests_tasks - vars: - variant: e2e_operator_perf_thirty - size: small - - - name: release_database - tags: [ "image_release" ] - allowed_requesters: [ "patch", "github_tag" ] - commands: - - func: clone - - func: setup_building_host - - func: quay_login - - func: setup_docker_sbom - - func: pipeline - vars: - image_name: database - - - name: build_om_images + # add a noop task because if the only task in a variant is git_tag_only: true Evergreen doesn't start it at all + - name: noop commands: - - func: clone - - func: setup_building_host - - func: pipeline - vars: - image_name: ops-manager - skip_tags: release - - - name: publish_ops_manager - commands: - - func: clone - - func: setup_building_host - - func: quay_login - - func: setup_docker_sbom - - func: pipeline - vars: - image_name: ops-manager - include_tags: release - - - name: prepare_and_upload_openshift_bundles_for_e2e - commands: - - func: clone - - func: setup_building_host - - func: download_kube_tools - - func: setup_prepare_openshift_bundles - - func: prepare_openshift_bundles_for_e2e - - - name: prepare_and_upload_openshift_bundles - tags: [ "openshift_bundles" ] - commands: - - func: clone - - func: setup_aws - - func: configure_docker_auth - - func: setup_prepare_openshift_bundles - - func: prepare_openshift_bundles - - func: update_evergreen_expansions - - func: upload_openshift_bundle - vars: - # mongoDbOperator expansion is added in update_evergreen_expansions func from release.json - bundle_file_name: "mck-operator-certified-${mongodbOperator}.tgz" - - - name: run_conditionally_prepare_and_upload_openshift_bundles - tags: [ "openshift_bundles" ] - commands: - - func: clone - - func: run_task_conditionally - vars: - condition_script: scripts/evergreen/should_prepare_openshift_bundles.sh - variant: prepare_openshift_bundles - task: prepare_and_upload_openshift_bundles - - - name: backup_csv_images_dry_run - commands: - - func: clone - - func: quay_login - - command: subprocess.exec - params: - working_dir: src/github.com/mongodb/mongodb-kubernetes - binary: python3 - args: - - scripts/dev/release/backup_csv_images.py - - scripts/dev/release/1.2.0.clusterserviceversion.yaml - - --dry-run - - --verbose - - name: backup_csv_images_limit_3 - commands: - - func: clone - - func: quay_login - - command: subprocess.exec - params: - working_dir: src/github.com/mongodb/mongodb-kubernetes - binary: python3 - args: - - scripts/dev/release/backup_csv_images.py - - scripts/dev/release/1.2.0.clusterserviceversion.yaml - - --limit - - "3" - - --verbose - - name: backup_csv_images_all - commands: - - func: clone - - func: quay_login - - command: subprocess.exec + - command: shell.exec params: - working_dir: src/github.com/mongodb/mongodb-kubernetes - binary: python3 - args: - - scripts/dev/release/backup_csv_images.py - - scripts/dev/release/1.2.0.clusterserviceversion.yaml - - --verbose - -task_groups: - - name: unit_task_group - max_hosts: -1 - setup_group_can_fail_task: true - setup_group: - - func: clone - - func: python_venv - - func: download_kube_tools - - func: setup_shellcheck - tasks: - - lint_repo - - unit_tests_golang - - unit_tests_python - - sbom_tests - - - name: gke_code_snippets_task_group - <<: *setup_and_teardown_group_gke_code_snippets - max_hosts: -1 - tasks: - - task_gke_multi_cluster_snippets - - task_gke_multi_cluster_no_mesh_snippets - - - name: kind_code_snippets_task_group - <<: *setup_and_teardown_group_kind_code_snippets - max_hosts: -1 - tasks: - - task_kind_community_search_snippets - - # Task group for deploying mongodbcommunity resources and testing the (former) MCO - - name: e2e_mdb_community_task_group - max_hosts: -1 - <<: *setup_group - <<: *setup_and_teardown_task - tasks: - - e2e_community_replicaset_scale - - e2e_search_community_basic - - # This is the task group that contains all the tests run in the e2e_mdb_kind_ubuntu_cloudqa build variant - - name: e2e_mdb_kind_cloudqa_task_group - max_hosts: -1 - <<: *setup_group - <<: *setup_and_teardown_task_cloudqa - tasks: - # e2e_kube_only_task_group - - e2e_crd_validation - - e2e_replica_set_config_map - - e2e_replica_set_exposed_externally - - e2e_replica_set_pv - - e2e_replica_set_pv_multiple - - e2e_replica_set_schema_validation - - e2e_replica_set_statefulset_status - - e2e_sharded_cluster_pv - - e2e_sharded_cluster_recovery - - e2e_sharded_cluster_schema_validation - - e2e_sharded_cluster_statefulset_status - - e2e_standalone_config_map - - e2e_standalone_recovery - - e2e_standalone_schema_validation - - e2e_users_schema_validation - - e2e_replica_set_tls_default - - e2e_replica_set_tls_override - - e2e_replica_set_ignore_unknown_users - # e2e_core_task_group - - e2e_all_mongodb_resources_parallel - - e2e_multiple_cluster_failures - - e2e_replica_set_custom_podspec - - e2e_replica_set_custom_sa - - e2e_replica_set_report_pending_pods - - e2e_replica_set_recovery - - e2e_replica_set_upgrade_downgrade - - e2e_replica_set_agent_flags_and_readinessProbe - - e2e_sharded_cluster - - e2e_sharded_cluster_secret - - e2e_sharded_cluster_upgrade_downgrade - - e2e_sharded_cluster_custom_podspec - - e2e_sharded_cluster_agent_flags - - e2e_standalone_upgrade_downgrade - - e2e_standalone_custom_podspec - - e2e_standalone_agent_flags - - e2e_replica_set_process_hostnames - - e2e_replica_set_member_options - # e2e_tls_task_group - - e2e_replica_set_tls_allow - - e2e_replica_set_tls_prefer - - e2e_replica_set_tls_require_upgrade - - e2e_tls_rs_additional_certs - - e2e_tls_sc_additional_certs - - e2e_tls_rs_intermediate_ca - - e2e_tls_sharded_cluster_certs_prefix - - e2e_sharded_cluster_migration - - e2e_standalone_no_tls_no_status_is_set - - e2e_feature_controls_authentication - - e2e_replica_set - - e2e_replica_set_liveness_probe - - e2e_replica_set_mongod_options - - e2e_replica_set_readiness_probe - - e2e_replica_set_update_delete_parallel - - e2e_sharded_cluster_scale_shards - - e2e_sharded_cluster_shard_overrides - - e2e_sharded_cluster_mongod_options_and_log_rotation - - e2e_tls_rs_external_access - - e2e_replica_set_tls_require - - e2e_replica_set_tls_certs_secret_prefix - - e2e_disable_tls_scale_up - - e2e_replica_set_tls_require_and_disable - # e2e_x509_task_group - - e2e_configure_tls_and_x509_simultaneously_st - - e2e_configure_tls_and_x509_simultaneously_rs - - e2e_configure_tls_and_x509_simultaneously_sc - - e2e_tls_x509_rs - - e2e_tls_x509_sc - - e2e_tls_x509_configure_all_options_rs - - e2e_tls_x509_configure_all_options_sc - - e2e_tls_x509_user_connectivity - - e2e_tls_x509_users_addition_removal - - e2e_replica_set_tls_process_hostnames - # e2e_ldap_task_group - - e2e_replica_set_ldap - - e2e_sharded_cluster_ldap - - e2e_replica_set_ldap_tls - - e2e_replica_set_ldap_user_to_dn_mapping - # e2e_replica_set_ldap_agent_auth - - e2e_replica_set_ldap_agent_client_certs - - e2e_replica_set_ldap_custom_roles - - e2e_replica_set_update_roles_no_privileges - - e2e_replica_set_ldap_group_dn - - e2e_replica_set_ldap_group_dn_with_x509_agent - # e2e_scram_sha_task_group_with_manually_generated_certs - - e2e_replica_set_scram_sha_256_user_connectivity - - e2e_replica_set_scram_sha_256_user_first - - e2e_replica_set_scram_sha_1_user_connectivity - - e2e_replica_set_scram_sha_1_upgrade - - e2e_replica_set_scram_x509_ic_manual_certs - - e2e_sharded_cluster_scram_sha_1_upgrade - - e2e_sharded_cluster_scram_sha_256_user_connectivity - - e2e_sharded_cluster_scram_sha_1_user_connectivity - - e2e_sharded_cluster_scram_x509_ic_manual_certs - - e2e_sharded_cluster_external_access - # e2e_auth_transitions_task_group - - e2e_replica_set_scram_sha_and_x509 - - e2e_replica_set_x509_to_scram_transition - - e2e_sharded_cluster_scram_sha_and_x509 - - e2e_sharded_cluster_x509_to_scram_transition - - e2e_sharded_cluster_internal_cluster_transition - # e2e_webhook_validation_task_group - - e2e_mongodb_validation_webhook - - e2e_mongodb_roles_validation_webhook - - e2e_vault_setup - - e2e_vault_setup_tls - # this test will either migrate from static to non-static or the other way around - - e2e_replica_set_migration - - e2e_replica_set_pv_resize - - e2e_sharded_cluster_pv_resize - - e2e_community_and_meko_replicaset_scale - # OIDC test group - - e2e_replica_set_oidc_m2m_group - - e2e_replica_set_oidc_m2m_user - - e2e_replica_set_oidc_workforce - - e2e_sharded_cluster_oidc_m2m_group - - e2e_sharded_cluster_oidc_m2m_user - <<: *teardown_group - - # this task group contains just a one task, which is smoke testing whether the operator - # works correctly when we run it without installing webhook cluster roles - - name: e2e_mdb_kind_no_webhook_roles_cloudqa_task_group - max_hosts: -1 - <<: *setup_group - <<: *setup_and_teardown_task_cloudqa - tasks: - - e2e_replica_set_agent_flags_and_readinessProbe - <<: *teardown_group - - # This task group mostly duplicates tests running in e2e_mdb_kind_ubi_cloudqa - # This is mostly a smoke test, so we execute only a few essential ones. - - name: e2e_mdb_openshift_ubi_cloudqa_task_group - # OM tests are also run on the same Openshift cluster, so we use 1 max host to not - # allow the helm installations to interfere with each other during the setup of the tests. - max_hosts: 1 - <<: *setup_group - <<: *setup_and_teardown_task_cloudqa - tasks: - - e2e_crd_validation - - e2e_replica_set_scram_sha_256_user_connectivity - - e2e_sharded_cluster_scram_sha_256_user_connectivity - - e2e_replica_set_pv - - e2e_sharded_cluster_pv - - - name: e2e_custom_domain_task_group - max_hosts: -1 - <<: *setup_group - <<: *setup_and_teardown_task_cloudqa - tasks: - - e2e_replica_set - - # e2e_operator_task_group includes the tests for the specific Operator configuration/behavior. They may deal with - # cluster-wide resources so should be run in isolated K8s clusters only (Kind or Minikube). - - name: e2e_operator_task_group - max_hosts: -1 - <<: *setup_group - <<: *setup_and_teardown_task_cloudqa - tasks: - - e2e_operator_upgrade_replica_set - - e2e_sharded_cluster_operator_upgrade_v1_27_to_mck - - e2e_operator_upgrade_ops_manager - - e2e_operator_proxy - - e2e_operator_partial_crd - - e2e_operator_clusterwide - - e2e_operator_multi_namespaces - - e2e_appdb_tls_operator_upgrade_v1_32_to_mck - - e2e_meko_mck_upgrade - <<: *teardown_group - - # e2e_operator_race_with_telemetry_task_group includes the tests for testing the operator with race detector enabled - # additionally, it sends telemetry to cloud-dev; more here: https://wiki.corp.mongodb.com/display/MMS/Telemetry - - name: e2e_operator_race_with_telemetry_task_group - max_hosts: -1 - <<: *setup_group_multi_cluster - <<: *setup_and_teardown_task - tasks: - - e2e_om_reconcile_race_with_telemetry - <<: *teardown_group - - # e2e_operator_task_group includes the tests for the specific Operator configuration/behavior. They may deal with - # cluster-wide resources so should be run in isolated K8s clusters only (Kind or Minikube). - - name: e2e_static_operator_task_group - max_hosts: -1 - <<: *setup_group - <<: *setup_and_teardown_task_cloudqa - tasks: - - e2e_operator_upgrade_replica_set - - e2e_sharded_cluster_operator_upgrade_v1_27_to_mck - - e2e_operator_upgrade_ops_manager - - e2e_operator_proxy - - e2e_operator_partial_crd - - e2e_operator_clusterwide - - e2e_operator_multi_namespaces - - e2e_appdb_tls_operator_upgrade_v1_32_to_mck - - e2e_meko_mck_upgrade - <<: *teardown_group - - - name: e2e_multi_cluster_kind_task_group - max_hosts: -1 - <<: *setup_group - <<: *setup_and_teardown_task_cloudqa - tasks: - - e2e_multi_cluster_replica_set - - e2e_multi_cluster_replica_set_migration - - e2e_multi_cluster_replica_set_member_options - - e2e_multi_cluster_recover - - e2e_multi_cluster_recover_clusterwide - - e2e_multi_cluster_specific_namespaces - - e2e_multi_cluster_scram - - e2e_multi_cluster_tls_with_x509 - - e2e_multi_cluster_tls_no_mesh - - e2e_multi_cluster_enable_tls - # e2e_multi_cluster_with_ldap - # e2e_multi_cluster_with_ldap_custom_roles - - e2e_multi_cluster_mtls_test - - e2e_multi_cluster_replica_set_deletion - - e2e_multi_cluster_replica_set_scale_up - - e2e_multi_cluster_scale_up_cluster - - e2e_multi_cluster_scale_up_cluster_new_cluster - - e2e_multi_cluster_replica_set_scale_down - - e2e_multi_cluster_scale_down_cluster - - e2e_multi_sts_override - - e2e_multi_cluster_tls_with_scram - - e2e_multi_cluster_upgrade_downgrade - - e2e_multi_cluster_backup_restore - - e2e_multi_cluster_backup_restore_no_mesh - - e2e_multi_cluster_disaster_recovery - - e2e_multi_cluster_multi_disaster_recovery - - e2e_multi_cluster_recover_network_partition - - e2e_multi_cluster_validation - - e2e_multi_cluster_agent_flags - - e2e_multi_cluster_replica_set_ignore_unknown_users - - e2e_multi_cluster_pvc_resize - - e2e_multi_cluster_sharded_geo_sharding - - e2e_multi_cluster_sharded_scaling - - e2e_multi_cluster_sharded_scaling_all_shard_overrides - - e2e_multi_cluster_sharded_simplest - - e2e_multi_cluster_sharded_snippets - - e2e_multi_cluster_sharded_simplest_no_mesh - - e2e_multi_cluster_sharded_external_access_no_ext_domain - - e2e_multi_cluster_sharded_tls_no_mesh - - e2e_multi_cluster_sharded_tls - - e2e_multi_cluster_sharded_disaster_recovery - - e2e_sharded_cluster - - e2e_sharded_cluster_agent_flags - - e2e_sharded_cluster_custom_podspec - - e2e_sharded_cluster_mongod_options_and_log_rotation - - e2e_sharded_cluster_migration - - e2e_sharded_cluster_pv - - e2e_sharded_cluster_pv_resize - - e2e_sharded_cluster_recovery - - e2e_sharded_cluster_scale_shards - - e2e_sharded_cluster_shard_overrides - - e2e_sharded_cluster_secret - - e2e_sharded_cluster_statefulset_status - - e2e_sharded_cluster_upgrade_downgrade - - e2e_configure_tls_and_x509_simultaneously_sc - - e2e_sharded_cluster_tls_require_custom_ca - - e2e_tls_sharded_cluster_certs_prefix - - e2e_tls_sc_additional_certs - - e2e_tls_x509_configure_all_options_sc - - e2e_tls_x509_sc - - e2e_meko_mck_upgrade - - e2e_mongodb_custom_roles - - e2e_sharded_cluster_oidc_m2m_group - - e2e_sharded_cluster_oidc_m2m_user - - e2e_multi_cluster_oidc_m2m_group - - e2e_multi_cluster_oidc_m2m_user - - <<: *teardown_group - - - name: e2e_multi_cluster_2_clusters_task_group - max_hosts: -1 - <<: *setup_group - <<: *setup_and_teardown_task_cloudqa - tasks: - - e2e_multi_cluster_2_clusters_replica_set - - e2e_multi_cluster_2_clusters_clusterwide - <<: *teardown_group - - - name: e2e_multi_cluster_om_appdb_task_group - max_hosts: -1 - <<: *setup_group_multi_cluster - <<: *setup_and_teardown_task - tasks: - # Dedicated AppDB Multi-Cluster tests - - e2e_multi_cluster_appdb_validation - - e2e_multi_cluster_om_validation - - e2e_multi_cluster_appdb - - e2e_multi_cluster_appdb_cleanup - - e2e_multi_cluster_appdb_s3_based_backup_restore - - e2e_multi_cluster_appdb_disaster_recovery - - e2e_multi_cluster_appdb_disaster_recovery_force_reconfigure - - e2e_multi_cluster_om_networking_clusterwide - - e2e_multi_cluster_om_appdb_no_mesh - # Reused OM tests with AppDB Multi-Cluster topology - - e2e_appdb_tls_operator_upgrade_v1_32_to_mck - - e2e_om_appdb_flags_and_config - - e2e_om_appdb_upgrade - - e2e_om_appdb_monitoring_tls - - e2e_om_ops_manager_backup - - e2e_om_ops_manager_backup_light - - e2e_om_ops_manager_backup_liveness_probe - - e2e_om_ops_manager_pod_spec - - e2e_om_ops_manager_secure_config - - e2e_om_appdb_validation - - e2e_om_appdb_scram - - e2e_om_external_connectivity - - e2e_om_jvm_params - - e2e_om_migration - - e2e_om_localmode - - e2e_om_ops_manager_enable_local_mode_running_om - - e2e_om_localmode_multiple_pv - - e2e_om_weak_password - - e2e_om_ops_manager_backup_delete_sts_and_log_rotation - - e2e_om_ops_manager_backup_tls - - e2e_om_ops_manager_backup_s3_tls - - e2e_om_ops_manager_backup_tls_custom_ca - - e2e_om_ops_manager_backup_sharded_cluster - - e2e_om_ops_manager_backup_kmip - - e2e_om_ops_manager_https_enabled - - e2e_om_ops_manager_https_enabled_hybrid - - e2e_om_ops_manager_https_enabled_internet_mode - - e2e_om_ops_manager_https_enabled_prefix - - e2e_om_ops_manager_scale - - e2e_om_ops_manager_upgrade - - e2e_om_update_before_reconciliation - - e2e_om_feature_controls - - e2e_multi_cluster_appdb_upgrade_downgrade_v1_27_to_mck - - e2e_multi_cluster_sharded_disaster_recovery - # disabled tests: - # - e2e_om_multiple # multi-cluster failures in EVG - # - e2e_om_appdb_scale_up_down # test not "reused" for multi-cluster appdb - <<: *teardown_group - - - name: e2e_static_multi_cluster_om_appdb_task_group - max_hosts: -1 - <<: *setup_group_multi_cluster - <<: *setup_and_teardown_task - tasks: - # Dedicated AppDB Multi-Cluster tests - - e2e_multi_cluster_appdb_validation - - e2e_multi_cluster_om_validation - - e2e_multi_cluster_appdb - - e2e_multi_cluster_appdb_cleanup - - e2e_multi_cluster_appdb_s3_based_backup_restore - - e2e_multi_cluster_appdb_disaster_recovery - - e2e_multi_cluster_appdb_disaster_recovery_force_reconfigure - - e2e_multi_cluster_om_networking_clusterwide - - e2e_multi_cluster_om_appdb_no_mesh - # Reused OM tests with AppDB Multi-Cluster topology - - e2e_om_appdb_flags_and_config - - e2e_om_appdb_upgrade - - e2e_om_appdb_monitoring_tls - - e2e_om_ops_manager_backup - - e2e_om_ops_manager_backup_light - - e2e_om_ops_manager_backup_liveness_probe - - e2e_om_ops_manager_pod_spec - - e2e_om_ops_manager_secure_config - - e2e_om_appdb_validation - - e2e_om_appdb_scram - - e2e_om_external_connectivity - - e2e_om_jvm_params - - e2e_om_migration - # Deactivated tests - # - e2e_om_localmode - # - e2e_om_localmode_multiple_pv - # - e2e_om_ops_manager_https_enabled - # - e2e_om_ops_manager_https_enabled_hybrid - # - e2e_om_ops_manager_https_enabled_internet_mode - - e2e_om_ops_manager_https_enabled_prefix - - e2e_om_ops_manager_enable_local_mode_running_om - - e2e_om_weak_password - - e2e_om_ops_manager_backup_delete_sts_and_log_rotation - - e2e_om_ops_manager_backup_tls - - e2e_om_ops_manager_backup_s3_tls - - e2e_om_ops_manager_backup_tls_custom_ca - - e2e_om_ops_manager_backup_sharded_cluster - - e2e_om_ops_manager_backup_kmip - - e2e_om_ops_manager_scale - - e2e_om_ops_manager_upgrade - - e2e_multi_cluster_appdb_upgrade_downgrade_v1_27_to_mck - - e2e_om_update_before_reconciliation - - e2e_om_feature_controls - - e2e_multi_cluster_sharded_disaster_recovery - <<: *teardown_group - - # Dedicated task group for deploying OM Multi-Cluster when the operator is in the central cluster - # that is not in the mesh - - name: e2e_multi_cluster_om_operator_not_in_mesh_task_group - max_hosts: -1 - <<: *setup_group_multi_cluster - <<: *setup_and_teardown_task - tasks: - - e2e_multi_cluster_om_clusterwide_operator_not_in_mesh_networking - <<: *teardown_group - - - # This task group runs on Kind clusters. In theory ALL Ops Manager tests should be added here - # including the cluster-wide resources changing. Uses Cloud-qa. TODO why Cloud-qa? those are OM tests - - name: e2e_ops_manager_kind_only_task_group - max_hosts: -1 - <<: *setup_group - <<: *setup_and_teardown_task - tasks: - - e2e_om_appdb_external_connectivity - - e2e_om_appdb_flags_and_config - - e2e_om_appdb_monitoring_tls - - e2e_om_appdb_multi_change - - e2e_om_appdb_scale_up_down - - e2e_om_appdb_upgrade - - e2e_om_appdb_validation - - e2e_om_appdb_scram - - e2e_om_external_connectivity - - e2e_om_jvm_params - - e2e_om_migration - - e2e_om_localmode - - e2e_om_ops_manager_enable_local_mode_running_om - - e2e_om_localmode_multiple_pv - - e2e_om_weak_password - - e2e_om_ops_manager_backup_delete_sts_and_log_rotation - - e2e_om_ops_manager_backup_light - - e2e_om_ops_manager_backup_tls - - e2e_om_ops_manager_backup_s3_tls - - e2e_om_ops_manager_backup_restore_minio - - e2e_om_ops_manager_backup_tls_custom_ca - - e2e_om_ops_manager_backup_liveness_probe - - e2e_om_ops_manager_backup_sharded_cluster - - e2e_om_ops_manager_backup_kmip - - e2e_om_ops_manager_pod_spec - - e2e_om_ops_manager_https_enabled - - e2e_om_ops_manager_https_enabled_hybrid - - e2e_om_ops_manager_https_enabled_internet_mode - - e2e_om_ops_manager_https_enabled_prefix - - e2e_om_ops_manager_scale - - e2e_om_validation_webhook - - e2e_om_ops_manager_secure_config - - e2e_om_update_before_reconciliation - - e2e_om_multiple - - e2e_om_feature_controls - - e2e_vault_setup_om - - e2e_vault_setup_om_backup - <<: *teardown_group - - - name: e2e_static_ops_manager_kind_only_task_group - max_hosts: -1 - <<: *setup_group - <<: *setup_and_teardown_task - tasks: - - e2e_om_appdb_external_connectivity - - e2e_om_appdb_flags_and_config - - e2e_om_appdb_monitoring_tls - - e2e_om_appdb_multi_change - - e2e_om_appdb_scale_up_down - - e2e_om_appdb_upgrade - - e2e_om_appdb_validation - - e2e_om_appdb_scram - - e2e_om_external_connectivity - - e2e_om_jvm_params - - e2e_om_weak_password - - e2e_om_ops_manager_backup_delete_sts_and_log_rotation - - e2e_om_ops_manager_backup_light - - e2e_om_ops_manager_backup_tls - - e2e_om_ops_manager_backup_s3_tls - - e2e_om_ops_manager_backup_restore_minio - - e2e_om_ops_manager_backup_tls_custom_ca - - e2e_om_ops_manager_backup_liveness_probe - - e2e_om_ops_manager_backup_sharded_cluster - - e2e_om_ops_manager_backup_kmip - - e2e_om_ops_manager_pod_spec - - e2e_om_ops_manager_scale - - e2e_om_validation_webhook - - e2e_om_ops_manager_secure_config - - e2e_om_update_before_reconciliation - - e2e_om_multiple - - e2e_om_feature_controls - - e2e_vault_setup_om - - e2e_vault_setup_om_backup - # Deactivated tests - # - e2e_om_localmode_multiple_pv - # - e2e_om_localmode - # - e2e_om_ops_manager_https_enabled_hybrid - # - e2e_om_ops_manager_https_enabled_internet_mode - # - e2e_om_ops_manager_https_enabled - # - e2e_om_ops_manager_https_enabled_prefix - <<: *teardown_group - - - name: e2e_smoke_task_group - max_hosts: -1 - <<: *setup_group - <<: *setup_and_teardown_task - tasks: - - e2e_om_ops_manager_backup - <<: *teardown_group - - - name: e2e_ops_manager_kind_5_0_only_task_group - max_hosts: -1 - <<: *setup_group - <<: *setup_and_teardown_task - tasks: - - e2e_om_remotemode - - e2e_om_ops_manager_backup_restore - - e2e_om_ops_manager_queryable_backup - - e2e_om_ops_manager_backup - <<: *teardown_group - - # Tests features only supported on OM60 - - name: e2e_ops_manager_kind_6_0_only_task_group - max_hosts: -1 - <<: *setup_group - <<: *setup_and_teardown_task - tasks: - - e2e_om_ops_manager_prometheus - <<: *teardown_group - - # Tests features only supported on OM70 and OM80, its only upgrade test as we test upgrading from 6 to 7 or 7 to 8 - - name: e2e_ops_manager_upgrade_only_task_group - max_hosts: -1 - <<: *setup_group - <<: *setup_and_teardown_task - tasks: - - e2e_om_ops_manager_upgrade - <<: *teardown_group - - # Tests features only supported on OM60 - - name: e2e_static_ops_manager_kind_6_0_only_task_group - max_hosts: -1 - <<: *setup_group - <<: *setup_and_teardown_task - tasks: - - e2e_om_ops_manager_prometheus - <<: *teardown_group - - - name: e2e_kind_olm_group - max_hosts: -1 - <<: *setup_group - <<: *setup_and_teardown_task - setup_task_can_fail_task: true - setup_task: - # Even if the two first tasks are already part of setup_and_teardown_task, - # we need to repeat them because we are overriding the setup_task variable of the YAML anchor - - func: cleanup_exec_environment - - func: configure_docker_auth - - func: setup_kubernetes_environment - - func: setup_prepare_openshift_bundles - - func: install_olm - tasks: - - e2e_olm_operator_upgrade - - e2e_olm_operator_upgrade_with_resources - - e2e_olm_meko_operator_upgrade_with_resources - <<: *teardown_group + shell: bash + script: echo "this is the noop task" buildvariants: - ## Unit tests + lint build variant - - - name: unit_tests - display_name: "unit_tests" - tags: [ "unit_tests" ] + # This variant is run when a new tag is out similar to github actions. + - name: noop + display_name: Noop run_on: - ubuntu2204-small tasks: - - name: "unit_task_group" - - ## Build variants for E2E tests - - # The pattern for naming build variants for E2E tests: - # e2e___[_] - # where is any of mdb|om|operator - # where is any of kind|openshift - # where is any of ubuntu|ubi - # where denotes the OM version tested (e.g. om50, om60, cloudqa) - used only for MDB tests - - # MongoDBCommunity build variant - - name: e2e_mdb_community - display_name: e2e_mdb_community - tags: [ "e2e_test_suite"] - run_on: - - ubuntu2204-large - <<: *community_dependency - tasks: - - name: e2e_mdb_community_task_group - - ## MongoDB build variants - - name: e2e_mdb_kind_ubi_cloudqa - display_name: e2e_mdb_kind_ubi_cloudqa - tags: [ "e2e_test_suite", "cloudqa" ] - run_on: - - ubuntu2204-large - <<: *base_no_om_image_dependency - tasks: - - name: e2e_mdb_kind_cloudqa_task_group - - - name: e2e_custom_domain_mdb_kind_ubi_cloudqa - display_name: e2e_custom_domain_mdb_kind_ubi_cloudqa - tags: [ "e2e_test_suite", "cloudqa" ] - run_on: - - ubuntu2204-large - <<: *base_no_om_image_dependency - tasks: - - name: e2e_custom_domain_task_group - - - name: e2e_static_mdb_kind_ubi_cloudqa - display_name: e2e_static_mdb_kind_ubi_cloudqa - tags: [ "e2e_test_suite", "cloudqa" ] - run_on: - - ubuntu2204-large - <<: *base_no_om_image_dependency - tasks: - - name: e2e_mdb_kind_cloudqa_task_group - - - name: e2e_static_custom_domain_mdb_kind_ubi_cloudqa - display_name: e2e_static_custom_domain_mdb_kind_ubi_cloudqa - tags: [ "e2e_test_suite", "cloudqa" ] - run_on: - - ubuntu2204-large - depends_on: - - name: build_operator_ubi - variant: init_test_run - - name: build_test_image - variant: init_test_run - - name: build_init_appdb_images_ubi - variant: init_test_run - - name: build_init_om_images_ubi - variant: init_test_run - - name: build_init_database_image_ubi - variant: init_test_run - - name: build_agent_images_ubi - variant: init_test_run - tasks: - - name: e2e_custom_domain_task_group - - - name: e2e_mdb_openshift_ubi_cloudqa - display_name: e2e_mdb_openshift_ubi_cloudqa - tags: [ "e2e_openshift_test_suite", "cloudqa" ] - depends_on: - - name: build_operator_ubi - variant: init_test_run - - name: build_init_database_image_ubi - variant: init_test_run - - name: build_database_image_ubi - variant: init_test_run - - name: build_test_image - variant: init_test_run - run_on: - - ubuntu2204-small - tasks: - - name: e2e_mdb_openshift_ubi_cloudqa_task_group - - # This name is on purpose reversed from e2e_static_openshift to e2e_openshift_static. - # That is because we run a regex - # in evergreen for all variants matching e2e_static-*, but we do not want to run openshift variants on every pr. - - name: e2e_openshift_static_mdb_ubi_cloudqa - display_name: e2e_openshift_static_mdb_ubi_cloudqa - tags: [ "e2e_openshift_test_suite", "cloudqa" ] - depends_on: - - name: build_operator_ubi - variant: init_test_run - - name: build_test_image - variant: init_test_run - - name: build_init_database_image_ubi - variant: init_test_run - - name: build_agent_images_ubi - variant: init_test_run - run_on: - - ubuntu2204-small - tasks: - - name: e2e_mdb_openshift_ubi_cloudqa_task_group - - ## Ops Manager build variants - - # Isolated Ops Manager Tests for 6.0 version - - name: e2e_om60_kind_ubi - display_name: e2e_om60_kind_ubi - tags: [ "e2e_test_suite" ] - run_on: - - ubuntu2204-large - <<: *base_om6_dependency - tasks: - - name: e2e_ops_manager_kind_only_task_group - - name: e2e_ops_manager_kind_5_0_only_task_group - - name: e2e_ops_manager_kind_6_0_only_task_group - - # Isolated Ops Manager Tests for 6.0 version - - name: e2e_static_om60_kind_ubi - display_name: e2e_static_om60_kind_ubi - tags: [ "e2e_test_suite" ] - run_on: - - ubuntu2204-large - <<: *base_om6_dependency - tasks: - - name: e2e_static_ops_manager_kind_only_task_group - - name: e2e_static_ops_manager_kind_6_0_only_task_group - - - name: e2e_om70_kind_ubi - display_name: e2e_om70_kind_ubi - tags: [ "e2e_test_suite" ] - run_on: - - ubuntu2204-large - <<: *base_om7_dependency - tasks: - - name: e2e_ops_manager_kind_only_task_group - - name: e2e_ops_manager_kind_5_0_only_task_group - - name: e2e_ops_manager_kind_6_0_only_task_group - - name: e2e_ops_manager_upgrade_only_task_group - - - name: e2e_static_om70_kind_ubi - display_name: e2e_static_om70_kind_ubi - tags: [ "e2e_test_suite" ] - run_on: - - ubuntu2204-large - <<: *base_om7_dependency - tasks: - - name: e2e_static_ops_manager_kind_only_task_group - - name: e2e_static_ops_manager_kind_6_0_only_task_group - - name: e2e_ops_manager_upgrade_only_task_group - - - name: e2e_om80_kind_ubi - display_name: e2e_om80_kind_ubi - tags: [ "e2e_test_suite" ] - run_on: - - ubuntu2204-large - <<: *base_om8_dependency - tasks: - - name: e2e_ops_manager_kind_only_task_group - - name: e2e_ops_manager_kind_5_0_only_task_group - - name: e2e_ops_manager_kind_6_0_only_task_group - - name: e2e_ops_manager_upgrade_only_task_group - - - name: e2e_static_om80_kind_ubi - display_name: e2e_static_om80_kind_ubi - tags: [ "e2e_test_suite" ] - run_on: - - ubuntu2204-large - <<: *base_om8_dependency - tasks: - - name: e2e_static_ops_manager_kind_only_task_group - - name: e2e_static_ops_manager_kind_6_0_only_task_group - - name: e2e_ops_manager_upgrade_only_task_group - - - name: e2e_operator_race_ubi_with_telemetry - display_name: e2e_operator_race_ubi_with_telemetry - tags: [ "e2e_test_suite" ] - run_on: - - ubuntu1804-xlarge - <<: *base_om7_dependency - tasks: - - name: e2e_operator_race_with_telemetry_task_group - - - name: e2e_smoke - display_name: e2e_smoke - tags: [ "e2e_test_suite", "e2e_smoke_release_test_suite" ] - run_on: - - ubuntu2204-large - allowed_requesters: [ "patch", "github_tag" ] - depends_on: - - name: build_test_image - variant: init_test_run - tasks: - - name: e2e_smoke_task_group - - - name: e2e_static_smoke - display_name: e2e_static_smoke - tags: [ "e2e_test_suite", "e2e_smoke_release_test_suite" ] - run_on: - - ubuntu2204-large - allowed_requesters: [ "patch", "github_tag" ] - depends_on: - - name: build_test_image - variant: init_test_run - tasks: - - name: e2e_smoke_task_group - - - name: e2e_multi_cluster_kind - display_name: e2e_multi_cluster_kind - tags: [ "e2e_test_suite" ] - run_on: - - ubuntu2204-large - <<: *base_om6_dependency - tasks: - - name: e2e_multi_cluster_kind_task_group - - - name: e2e_static_multi_cluster_kind - display_name: e2e_static_multi_cluster_kind - tags: [ "e2e_test_suite" ] - run_on: - - ubuntu2204-large - <<: *base_om6_dependency - tasks: - - name: e2e_multi_cluster_kind_task_group - - - name: e2e_multi_cluster_2_clusters - display_name: e2e_multi_cluster_2_clusters - tags: [ "e2e_test_suite" ] - run_on: - - ubuntu2204-large - <<: *base_om6_dependency - tasks: - - name: e2e_multi_cluster_2_clusters_task_group - - - name: e2e_static_multi_cluster_2_clusters - display_name: e2e_static_multi_cluster_2_clusters - tags: [ "e2e_test_suite" ] - run_on: - - ubuntu2204-large - <<: *base_om6_dependency - tasks: - - name: e2e_multi_cluster_2_clusters_task_group - - - name: e2e_multi_cluster_om_appdb - display_name: e2e_multi_cluster_om_appdb - tags: [ "e2e_test_suite" ] - run_on: - - ubuntu2204-large - <<: *base_om6_dependency - tasks: - - name: e2e_multi_cluster_om_appdb_task_group - - - name: e2e_static_multi_cluster_om_appdb - display_name: e2e_static_multi_cluster_om_appdb - tags: [ "e2e_test_suite" ] - run_on: - - ubuntu2204-large - <<: *base_om6_dependency - tasks: - - name: e2e_static_multi_cluster_om_appdb_task_group - - - name: e2e_multi_cluster_om_operator_not_in_mesh - display_name: e2e_multi_cluster_om_operator_not_in_mesh - tags: [ "e2e_test_suite" ] - run_on: - - ubuntu2204-large - <<: *base_om7_dependency - tasks: - - name: e2e_multi_cluster_om_operator_not_in_mesh_task_group - - ## Operator tests build variants - - - name: e2e_operator_kind_ubi_cloudqa - display_name: e2e_operator_kind_ubi_cloudqa - tags: [ "e2e_test_suite", "cloudqa" ] - run_on: - - ubuntu2204-large - <<: *base_no_om_image_dependency - tasks: - - name: e2e_operator_task_group - - - name: e2e_static_operator_kind_ubi_cloudqa - display_name: e2e_static_operator_kind_ubi_cloudqa - tags: [ "e2e_test_suite", "cloudqa" ] - run_on: - - ubuntu2204-large - <<: *base_no_om_image_dependency - tasks: - - name: e2e_static_operator_task_group - - - name: e2e_operator_no_webhook_roles_cloudqa - display_name: e2e_operator_no_webhook_roles_cloudqa - tags: [ "e2e_test_suite", "cloudqa" ] - run_on: - - ubuntu2204-large - <<: *base_no_om_image_dependency - tasks: - - name: e2e_mdb_kind_no_webhook_roles_cloudqa_task_group - - - name: e2e_kind_olm_ubi - display_name: e2e_kind_olm_ubi - tags: [ "e2e_test_suite" ] - run_on: - - ubuntu2204-large - depends_on: - - name: build_om_images - variant: build_om60_images - - name: build_operator_ubi - variant: init_test_run - - name: build_test_image - variant: init_test_run - - name: build_init_om_images_ubi - variant: init_test_run - - name: build_init_database_image_ubi - variant: init_test_run - - name: build_database_image_ubi - variant: init_test_run - - name: build_init_appdb_images_ubi - variant: init_test_run - - name: prepare_and_upload_openshift_bundles_for_e2e - variant: init_tests_with_olm - tasks: - - name: e2e_kind_olm_group - - - name: e2e_static_kind_olm_ubi - display_name: e2e_static_kind_olm_ubi - tags: [ "e2e_test_suite" ] - run_on: - - ubuntu2204-large - depends_on: - - name: build_om_images - variant: build_om60_images - - name: build_operator_ubi - variant: init_test_run - - name: build_test_image - variant: init_test_run - - name: build_init_om_images_ubi - variant: init_test_run - - name: build_init_appdb_images_ubi - variant: init_test_run - - name: prepare_and_upload_openshift_bundles_for_e2e - variant: init_tests_with_olm - - name: build_init_database_image_ubi - variant: init_test_run - - name: build_agent_images_ubi - variant: init_test_run - tasks: - - name: e2e_kind_olm_group - - # This variants runs the tests from MCO with the MEKO operator binary - - name: e2e_mco_tests - display_name: "e2e_mco_tests" - tags: [ "e2e_mco_test_suite"] - <<: *community_dependency - run_on: - - ubuntu2004-large - - ## Manual (patch) E2E tests not run for every PR and commit - - - name: e2e_operator_perf - display_name: e2e_operator_perf - tags: [ "e2e_perf_test_suite" ] - allowed_requesters: [ "patch" ] - run_on: - - ubuntu1804-xlarge - <<: *base_om7_dependency - tasks: - - name: generate_perf_tasks_10_thread - - - name: e2e_operator_perf_one_thread - display_name: e2e_operator_perf_one_thread - tags: [ "e2e_perf_test_suite" ] - allowed_requesters: [ "patch" ] - run_on: - - ubuntu1804-xlarge - <<: *base_om7_dependency - tasks: - - name: generate_perf_tasks_one_thread - - - name: e2e_operator_perf_thirty - display_name: e2e_operator_perf_thirty - tags: [ "e2e_perf_test_suite" ] - allowed_requesters: [ "patch" ] - run_on: - - ubuntu1804-xlarge - <<: *base_om7_dependency - tasks: - - name: generate_perf_tasks_30_thread - - ### Prerequisites for E2E test suite - - - name: init_test_run - display_name: init_test_run - max_hosts: -1 - run_on: - - ubuntu2204-small - tasks: - - name: build_operator_ubi - - name: build_test_image - - name: build_mco_test_image - - name: build_init_appdb_images_ubi - - name: build_init_om_images_ubi - - name: build_init_database_image_ubi - - name: build_database_image_ubi - - name: build_agent_images_ubi - - name: build_readiness_probe_image - - name: build_upgrade_hook_image - - name: prepare_aws - - - name: init_release_agents_on_ecr - display_name: init_release_agents_on_ecr - # this enables us to run this variant either manually (patch) which pct does or during an OM bump (github_pr) - allowed_requesters: [ "patch", "github_pr" ] - tags: [ "release_agents_on_ecr" ] - # We want that to run first and finish asap. Digest pinning depends on this to succeed. - priority: 70 - run_on: - - ubuntu2204-large - tasks: - - name: release_agents_on_ecr_conditional - - - name: run_pre_commit - priority: 70 - display_name: run_pre_commit - allowed_requesters: [ "patch", "github_pr" ] - tags: [ "auto_bump" ] - run_on: - - ubuntu2204-small - tasks: - - name: run_precommit_and_push - - - name: init_tests_with_olm - display_name: init_tests_with_olm - depends_on: - - name: build_om_images - variant: build_om60_images - - name: build_operator_ubi - variant: init_test_run - - name: build_init_database_image_ubi - variant: init_test_run - - name: build_database_image_ubi - variant: init_test_run - - name: build_test_image - variant: init_test_run - - name: build_init_appdb_images_ubi - variant: init_test_run - - name: build_init_om_images_ubi - variant: init_test_run - - name: build_agent_images_ubi - variant: init_test_run - run_on: - - ubuntu2204-small - tasks: - - name: prepare_and_upload_openshift_bundles_for_e2e - - ### End of build variants for E2E - - ### Variants that run on each commit - - - name: preflight_release_images_check_only - display_name: preflight_release_images_check_only - run_on: - - rhel90-large - tasks: - - name: preflight_images_task_group - - - name: build_om60_images - display_name: build_om60_images - run_on: - - ubuntu2204-small - tasks: - - name: build_om_images - - - name: preflight_om60_images - display_name: preflight_om60_images - run_on: - - rhel90-large - tasks: - - name: preflight_om_image - - - name: build_om70_images - display_name: build_om70_images - run_on: - - ubuntu2204-small - tasks: - - name: build_om_images - - - name: preflight_om70_images - display_name: preflight_om70_images - run_on: - - rhel90-large - tasks: - - name: preflight_om_image - - - name: build_om80_images - display_name: build_om80_images - run_on: - - ubuntu2204-small - tasks: - - name: build_om_images - - - name: preflight_om80_images - display_name: preflight_om80_images - run_on: - - rhel90-large - tasks: - - name: preflight_om_image - - ### Release build variants - - ## Adds versions as supported in the supported versions Database. - - name: release_images - display_name: release_images - tags: [ "release" ] - allowed_requesters: [ "patch", "github_tag" ] - max_hosts: -1 - run_on: - - release-ubuntu2204-large # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780 - depends_on: - - name: build_operator_ubi - variant: init_test_run - - name: build_init_om_images_ubi - variant: init_test_run - - name: build_init_appdb_images_ubi - variant: init_test_run - - name: build_init_database_image_ubi - variant: init_test_run - - name: build_database_image_ubi - variant: init_test_run - tasks: - - name: release_operator - - name: release_init_appdb - - name: release_init_database - - name: release_init_ops_manager - - name: release_database - # Once we release the operator, we will also release the init databases, we require them to be out first - # such that we can reference them and retrieve those binaries. - # Since we immediately run daily rebuild after creating the image, we can ensure that the init_database is out - # such that the agent image build can use it. - - name: release_agent_operator_release - depends_on: - - name: release_init_database - - - name: preflight_release_images - display_name: preflight_release_images - tags: [ "release" ] - allowed_requesters: [ "patch", "github_tag" ] - depends_on: - - name: "*" - variant: release_images - run_on: - - rhel90-large - expansions: - preflight_submit: true - tasks: - - name: preflight_images_task_group - - - name: prepare_openshift_bundles - display_name: prepare_openshift_bundles - tags: [ "release" ] - allowed_requesters: [ "patch", "github_tag" ] - depends_on: - - name: "*" - variant: release_images - - name: "*" - variant: preflight_release_images - run_on: - - ubuntu2204-large - tasks: - - name: run_conditionally_prepare_and_upload_openshift_bundles - - # It will be called by pct while bumping the agent cloud manager image - - name: release_agent - display_name: (Static Containers) Release Agent matrix - tags: [ "release_agent" ] - run_on: - - release-ubuntu2204-large # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780 - depends_on: - - variant: init_release_agents_on_ecr - name: '*' - - variant: e2e_multi_cluster_kind - name: '*' - - variant: e2e_static_multi_cluster_2_clusters - name: '*' - - variant: e2e_mdb_kind_ubi_cloudqa - name: '*' - - variant: e2e_static_mdb_kind_ubi_cloudqa - name: '*' - tasks: - - name: release_agent - - # Only called manually, It's used for testing the task release_agents_on_ecr in case the release.json - # has not changed, and you still want to push the images to ecr. - - name: manual_ecr_release_agent - display_name: Manual Agent Release for all versions - tags: [ "release_all_agents_manually" ] - run_on: - - ubuntu2204-large - tasks: - - name: release_all_agents_on_ecr - - # These variants are used to test the code snippets and each one can be used in patches - # Prerelease is especially used when the repo is tagged - # More details in the TD: https://docs.google.com/document/d/1fuTxfRtP8QPtn7sKYxQM_AGcD6xycTZH8svngGxyKhc/edit?tab=t.0#bookmark=id.e8uva0393mbe - - name: public_gke_code_snippets - display_name: public_gke_code_snippets - allowed_requesters: ["patch"] - run_on: - - ubuntu2204-small - tasks: - - name: gke_code_snippets_task_group - - - name: prerelease_gke_code_snippets - display_name: prerelease_gke_code_snippets - tags: [ "release" ] - allowed_requesters: ["patch", "github_tag"] - depends_on: - - variant: release_images - name: '*' - patch_optional: true - run_on: - - ubuntu2204-small - tasks: - - name: gke_code_snippets_task_group - - - name: private_gke_code_snippets - display_name: private_gke_code_snippets - allowed_requesters: ["patch"] - run_on: - - ubuntu2204-small - <<: *base_om8_dependency - tasks: - - name: gke_code_snippets_task_group - - - name: private_kind_code_snippets - display_name: private_kind_code_snippets - tags: [ "e2e_test_suite" ] - allowed_requesters: ["patch", "github_pr"] - run_on: - - ubuntu2204-large - <<: *base_om8_dependency - tasks: - - name: kind_code_snippets_task_group - - - name: prerelease_kind_code_snippets - display_name: prerelease_kind_code_snippets - allowed_requesters: ["patch"] - run_on: - - ubuntu2204-large - tasks: - - name: kind_code_snippets_task_group - - - name: public_kind_code_snippets - display_name: public_kind_code_snippets - allowed_requesters: ["patch"] - run_on: - - ubuntu2204-large - tasks: - - name: kind_code_snippets_task_group - - ### Build variants for manual patch only - - - name: backup_csv_images - display_name: "Backup CSV Images" - allowed_requesters: [ "patch" ] - run_on: - - ubuntu2204-small - tasks: - - name: backup_csv_images_dry_run - - name: backup_csv_images_limit_3 - - name: backup_csv_images_all - - - name: publish_om60_images - display_name: publish_om60_images - allowed_requesters: [ "patch", "github_pr" ] - run_on: - - release-ubuntu2204-large # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780 - depends_on: - - variant: e2e_om60_kind_ubi - name: '*' - - variant: e2e_static_om60_kind_ubi - name: '*' - tasks: - - name: publish_ops_manager - - name: release_agent - - - name: publish_om70_images - display_name: publish_om70_images - allowed_requesters: [ "patch", "github_pr" ] - run_on: - - release-ubuntu2204-large # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780 - depends_on: - - variant: e2e_om70_kind_ubi - name: '*' - - variant: e2e_static_om70_kind_ubi - name: '*' - tasks: - - name: publish_ops_manager - - name: release_agent - - - name: publish_om80_images - display_name: publish_om80_images - allowed_requesters: [ "patch", "github_pr" ] - run_on: - - release-ubuntu2204-large # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780 - depends_on: - - variant: e2e_om80_kind_ubi - name: '*' - - variant: e2e_static_om80_kind_ubi - name: '*' - tasks: - - name: publish_ops_manager - - name: release_agent + - name: noop diff --git a/.evergreen/functions/build.yml b/.evergreen/functions/build.yml new file mode 100644 index 000000000..d64dd6202 --- /dev/null +++ b/.evergreen/functions/build.yml @@ -0,0 +1,128 @@ +# Build Functions +# Image building, compilation, and artifact creation functions + +variables: + - &e2e_include_expansions_in_env + include_expansions_in_env: + - workdir + - project + - branch_name + - github_commit + - revision + - github_pr_number + - project_identifier + - revision_order_id + - version_id + - build_id + - build_variant + - execution + - is_patch + - task_id + - task_name + +functions: + ### Core Build Functions ### + + build_multi_cluster_binary: + - command: subprocess.exec + type: setup + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/build_multi_cluster_kubeconfig_creator.sh + + build_and_push_appdb_database: + - command: subprocess.exec + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes/docker/mongodb-kubernetes-appdb-database + binary: ./build_and_push_appdb_database_images.sh + add_to_path: + - ${workdir}/bin + - ${workdir} + + ### Main Pipeline Build Function ### + + pipeline: + - func: switch_context + - command: shell.exec + type: setup + params: + shell: bash + script: | + # Docker Hub workaround + # docker buildx needs the moby/buildkit image when setting up a builder so we pull it from our mirror + docker buildx create --driver=docker-container --driver-opt=image=268558157000.dkr.ecr.eu-west-1.amazonaws.com/docker-hub-mirrors/moby/buildkit:buildx-stable-1 --use + docker buildx inspect --bootstrap + - command: ec2.assume_role + display_name: Assume IAM role with permissions to pull Kondukto API token + params: + role_arn: ${kondukto_role_arn} + - command: shell.exec + display_name: Pull Kondukto API token from AWS Secrets Manager and write it to file + params: + silent: true + shell: bash + include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN] + script: | + set -e + # use AWS CLI to get the Kondukto API token from AWS Secrets Manager + kondukto_token=$(aws secretsmanager get-secret-value --secret-id "kondukto-token" --region "us-east-1" --query 'SecretString' --output text) + # write the KONDUKTO_TOKEN environment variable to Silkbomb environment file + echo "KONDUKTO_TOKEN=$kondukto_token" > ${workdir}/silkbomb.env + - command: subprocess.exec + retry_on_failure: true + type: setup + params: + shell: bash + <<: *e2e_include_expansions_in_env + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/run_python.sh pipeline.py --include ${image_name} --parallel --sign + + ### Registry and Release Functions ### + + release_docker_image_to_registry: + - command: subprocess.exec + type: system + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/release_docker_image_to_registry.sh + + ### AWS Resource Management ### + + prepare_aws: &prepare_aws + command: subprocess.exec + type: setup + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/prepare_aws.sh + + ### OpenShift Bundle Functions ### + + prepare_openshift_bundles: + - command: subprocess.exec + type: setup + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/prepare_openshift_bundles.sh + + upload_openshift_bundle: + - command: subprocess.exec + type: setup + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/upload_openshift_bundle.sh + + ### Performance Test Generation ### + + generate_perf_tests_tasks: + - command: subprocess.exec + type: setup + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/generate_perf_tests_tasks.sh + + ### Utility Functions ### + + update_evergreen_expansions: + - command: expansions.update + params: + file: src/github.com/mongodb/mongodb-kubernetes/expansions.yml diff --git a/.evergreen/functions/deploy.yml b/.evergreen/functions/deploy.yml new file mode 100644 index 000000000..152d2ab7a --- /dev/null +++ b/.evergreen/functions/deploy.yml @@ -0,0 +1,68 @@ +# Deploy Functions +# Deployment, cleanup, and infrastructure management functions + +functions: + ### Environment Teardown Functions ### + + teardown_kubernetes_environment: + - command: shell.exec + type: setup + params: + shell: bash + working_dir: src/github.com/mongodb/mongodb-kubernetes + script: | + scripts/evergreen/teardown_kubernetes_environment.sh + + teardown_cloud_qa: + - command: shell.exec + type: setup + params: + shell: bash + working_dir: src/github.com/mongodb/mongodb-kubernetes + script: | + source .generated/context.export.env + scripts/evergreen/e2e/setup_cloud_qa.py delete + + teardown_cloud_qa_all: + - func: switch_context + - command: shell.exec + type: setup + params: + shell: bash + working_dir: src/github.com/mongodb/mongodb-kubernetes + script: | + source .generated/context.export.env + scripts/evergreen/e2e/setup_cloud_qa.py delete-all + + ### Resource Cleanup Functions ### + + prune_docker_resources: + - command: shell.exec + type: setup + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + script: | + scripts/evergreen/prune_docker_resources.sh + + cleanup_aws: + - func: switch_context + - command: shell.exec + type: setup + params: + shell: bash + working_dir: src/github.com/mongodb/mongodb-kubernetes + script: | + source .generated/context.export.env + scripts/evergreen/cleanup_aws.py + + ### Retry and Recovery Functions ### + + run_retry_script: + - command: shell.exec + type: setup + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + script: | + scripts/evergreen/run_retry_script.sh + + diff --git a/.evergreen/functions/setup.yml b/.evergreen/functions/setup.yml new file mode 100644 index 000000000..ad39cd06e --- /dev/null +++ b/.evergreen/functions/setup.yml @@ -0,0 +1,259 @@ +# Setup Functions +# Environment setup, tool installation, and initialization functions + +variables: + - &e2e_include_expansions_in_env + include_expansions_in_env: + - cognito_user_pool_id + - cognito_workload_federation_client_id + - cognito_user_name + - cognito_workload_federation_client_secret + - cognito_user_password + - cognito_workload_url + - cognito_workload_user_id + - ARTIFACTORY_PASSWORD + - ARTIFACTORY_USERNAME + - GRS_PASSWORD + - GRS_USERNAME + - OVERRIDE_VERSION_ID + - PKCS11_URI + - branch_name + - build_id + - build_variant + - distro + - e2e_cloud_qa_apikey_owner_ubi_cloudqa + - e2e_cloud_qa_orgid_owner_ubi_cloudqa + - e2e_cloud_qa_user_owner_ubi_cloudqa + - ecr_registry + - ecr_registry_needs_auth + - execution + - github_commit + - github_pr_number + - is_patch + - project + - project_identifier + - revision + - revision_order_id + - task_id + - task_name + - version_id + - workdir + +functions: + ### Core Setup Functions ### + + setup_context: &setup_context # Running the first switch is important to fill the workdir and other important initial env vars + command: shell.exec + type: setup + params: + shell: bash + working_dir: src/github.com/mongodb/mongodb-kubernetes + <<: *e2e_include_expansions_in_env + script: | + echo "Initializing context files" + cp scripts/dev/contexts/evg-private-context scripts/dev/contexts/private-context + scripts/dev/switch_context.sh root-context + echo "Finished initializing to the root context" + + switch_context: &switch_context + command: shell.exec + type: setup + params: + shell: bash + working_dir: src/github.com/mongodb/mongodb-kubernetes + <<: *e2e_include_expansions_in_env + add_to_path: + - ${workdir}/bin + - ${workdir}/google-cloud-sdk/bin + script: | + echo "Switching context" + scripts/dev/switch_context.sh "${build_variant}" + echo "Finished switching context" + + python_venv: &python_venv + command: subprocess.exec + type: setup + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + command: scripts/dev/recreate_python_venv.sh + + "clone": + - command: subprocess.exec + type: setup + params: + command: "mkdir -p src/github.com/mongodb" + - command: git.get_project + type: setup + params: + directory: src/github.com/mongodb/mongodb-kubernetes + - command: subprocess.exec + type: setup + params: + command: "git config --global user.name 'Evergreen'" + - command: subprocess.exec + type: setup + params: + command: "git config --global user.email 'kubernetes-hosted-team@mongodb.com'" + - *setup_context + + ### Tool Setup Functions ### + + setup_kubectl: &setup_kubectl + command: subprocess.exec + type: setup + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/setup_kubectl.sh + + setup_jq: &setup_jq + command: subprocess.exec + type: setup + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/setup_jq.sh + + setup_shellcheck: + - command: subprocess.exec + type: setup + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/setup_shellcheck.sh + + download_kube_tools: + - command: subprocess.exec + type: setup + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/download_kube_tools.sh + + ### Cloud Provider Setup ### + + setup_aws: &setup_aws + command: subprocess.exec + type: setup + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/setup_aws.sh + + setup_gcloud_cli: + - command: subprocess.exec + type: setup + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/setup_gcloud_cli.sh + + setup_mongosh: + - command: subprocess.exec + type: setup + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/setup_mongosh.sh + + ### Kubernetes Environment Setup ### + + setup_kind: &setup_kind + command: subprocess.exec + type: setup + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/setup_kind.sh + + setup_kubernetes_environment_p: &setup_kubernetes_environment_p + command: subprocess.exec + type: setup + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/setup_kubernetes_environment.sh + + setup_kubernetes_environment: + - func: setup_kubernetes_environment_p + - func: setup_kind + + cleanup_exec_environment: + - command: subprocess.exec + type: setup + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/cleanup_exec_environment.sh + + ### Docker and Registry Setup ### + + configure_docker_auth: &configure_docker_auth + command: subprocess.exec + type: setup + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/configure_docker_auth.sh + + quay_login: + - command: subprocess.exec + type: setup + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/quay_login.sh + + setup_docker_sbom: + - command: subprocess.exec + type: setup + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/setup_docker_sbom.sh + + ### Host and Environment Setup ### + + setup_evg_host: &setup_evg_host + command: subprocess.exec + type: setup + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/setup_evg_host.sh + + setup_building_host: + - func: setup_evg_host + - func: enable_QEMU + + enable_QEMU: + - command: subprocess.exec + type: setup + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/enable_QEMU.sh + + ### Cloud QA Setup ### + + setup_cloud_qa: + - command: subprocess.exec + type: setup + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/setup_cloud_qa.sh + + ### OpenShift and OLM Setup ### + + setup_preflight: + - command: subprocess.exec + type: setup + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/setup_preflight.sh + + setup_prepare_openshift_bundles: + - command: subprocess.exec + type: setup + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/setup_prepare_openshift_bundles.sh + + install_olm: + - command: subprocess.exec + type: setup + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/install_olm.sh + + prepare_openshift_bundles_for_e2e: + - command: subprocess.exec + type: setup + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/prepare_openshift_bundles_for_e2e.sh diff --git a/.evergreen/functions/test.yml b/.evergreen/functions/test.yml new file mode 100644 index 000000000..00fbdea6c --- /dev/null +++ b/.evergreen/functions/test.yml @@ -0,0 +1,162 @@ +# Test Functions +# Testing, validation, and quality assurance functions + +functions: + ### E2E Test Functions ### + + # Please note: There are many ENV variables passed to the `e2e` script, so try + # to not add more. If this is required, discuss your use case with the team first. + e2e_test: + - command: subprocess.exec + type: test + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + include_expansions_in_env: + - otel_parent_id + - branch_name + - github_commit + - revision + - github_pr_number + - project_identifier + - revision_order_id + add_to_path: + - ${workdir}/bin + binary: scripts/evergreen/e2e/e2e.sh + + e2e_test_perf: + - command: subprocess.exec + type: test + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + include_expansions_in_env: + - otel_parent_id + - branch_name + - github_commit + - revision + - github_pr_number + - project_identifier + - revision_order_id + add_to_path: + - ${workdir}/bin + env: + PERF_TASK_DEPLOYMENTS: ${PERF_TASK_DEPLOYMENTS} + PERF_TASK_REPLICAS: ${PERF_TASK_REPLICAS} + TEST_NAME_OVERRIDE: ${TEST_NAME_OVERRIDE} + binary: scripts/evergreen/e2e/e2e.sh + + ### Unit Test Functions ### + + test_golang_unit: + - command: shell.exec + type: test + params: + shell: bash + working_dir: src/github.com/mongodb/mongodb-kubernetes + script: | + source .generated/context.export.env + make test-race + - command: gotest.parse_files + params: + files: [ "src/github.com/mongodb/mongodb-kubernetes/*.suite", "src/github.com/mongodb/mongodb-kubernetes/public/tools/multicluster/*.suite", "src/github.com/mongodb/mongodb-kubernetes/docker/mongodb-kubernetes-init-ops-manager/mmsconfiguration/*.suite" ] + + test_python_unit: + - command: shell.exec + type: test + params: + shell: bash + working_dir: src/github.com/mongodb/mongodb-kubernetes + script: | + source .generated/context.export.env + make python-tests + + test_sboms: + - command: shell.exec + type: test + params: + shell: bash + working_dir: src/github.com/mongodb/mongodb-kubernetes + script: | + source .generated/context.export.env + make sbom-tests + + ### Validation and Quality Functions ### + + preflight_image: + - command: subprocess.exec + type: test + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/preflight_image.sh + + lint_repo: + - command: shell.exec + type: test + params: + shell: bash + working_dir: src/github.com/mongodb/mongodb-kubernetes + script: | + source .generated/context.export.env + make lint + + ### Log Upload and Diagnostic Functions ### + + upload_e2e_logs: + - command: s3.put + type: system + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_files_include_filter: + - src/github.com/mongodb/mongodb-kubernetes/tmp/manager_logs/**/* + remote_file: ${project}/${build_variant}/${revision}/${version_id}/${build_id}/logs/ + bucket: mciuploads + permissions: public-read + content_type: text/plain + display_name: "manager logs" + + upload_e2e_logs_gotest: + - command: s3.put + type: system + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_files_include_filter: + - src/github.com/mongodb/mongodb-kubernetes/tmp/manager_logs/**/* + remote_file: ${project}/${build_variant}/${revision}/${version_id}/${build_id}/logs/ + bucket: mciuploads + permissions: public-read + content_type: text/plain + display_name: "manager logs" + + upload_code_snippets_logs: + - command: s3.put + type: system + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_files_include_filter: + - src/github.com/mongodb/mongodb-kubernetes/tmp/code_snippets_logs/**/* + remote_file: ${project}/${build_variant}/${revision}/${version_id}/${build_id}/logs/ + bucket: mciuploads + permissions: public-read + content_type: text/plain + display_name: "code snippets logs" + + dump_diagnostic_information_from_all_namespaces: + - command: subprocess.exec + type: system + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/dump_diagnostic_information_from_all_namespaces.sh + + ### Sample and Demo Functions ### + + sample_commit_output: + - command: github.generate_token + params: + expansion_name: GH_TOKEN + - command: subprocess.exec + type: system + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/sample_commit_output.sh diff --git a/.evergreen/main.yml b/.evergreen/main.yml new file mode 100644 index 000000000..26ccfe42d --- /dev/null +++ b/.evergreen/main.yml @@ -0,0 +1,31 @@ +# Main Evergreen Configuration +# Modular CI configuration entry point + +# Global timeout setting +exec_timeout_secs: 7200 + +include: + # Variables - Core configuration and reusable definitions + - filename: .evergreen/variables/common.yml + - filename: .evergreen/variables/versions.yml + - filename: .evergreen/variables/dependencies.yml + + # Functions - Organized by purpose + - filename: .evergreen/functions/setup.yml + - filename: .evergreen/functions/build.yml + - filename: .evergreen/functions/test.yml + - filename: .evergreen/functions/deploy.yml + + # Tasks - Organized by functionality + - filename: .evergreen/tasks/unit-tests.yml + - filename: .evergreen/tasks/e2e-tests.yml + - filename: .evergreen/tasks/image-builds.yml + - filename: .evergreen/tasks/mco-tests.yml + - filename: .evergreen/tasks/preflight.yml + + # Build Variants - Organized by purpose + - filename: .evergreen/variants/unit-tests.yml + - filename: .evergreen/variants/e2e-community.yml + - filename: .evergreen/variants/e2e-enterprise.yml + - filename: .evergreen/variants/image-builds.yml + - filename: .evergreen/variants/releases.yml diff --git a/.evergreen/periodic/scheduled-builds.yml b/.evergreen/periodic/scheduled-builds.yml new file mode 100644 index 000000000..55a252347 --- /dev/null +++ b/.evergreen/periodic/scheduled-builds.yml @@ -0,0 +1,112 @@ +# Scheduled/Periodic Builds +# Configuration for periodic and scheduled build tasks + +include: + - filename: ../.evergreen/variables/common.yml + - filename: ../.evergreen/variables/versions.yml + - filename: ../.evergreen/variables/dependencies.yml + - filename: ../.evergreen/functions/setup.yml + - filename: ../.evergreen/functions/build.yml + - filename: ../.evergreen/functions/test.yml + - filename: ../.evergreen/functions/deploy.yml + +parameters: + - key: pin_tag_at + value: 00:00 + description: Pin tags at this time of the day. Midnight by default. + +tasks: + ### Periodic Build Tasks ### + + - name: periodic_build_operator + commands: + - func: pipeline + vars: + image_name: operator-daily + + - name: periodic_build_init_appdb + commands: + - func: pipeline + vars: + image_name: init-appdb-daily + + - name: periodic_build_init_database + commands: + - func: pipeline + vars: + image_name: init-database-daily + + - name: periodic_build_init_opsmanager + commands: + - func: pipeline + vars: + image_name: init-ops-manager-daily + + - name: periodic_build_agent + commands: + - func: pipeline + vars: + image_name: agent-daily + + - name: periodic_build_database + commands: + - func: pipeline + vars: + image_name: database-daily + + - name: periodic_build_readiness_probe + commands: + - func: pipeline + vars: + image_name: readiness-probe-daily + + - name: periodic_build_upgrade_hook + commands: + - func: pipeline + vars: + image_name: upgrade-hook-daily + + - name: periodic_build_test_image + commands: + - func: pipeline + vars: + image_name: test-daily + + ### Periodic Cleanup Tasks ### + + - name: periodic_teardown_aws + commands: + - func: cleanup_aws + + - name: periodic_teardown_cloudqa + commands: + - func: teardown_cloud_qa_all + +buildvariants: + ### Periodic Build Variants ### + + - name: periodic_builds + display_name: periodic_builds + cron: "0 0 * * *" # Daily at midnight + run_on: + - ubuntu2204-large + <<: *setup_group + tasks: + - name: periodic_build_operator + - name: periodic_build_init_appdb + - name: periodic_build_init_database + - name: periodic_build_init_opsmanager + - name: periodic_build_agent + - name: periodic_build_database + - name: periodic_build_readiness_probe + - name: periodic_build_upgrade_hook + - name: periodic_build_test_image + + - name: periodic_cleanup + display_name: periodic_cleanup + cron: "0 2 * * *" # Daily at 2 AM + run_on: + - ubuntu2204-small + tasks: + - name: periodic_teardown_aws + - name: periodic_teardown_cloudqa diff --git a/.evergreen/tasks/e2e-tests.yml b/.evergreen/tasks/e2e-tests.yml new file mode 100644 index 000000000..a80c17425 --- /dev/null +++ b/.evergreen/tasks/e2e-tests.yml @@ -0,0 +1,117 @@ +# E2E Test Tasks +# End-to-end test task definitions organized by test type + +tasks: + ### Community E2E Tests ### + + - name: e2e_multiple_cluster_failures + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_community_and_meko_replicaset_scale + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_standalone_custom_podspec + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_standalone_schema_validation + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_schema_validation + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_sharded_cluster_schema_validation + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_sharded_cluster_external_access + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_users_schema_validation + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_crd_validation + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_standalone_config_map + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_standalone_groups + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + ### TLS and Security Tests ### + + - name: e2e_tls_sc_additional_certs + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_tls_x509_rs + tags: [ "patch-run" ] + # longer timeout than usual as this test tests recovery from bad states which can take some time + commands: + - func: "e2e_test" + + ### Multi-Cluster Tests ### + + - name: e2e_multi_cluster_scram + tags: [ "patch-run" ] + commands: + - func: e2e_test + + ### Ops Manager E2E Tests ### + + - name: e2e_om_appdb_multi_change + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_om_reconcile_perf + tags: [ "patch-run" ] + commands: + - func: e2e_test + + ### Performance Tests ### + + - name: e2e_perf_test_one_thread + tags: [ "patch-run" ] + commands: + - func: e2e_test_perf + + - name: e2e_perf_test_10_thread + tags: [ "patch-run" ] + commands: + - func: e2e_test_perf + + - name: e2e_perf_test_30_thread + tags: [ "patch-run" ] + commands: + - func: e2e_test_perf + + ### Code Snippets Tests ### + + - name: test_code_snippets + tags: [ "patch-run" ] + commands: + - func: test_code_snippets + - func: sample_commit_output diff --git a/.evergreen/tasks/image-builds.yml b/.evergreen/tasks/image-builds.yml new file mode 100644 index 000000000..2853a5544 --- /dev/null +++ b/.evergreen/tasks/image-builds.yml @@ -0,0 +1,248 @@ +# Image Build Tasks +# Docker image building and related tasks + +tasks: + ### Core Image Build Tasks ### + + - name: build_test_image + commands: + - func: clone + - func: setup_building_host + - func: build_multi_cluster_binary + - func: pipeline + vars: + image_name: test + + - name: build_mco_test_image + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + image_name: mco-test + + - name: build_operator_ubi + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + skip_tags: ubuntu,release + distro: ubi + image_name: operator + + ### Init Image Build Tasks ### + + - name: build_init_om_images_ubi + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + image_name: init-ops-manager + skip_tags: ubuntu,release + + - name: build_init_appdb_images_ubi + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + image_name: init-appdb + skip_tags: ubuntu,release + + - name: build_init_database_image_ubi + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + image_name: init-database + skip_tags: ubuntu,release + + ### Agent and Database Image Build Tasks ### + + - name: build_agent_images_ubi + depends_on: + - name: build_init_database_image_ubi + variant: init_test_run + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + image_name: agent + skip_tags: ubuntu,release + + - name: build_database_image_ubi + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + image_name: database + skip_tags: ubuntu,release + + ### Community Image Build Tasks ### + + - name: build_readiness_probe_image + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + image_name: readiness-probe + skip_tags: ubuntu,release + + - name: build_upgrade_hook_image + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + image_name: upgrade-hook + skip_tags: ubuntu,release + + ### Ops Manager Image Build Tasks ### + + - name: build_om_images + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + image_name: ops-manager + + ### Release Image Tasks ### + + - name: release_operator + tags: [ "image_release" ] + allowed_requesters: [ "patch", "github_tag" ] + commands: + - func: clone + - func: setup_building_host + - func: quay_login + - func: setup_docker_sbom + - func: pipeline + vars: + image_name: operator + include_tags: release + + - name: release_init_appdb + tags: [ "image_release" ] + allowed_requesters: [ "patch", "github_tag" ] + commands: + - func: clone + - func: setup_building_host + - func: quay_login + - func: setup_docker_sbom + - func: pipeline + vars: + image_name: init-appdb + include_tags: release + + - name: release_init_database + tags: [ "image_release" ] + allowed_requesters: [ "patch", "github_tag" ] + commands: + - func: clone + - func: setup_building_host + - func: quay_login + - func: setup_docker_sbom + - func: pipeline + vars: + image_name: init-database + include_tags: release + + - name: release_init_ops_manager + tags: [ "image_release" ] + allowed_requesters: [ "patch", "github_tag" ] + commands: + - func: clone + - func: setup_building_host + - func: quay_login + - func: setup_docker_sbom + - func: pipeline + vars: + image_name: init-ops-manager + include_tags: release + + - name: release_agent_operator_release + tags: [ "image_release" ] + allowed_requesters: [ "patch", "github_tag" ] + commands: + - func: clone + - func: setup_building_host + - func: quay_login + - func: setup_docker_sbom + - func: pipeline + vars: + image_name: agent-operator-release + include_tags: release + + - name: release_agent + tags: [ "image_release" ] + allowed_requesters: [ "patch", "github_tag" ] + commands: + - func: clone + - func: setup_building_host + - func: quay_login + - func: setup_docker_sbom + - func: pipeline + vars: + image_name: agent + include_tags: release + + - name: release_database + tags: [ "image_release" ] + allowed_requesters: [ "patch", "github_tag" ] + commands: + - func: clone + - func: setup_building_host + - func: quay_login + - func: setup_docker_sbom + - func: pipeline + vars: + image_name: database + include_tags: release + + ### Agent Release Tasks ### + + - name: release_agents_on_ecr_conditional + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + image_name: agent-conditional + + - name: release_agents_on_ecr + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + image_name: agent-ecr + + - name: release_all_agents_on_ecr + # this enables us to run this manually (patch) and release all agent versions to ECR + # it's needed during operator new version release process - e2e tests (especially olm tests) + # will look for agent with new operator version suffix, but during PR checks we only build + # agent versions for most recent major OM versions and the tests will fail. Before running the PR + # checks, we need to run this task to build all agent versions. + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + image_name: agent-all-ecr + + ### Ops Manager Publishing ### + + - name: publish_ops_manager + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + image_name: ops-manager-publish diff --git a/.evergreen/tasks/mco-tests.yml b/.evergreen/tasks/mco-tests.yml new file mode 100644 index 000000000..41c5334d0 --- /dev/null +++ b/.evergreen/tasks/mco-tests.yml @@ -0,0 +1,173 @@ +# MCO Test Tasks +# MongoDB Community Operator specific test tasks + +tasks: + ### MCO E2E Test Tasks ### + + - name: feature_compatibility_version + commands: + - func: e2e_test + + - name: prometheus + commands: + - func: e2e_test + + - name: replica_set + commands: + - func: e2e_test + + - name: replica_set_arbiter + commands: + - func: e2e_test + + - name: replica_set_change_version + commands: + - func: e2e_test + + - name: replica_set_connection_string_options + commands: + - func: e2e_test + + - name: replica_set_cross_namespace_deploy + commands: + - func: e2e_test + + - name: replica_set_custom_persistent_volume + commands: + - func: e2e_test + + - name: replica_set_custom_role + commands: + - func: e2e_test + + - name: replica_set_enterprise_upgrade_4_5 + commands: + - func: e2e_test + + - name: replica_set_enterprise_upgrade_5_6 + commands: + - func: e2e_test + + - name: replica_set_enterprise_upgrade_6_7 + commands: + - func: e2e_test + + - name: replica_set_enterprise_upgrade_7_8 + commands: + - func: e2e_test + + - name: replica_set_mongod_config + commands: + - func: e2e_test + + - name: replica_set_mongod_port_change_with_arbiters + commands: + - func: e2e_test + + - name: replica_set_mongod_readiness + commands: + - func: e2e_test + + - name: replica_set_mount_connection_string + commands: + - func: e2e_test + + - name: replica_set_operator_upgrade + commands: + - func: e2e_test + + - name: replica_set_recovery + commands: + - func: e2e_test + + - name: replica_set_remove_user + commands: + - func: e2e_test + + - name: replica_set_scale + commands: + - func: e2e_test + + - name: replica_set_scale_down + commands: + - func: e2e_test + + - name: replica_set_tls + commands: + - func: e2e_test + + - name: replica_set_tls_recreate_mdbc + commands: + - func: e2e_test + + - name: replica_set_tls_rotate + commands: + - func: e2e_test + + - name: replica_set_tls_rotate_delete_sts + commands: + - func: e2e_test + + - name: replica_set_tls_upgrade + commands: + - func: e2e_test + + - name: replica_set_x509 + commands: + - func: e2e_test + + - name: statefulset_arbitrary_config + commands: + - func: e2e_test + + - name: statefulset_arbitrary_config_update + commands: + - func: e2e_test + +task_groups: + ### MCO Task Group ### + + - name: e2e_mco_task_group + max_hosts: -1 + setup_group: + - func: clone + - func: download_kube_tools + - func: setup_building_host + setup_task: + - func: cleanup_exec_environment + - func: configure_docker_auth + - func: setup_kubernetes_environment + tasks: + - feature_compatibility_version + - prometheus + - replica_set + - replica_set_arbiter + - replica_set_change_version + - replica_set_connection_string_options + - replica_set_cross_namespace_deploy + - replica_set_custom_persistent_volume + - replica_set_custom_role + - replica_set_enterprise_upgrade_4_5 + - replica_set_enterprise_upgrade_5_6 + - replica_set_enterprise_upgrade_6_7 + - replica_set_enterprise_upgrade_7_8 + - replica_set_mongod_config + - replica_set_mongod_port_change_with_arbiters + - replica_set_mongod_readiness + - replica_set_mount_connection_string + - replica_set_operator_upgrade + - replica_set_recovery + - replica_set_remove_user + - replica_set_scale + - replica_set_scale_down + - replica_set_tls + - replica_set_tls_recreate_mdbc + - replica_set_tls_rotate + - replica_set_tls_rotate_delete_sts + - replica_set_tls_upgrade + - replica_set_x509 + - statefulset_arbitrary_config + - statefulset_arbitrary_config_update + teardown_group: + - func: prune_docker_resources + - func: run_retry_script diff --git a/.evergreen/tasks/preflight.yml b/.evergreen/tasks/preflight.yml new file mode 100644 index 000000000..cac25c101 --- /dev/null +++ b/.evergreen/tasks/preflight.yml @@ -0,0 +1,163 @@ +# Preflight and Validation Tasks +# Image validation, preflight checks, and OpenShift bundle tasks + +tasks: + ### Preflight Image Validation Tasks ### + + - name: preflight_images + tags: [ "image_preflight" ] + commands: + - func: clone + - func: python_venv + - func: setup_preflight + - func: preflight_image + vars: + image_name: mongodb-kubernetes + - func: preflight_image + vars: + image_name: init-appdb + - func: preflight_image + vars: + image_name: init-database + - func: preflight_image + vars: + image_name: init-ops-manager + - func: preflight_image + vars: + image_name: database + + - name: preflight_ops_manager + tags: [ "image_preflight" ] + commands: + - func: clone + - func: python_venv + - func: setup_preflight + - func: preflight_image + vars: + image_name: ops-manager + + - name: preflight_official_database_image + tags: [ "image_preflight" ] + commands: + - func: clone + - func: python_venv + - func: setup_preflight + - func: preflight_image + vars: + image_name: official-database + + - name: preflight_mongodb_agent_image + tags: [ "image_preflight" ] + commands: + - func: clone + - func: python_venv + - func: setup_preflight + - func: preflight_image + vars: + image_name: mongodb-agent + + ### OpenShift Bundle Tasks ### + + - name: prepare_and_upload_openshift_bundles_for_e2e + commands: + - func: clone + - func: setup_aws + - func: prepare_openshift_bundles_for_e2e + + - name: prepare_and_upload_openshift_bundles + tags: [ "openshift_bundles" ] + commands: + - func: clone + - func: setup_aws + - func: prepare_openshift_bundles + - func: upload_openshift_bundle + + - name: run_conditionally_prepare_and_upload_openshift_bundles + commands: + - func: clone + - func: setup_aws + - func: prepare_openshift_bundles + - func: upload_openshift_bundle + + ### CSV Backup Tasks ### + + - name: backup_csv_images_dry_run + commands: + - func: clone + - func: setup_aws + - func: backup_csv_images + vars: + dry_run: "true" + + - name: backup_csv_images_limit_3 + commands: + - func: clone + - func: setup_aws + - func: backup_csv_images + vars: + limit: "3" + + - name: backup_csv_images_all + commands: + - func: clone + - func: setup_aws + - func: backup_csv_images + + ### Utility Tasks ### + + - name: prepare_aws + priority: 59 + commands: + - func: clone + - func: setup_jq + - func: setup_aws + - func: prepare_aws + + - name: run_retry_script + commands: + - func: run_retry_script + + ### Performance Test Generation Tasks ### + + - name: generate_perf_tasks_one_thread + commands: + - func: clone + - func: generate_perf_tests_tasks + vars: + variant: e2e_perf_test_one_thread + size: 1 + + - name: generate_perf_tasks_10_thread + commands: + - func: clone + - func: generate_perf_tests_tasks + vars: + variant: e2e_perf_test_10_thread + size: 10 + + - name: generate_perf_tasks_30_thread + commands: + - func: clone + - func: generate_perf_tests_tasks + vars: + variant: e2e_perf_test_30_thread + size: 30 + + ### Pre-commit Tasks ### + + - name: run_precommit_and_push + commands: + - func: clone + - func: setup_building_host + - func: run_precommit_and_push + +task_groups: + ### Preflight Task Group ### + + - name: preflight_images_task_group + max_hosts: -1 + tasks: + - preflight_images + - preflight_official_database_image + - preflight_mongodb_agent_image + - preflight_ops_manager diff --git a/.evergreen/tasks/unit-tests.yml b/.evergreen/tasks/unit-tests.yml new file mode 100644 index 000000000..447aa4ee9 --- /dev/null +++ b/.evergreen/tasks/unit-tests.yml @@ -0,0 +1,46 @@ +# Unit Test Tasks +# Unit testing, linting, and code quality tasks + +tasks: + ### Core Unit Test Tasks ### + + - name: unit_tests_golang + tags: [ "unit_tests" ] + commands: + - func: "test_golang_unit" + + - name: unit_tests_python + tags: [ "unit_tests" ] + commands: + - func: "test_python_unit" + + - name: sbom_tests + tags: [ "unit_tests" ] + # The SBOM tests run only on commit builds. Running this on patches might cause false-positive failures + # because certain images might not be there yet. Such situation happens for OM image upgrades for example. + # See https://docs.devprod.prod.corp.mongodb.com/evergreen/Project-Configuration/Project-Configuration-Files#limiting-when-a-task-or-variant-will-run + patchable: false + commands: + - func: "test_sboms" + + ### Code Quality Tasks ### + + - name: lint_repo + tags: [ "unit_tests" ] + commands: + - func: lint_repo + +task_groups: + ### Unit Test Task Group ### + + - name: unit_task_group + max_hosts: -1 + setup_group_can_fail_task: true + setup_group: + - func: clone + - func: python_venv + tasks: + - unit_tests_golang + - unit_tests_python + - sbom_tests + - lint_repo diff --git a/.evergreen/variables/common.yml b/.evergreen/variables/common.yml new file mode 100644 index 000000000..7dedbcbde --- /dev/null +++ b/.evergreen/variables/common.yml @@ -0,0 +1,150 @@ +# Common Variables and Anchors +# Global configuration settings and reusable YAML anchors + +variables: + # Environment expansion anchor for E2E tests + - &e2e_include_expansions_in_env + include_expansions_in_env: + - workdir + - project + - branch_name + - github_commit + - revision + - github_pr_number + - project_identifier + - revision_order_id + - version_id + - build_id + - build_variant + - execution + - is_patch + - task_id + - task_name + + # Global timeout setting + - &global_timeout 7200 + + # Common setup group configurations + - &setup_group + setup_group_can_fail_task: true + setup_group: + - func: clone + - func: download_kube_tools + - func: setup_building_host + + - &setup_group_multi_cluster + setup_group_can_fail_task: true + setup_group: + - func: clone + - func: download_kube_tools + - func: setup_building_host + - func: build_multi_cluster_binary + + # Setup and teardown patterns for different environments + - &setup_and_teardown_group_gke_code_snippets + setup_task_can_fail_task: true + setup_group: + - func: clone + - func: setup_gcloud_cli + - func: setup_mongosh + - func: download_kube_tools + - func: build_multi_cluster_binary + teardown_group: + - func: upload_code_snippets_logs + + - &setup_and_teardown_group_kind_code_snippets + setup_task_can_fail_task: true + setup_group: + - func: clone + - func: cleanup_exec_environment + - func: download_kube_tools + - func: configure_docker_auth + - func: setup_kubernetes_environment + teardown_task: + - func: upload_e2e_logs + - func: upload_code_snippets_logs + + # Task-level setup and teardown patterns + - &setup_and_teardown_task_cloudqa + setup_task_can_fail_task: true + setup_task: + - func: cleanup_exec_environment + - func: configure_docker_auth + - func: setup_kubernetes_environment + - func: setup_cloud_qa + teardown_task_can_fail_task: true + teardown_task: + - func: upload_e2e_logs + - func: teardown_kubernetes_environment + - func: teardown_cloud_qa + + - &setup_and_teardown_task + setup_task_can_fail_task: true + setup_task: + - func: cleanup_exec_environment + - func: configure_docker_auth + - func: setup_kubernetes_environment + teardown_task_can_fail_task: true + teardown_task: + - func: upload_e2e_logs + - func: teardown_kubernetes_environment + + # Common teardown group + - &teardown_group + teardown_group: + - func: prune_docker_resources + - func: run_retry_script + + # Environment variable expansions for E2E tests + - &e2e_include_expansions_in_env + include_expansions_in_env: + - cognito_user_pool_id + - cognito_workload_federation_client_id + - cognito_user_name + - cognito_workload_federation_client_secret + - cognito_user_password + - cognito_workload_url + - cognito_workload_user_id + - ARTIFACTORY_PASSWORD + - ARTIFACTORY_USERNAME + - GRS_PASSWORD + - GRS_USERNAME + - OVERRIDE_VERSION_ID + - PKCS11_URI + - branch_name + - build_id + - build_variant + - distro + - e2e_cloud_qa_apikey_owner_ubi_cloudqa + - e2e_cloud_qa_orgid_owner_ubi_cloudqa + - e2e_cloud_qa_user_owner_ubi_cloudqa + - ecr_registry + - ecr_registry_needs_auth + - execution + - github_commit + - image_name + - include_tags + - is_patch + - mms_eng_test_aws_access_key + - mms_eng_test_aws_region + - mms_eng_test_aws_secret + - openshift_token + - openshift_url + - otel_collector_endpoint + - otel_parent_id + - otel_trace_id + - pin_tag_at + - registry + - requester + - skip_tags + - task_name + - triggered_by_git_tag + - version_id + - workdir + # temporary secret to pull community private preview image from quay.io + - community_private_preview_pullsecret_dockerconfigjson + +parameters: + - key: evergreen_retry + value: "true" + description: set this to false to suppress retries on failure diff --git a/.evergreen/variables/dependencies.yml b/.evergreen/variables/dependencies.yml new file mode 100644 index 000000000..666e2c36b --- /dev/null +++ b/.evergreen/variables/dependencies.yml @@ -0,0 +1,162 @@ +# Dependency Definitions +# Centralized build dependency patterns for different test scenarios + +variables: + # Build dependency anchors for different test scenarios + # These define which images need to be built before running tests + + # Community dependency - for MongoDB Community Operator tests + - &community_dependency + depends_on: + - name: build_operator_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run + - name: build_readiness_probe_image + variant: init_test_run + - name: build_upgrade_hook_image + variant: init_test_run + - name: build_mco_test_image + variant: init_test_run + + # Base dependency without OM images - for CloudQA tests + - &base_no_om_image_dependency + depends_on: + - name: build_om_images + variant: build_om70_images + - name: build_operator_ubi + variant: init_test_run + - name: build_init_database_image_ubi + variant: init_test_run + - name: build_database_image_ubi + variant: init_test_run + - name: build_init_om_images_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + - name: build_init_appdb_images_ubi + variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run + + # Base dependencies for Ops Manager 6.0 tests + # The dependency unification between static and non-static is intentional here. + # Even though some images are exclusive, in EVG they all are built once and in parallel. + # It is not worth the effort of splitting them out. + # Once Static Containers are default, this piece can be cleaned up. + - &base_om6_dependency + depends_on: + - name: build_om_images + variant: build_om60_images + - name: build_operator_ubi + variant: init_test_run + - name: build_init_database_image_ubi + variant: init_test_run + - name: build_database_image_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + - name: build_init_appdb_images_ubi + variant: init_test_run + - name: build_init_om_images_ubi + variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run + + # Base dependencies for Ops Manager 7.0 tests + - &base_om7_dependency + depends_on: + - name: build_om_images + variant: build_om70_images + - name: build_operator_ubi + variant: init_test_run + - name: build_init_database_image_ubi + variant: init_test_run + - name: build_database_image_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + - name: build_init_appdb_images_ubi + variant: init_test_run + - name: build_init_om_images_ubi + variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run + + # Base dependencies for Ops Manager 8.0 tests + - &base_om8_dependency + depends_on: + - name: build_om_images + variant: build_om80_images + - name: build_operator_ubi + variant: init_test_run + - name: build_init_database_image_ubi + variant: init_test_run + - name: build_database_image_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + - name: build_init_appdb_images_ubi + variant: init_test_run + - name: build_init_om_images_ubi + variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run + + # Dependencies for tests that don't require OM images + - &base_no_om_image_dependency + depends_on: + - name: build_om_images + variant: build_om70_images + - name: build_operator_ubi + variant: init_test_run + - name: build_init_database_image_ubi + variant: init_test_run + - name: build_database_image_ubi + variant: init_test_run + - name: build_init_om_images_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + - name: build_init_appdb_images_ubi + variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run + + # Dependencies for community tests (no Ops Manager) + - &community_dependency + depends_on: + - name: build_operator_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run + - name: build_readiness_probe_image + variant: init_test_run + - name: build_upgrade_hook_image + variant: init_test_run + - name: build_mco_test_image + variant: init_test_run + + # Common image build dependencies (reusable pattern) + - &common_image_dependencies + - name: build_operator_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run + + # Ops Manager specific dependencies (reusable pattern) + - &om_image_dependencies + - name: build_init_database_image_ubi + variant: init_test_run + - name: build_database_image_ubi + variant: init_test_run + - name: build_init_appdb_images_ubi + variant: init_test_run + - name: build_init_om_images_ubi + variant: init_test_run diff --git a/.evergreen/variables/versions.yml b/.evergreen/variables/versions.yml new file mode 100644 index 000000000..181eb068e --- /dev/null +++ b/.evergreen/variables/versions.yml @@ -0,0 +1,16 @@ +# Version Definitions +# Centralized version specifications for all components + +variables: + # Ops Manager Version Anchors + # IMPORTANT: The order/index is important, since these are anchors. Please do not change + - &ops_manager_60_latest 6.0.27 + - &ops_manager_70_latest 7.0.15 + - &ops_manager_80_latest 8.0.7 + + # Version update documentation: + # To update Ops Manager versions: + # 1. Update the version number above + # 2. Ensure corresponding build variants are updated + # 3. Test with a patch build before merging + # 4. Coordinate with team for any breaking changes diff --git a/.evergreen/variants/e2e-community.yml b/.evergreen/variants/e2e-community.yml new file mode 100644 index 000000000..4fa27ce47 --- /dev/null +++ b/.evergreen/variants/e2e-community.yml @@ -0,0 +1,80 @@ +# Community E2E Build Variants +# Build variants for MongoDB Community Operator E2E tests + +buildvariants: + # The pattern for naming build variants for E2E tests: + # e2e___[_] + # where is any of mdb|om|operator + # where is any of kind|openshift + # where is any of ubuntu|ubi + # where denotes the OM version tested (e.g. om50, om60, cloudqa) - used only for MDB tests + + ## MongoDB Community Build Variants ## + + # MongoDBCommunity build variant + - name: e2e_mdb_community + display_name: e2e_mdb_community + tags: [ "e2e_test_suite"] + run_on: + - ubuntu2204-large + <<: *community_dependency + tasks: + - name: e2e_mdb_community_task_group + + ## MongoDB KIND CloudQA Build Variants ## + + - name: e2e_mdb_kind_ubi_cloudqa + display_name: e2e_mdb_kind_ubi_cloudqa + tags: [ "e2e_test_suite", "cloudqa" ] + run_on: + - ubuntu2204-large + <<: *base_no_om_image_dependency + tasks: + - name: e2e_mdb_kind_cloudqa_task_group + + - name: e2e_custom_domain_mdb_kind_ubi_cloudqa + display_name: e2e_custom_domain_mdb_kind_ubi_cloudqa + tags: [ "e2e_test_suite", "cloudqa" ] + run_on: + - ubuntu2204-large + <<: *base_no_om_image_dependency + tasks: + - name: e2e_custom_domain_task_group + + - name: e2e_static_mdb_kind_ubi_cloudqa + display_name: e2e_static_mdb_kind_ubi_cloudqa + tags: [ "e2e_test_suite", "cloudqa" ] + run_on: + - ubuntu2204-large + <<: *base_no_om_image_dependency + tasks: + - name: e2e_static_operator_task_group + + - name: e2e_mdb_kind_no_webhook_roles_ubi_cloudqa + display_name: e2e_mdb_kind_no_webhook_roles_ubi_cloudqa + tags: [ "e2e_test_suite", "cloudqa" ] + run_on: + - ubuntu2204-large + <<: *base_no_om_image_dependency + tasks: + - name: e2e_mdb_kind_no_webhook_roles_cloudqa_task_group + + ## Multi-Cluster Build Variants ## + + - name: e2e_multi_cluster_kind_ubi + display_name: e2e_multi_cluster_kind_ubi + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *community_dependency + tasks: + - name: e2e_multi_cluster_kind_task_group + + ## MCO Build Variants ## + + - name: e2e_mco_tests + display_name: e2e_mco_tests + run_on: + - ubuntu2204-large + tasks: + - name: e2e_mco_task_group diff --git a/.evergreen/variants/e2e-enterprise.yml b/.evergreen/variants/e2e-enterprise.yml new file mode 100644 index 000000000..94f219e94 --- /dev/null +++ b/.evergreen/variants/e2e-enterprise.yml @@ -0,0 +1,134 @@ +# Enterprise E2E Build Variants +# Build variants for MongoDB Enterprise Operator E2E tests with Ops Manager + +buildvariants: + ## Ops Manager 6.0 Build Variants ## + + - name: e2e_static_om60_kind_ubi + display_name: e2e_static_om60_kind_ubi + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om6_dependency + tasks: + - name: e2e_static_operator_task_group + + - name: e2e_om60_kind_ubi + display_name: e2e_om60_kind_ubi + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om6_dependency + tasks: + - name: e2e_operator_task_group + + - name: e2e_om60_appdb_kind_ubi + display_name: e2e_om60_appdb_kind_ubi + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om6_dependency + tasks: + - name: e2e_multi_cluster_om_appdb_task_group + + ## Ops Manager 7.0 Build Variants ## + + - name: e2e_static_om70_kind_ubi + display_name: e2e_static_om70_kind_ubi + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om7_dependency + tasks: + - name: e2e_static_operator_task_group + + - name: e2e_om70_kind_ubi + display_name: e2e_om70_kind_ubi + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om7_dependency + tasks: + - name: e2e_operator_task_group + + - name: e2e_om70_appdb_kind_ubi + display_name: e2e_om70_appdb_kind_ubi + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om7_dependency + tasks: + - name: e2e_multi_cluster_om_appdb_task_group + + ## Ops Manager 8.0 Build Variants ## + + - name: e2e_static_om80_kind_ubi + display_name: e2e_static_om80_kind_ubi + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om8_dependency + tasks: + - name: e2e_static_operator_task_group + + - name: e2e_om80_kind_ubi + display_name: e2e_om80_kind_ubi + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om8_dependency + tasks: + - name: e2e_operator_task_group + + - name: e2e_om80_appdb_kind_ubi + display_name: e2e_om80_appdb_kind_ubi + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om8_dependency + tasks: + - name: e2e_multi_cluster_om_appdb_task_group + + ## OpenShift Build Variants ## + + - name: e2e_mdb_openshift_ubi_cloudqa + display_name: e2e_mdb_openshift_ubi_cloudqa + tags: [ "e2e_test_suite", "cloudqa" ] + run_on: + - ubuntu2204-large + <<: *base_no_om_image_dependency + tasks: + - name: e2e_mdb_openshift_ubi_cloudqa_task_group + + ## OLM Build Variants ## + + - name: e2e_kind_olm_ubi + display_name: e2e_kind_olm_ubi + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om7_dependency + tasks: + - name: e2e_kind_olm_group + + ## Performance Build Variants ## + + - name: e2e_operator_race_with_telemetry_kind_ubi + display_name: e2e_operator_race_with_telemetry_kind_ubi + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om7_dependency + tasks: + - name: e2e_operator_race_with_telemetry_task_group + + ## Ops Manager Only Build Variants ## + + - name: e2e_ops_manager_kind_only_ubi + display_name: e2e_ops_manager_kind_only_ubi + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om7_dependency + tasks: + - name: e2e_ops_manager_kind_only_task_group diff --git a/.evergreen/variants/image-builds.yml b/.evergreen/variants/image-builds.yml new file mode 100644 index 000000000..a463acf87 --- /dev/null +++ b/.evergreen/variants/image-builds.yml @@ -0,0 +1,151 @@ +# Image Build Variants +# Build variants for Docker image building and preflight validation + +buildvariants: + ## Main Image Build Variant ## + + - name: init_test_run + display_name: init_test_run + run_on: + - ubuntu2204-large + tasks: + - name: build_test_image + - name: build_mco_test_image + - name: build_operator_ubi + - name: build_init_om_images_ubi + - name: build_init_appdb_images_ubi + - name: build_agent_images_ubi + - name: build_init_database_image_ubi + - name: build_database_image_ubi + - name: build_readiness_probe_image + - name: build_upgrade_hook_image + + ## Ops Manager Image Build Variants ## + + - name: build_om60_images + display_name: build_om60_images + run_on: + - ubuntu2204-large + tasks: + - name: build_om_images + vars: + om_version: *ops_manager_60_latest + + - name: build_om70_images + display_name: build_om70_images + run_on: + - ubuntu2204-large + tasks: + - name: build_om_images + vars: + om_version: *ops_manager_70_latest + + - name: build_om80_images + display_name: build_om80_images + run_on: + - ubuntu2204-large + tasks: + - name: build_om_images + vars: + om_version: *ops_manager_80_latest + + ## Preflight Validation Variants ## + + - name: preflight_release_images_check_only + display_name: preflight_release_images_check_only + run_on: + - rhel90-large + tasks: + - name: preflight_images_task_group + + ## Code Snippets Build Variants ## + + - name: gke_code_snippets + display_name: gke_code_snippets + tags: [ "release" ] + allowed_requesters: ["patch", "github_tag"] + depends_on: + - name: build_operator_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run + - name: build_readiness_probe_image + variant: init_test_run + - name: build_upgrade_hook_image + variant: init_test_run + - name: build_mco_test_image + variant: init_test_run + run_on: + - ubuntu2204-large + tasks: + - name: gke_code_snippets_task_group + + - name: kind_code_snippets + display_name: kind_code_snippets + tags: [ "release" ] + allowed_requesters: ["patch", "github_tag"] + depends_on: + - name: build_operator_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run + - name: build_readiness_probe_image + variant: init_test_run + - name: build_upgrade_hook_image + variant: init_test_run + - name: build_mco_test_image + variant: init_test_run + run_on: + - ubuntu2204-large + tasks: + - name: kind_code_snippets_task_group + + ## Pre-release Build Variants ## + + - name: prerelease_gke_code_snippets + display_name: prerelease_gke_code_snippets + tags: [ "release" ] + allowed_requesters: ["patch", "github_tag"] + depends_on: + - name: build_operator_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run + - name: build_readiness_probe_image + variant: init_test_run + - name: build_upgrade_hook_image + variant: init_test_run + - name: build_mco_test_image + variant: init_test_run + run_on: + - ubuntu2204-large + tasks: + - name: gke_code_snippets_task_group + + - name: prerelease_kind_code_snippets + display_name: prerelease_kind_code_snippets + tags: [ "release" ] + allowed_requesters: ["patch", "github_tag"] + depends_on: + - name: build_operator_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run + - name: build_readiness_probe_image + variant: init_test_run + - name: build_upgrade_hook_image + variant: init_test_run + - name: build_mco_test_image + variant: init_test_run + run_on: + - ubuntu2204-large + tasks: + - name: kind_code_snippets_task_group diff --git a/.evergreen/variants/releases.yml b/.evergreen/variants/releases.yml new file mode 100644 index 000000000..bbdc0331e --- /dev/null +++ b/.evergreen/variants/releases.yml @@ -0,0 +1,70 @@ +# Release Build Variants +# Build variants for releases, OpenShift bundles, and production deployments + +buildvariants: + ## Release Build Variants ## + + - name: release + display_name: release + tags: [ "release" ] + allowed_requesters: ["patch", "github_tag"] + run_on: + - ubuntu2204-large + tasks: + - name: release_operator + - name: release_init_appdb + - name: release_init_database + - name: release_init_ops_manager + - name: release_agent_operator_release + - name: release_agent + - name: release_database + - name: release_agents_on_ecr_conditional + - name: release_agents_on_ecr + - name: release_all_agents_on_ecr + - name: publish_ops_manager + + ## OpenShift Bundle Variants ## + + - name: openshift_bundles + display_name: openshift_bundles + tags: [ "openshift_bundles" ] + allowed_requesters: ["patch", "github_tag"] + run_on: + - ubuntu2204-large + tasks: + - name: prepare_and_upload_openshift_bundles_for_e2e + - name: prepare_and_upload_openshift_bundles + - name: run_conditionally_prepare_and_upload_openshift_bundles + + ## CSV Backup Variants ## + + - name: csv_backup + display_name: csv_backup + run_on: + - ubuntu2204-large + tasks: + - name: backup_csv_images_dry_run + - name: backup_csv_images_limit_3 + - name: backup_csv_images_all + + ## Performance Test Generation Variants ## + + - name: perf_test_generation + display_name: perf_test_generation + run_on: + - ubuntu2204-large + tasks: + - name: generate_perf_tasks_one_thread + - name: generate_perf_tasks_10_thread + - name: generate_perf_tasks_30_thread + + ## Pre-commit and Utility Variants ## + + - name: utilities + display_name: utilities + run_on: + - ubuntu2204-large + tasks: + - name: run_precommit_and_push + - name: prepare_aws + - name: run_retry_script diff --git a/.evergreen/variants/unit-tests.yml b/.evergreen/variants/unit-tests.yml new file mode 100644 index 000000000..a05382104 --- /dev/null +++ b/.evergreen/variants/unit-tests.yml @@ -0,0 +1,13 @@ +# Unit Test Build Variants +# Build variants for unit testing, linting, and code quality + +buildvariants: + ## Unit tests + lint build variant + + - name: unit_tests + display_name: "unit_tests" + tags: [ "unit_tests" ] + run_on: + - ubuntu2204-small + tasks: + - name: "unit_task_group" diff --git a/CI_MIGRATION_PLAN.md b/CI_MIGRATION_PLAN.md new file mode 100644 index 000000000..e9acde85c --- /dev/null +++ b/CI_MIGRATION_PLAN.md @@ -0,0 +1,322 @@ +# Evergreen CI Configuration Migration Plan + +## Overview +This document outlines a step-by-step migration plan to refactor the current monolithic Evergreen CI configuration into a modular, maintainable structure. + +## Current State Analysis + +### Existing Files +- `.evergreen.yml` (~2000 lines) - Main configuration +- `.evergreen-functions.yml` (~800 lines) - Function definitions +- `.evergreen-tasks.yml` (~1300 lines) - Task definitions +- `.evergreen-mco.yml` (~150 lines) - MCO-specific tasks +- `.evergreen-periodic-builds.yaml` (~180 lines) - Periodic builds + +### Key Pain Points +1. **Monolithic structure** - Hard to navigate and maintain +2. **Repetitive dependency definitions** - Multiple YAML anchors for similar dependencies +3. **Mixed concerns** - Unit tests, E2E tests, builds, and releases intermingled +4. **Complex task groups** - Overlapping setup/teardown logic +5. **Hardcoded values** - Registry URLs and versions scattered throughout + +## Target Architecture + +``` +.evergreen/ +├── main.yml # Main entry point (minimal) +├── variables/ +│ ├── common.yml # Global variables and anchors +│ ├── dependencies.yml # Build dependency definitions +│ └── versions.yml # Version specifications +├── functions/ +│ ├── setup.yml # Setup and environment functions +│ ├── build.yml # Build-related functions +│ ├── test.yml # Testing functions +│ └── deploy.yml # Deployment and release functions +├── tasks/ +│ ├── unit-tests.yml # Unit test task definitions +│ ├── e2e-tests.yml # E2E test task definitions +│ ├── image-builds.yml # Image building tasks +│ └── preflight.yml # Preflight and validation tasks +├── variants/ +│ ├── unit-tests.yml # Unit test build variants +│ ├── e2e-community.yml # Community E2E variants +│ ├── e2e-enterprise.yml # Enterprise E2E variants +│ ├── image-builds.yml # Image building variants +│ └── releases.yml # Release variants +└── periodic/ + └── scheduled-builds.yml # Periodic/scheduled builds +``` + +## Migration Strategy + +### Phase 1: Preparation and Validation (Week 1) +**Goal**: Set up infrastructure and validate current configuration + +#### Step 1.1: Create Migration Branch +```bash +git checkout -b ci-config-refactor +``` + +#### Step 1.2: Backup Current Configuration +```bash +mkdir -p .evergreen-backup +cp .evergreen*.yml .evergreen-backup/ +cp .evergreen*.yaml .evergreen-backup/ +``` + +#### Step 1.3: Validate Current Configuration +- Run existing CI pipeline to establish baseline +- Document all current build variants and their purposes +- Identify critical paths and dependencies + +#### Step 1.4: Create New Directory Structure +```bash +mkdir -p .evergreen/{variables,functions,tasks,variants,periodic} +``` + +### Phase 2: Extract Variables and Common Definitions (Week 1-2) +**Goal**: Centralize and organize variable definitions + +#### Step 2.1: Extract Global Variables +Create `.evergreen/variables/common.yml`: +- Move `exec_timeout_secs` and global parameters +- Extract common YAML anchors (`&setup_group`, `&teardown_group`, etc.) +- Standardize naming conventions + +#### Step 2.2: Extract Version Definitions +Create `.evergreen/variables/versions.yml`: +- Move Ops Manager version anchors (`&ops_manager_60_latest`, etc.) +- Centralize all version-related variables +- Add documentation for version update process + +#### Step 2.3: Extract Dependency Definitions +Create `.evergreen/variables/dependencies.yml`: +- Move dependency anchors (`&base_om6_dependency`, `&community_dependency`, etc.) +- Simplify and deduplicate dependency patterns +- Create reusable dependency templates + +### Phase 3: Modularize Functions (Week 2) +**Goal**: Organize functions by purpose + +#### Step 3.1: Extract Setup Functions +Create `.evergreen/functions/setup.yml`: +- `clone`, `download_kube_tools`, `setup_building_host` +- `setup_docker_sbom`, `configure_docker_auth` +- `setup_kubernetes_environment`, `setup_gcloud_cli` + +#### Step 3.2: Extract Build Functions +Create `.evergreen/functions/build.yml`: +- `pipeline`, `build_multi_cluster_binary` +- `release_docker_image_to_registry` +- Image building related functions + +#### Step 3.3: Extract Test Functions +Create `.evergreen/functions/test.yml`: +- `e2e_test`, `test_golang_unit` +- `preflight_image`, `setup_preflight` +- Test environment setup functions + +#### Step 3.4: Extract Deploy Functions +Create `.evergreen/functions/deploy.yml`: +- Release and deployment related functions +- Registry and artifact management functions + +### Phase 4: Organize Tasks by Type (Week 2-3) +**Goal**: Group tasks by functionality + +#### Step 4.1: Extract Unit Test Tasks +Create `.evergreen/tasks/unit-tests.yml`: +- `unit_tests_golang`, `lint_yaml`, `security_scan` +- Unit test task groups + +#### Step 4.2: Extract E2E Test Tasks +Create `.evergreen/tasks/e2e-tests.yml`: +- All E2E test task definitions from `.evergreen-tasks.yml` +- Organize by test type (community, enterprise, multi-cluster) + +#### Step 4.3: Extract Image Build Tasks +Create `.evergreen/tasks/image-builds.yml`: +- Image building tasks +- Preflight tasks from `.evergreen-tasks.yml` + +#### Step 4.4: Extract MCO Tasks +Create `.evergreen/tasks/mco-tests.yml`: +- Move content from `.evergreen-mco.yml` +- Integrate with main task structure + +### Phase 5: Organize Build Variants (Week 3) +**Goal**: Group build variants by purpose + +#### Step 5.1: Extract Unit Test Variants +Create `.evergreen/variants/unit-tests.yml`: +- `unit_tests` variant +- Lint and security scan variants + +#### Step 5.2: Extract E2E Variants +Create `.evergreen/variants/e2e-community.yml` and `.evergreen/variants/e2e-enterprise.yml`: +- Community E2E variants (KIND-based tests) +- Enterprise E2E variants (Ops Manager tests) +- Multi-cluster variants + +#### Step 5.3: Extract Build Variants +Create `.evergreen/variants/image-builds.yml`: +- Image building variants +- Preflight variants + +#### Step 5.4: Extract Release Variants +Create `.evergreen/variants/releases.yml`: +- Release-related variants +- OpenShift bundle variants + +### Phase 6: Handle Periodic Builds (Week 3) +**Goal**: Integrate periodic builds into new structure + +#### Step 6.1: Refactor Periodic Configuration +Create `.evergreen/periodic/scheduled-builds.yml`: +- Move content from `.evergreen-periodic-builds.yaml` +- Align with new function and task structure + +### Phase 7: Create Main Configuration (Week 3-4) +**Goal**: Create minimal main entry point + +#### Step 7.1: Create New Main Configuration +Create `.evergreen/main.yml`: +```yaml +# Main Evergreen Configuration +exec_timeout_secs: 7200 + +include: + # Variables + - filename: .evergreen/variables/common.yml + - filename: .evergreen/variables/versions.yml + - filename: .evergreen/variables/dependencies.yml + + # Functions + - filename: .evergreen/functions/setup.yml + - filename: .evergreen/functions/build.yml + - filename: .evergreen/functions/test.yml + - filename: .evergreen/functions/deploy.yml + + # Tasks + - filename: .evergreen/tasks/unit-tests.yml + - filename: .evergreen/tasks/e2e-tests.yml + - filename: .evergreen/tasks/image-builds.yml + - filename: .evergreen/tasks/mco-tests.yml + + # Build Variants + - filename: .evergreen/variants/unit-tests.yml + - filename: .evergreen/variants/e2e-community.yml + - filename: .evergreen/variants/e2e-enterprise.yml + - filename: .evergreen/variants/image-builds.yml + - filename: .evergreen/variants/releases.yml + +parameters: + - key: evergreen_retry + value: "true" + description: set this to false to suppress retries on failure +``` + +### Phase 8: Testing and Validation (Week 4) +**Goal**: Ensure new configuration works correctly + +#### Step 8.1: Syntax Validation +```bash +# Validate YAML syntax +yamllint .evergreen/ + +# Test Evergreen configuration parsing +evergreen validate --file .evergreen/main.yml +``` + +#### Step 8.2: Incremental Testing +- Test individual components (unit tests first) +- Validate task dependencies +- Test build variants one by one + +#### Step 8.3: Full Pipeline Testing +- Run complete CI pipeline with new configuration +- Compare results with baseline from Phase 1 +- Fix any issues discovered + +### Phase 9: Migration and Cleanup (Week 4) +**Goal**: Switch to new configuration and clean up + +#### Step 9.1: Update Main Evergreen File +Replace `.evergreen.yml` content: +```yaml +include: + - filename: .evergreen/main.yml +``` + +#### Step 9.2: Archive Old Configuration +```bash +mkdir -p .evergreen-legacy +mv .evergreen-functions.yml .evergreen-legacy/ +mv .evergreen-tasks.yml .evergreen-legacy/ +mv .evergreen-mco.yml .evergreen-legacy/ +mv .evergreen-periodic-builds.yaml .evergreen-legacy/ +``` + +#### Step 9.3: Update Documentation +- Update README with new CI structure +- Document maintenance procedures +- Create troubleshooting guide + +## Risk Mitigation + +### Rollback Plan +1. Keep original files in `.evergreen-backup/` +2. Maintain feature branch until validation complete +3. Quick rollback procedure documented + +### Testing Strategy +1. **Incremental validation** - Test each phase separately +2. **Parallel testing** - Run old and new configs side by side +3. **Critical path focus** - Prioritize release and main branch workflows + +### Communication Plan +1. **Team notification** - Inform team before starting migration +2. **Progress updates** - Weekly status updates +3. **Training session** - Walk through new structure with team + +## Success Criteria + +### Technical Metrics +- [ ] All existing functionality preserved +- [ ] CI pipeline execution time unchanged or improved +- [ ] Configuration files reduced in size by >50% +- [ ] Zero breaking changes to existing workflows + +### Maintainability Metrics +- [ ] New team members can understand structure in <30 minutes +- [ ] Adding new test takes <5 minutes +- [ ] Configuration changes isolated to single files +- [ ] Dependency management centralized and clear + +## Timeline Summary + +| Phase | Duration | Key Deliverables | +|-------|----------|------------------| +| 1 | Week 1 | Preparation, validation, directory structure | +| 2 | Week 1-2 | Variables extracted and organized | +| 3 | Week 2 | Functions modularized | +| 4 | Week 2-3 | Tasks organized by type | +| 5 | Week 3 | Build variants restructured | +| 6 | Week 3 | Periodic builds integrated | +| 7 | Week 3-4 | Main configuration created | +| 8 | Week 4 | Testing and validation | +| 9 | Week 4 | Migration and cleanup | + +**Total Duration**: 4 weeks + +## Next Steps + +1. **Review and approve** this migration plan +2. **Schedule migration window** - Coordinate with team +3. **Begin Phase 1** - Create migration branch and backup +4. **Set up monitoring** - Track migration progress and issues + +--- + +*This migration plan ensures a systematic, low-risk approach to modernizing the Evergreen CI configuration while maintaining all existing functionality.*