From fd701b22266a511ca5fe45b06e95b095ace1bef5 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Tue, 29 Jul 2025 15:58:56 +0200 Subject: [PATCH 01/23] add minikube scripts --- .evergreen-functions.yml | 18 ++++++++++++++++++ .evergreen.yml | 27 +++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/.evergreen-functions.yml b/.evergreen-functions.yml index c004dd098..3cd67d524 100644 --- a/.evergreen-functions.yml +++ b/.evergreen-functions.yml @@ -236,6 +236,15 @@ functions: - ${workdir}/bin binary: scripts/dev/setup_evg_host.sh + setup_ibm_host: &setup_ibm_host + command: subprocess.exec + type: setup + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + add_to_path: + - ${workdir}/bin + binary: scripts/dev/setup_minikube_host.sh + lint_repo: - command: subprocess.exec type: setup @@ -261,6 +270,15 @@ functions: - *setup_evg_host - *python_venv + # This differs for normal evg_host as we require minikube instead of kind for + # IBM machines + setup_building_host_minikube: + - *switch_context + - *setup_aws + - *configure_docker_auth + - *setup_ibm_host + - *python_venv + prune_docker_resources: - command: subprocess.exec type: setup diff --git a/.evergreen.yml b/.evergreen.yml index 209bf152a..17e882f95 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -77,6 +77,13 @@ variables: - func: download_kube_tools - func: setup_building_host + - &setup_group_ibm + setup_group_can_fail_task: true + setup_group: + - func: clone + - func: download_kube_tools + - func: setup_building_host_minikube + - &setup_group_multi_cluster setup_group_can_fail_task: true setup_group: @@ -1188,6 +1195,14 @@ task_groups: - e2e_om_ops_manager_backup <<: *teardown_group + - name: e2e_smoke_ibm_task_group + max_hosts: -1 + <<: *setup_group_ibm + <<: *setup_and_teardown_task + tasks: + - e2e_om_ops_manager_backup + <<: *teardown_group + - name: e2e_ops_manager_kind_5_0_only_task_group max_hosts: -1 <<: *setup_group @@ -1453,6 +1468,18 @@ buildvariants: tasks: - name: e2e_smoke_task_group + - name: e2e_smoke_ibm + display_name: e2e_smoke_ibm + tags: [ "e2e_test_suite", "e2e_smoke_release_test_suite" ] + run_on: + - rhel9-power-large + allowed_requesters: [ "patch", "github_tag" ] +# depends_on: +# - name: build_test_image +# variant: init_test_run + tasks: + - name: e2e_smoke_ibm_task_group + - name: e2e_static_smoke display_name: e2e_static_smoke tags: [ "e2e_test_suite", "e2e_smoke_release_test_suite" ] From 05cbe235bcd8af5aa2547631b891e0e0e783717f Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Tue, 29 Jul 2025 16:01:39 +0200 Subject: [PATCH 02/23] smoke test replicacset --- .evergreen.yml | 2 +- scripts/dev/contexts/e2e_smoke_ibm | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 scripts/dev/contexts/e2e_smoke_ibm diff --git a/.evergreen.yml b/.evergreen.yml index 17e882f95..2d2e113ae 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -1200,7 +1200,7 @@ task_groups: <<: *setup_group_ibm <<: *setup_and_teardown_task tasks: - - e2e_om_ops_manager_backup + - e2e_replica_set <<: *teardown_group - name: e2e_ops_manager_kind_5_0_only_task_group diff --git a/scripts/dev/contexts/e2e_smoke_ibm b/scripts/dev/contexts/e2e_smoke_ibm new file mode 100644 index 000000000..03384c26c --- /dev/null +++ b/scripts/dev/contexts/e2e_smoke_ibm @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" + +export ops_manager_version="cloud_qa" + +# This is required to be able to rebuild the om image and use that image which has been rebuild +export OPS_MANAGER_REGISTRY=268558157000.dkr.ecr.us-east-1.amazonaws.com/dev +CUSTOM_OM_VERSION=$(grep -E "^\s*-\s*&ops_manager_70_latest\s+(\S+)\s+#" <"${script_dir}"/../../../.evergreen.yml | awk '{print $3}') +export CUSTOM_OM_VERSION + +export CUSTOM_MDB_VERSION=6.0.5 +export CUSTOM_MDB_PREV_VERSION=5.0.7 From a1f3d90ed86606d7428f2499cfa3f140a19120c7 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Tue, 29 Jul 2025 16:26:38 +0200 Subject: [PATCH 03/23] smoke test replicacset --- scripts/evergreen/setup_kubectl.sh | 40 ++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/scripts/evergreen/setup_kubectl.sh b/scripts/evergreen/setup_kubectl.sh index ab9066ac1..00cf975fd 100755 --- a/scripts/evergreen/setup_kubectl.sh +++ b/scripts/evergreen/setup_kubectl.sh @@ -3,22 +3,52 @@ set -Eeou pipefail source scripts/dev/set_env_context.sh +# Detect system architecture and map to kubectl/helm architecture names +detect_architecture() { + local arch + arch=$(uname -m) + + case "${arch}" in + x86_64) + echo "amd64" + ;; + aarch64|arm64) + echo "arm64" + ;; + ppc64le) + echo "ppc64le" + ;; + s390x) + echo "s390x" + ;; + *) + echo "Unsupported architecture: ${arch}" >&2 + echo "Supported architectures: x86_64 (amd64), aarch64 (arm64), ppc64le, s390x" >&2 + exit 1 + ;; + esac +} + +# Detect the current architecture +ARCH=$(detect_architecture) +echo "Detected architecture: ${ARCH}" + bindir="${PROJECT_DIR}/bin" tmpdir="${PROJECT_DIR}/tmp" mkdir -p "${bindir}" "${tmpdir}" -echo "Downloading latest kubectl" -curl -s --retry 3 -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" +echo "Downloading latest kubectl for ${ARCH}" +curl -s --retry 3 -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/${ARCH}/kubectl" chmod +x kubectl echo "kubectl version --client" ./kubectl version --client mv kubectl "${bindir}" -echo "Downloading helm" +echo "Downloading helm for ${ARCH}" helm_archive="${tmpdir}/helm.tgz" helm_version="v3.17.1" -curl -s https://get.helm.sh/helm-${helm_version}-linux-amd64.tar.gz --output "${helm_archive}" +curl -s https://get.helm.sh/helm-${helm_version}-linux-${ARCH}.tar.gz --output "${helm_archive}" tar xfz "${helm_archive}" -C "${tmpdir}" &> /dev/null -mv "${tmpdir}/linux-amd64/helm" "${bindir}" +mv "${tmpdir}/linux-${ARCH}/helm" "${bindir}" "${bindir}"/helm version From 2d335f398673fff6ad51eeaeac3a1d0b29199292 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Tue, 29 Jul 2025 16:48:48 +0200 Subject: [PATCH 04/23] add minikube and make aws multiarch --- scripts/evergreen/setup_aws.sh | 37 +++- scripts/minikube/install-docker.sh | 108 +++++++++++ scripts/minikube/install-minikube.sh | 190 ++++++++++++++++++++ scripts/minikube/minikube_host.sh | 226 ++++++++++++++++++++++++ scripts/minikube/setup_minikube_host.sh | 115 ++++++++++++ 5 files changed, 675 insertions(+), 1 deletion(-) create mode 100755 scripts/minikube/install-docker.sh create mode 100755 scripts/minikube/install-minikube.sh create mode 100755 scripts/minikube/minikube_host.sh create mode 100755 scripts/minikube/setup_minikube_host.sh diff --git a/scripts/evergreen/setup_aws.sh b/scripts/evergreen/setup_aws.sh index 931eb0a36..072900639 100755 --- a/scripts/evergreen/setup_aws.sh +++ b/scripts/evergreen/setup_aws.sh @@ -3,6 +3,40 @@ set -Eeou pipefail source scripts/dev/set_env_context.sh +# Detect system architecture and map to AWS CLI architecture names +detect_aws_architecture() { + local arch + arch=$(uname -m) + + case "${arch}" in + x86_64) + echo "x86_64" + ;; + aarch64|arm64) + echo "aarch64" + ;; + ppc64le) + echo "Skipping AWS CLI installation: ppc64le (IBM Power) architecture is not supported by AWS CLI v2." >&2 + echo "AWS CLI v2 only supports: x86_64 (amd64), aarch64 (arm64)" >&2 + exit 0 + ;; + s390x) + echo "Skipping AWS CLI installation: s390x (IBM Z) architecture is not supported by AWS CLI v2." >&2 + echo "AWS CLI v2 only supports: x86_64 (amd64), aarch64 (arm64)" >&2 + exit 0 + ;; + *) + echo "Skipping AWS CLI installation: Unsupported architecture: ${arch}" >&2 + echo "AWS CLI v2 only supports: x86_64 (amd64), aarch64 (arm64)" >&2 + exit 0 + ;; + esac +} + +# Detect the current architecture +ARCH=$(detect_aws_architecture) +echo "Detected architecture: ${ARCH} (AWS CLI v2 supported)" + INSTALL_DIR="${workdir:?}/.local/lib/aws" BIN_LOCATION="${workdir}/bin" @@ -11,7 +45,8 @@ mkdir -p "${BIN_LOCATION}" tmpdir=$(mktemp -d) cd "${tmpdir}" -curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" +echo "Downloading AWS CLI v2 for ${ARCH}..." +curl "https://awscli.amazonaws.com/awscli-exe-linux-${ARCH}.zip" -o "awscliv2.zip" unzip awscliv2.zip &> /dev/null docker_dir="/home/${USER}/.docker" diff --git a/scripts/minikube/install-docker.sh b/scripts/minikube/install-docker.sh new file mode 100755 index 000000000..04ae3f7d8 --- /dev/null +++ b/scripts/minikube/install-docker.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +set -Eeou pipefail + +# Script to install Docker on s390x architecture (specifically for RHEL/Ubuntu based systems) + +print_usage() { + echo "Usage: $0 [options]" + echo "Options:" + echo " -h, --help Show this help message" + echo " -u, --user Username to add to docker group (optional)" + echo "" + echo "This script installs Docker on s390x architecture systems." +} + +DOCKER_USER="" + +while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + print_usage + exit 0 + ;; + -u|--user) + DOCKER_USER="$2" + shift 2 + ;; + *) + echo "Unknown option: $1" + print_usage + exit 1 + ;; + esac +done + +echo "Installing Docker on s390x architecture..." + +# Detect OS +if [[ -f /etc/redhat-release ]]; then + OS_TYPE="rhel" +elif [[ -f /etc/debian_version ]]; then + OS_TYPE="debian" +else + echo "Unsupported OS. This script supports RHEL/CentOS and Ubuntu/Debian." + exit 1 +fi + +# Install Docker based on OS +if [[ "$OS_TYPE" == "rhel" ]]; then + echo "Detected RHEL/CentOS system..." + + # Remove any existing Docker packages + sudo yum remove -y docker docker-client docker-client-latest docker-common docker-latest docker-latest-logrotate docker-logrotate docker-engine || true + + # Install required packages (some may not exist on newer RHEL versions) + sudo yum install -y yum-utils || echo "yum-utils already installed or unavailable" + sudo yum install -y device-mapper-persistent-data lvm2 || echo "device-mapper packages may not be available on this system" + + # Add Docker repository + sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo + + # Install Docker CE + sudo yum install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + +elif [[ "$OS_TYPE" == "debian" ]]; then + echo "Detected Ubuntu/Debian system..." + + # Remove any existing Docker packages + sudo apt-get remove -y docker docker-engine docker.io containerd runc || true + + # Update package index + sudo apt-get update + + # Install required packages + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release + + # Add Docker's official GPG key + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + + # Set up stable repository + echo "deb [arch=s390x signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + + # Update package index again + sudo apt-get update + + # Install Docker CE + sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin +fi + +# Start and enable Docker service +sudo systemctl start docker +sudo systemctl enable docker + +# Add user to docker group if specified +if [[ -n "$DOCKER_USER" ]]; then + echo "Adding user '$DOCKER_USER' to docker group..." + sudo usermod -aG docker "$DOCKER_USER" + echo "Note: User '$DOCKER_USER' needs to log out and log back in for group membership to take effect." +fi + +# Verify installation +echo "Verifying Docker installation..." +sudo docker --version +sudo docker run --rm hello-world + +echo "Docker installation completed successfully!" +echo "" +echo "If you added a user to the docker group, they need to log out and log back in." +echo "You can also run 'newgrp docker' to apply the group membership in the current session." \ No newline at end of file diff --git a/scripts/minikube/install-minikube.sh b/scripts/minikube/install-minikube.sh new file mode 100755 index 000000000..3e6dd4e13 --- /dev/null +++ b/scripts/minikube/install-minikube.sh @@ -0,0 +1,190 @@ +#!/usr/bin/env bash +set -Eeou pipefail + +# Script to install and configure minikube on s390x architecture + +print_usage() { + echo "Usage: $0 [options]" + echo "Options:" + echo " -h, --help Show this help message" + echo " -v, --version VERSION Minikube version to install (default: latest)" + echo " -k, --kubernetes VER Kubernetes version (default: latest stable)" + echo " -m, --memory MEMORY Memory allocation (default: 8192mb)" + echo " -c, --cpus CPUS CPU allocation (default: 4)" + echo " --profile PROFILE Minikube profile name (default: minikube)" + echo " --start Start minikube after installation" + echo "" + echo "This script installs minikube on s390x architecture and configures it for MongoDB Kubernetes e2e testing." +} + +MINIKUBE_VERSION="latest" +K8S_VERSION="" +MEMORY="8192" +CPUS="4" +PROFILE="minikube" +START_MINIKUBE="false" + +while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + print_usage + exit 0 + ;; + -v|--version) + MINIKUBE_VERSION="$2" + shift 2 + ;; + -k|--kubernetes) + K8S_VERSION="$2" + shift 2 + ;; + -m|--memory) + MEMORY="$2" + shift 2 + ;; + -c|--cpus) + CPUS="$2" + shift 2 + ;; + --profile) + PROFILE="$2" + shift 2 + ;; + --start) + START_MINIKUBE="true" + shift + ;; + *) + echo "Unknown option: $1" + print_usage + exit 1 + ;; + esac +done + +echo "Installing minikube on s390x architecture..." + +# Verify Docker is installed +if ! command -v docker &> /dev/null; then + echo "Error: Docker is required but not installed. Please install Docker first." + echo "You can use the install-docker.sh script in this directory." + exit 1 +fi + +# Verify Docker is running +if ! docker info &> /dev/null; then + echo "Error: Docker is not running. Please start Docker service." + exit 1 +fi + +# Install kubectl if not present +if ! command -v kubectl &> /dev/null; then + echo "Installing kubectl..." + + # Get the latest kubectl version if not specified + if [[ -z "$K8S_VERSION" ]]; then + K8S_VERSION=$(curl -L -s https://dl.k8s.io/release/stable.txt) + fi + + # Download kubectl for s390x + curl -LO "https://dl.k8s.io/release/${K8S_VERSION}/bin/linux/s390x/kubectl" + + # Verify the binary + curl -LO "https://dl.k8s.io/${K8S_VERSION}/bin/linux/s390x/kubectl.sha256" + echo "$(cat kubectl.sha256) kubectl" | sha256sum --check + + # Install kubectl + sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl + rm -f kubectl kubectl.sha256 + + echo "kubectl installed successfully" +fi + +# Install minikube +echo "Installing minikube..." + +if [[ "$MINIKUBE_VERSION" == "latest" ]]; then + # Get the latest minikube version + MINIKUBE_VERSION=$(curl -s https://api.github.com/repos/kubernetes/minikube/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/') +fi + +# Download minikube for s390x +curl -Lo minikube "https://github.com/kubernetes/minikube/releases/download/${MINIKUBE_VERSION}/minikube-linux-s390x" + +# Make it executable and install +chmod +x minikube +sudo install minikube /usr/local/bin/ + +# Clean up +rm -f minikube + +echo "Minikube ${MINIKUBE_VERSION} installed successfully" + +# Configure minikube for MongoDB Kubernetes testing +echo "Configuring minikube for MongoDB Kubernetes e2e testing..." + +# Set default driver to docker +minikube config set driver docker + +# Configure resource limits +minikube config set memory "${MEMORY}mb" +minikube config set cpus "${CPUS}" + +# Enable required addons for testing +ADDONS=( + "storage-provisioner" + "default-storageclass" + "volumesnapshots" + "csi-hostpath-driver" +) + +echo "Minikube configuration completed." + +if [[ "$START_MINIKUBE" == "true" ]]; then + echo "Starting minikube cluster with profile '${PROFILE}'..." + + # Start minikube with specific configuration for MongoDB testing + minikube start \ + --profile="${PROFILE}" \ + --driver=docker \ + --memory="${MEMORY}mb" \ + --cpus="${CPUS}" \ + --disk-size=50g \ + --extra-config=kubelet.authentication-token-webhook=true \ + --extra-config=kubelet.authorization-mode=Webhook \ + --extra-config=scheduler.bind-address=0.0.0.0 \ + --extra-config=controller-manager.bind-address=0.0.0.0 \ + ${K8S_VERSION:+--kubernetes-version=$K8S_VERSION} + + # Wait for cluster to be ready + echo "Waiting for cluster to be ready..." + kubectl wait --for=condition=Ready nodes --all --timeout=300s + + # Enable addons + for addon in "${ADDONS[@]}"; do + echo "Enabling addon: $addon" + minikube addons enable "$addon" --profile="${PROFILE}" || true + done + + # Create directories that MongoDB tests expect (similar to kind setup) + echo "Setting up test directories..." + minikube ssh --profile="${PROFILE}" -- 'sudo mkdir -p /opt/data/mongo-data-{0..2} /opt/data/mongo-logs-{0..2}' + minikube ssh --profile="${PROFILE}" -- 'sudo chmod 777 /opt/data/mongo-data-* /opt/data/mongo-logs-*' + + echo "Minikube cluster started successfully!" + echo "" + echo "To use this cluster:" + echo " export KUBECONFIG=\$(minikube kubeconfig --profile=${PROFILE})" + echo " kubectl get nodes" + echo "" + echo "To stop the cluster:" + echo " minikube stop --profile=${PROFILE}" +else + echo "" + echo "Minikube installed but not started." + echo "To start minikube later, run:" + echo " minikube start --profile=${PROFILE} --driver=docker --memory=${MEMORY}mb --cpus=${CPUS}" +fi + +echo "" +echo "Installation completed successfully!" \ No newline at end of file diff --git a/scripts/minikube/minikube_host.sh b/scripts/minikube/minikube_host.sh new file mode 100755 index 000000000..ab410d7bc --- /dev/null +++ b/scripts/minikube/minikube_host.sh @@ -0,0 +1,226 @@ +#!/usr/bin/env bash + +# This is a helper script for running tests on s390x Hosts. +# It allows to configure minikube clusters and expose remote API servers on a local machine to +# enable local development while running minikube cluster on s390x instance. +# Run "minikube_host.sh help" command to see the full usage. +# Similar to evg_host.sh but uses minikube instead of kind. + +set -Eeou pipefail + +test "${MDB_BASH_DEBUG:-0}" -eq 1 && set -x + +source scripts/dev/set_env_context.sh +source scripts/funcs/printing + +if [[ -z "${S390_HOST_NAME}" ]]; then + echo "S390_HOST_NAME env var is missing" + echo "Set it to your s390x host connection string (e.g., user@hostname)" + exit 1 +fi + +get_host_url() { + echo "${S390_HOST_NAME}" +} + +cmd=${1-""} + +if [[ "${cmd}" != "" && "${cmd}" != "help" ]]; then + host_url=$(get_host_url) +fi + +kubeconfig_path="${HOME}/.operator-dev/s390-host.kubeconfig" + +configure() { + shift 1 + arch=${1-"$(uname -m)"} + + echo "Configuring minikube host ${S390_HOST_NAME} (${host_url}) with architecture ${arch}" + + if [[ "${cmd}" == "configure" && ! "${arch}" =~ ^(s390x|ppc64le|x86_64|aarch64)$ ]]; then + echo "'configure' command supports the following architectures: s390x, ppc64le, x86_64, aarch64" + exit 1 + fi + + ssh -T -q "${host_url}" "sudo chown \$(whoami):\$(whoami) ~/.docker || true; mkdir -p ~/.docker" + if [[ -f "${HOME}/.docker/config.json" ]]; then + echo "Copying local ~/.docker/config.json authorization credentials to s390x host" + jq '. | with_entries(select(.key == "auths"))' "${HOME}/.docker/config.json" | ssh -T -q "${host_url}" 'cat > ~/.docker/config.json' + fi + + sync + + ssh -T -q "${host_url}" "cd ~/mongodb-kubernetes; scripts/dev/switch_context.sh root-context; scripts/minikube/setup_minikube_host.sh ${arch}" +} + +sync() { + rsync --verbose --archive --compress --human-readable --recursive --progress \ + --delete --delete-excluded --max-size=1000000 --prune-empty-dirs \ + -e ssh \ + --include-from=.rsyncinclude \ + --exclude-from=.gitignore \ + --exclude-from=.rsyncignore \ + ./ "${host_url}:~/mongodb-kubernetes/" + + rsync --verbose --no-links --recursive --prune-empty-dirs --archive --compress --human-readable \ + --max-size=1000000 \ + -e ssh \ + ~/.operator-dev/ \ + "${host_url}:~/.operator-dev" & + + wait +} + +remote-prepare-local-e2e-run() { + set -x + sync + cmd make switch context=e2e_mdb_kind_ubi_cloudqa + cmd scripts/dev/prepare_local_e2e_run.sh + rsync --verbose --no-links --recursive --prune-empty-dirs --archive --compress --human-readable \ + --max-size=1000000 \ + -e ssh \ + "${host_url}:~/mongodb-kubernetes/.multi_cluster_local_test_files" \ + ./ & + scp "${host_url}:~/.operator-dev/multicluster_kubeconfig" "${KUBE_CONFIG_PATH}" & + + wait +} + +get-kubeconfig() { + # For minikube, we need to get the kubeconfig and certificates + echo "Getting kubeconfig from minikube on s390x host..." + local profile=${MINIKUBE_PROFILE:-mongodb-e2e} + + # Create local minikube directory structure + mkdir -p "${HOME}/.minikube/profiles/${profile}" + + # Copy certificates from remote host + echo "Copying minikube certificates..." + scp "${host_url}:~/.minikube/ca.crt" "${HOME}/.minikube/" + scp "${host_url}:~/.minikube/profiles/${profile}/client.crt" "${HOME}/.minikube/profiles/${profile}/" + scp "${host_url}:~/.minikube/profiles/${profile}/client.key" "${HOME}/.minikube/profiles/${profile}/" + + # Get kubeconfig and update paths to local ones + ssh -T -q "${host_url}" "cd ~/mongodb-kubernetes; export KUBE_ENVIRONMENT_NAME=minikube; export MINIKUBE_PROFILE=${profile}; kubectl config view --raw" > "${kubeconfig_path}" + + # Update certificate paths to local paths + sed -i '' "s|/home/cloud-user/.minikube|${HOME}/.minikube|g" "${kubeconfig_path}" + + # Update server addresses to use localhost for tunneling + sed -i '' "s|https://192.168.[0-9]*.[0-9]*:\([0-9]*\)|https://127.0.0.1:\1|g" "${kubeconfig_path}" + + echo "Copied minikube kubeconfig and certificates to ${kubeconfig_path}" +} + +recreate-minikube-cluster() { + shift 1 + profile_name=${1:-mongodb-e2e} + configure "$(uname -m)" 2>&1| prepend "minikube_host.sh configure" + echo "Recreating minikube cluster ${profile_name} on ${S390_HOST_NAME} (${host_url})..." + # shellcheck disable=SC2088 + ssh -T "${host_url}" "cd ~/mongodb-kubernetes; export KUBE_ENVIRONMENT_NAME=minikube; export MINIKUBE_PROFILE=${profile_name}; minikube delete --profile=${profile_name} || true; minikube start --profile=${profile_name} --driver=docker --memory=8192mb --cpus=4" + echo "Copying kubeconfig to ${kubeconfig_path}" + get-kubeconfig +} + +tunnel() { + shift 1 + echo "Setting up tunnel for minikube cluster..." + local profile=${MINIKUBE_PROFILE:-mongodb-e2e} + + # Get the minikube API server port from remote host + local api_port + api_port=$(ssh -T -q "${host_url}" "cd ~/mongodb-kubernetes; export MINIKUBE_PROFILE=${profile}; minikube ip --profile=${profile} 2>/dev/null && echo ':8443' | tr -d '\n'") + + if [[ -z "${api_port}" ]]; then + echo "Could not determine minikube API server details. Is the cluster running?" + return 1 + fi + + # Extract just the port (8443) + local port="8443" + echo "Forwarding localhost:${port} to minikube cluster API server" + + # Forward the API server port through minikube + set -x + # shellcheck disable=SC2029 + ssh -L "${port}:$(ssh -T -q "${host_url}" "export MINIKUBE_PROFILE=${profile}; minikube ip --profile=${profile}"):${port}" "${host_url}" "$@" + set +x +} + +retry_with_sleep() { + shift 1 + cmd=$1 + local sleep_time + sleep_time=5 + + while true; do + ${cmd} || true + echo "Retrying command after ${sleep_time} of sleep: ${cmd}" + sleep 5; + done +} + +ssh_to_host() { + shift 1 + # shellcheck disable=SC2029 + ssh "$@" "${host_url}" +} + +upload-my-ssh-private-key() { + ssh -T -q "${host_url}" "mkdir -p ~/.ssh" + scp "${HOME}/.ssh/id_rsa" "${host_url}:~/.ssh/id_rsa" + scp "${HOME}/.ssh/id_rsa.pub" "${host_url}:~/.ssh/id_rsa.pub" + ssh -T -q "${host_url}" "chmod 700 ~/.ssh && chown -R \$(whoami):\$(whoami) ~/.ssh" +} + +cmd() { + if [[ "$1" == "cmd" ]]; then + shift 1 + fi + + cmd="cd ~/mongodb-kubernetes; $*" + ssh -T -q "${host_url}" "${cmd}" +} + +usage() { + echo "USAGE: + minikube_host.sh + +PREREQUISITES: + - s390x host with SSH access + - define S390_HOST_NAME env var (e.g., export S390_HOST_NAME=user@hostname) + - SSH key-based authentication configured + +COMMANDS: + configure installs on a host: calls sync, switches context, installs necessary software (auto-detects arch) + sync rsync of project directory + recreate-minikube-cluster recreates minikube cluster with specific profile and executes get-kubeconfig + remote-prepare-local-e2e-run executes prepare-local-e2e on the remote host + get-kubeconfig copies remote minikube kubeconfig locally to ~/.operator-dev/s390-host.kubeconfig + tunnel [args] creates ssh session with tunneling to all API servers + ssh [args] creates ssh session passing optional arguments to ssh + cmd [command with args] execute command as if being on s390x host + upload-my-ssh-private-key uploads your ssh keys (~/.ssh/id_rsa) to s390x host + help this message + +EXAMPLES: + export S390_HOST_NAME=user@ibmz8 + minikube_host.sh tunnel + minikube_host.sh cmd 'make e2e test=replica_set' +" +} + +case ${cmd} in +configure) configure "$@" ;; +recreate-minikube-cluster) recreate-minikube-cluster "$@" ;; +get-kubeconfig) get-kubeconfig ;; +remote-prepare-local-e2e-run) remote-prepare-local-e2e-run ;; +ssh) ssh_to_host "$@" ;; +tunnel) retry_with_sleep tunnel "$@" ;; +sync) sync ;; +cmd) cmd "$@" ;; +upload-my-ssh-private-key) upload-my-ssh-private-key ;; +help) usage ;; +*) usage ;; +esac diff --git a/scripts/minikube/setup_minikube_host.sh b/scripts/minikube/setup_minikube_host.sh new file mode 100755 index 000000000..7628c388c --- /dev/null +++ b/scripts/minikube/setup_minikube_host.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash + +# this script downloads necessary tooling for alternative architectures (s390x, ppc64le) using minikube (similar to setup_evg_host.sh) + +set -Eeou pipefail + +check_disk_space() { + echo "Checking available disk space..." + local available_gb + available_gb=$(df / | awk 'NR==2 {print int($4/1024/1024)}') + + if [[ $available_gb -lt 5 ]]; then + echo "ERROR: Insufficient disk space. Available: ${available_gb}GB, Required: 5GB minimum" + echo "Please clean up disk space before continuing:" + echo " sudo dnf clean all" + echo " sudo rm -rf /var/cache/dnf/* /tmp/* /var/tmp/*" + echo " docker system prune -af" + return 1 + fi + + echo "Disk space check passed: ${available_gb}GB available" +} + +set_limits() { + echo "Increasing fs.inotify.max_user_instances" + sudo sysctl -w fs.inotify.max_user_instances=8192 + + echo "Increasing fs.inotify.max_user_watches" + sudo sysctl -w fs.inotify.max_user_watches=10485760 + + echo "Increasing the number of open files" + nofile_max=$(cat /proc/sys/fs/nr_open) + nproc_max=$(ulimit -u) + sudo tee -a /etc/security/limits.conf <> ~/.bashrc +} + +check_disk_space +set_limits +setup_group + +download_minikube & +download_docker & + +wait + +echo "Setting up minikube environment variables..." +echo 'export KUBE_ENVIRONMENT_NAME=minikube' >> ~/.bashrc +echo 'export MINIKUBE_PROFILE=${MINIKUBE_PROFILE:-mongodb-e2e}' >> ~/.bashrc +echo 'export KUBECONFIG=$(minikube kubeconfig --profile=${MINIKUBE_PROFILE:-mongodb-e2e} 2>/dev/null || echo ~/.kube/config)' >> ~/.bashrc + +echo "Minikube host setup completed successfully for ${ARCH}!" +echo "SETUP_GROUP: ${SETUP_GROUP}" From 46f0998d2335fad9311734cbede5b14e6864415d Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Wed, 30 Jul 2025 13:52:17 +0200 Subject: [PATCH 05/23] make jq multiarch and all other scriupts as well --- .evergreen-functions.yml | 15 +-- .evergreen.yml | 1 - scripts/dev/recreate_python_venv.sh | 87 ++++++++++++++-- scripts/evergreen/setup_aws.sh | 125 +++++++++++++++++------ scripts/evergreen/setup_jq.sh | 4 +- scripts/evergreen/setup_minikube_host.sh | 66 ++++++++++++ scripts/minikube/minikube_host.sh | 12 +-- 7 files changed, 254 insertions(+), 56 deletions(-) create mode 100755 scripts/evergreen/setup_minikube_host.sh diff --git a/.evergreen-functions.yml b/.evergreen-functions.yml index 3cd67d524..71837ab97 100644 --- a/.evergreen-functions.yml +++ b/.evergreen-functions.yml @@ -243,7 +243,7 @@ functions: working_dir: src/github.com/mongodb/mongodb-kubernetes add_to_path: - ${workdir}/bin - binary: scripts/dev/setup_minikube_host.sh + binary: scripts/minikube/setup_minikube_host.sh lint_repo: - command: subprocess.exec @@ -271,13 +271,16 @@ functions: - *python_venv # This differs for normal evg_host as we require minikube instead of kind for - # IBM machines + # IBM machines and install aws cli via pip instead setup_building_host_minikube: - *switch_context - - *setup_aws - - *configure_docker_auth - - *setup_ibm_host - - *python_venv + - command: subprocess.exec + type: setup + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + add_to_path: + - ${workdir}/bin + command: scripts/evergreen/setup_minikube_host.sh prune_docker_resources: - command: subprocess.exec diff --git a/.evergreen.yml b/.evergreen.yml index 2d2e113ae..31655847c 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -81,7 +81,6 @@ variables: setup_group_can_fail_task: true setup_group: - func: clone - - func: download_kube_tools - func: setup_building_host_minikube - &setup_group_multi_cluster diff --git a/scripts/dev/recreate_python_venv.sh b/scripts/dev/recreate_python_venv.sh index fb1f9ab8f..a5388e40f 100755 --- a/scripts/dev/recreate_python_venv.sh +++ b/scripts/dev/recreate_python_venv.sh @@ -4,26 +4,101 @@ set -Eeou pipefail +# Parse command line arguments +INSTALL_REQUIREMENTS=true + +while [[ $# -gt 0 ]]; do + case $1 in + --skip-requirements) + INSTALL_REQUIREMENTS=false + shift + ;; + -h|--help) + echo "Usage: $0 [--skip-requirements]" + echo " --skip-requirements Skip installing requirements.txt" + echo " -h, --help Show this help message" + exit 0 + ;; + *) + echo "Unknown option: $1" + echo "Use -h or --help for usage information" + exit 1 + ;; + esac +done + source scripts/dev/set_env_context.sh +# Ensure Python 3.10 is available, install if needed +ensure_python310() { + echo "Checking current Python version..." >&2 + + # Check if current python is 3.10 + if command -v python3 &> /dev/null; then + local version + if version=$(python3 --version 2>&1) && [[ "${version}" == *"Python 3.10"* ]]; then + echo "Found Python 3.10: ${version}" >&2 + echo "python3" + return 0 + else + echo "Current python3 version: ${version}" >&2 + fi + fi + + # Try to install Python 3.10 using pyenv if available + if command -v pyenv &> /dev/null; then + echo "Python 3.10 not found. Attempting to install via pyenv..." >&2 + + # Check if any 3.10 version is already installed + if pyenv versions --bare | grep -q "^3\.10\."; then + local installed_version + installed_version=$(pyenv versions --bare | grep "^3\.10\." | head -1) + echo "Found existing pyenv Python 3.10: ${installed_version}" >&2 + pyenv global "${installed_version}" + echo "python3" + return 0 + fi + + # Install latest Python 3.10 + local latest_310 + latest_310=$(pyenv install --list | grep -E "^[[:space:]]*3\.10\.[0-9]+$" | tail -1 | xargs) + if [[ -n "${latest_310}" ]]; then + echo "Installing Python ${latest_310} via pyenv..." >&2 + if pyenv install "${latest_310}"; then + pyenv global "${latest_310}" + echo "python3" + return 0 + fi + fi + fi + + echo "Error: No suitable Python 3.10 installation found and unable to install via pyenv." >&2 + echo "Please ensure Python 3.10 is installed or pyenv is available." >&2 + return 1 +} + if [[ -d "${PROJECT_DIR}"/venv ]]; then echo "Removing venv..." cd "${PROJECT_DIR}" rm -rf "venv" fi -# in our EVG hosts, python versions are always in /opt/python -python_bin="/opt/python/${PYTHON_VERSION}/bin/python3" -if [[ "$(uname)" == "Darwin" ]]; then - python_bin="python${PYTHON_VERSION}" -fi +# Ensure Python 3.10 is available +python_bin=$(ensure_python310) echo "Using python from the following path: ${python_bin}" "${python_bin}" -m venv venv source venv/bin/activate pip install --upgrade pip -pip install -r requirements.txt + +if [[ "${INSTALL_REQUIREMENTS}" == "true" ]]; then + echo "Installing requirements.txt..." + pip install -r requirements.txt +else + echo "Skipping requirements.txt installation (--skip-requirements flag used)" +fi + echo "Python venv was recreated successfully." echo "Current python path: $(which python)" python --version diff --git a/scripts/evergreen/setup_aws.sh b/scripts/evergreen/setup_aws.sh index 072900639..5563a8a30 100755 --- a/scripts/evergreen/setup_aws.sh +++ b/scripts/evergreen/setup_aws.sh @@ -3,51 +3,116 @@ set -Eeou pipefail source scripts/dev/set_env_context.sh -# Detect system architecture and map to AWS CLI architecture names -detect_aws_architecture() { +# Detect system architecture +detect_architecture() { local arch arch=$(uname -m) + echo "Detected architecture: ${arch}" >&2 + echo "${arch}" +} + +# Install AWS CLI v2 via binary download (for x86_64 and aarch64) +install_aws_cli_binary() { + local arch="$1" + echo "Installing AWS CLI v2 via binary download for ${arch}..." + # Map architecture names for AWS CLI download URLs + local aws_arch case "${arch}" in x86_64) - echo "x86_64" + aws_arch="x86_64" ;; aarch64|arm64) - echo "aarch64" - ;; - ppc64le) - echo "Skipping AWS CLI installation: ppc64le (IBM Power) architecture is not supported by AWS CLI v2." >&2 - echo "AWS CLI v2 only supports: x86_64 (amd64), aarch64 (arm64)" >&2 - exit 0 - ;; - s390x) - echo "Skipping AWS CLI installation: s390x (IBM Z) architecture is not supported by AWS CLI v2." >&2 - echo "AWS CLI v2 only supports: x86_64 (amd64), aarch64 (arm64)" >&2 - exit 0 + aws_arch="aarch64" ;; *) - echo "Skipping AWS CLI installation: Unsupported architecture: ${arch}" >&2 - echo "AWS CLI v2 only supports: x86_64 (amd64), aarch64 (arm64)" >&2 - exit 0 + echo "Error: Unsupported architecture for binary installation: ${arch}" >&2 + return 1 ;; esac + + # Download and install AWS CLI v2 + local temp_dir + temp_dir=$(mktemp -d) + cd "${temp_dir}" + + echo "Downloading AWS CLI v2 for ${aws_arch}..." + curl -s "https://awscli.amazonaws.com/awscli-exe-linux-${aws_arch}.zip" -o "awscliv2.zip" + + unzip -q awscliv2.zip + sudo ./aws/install --update + + # Clean up + cd - > /dev/null + rm -rf "${temp_dir}" + + # Verify installation + if command -v aws &> /dev/null; then + echo "AWS CLI v2 installed successfully:" + aws --version + else + echo "Error: AWS CLI v2 installation failed" >&2 + return 1 + fi } -# Detect the current architecture -ARCH=$(detect_aws_architecture) -echo "Detected architecture: ${ARCH} (AWS CLI v2 supported)" +# Install AWS CLI v1 via pip (for IBM architectures: ppc64le, s390x) +install_aws_cli_pip() { + echo "Installing AWS CLI v1 via pip (for IBM architectures)..." + + # Ensure pip is available + if ! command -v pip3 &> /dev/null && ! command -v pip &> /dev/null; then + echo "Error: pip is not available. Please install Python and pip first." >&2 + return 1 + fi -INSTALL_DIR="${workdir:?}/.local/lib/aws" -BIN_LOCATION="${workdir}/bin" + # Use pip3 if available, otherwise pip + local pip_cmd="pip3" + if ! command -v pip3 &> /dev/null; then + pip_cmd="pip" + fi -mkdir -p "${BIN_LOCATION}" + echo "Installing AWS CLI using ${pip_cmd}..." + ${pip_cmd} install --user awscli -tmpdir=$(mktemp -d) -cd "${tmpdir}" + # Add ~/.local/bin to PATH if not already there (where pip --user installs) + if [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" + echo "Added ~/.local/bin to PATH" + fi + + # Verify installation + if command -v aws &> /dev/null; then + echo "AWS CLI v1 installed successfully:" + aws --version + else + echo "Error: AWS CLI v1 installation failed or not found in PATH" >&2 + return 1 + fi +} + +# Main installation logic +install_aws_cli() { + local arch + arch=$(detect_architecture) + + case "${arch}" in + ppc64le|s390x) + echo "IBM architecture detected (${arch}). Using pip installation..." + install_aws_cli_pip + ;; + x86_64|aarch64|arm64) + echo "Standard architecture detected (${arch}). Using binary installation..." + install_aws_cli_binary "${arch}" + ;; + *) + echo "Warning: Unknown architecture ${arch}. Falling back to pip installation..." + install_aws_cli_pip + ;; + esac +} -echo "Downloading AWS CLI v2 for ${ARCH}..." -curl "https://awscli.amazonaws.com/awscli-exe-linux-${ARCH}.zip" -o "awscliv2.zip" -unzip awscliv2.zip &> /dev/null +install_aws_cli docker_dir="/home/${USER}/.docker" if [[ ! -d "${docker_dir}" ]]; then @@ -56,7 +121,5 @@ fi sudo chown "${USER}":"${USER}" "${docker_dir}" -R sudo chmod g+rwx "${docker_dir}" -R -sudo ./aws/install --bin-dir "${BIN_LOCATION}" --install-dir "${INSTALL_DIR}" --update -cd - -rm -rf "${tmpdir}" +echo "AWS CLI setup completed successfully." diff --git a/scripts/evergreen/setup_jq.sh b/scripts/evergreen/setup_jq.sh index e21d4a07e..23ad3e723 100755 --- a/scripts/evergreen/setup_jq.sh +++ b/scripts/evergreen/setup_jq.sh @@ -10,4 +10,6 @@ set -Eeou pipefail source scripts/dev/set_env_context.sh source scripts/funcs/install -download_and_install_binary "${PROJECT_DIR:-.}/bin" jq "https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64" +arch=$(uname -m) + +download_and_install_binary "${PROJECT_DIR:-.}/bin" jq "https://github.com/stedolan/jq/releases/download/jq-1.8.1/jq-linux-${arch}" diff --git a/scripts/evergreen/setup_minikube_host.sh b/scripts/evergreen/setup_minikube_host.sh new file mode 100755 index 000000000..eb2e7906c --- /dev/null +++ b/scripts/evergreen/setup_minikube_host.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +# Consolidated setup script for minikube host with multi-architecture support +# This script groups all the setup steps needed for IBM machines and other architectures +# Can be run on static hosts for testing and verification + +source scripts/dev/set_env_context.sh +set -Eeou pipefail + +echo "==========================================" +echo "Setting up minikube host with multi-architecture support" +echo "Architecture: $(uname -m)" +echo "OS: $(uname -s)" +echo "==========================================" + +# Function to run a setup step with error handling and logging +run_setup_step() { + local step_name="$1" + shift + local script_command=("$@") + + echo "" + echo ">>> Running: ${step_name}" + echo ">>> Command: ${script_command[*]}" + + local script_path="${script_command[0]}" + if [[ -f "${script_path}" ]]; then + if "${script_command[@]}"; then + echo "✅ ${step_name} completed successfully" + else + echo "❌ ${step_name} failed" + exit 1 + fi + else + echo "❌ Script not found: ${script_path}" + exit 1 + fi +} + +# Setup Python environment (needed for AWS CLI pip installation) +run_setup_step "Python Virtual Environment" "scripts/dev/recreate_python_venv.sh" "--skip-requirements" + +run_setup_step "AWS CLI Setup" "scripts/evergreen/setup_aws.sh" + +run_setup_step "kubectl and helm Setup" "scripts/evergreen/setup_kubectl.sh" + +run_setup_step "jq Setup" "scripts/evergreen/setup_jq.sh" + +run_setup_step "IBM Host Setup" "scripts/minikube/setup_minikube_host.sh" + +run_setup_step "Docker Authentication" "scripts/dev/configure_docker_auth.sh" + +echo "" +echo "==========================================" +echo "✅ Minikube host setup completed successfully!" +echo "==========================================" +echo "" +echo "Installed tools summary:" +echo "- Python: $(python --version 2>/dev/null || python3 --version 2>/dev/null || echo 'Not found')" +echo "- AWS CLI: $(aws --version 2>/dev/null || echo 'Not found')" +echo "- kubectl: $(kubectl version --client 2>/dev/null || echo 'Not found')" +echo "- helm: $(helm version --short 2>/dev/null || echo 'Not found')" +echo "- jq: $(jq --version 2>/dev/null || echo 'Not found')" +echo "- Docker: $(docker --version 2>/dev/null || echo 'Not found')" +echo "" +echo "Setup complete! Host is ready for minikube operations." diff --git a/scripts/minikube/minikube_host.sh b/scripts/minikube/minikube_host.sh index ab410d7bc..c383ac09c 100755 --- a/scripts/minikube/minikube_host.sh +++ b/scripts/minikube/minikube_host.sh @@ -32,16 +32,6 @@ fi kubeconfig_path="${HOME}/.operator-dev/s390-host.kubeconfig" configure() { - shift 1 - arch=${1-"$(uname -m)"} - - echo "Configuring minikube host ${S390_HOST_NAME} (${host_url}) with architecture ${arch}" - - if [[ "${cmd}" == "configure" && ! "${arch}" =~ ^(s390x|ppc64le|x86_64|aarch64)$ ]]; then - echo "'configure' command supports the following architectures: s390x, ppc64le, x86_64, aarch64" - exit 1 - fi - ssh -T -q "${host_url}" "sudo chown \$(whoami):\$(whoami) ~/.docker || true; mkdir -p ~/.docker" if [[ -f "${HOME}/.docker/config.json" ]]; then echo "Copying local ~/.docker/config.json authorization credentials to s390x host" @@ -50,7 +40,7 @@ configure() { sync - ssh -T -q "${host_url}" "cd ~/mongodb-kubernetes; scripts/dev/switch_context.sh root-context; scripts/minikube/setup_minikube_host.sh ${arch}" + ssh -T -q "${host_url}" "cd ~/mongodb-kubernetes; scripts/dev/switch_context.sh root-context; scripts/minikube/setup_minikube_host.sh " } sync() { From 433cdc17eaa8eb06ac72d150a6950fe29f528c15 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Wed, 30 Jul 2025 14:02:49 +0200 Subject: [PATCH 06/23] fix jq --- scripts/evergreen/setup_jq.sh | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/scripts/evergreen/setup_jq.sh b/scripts/evergreen/setup_jq.sh index 23ad3e723..1f260883a 100755 --- a/scripts/evergreen/setup_jq.sh +++ b/scripts/evergreen/setup_jq.sh @@ -10,6 +10,32 @@ set -Eeou pipefail source scripts/dev/set_env_context.sh source scripts/funcs/install -arch=$(uname -m) +# Detect and map architecture for jq releases +detect_jq_architecture() { + local arch + arch=$(uname -m) + + case "${arch}" in + x86_64) + echo "amd64" + ;; + aarch64|arm64) + echo "arm64" + ;; + ppc64le) + echo "ppc64el" # jq uses ppc64el instead of ppc64le + ;; + s390x) + echo "s390x" + ;; + *) + echo "Error: Unsupported architecture for jq: ${arch}" >&2 + exit 1 + ;; + esac +} -download_and_install_binary "${PROJECT_DIR:-.}/bin" jq "https://github.com/stedolan/jq/releases/download/jq-1.8.1/jq-linux-${arch}" +jq_arch=$(detect_jq_architecture) +echo "Detected architecture: $(uname -m), using jq architecture: ${jq_arch}" + +download_and_install_binary "${PROJECT_DIR:-.}/bin" jq "https://github.com/stedolan/jq/releases/download/jq-1.8.1/jq-linux-${jq_arch}" From 161252561c1f457b636d2dc8827f065fc84f0ba5 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Wed, 30 Jul 2025 15:35:54 +0200 Subject: [PATCH 07/23] fix docker, minikube etc --- .evergreen.yml | 8 +- scripts/evergreen/e2e/e2e.sh | 2 +- scripts/evergreen/setup_minikube_host.sh | 18 ++ scripts/minikube/install-docker.sh | 58 ++----- scripts/minikube/install-minikube.sh | 200 ++++------------------- scripts/minikube/setup_minikube_host.sh | 67 ++++---- 6 files changed, 106 insertions(+), 247 deletions(-) diff --git a/.evergreen.yml b/.evergreen.yml index 31655847c..98eaf8a75 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -1197,7 +1197,13 @@ task_groups: - name: e2e_smoke_ibm_task_group max_hosts: -1 <<: *setup_group_ibm - <<: *setup_and_teardown_task + setup_task_can_fail_task: true + setup_task: + - func: cleanup_exec_environment + teardown_task_can_fail_task: true + teardown_task: + - func: upload_e2e_logs + - func: teardown_kubernetes_environment tasks: - e2e_replica_set <<: *teardown_group diff --git a/scripts/evergreen/e2e/e2e.sh b/scripts/evergreen/e2e/e2e.sh index 12f839b86..eb6140a22 100755 --- a/scripts/evergreen/e2e/e2e.sh +++ b/scripts/evergreen/e2e/e2e.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -Eeou pipefail +set -Eeoux pipefail start_time=$(date +%s) diff --git a/scripts/evergreen/setup_minikube_host.sh b/scripts/evergreen/setup_minikube_host.sh index eb2e7906c..c96cfe9c5 100755 --- a/scripts/evergreen/setup_minikube_host.sh +++ b/scripts/evergreen/setup_minikube_host.sh @@ -50,6 +50,24 @@ run_setup_step "IBM Host Setup" "scripts/minikube/setup_minikube_host.sh" run_setup_step "Docker Authentication" "scripts/dev/configure_docker_auth.sh" +# Setup Kubernetes cluster after Docker is properly configured +echo "" +echo ">>> Setting up Kubernetes cluster" +if [[ "${KUBE_ENVIRONMENT_NAME:-}" == "kind" ]]; then + run_setup_step "Kind Kubernetes Cluster" "scripts/dev/recreate_kind_cluster.sh" "kind" +elif [[ "${KUBE_ENVIRONMENT_NAME:-}" == "minikube" ]]; then + echo ">>> Running: Minikube Kubernetes Cluster" + echo ">>> Command: minikube start --profile=${MINIKUBE_PROFILE:-mongodb-e2e} --driver=docker --memory=8192mb --cpus=4" + if minikube start --profile="${MINIKUBE_PROFILE:-mongodb-e2e}" --driver=docker --memory=8192mb --cpus=4; then + echo "✅ Minikube Kubernetes Cluster completed successfully" + else + echo "❌ Minikube Kubernetes Cluster failed" + exit 1 + fi +else + echo "⚠️ No Kubernetes environment specified (KUBE_ENVIRONMENT_NAME not set)" +fi + echo "" echo "==========================================" echo "✅ Minikube host setup completed successfully!" diff --git a/scripts/minikube/install-docker.sh b/scripts/minikube/install-docker.sh index 04ae3f7d8..164d94c3a 100755 --- a/scripts/minikube/install-docker.sh +++ b/scripts/minikube/install-docker.sh @@ -1,25 +1,10 @@ #!/usr/bin/env bash set -Eeou pipefail -# Script to install Docker on s390x architecture (specifically for RHEL/Ubuntu based systems) - -print_usage() { - echo "Usage: $0 [options]" - echo "Options:" - echo " -h, --help Show this help message" - echo " -u, --user Username to add to docker group (optional)" - echo "" - echo "This script installs Docker on s390x architecture systems." -} - DOCKER_USER="" while [[ $# -gt 0 ]]; do case $1 in - -h|--help) - print_usage - exit 0 - ;; -u|--user) DOCKER_USER="$2" shift 2 @@ -32,7 +17,7 @@ while [[ $# -gt 0 ]]; do esac done -echo "Installing Docker on s390x architecture..." +echo "Installing Docker" # Detect OS if [[ -f /etc/redhat-release ]]; then @@ -47,41 +32,31 @@ fi # Install Docker based on OS if [[ "$OS_TYPE" == "rhel" ]]; then echo "Detected RHEL/CentOS system..." - - # Remove any existing Docker packages - sudo yum remove -y docker docker-client docker-client-latest docker-common docker-latest docker-latest-logrotate docker-logrotate docker-engine || true - - # Install required packages (some may not exist on newer RHEL versions) - sudo yum install -y yum-utils || echo "yum-utils already installed or unavailable" - sudo yum install -y device-mapper-persistent-data lvm2 || echo "device-mapper packages may not be available on this system" - + # Add Docker repository sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo - + # Install Docker CE sudo yum install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin - + elif [[ "$OS_TYPE" == "debian" ]]; then echo "Detected Ubuntu/Debian system..." - - # Remove any existing Docker packages - sudo apt-get remove -y docker docker-engine docker.io containerd runc || true - + # Update package index sudo apt-get update - + # Install required packages sudo apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release - + # Add Docker's official GPG key curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg - + # Set up stable repository echo "deb [arch=s390x signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null - + # Update package index again sudo apt-get update - + # Install Docker CE sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin fi @@ -100,9 +75,12 @@ fi # Verify installation echo "Verifying Docker installation..." sudo docker --version -sudo docker run --rm hello-world -echo "Docker installation completed successfully!" -echo "" -echo "If you added a user to the docker group, they need to log out and log back in." -echo "You can also run 'newgrp docker' to apply the group membership in the current session." \ No newline at end of file +# Test docker access with newgrp (temporary group membership) +echo "Testing Docker access..." +if newgrp docker -c 'docker ps' >/dev/null 2>&1; then + echo "✅ Docker access confirmed" +else + echo "⚠️ Docker group membership requires logout/login to take effect" + echo "Continuing with setup..." +fi diff --git a/scripts/minikube/install-minikube.sh b/scripts/minikube/install-minikube.sh index 3e6dd4e13..903351960 100755 --- a/scripts/minikube/install-minikube.sh +++ b/scripts/minikube/install-minikube.sh @@ -1,73 +1,35 @@ #!/usr/bin/env bash set -Eeou pipefail -# Script to install and configure minikube on s390x architecture - -print_usage() { - echo "Usage: $0 [options]" - echo "Options:" - echo " -h, --help Show this help message" - echo " -v, --version VERSION Minikube version to install (default: latest)" - echo " -k, --kubernetes VER Kubernetes version (default: latest stable)" - echo " -m, --memory MEMORY Memory allocation (default: 8192mb)" - echo " -c, --cpus CPUS CPU allocation (default: 4)" - echo " --profile PROFILE Minikube profile name (default: minikube)" - echo " --start Start minikube after installation" - echo "" - echo "This script installs minikube on s390x architecture and configures it for MongoDB Kubernetes e2e testing." -} - -MINIKUBE_VERSION="latest" -K8S_VERSION="" -MEMORY="8192" -CPUS="4" -PROFILE="minikube" -START_MINIKUBE="false" - -while [[ $# -gt 0 ]]; do - case $1 in - -h|--help) - print_usage - exit 0 - ;; - -v|--version) - MINIKUBE_VERSION="$2" - shift 2 - ;; - -k|--kubernetes) - K8S_VERSION="$2" - shift 2 - ;; - -m|--memory) - MEMORY="$2" - shift 2 - ;; - -c|--cpus) - CPUS="$2" - shift 2 - ;; - --profile) - PROFILE="$2" - shift 2 - ;; - --start) - START_MINIKUBE="true" - shift - ;; - *) - echo "Unknown option: $1" - print_usage - exit 1 - ;; - esac -done - -echo "Installing minikube on s390x architecture..." +source scripts/dev/set_env_context.sh + +# Detect architecture +ARCH=$(uname -m) +case "${ARCH}" in + x86_64) + MINIKUBE_ARCH="amd64" + ;; + aarch64) + MINIKUBE_ARCH="arm64" + ;; + ppc64le) + MINIKUBE_ARCH="ppc64le" + ;; + s390x) + MINIKUBE_ARCH="s390x" + ;; + *) + echo "Error: Unsupported architecture: ${ARCH}" + echo "Supported architectures: x86_64, aarch64, ppc64le, s390x" + exit 1 + ;; +esac + +echo "Installing minikube on ${ARCH} architecture..." # Verify Docker is installed if ! command -v docker &> /dev/null; then echo "Error: Docker is required but not installed. Please install Docker first." - echo "You can use the install-docker.sh script in this directory." exit 1 fi @@ -77,114 +39,12 @@ if ! docker info &> /dev/null; then exit 1 fi -# Install kubectl if not present -if ! command -v kubectl &> /dev/null; then - echo "Installing kubectl..." - - # Get the latest kubectl version if not specified - if [[ -z "$K8S_VERSION" ]]; then - K8S_VERSION=$(curl -L -s https://dl.k8s.io/release/stable.txt) - fi - - # Download kubectl for s390x - curl -LO "https://dl.k8s.io/release/${K8S_VERSION}/bin/linux/s390x/kubectl" - - # Verify the binary - curl -LO "https://dl.k8s.io/${K8S_VERSION}/bin/linux/s390x/kubectl.sha256" - echo "$(cat kubectl.sha256) kubectl" | sha256sum --check - - # Install kubectl - sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl - rm -f kubectl kubectl.sha256 - - echo "kubectl installed successfully" -fi - # Install minikube -echo "Installing minikube..." - -if [[ "$MINIKUBE_VERSION" == "latest" ]]; then - # Get the latest minikube version - MINIKUBE_VERSION=$(curl -s https://api.github.com/repos/kubernetes/minikube/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/') -fi - -# Download minikube for s390x -curl -Lo minikube "https://github.com/kubernetes/minikube/releases/download/${MINIKUBE_VERSION}/minikube-linux-s390x" +echo "Installing minikube for ${ARCH}..." -# Make it executable and install -chmod +x minikube -sudo install minikube /usr/local/bin/ +MINIKUBE_VERSION=$(curl -s https://api.github.com/repos/kubernetes/minikube/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/') -# Clean up -rm -f minikube - -echo "Minikube ${MINIKUBE_VERSION} installed successfully" - -# Configure minikube for MongoDB Kubernetes testing -echo "Configuring minikube for MongoDB Kubernetes e2e testing..." - -# Set default driver to docker -minikube config set driver docker - -# Configure resource limits -minikube config set memory "${MEMORY}mb" -minikube config set cpus "${CPUS}" - -# Enable required addons for testing -ADDONS=( - "storage-provisioner" - "default-storageclass" - "volumesnapshots" - "csi-hostpath-driver" -) - -echo "Minikube configuration completed." - -if [[ "$START_MINIKUBE" == "true" ]]; then - echo "Starting minikube cluster with profile '${PROFILE}'..." - - # Start minikube with specific configuration for MongoDB testing - minikube start \ - --profile="${PROFILE}" \ - --driver=docker \ - --memory="${MEMORY}mb" \ - --cpus="${CPUS}" \ - --disk-size=50g \ - --extra-config=kubelet.authentication-token-webhook=true \ - --extra-config=kubelet.authorization-mode=Webhook \ - --extra-config=scheduler.bind-address=0.0.0.0 \ - --extra-config=controller-manager.bind-address=0.0.0.0 \ - ${K8S_VERSION:+--kubernetes-version=$K8S_VERSION} - - # Wait for cluster to be ready - echo "Waiting for cluster to be ready..." - kubectl wait --for=condition=Ready nodes --all --timeout=300s - - # Enable addons - for addon in "${ADDONS[@]}"; do - echo "Enabling addon: $addon" - minikube addons enable "$addon" --profile="${PROFILE}" || true - done - - # Create directories that MongoDB tests expect (similar to kind setup) - echo "Setting up test directories..." - minikube ssh --profile="${PROFILE}" -- 'sudo mkdir -p /opt/data/mongo-data-{0..2} /opt/data/mongo-logs-{0..2}' - minikube ssh --profile="${PROFILE}" -- 'sudo chmod 777 /opt/data/mongo-data-* /opt/data/mongo-logs-*' - - echo "Minikube cluster started successfully!" - echo "" - echo "To use this cluster:" - echo " export KUBECONFIG=\$(minikube kubeconfig --profile=${PROFILE})" - echo " kubectl get nodes" - echo "" - echo "To stop the cluster:" - echo " minikube stop --profile=${PROFILE}" -else - echo "" - echo "Minikube installed but not started." - echo "To start minikube later, run:" - echo " minikube start --profile=${PROFILE} --driver=docker --memory=${MEMORY}mb --cpus=${CPUS}" -fi +# Download minikube for detected architecture +download_and_install_binary "${PROJECT_DIR:-.}/bin" minikube "https://github.com/kubernetes/minikube/releases/download/${MINIKUBE_VERSION}/minikube-linux-${MINIKUBE_ARCH}" -echo "" -echo "Installation completed successfully!" \ No newline at end of file +echo "Minikube ${MINIKUBE_VERSION} installed successfully for ${ARCH}" diff --git a/scripts/minikube/setup_minikube_host.sh b/scripts/minikube/setup_minikube_host.sh index 7628c388c..a8c4cf70c 100755 --- a/scripts/minikube/setup_minikube_host.sh +++ b/scripts/minikube/setup_minikube_host.sh @@ -65,51 +65,48 @@ download_minikube() { download_docker() { echo "Installing Docker for ${ARCH}..." + scripts/minikube/install-docker.sh --user "$(whoami)" } -# Setup group for minikube based on architecture -setup_group() { - echo "Setting up group configuration for ${ARCH}..." - - # Set SETUP_GROUP evergreen variable based on architecture - case "${ARCH}" in - s390x) - export SETUP_GROUP="s390x_minikube" - ;; - ppc64le) - export SETUP_GROUP="ppc64le_minikube" - ;; - x86_64) - export SETUP_GROUP="x86_64_minikube" - ;; - aarch64) - export SETUP_GROUP="arm64_minikube" - ;; - *) - export SETUP_GROUP="unknown_minikube" - ;; - esac - - echo "SETUP_GROUP set to: ${SETUP_GROUP}" - - # Add to bashrc for persistence - echo "export SETUP_GROUP=${SETUP_GROUP}" >> ~/.bashrc +start_minikube() { + echo "Starting minikube cluster..." + local profile=${MINIKUBE_PROFILE:-mongodb-e2e} + + if minikube start --profile="${profile}" --driver=docker --memory=8192mb --cpus=4; then + echo "✅ Minikube cluster started successfully" + + # Test cluster connectivity + if kubectl --kubeconfig="$(minikube kubeconfig --profile="${profile}")" get nodes >/dev/null 2>&1; then + echo "✅ Cluster connectivity verified" + else + echo "⚠️ Cluster connectivity test failed - may need manual intervention" + fi + else + echo "⚠️ Minikube start failed - likely due to docker permissions" + echo "This will be resolved after logout/login" + fi } check_disk_space set_limits -setup_group - -download_minikube & download_docker & +download_minikube & wait -echo "Setting up minikube environment variables..." -echo 'export KUBE_ENVIRONMENT_NAME=minikube' >> ~/.bashrc -echo 'export MINIKUBE_PROFILE=${MINIKUBE_PROFILE:-mongodb-e2e}' >> ~/.bashrc -echo 'export KUBECONFIG=$(minikube kubeconfig --profile=${MINIKUBE_PROFILE:-mongodb-e2e} 2>/dev/null || echo ~/.kube/config)' >> ~/.bashrc +# Start minikube cluster +start_minikube echo "Minikube host setup completed successfully for ${ARCH}!" -echo "SETUP_GROUP: ${SETUP_GROUP}" + +# Final status +echo "" +echo "==========================================" +echo "✅ Setup Summary" +echo "==========================================" +echo "Architecture: ${ARCH}" +echo "Minikube Profile: ${MINIKUBE_PROFILE:-mongodb-e2e}" +echo "" +echo "If docker permissions failed, logout and login, then run:" +echo " minikube start --profile=\${MINIKUBE_PROFILE:-mongodb-e2e} --driver=docker --memory=8192mb --cpus=4" From bf8e64ae74eacb2a6721418cd9fb63de64cc1a03 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Wed, 30 Jul 2025 15:57:20 +0200 Subject: [PATCH 08/23] remove docker group management --- scripts/evergreen/setup_minikube_host.sh | 18 ++++++--------- scripts/minikube/install-docker.sh | 11 ++++++--- scripts/minikube/setup_minikube_host.sh | 29 ++---------------------- 3 files changed, 17 insertions(+), 41 deletions(-) diff --git a/scripts/evergreen/setup_minikube_host.sh b/scripts/evergreen/setup_minikube_host.sh index c96cfe9c5..36aa423ee 100755 --- a/scripts/evergreen/setup_minikube_host.sh +++ b/scripts/evergreen/setup_minikube_host.sh @@ -53,17 +53,13 @@ run_setup_step "Docker Authentication" "scripts/dev/configure_docker_auth.sh" # Setup Kubernetes cluster after Docker is properly configured echo "" echo ">>> Setting up Kubernetes cluster" -if [[ "${KUBE_ENVIRONMENT_NAME:-}" == "kind" ]]; then - run_setup_step "Kind Kubernetes Cluster" "scripts/dev/recreate_kind_cluster.sh" "kind" -elif [[ "${KUBE_ENVIRONMENT_NAME:-}" == "minikube" ]]; then - echo ">>> Running: Minikube Kubernetes Cluster" - echo ">>> Command: minikube start --profile=${MINIKUBE_PROFILE:-mongodb-e2e} --driver=docker --memory=8192mb --cpus=4" - if minikube start --profile="${MINIKUBE_PROFILE:-mongodb-e2e}" --driver=docker --memory=8192mb --cpus=4; then - echo "✅ Minikube Kubernetes Cluster completed successfully" - else - echo "❌ Minikube Kubernetes Cluster failed" - exit 1 - fi +echo ">>> Command: minikube start --profile=${MINIKUBE_PROFILE:-mongodb-e2e} --driver=docker --memory=8192mb --cpus=4" +if minikube start --profile="${MINIKUBE_PROFILE:-mongodb-e2e}" --driver=docker --memory=8192mb --cpus=4; then + echo "✅ Minikube Kubernetes Cluster completed successfully" +else + echo "❌ Minikube Kubernetes Cluster failed" + exit 1 +fi else echo "⚠️ No Kubernetes environment specified (KUBE_ENVIRONMENT_NAME not set)" fi diff --git a/scripts/minikube/install-docker.sh b/scripts/minikube/install-docker.sh index 164d94c3a..f645a6a97 100755 --- a/scripts/minikube/install-docker.sh +++ b/scripts/minikube/install-docker.sh @@ -76,11 +76,16 @@ fi echo "Verifying Docker installation..." sudo docker --version -# Test docker access with newgrp (temporary group membership) +# Test docker access echo "Testing Docker access..." -if newgrp docker -c 'docker ps' >/dev/null 2>&1; then +if docker ps >/dev/null 2>&1; then echo "✅ Docker access confirmed" else - echo "⚠️ Docker group membership requires logout/login to take effect" + echo "⚠️ Docker access test failed - checking if running as root..." + if [[ $(id -u) -eq 0 ]]; then + echo "Running as root - Docker should work" + else + echo "Docker group membership may require logout/login to take effect" + fi echo "Continuing with setup..." fi diff --git a/scripts/minikube/setup_minikube_host.sh b/scripts/minikube/setup_minikube_host.sh index a8c4cf70c..bd2ac72bf 100755 --- a/scripts/minikube/setup_minikube_host.sh +++ b/scripts/minikube/setup_minikube_host.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # this script downloads necessary tooling for alternative architectures (s390x, ppc64le) using minikube (similar to setup_evg_host.sh) +source scripts/dev/set_env_context.sh set -Eeou pipefail @@ -65,27 +66,7 @@ download_minikube() { download_docker() { echo "Installing Docker for ${ARCH}..." - - scripts/minikube/install-docker.sh --user "$(whoami)" -} - -start_minikube() { - echo "Starting minikube cluster..." - local profile=${MINIKUBE_PROFILE:-mongodb-e2e} - - if minikube start --profile="${profile}" --driver=docker --memory=8192mb --cpus=4; then - echo "✅ Minikube cluster started successfully" - - # Test cluster connectivity - if kubectl --kubeconfig="$(minikube kubeconfig --profile="${profile}")" get nodes >/dev/null 2>&1; then - echo "✅ Cluster connectivity verified" - else - echo "⚠️ Cluster connectivity test failed - may need manual intervention" - fi - else - echo "⚠️ Minikube start failed - likely due to docker permissions" - echo "This will be resolved after logout/login" - fi + scripts/minikube/install-docker.sh } check_disk_space @@ -95,9 +76,6 @@ download_minikube & wait -# Start minikube cluster -start_minikube - echo "Minikube host setup completed successfully for ${ARCH}!" # Final status @@ -107,6 +85,3 @@ echo "✅ Setup Summary" echo "==========================================" echo "Architecture: ${ARCH}" echo "Minikube Profile: ${MINIKUBE_PROFILE:-mongodb-e2e}" -echo "" -echo "If docker permissions failed, logout and login, then run:" -echo " minikube start --profile=\${MINIKUBE_PROFILE:-mongodb-e2e} --driver=docker --memory=8192mb --cpus=4" From 540b4200a160fea89a7837b3fbd19b3b9e6b6379 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Wed, 30 Jul 2025 18:07:25 +0200 Subject: [PATCH 09/23] installation works, minikube starts but docker doesn't start --- scripts/evergreen/setup_minikube_host.sh | 7 ++--- scripts/minikube/install-docker.sh | 40 +++++++++--------------- scripts/minikube/install-minikube.sh | 14 +-------- scripts/minikube/setup_minikube_host.sh | 32 +++++++++++++++++++ 4 files changed, 50 insertions(+), 43 deletions(-) diff --git a/scripts/evergreen/setup_minikube_host.sh b/scripts/evergreen/setup_minikube_host.sh index 36aa423ee..ffc571d3b 100755 --- a/scripts/evergreen/setup_minikube_host.sh +++ b/scripts/evergreen/setup_minikube_host.sh @@ -5,7 +5,7 @@ # Can be run on static hosts for testing and verification source scripts/dev/set_env_context.sh -set -Eeou pipefail +set -Eeoux pipefail echo "==========================================" echo "Setting up minikube host with multi-architecture support" @@ -54,15 +54,14 @@ run_setup_step "Docker Authentication" "scripts/dev/configure_docker_auth.sh" echo "" echo ">>> Setting up Kubernetes cluster" echo ">>> Command: minikube start --profile=${MINIKUBE_PROFILE:-mongodb-e2e} --driver=docker --memory=8192mb --cpus=4" + +# Start minikube cluster for CI if minikube start --profile="${MINIKUBE_PROFILE:-mongodb-e2e}" --driver=docker --memory=8192mb --cpus=4; then echo "✅ Minikube Kubernetes Cluster completed successfully" else echo "❌ Minikube Kubernetes Cluster failed" exit 1 fi -else - echo "⚠️ No Kubernetes environment specified (KUBE_ENVIRONMENT_NAME not set)" -fi echo "" echo "==========================================" diff --git a/scripts/minikube/install-docker.sh b/scripts/minikube/install-docker.sh index f645a6a97..770c75e5a 100755 --- a/scripts/minikube/install-docker.sh +++ b/scripts/minikube/install-docker.sh @@ -1,21 +1,7 @@ #!/usr/bin/env bash set -Eeou pipefail -DOCKER_USER="" - -while [[ $# -gt 0 ]]; do - case $1 in - -u|--user) - DOCKER_USER="$2" - shift 2 - ;; - *) - echo "Unknown option: $1" - print_usage - exit 1 - ;; - esac -done +source scripts/dev/set_env_context.sh echo "Installing Docker" @@ -65,11 +51,15 @@ fi sudo systemctl start docker sudo systemctl enable docker -# Add user to docker group if specified -if [[ -n "$DOCKER_USER" ]]; then - echo "Adding user '$DOCKER_USER' to docker group..." - sudo usermod -aG docker "$DOCKER_USER" - echo "Note: User '$DOCKER_USER' needs to log out and log back in for group membership to take effect." +# Add current CI user to docker group and change socket permissions +current_user=$(whoami) +if [[ "$current_user" != "root" ]]; then + echo "Adding CI user '$current_user' to docker group..." + sudo usermod -aG docker "$current_user" + + # For CI: Change docker socket permissions to allow immediate access + echo "Setting docker socket permissions for CI..." + sudo chmod 666 /var/run/docker.sock fi # Verify installation @@ -81,11 +71,9 @@ echo "Testing Docker access..." if docker ps >/dev/null 2>&1; then echo "✅ Docker access confirmed" else - echo "⚠️ Docker access test failed - checking if running as root..." - if [[ $(id -u) -eq 0 ]]; then - echo "Running as root - Docker should work" - else - echo "Docker group membership may require logout/login to take effect" + echo "❌ Docker access failed - CI may not work properly" + echo "Trying with sudo..." + if sudo docker ps >/dev/null 2>&1; then + echo "⚠️ Docker only works with sudo" fi - echo "Continuing with setup..." fi diff --git a/scripts/minikube/install-minikube.sh b/scripts/minikube/install-minikube.sh index 903351960..b82c829b6 100755 --- a/scripts/minikube/install-minikube.sh +++ b/scripts/minikube/install-minikube.sh @@ -2,6 +2,7 @@ set -Eeou pipefail source scripts/dev/set_env_context.sh +source scripts/funcs/install # Detect architecture ARCH=$(uname -m) @@ -26,19 +27,6 @@ case "${ARCH}" in esac echo "Installing minikube on ${ARCH} architecture..." - -# Verify Docker is installed -if ! command -v docker &> /dev/null; then - echo "Error: Docker is required but not installed. Please install Docker first." - exit 1 -fi - -# Verify Docker is running -if ! docker info &> /dev/null; then - echo "Error: Docker is not running. Please start Docker service." - exit 1 -fi - # Install minikube echo "Installing minikube for ${ARCH}..." diff --git a/scripts/minikube/setup_minikube_host.sh b/scripts/minikube/setup_minikube_host.sh index bd2ac72bf..a455fca57 100755 --- a/scripts/minikube/setup_minikube_host.sh +++ b/scripts/minikube/setup_minikube_host.sh @@ -76,6 +76,36 @@ download_minikube & wait +echo "" +echo ">>> Verifying minikube installation..." +if command -v minikube &> /dev/null; then + minikube_version=$(minikube version --short 2>/dev/null || minikube version 2>/dev/null | head -n1) + echo "✅ Minikube installed successfully: ${minikube_version}" +else + echo "❌ Minikube installation failed - minikube command not found" + echo "Please check the installation logs above for errors" + exit 1 +fi + +echo "" +echo ">>> Verifying docker installation..." +if command -v docker &> /dev/null; then + docker_version=$(docker --version 2>/dev/null) + echo "✅ Docker installed successfully: ${docker_version}" + + # Check if docker service is running + if systemctl is-active --quiet docker 2>/dev/null || docker info &>/dev/null; then + echo "✅ Docker service is running" + else + echo "⚠️ Docker is installed but service may not be running" + echo "You may need to start docker service: sudo systemctl start docker" + fi +else + echo "❌ Docker installation failed - docker command not found" + echo "Please check the installation logs above for errors" + exit 1 +fi + echo "Minikube host setup completed successfully for ${ARCH}!" # Final status @@ -85,3 +115,5 @@ echo "✅ Setup Summary" echo "==========================================" echo "Architecture: ${ARCH}" echo "Minikube Profile: ${MINIKUBE_PROFILE:-mongodb-e2e}" +echo "Minikube: ${minikube_version}" +echo "Docker: ${docker_version}" From fd8db45fc696456e28e5a4f8011cf6be770b8548 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Thu, 31 Jul 2025 16:16:09 +0200 Subject: [PATCH 10/23] podman support --- scripts/dev/configure_docker_auth.sh | 100 ++++++++--- scripts/evergreen/setup_minikube_host.sh | 21 +-- scripts/minikube/install-docker.sh | 79 --------- scripts/minikube/install-minikube.sh | 41 ++++- scripts/minikube/minikube_host.sh | 22 +-- scripts/minikube/setup_minikube_host.sh | 212 ++++++++++++++++++----- 6 files changed, 303 insertions(+), 172 deletions(-) delete mode 100755 scripts/minikube/install-docker.sh diff --git a/scripts/dev/configure_docker_auth.sh b/scripts/dev/configure_docker_auth.sh index dfcb14f0b..74e0d9abc 100755 --- a/scripts/dev/configure_docker_auth.sh +++ b/scripts/dev/configure_docker_auth.sh @@ -8,7 +8,33 @@ source scripts/funcs/checks source scripts/funcs/printing source scripts/funcs/kubernetes +# Detect available container runtime +detect_container_runtime() { + if command -v podman &> /dev/null && podman info &> /dev/null; then + CONTAINER_RUNTIME="podman" + CONFIG_PATH="${HOME}/.config/containers/auth.json" + mkdir -p "$(dirname "${CONFIG_PATH}")" + echo "Using Podman for container authentication" + return 0 + elif command -v docker &> /dev/null; then + CONTAINER_RUNTIME="docker" + CONFIG_PATH="${HOME}/.docker/config.json" + mkdir -p "$(dirname "${CONFIG_PATH}")" + echo "Using Docker for container authentication" + return 0 + else + echo "Error: Neither Docker nor Podman is available" + exit 1 + fi +} + check_docker_daemon_is_running() { + if [[ "${CONTAINER_RUNTIME}" == "podman" ]]; then + # Podman doesn't require a daemon + echo "Using Podman (no daemon required)" + return 0 + fi + if [[ "$(uname -s)" != "Linux" ]]; then echo "Skipping docker daemon check when not running in Linux" return 0 @@ -34,71 +60,95 @@ check_docker_daemon_is_running() { remove_element() { config_option="${1}" tmpfile=$(mktemp) - jq 'del(.'"${config_option}"')' ~/.docker/config.json >"${tmpfile}" - cp "${tmpfile}" ~/.docker/config.json + + # Initialize config file if it doesn't exist + if [[ ! -f "${CONFIG_PATH}" ]]; then + echo '{}' > "${CONFIG_PATH}" + fi + + jq 'del(.'"${config_option}"')' "${CONFIG_PATH}" >"${tmpfile}" + cp "${tmpfile}" "${CONFIG_PATH}" rm "${tmpfile}" } -# This is the script which performs docker authentication to different registries that we use (so far ECR and RedHat) -# As the result of this login the ~/.docker/config.json will have all the 'auth' information necessary to work with docker registries +# Container runtime login wrapper +container_login() { + local username="$1" + local registry="$2" + + if [[ "${CONTAINER_RUNTIME}" == "podman" ]]; then + podman login --username "${username}" --password-stdin "${registry}" + else + docker login --username "${username}" --password-stdin "${registry}" + fi +} + +# This is the script which performs container authentication to different registries that we use (so far ECR and RedHat) +# As the result of this login the config file will have all the 'auth' information necessary to work with container registries + +# Detect container runtime and set appropriate config path +detect_container_runtime check_docker_daemon_is_running -if [[ -f ~/.docker/config.json ]]; then +if [[ -f "${CONFIG_PATH}" ]]; then if [[ "${RUNNING_IN_EVG:-"false"}" != "true" ]]; then - # Check if login is actually required by making a HEAD request to ECR using existing Docker config - echo "Checking if Docker credentials are valid..." - ecr_auth=$(jq -r '.auths."268558157000.dkr.ecr.us-east-1.amazonaws.com".auth // empty' ~/.docker/config.json) + # Check if login is actually required by making a HEAD request to ECR using existing credentials + echo "Checking if container registry credentials are valid..." + ecr_auth=$(jq -r '.auths."268558157000.dkr.ecr.us-east-1.amazonaws.com".auth // empty' "${CONFIG_PATH}") if [[ -n "${ecr_auth}" ]]; then http_status=$(curl --head -s -o /dev/null -w "%{http_code}" --max-time 3 "https://268558157000.dkr.ecr.us-east-1.amazonaws.com/v2/dev/mongodb-kubernetes/manifests/latest" \ -H "Authorization: Basic ${ecr_auth}" 2>/dev/null || echo "error/timeout") if [[ "${http_status}" != "401" && "${http_status}" != "403" && "${http_status}" != "error/timeout" ]]; then - echo "Docker credentials are up to date - not performing the new login!" + echo "Container registry credentials are up to date - not performing the new login!" exit fi - echo "Docker login required (HTTP status: ${http_status})" + echo "Container login required (HTTP status: ${http_status})" else - echo "No ECR credentials found in Docker config - login required" + echo "No ECR credentials found in container config - login required" fi fi - title "Performing docker login to ECR registries" + title "Performing container login to ECR registries" - # There could be some leftovers on Evergreen - if grep -q "credsStore" ~/.docker/config.json; then - remove_element "credsStore" - fi - if grep -q "credHelpers" ~/.docker/config.json; then - remove_element "credHelpers" + # There could be some leftovers on Evergreen (Docker-specific, skip for Podman) + if [[ "${CONTAINER_RUNTIME}" == "docker" ]]; then + if grep -q "credsStore" "${CONFIG_PATH}"; then + remove_element "credsStore" + fi + if grep -q "credHelpers" "${CONFIG_PATH}"; then + remove_element "credHelpers" + fi fi fi echo "$(aws --version)}" -aws ecr get-login-password --region "us-east-1" | docker login --username AWS --password-stdin 268558157000.dkr.ecr.us-east-1.amazonaws.com +aws ecr get-login-password --region "us-east-1" | container_login "AWS" "268558157000.dkr.ecr.us-east-1.amazonaws.com" # by default docker tries to store credentials in an external storage (e.g. OS keychain) - not in the config.json # We need to store it as base64 string in config.json instead so we need to remove the "credsStore" element -if grep -q "credsStore" ~/.docker/config.json; then +# This is Docker-specific behavior, Podman stores credentials directly in auth.json +if [[ "${CONTAINER_RUNTIME}" == "docker" ]] && grep -q "credsStore" "${CONFIG_PATH}"; then remove_element "credsStore" # login again to store the credentials into the config.json - aws ecr get-login-password --region "us-east-1" | docker login --username AWS --password-stdin 268558157000.dkr.ecr.us-east-1.amazonaws.com + aws ecr get-login-password --region "us-east-1" | container_login "AWS" "268558157000.dkr.ecr.us-east-1.amazonaws.com" fi -aws ecr get-login-password --region "eu-west-1" | docker login --username AWS --password-stdin 268558157000.dkr.ecr.eu-west-1.amazonaws.com +aws ecr get-login-password --region "eu-west-1" | container_login "AWS" "268558157000.dkr.ecr.eu-west-1.amazonaws.com" if [[ -n "${COMMUNITY_PRIVATE_PREVIEW_PULLSECRET_DOCKERCONFIGJSON:-}" ]]; then # log in to quay.io for the mongodb/mongodb-search-community private repo # TODO remove once we switch to the official repo in Public Preview quay_io_auth_file=$(mktemp) - docker_configjson_tmp=$(mktemp) + config_tmp=$(mktemp) echo "${COMMUNITY_PRIVATE_PREVIEW_PULLSECRET_DOCKERCONFIGJSON}" | base64 -d > "${quay_io_auth_file}" - jq -s '.[0] * .[1]' "${quay_io_auth_file}" ~/.docker/config.json > "${docker_configjson_tmp}" - mv "${docker_configjson_tmp}" ~/.docker/config.json + jq -s '.[0] * .[1]' "${quay_io_auth_file}" "${CONFIG_PATH}" > "${config_tmp}" + mv "${config_tmp}" "${CONFIG_PATH}" rm "${quay_io_auth_file}" fi diff --git a/scripts/evergreen/setup_minikube_host.sh b/scripts/evergreen/setup_minikube_host.sh index ffc571d3b..9d4884530 100755 --- a/scripts/evergreen/setup_minikube_host.sh +++ b/scripts/evergreen/setup_minikube_host.sh @@ -46,22 +46,14 @@ run_setup_step "kubectl and helm Setup" "scripts/evergreen/setup_kubectl.sh" run_setup_step "jq Setup" "scripts/evergreen/setup_jq.sh" -run_setup_step "IBM Host Setup" "scripts/minikube/setup_minikube_host.sh" +run_setup_step "Minikube Host Setup with Container Runtime Detection" "scripts/minikube/setup_minikube_host.sh" -run_setup_step "Docker Authentication" "scripts/dev/configure_docker_auth.sh" +run_setup_step "Container Registry Authentication" "scripts/dev/configure_docker_auth.sh" -# Setup Kubernetes cluster after Docker is properly configured +# The minikube cluster is already started by the setup_minikube_host.sh script echo "" -echo ">>> Setting up Kubernetes cluster" -echo ">>> Command: minikube start --profile=${MINIKUBE_PROFILE:-mongodb-e2e} --driver=docker --memory=8192mb --cpus=4" - -# Start minikube cluster for CI -if minikube start --profile="${MINIKUBE_PROFILE:-mongodb-e2e}" --driver=docker --memory=8192mb --cpus=4; then - echo "✅ Minikube Kubernetes Cluster completed successfully" -else - echo "❌ Minikube Kubernetes Cluster failed" - exit 1 -fi +echo ">>> Minikube cluster startup completed by setup_minikube_host.sh" +echo "✅ Minikube cluster is ready for use" echo "" echo "==========================================" @@ -74,6 +66,7 @@ echo "- AWS CLI: $(aws --version 2>/dev/null || echo 'Not found')" echo "- kubectl: $(kubectl version --client 2>/dev/null || echo 'Not found')" echo "- helm: $(helm version --short 2>/dev/null || echo 'Not found')" echo "- jq: $(jq --version 2>/dev/null || echo 'Not found')" -echo "- Docker: $(docker --version 2>/dev/null || echo 'Not found')" +echo "- Container Runtime: $(command -v podman &>/dev/null && echo "Podman $(podman --version 2>/dev/null)" || command -v docker &>/dev/null && echo "Docker $(docker --version 2>/dev/null)" || echo "Not found")" +echo "- Minikube: $(./bin/minikube version --short 2>/dev/null || echo 'Not found')" echo "" echo "Setup complete! Host is ready for minikube operations." diff --git a/scripts/minikube/install-docker.sh b/scripts/minikube/install-docker.sh deleted file mode 100755 index 770c75e5a..000000000 --- a/scripts/minikube/install-docker.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env bash -set -Eeou pipefail - -source scripts/dev/set_env_context.sh - -echo "Installing Docker" - -# Detect OS -if [[ -f /etc/redhat-release ]]; then - OS_TYPE="rhel" -elif [[ -f /etc/debian_version ]]; then - OS_TYPE="debian" -else - echo "Unsupported OS. This script supports RHEL/CentOS and Ubuntu/Debian." - exit 1 -fi - -# Install Docker based on OS -if [[ "$OS_TYPE" == "rhel" ]]; then - echo "Detected RHEL/CentOS system..." - - # Add Docker repository - sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo - - # Install Docker CE - sudo yum install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin - -elif [[ "$OS_TYPE" == "debian" ]]; then - echo "Detected Ubuntu/Debian system..." - - # Update package index - sudo apt-get update - - # Install required packages - sudo apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release - - # Add Docker's official GPG key - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg - - # Set up stable repository - echo "deb [arch=s390x signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null - - # Update package index again - sudo apt-get update - - # Install Docker CE - sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -fi - -# Start and enable Docker service -sudo systemctl start docker -sudo systemctl enable docker - -# Add current CI user to docker group and change socket permissions -current_user=$(whoami) -if [[ "$current_user" != "root" ]]; then - echo "Adding CI user '$current_user' to docker group..." - sudo usermod -aG docker "$current_user" - - # For CI: Change docker socket permissions to allow immediate access - echo "Setting docker socket permissions for CI..." - sudo chmod 666 /var/run/docker.sock -fi - -# Verify installation -echo "Verifying Docker installation..." -sudo docker --version - -# Test docker access -echo "Testing Docker access..." -if docker ps >/dev/null 2>&1; then - echo "✅ Docker access confirmed" -else - echo "❌ Docker access failed - CI may not work properly" - echo "Trying with sudo..." - if sudo docker ps >/dev/null 2>&1; then - echo "⚠️ Docker only works with sudo" - fi -fi diff --git a/scripts/minikube/install-minikube.sh b/scripts/minikube/install-minikube.sh index b82c829b6..527b27543 100755 --- a/scripts/minikube/install-minikube.sh +++ b/scripts/minikube/install-minikube.sh @@ -27,12 +27,49 @@ case "${ARCH}" in esac echo "Installing minikube on ${ARCH} architecture..." + +# Install crictl (container runtime CLI) +echo "Installing crictl for ${ARCH}..." +CRICTL_VERSION=$(curl -s https://api.github.com/repos/kubernetes-sigs/cri-tools/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/') + +# Download and extract crictl tar.gz +mkdir -p "${PROJECT_DIR:-.}/bin" +CRICTL_URL="https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-${MINIKUBE_ARCH}.tar.gz" +echo "Downloading ${CRICTL_URL}" +TEMP_DIR=$(mktemp -d) +curl --retry 3 --silent -L "${CRICTL_URL}" -o "${TEMP_DIR}/crictl.tar.gz" +tar -xzf "${TEMP_DIR}/crictl.tar.gz" -C "${TEMP_DIR}/" +chmod +x "${TEMP_DIR}/crictl" +mv "${TEMP_DIR}/crictl" "${PROJECT_DIR:-.}/bin/crictl" +rm -rf "${TEMP_DIR}" +echo "Installed crictl to ${PROJECT_DIR:-.}/bin" + +# Also install crictl system-wide so minikube can find it +echo "Installing crictl system-wide..." +if [[ -f "${PROJECT_DIR:-.}/bin/crictl" ]]; then + # Install to both /usr/local/bin and /usr/bin for better PATH coverage + sudo cp "${PROJECT_DIR:-.}/bin/crictl" /usr/local/bin/crictl + sudo cp "${PROJECT_DIR:-.}/bin/crictl" /usr/bin/crictl + sudo chmod +x /usr/local/bin/crictl + sudo chmod +x /usr/bin/crictl + echo "✅ crictl installed to /usr/local/bin/ and /usr/bin/" + + # Verify installation + if command -v crictl >/dev/null 2>&1; then + echo "✅ crictl is now available in PATH: $(which crictl)" + echo "✅ crictl version: $(crictl --version 2>/dev/null || echo 'version check failed')" + else + echo "⚠️ crictl installed but not found in PATH" + fi +else + echo "⚠️ crictl not found in project bin, minikube may have issues" +fi + # Install minikube echo "Installing minikube for ${ARCH}..." - MINIKUBE_VERSION=$(curl -s https://api.github.com/repos/kubernetes/minikube/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/') # Download minikube for detected architecture download_and_install_binary "${PROJECT_DIR:-.}/bin" minikube "https://github.com/kubernetes/minikube/releases/download/${MINIKUBE_VERSION}/minikube-linux-${MINIKUBE_ARCH}" -echo "Minikube ${MINIKUBE_VERSION} installed successfully for ${ARCH}" +echo "Crictl ${CRICTL_VERSION} and Minikube ${MINIKUBE_VERSION} installed successfully for ${ARCH}" diff --git a/scripts/minikube/minikube_host.sh b/scripts/minikube/minikube_host.sh index c383ac09c..09c5b80f7 100755 --- a/scripts/minikube/minikube_host.sh +++ b/scripts/minikube/minikube_host.sh @@ -79,19 +79,18 @@ remote-prepare-local-e2e-run() { get-kubeconfig() { # For minikube, we need to get the kubeconfig and certificates echo "Getting kubeconfig from minikube on s390x host..." - local profile=${MINIKUBE_PROFILE:-mongodb-e2e} # Create local minikube directory structure - mkdir -p "${HOME}/.minikube/profiles/${profile}" + mkdir -p "${HOME}/.minikube" # Copy certificates from remote host echo "Copying minikube certificates..." scp "${host_url}:~/.minikube/ca.crt" "${HOME}/.minikube/" - scp "${host_url}:~/.minikube/profiles/${profile}/client.crt" "${HOME}/.minikube/profiles/${profile}/" - scp "${host_url}:~/.minikube/profiles/${profile}/client.key" "${HOME}/.minikube/profiles/${profile}/" + scp "${host_url}:~/.minikube/client.crt" "${HOME}/.minikube/" + scp "${host_url}:~/.minikube/client.key" "${HOME}/.minikube/" # Get kubeconfig and update paths to local ones - ssh -T -q "${host_url}" "cd ~/mongodb-kubernetes; export KUBE_ENVIRONMENT_NAME=minikube; export MINIKUBE_PROFILE=${profile}; kubectl config view --raw" > "${kubeconfig_path}" + ssh -T -q "${host_url}" "cd ~/mongodb-kubernetes; export KUBE_ENVIRONMENT_NAME=minikube; kubectl config view --raw" > "${kubeconfig_path}" # Update certificate paths to local paths sed -i '' "s|/home/cloud-user/.minikube|${HOME}/.minikube|g" "${kubeconfig_path}" @@ -103,12 +102,10 @@ get-kubeconfig() { } recreate-minikube-cluster() { - shift 1 - profile_name=${1:-mongodb-e2e} configure "$(uname -m)" 2>&1| prepend "minikube_host.sh configure" - echo "Recreating minikube cluster ${profile_name} on ${S390_HOST_NAME} (${host_url})..." + echo "Recreating minikube cluster on ${S390_HOST_NAME} (${host_url})..." # shellcheck disable=SC2088 - ssh -T "${host_url}" "cd ~/mongodb-kubernetes; export KUBE_ENVIRONMENT_NAME=minikube; export MINIKUBE_PROFILE=${profile_name}; minikube delete --profile=${profile_name} || true; minikube start --profile=${profile_name} --driver=docker --memory=8192mb --cpus=4" + ssh -T "${host_url}" "cd ~/mongodb-kubernetes; export KUBE_ENVIRONMENT_NAME=minikube; minikube delete || true; minikube start --driver=podman --memory=8192mb --cpus=4" echo "Copying kubeconfig to ${kubeconfig_path}" get-kubeconfig } @@ -116,11 +113,10 @@ recreate-minikube-cluster() { tunnel() { shift 1 echo "Setting up tunnel for minikube cluster..." - local profile=${MINIKUBE_PROFILE:-mongodb-e2e} # Get the minikube API server port from remote host local api_port - api_port=$(ssh -T -q "${host_url}" "cd ~/mongodb-kubernetes; export MINIKUBE_PROFILE=${profile}; minikube ip --profile=${profile} 2>/dev/null && echo ':8443' | tr -d '\n'") + api_port=$(ssh -T -q "${host_url}" "cd ~/mongodb-kubernetes; minikube ip 2>/dev/null && echo ':8443' | tr -d '\n'") if [[ -z "${api_port}" ]]; then echo "Could not determine minikube API server details. Is the cluster running?" @@ -134,7 +130,7 @@ tunnel() { # Forward the API server port through minikube set -x # shellcheck disable=SC2029 - ssh -L "${port}:$(ssh -T -q "${host_url}" "export MINIKUBE_PROFILE=${profile}; minikube ip --profile=${profile}"):${port}" "${host_url}" "$@" + ssh -L "${port}:$(ssh -T -q "${host_url}" "minikube ip"):${port}" "${host_url}" "$@" set +x } @@ -185,7 +181,7 @@ PREREQUISITES: COMMANDS: configure installs on a host: calls sync, switches context, installs necessary software (auto-detects arch) sync rsync of project directory - recreate-minikube-cluster recreates minikube cluster with specific profile and executes get-kubeconfig + recreate-minikube-cluster recreates minikube cluster and executes get-kubeconfig remote-prepare-local-e2e-run executes prepare-local-e2e on the remote host get-kubeconfig copies remote minikube kubeconfig locally to ~/.operator-dev/s390-host.kubeconfig tunnel [args] creates ssh session with tunneling to all API servers diff --git a/scripts/minikube/setup_minikube_host.sh b/scripts/minikube/setup_minikube_host.sh index a455fca57..8726e8e3f 100755 --- a/scripts/minikube/setup_minikube_host.sh +++ b/scripts/minikube/setup_minikube_host.sh @@ -5,23 +5,6 @@ source scripts/dev/set_env_context.sh set -Eeou pipefail -check_disk_space() { - echo "Checking available disk space..." - local available_gb - available_gb=$(df / | awk 'NR==2 {print int($4/1024/1024)}') - - if [[ $available_gb -lt 5 ]]; then - echo "ERROR: Insufficient disk space. Available: ${available_gb}GB, Required: 5GB minimum" - echo "Please clean up disk space before continuing:" - echo " sudo dnf clean all" - echo " sudo rm -rf /var/cache/dnf/* /tmp/* /var/tmp/*" - echo " docker system prune -af" - return 1 - fi - - echo "Disk space check passed: ${available_gb}GB available" -} - set_limits() { echo "Increasing fs.inotify.max_user_instances" sudo sysctl -w fs.inotify.max_user_instances=8192 @@ -64,17 +47,125 @@ download_minikube() { scripts/minikube/install-minikube.sh } -download_docker() { - echo "Installing Docker for ${ARCH}..." - scripts/minikube/install-docker.sh +# Fix crictl for ppc64le kicbase images +fix_crictl_ppc64le() { + if [[ "${ARCH}" == "ppc64le" ]]; then + echo ">>> Applying ppc64le crictl fix for podman driver..." + + # Ensure crictl is available on host + if ! command -v crictl &> /dev/null; then + echo "❌ crictl not found on host - this is required for ppc64le fix" + return 1 + fi + + # Wait for minikube container to be created + local container_name="minikube" + local max_wait=60 + local waited=0 + + echo "Waiting for minikube container '${container_name}' to be ready..." + while ! podman ps --format '{{.Names}}' | grep -q "^${container_name}$" && [[ $waited -lt $max_wait ]]; do + sleep 2 + waited=$((waited + 2)) + done + + if [[ $waited -ge $max_wait ]]; then + echo "⚠️ Timeout waiting for minikube container - crictl fix may be needed manually" + return 1 + fi + + # Copy crictl from host to container using podman + echo "Copying crictl to minikube container using podman..." + if podman cp "$(which crictl)" "${container_name}:/usr/bin/crictl"; then + podman exec "${container_name}" chmod +x /usr/bin/crictl + echo "✅ crictl fix applied successfully with podman" + + # Verify the fix + if podman exec "${container_name}" crictl --version &>/dev/null; then + echo "✅ crictl is now working in minikube container" + return 0 + else + echo "⚠️ crictl copy succeeded but binary may not be working" + return 1 + fi + else + echo "❌ Failed to copy crictl to container via podman" + return 1 + fi + fi + return 0 } -check_disk_space -set_limits -download_docker & -download_minikube & +# Start minikube with podman driver +start_minikube_cluster() { + echo ">>> Starting minikube cluster with podman driver..." + + local start_args=("--driver=podman") + + # Add Calico CNI for better compatibility + start_args+=("--cni=calico") + + # For ppc64le, we need to handle the crictl fix + if [[ "${ARCH}" == "ppc64le" ]]; then + echo "Starting minikube (ppc64le requires crictl fix for podman)..." + + # Start minikube in background to let container initialize + timeout 120 "${PROJECT_DIR:-.}/bin/minikube" start "${start_args[@]}" & + local minikube_pid=$! -wait + # Wait a bit for container to be created + sleep 15 + + # Apply crictl fix while minikube is starting + if fix_crictl_ppc64le; then + echo "✅ crictl fix applied, waiting for minikube to complete..." + else + echo "⚠️ crictl fix failed, but continuing..." + fi + + # Wait for minikube to finish + if wait $minikube_pid; then + echo "✅ Minikube started successfully with crictl fix" + else + echo "❌ Minikube failed to start" + return 1 + fi + else + # Standard minikube start + if "${PROJECT_DIR:-.}/bin/minikube" start "${start_args[@]}"; then + echo "✅ Minikube started successfully" + else + echo "❌ Minikube failed to start" + return 1 + fi + fi +} + +setup_podman() { + echo "Setting up podman for ${ARCH}..." + + # Check if podman is already available + if command -v podman &> /dev/null; then + echo "✅ Podman already installed" + else + echo "Installing podman..." + sudo dnf install -y podman + fi + + # Configure podman + echo "Configuring Podman for ${ARCH}..." + + # Start podman service if not running + systemctl --user enable podman.socket 2>/dev/null || true + systemctl --user start podman.socket 2>/dev/null || true + + echo "✅ Podman configured successfully" +} + +# Setup podman and container runtime +setup_podman +set_limits +download_minikube echo "" echo ">>> Verifying minikube installation..." @@ -88,24 +179,61 @@ else fi echo "" -echo ">>> Verifying docker installation..." -if command -v docker &> /dev/null; then - docker_version=$(docker --version 2>/dev/null) - echo "✅ Docker installed successfully: ${docker_version}" - - # Check if docker service is running - if systemctl is-active --quiet docker 2>/dev/null || docker info &>/dev/null; then - echo "✅ Docker service is running" +echo ">>> Verifying podman installation..." +if command -v podman &> /dev/null; then + podman_version=$(podman --version 2>/dev/null) + echo "✅ Podman installed successfully: ${podman_version}" + + # Check podman info + if podman info &>/dev/null; then + echo "✅ Podman is working correctly" else - echo "⚠️ Docker is installed but service may not be running" - echo "You may need to start docker service: sudo systemctl start docker" + echo "⚠️ Podman may need additional configuration" fi else - echo "❌ Docker installation failed - docker command not found" - echo "Please check the installation logs above for errors" + echo "❌ Podman installation failed - podman command not found" exit 1 fi +echo "" +echo ">>> Verifying crictl is available in PATH..." +if command -v crictl &> /dev/null; then + crictl_version=$(crictl --version 2>/dev/null || echo "crictl available") + crictl_path=$(which crictl 2>/dev/null || echo "unknown path") + echo "✅ crictl found in PATH: ${crictl_version}" + echo "✅ crictl location: ${crictl_path}" +else + echo "❌ crictl not found in PATH - this may cause minikube issues" + echo "Checking if crictl exists in project bin..." + if [[ -f "${PROJECT_DIR:-.}/bin/crictl" ]]; then + echo "Found crictl in project bin, installing to system directories..." + sudo cp "${PROJECT_DIR:-.}/bin/crictl" /usr/local/bin/crictl + sudo cp "${PROJECT_DIR:-.}/bin/crictl" /usr/bin/crictl + sudo chmod +x /usr/local/bin/crictl + sudo chmod +x /usr/bin/crictl + echo "✅ crictl installed to /usr/local/bin/ and /usr/bin/" + + # Force PATH refresh and verify + hash -r 2>/dev/null || true + if command -v crictl &> /dev/null; then + echo "✅ crictl now available: $(which crictl)" + else + echo "⚠️ crictl installation may not be working properly" + fi + else + echo "❌ crictl not found in project bin either" + fi +fi + +# Start the minikube cluster +start_minikube_cluster + +# Update kubectl context to point to the running cluster +echo "" +echo ">>> Updating kubectl context for minikube cluster..." +"${PROJECT_DIR:-.}/bin/minikube" update-context +echo "✅ Kubectl context updated successfully" + echo "Minikube host setup completed successfully for ${ARCH}!" # Final status @@ -114,6 +242,12 @@ echo "==========================================" echo "✅ Setup Summary" echo "==========================================" echo "Architecture: ${ARCH}" -echo "Minikube Profile: ${MINIKUBE_PROFILE:-mongodb-e2e}" +echo "Container Runtime: podman" +echo "Minikube Driver: podman" +echo "Minikube: Default cluster" echo "Minikube: ${minikube_version}" -echo "Docker: ${docker_version}" +echo "Podman: ${podman_version}" +echo "CNI: Calico" +if [[ "${ARCH}" == "ppc64le" ]]; then + echo "Special Config: ppc64le crictl fix applied for podman" +fi From 284c7e19beed1ab20b1e2ebd619b0efeef6aa25f Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Thu, 31 Jul 2025 16:21:51 +0200 Subject: [PATCH 11/23] z series support --- .evergreen.yml | 14 +++++++++++++- scripts/minikube/setup_minikube_host.sh | 4 ++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.evergreen.yml b/.evergreen.yml index 98eaf8a75..fdcb4eaae 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -1473,7 +1473,7 @@ buildvariants: tasks: - name: e2e_smoke_task_group - - name: e2e_smoke_ibm + - name: e2e_smoke_ibm_power display_name: e2e_smoke_ibm tags: [ "e2e_test_suite", "e2e_smoke_release_test_suite" ] run_on: @@ -1481,6 +1481,18 @@ buildvariants: allowed_requesters: [ "patch", "github_tag" ] # depends_on: # - name: build_test_image +# variant: init_test_run + tasks: + - name: e2e_smoke_ibm_task_group + + - name: e2e_smoke_ibm_z + display_name: e2e_smoke_ibm + tags: [ "e2e_test_suite", "e2e_smoke_release_test_suite" ] + run_on: + - rhel9-zseries-large + allowed_requesters: [ "patch", "github_tag" ] +# depends_on: +# - name: build_test_image # variant: init_test_run tasks: - name: e2e_smoke_ibm_task_group diff --git a/scripts/minikube/setup_minikube_host.sh b/scripts/minikube/setup_minikube_host.sh index 8726e8e3f..cf38652dc 100755 --- a/scripts/minikube/setup_minikube_host.sh +++ b/scripts/minikube/setup_minikube_host.sh @@ -102,8 +102,8 @@ start_minikube_cluster() { local start_args=("--driver=podman") - # Add Calico CNI for better compatibility - start_args+=("--cni=calico") + # Use default bridge CNI to avoid Docker Hub rate limiting issues + # start_args+=("--cni=bridge") # For ppc64le, we need to handle the crictl fix if [[ "${ARCH}" == "ppc64le" ]]; then From 00439c0e67f64d59e62da3a7a34a8977ef8c7435 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Thu, 31 Jul 2025 18:32:27 +0200 Subject: [PATCH 12/23] custom container with working power critcl --- scripts/minikube/setup_minikube_host.sh | 246 ++++++++++++++---------- 1 file changed, 141 insertions(+), 105 deletions(-) diff --git a/scripts/minikube/setup_minikube_host.sh b/scripts/minikube/setup_minikube_host.sh index cf38652dc..f4af288c0 100755 --- a/scripts/minikube/setup_minikube_host.sh +++ b/scripts/minikube/setup_minikube_host.sh @@ -47,51 +47,97 @@ download_minikube() { scripts/minikube/install-minikube.sh } -# Fix crictl for ppc64le kicbase images -fix_crictl_ppc64le() { +# Setup local registry and build custom kicbase image for ppc64le with crictl +setup_local_registry_and_custom_image() { if [[ "${ARCH}" == "ppc64le" ]]; then - echo ">>> Applying ppc64le crictl fix for podman driver..." - - # Ensure crictl is available on host - if ! command -v crictl &> /dev/null; then - echo "❌ crictl not found on host - this is required for ppc64le fix" - return 1 + echo ">>> Setting up local registry and custom kicbase image for ppc64le..." + + # Check if local registry is running + if ! podman ps --filter "name=registry" --format "{{.Names}}" | grep -q "^registry$"; then + echo "Starting local container registry on port 5000..." + podman run -d -p 5000:5000 --name registry --restart=always docker.io/library/registry:2 || { + echo "Registry might already exist, trying to start it..." + podman start registry || { + echo "Removing existing registry and creating new one..." + podman rm -f registry 2>/dev/null || true + podman run -d -p 5000:5000 --name registry --restart=always docker.io/library/registry:2 + } + } + + # Wait for registry to be ready + echo "Waiting for registry to be ready..." + for i in {1..30}; do + if curl -s http://localhost:5000/v2/_catalog >/dev/null 2>&1; then + break + fi + sleep 1 + done + else + echo "✅ Local registry already running" fi - - # Wait for minikube container to be created - local container_name="minikube" - local max_wait=60 - local waited=0 - - echo "Waiting for minikube container '${container_name}' to be ready..." - while ! podman ps --format '{{.Names}}' | grep -q "^${container_name}$" && [[ $waited -lt $max_wait ]]; do - sleep 2 - waited=$((waited + 2)) - done - - if [[ $waited -ge $max_wait ]]; then - echo "⚠️ Timeout waiting for minikube container - crictl fix may be needed manually" - return 1 + + # Configure per-user podman to trust local registry (no system-wide changes) + echo "Configuring per-user podman registry settings..." + mkdir -p ~/.config/containers + + # Create user-specific registries.conf that includes insecure local registry + cat > ~/.config/containers/registries.conf << 'EOF' +unqualified-search-registries = ["registry.access.redhat.com", "registry.redhat.io", "docker.io"] + +[[registry]] +location = "localhost:5000" +insecure = true + +short-name-mode = "permissive" +EOF + + # Check if custom image already exists in local registry + if curl -s http://localhost:5000/v2/kicbase/tags/list | grep -q "v0.0.47"; then + echo "✅ Custom kicbase image already exists in local registry" + return 0 fi - - # Copy crictl from host to container using podman - echo "Copying crictl to minikube container using podman..." - if podman cp "$(which crictl)" "${container_name}:/usr/bin/crictl"; then - podman exec "${container_name}" chmod +x /usr/bin/crictl - echo "✅ crictl fix applied successfully with podman" - - # Verify the fix - if podman exec "${container_name}" crictl --version &>/dev/null; then - echo "✅ crictl is now working in minikube container" - return 0 - else - echo "⚠️ crictl copy succeeded but binary may not be working" - return 1 - fi - else - echo "❌ Failed to copy crictl to container via podman" - return 1 + + # Build custom kicbase image with crictl + echo "Building custom kicbase image with crictl for ppc64le..." + + # Create build directory if it doesn't exist + mkdir -p "${PROJECT_DIR:-.}/scripts/minikube/kicbase" + + # Create Dockerfile for custom kicbase + cat > "${PROJECT_DIR:-.}/scripts/minikube/kicbase/Dockerfile" << 'EOF' +FROM gcr.io/k8s-minikube/kicbase:v0.0.47 + +# Install crictl for ppc64le if needed +RUN if [ "$(uname -m)" = "ppc64le" ]; then \ + echo "Installing crictl for ppc64le architecture..." && \ + CRICTL_VERSION="v1.28.0" && \ + curl -L "https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-ppc64le.tar.gz" \ + -o /tmp/crictl.tar.gz && \ + tar -C /usr/bin -xzf /tmp/crictl.tar.gz && \ + chmod +x /usr/bin/crictl && \ + rm /tmp/crictl.tar.gz && \ + echo "crictl installed successfully" && \ + crictl --version; \ + else \ + echo "Not ppc64le architecture, skipping crictl installation"; \ fi + +# Verify crictl is available +RUN command -v crictl >/dev/null 2>&1 && echo "crictl is available" || echo "crictl not found" +EOF + + # Build and push to local registry + echo "Building custom kicbase image..." + cd "${PROJECT_DIR:-.}/scripts/minikube/kicbase" + + podman build -t localhost:5000/kicbase:v0.0.47 . + + echo "Pushing custom image to local registry..." + podman push localhost:5000/kicbase:v0.0.47 --tls-verify=false + + cd - > /dev/null + + echo "✅ Custom kicbase image with crictl ready in local registry" fi return 0 } @@ -100,44 +146,36 @@ fix_crictl_ppc64le() { start_minikube_cluster() { echo ">>> Starting minikube cluster with podman driver..." - local start_args=("--driver=podman") + # Clean up any existing minikube state to avoid cached configuration issues + echo "Cleaning up any existing minikube state..." + if [[ -d ~/.minikube/machines/minikube ]]; then + echo "Removing ~/.minikube/machines/minikube directory..." + rm -rf ~/.minikube/machines/minikube + fi + + # Delete any existing minikube cluster to start fresh + echo "Ensuring clean minikube state..." + "${PROJECT_DIR:-.}/bin/minikube" delete 2>/dev/null || true - # Use default bridge CNI to avoid Docker Hub rate limiting issues - # start_args+=("--cni=bridge") + local start_args=("--driver=podman") - # For ppc64le, we need to handle the crictl fix + # Use custom kicbase image for ppc64le with crictl included if [[ "${ARCH}" == "ppc64le" ]]; then - echo "Starting minikube (ppc64le requires crictl fix for podman)..." - - # Start minikube in background to let container initialize - timeout 120 "${PROJECT_DIR:-.}/bin/minikube" start "${start_args[@]}" & - local minikube_pid=$! - - # Wait a bit for container to be created - sleep 15 + echo "Using custom kicbase image for ppc64le with crictl..." + start_args+=("--base-image=localhost:5000/kicbase:v0.0.47") + fi - # Apply crictl fix while minikube is starting - if fix_crictl_ppc64le; then - echo "✅ crictl fix applied, waiting for minikube to complete..." - else - echo "⚠️ crictl fix failed, but continuing..." - fi + # Use default bridge CNI to avoid Docker Hub rate limiting issues + # start_args+=("--cni=bridge") - # Wait for minikube to finish - if wait $minikube_pid; then - echo "✅ Minikube started successfully with crictl fix" - else - echo "❌ Minikube failed to start" - return 1 - fi + echo "Starting minikube with args: ${start_args[*]}" + if "${PROJECT_DIR:-.}/bin/minikube" start "${start_args[@]}"; then + echo "✅ Minikube started successfully" else - # Standard minikube start - if "${PROJECT_DIR:-.}/bin/minikube" start "${start_args[@]}"; then - echo "✅ Minikube started successfully" - else - echo "❌ Minikube failed to start" - return 1 - fi + echo "❌ Minikube failed to start" + echo "Minikube logs:" + "${PROJECT_DIR:-.}/bin/minikube" logs | tail -20 + return 1 fi } @@ -152,14 +190,31 @@ setup_podman() { sudo dnf install -y podman fi - # Configure podman - echo "Configuring Podman for ${ARCH}..." + # Configure podman for CI environment + echo "Configuring Podman for ${ARCH} CI environment..." + + # Enable lingering for the current user to fix systemd session issues + current_user=$(whoami) + current_uid=$(id -u) + echo "Enabling systemd lingering for user ${current_user} (UID: ${current_uid})" + sudo loginctl enable-linger "${current_uid}" 2>/dev/null || true + + # Configure podman to use cgroupfs instead of systemd in CI + mkdir -p ~/.config/containers + cat > ~/.config/containers/containers.conf << EOF +[containers] +cgroup_manager = "cgroupfs" +events_logger = "file" + +[engine] +cgroup_manager = "cgroupfs" +EOF # Start podman service if not running systemctl --user enable podman.socket 2>/dev/null || true systemctl --user start podman.socket 2>/dev/null || true - echo "✅ Podman configured successfully" + echo "✅ Podman configured successfully for CI" } # Setup podman and container runtime @@ -167,6 +222,9 @@ setup_podman set_limits download_minikube +# Setup local registry and custom kicbase image for ppc64le if needed +setup_local_registry_and_custom_image + echo "" echo ">>> Verifying minikube installation..." if command -v minikube &> /dev/null; then @@ -195,34 +253,12 @@ else exit 1 fi -echo "" -echo ">>> Verifying crictl is available in PATH..." -if command -v crictl &> /dev/null; then - crictl_version=$(crictl --version 2>/dev/null || echo "crictl available") - crictl_path=$(which crictl 2>/dev/null || echo "unknown path") - echo "✅ crictl found in PATH: ${crictl_version}" - echo "✅ crictl location: ${crictl_path}" +if [[ "${ARCH}" == "ppc64le" ]]; then + echo "" + echo ">>> Note: crictl will be patched into the minikube container after startup" else - echo "❌ crictl not found in PATH - this may cause minikube issues" - echo "Checking if crictl exists in project bin..." - if [[ -f "${PROJECT_DIR:-.}/bin/crictl" ]]; then - echo "Found crictl in project bin, installing to system directories..." - sudo cp "${PROJECT_DIR:-.}/bin/crictl" /usr/local/bin/crictl - sudo cp "${PROJECT_DIR:-.}/bin/crictl" /usr/bin/crictl - sudo chmod +x /usr/local/bin/crictl - sudo chmod +x /usr/bin/crictl - echo "✅ crictl installed to /usr/local/bin/ and /usr/bin/" - - # Force PATH refresh and verify - hash -r 2>/dev/null || true - if command -v crictl &> /dev/null; then - echo "✅ crictl now available: $(which crictl)" - else - echo "⚠️ crictl installation may not be working properly" - fi - else - echo "❌ crictl not found in project bin either" - fi + echo "" + echo ">>> Using standard kicbase image (crictl included for x86_64/aarch64/s390x)" fi # Start the minikube cluster @@ -247,7 +283,7 @@ echo "Minikube Driver: podman" echo "Minikube: Default cluster" echo "Minikube: ${minikube_version}" echo "Podman: ${podman_version}" -echo "CNI: Calico" +echo "CNI: bridge (default)" if [[ "${ARCH}" == "ppc64le" ]]; then - echo "Special Config: ppc64le crictl fix applied for podman" + echo "Special Config: Custom kicbase image with crictl via local registry" fi From c675c83bda19fa5431c407ce06a3750e487f6c8e Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Thu, 31 Jul 2025 18:33:59 +0200 Subject: [PATCH 13/23] custom container with working power critcl --- .evergreen.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.evergreen.yml b/.evergreen.yml index fdcb4eaae..fd3dc78b0 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -1474,7 +1474,7 @@ buildvariants: - name: e2e_smoke_task_group - name: e2e_smoke_ibm_power - display_name: e2e_smoke_ibm + display_name: e2e_smoke_ibm_power tags: [ "e2e_test_suite", "e2e_smoke_release_test_suite" ] run_on: - rhel9-power-large @@ -1486,7 +1486,7 @@ buildvariants: - name: e2e_smoke_ibm_task_group - name: e2e_smoke_ibm_z - display_name: e2e_smoke_ibm + display_name: e2e_smoke_ibm_z tags: [ "e2e_test_suite", "e2e_smoke_release_test_suite" ] run_on: - rhel9-zseries-large From 37c31de408372163ffb6fefdf8e53a9ab7d05014 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Thu, 31 Jul 2025 21:40:47 +0200 Subject: [PATCH 14/23] add insecure --- scripts/minikube/setup_minikube_host.sh | 44 ++++++++++++++----------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/scripts/minikube/setup_minikube_host.sh b/scripts/minikube/setup_minikube_host.sh index f4af288c0..88afd424f 100755 --- a/scripts/minikube/setup_minikube_host.sh +++ b/scripts/minikube/setup_minikube_host.sh @@ -51,7 +51,7 @@ download_minikube() { setup_local_registry_and_custom_image() { if [[ "${ARCH}" == "ppc64le" ]]; then echo ">>> Setting up local registry and custom kicbase image for ppc64le..." - + # Check if local registry is running if ! podman ps --filter "name=registry" --format "{{.Names}}" | grep -q "^registry$"; then echo "Starting local container registry on port 5000..." @@ -63,7 +63,7 @@ setup_local_registry_and_custom_image() { podman run -d -p 5000:5000 --name registry --restart=always docker.io/library/registry:2 } } - + # Wait for registry to be ready echo "Waiting for registry to be ready..." for i in {1..30}; do @@ -75,13 +75,18 @@ setup_local_registry_and_custom_image() { else echo "✅ Local registry already running" fi - - # Configure per-user podman to trust local registry (no system-wide changes) - echo "Configuring per-user podman registry settings..." - mkdir -p ~/.config/containers - - # Create user-specific registries.conf that includes insecure local registry - cat > ~/.config/containers/registries.conf << 'EOF' + + # Configure system-wide podman to trust local registry (with backup) + echo "Configuring system registries.conf to trust local registry..." + + # Backup existing registries.conf if it exists + if [[ -f /etc/containers/registries.conf ]]; then + echo "Backing up existing registries.conf..." + sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.minikube-backup + fi + + # Create a clean registries.conf that includes insecure local registry + sudo tee /etc/containers/registries.conf << 'EOF' unqualified-search-registries = ["registry.access.redhat.com", "registry.redhat.io", "docker.io"] [[registry]] @@ -90,19 +95,19 @@ insecure = true short-name-mode = "permissive" EOF - + # Check if custom image already exists in local registry if curl -s http://localhost:5000/v2/kicbase/tags/list | grep -q "v0.0.47"; then echo "✅ Custom kicbase image already exists in local registry" return 0 fi - + # Build custom kicbase image with crictl echo "Building custom kicbase image with crictl for ppc64le..." - + # Create build directory if it doesn't exist mkdir -p "${PROJECT_DIR:-.}/scripts/minikube/kicbase" - + # Create Dockerfile for custom kicbase cat > "${PROJECT_DIR:-.}/scripts/minikube/kicbase/Dockerfile" << 'EOF' FROM gcr.io/k8s-minikube/kicbase:v0.0.47 @@ -125,18 +130,18 @@ RUN if [ "$(uname -m)" = "ppc64le" ]; then \ # Verify crictl is available RUN command -v crictl >/dev/null 2>&1 && echo "crictl is available" || echo "crictl not found" EOF - + # Build and push to local registry echo "Building custom kicbase image..." cd "${PROJECT_DIR:-.}/scripts/minikube/kicbase" - + podman build -t localhost:5000/kicbase:v0.0.47 . - + echo "Pushing custom image to local registry..." podman push localhost:5000/kicbase:v0.0.47 --tls-verify=false - + cd - > /dev/null - + echo "✅ Custom kicbase image with crictl ready in local registry" fi return 0 @@ -152,7 +157,7 @@ start_minikube_cluster() { echo "Removing ~/.minikube/machines/minikube directory..." rm -rf ~/.minikube/machines/minikube fi - + # Delete any existing minikube cluster to start fresh echo "Ensuring clean minikube state..." "${PROJECT_DIR:-.}/bin/minikube" delete 2>/dev/null || true @@ -163,6 +168,7 @@ start_minikube_cluster() { if [[ "${ARCH}" == "ppc64le" ]]; then echo "Using custom kicbase image for ppc64le with crictl..." start_args+=("--base-image=localhost:5000/kicbase:v0.0.47") + start_args+=("--insecure-registry=localhost:5000") fi # Use default bridge CNI to avoid Docker Hub rate limiting issues From 148779d4c1053b09b69ef39bfe83c7b9d1379dc7 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Thu, 31 Jul 2025 23:43:02 +0200 Subject: [PATCH 15/23] add empty config handling and broken podman making --- .evergreen.yml | 4 ++-- scripts/dev/configure_docker_auth.sh | 5 +++++ scripts/minikube/setup_minikube_host.sh | 7 +++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.evergreen.yml b/.evergreen.yml index fd3dc78b0..2275ab0e4 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -1477,7 +1477,7 @@ buildvariants: display_name: e2e_smoke_ibm_power tags: [ "e2e_test_suite", "e2e_smoke_release_test_suite" ] run_on: - - rhel9-power-large + - rhel9-power-small allowed_requesters: [ "patch", "github_tag" ] # depends_on: # - name: build_test_image @@ -1489,7 +1489,7 @@ buildvariants: display_name: e2e_smoke_ibm_z tags: [ "e2e_test_suite", "e2e_smoke_release_test_suite" ] run_on: - - rhel9-zseries-large + - rhel9-zseries-small allowed_requesters: [ "patch", "github_tag" ] # depends_on: # - name: build_test_image diff --git a/scripts/dev/configure_docker_auth.sh b/scripts/dev/configure_docker_auth.sh index 74e0d9abc..2e90328ed 100755 --- a/scripts/dev/configure_docker_auth.sh +++ b/scripts/dev/configure_docker_auth.sh @@ -91,6 +91,11 @@ detect_container_runtime check_docker_daemon_is_running +# Initialize config file if it doesn't exist +if [[ ! -f "${CONFIG_PATH}" ]]; then + echo '{}' > "${CONFIG_PATH}" +fi + if [[ -f "${CONFIG_PATH}" ]]; then if [[ "${RUNNING_IN_EVG:-"false"}" != "true" ]]; then # Check if login is actually required by making a HEAD request to ECR using existing credentials diff --git a/scripts/minikube/setup_minikube_host.sh b/scripts/minikube/setup_minikube_host.sh index 88afd424f..cf39e8f4d 100755 --- a/scripts/minikube/setup_minikube_host.sh +++ b/scripts/minikube/setup_minikube_host.sh @@ -191,6 +191,13 @@ setup_podman() { # Check if podman is already available if command -v podman &> /dev/null; then echo "✅ Podman already installed" + + # Reset podman if it's in an invalid state + if podman info 2>&1 | grep -q "invalid internal status"; then + echo "Resetting podman due to invalid internal status..." + podman system migrate || true + podman system reset --force || true + fi else echo "Installing podman..." sudo dnf install -y podman From 10811d7440721f34ceeb90a231630ec4ea466a12 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Thu, 31 Jul 2025 23:48:57 +0200 Subject: [PATCH 16/23] handle edge cases --- .../{e2e_smoke_ibm => e2e_smoke_ibm_power} | 0 scripts/dev/contexts/e2e_smoke_ibm_z | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+) rename scripts/dev/contexts/{e2e_smoke_ibm => e2e_smoke_ibm_power} (100%) create mode 100644 scripts/dev/contexts/e2e_smoke_ibm_z diff --git a/scripts/dev/contexts/e2e_smoke_ibm b/scripts/dev/contexts/e2e_smoke_ibm_power similarity index 100% rename from scripts/dev/contexts/e2e_smoke_ibm rename to scripts/dev/contexts/e2e_smoke_ibm_power diff --git a/scripts/dev/contexts/e2e_smoke_ibm_z b/scripts/dev/contexts/e2e_smoke_ibm_z new file mode 100644 index 000000000..03384c26c --- /dev/null +++ b/scripts/dev/contexts/e2e_smoke_ibm_z @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" + +export ops_manager_version="cloud_qa" + +# This is required to be able to rebuild the om image and use that image which has been rebuild +export OPS_MANAGER_REGISTRY=268558157000.dkr.ecr.us-east-1.amazonaws.com/dev +CUSTOM_OM_VERSION=$(grep -E "^\s*-\s*&ops_manager_70_latest\s+(\S+)\s+#" <"${script_dir}"/../../../.evergreen.yml | awk '{print $3}') +export CUSTOM_OM_VERSION + +export CUSTOM_MDB_VERSION=6.0.5 +export CUSTOM_MDB_PREV_VERSION=5.0.7 From a4cc24fdc7eb8f7f2e351745c52f2aaa1c4b463b Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Fri, 1 Aug 2025 17:00:36 +0200 Subject: [PATCH 17/23] use python wheel locacl build, unify teardown for minikube, use run for cloudqa --- .evergreen-functions.yml | 11 +- .evergreen.yml | 12 +- scripts/dev/configure_docker_auth.sh | 74 +++++--- scripts/dev/contexts/e2e_smoke_ibm_power | 1 + scripts/dev/contexts/e2e_smoke_ibm_z | 1 + scripts/dev/recreate_python_venv.sh | 90 ++++------ .../evergreen/setup_kubernetes_environment.sh | 2 + scripts/evergreen/setup_minikube_host.sh | 4 +- .../teardown_kubernetes_environment.sh | 8 +- scripts/minikube/setup_minikube_host.sh | 161 +++++++----------- 10 files changed, 159 insertions(+), 205 deletions(-) diff --git a/.evergreen-functions.yml b/.evergreen-functions.yml index 71837ab97..c67634152 100644 --- a/.evergreen-functions.yml +++ b/.evergreen-functions.yml @@ -236,15 +236,6 @@ functions: - ${workdir}/bin binary: scripts/dev/setup_evg_host.sh - setup_ibm_host: &setup_ibm_host - command: subprocess.exec - type: setup - params: - working_dir: src/github.com/mongodb/mongodb-kubernetes - add_to_path: - - ${workdir}/bin - binary: scripts/minikube/setup_minikube_host.sh - lint_repo: - command: subprocess.exec type: setup @@ -349,7 +340,7 @@ functions: working_dir: src/github.com/mongodb/mongodb-kubernetes script: | source .generated/context.export.env - scripts/evergreen/e2e/setup_cloud_qa.py create + scripts/evergreen/run_python.sh scripts/evergreen/e2e/setup_cloud_qa.py create # The additional switch is needed, since we now have created the needed OM exports. - *switch_context diff --git a/.evergreen.yml b/.evergreen.yml index 2275ab0e4..136212703 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -140,7 +140,7 @@ variables: - &teardown_group teardown_group: - - func: prune_docker_resources + - func: teardown_kubernetes_environment - func: run_retry_script - &base_om7_dependency @@ -1197,13 +1197,7 @@ task_groups: - name: e2e_smoke_ibm_task_group max_hosts: -1 <<: *setup_group_ibm - setup_task_can_fail_task: true - setup_task: - - func: cleanup_exec_environment - teardown_task_can_fail_task: true - teardown_task: - - func: upload_e2e_logs - - func: teardown_kubernetes_environment + <<: *setup_and_teardown_task_cloudqa tasks: - e2e_replica_set <<: *teardown_group @@ -1477,7 +1471,7 @@ buildvariants: display_name: e2e_smoke_ibm_power tags: [ "e2e_test_suite", "e2e_smoke_release_test_suite" ] run_on: - - rhel9-power-small + - rhel9-power-large allowed_requesters: [ "patch", "github_tag" ] # depends_on: # - name: build_test_image diff --git a/scripts/dev/configure_docker_auth.sh b/scripts/dev/configure_docker_auth.sh index 2e90328ed..4edf68817 100755 --- a/scripts/dev/configure_docker_auth.sh +++ b/scripts/dev/configure_docker_auth.sh @@ -10,11 +10,12 @@ source scripts/funcs/kubernetes # Detect available container runtime detect_container_runtime() { - if command -v podman &> /dev/null && podman info &> /dev/null; then + if command -v podman &> /dev/null && (podman info &> /dev/null || sudo podman info &> /dev/null); then CONTAINER_RUNTIME="podman" - CONFIG_PATH="${HOME}/.config/containers/auth.json" - mkdir -p "$(dirname "${CONFIG_PATH}")" - echo "Using Podman for container authentication" + # Use root's auth.json since minikube uses sudo podman + CONFIG_PATH="/root/.config/containers/auth.json" + sudo mkdir -p "$(dirname "${CONFIG_PATH}")" + echo "Using Podman for container authentication (sudo mode)" return 0 elif command -v docker &> /dev/null; then CONTAINER_RUNTIME="docker" @@ -60,14 +61,23 @@ check_docker_daemon_is_running() { remove_element() { config_option="${1}" tmpfile=$(mktemp) - + # Initialize config file if it doesn't exist if [[ ! -f "${CONFIG_PATH}" ]]; then - echo '{}' > "${CONFIG_PATH}" + if [[ "${CONFIG_PATH}" == "/root/.config/containers/auth.json" ]]; then + echo '{}' | sudo tee "${CONFIG_PATH}" > /dev/null + else + echo '{}' > "${CONFIG_PATH}" + fi + fi + + if [[ "${CONFIG_PATH}" == "/root/.config/containers/auth.json" ]]; then + sudo "${PROJECT_DIR:-.}/bin/jq" 'del(.'"${config_option}"')' "${CONFIG_PATH}" >"${tmpfile}" + sudo cp "${tmpfile}" "${CONFIG_PATH}" + else + "${PROJECT_DIR:-.}/bin/jq" 'del(.'"${config_option}"')' "${CONFIG_PATH}" >"${tmpfile}" + cp "${tmpfile}" "${CONFIG_PATH}" fi - - jq 'del(.'"${config_option}"')' "${CONFIG_PATH}" >"${tmpfile}" - cp "${tmpfile}" "${CONFIG_PATH}" rm "${tmpfile}" } @@ -75,9 +85,9 @@ remove_element() { container_login() { local username="$1" local registry="$2" - + if [[ "${CONTAINER_RUNTIME}" == "podman" ]]; then - podman login --username "${username}" --password-stdin "${registry}" + sudo podman login --username "${username}" --password-stdin "${registry}" else docker login --username "${username}" --password-stdin "${registry}" fi @@ -93,14 +103,22 @@ check_docker_daemon_is_running # Initialize config file if it doesn't exist if [[ ! -f "${CONFIG_PATH}" ]]; then - echo '{}' > "${CONFIG_PATH}" + if [[ "${CONFIG_PATH}" == "/root/.config/containers/auth.json" ]]; then + echo '{}' | sudo tee "${CONFIG_PATH}" > /dev/null + else + echo '{}' > "${CONFIG_PATH}" + fi fi if [[ -f "${CONFIG_PATH}" ]]; then if [[ "${RUNNING_IN_EVG:-"false"}" != "true" ]]; then # Check if login is actually required by making a HEAD request to ECR using existing credentials echo "Checking if container registry credentials are valid..." - ecr_auth=$(jq -r '.auths."268558157000.dkr.ecr.us-east-1.amazonaws.com".auth // empty' "${CONFIG_PATH}") + if [[ "${CONFIG_PATH}" == "/root/.config/containers/auth.json" ]]; then + ecr_auth=$(sudo "${PROJECT_DIR:-.}/bin/jq" -r '.auths."268558157000.dkr.ecr.us-east-1.amazonaws.com".auth // empty' "${CONFIG_PATH}") + else + ecr_auth=$("${PROJECT_DIR:-.}/bin/jq" -r '.auths."268558157000.dkr.ecr.us-east-1.amazonaws.com".auth // empty' "${CONFIG_PATH}") + fi if [[ -n "${ecr_auth}" ]]; then http_status=$(curl --head -s -o /dev/null -w "%{http_code}" --max-time 3 "https://268558157000.dkr.ecr.us-east-1.amazonaws.com/v2/dev/mongodb-kubernetes/manifests/latest" \ @@ -120,11 +138,20 @@ if [[ -f "${CONFIG_PATH}" ]]; then # There could be some leftovers on Evergreen (Docker-specific, skip for Podman) if [[ "${CONTAINER_RUNTIME}" == "docker" ]]; then - if grep -q "credsStore" "${CONFIG_PATH}"; then - remove_element "credsStore" - fi - if grep -q "credHelpers" "${CONFIG_PATH}"; then - remove_element "credHelpers" + if [[ "${CONFIG_PATH}" == "/root/.config/containers/auth.json" ]]; then + if sudo grep -q "credsStore" "${CONFIG_PATH}"; then + remove_element "credsStore" + fi + if sudo grep -q "credHelpers" "${CONFIG_PATH}"; then + remove_element "credHelpers" + fi + else + if grep -q "credsStore" "${CONFIG_PATH}"; then + remove_element "credsStore" + fi + if grep -q "credHelpers" "${CONFIG_PATH}"; then + remove_element "credHelpers" + fi fi fi fi @@ -137,7 +164,7 @@ aws ecr get-login-password --region "us-east-1" | container_login "AWS" "2685581 # by default docker tries to store credentials in an external storage (e.g. OS keychain) - not in the config.json # We need to store it as base64 string in config.json instead so we need to remove the "credsStore" element # This is Docker-specific behavior, Podman stores credentials directly in auth.json -if [[ "${CONTAINER_RUNTIME}" == "docker" ]] && grep -q "credsStore" "${CONFIG_PATH}"; then +if [[ "${CONTAINER_RUNTIME}" == "docker" ]] && (([[ "${CONFIG_PATH}" == "/root/.config/containers/auth.json" ]] && sudo grep -q "credsStore" "${CONFIG_PATH}") || ([[ "${CONFIG_PATH}" != "/root/.config/containers/auth.json" ]] && grep -q "credsStore" "${CONFIG_PATH}")); then remove_element "credsStore" # login again to store the credentials into the config.json @@ -152,8 +179,13 @@ if [[ -n "${COMMUNITY_PRIVATE_PREVIEW_PULLSECRET_DOCKERCONFIGJSON:-}" ]]; then quay_io_auth_file=$(mktemp) config_tmp=$(mktemp) echo "${COMMUNITY_PRIVATE_PREVIEW_PULLSECRET_DOCKERCONFIGJSON}" | base64 -d > "${quay_io_auth_file}" - jq -s '.[0] * .[1]' "${quay_io_auth_file}" "${CONFIG_PATH}" > "${config_tmp}" - mv "${config_tmp}" "${CONFIG_PATH}" + if [[ "${CONFIG_PATH}" == "/root/.config/containers/auth.json" ]]; then + sudo jq -s '.[0] * .[1]' "${quay_io_auth_file}" "${CONFIG_PATH}" > "${config_tmp}" + sudo mv "${config_tmp}" "${CONFIG_PATH}" + else + jq -s '.[0] * .[1]' "${quay_io_auth_file}" "${CONFIG_PATH}" > "${config_tmp}" + mv "${config_tmp}" "${CONFIG_PATH}" + fi rm "${quay_io_auth_file}" fi diff --git a/scripts/dev/contexts/e2e_smoke_ibm_power b/scripts/dev/contexts/e2e_smoke_ibm_power index 03384c26c..4ba998050 100644 --- a/scripts/dev/contexts/e2e_smoke_ibm_power +++ b/scripts/dev/contexts/e2e_smoke_ibm_power @@ -16,3 +16,4 @@ export CUSTOM_OM_VERSION export CUSTOM_MDB_VERSION=6.0.5 export CUSTOM_MDB_PREV_VERSION=5.0.7 +export KUBE_ENVIRONMENT_NAME=minikube diff --git a/scripts/dev/contexts/e2e_smoke_ibm_z b/scripts/dev/contexts/e2e_smoke_ibm_z index 03384c26c..4ba998050 100644 --- a/scripts/dev/contexts/e2e_smoke_ibm_z +++ b/scripts/dev/contexts/e2e_smoke_ibm_z @@ -16,3 +16,4 @@ export CUSTOM_OM_VERSION export CUSTOM_MDB_VERSION=6.0.5 export CUSTOM_MDB_PREV_VERSION=5.0.7 +export KUBE_ENVIRONMENT_NAME=minikube diff --git a/scripts/dev/recreate_python_venv.sh b/scripts/dev/recreate_python_venv.sh index a5388e40f..bd8b411db 100755 --- a/scripts/dev/recreate_python_venv.sh +++ b/scripts/dev/recreate_python_venv.sh @@ -4,76 +4,54 @@ set -Eeou pipefail -# Parse command line arguments -INSTALL_REQUIREMENTS=true +ensure_required_python() { + local required_version="${PYTHON_VERSION:-3.10}" + local major_minor + major_minor=$(echo "${required_version}" | grep -oE '^[0-9]+\.[0-9]+') -while [[ $# -gt 0 ]]; do - case $1 in - --skip-requirements) - INSTALL_REQUIREMENTS=false - shift - ;; - -h|--help) - echo "Usage: $0 [--skip-requirements]" - echo " --skip-requirements Skip installing requirements.txt" - echo " -h, --help Show this help message" - exit 0 - ;; - *) - echo "Unknown option: $1" - echo "Use -h or --help for usage information" - exit 1 - ;; - esac -done + echo "Checking for Python ${required_version} (${major_minor}.x)..." >&2 -source scripts/dev/set_env_context.sh - -# Ensure Python 3.10 is available, install if needed -ensure_python310() { - echo "Checking current Python version..." >&2 - - # Check if current python is 3.10 + # Check if current python matches required version if command -v python3 &> /dev/null; then local version - if version=$(python3 --version 2>&1) && [[ "${version}" == *"Python 3.10"* ]]; then - echo "Found Python 3.10: ${version}" >&2 + if version=$(python3 --version 2>&1) && [[ "${version}" == *"Python ${major_minor}"* ]]; then + echo "Found Python ${major_minor}: ${version}" >&2 echo "python3" return 0 else echo "Current python3 version: ${version}" >&2 fi fi - - # Try to install Python 3.10 using pyenv if available + + # Try to install required Python version using pyenv if available if command -v pyenv &> /dev/null; then - echo "Python 3.10 not found. Attempting to install via pyenv..." >&2 - - # Check if any 3.10 version is already installed - if pyenv versions --bare | grep -q "^3\.10\."; then + echo "Python ${major_minor} not found. Attempting to install via pyenv..." >&2 + + # Check if any version in the required series is already installed + if pyenv versions --bare | grep -q "^${major_minor}\."; then local installed_version - installed_version=$(pyenv versions --bare | grep "^3\.10\." | head -1) - echo "Found existing pyenv Python 3.10: ${installed_version}" >&2 + installed_version=$(pyenv versions --bare | grep "^${major_minor}\." | head -1) + echo "Found existing pyenv Python ${major_minor}: ${installed_version}" >&2 pyenv global "${installed_version}" echo "python3" return 0 fi - - # Install latest Python 3.10 - local latest_310 - latest_310=$(pyenv install --list | grep -E "^[[:space:]]*3\.10\.[0-9]+$" | tail -1 | xargs) - if [[ -n "${latest_310}" ]]; then - echo "Installing Python ${latest_310} via pyenv..." >&2 - if pyenv install "${latest_310}"; then - pyenv global "${latest_310}" + + # Install latest version in the required series + local latest_version + latest_version=$(pyenv install --list | grep -E "^[[:space:]]*${major_minor}\.[0-9]+$" | tail -1 | xargs) + if [[ -n "${latest_version}" ]]; then + echo "Installing Python ${latest_version} via pyenv..." >&2 + if pyenv install "${latest_version}"; then + pyenv global "${latest_version}" echo "python3" return 0 fi fi fi - - echo "Error: No suitable Python 3.10 installation found and unable to install via pyenv." >&2 - echo "Please ensure Python 3.10 is installed or pyenv is available." >&2 + + echo "Error: No suitable Python ${major_minor} installation found and unable to install via pyenv." >&2 + echo "Please ensure Python ${major_minor} is installed or pyenv is available." >&2 return 1 } @@ -83,22 +61,16 @@ if [[ -d "${PROJECT_DIR}"/venv ]]; then rm -rf "venv" fi -# Ensure Python 3.10 is available -python_bin=$(ensure_python310) +# Ensure required Python version is available +python_bin=$(ensure_required_python) echo "Using python from the following path: ${python_bin}" "${python_bin}" -m venv venv source venv/bin/activate pip install --upgrade pip - -if [[ "${INSTALL_REQUIREMENTS}" == "true" ]]; then - echo "Installing requirements.txt..." - pip install -r requirements.txt -else - echo "Skipping requirements.txt installation (--skip-requirements flag used)" -fi - +echo "Installing requirements.txt..." +pip install -r requirements.txt echo "Python venv was recreated successfully." echo "Current python path: $(which python)" python --version diff --git a/scripts/evergreen/setup_kubernetes_environment.sh b/scripts/evergreen/setup_kubernetes_environment.sh index 707231c9f..6edaad50d 100755 --- a/scripts/evergreen/setup_kubernetes_environment.sh +++ b/scripts/evergreen/setup_kubernetes_environment.sh @@ -30,6 +30,8 @@ elif [ "${KUBE_ENVIRONMENT_NAME}" = "kind" ] || [ "${KUBE_ENVIRONMENT_NAME}" = " scripts/dev/recreate_kind_cluster.sh "kind" elif [[ "${KUBE_ENVIRONMENT_NAME}" = "multi" && "${CLUSTER_TYPE}" == "kind" ]]; then scripts/dev/recreate_kind_clusters.sh +elif [[ "${KUBE_ENVIRONMENT_NAME}" = "minikube" ]]; then + echo "Nothing to do for minikube" else echo "KUBE_ENVIRONMENT_NAME not recognized" echo "value is <<${KUBE_ENVIRONMENT_NAME}>>. If empty it means it was not set" diff --git a/scripts/evergreen/setup_minikube_host.sh b/scripts/evergreen/setup_minikube_host.sh index 9d4884530..f4f5d643b 100755 --- a/scripts/evergreen/setup_minikube_host.sh +++ b/scripts/evergreen/setup_minikube_host.sh @@ -38,7 +38,9 @@ run_setup_step() { } # Setup Python environment (needed for AWS CLI pip installation) -run_setup_step "Python Virtual Environment" "scripts/dev/recreate_python_venv.sh" "--skip-requirements" + +export GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 +run_setup_step "Python Virtual Environment" "scripts/dev/recreate_python_venv.sh" run_setup_step "AWS CLI Setup" "scripts/evergreen/setup_aws.sh" diff --git a/scripts/evergreen/teardown_kubernetes_environment.sh b/scripts/evergreen/teardown_kubernetes_environment.sh index e5e2bd869..ec0c59966 100755 --- a/scripts/evergreen/teardown_kubernetes_environment.sh +++ b/scripts/evergreen/teardown_kubernetes_environment.sh @@ -1,10 +1,16 @@ #!/usr/bin/env bash -set -Eeou pipefail +set -Eeoux pipefail source scripts/dev/set_env_context.sh if [ "${KUBE_ENVIRONMENT_NAME}" = "kind" ]; then + docker system prune -a -f echo "Deleting Kind cluster" kind delete clusters --all fi + +if [ "${KUBE_ENVIRONMENT_NAME}" = "minikube" ]; then + echo "Deleting minikube cluster" + "${PROJECT_DIR:-.}/bin/minikube" delete +fi diff --git a/scripts/minikube/setup_minikube_host.sh b/scripts/minikube/setup_minikube_host.sh index cf39e8f4d..16f1c1c31 100755 --- a/scripts/minikube/setup_minikube_host.sh +++ b/scripts/minikube/setup_minikube_host.sh @@ -30,17 +30,7 @@ EOF # retrieve arch variable off the shell command line ARCH=${1-"$(uname -m)"} -# Validate architecture -case "${ARCH}" in - s390x|ppc64le|x86_64|aarch64) - echo "Setting up minikube host for architecture: ${ARCH}" - ;; - *) - echo "ERROR: Unsupported architecture: ${ARCH}" - echo "Supported architectures: s390x, ppc64le, x86_64, aarch64" - exit 1 - ;; -esac +echo "Setting up minikube host for architecture: ${ARCH}" download_minikube() { echo "Downloading minikube for ${ARCH}..." @@ -52,17 +42,23 @@ setup_local_registry_and_custom_image() { if [[ "${ARCH}" == "ppc64le" ]]; then echo ">>> Setting up local registry and custom kicbase image for ppc64le..." - # Check if local registry is running - if ! podman ps --filter "name=registry" --format "{{.Names}}" | grep -q "^registry$"; then + # Check if local registry is running (with fallback for namespace issues) + registry_running=false + if curl -s http://localhost:5000/v2/_catalog >/dev/null 2>&1; then + echo "Registry detected via HTTP check (podman ps failed)" + registry_running=true + fi + + if ! $registry_running; then echo "Starting local container registry on port 5000..." - podman run -d -p 5000:5000 --name registry --restart=always docker.io/library/registry:2 || { - echo "Registry might already exist, trying to start it..." - podman start registry || { - echo "Removing existing registry and creating new one..." - podman rm -f registry 2>/dev/null || true - podman run -d -p 5000:5000 --name registry --restart=always docker.io/library/registry:2 - } - } + + # Clean up any existing registry first + sudo podman rm -f registry 2>/dev/null || true + + if ! sudo podman run -d -p 5000:5000 --name registry --restart=always docker.io/library/registry:2; then + echo "❌ Failed to start local registry - trying alternative approach" + exit 1 + fi # Wait for registry to be ready echo "Waiting for registry to be ready..." @@ -76,73 +72,63 @@ setup_local_registry_and_custom_image() { echo "✅ Local registry already running" fi - # Configure system-wide podman to trust local registry (with backup) - echo "Configuring system registries.conf to trust local registry..." - - # Backup existing registries.conf if it exists - if [[ -f /etc/containers/registries.conf ]]; then - echo "Backing up existing registries.conf..." - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.minikube-backup - fi - - # Create a clean registries.conf that includes insecure local registry - sudo tee /etc/containers/registries.conf << 'EOF' -unqualified-search-registries = ["registry.access.redhat.com", "registry.redhat.io", "docker.io"] + # Configure podman to trust local registry (both user and root level for minikube) + echo "Configuring registries.conf to trust local registry..." + # User-level config + mkdir -p ~/.config/containers + cat > ~/.config/containers/registries.conf << 'EOF' [[registry]] location = "localhost:5000" insecure = true +EOF -short-name-mode = "permissive" + # Root-level config (since minikube uses sudo podman) + sudo mkdir -p /root/.config/containers + sudo tee /root/.config/containers/registries.conf << 'EOF' >/dev/null +[[registry]] +location = "localhost:5000" +insecure = true EOF - # Check if custom image already exists in local registry + echo "✅ Registry configuration created for both user and root" + custom_image_tag="localhost:5000/kicbase:v0.0.47" + + # Determine image tag + custom_image_tag="localhost:5000/kicbase:v0.0.47" if curl -s http://localhost:5000/v2/kicbase/tags/list | grep -q "v0.0.47"; then - echo "✅ Custom kicbase image already exists in local registry" + echo "Custom kicbase image already exists in local registry" return 0 fi # Build custom kicbase image with crictl echo "Building custom kicbase image with crictl for ppc64le..." - # Create build directory if it doesn't exist + # Build custom kicbase image mkdir -p "${PROJECT_DIR:-.}/scripts/minikube/kicbase" - - # Create Dockerfile for custom kicbase cat > "${PROJECT_DIR:-.}/scripts/minikube/kicbase/Dockerfile" << 'EOF' FROM gcr.io/k8s-minikube/kicbase:v0.0.47 - -# Install crictl for ppc64le if needed RUN if [ "$(uname -m)" = "ppc64le" ]; then \ - echo "Installing crictl for ppc64le architecture..." && \ CRICTL_VERSION="v1.28.0" && \ curl -L "https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-ppc64le.tar.gz" \ -o /tmp/crictl.tar.gz && \ tar -C /usr/bin -xzf /tmp/crictl.tar.gz && \ chmod +x /usr/bin/crictl && \ - rm /tmp/crictl.tar.gz && \ - echo "crictl installed successfully" && \ - crictl --version; \ - else \ - echo "Not ppc64le architecture, skipping crictl installation"; \ + rm /tmp/crictl.tar.gz; \ fi - -# Verify crictl is available -RUN command -v crictl >/dev/null 2>&1 && echo "crictl is available" || echo "crictl not found" EOF - # Build and push to local registry - echo "Building custom kicbase image..." cd "${PROJECT_DIR:-.}/scripts/minikube/kicbase" - - podman build -t localhost:5000/kicbase:v0.0.47 . - - echo "Pushing custom image to local registry..." - podman push localhost:5000/kicbase:v0.0.47 --tls-verify=false - - cd - > /dev/null - - echo "✅ Custom kicbase image with crictl ready in local registry" + sudo podman build -t "${custom_image_tag}" . || { + echo "Failed to build custom image" + return 1 + } + sudo podman push "${custom_image_tag}" --tls-verify=false || { + echo "Failed to push to registry" + return 1 + } + cd - >/dev/null + echo "Custom kicbase image ready: ${custom_image_tag}" fi return 0 } @@ -158,15 +144,14 @@ start_minikube_cluster() { rm -rf ~/.minikube/machines/minikube fi - # Delete any existing minikube cluster to start fresh echo "Ensuring clean minikube state..." "${PROJECT_DIR:-.}/bin/minikube" delete 2>/dev/null || true local start_args=("--driver=podman") - # Use custom kicbase image for ppc64le with crictl included if [[ "${ARCH}" == "ppc64le" ]]; then echo "Using custom kicbase image for ppc64le with crictl..." + start_args+=("--base-image=localhost:5000/kicbase:v0.0.47") start_args+=("--insecure-registry=localhost:5000") fi @@ -191,26 +176,17 @@ setup_podman() { # Check if podman is already available if command -v podman &> /dev/null; then echo "✅ Podman already installed" - - # Reset podman if it's in an invalid state - if podman info 2>&1 | grep -q "invalid internal status"; then - echo "Resetting podman due to invalid internal status..." - podman system migrate || true - podman system reset --force || true - fi - else - echo "Installing podman..." - sudo dnf install -y podman - fi - # Configure podman for CI environment - echo "Configuring Podman for ${ARCH} CI environment..." + # Diagnose podman state + echo "=== Podman Diagnostics ===" + echo "User: $(whoami), UID: $(id -u)" + echo "User namespace support: $(cat /proc/self/uid_map 2>/dev/null || echo 'not available')" + echo "Systemctl user status:" + systemctl --user status podman.socket 2>/dev/null || echo "podman.socket not active" + echo "Running 'sudo podman info' command..." + sudo podman info 2>&1 + fi - # Enable lingering for the current user to fix systemd session issues - current_user=$(whoami) - current_uid=$(id -u) - echo "Enabling systemd lingering for user ${current_user} (UID: ${current_uid})" - sudo loginctl enable-linger "${current_uid}" 2>/dev/null || true # Configure podman to use cgroupfs instead of systemd in CI mkdir -p ~/.config/containers @@ -223,11 +199,6 @@ events_logger = "file" cgroup_manager = "cgroupfs" EOF - # Start podman service if not running - systemctl --user enable podman.socket 2>/dev/null || true - systemctl --user start podman.socket 2>/dev/null || true - - echo "✅ Podman configured successfully for CI" } # Setup podman and container runtime @@ -249,23 +220,6 @@ else exit 1 fi -echo "" -echo ">>> Verifying podman installation..." -if command -v podman &> /dev/null; then - podman_version=$(podman --version 2>/dev/null) - echo "✅ Podman installed successfully: ${podman_version}" - - # Check podman info - if podman info &>/dev/null; then - echo "✅ Podman is working correctly" - else - echo "⚠️ Podman may need additional configuration" - fi -else - echo "❌ Podman installation failed - podman command not found" - exit 1 -fi - if [[ "${ARCH}" == "ppc64le" ]]; then echo "" echo ">>> Note: crictl will be patched into the minikube container after startup" @@ -295,7 +249,6 @@ echo "Container Runtime: podman" echo "Minikube Driver: podman" echo "Minikube: Default cluster" echo "Minikube: ${minikube_version}" -echo "Podman: ${podman_version}" echo "CNI: bridge (default)" if [[ "${ARCH}" == "ppc64le" ]]; then echo "Special Config: Custom kicbase image with crictl via local registry" From d72017af3825f5f99393a87e9d40b49037a53aff Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Fri, 1 Aug 2025 17:27:27 +0200 Subject: [PATCH 18/23] handle kube secret for podman --- scripts/funcs/kubernetes | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/scripts/funcs/kubernetes b/scripts/funcs/kubernetes index 5377d8927..d4426ef3c 100644 --- a/scripts/funcs/kubernetes +++ b/scripts/funcs/kubernetes @@ -98,12 +98,25 @@ create_image_registries_secret() { context=$1 namespace=$2 secret_name=$3 + + # Detect the correct config file path based on container runtime + local config_file + if command -v podman &> /dev/null && (podman info &> /dev/null || sudo podman info &> /dev/null); then + # For Podman, use root's auth.json since minikube uses sudo podman + config_file="/root/.config/containers/auth.json" + echo "Using Podman config: ${config_file}" + else + # For Docker, use standard docker config + config_file="${HOME}/.docker/config.json" + echo "Using Docker config: ${config_file}" + fi + # shellcheck disable=SC2154 if kubectl --context "${context}" get namespace "${namespace}"; then kubectl --context "${context}" -n "${namespace}" delete secret "${secret_name}" --ignore-not-found echo "${context}: Creating ${namespace}/${secret_name} pull secret" kubectl --context "${context}" -n "${namespace}" create secret generic "${secret_name}" \ - --from-file=.dockerconfigjson="${HOME}/.docker/config.json" --type=kubernetes.io/dockerconfigjson + --from-file=.dockerconfigjson="${config_file}" --type=kubernetes.io/dockerconfigjson else echo "Skipping creating pull secret in ${context}/${namespace}. The namespace doesn't exist yet." fi From 02073269560d6b9413690a3e08aea7f24f57562d Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Fri, 1 Aug 2025 17:38:59 +0200 Subject: [PATCH 19/23] handle path --- scripts/dev/configure_docker_auth.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/dev/configure_docker_auth.sh b/scripts/dev/configure_docker_auth.sh index 4edf68817..a36896970 100755 --- a/scripts/dev/configure_docker_auth.sh +++ b/scripts/dev/configure_docker_auth.sh @@ -87,7 +87,7 @@ container_login() { local registry="$2" if [[ "${CONTAINER_RUNTIME}" == "podman" ]]; then - sudo podman login --username "${username}" --password-stdin "${registry}" + sudo podman login --authfile "${CONFIG_PATH}" --username "${username}" --password-stdin "${registry}" else docker login --username "${username}" --password-stdin "${registry}" fi From 0bd74d88434f361420c2f29da20b304ef6351155 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Fri, 1 Aug 2025 18:05:12 +0200 Subject: [PATCH 20/23] handle auth issue for secret --- scripts/funcs/kubernetes | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/funcs/kubernetes b/scripts/funcs/kubernetes index d4426ef3c..11250422d 100644 --- a/scripts/funcs/kubernetes +++ b/scripts/funcs/kubernetes @@ -101,10 +101,17 @@ create_image_registries_secret() { # Detect the correct config file path based on container runtime local config_file + local temp_config_file="" if command -v podman &> /dev/null && (podman info &> /dev/null || sudo podman info &> /dev/null); then # For Podman, use root's auth.json since minikube uses sudo podman config_file="/root/.config/containers/auth.json" echo "Using Podman config: ${config_file}" + + # Create a temporary copy that the current user can read + temp_config_file=$(mktemp) + sudo cp "${config_file}" "${temp_config_file}" + sudo chown "$(whoami):$(whoami)" "${temp_config_file}" + config_file="${temp_config_file}" else # For Docker, use standard docker config config_file="${HOME}/.docker/config.json" @@ -120,6 +127,11 @@ create_image_registries_secret() { else echo "Skipping creating pull secret in ${context}/${namespace}. The namespace doesn't exist yet." fi + + # Clean up temporary file + if [[ -n "${temp_config_file}" ]] && [[ -f "${temp_config_file}" ]]; then + rm -f "${temp_config_file}" + fi } echo "Creating/updating pull secret from docker configured file" From 5cfdd82897e053dc1c641d69a6b72fa34efa00df Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Fri, 1 Aug 2025 19:38:51 +0200 Subject: [PATCH 21/23] use lucian evg and remove debug mode --- scripts/dev/contexts/evg-private-context | 8 ++++++++ scripts/evergreen/e2e/e2e.sh | 2 +- scripts/evergreen/setup_minikube_host.sh | 2 +- scripts/evergreen/teardown_kubernetes_environment.sh | 2 +- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/scripts/dev/contexts/evg-private-context b/scripts/dev/contexts/evg-private-context index 8f25842e8..8c1dfd672 100644 --- a/scripts/dev/contexts/evg-private-context +++ b/scripts/dev/contexts/evg-private-context @@ -122,3 +122,11 @@ export cognito_workload_url="${cognito_workload_url}" export cognito_workload_user_id="${cognito_workload_user_id}" export MDB_UPDATE_LICENSES=true + +export BASE_REPO_URL="268558157000.dkr.ecr.us-east-1.amazonaws.com/lucian.tosa" +export REGISTRY="${BASE_REPO_URL}" +export INIT_DATABASE_IMAGE_REPOSITORY="${BASE_REPO_URL}/mongodb-kubernetes-init-database" +export OPERATOR_REGISTRY=${BASE_REPO_URL} +export DATABASE_REGISTRY=${BASE_REPO_URL} +export INIT_DATABASE_REGISTRY=${BASE_REPO_URL} + diff --git a/scripts/evergreen/e2e/e2e.sh b/scripts/evergreen/e2e/e2e.sh index eb6140a22..12f839b86 100755 --- a/scripts/evergreen/e2e/e2e.sh +++ b/scripts/evergreen/e2e/e2e.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -Eeoux pipefail +set -Eeou pipefail start_time=$(date +%s) diff --git a/scripts/evergreen/setup_minikube_host.sh b/scripts/evergreen/setup_minikube_host.sh index f4f5d643b..cc85dbcd4 100755 --- a/scripts/evergreen/setup_minikube_host.sh +++ b/scripts/evergreen/setup_minikube_host.sh @@ -5,7 +5,7 @@ # Can be run on static hosts for testing and verification source scripts/dev/set_env_context.sh -set -Eeoux pipefail +set -Eeou pipefail echo "==========================================" echo "Setting up minikube host with multi-architecture support" diff --git a/scripts/evergreen/teardown_kubernetes_environment.sh b/scripts/evergreen/teardown_kubernetes_environment.sh index ec0c59966..a9babfbed 100755 --- a/scripts/evergreen/teardown_kubernetes_environment.sh +++ b/scripts/evergreen/teardown_kubernetes_environment.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -Eeoux pipefail +set -Eeou pipefail source scripts/dev/set_env_context.sh From 7c0cfbd3e2ba1dfeb17d1f3beec0c59a464730be Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Mon, 4 Aug 2025 11:07:17 +0200 Subject: [PATCH 22/23] fix z and workdir setup --- .evergreen-functions.yml | 15 ++++++++------- scripts/evergreen/setup_jq.sh | 5 ++--- scripts/evergreen/setup_minikube_host.sh | 2 +- scripts/funcs/printing | 23 +++++++++++++++++++++++ 4 files changed, 34 insertions(+), 11 deletions(-) diff --git a/.evergreen-functions.yml b/.evergreen-functions.yml index c67634152..8cf892a2b 100644 --- a/.evergreen-functions.yml +++ b/.evergreen-functions.yml @@ -51,6 +51,13 @@ functions: ### Setup Functions ### + setup_jq: &setup_jq + command: subprocess.exec + type: setup + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/setup_jq.sh + setup_context: &setup_context # Running the first switch is important to fill the workdir and other important initial env vars command: shell.exec type: setup @@ -103,6 +110,7 @@ functions: type: setup params: command: "git config --global user.email 'kubernetes-hosted-team@mongodb.com'" + - *setup_jq # we need jq in the context - *setup_context setup_kubectl: &setup_kubectl @@ -112,13 +120,6 @@ functions: working_dir: src/github.com/mongodb/mongodb-kubernetes binary: scripts/evergreen/setup_kubectl.sh - setup_jq: &setup_jq - command: subprocess.exec - type: setup - params: - working_dir: src/github.com/mongodb/mongodb-kubernetes - binary: scripts/evergreen/setup_jq.sh - setup_shellcheck: command: subprocess.exec type: setup diff --git a/scripts/evergreen/setup_jq.sh b/scripts/evergreen/setup_jq.sh index 1f260883a..8905d4d46 100755 --- a/scripts/evergreen/setup_jq.sh +++ b/scripts/evergreen/setup_jq.sh @@ -7,14 +7,13 @@ set -Eeou pipefail -source scripts/dev/set_env_context.sh source scripts/funcs/install # Detect and map architecture for jq releases detect_jq_architecture() { local arch arch=$(uname -m) - + case "${arch}" in x86_64) echo "amd64" @@ -38,4 +37,4 @@ detect_jq_architecture() { jq_arch=$(detect_jq_architecture) echo "Detected architecture: $(uname -m), using jq architecture: ${jq_arch}" -download_and_install_binary "${PROJECT_DIR:-.}/bin" jq "https://github.com/stedolan/jq/releases/download/jq-1.8.1/jq-linux-${jq_arch}" +download_and_install_binary "${PROJECT_DIR:-${workdir:-.}}/bin" jq "https://github.com/stedolan/jq/releases/download/jq-1.8.1/jq-linux-${jq_arch}" diff --git a/scripts/evergreen/setup_minikube_host.sh b/scripts/evergreen/setup_minikube_host.sh index cc85dbcd4..f4f5d643b 100755 --- a/scripts/evergreen/setup_minikube_host.sh +++ b/scripts/evergreen/setup_minikube_host.sh @@ -5,7 +5,7 @@ # Can be run on static hosts for testing and verification source scripts/dev/set_env_context.sh -set -Eeou pipefail +set -Eeoux pipefail echo "==========================================" echo "Setting up minikube host with multi-architecture support" diff --git a/scripts/funcs/printing b/scripts/funcs/printing index 0ee4b5ec0..7d55e5af9 100644 --- a/scripts/funcs/printing +++ b/scripts/funcs/printing @@ -21,3 +21,26 @@ prepend() { export RED='\033[0;31m' export NO_COLOR='\033[0m' + +run_setup_step() { + local step_name="$1" + shift + local script_command=("$@") + + echo "" + echo ">>> Running: ${step_name}" + echo ">>> Command: ${script_command[*]}" + + local script_path="${script_command[0]}" + if [[ -f "${script_path}" ]]; then + if "${script_command[@]}"; then + echo "✅ ${step_name} completed successfully" + else + echo "❌ ${step_name} failed" + exit 1 + fi + else + echo "❌ Script not found: ${script_path}" + exit 1 + fi +} From 718d3cc409e2f3596ebc322924fce60eb26059b4 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Mon, 4 Aug 2025 11:07:32 +0200 Subject: [PATCH 23/23] fix z and workdir setup --- scripts/funcs/printing | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/scripts/funcs/printing b/scripts/funcs/printing index 7d55e5af9..0ee4b5ec0 100644 --- a/scripts/funcs/printing +++ b/scripts/funcs/printing @@ -21,26 +21,3 @@ prepend() { export RED='\033[0;31m' export NO_COLOR='\033[0m' - -run_setup_step() { - local step_name="$1" - shift - local script_command=("$@") - - echo "" - echo ">>> Running: ${step_name}" - echo ">>> Command: ${script_command[*]}" - - local script_path="${script_command[0]}" - if [[ -f "${script_path}" ]]; then - if "${script_command[@]}"; then - echo "✅ ${step_name} completed successfully" - else - echo "❌ ${step_name} failed" - exit 1 - fi - else - echo "❌ Script not found: ${script_path}" - exit 1 - fi -}