From 200b87d18704dc5414d723a90ec695d7c52b7c2a Mon Sep 17 00:00:00 2001 From: Vivek Singh Date: Wed, 8 Oct 2025 16:10:57 +0200 Subject: [PATCH 01/11] Publish helm chart to OCI registry for PR patches --- .evergreen.yml | 10 ++++++++ scripts/dev/helm_registry_login.sh | 39 ++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100755 scripts/dev/helm_registry_login.sh diff --git a/.evergreen.yml b/.evergreen.yml index 5223bae84..68a3d81b4 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -445,6 +445,16 @@ tasks: - func: helm_registry_login - func: publish_helm_chart + - name: publish_helm_chart + commands: + - func: clone + - func: setup_kubectl + - func: setup_aws + - func: prepare_aws + - func: helm_registry_login + - func: python_venv + - func: publish_helm_chart + - name: prepare_aws priority: 59 commands: diff --git a/scripts/dev/helm_registry_login.sh b/scripts/dev/helm_registry_login.sh new file mode 100755 index 000000000..1d3ca2e3f --- /dev/null +++ b/scripts/dev/helm_registry_login.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +ECR_REGISTRY="268558157000.dkr.ecr.us-east-1.amazonaws.com" +AWS_REGION="us-east-1" + +source scripts/dev/set_env_context.sh + +export PATH=${PROJECT_DIR}/bin:$PATH + +echo "Checking if helm CLI is installed..." +if ! command -v helm &> /dev/null +then + echo "Error: helm CLI could not be found." + echo "Please ensure helm is installed and in your system's PATH." + exit 1 +fi + +echo "Checking if aws CLI is installed..." +if ! command -v aws &> /dev/null +then + echo "Error: aws CLI could not be found." + echo "Please ensure aws CLI is installed and configured." + exit 1 +fi + +echo "Logging into OCI Registry: ${ECR_REGISTRY} in region ${AWS_REGION}..." + +if aws ecr get-login-password --region "$AWS_REGION" | helm registry login \ + --username AWS \ + --password-stdin \ + "$ECR_REGISTRY" +then + echo "Helm successfully logged into the OCI registry." + exit 0 +else + echo "ERROR: Helm login to ECR registry failed." + echo "Please ensure your AWS credentials have permission to access ECR in the specified region." + exit 1 +fi From 139c2f3c4d58dcdd548ca7f3ebf89bb58c523dc1 Mon Sep 17 00:00:00 2001 From: Vivek Singh Date: Wed, 8 Oct 2025 16:39:43 +0200 Subject: [PATCH 02/11] Install published helm chart instead of local one, during E2E --- .evergreen.yml | 2 + docker/mongodb-kubernetes-tests/Dockerfile | 8 +- .../kubetester/helm.py | 84 ++++++++++++++++++- 3 files changed, 92 insertions(+), 2 deletions(-) diff --git a/.evergreen.yml b/.evergreen.yml index 68a3d81b4..9ff87edaf 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -54,6 +54,8 @@ variables: variant: init_test_run - name: build_init_appdb_images_ubi variant: init_test_run + - name: publish_helm_chart + variant: init_test_run - &community_dependency depends_on: diff --git a/docker/mongodb-kubernetes-tests/Dockerfile b/docker/mongodb-kubernetes-tests/Dockerfile index 88be6b11d..dd1b8aca7 100644 --- a/docker/mongodb-kubernetes-tests/Dockerfile +++ b/docker/mongodb-kubernetes-tests/Dockerfile @@ -45,7 +45,8 @@ RUN apt-get -qq update \ libldap2-dev \ libsasl2-dev \ git \ - openssl + openssl \ + unzip RUN mkdir -p /tmp/mongodb-tools && \ tar xfz /tmp/mongodb-tools.tgz -C /tmp/mongodb-tools && \ @@ -66,6 +67,11 @@ RUN curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl - && chmod +x ./kubectl \ && mv ./kubectl /usr/local/bin/kubectl +# install aws, required to run helm registry login while running the tests +RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \ + && unzip -q awscliv2.zip \ + && ./aws/install + COPY --from=builder /venv /venv ENV PATH="/venv/bin:${PATH}" diff --git a/docker/mongodb-kubernetes-tests/kubetester/helm.py b/docker/mongodb-kubernetes-tests/kubetester/helm.py index 9e0cca8fe..60829ab2f 100644 --- a/docker/mongodb-kubernetes-tests/kubetester/helm.py +++ b/docker/mongodb-kubernetes-tests/kubetester/helm.py @@ -10,6 +10,8 @@ logger = test_logger.get_test_logger(__name__) +HELM_REGISTRY_AWS_REGION = "us-east-1" +HELM_ECR_REGISTRY = "268558157000.dkr.ecr.us-east-1.amazonaws.com" def helm_template( helm_args: Dict, @@ -162,8 +164,14 @@ def helm_upgrade( chart_dir = helm_chart_path if helm_override_path else _helm_chart_dir(helm_chart_path) if apply_crds_first: + # right now tests image has the entire helm_chart directory, maybe we should just copy the CRDs apply_crds_from_chart(chart_dir) + # login to helm registry because we are going to install published helm chart + if not helm_registry_login(): + raise Exception(f"Failed logging in to the helm registry {HELM_ECR_REGISTRY}") + + chart_dir = f"oci://{HELM_ECR_REGISTRY}/dev/helm-charts/mongodb-kubernetes" command_args = _create_helm_args(helm_args, helm_options) args = [ "helm", @@ -173,7 +181,7 @@ def helm_upgrade( *command_args, name, ] - + custom_operator_version = "0.0.0+68e3eec04df1df00072e1bb2" if custom_operator_version: args.append(f"--version={custom_operator_version}") @@ -182,6 +190,80 @@ def helm_upgrade( command = " ".join(args) process_run_and_check(command, check=True, capture_output=True, shell=True) +def helm_registry_login(): + logger.info(f"Attempting to log into ECR registry: {HELM_ECR_REGISTRY}, using helm registry login.") + + aws_command = [ + "aws", + "ecr", + "get-login-password", + "--region", + HELM_REGISTRY_AWS_REGION + ] + + # as we can see the password is being provided by stdin, that would mean we will have to + # pipe the aws_command (it figures out the password) into helm_command. + helm_command = [ + "helm", + "registry", + "login", + "--username", + "AWS", + "--password-stdin", + HELM_ECR_REGISTRY + ] + + try: + logger.info("Starting AWS ECR credential retrieval.") + aws_proc = subprocess.Popen( + aws_command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True # Treat input/output as text strings + ) + + logger.info("Starting Helm registry login.") + helm_proc = subprocess.Popen( + helm_command, + stdin=aws_proc.stdout, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + + # Close the stdout stream of aws_proc in the parent process + # to prevent resource leakage (only needed if you plan to do more processing) + aws_proc.stdout.close() + + # Wait for the Helm command (helm_proc) to finish and capture its output + helm_stdout, helm_stderr = helm_proc.communicate() + + # Wait for the AWS process to finish as well + aws_proc.wait() + + if aws_proc.returncode != 0: + logger.error(f"aws command to get password failed, (Exit Code {aws_proc.returncode}).") + # We captured AWS stderr directly, so print it + _, aws_stderr = aws_proc.communicate() + logger.error(aws_stderr) + return False + + if helm_proc.returncode == 0: + logger.info("Login to helm registry was successful.") + logger.info(helm_stdout.strip()) + return True + else: + logger.error(f"Login to helm registry failed, (Exit Code {helm_proc.returncode}).") + logger.error(helm_stderr.strip()) + return False + + except FileNotFoundError as e: + # This catches errors if 'aws' or 'helm' are not in the PATH + logger.error(f"Command not found. Please ensure '{e.filename}' is installed and in your system's PATH.") + return False + except Exception as e: + logger.error(f"An unexpected error occurred: {e}.") + return False def apply_crds_from_chart(chart_dir: str): crd_files = glob.glob(os.path.join(chart_dir, "crds", "*.yaml")) From c8d24c0e9dc9c1268564883199a9229295134499 Mon Sep 17 00:00:00 2001 From: Vivek Singh Date: Wed, 8 Oct 2025 17:47:41 +0200 Subject: [PATCH 03/11] Don't copy entire helm_chart dir, just copy CRDs --- docker/mongodb-kubernetes-tests/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/mongodb-kubernetes-tests/Dockerfile b/docker/mongodb-kubernetes-tests/Dockerfile index dd1b8aca7..b91ef0516 100644 --- a/docker/mongodb-kubernetes-tests/Dockerfile +++ b/docker/mongodb-kubernetes-tests/Dockerfile @@ -81,8 +81,9 @@ WORKDIR /tests # copying the test files after python build, otherwise pip install will be called each time the tests change COPY . /tests -# copying the helm_chart directory as well to support installation of the Operator from the test application -COPY helm_chart /helm_chart +# copying the helm_chart/crds directory so that we can install CRDs before installing the operator helm chart to run a test. +# operator is installed via published OCI helm repo and not the local helm repo. +COPY helm_chart/crds /helm_chart/crds COPY release.json /release.json # we use the public directory to automatically test resources samples COPY public /mongodb-kubernetes/public From 08649ced4ce7ca954a98afd560c6dfb0173e12f9 Mon Sep 17 00:00:00 2001 From: Vivek Singh Date: Fri, 10 Oct 2025 17:00:24 +0200 Subject: [PATCH 04/11] Delete not needed file --- scripts/dev/helm_registry_login.sh | 39 ------------------------------ 1 file changed, 39 deletions(-) delete mode 100755 scripts/dev/helm_registry_login.sh diff --git a/scripts/dev/helm_registry_login.sh b/scripts/dev/helm_registry_login.sh deleted file mode 100755 index 1d3ca2e3f..000000000 --- a/scripts/dev/helm_registry_login.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -ECR_REGISTRY="268558157000.dkr.ecr.us-east-1.amazonaws.com" -AWS_REGION="us-east-1" - -source scripts/dev/set_env_context.sh - -export PATH=${PROJECT_DIR}/bin:$PATH - -echo "Checking if helm CLI is installed..." -if ! command -v helm &> /dev/null -then - echo "Error: helm CLI could not be found." - echo "Please ensure helm is installed and in your system's PATH." - exit 1 -fi - -echo "Checking if aws CLI is installed..." -if ! command -v aws &> /dev/null -then - echo "Error: aws CLI could not be found." - echo "Please ensure aws CLI is installed and configured." - exit 1 -fi - -echo "Logging into OCI Registry: ${ECR_REGISTRY} in region ${AWS_REGION}..." - -if aws ecr get-login-password --region "$AWS_REGION" | helm registry login \ - --username AWS \ - --password-stdin \ - "$ECR_REGISTRY" -then - echo "Helm successfully logged into the OCI registry." - exit 0 -else - echo "ERROR: Helm login to ECR registry failed." - echo "Please ensure your AWS credentials have permission to access ECR in the specified region." - exit 1 -fi From 5d55e9f9e8f37c39e96b9d9d357563b6b831e872 Mon Sep 17 00:00:00 2001 From: Vivek Singh Date: Wed, 15 Oct 2025 16:36:21 +0200 Subject: [PATCH 05/11] Start installing chart from OCI registry in E2E --- .evergreen.yml | 10 -- .../kubetester/helm.py | 147 ++++++++---------- .../templates/mongodb-enterprise-tests.yaml | 16 ++ .../deployments/test-app/values.yaml | 11 ++ scripts/evergreen/e2e/single_e2e.sh | 21 +++ scripts/release/oci_chart_info.py | 24 +++ 6 files changed, 139 insertions(+), 90 deletions(-) create mode 100644 scripts/release/oci_chart_info.py diff --git a/.evergreen.yml b/.evergreen.yml index 9ff87edaf..522ad4863 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -447,16 +447,6 @@ tasks: - func: helm_registry_login - func: publish_helm_chart - - name: publish_helm_chart - commands: - - func: clone - - func: setup_kubectl - - func: setup_aws - - func: prepare_aws - - func: helm_registry_login - - func: python_venv - - func: publish_helm_chart - - name: prepare_aws priority: 59 commands: diff --git a/docker/mongodb-kubernetes-tests/kubetester/helm.py b/docker/mongodb-kubernetes-tests/kubetester/helm.py index 60829ab2f..117665439 100644 --- a/docker/mongodb-kubernetes-tests/kubetester/helm.py +++ b/docker/mongodb-kubernetes-tests/kubetester/helm.py @@ -10,9 +10,6 @@ logger = test_logger.get_test_logger(__name__) -HELM_REGISTRY_AWS_REGION = "us-east-1" -HELM_ECR_REGISTRY = "268558157000.dkr.ecr.us-east-1.amazonaws.com" - def helm_template( helm_args: Dict, helm_chart_path: Optional[str] = "helm_chart", @@ -146,6 +143,53 @@ def process_run_and_check(args, **kwargs): logger.error(f"output: {exc.output}") raise +def oci_helm_registry_login(helm_registry: str, region: str): + logger.info(f"Attempting to log into ECR registry: {helm_registry}, using helm registry login.") + + aws_command = ["aws", "ecr", "get-login-password", "--region", region] + + # as we can see the password is being provided by stdin, that would mean we will have to + # pipe the aws_command (it figures out the password) into helm_command. + helm_command = ["helm", "registry", "login", "--username", "AWS", "--password-stdin", helm_registry] + + try: + logger.info("Starting AWS ECR credential retrieval.") + aws_proc = subprocess.Popen( + aws_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True # Treat input/output as text strings + ) + + logger.info("Starting Helm registry login.") + helm_proc = subprocess.Popen( + helm_command, stdin=aws_proc.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + ) + + # Close the stdout stream of aws_proc in the parent process + # to prevent resource leakage (only needed if you plan to do more processing) + aws_proc.stdout.close() + + # Wait for the Helm command (helm_proc) to finish and capture its output + helm_stdout, helm_stderr = helm_proc.communicate() + + # Wait for the AWS process to finish as well + aws_proc.wait() + + if aws_proc.returncode != 0: + _, aws_stderr = aws_proc.communicate() + raise Exception(f"aws command to get password failed. Error: {aws_stderr}") + + if helm_proc.returncode == 0: + logger.info("Login to helm registry was successful.") + logger.info(helm_stdout.strip()) + else: + raise Exception( + f"Login to helm registry failed, Exit code: {helm_proc.returncode}, Error: {helm_stderr.strip()}" + ) + + except FileNotFoundError as e: + # This catches errors if 'aws' or 'helm' are not in the PATH + raise Exception(f"Command not found. Please ensure '{e.filename}' is installed and in your system's PATH.") + except Exception as e: + raise Exception(f"An unexpected error occurred: {e}.") def helm_upgrade( name: str, @@ -168,10 +212,14 @@ def helm_upgrade( apply_crds_from_chart(chart_dir) # login to helm registry because we are going to install published helm chart - if not helm_registry_login(): - raise Exception(f"Failed logging in to the helm registry {HELM_ECR_REGISTRY}") + try: + registry, repository, region = oci_chart_info() + + oci_helm_registry_login(registry, region) + except Exception as e: + raise Exception(f"Failed logging in to the helm registry {registry}. Error: {e}") - chart_dir = f"oci://{HELM_ECR_REGISTRY}/dev/helm-charts/mongodb-kubernetes" + chart_uri = f"oci://{registry}/{repository}" command_args = _create_helm_args(helm_args, helm_options) args = [ "helm", @@ -181,89 +229,28 @@ def helm_upgrade( *command_args, name, ] - custom_operator_version = "0.0.0+68e3eec04df1df00072e1bb2" + if custom_operator_version: args.append(f"--version={custom_operator_version}") + else: + published_chart_version = os.environ.get("OPERATOR_VERSION") + if not published_chart_version: + logger.info("OPERATOR_VERSION env var is not set") + args.append(f"--version=0.0.0+{published_chart_version}") - args.append(chart_dir) + args.append(chart_uri) command = " ".join(args) process_run_and_check(command, check=True, capture_output=True, shell=True) -def helm_registry_login(): - logger.info(f"Attempting to log into ECR registry: {HELM_ECR_REGISTRY}, using helm registry login.") - - aws_command = [ - "aws", - "ecr", - "get-login-password", - "--region", - HELM_REGISTRY_AWS_REGION - ] - - # as we can see the password is being provided by stdin, that would mean we will have to - # pipe the aws_command (it figures out the password) into helm_command. - helm_command = [ - "helm", - "registry", - "login", - "--username", - "AWS", - "--password-stdin", - HELM_ECR_REGISTRY - ] +def oci_chart_info(): + registry = os.environ.get("OCI_HELM_REGISTRY") + repository = os.environ.get("OCI_HELM_REPOSITORY") + region = os.environ.get("OCI_HELM_REGION") - try: - logger.info("Starting AWS ECR credential retrieval.") - aws_proc = subprocess.Popen( - aws_command, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True # Treat input/output as text strings - ) - - logger.info("Starting Helm registry login.") - helm_proc = subprocess.Popen( - helm_command, - stdin=aws_proc.stdout, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True - ) - - # Close the stdout stream of aws_proc in the parent process - # to prevent resource leakage (only needed if you plan to do more processing) - aws_proc.stdout.close() - - # Wait for the Helm command (helm_proc) to finish and capture its output - helm_stdout, helm_stderr = helm_proc.communicate() - - # Wait for the AWS process to finish as well - aws_proc.wait() - - if aws_proc.returncode != 0: - logger.error(f"aws command to get password failed, (Exit Code {aws_proc.returncode}).") - # We captured AWS stderr directly, so print it - _, aws_stderr = aws_proc.communicate() - logger.error(aws_stderr) - return False - - if helm_proc.returncode == 0: - logger.info("Login to helm registry was successful.") - logger.info(helm_stdout.strip()) - return True - else: - logger.error(f"Login to helm registry failed, (Exit Code {helm_proc.returncode}).") - logger.error(helm_stderr.strip()) - return False + print(f"oci chart details in test image is registry {registry}, repo {repository}, region {region}") - except FileNotFoundError as e: - # This catches errors if 'aws' or 'helm' are not in the PATH - logger.error(f"Command not found. Please ensure '{e.filename}' is installed and in your system's PATH.") - return False - except Exception as e: - logger.error(f"An unexpected error occurred: {e}.") - return False + return registry, f"{repository}/mongodb-kubernetes", region def apply_crds_from_chart(chart_dir: str): crd_files = glob.glob(os.path.join(chart_dir, "crds", "*.yaml")) diff --git a/scripts/evergreen/deployments/test-app/templates/mongodb-enterprise-tests.yaml b/scripts/evergreen/deployments/test-app/templates/mongodb-enterprise-tests.yaml index 2a3e37ff3..7abfd58ab 100644 --- a/scripts/evergreen/deployments/test-app/templates/mongodb-enterprise-tests.yaml +++ b/scripts/evergreen/deployments/test-app/templates/mongodb-enterprise-tests.yaml @@ -174,6 +174,22 @@ spec: - name: OM_DEBUG_HTTP value: "{{ .Values.omDebugHttp }}" {{ end }} + {{ if .Values.helm.oci.registry }} + - name: OCI_HELM_REGISTRY + value: "{{ .Values.helm.oci.registry }}" + {{ end }} + {{ if .Values.operator.version }} + - name: OPERATOR_VERSION + value: "{{ .Values.operator.version }}" + {{ end }} + {{ if .Values.helm.oci.repository }} + - name: OCI_HELM_REPOSITORY + value: "{{ .Values.helm.oci.repository }}" + {{ end }} + {{ if .Values.helm.oci.region }} + - name: OCI_HELM_REGION + value: "{{ .Values.helm.oci.region }}" + {{ end }} - name: ops_manager_version value: "{{ .Values.opsManagerVersion }}" - name: cognito_user_pool_id diff --git a/scripts/evergreen/deployments/test-app/values.yaml b/scripts/evergreen/deployments/test-app/values.yaml index 13aba3df8..09ab966c7 100644 --- a/scripts/evergreen/deployments/test-app/values.yaml +++ b/scripts/evergreen/deployments/test-app/values.yaml @@ -43,3 +43,14 @@ mdbImageType: "ubi8" # set to "true" to set OM_DEBUG_HTTP=true for the operator omDebugHttp: + +# sets the configuration using which we can figure out the helm OCI repo of the MCK operator while deploying it +# to run a test. +helm: + oci: + registry: "" + repository: "" + region: "" + +operator: + version: "" diff --git a/scripts/evergreen/e2e/single_e2e.sh b/scripts/evergreen/e2e/single_e2e.sh index 054470eef..03776c5ad 100755 --- a/scripts/evergreen/e2e/single_e2e.sh +++ b/scripts/evergreen/e2e/single_e2e.sh @@ -45,6 +45,11 @@ deploy_test_app() { BUILD_ID="${BUILD_ID:-default_build_id}" BUILD_VARIANT="${BUILD_VARIANT:-default_build_variant}" + chart_info=$(scripts/dev/run_python.sh scripts/release/oci_chart_info.py) + helm_oci_regisry=$(echo "$chart_info" | jq -r '.registry' ) + helm_oci_repository=$(echo "$chart_info" | jq -r '.repository' ) + helm_oci_registry_region=$(echo "$chart_info" | jq -r '.region' ) + # note, that the 4 last parameters are used only for Mongodb resource testing - not for Ops Manager helm_params=( "--set" "taskId=${task_id:-'not-specified'}" @@ -139,6 +144,22 @@ deploy_test_app() { helm_params+=("--set" "omDebugHttp=true") fi + if [[ -n "${helm_oci_regisry:-}" ]]; then + helm_params+=("--set" "helm.oci.registry=${helm_oci_regisry}") + fi + + if [[ -n "${helm_oci_repository:-}" ]]; then + helm_params+=("--set" "helm.oci.repository=${helm_oci_repository}") + fi + + if [[ -n "${helm_oci_registry_region:-}" ]]; then + helm_params+=("--set" "helm.oci.region=${helm_oci_registry_region}") + fi + + if [[ -n "${OPERATOR_VERSION:-}" ]]; then + helm_params+=("--set" "operator.version=${OPERATOR_VERSION}") + fi + helm_params+=("--set" "opsManagerVersion=${ops_manager_version}") helm template "scripts/evergreen/deployments/test-app" "${helm_params[@]}" > "${helm_template_file}" || exit 1 diff --git a/scripts/release/oci_chart_info.py b/scripts/release/oci_chart_info.py new file mode 100644 index 000000000..7272658c6 --- /dev/null +++ b/scripts/release/oci_chart_info.py @@ -0,0 +1,24 @@ +import os +import sys +import json + +from dataclasses import asdict +from lib.base_logger import logger +from scripts.release.build.build_info import load_build_info + +def main(): + build_scenario = os.environ.get("BUILD_SCENARIO") + build_info = load_build_info(build_scenario) + chart_info = build_info.helm_charts["mongodb-kubernetes"] + + j = json.dumps(asdict(chart_info)) + print(j) + + + +if __name__ == "__main__": + try: + main() + except Exception as e: + logger.error(f"Failed while dumping the chart_info as json. Error: {e}") + sys.exit(1) \ No newline at end of file From a5b3fbf9e9c345c97b4f348da44f77c49f3a1037 Mon Sep 17 00:00:00 2001 From: Vivek Singh Date: Wed, 15 Oct 2025 16:43:34 +0200 Subject: [PATCH 06/11] Add newline --- scripts/release/oci_chart_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/release/oci_chart_info.py b/scripts/release/oci_chart_info.py index 7272658c6..0e5b6bf54 100644 --- a/scripts/release/oci_chart_info.py +++ b/scripts/release/oci_chart_info.py @@ -21,4 +21,4 @@ def main(): main() except Exception as e: logger.error(f"Failed while dumping the chart_info as json. Error: {e}") - sys.exit(1) \ No newline at end of file + sys.exit(1) From 30666e8ad8393a10d09e900a4057bb23a3f74009 Mon Sep 17 00:00:00 2001 From: Vivek Singh Date: Thu, 16 Oct 2025 13:57:25 +0200 Subject: [PATCH 07/11] Refactor 1. Move figuring out of chart dir from helm.py to other places to Operator() and _install_multi_cluster_operator 2. Run precommit --- .../kubetester/helm.py | 24 +++++---------- .../kubetester/operator.py | 30 +++++++++++++++++-- .../tests/conftest.py | 28 +++++++++++++++-- scripts/evergreen/e2e/single_e2e.sh | 6 ++-- scripts/release/oci_chart_info.py | 6 ++-- 5 files changed, 65 insertions(+), 29 deletions(-) diff --git a/docker/mongodb-kubernetes-tests/kubetester/helm.py b/docker/mongodb-kubernetes-tests/kubetester/helm.py index 117665439..40533b2b6 100644 --- a/docker/mongodb-kubernetes-tests/kubetester/helm.py +++ b/docker/mongodb-kubernetes-tests/kubetester/helm.py @@ -10,6 +10,7 @@ logger = test_logger.get_test_logger(__name__) + def helm_template( helm_args: Dict, helm_chart_path: Optional[str] = "helm_chart", @@ -143,6 +144,7 @@ def process_run_and_check(args, **kwargs): logger.error(f"output: {exc.output}") raise + def oci_helm_registry_login(helm_registry: str, region: str): logger.info(f"Attempting to log into ECR registry: {helm_registry}, using helm registry login.") @@ -191,6 +193,7 @@ def oci_helm_registry_login(helm_registry: str, region: str): except Exception as e: raise Exception(f"An unexpected error occurred: {e}.") + def helm_upgrade( name: str, namespace: str, @@ -208,18 +211,8 @@ def helm_upgrade( chart_dir = helm_chart_path if helm_override_path else _helm_chart_dir(helm_chart_path) if apply_crds_first: - # right now tests image has the entire helm_chart directory, maybe we should just copy the CRDs apply_crds_from_chart(chart_dir) - # login to helm registry because we are going to install published helm chart - try: - registry, repository, region = oci_chart_info() - - oci_helm_registry_login(registry, region) - except Exception as e: - raise Exception(f"Failed logging in to the helm registry {registry}. Error: {e}") - - chart_uri = f"oci://{registry}/{repository}" command_args = _create_helm_args(helm_args, helm_options) args = [ "helm", @@ -229,20 +222,16 @@ def helm_upgrade( *command_args, name, ] - + if custom_operator_version: args.append(f"--version={custom_operator_version}") - else: - published_chart_version = os.environ.get("OPERATOR_VERSION") - if not published_chart_version: - logger.info("OPERATOR_VERSION env var is not set") - args.append(f"--version=0.0.0+{published_chart_version}") - args.append(chart_uri) + args.append(chart_dir) command = " ".join(args) process_run_and_check(command, check=True, capture_output=True, shell=True) + def oci_chart_info(): registry = os.environ.get("OCI_HELM_REGISTRY") repository = os.environ.get("OCI_HELM_REPOSITORY") @@ -252,6 +241,7 @@ def oci_chart_info(): return registry, f"{repository}/mongodb-kubernetes", region + def apply_crds_from_chart(chart_dir: str): crd_files = glob.glob(os.path.join(chart_dir, "crds", "*.yaml")) diff --git a/docker/mongodb-kubernetes-tests/kubetester/operator.py b/docker/mongodb-kubernetes-tests/kubetester/operator.py index fa2b90344..f75a321fb 100644 --- a/docker/mongodb-kubernetes-tests/kubetester/operator.py +++ b/docker/mongodb-kubernetes-tests/kubetester/operator.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +import os import time from typing import Dict, List, Optional @@ -16,6 +17,8 @@ helm_template, helm_uninstall, helm_upgrade, + oci_chart_info, + oci_helm_registry_login, ) from tests import test_logger @@ -43,13 +46,27 @@ def __init__( namespace: str, helm_args: Optional[Dict] = None, helm_options: Optional[List[str]] = None, - helm_chart_path: Optional[str] = "helm_chart", + helm_chart_path: Optional[str] = None, name: Optional[str] = "mongodb-kubernetes-operator", api_client: Optional[client.api_client.ApiClient] = None, + operator_version: Optional[str] = None, ): + if not helm_chart_path: + # login to the OCI container registry + registry, repository, region = oci_chart_info() + try: + oci_helm_registry_login(registry, region) + except Exception as e: + raise e - # The Operator will be installed from the following repo, so adding it first - helm_repo_add("mongodb", "https://mongodb.github.io/helm-charts") + # figure out the registry URI, based on dev/staging scenario + chart_uri = f"oci://{registry}/{repository}" + helm_chart_path = chart_uri + + if not operator_version: + # most probably we are trying to install current operator which will be installed + # from OCI registry. The version (dev/staging) is set in `OPERATOR_VERSION` + operator_version = os.environ.get("OPERATOR_VERSION") if helm_args is None: helm_args = {} @@ -69,6 +86,7 @@ def __init__( self.helm_chart_path = helm_chart_path self.name = name self.api_client = api_client + self.operator_version = operator_version def install_from_template(self): """Uses helm to generate yaml specification and then uses python K8s client to apply them to the cluster @@ -82,6 +100,9 @@ def install_from_template(self): def install(self, custom_operator_version: Optional[str] = None) -> Operator: """Installs the Operator to Kubernetes cluster using 'helm install', waits until it's running""" + if not custom_operator_version: + custom_operator_version = self.operator_version + helm_install( self.name, self.namespace, @@ -99,6 +120,9 @@ def upgrade( self, multi_cluster: bool = False, custom_operator_version: Optional[str] = None, apply_crds_first: bool = False ) -> Operator: """Upgrades the Operator in Kubernetes cluster using 'helm upgrade', waits until it's running""" + if not custom_operator_version: + custom_operator_version = self.operator_version + helm_upgrade( self.name, self.namespace, diff --git a/docker/mongodb-kubernetes-tests/tests/conftest.py b/docker/mongodb-kubernetes-tests/tests/conftest.py index 15366fc79..cb46e4755 100644 --- a/docker/mongodb-kubernetes-tests/tests/conftest.py +++ b/docker/mongodb-kubernetes-tests/tests/conftest.py @@ -20,7 +20,12 @@ update_configmap, ) from kubetester.awss3client import AwsS3Client -from kubetester.helm import helm_install_from_chart, helm_repo_add +from kubetester.helm import ( + helm_install_from_chart, + helm_repo_add, + oci_chart_info, + oci_helm_registry_login, +) from kubetester.kubetester import KubernetesTester from kubetester.kubetester import fixture as _fixture from kubetester.kubetester import running_locally @@ -827,14 +832,31 @@ def _install_multi_cluster_operator( helm_opts: dict[str, str], central_cluster_name: str, operator_name: Optional[str] = MULTI_CLUSTER_OPERATOR_NAME, - helm_chart_path: Optional[str] = LOCAL_HELM_CHART_DIR, + helm_chart_path: Optional[str] = None, custom_operator_version: Optional[str] = None, apply_crds_first: bool = False, ) -> Operator: multi_cluster_operator_installation_config.update(helm_opts) # The Operator will be installed from the following repo, so adding it first - helm_repo_add("mongodb", "https://mongodb.github.io/helm-charts") + # helm_repo_add("mongodb", "https://mongodb.github.io/helm-charts") + + # login to the OCI container registry + registry, repository, region = oci_chart_info() + try: + oci_helm_registry_login(registry, region) + except Exception as e: + raise e + + # figure out the registry URI, based on dev/staging scenario + chart_uri = f"oci://{registry}/{repository}" + if not helm_chart_path: + helm_chart_path = chart_uri + + if not custom_operator_version: + # most probably we are trying to install current operator which will be installed + # from OCI registry. The version (dev/staging) is set in `OPERATOR_VERSION` + custom_operator_version = os.environ.get("OPERATOR_VERSION") prepare_multi_cluster_namespaces( namespace, diff --git a/scripts/evergreen/e2e/single_e2e.sh b/scripts/evergreen/e2e/single_e2e.sh index 03776c5ad..d610df9c8 100755 --- a/scripts/evergreen/e2e/single_e2e.sh +++ b/scripts/evergreen/e2e/single_e2e.sh @@ -46,9 +46,9 @@ deploy_test_app() { BUILD_VARIANT="${BUILD_VARIANT:-default_build_variant}" chart_info=$(scripts/dev/run_python.sh scripts/release/oci_chart_info.py) - helm_oci_regisry=$(echo "$chart_info" | jq -r '.registry' ) - helm_oci_repository=$(echo "$chart_info" | jq -r '.repository' ) - helm_oci_registry_region=$(echo "$chart_info" | jq -r '.region' ) + helm_oci_regisry=$(echo "${chart_info}" | jq -r '.registry' ) + helm_oci_repository=$(echo "${chart_info}" | jq -r '.repository' ) + helm_oci_registry_region=$(echo "${chart_info}" | jq -r '.region' ) # note, that the 4 last parameters are used only for Mongodb resource testing - not for Ops Manager helm_params=( diff --git a/scripts/release/oci_chart_info.py b/scripts/release/oci_chart_info.py index 0e5b6bf54..c7c92ca71 100644 --- a/scripts/release/oci_chart_info.py +++ b/scripts/release/oci_chart_info.py @@ -1,11 +1,12 @@ +import json import os import sys -import json - from dataclasses import asdict + from lib.base_logger import logger from scripts.release.build.build_info import load_build_info + def main(): build_scenario = os.environ.get("BUILD_SCENARIO") build_info = load_build_info(build_scenario) @@ -15,7 +16,6 @@ def main(): print(j) - if __name__ == "__main__": try: main() From be67afe47e186a269cf249cc6f8b1e9f3f7504cb Mon Sep 17 00:00:00 2001 From: Vivek Singh Date: Thu, 16 Oct 2025 15:53:08 +0200 Subject: [PATCH 08/11] Add 0.0.0 while forming chart version --- docker/mongodb-kubernetes-tests/kubetester/operator.py | 3 ++- docker/mongodb-kubernetes-tests/tests/conftest.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/mongodb-kubernetes-tests/kubetester/operator.py b/docker/mongodb-kubernetes-tests/kubetester/operator.py index f75a321fb..87a0f7a4f 100644 --- a/docker/mongodb-kubernetes-tests/kubetester/operator.py +++ b/docker/mongodb-kubernetes-tests/kubetester/operator.py @@ -66,7 +66,8 @@ def __init__( if not operator_version: # most probably we are trying to install current operator which will be installed # from OCI registry. The version (dev/staging) is set in `OPERATOR_VERSION` - operator_version = os.environ.get("OPERATOR_VERSION") + non_semver_operator_version = os.environ.get("OPERATOR_VERSION") + operator_version = f"0.0.0+{non_semver_operator_version}" if helm_args is None: helm_args = {} diff --git a/docker/mongodb-kubernetes-tests/tests/conftest.py b/docker/mongodb-kubernetes-tests/tests/conftest.py index cb46e4755..0fdf7f405 100644 --- a/docker/mongodb-kubernetes-tests/tests/conftest.py +++ b/docker/mongodb-kubernetes-tests/tests/conftest.py @@ -856,7 +856,8 @@ def _install_multi_cluster_operator( if not custom_operator_version: # most probably we are trying to install current operator which will be installed # from OCI registry. The version (dev/staging) is set in `OPERATOR_VERSION` - custom_operator_version = os.environ.get("OPERATOR_VERSION") + non_semver_custom_operator_version = os.environ.get("OPERATOR_VERSION") + custom_operator_version = f"0.0.0+{non_semver_custom_operator_version}" prepare_multi_cluster_namespaces( namespace, From 03562d95932988b73f74d243a4946d96eec7bf2f Mon Sep 17 00:00:00 2001 From: Vivek Singh Date: Thu, 16 Oct 2025 19:08:56 +0200 Subject: [PATCH 09/11] Fix `e2e_meko_mck_upgrade` by using OCI version only when version is not provided and chart is not meko or mck --- .evergreen.yml | 2 ++ docker/mongodb-kubernetes-tests/kubetester/operator.py | 5 ++++- docker/mongodb-kubernetes-tests/tests/conftest.py | 4 ++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.evergreen.yml b/.evergreen.yml index 522ad4863..d57d4c3a1 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -37,6 +37,8 @@ variables: variant: init_test_run - name: build_init_om_images_ubi variant: init_test_run + - name: publish_helm_chart + variant: init_test_run - &base_no_om_image_dependency depends_on: diff --git a/docker/mongodb-kubernetes-tests/kubetester/operator.py b/docker/mongodb-kubernetes-tests/kubetester/operator.py index 87a0f7a4f..44a417dc2 100644 --- a/docker/mongodb-kubernetes-tests/kubetester/operator.py +++ b/docker/mongodb-kubernetes-tests/kubetester/operator.py @@ -51,6 +51,9 @@ def __init__( api_client: Optional[client.api_client.ApiClient] = None, operator_version: Optional[str] = None, ): + # The Operator will be installed from the following repo, so adding it first + helm_repo_add("mongodb", "https://mongodb.github.io/helm-charts") + if not helm_chart_path: # login to the OCI container registry registry, repository, region = oci_chart_info() @@ -63,7 +66,7 @@ def __init__( chart_uri = f"oci://{registry}/{repository}" helm_chart_path = chart_uri - if not operator_version: + if not operator_version and helm_chart_path not in ("mongodb/mongodb-kubernetes", "mongodb/enterprise-operator"): # most probably we are trying to install current operator which will be installed # from OCI registry. The version (dev/staging) is set in `OPERATOR_VERSION` non_semver_operator_version = os.environ.get("OPERATOR_VERSION") diff --git a/docker/mongodb-kubernetes-tests/tests/conftest.py b/docker/mongodb-kubernetes-tests/tests/conftest.py index 0fdf7f405..c9242b84d 100644 --- a/docker/mongodb-kubernetes-tests/tests/conftest.py +++ b/docker/mongodb-kubernetes-tests/tests/conftest.py @@ -839,7 +839,7 @@ def _install_multi_cluster_operator( multi_cluster_operator_installation_config.update(helm_opts) # The Operator will be installed from the following repo, so adding it first - # helm_repo_add("mongodb", "https://mongodb.github.io/helm-charts") + helm_repo_add("mongodb", "https://mongodb.github.io/helm-charts") # login to the OCI container registry registry, repository, region = oci_chart_info() @@ -853,7 +853,7 @@ def _install_multi_cluster_operator( if not helm_chart_path: helm_chart_path = chart_uri - if not custom_operator_version: + if not custom_operator_version and helm_chart_path not in (MCK_HELM_CHART, LEGACY_OPERATOR_CHART): # most probably we are trying to install current operator which will be installed # from OCI registry. The version (dev/staging) is set in `OPERATOR_VERSION` non_semver_custom_operator_version = os.environ.get("OPERATOR_VERSION") From 5f222bb282b994c4433dddf3e24e2f6cc85c09c3 Mon Sep 17 00:00:00 2001 From: Vivek Singh Date: Thu, 16 Oct 2025 19:10:25 +0200 Subject: [PATCH 10/11] Run precommit --- docker/mongodb-kubernetes-tests/kubetester/operator.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/mongodb-kubernetes-tests/kubetester/operator.py b/docker/mongodb-kubernetes-tests/kubetester/operator.py index 44a417dc2..955ef91da 100644 --- a/docker/mongodb-kubernetes-tests/kubetester/operator.py +++ b/docker/mongodb-kubernetes-tests/kubetester/operator.py @@ -66,7 +66,10 @@ def __init__( chart_uri = f"oci://{registry}/{repository}" helm_chart_path = chart_uri - if not operator_version and helm_chart_path not in ("mongodb/mongodb-kubernetes", "mongodb/enterprise-operator"): + if not operator_version and helm_chart_path not in ( + "mongodb/mongodb-kubernetes", + "mongodb/enterprise-operator", + ): # most probably we are trying to install current operator which will be installed # from OCI registry. The version (dev/staging) is set in `OPERATOR_VERSION` non_semver_operator_version = os.environ.get("OPERATOR_VERSION") From b485e0c8ebe14e7ce4e55deb5735e6a1ff273569 Mon Sep 17 00:00:00 2001 From: Vivek Singh Date: Thu, 16 Oct 2025 21:04:56 +0200 Subject: [PATCH 11/11] Fix CRDs not found issue by installing CRD from correct path --- docker/mongodb-kubernetes-tests/kubetester/helm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/mongodb-kubernetes-tests/kubetester/helm.py b/docker/mongodb-kubernetes-tests/kubetester/helm.py index 40533b2b6..87ef8001b 100644 --- a/docker/mongodb-kubernetes-tests/kubetester/helm.py +++ b/docker/mongodb-kubernetes-tests/kubetester/helm.py @@ -211,7 +211,8 @@ def helm_upgrade( chart_dir = helm_chart_path if helm_override_path else _helm_chart_dir(helm_chart_path) if apply_crds_first: - apply_crds_from_chart(chart_dir) + local_helm_chart_dir = "helm_chart" + apply_crds_from_chart(local_helm_chart_dir) command_args = _create_helm_args(helm_args, helm_options) args = [