Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .evergreen.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ variables:
variant: init_test_run
- name: build_init_om_images_ubi
variant: init_test_run
- name: publish_helm_chart
variant: init_test_run

- &base_no_om_image_dependency
depends_on:
Expand All @@ -54,6 +56,8 @@ variables:
variant: init_test_run
- name: build_init_appdb_images_ubi
variant: init_test_run
- name: publish_helm_chart
variant: init_test_run

- &community_dependency
depends_on:
Expand Down
13 changes: 10 additions & 3 deletions docker/mongodb-kubernetes-tests/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,8 @@ RUN apt-get -qq update \
libldap2-dev \
libsasl2-dev \
git \
openssl
openssl \
unzip

RUN mkdir -p /tmp/mongodb-tools && \
tar xfz /tmp/mongodb-tools.tgz -C /tmp/mongodb-tools && \
Expand All @@ -66,6 +67,11 @@ RUN curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -
&& chmod +x ./kubectl \
&& mv ./kubectl /usr/local/bin/kubectl

# install aws, required to run helm registry login while running the tests
RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \
&& unzip -q awscliv2.zip \
&& ./aws/install

COPY --from=builder /venv /venv

ENV PATH="/venv/bin:${PATH}"
Expand All @@ -75,8 +81,9 @@ WORKDIR /tests

# copying the test files after python build, otherwise pip install will be called each time the tests change
COPY . /tests
# copying the helm_chart directory as well to support installation of the Operator from the test application
COPY helm_chart /helm_chart
# copying the helm_chart/crds directory so that we can install CRDs before installing the operator helm chart to run a test.
# operator is installed via published OCI helm repo and not the local helm repo.
COPY helm_chart/crds /helm_chart/crds
COPY release.json /release.json
# we use the public directory to automatically test resources samples
COPY public /mongodb-kubernetes/public
Expand Down
62 changes: 61 additions & 1 deletion docker/mongodb-kubernetes-tests/kubetester/helm.py
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,55 @@ def process_run_and_check(args, **kwargs):
raise


def oci_helm_registry_login(helm_registry: str, region: str):
logger.info(f"Attempting to log into ECR registry: {helm_registry}, using helm registry login.")

aws_command = ["aws", "ecr", "get-login-password", "--region", region]

# as we can see the password is being provided by stdin, that would mean we will have to
# pipe the aws_command (it figures out the password) into helm_command.
helm_command = ["helm", "registry", "login", "--username", "AWS", "--password-stdin", helm_registry]

try:
logger.info("Starting AWS ECR credential retrieval.")
aws_proc = subprocess.Popen(
aws_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True # Treat input/output as text strings
)

logger.info("Starting Helm registry login.")
helm_proc = subprocess.Popen(
helm_command, stdin=aws_proc.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
)

# Close the stdout stream of aws_proc in the parent process
# to prevent resource leakage (only needed if you plan to do more processing)
aws_proc.stdout.close()

# Wait for the Helm command (helm_proc) to finish and capture its output
helm_stdout, helm_stderr = helm_proc.communicate()

# Wait for the AWS process to finish as well
aws_proc.wait()

if aws_proc.returncode != 0:
_, aws_stderr = aws_proc.communicate()
raise Exception(f"aws command to get password failed. Error: {aws_stderr}")

if helm_proc.returncode == 0:
logger.info("Login to helm registry was successful.")
logger.info(helm_stdout.strip())
else:
raise Exception(
f"Login to helm registry failed, Exit code: {helm_proc.returncode}, Error: {helm_stderr.strip()}"
)

except FileNotFoundError as e:
# This catches errors if 'aws' or 'helm' are not in the PATH
raise Exception(f"Command not found. Please ensure '{e.filename}' is installed and in your system's PATH.")
except Exception as e:
raise Exception(f"An unexpected error occurred: {e}.")


def helm_upgrade(
name: str,
namespace: str,
Expand All @@ -162,7 +211,8 @@ def helm_upgrade(
chart_dir = helm_chart_path if helm_override_path else _helm_chart_dir(helm_chart_path)

if apply_crds_first:
apply_crds_from_chart(chart_dir)
local_helm_chart_dir = "helm_chart"
apply_crds_from_chart(local_helm_chart_dir)

command_args = _create_helm_args(helm_args, helm_options)
args = [
Expand All @@ -183,6 +233,16 @@ def helm_upgrade(
process_run_and_check(command, check=True, capture_output=True, shell=True)


def oci_chart_info():
registry = os.environ.get("OCI_HELM_REGISTRY")
repository = os.environ.get("OCI_HELM_REPOSITORY")
region = os.environ.get("OCI_HELM_REGION")

print(f"oci chart details in test image is registry {registry}, repo {repository}, region {region}")

return registry, f"{repository}/mongodb-kubernetes", region


def apply_crds_from_chart(chart_dir: str):
crd_files = glob.glob(os.path.join(chart_dir, "crds", "*.yaml"))

Expand Down
35 changes: 33 additions & 2 deletions docker/mongodb-kubernetes-tests/kubetester/operator.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from __future__ import annotations

import logging
import os
import time
from typing import Dict, List, Optional

Expand All @@ -16,6 +17,8 @@
helm_template,
helm_uninstall,
helm_upgrade,
oci_chart_info,
oci_helm_registry_login,
)
from tests import test_logger

Expand Down Expand Up @@ -43,14 +46,35 @@ def __init__(
namespace: str,
helm_args: Optional[Dict] = None,
helm_options: Optional[List[str]] = None,
helm_chart_path: Optional[str] = "helm_chart",
helm_chart_path: Optional[str] = None,
name: Optional[str] = "mongodb-kubernetes-operator",
api_client: Optional[client.api_client.ApiClient] = None,
operator_version: Optional[str] = None,
):

# The Operator will be installed from the following repo, so adding it first
helm_repo_add("mongodb", "https://mongodb.github.io/helm-charts")

if not helm_chart_path:
# login to the OCI container registry
registry, repository, region = oci_chart_info()
try:
oci_helm_registry_login(registry, region)
except Exception as e:
raise e

# figure out the registry URI, based on dev/staging scenario
chart_uri = f"oci://{registry}/{repository}"
helm_chart_path = chart_uri

if not operator_version and helm_chart_path not in (
"mongodb/mongodb-kubernetes",
"mongodb/enterprise-operator",
):
# most probably we are trying to install current operator which will be installed
# from OCI registry. The version (dev/staging) is set in `OPERATOR_VERSION`
non_semver_operator_version = os.environ.get("OPERATOR_VERSION")
operator_version = f"0.0.0+{non_semver_operator_version}"

if helm_args is None:
helm_args = {}

Expand All @@ -69,6 +93,7 @@ def __init__(
self.helm_chart_path = helm_chart_path
self.name = name
self.api_client = api_client
self.operator_version = operator_version

def install_from_template(self):
"""Uses helm to generate yaml specification and then uses python K8s client to apply them to the cluster
Expand All @@ -82,6 +107,9 @@ def install_from_template(self):

def install(self, custom_operator_version: Optional[str] = None) -> Operator:
"""Installs the Operator to Kubernetes cluster using 'helm install', waits until it's running"""
if not custom_operator_version:
custom_operator_version = self.operator_version

helm_install(
self.name,
self.namespace,
Expand All @@ -99,6 +127,9 @@ def upgrade(
self, multi_cluster: bool = False, custom_operator_version: Optional[str] = None, apply_crds_first: bool = False
) -> Operator:
"""Upgrades the Operator in Kubernetes cluster using 'helm upgrade', waits until it's running"""
if not custom_operator_version:
custom_operator_version = self.operator_version

helm_upgrade(
self.name,
self.namespace,
Expand Down
27 changes: 25 additions & 2 deletions docker/mongodb-kubernetes-tests/tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,12 @@
update_configmap,
)
from kubetester.awss3client import AwsS3Client
from kubetester.helm import helm_install_from_chart, helm_repo_add
from kubetester.helm import (
helm_install_from_chart,
helm_repo_add,
oci_chart_info,
oci_helm_registry_login,
)
from kubetester.kubetester import KubernetesTester
from kubetester.kubetester import fixture as _fixture
from kubetester.kubetester import running_locally
Expand Down Expand Up @@ -827,7 +832,7 @@ def _install_multi_cluster_operator(
helm_opts: dict[str, str],
central_cluster_name: str,
operator_name: Optional[str] = MULTI_CLUSTER_OPERATOR_NAME,
helm_chart_path: Optional[str] = LOCAL_HELM_CHART_DIR,
helm_chart_path: Optional[str] = None,
custom_operator_version: Optional[str] = None,
apply_crds_first: bool = False,
) -> Operator:
Expand All @@ -836,6 +841,24 @@ def _install_multi_cluster_operator(
# The Operator will be installed from the following repo, so adding it first
helm_repo_add("mongodb", "https://mongodb.github.io/helm-charts")

# login to the OCI container registry
registry, repository, region = oci_chart_info()
try:
oci_helm_registry_login(registry, region)
except Exception as e:
raise e

# figure out the registry URI, based on dev/staging scenario
chart_uri = f"oci://{registry}/{repository}"
if not helm_chart_path:
helm_chart_path = chart_uri

if not custom_operator_version and helm_chart_path not in (MCK_HELM_CHART, LEGACY_OPERATOR_CHART):
# most probably we are trying to install current operator which will be installed
# from OCI registry. The version (dev/staging) is set in `OPERATOR_VERSION`
non_semver_custom_operator_version = os.environ.get("OPERATOR_VERSION")
custom_operator_version = f"0.0.0+{non_semver_custom_operator_version}"

prepare_multi_cluster_namespaces(
namespace,
multi_cluster_operator_installation_config,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,22 @@ spec:
- name: OM_DEBUG_HTTP
value: "{{ .Values.omDebugHttp }}"
{{ end }}
{{ if .Values.helm.oci.registry }}
- name: OCI_HELM_REGISTRY
value: "{{ .Values.helm.oci.registry }}"
{{ end }}
{{ if .Values.operator.version }}
- name: OPERATOR_VERSION
value: "{{ .Values.operator.version }}"
{{ end }}
{{ if .Values.helm.oci.repository }}
- name: OCI_HELM_REPOSITORY
value: "{{ .Values.helm.oci.repository }}"
{{ end }}
{{ if .Values.helm.oci.region }}
- name: OCI_HELM_REGION
value: "{{ .Values.helm.oci.region }}"
{{ end }}
- name: ops_manager_version
value: "{{ .Values.opsManagerVersion }}"
- name: cognito_user_pool_id
Expand Down
11 changes: 11 additions & 0 deletions scripts/evergreen/deployments/test-app/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -43,3 +43,14 @@ mdbImageType: "ubi8"

# set to "true" to set OM_DEBUG_HTTP=true for the operator
omDebugHttp:

# sets the configuration using which we can figure out the helm OCI repo of the MCK operator while deploying it
# to run a test.
helm:
oci:
registry: ""
repository: ""
region: ""

operator:
version: ""
21 changes: 21 additions & 0 deletions scripts/evergreen/e2e/single_e2e.sh
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,11 @@ deploy_test_app() {
BUILD_ID="${BUILD_ID:-default_build_id}"
BUILD_VARIANT="${BUILD_VARIANT:-default_build_variant}"

chart_info=$(scripts/dev/run_python.sh scripts/release/oci_chart_info.py)
helm_oci_regisry=$(echo "${chart_info}" | jq -r '.registry' )
helm_oci_repository=$(echo "${chart_info}" | jq -r '.repository' )
helm_oci_registry_region=$(echo "${chart_info}" | jq -r '.region' )

# note, that the 4 last parameters are used only for Mongodb resource testing - not for Ops Manager
helm_params=(
"--set" "taskId=${task_id:-'not-specified'}"
Expand Down Expand Up @@ -142,6 +147,22 @@ deploy_test_app() {
helm_params+=("--set" "omDebugHttp=true")
fi

if [[ -n "${helm_oci_regisry:-}" ]]; then
helm_params+=("--set" "helm.oci.registry=${helm_oci_regisry}")
fi

if [[ -n "${helm_oci_repository:-}" ]]; then
helm_params+=("--set" "helm.oci.repository=${helm_oci_repository}")
fi

if [[ -n "${helm_oci_registry_region:-}" ]]; then
helm_params+=("--set" "helm.oci.region=${helm_oci_registry_region}")
fi

if [[ -n "${OPERATOR_VERSION:-}" ]]; then
helm_params+=("--set" "operator.version=${OPERATOR_VERSION}")
fi

helm_params+=("--set" "opsManagerVersion=${ops_manager_version}")

helm template "scripts/evergreen/deployments/test-app" "${helm_params[@]}" > "${helm_template_file}" || exit 1
Expand Down
24 changes: 24 additions & 0 deletions scripts/release/oci_chart_info.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
import json
import os
import sys
from dataclasses import asdict

from lib.base_logger import logger
from scripts.release.build.build_info import load_build_info


def main():
build_scenario = os.environ.get("BUILD_SCENARIO")
build_info = load_build_info(build_scenario)
chart_info = build_info.helm_charts["mongodb-kubernetes"]

j = json.dumps(asdict(chart_info))
print(j)


if __name__ == "__main__":
try:
main()
except Exception as e:
logger.error(f"Failed while dumping the chart_info as json. Error: {e}")
sys.exit(1)