diff --git a/.evergreen-functions.yml b/.evergreen-functions.yml index 1abd6b451..2b7cac62d 100644 --- a/.evergreen-functions.yml +++ b/.evergreen-functions.yml @@ -490,12 +490,53 @@ functions: - rh_pyxis binary: scripts/dev/run_python.sh scripts/preflight_images.py --image ${image_name} --submit "${preflight_submit}" + install_macnotary: + - command: shell.exec + type: setup + params: + include_expansions_in_env: + - notary_service_url + script: | + set -Eeu pipefail + + curl "${notary_service_url}" --output macos-notary.zip + unzip -u macos-notary.zip + chmod 755 ./linux_amd64/macnotary + + install_goreleaser: + - command: shell.exec + type: setup + include_expansions_in_env: + - goreleaser_pro_tar_gz + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + script: | + set -Eeu pipefail + curl -fL "${goreleaser_pro_tar_gz}" --output goreleaser_Linux_x86_64.tar.gz + tar -xf goreleaser_Linux_x86_64.tar.gz + chmod 755 ./goreleaser + build_multi_cluster_binary: - command: subprocess.exec type: setup params: + include_expansions_in_env: + - github_commit + - GRS_USERNAME + - GRS_PASSWORD + - PKCS11_URI + - ARTIFACTORY_URL + - ARTIFACTORY_PASSWORD + - SIGNING_IMAGE_URI + - macos_notary_keyid + - macos_notary_secret + - workdir + - triggered_by_git_tag + env: + MACOS_NOTARY_KEY: ${macos_notary_keyid} + MACOS_NOTARY_SECRET: ${macos_notary_secret} working_dir: src/github.com/mongodb/mongodb-kubernetes - binary: scripts/evergreen/build_multi_cluster_kubeconfig_creator.sh + binary: scripts/dev/run_python.sh scripts/release/kubectl-mongodb/python/build_kubectl_plugin.py build_and_push_appdb_database: - command: subprocess.exec diff --git a/.evergreen.yml b/.evergreen.yml index f2e9fbb7e..ed87af213 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -83,6 +83,8 @@ variables: - func: clone - func: download_kube_tools - func: setup_building_host + - func: install_goreleaser + - func: install_macnotary - func: build_multi_cluster_binary - &setup_and_teardown_group_gke_code_snippets @@ -92,6 +94,8 @@ variables: - func: setup_gcloud_cli - func: setup_mongosh - func: download_kube_tools + - func: install_goreleaser + - func: install_macnotary - func: build_multi_cluster_binary teardown_group: - func: upload_code_snippets_logs @@ -394,6 +398,8 @@ tasks: commands: - func: clone - func: setup_building_host + - func: install_goreleaser + - func: install_macnotary - func: build_multi_cluster_binary - func: pipeline vars: diff --git a/build_info.json b/build_info.json index 286667428..35e58a964 100644 --- a/build_info.json +++ b/build_info.json @@ -279,14 +279,14 @@ "binaries": { "kubectl-mongodb": { "patch": { - "s3-store": "s3://kubectl-mongodb/dev", + "s3-store": "mongodb-kubernetes-dev", "platforms": [ "linux/amd64" ] }, "staging": { - "sign": true, - "s3-store": "s3://kubectl-mongodb/staging", + "sign": false, + "s3-store": "mongodb-kubernetes-staging", "platforms": [ "darwin/amd64", "darwin/arm64", @@ -296,7 +296,7 @@ }, "release": { "sign": true, - "s3-store": "s3://kubectl-mongodb/prod", + "s3-store": "mongodb-kubernetes-release", "platforms": [ "darwin/amd64", "darwin/arm64", diff --git a/scripts/release/kubectl-mongodb/python/build_kubectl_plugin.py b/scripts/release/kubectl-mongodb/python/build_kubectl_plugin.py new file mode 100755 index 000000000..71d56ab57 --- /dev/null +++ b/scripts/release/kubectl-mongodb/python/build_kubectl_plugin.py @@ -0,0 +1,115 @@ +import os +import sys + +import boto3 +import utils +from botocore.exceptions import ClientError, NoCredentialsError, PartialCredentialsError + +from lib.base_logger import logger +from scripts.release.build.build_info import ( + load_build_info, +) +from scripts.release.build.build_scenario import ( + BuildScenario, +) + +AWS_REGION = "eu-north-1" +S3_BUCKET_KUBECTL_PLUGIN_SUBPATH = utils.KUBECTL_PLUGIN_BINARY_NAME + +GORELEASER_DIST_DIR = "dist" + +# LOCAL_KUBECTL_PLUGIN_PATH is the full filename where tests image expects the kuebctl-mongodb binary to be available +LOCAL_KUBECTL_PLUGIN_PATH = "docker/mongodb-kubernetes-tests/multi-cluster-kube-config-creator_linux" + + +def run_goreleaser_build(): + + command = ["./goreleaser", "build", "--snapshot", "--clean", "--skip", "post-hooks"] + utils.run_goreleaser_command(command) + + +# upload_artifacts_to_s3 uploads the artifacts that are generated by goreleaser to S3 bucket at a specific path. +# The S3 bucket and version are figured out and passed to this function based on BuildScenario. +def upload_artifacts_to_s3(s3_bucket: str, version: str): + if not os.path.isdir(GORELEASER_DIST_DIR): + logger.info(f"ERROR: GoReleaser dist directory '{GORELEASER_DIST_DIR}' not found.") + sys.exit(1) + + try: + s3_client = boto3.client("s3", region_name=AWS_REGION) + except (NoCredentialsError, PartialCredentialsError): + logger.debug("ERROR: Failed to create S3 client. AWS credentials not found.") + sys.exit(1) + except Exception as e: + logger.debug(f"An error occurred connecting to S3: {e}") + sys.exit(1) + + uploaded_files = 0 + # iterate over all the files generated by goreleaser in the dist directory and upload them to S3 + for root, _, files in os.walk(GORELEASER_DIST_DIR): + for filename in files: + local_path = os.path.join(root, filename) + s3_key = s3_path(local_path, version) + + logger.info(f"Uploading artifact {local_path} to s3://{s3_bucket}/{s3_key}") + try: + s3_client.upload_file(local_path, s3_bucket, s3_key) + logger.info(f"Successfully uploaded the artifact {filename}") + uploaded_files += 1 + except Exception as e: + logger.debug(f"ERROR: Failed to upload file {filename}: {e}") + + if uploaded_files > 0: + logger.info(f"Successfully uploaded {uploaded_files} kubectl-mongodb plugin artifacts to S3.") + + +# s3_path returns the path where the artifacts should be uploaded to in S3 object store. +# For dev workflows it's going to be `kubectl-mongodb/{evg-patch-id}/{goreleaser-artifact}`, +# for staging workflows it would be `kubectl-mongodb/{commit-sha}/{goreleaser-artifact}`. +# The `version` string has the correct version (either patch id or commit sha), based on the BuildScenario. +def s3_path(local_path: str, version: str): + return f"{S3_BUCKET_KUBECTL_PLUGIN_SUBPATH}/{version}/{local_path}" + + +# download_plugin_for_tests_image downloads just the linux amd64 version of the binary and places it +# at the location LOCAL_KUBECTL_PLUGIN_PATH. +def download_plugin_for_tests_image(build_scenario: BuildScenario, s3_bucket: str, version: str): + try: + s3_client = boto3.client("s3", region_name=AWS_REGION) + except Exception as e: + logger.debug(f"An error occurred connecting to S3 to download kubectl plugin for tests image: {e}") + return + + plugin_path = f"{S3_BUCKET_KUBECTL_PLUGIN_SUBPATH}/{version}/dist/kubectl-mongodb_linux_amd64_v1/kubectl-mongodb" + + logger.info(f"Downloading s3://{s3_bucket}/{plugin_path} to {LOCAL_KUBECTL_PLUGIN_PATH}") + try: + s3_client.download_file(s3_bucket, plugin_path, LOCAL_KUBECTL_PLUGIN_PATH) + # change the file's permissions to make file executable + os.chmod(LOCAL_KUBECTL_PLUGIN_PATH, 0o755) + + logger.info(f"Successfully downloaded artifact to {LOCAL_KUBECTL_PLUGIN_PATH}") + except ClientError as e: + if e.response["Error"]["Code"] == "404": + logger.debug(f"ERROR: Artifact not found at s3://{s3_bucket}/{plugin_path} ") + else: + logger.debug(f"ERROR: Failed to download artifact. S3 Client Error: {e}") + except Exception as e: + logger.debug(f"An unexpected error occurred during download: {e}") + + +def main(): + build_scenario = BuildScenario.infer_scenario_from_environment() + kubectl_plugin_build_info = load_build_info(build_scenario).binaries[utils.KUBECTL_PLUGIN_BINARY_NAME] + + run_goreleaser_build() + + upload_artifacts_to_s3(kubectl_plugin_build_info.s3_store, kubectl_plugin_build_info.version) + + download_plugin_for_tests_image( + build_scenario, kubectl_plugin_build_info.s3_store, kubectl_plugin_build_info.version + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/release/kubectl-mongodb/python/release_kubectl_plugin.py b/scripts/release/kubectl-mongodb/python/release_kubectl_plugin.py new file mode 100644 index 000000000..5f385db5e --- /dev/null +++ b/scripts/release/kubectl-mongodb/python/release_kubectl_plugin.py @@ -0,0 +1,34 @@ +import build_kubectl_plugin +import utils + +from scripts.release.build.build_info import ( + load_build_info, +) +from scripts.release.build.build_scenario import ( + BuildScenario, +) + + +def run_goreleaser_release(): + + command = ["./goreleaser", "release", "--clean"] + utils.run_goreleaser_command(command) + + +# upload_artifacts uploads the artifacts that are generated by the `goreleaser release` +# command to the release S3 bucket. +def upload_artifacts(s3_bucket: str, version: str): + build_kubectl_plugin.upload_artifacts_to_s3(s3_bucket, version) + + +def main(): + run_goreleaser_release() + + build_scenario = BuildScenario.infer_scenario_from_environment() + kubectl_plugin_build_info = load_build_info(build_scenario).binaries[utils.KUBECTL_PLUGIN_BINARY_NAME] + + upload_artifacts(kubectl_plugin_build_info.s3_store, kubectl_plugin_build_info.version) + + +if __name__ == "__main__": + main() diff --git a/scripts/release/kubectl-mongodb/python/utils.py b/scripts/release/kubectl-mongodb/python/utils.py new file mode 100644 index 000000000..2c984ba52 --- /dev/null +++ b/scripts/release/kubectl-mongodb/python/utils.py @@ -0,0 +1,32 @@ +import subprocess +import sys + +from lib.base_logger import logger + +KUBECTL_PLUGIN_BINARY_NAME = "kubectl-mongodb" + + +def run_goreleaser_command(command: list[str]): + try: + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1) + + for log in iter(process.stdout.readline, ""): + print(log, end="") + + process.stdout.close() + exit_code = process.wait() + + if exit_code != 0: + logger.debug(f"GoReleaser command failed with exit code {exit_code}.") + sys.exit(1) + + logger.info("GoReleaser build completed successfully!") + + except FileNotFoundError: + logger.debug( + "ERROR: 'goreleaser' command not found. Please ensure goreleaser is installed and in your system's PATH." + ) + sys.exit(1) + except Exception as e: + logger.debug(f"An unexpected error occurred while running `goreleaser build`: {e}") + sys.exit(1) diff --git a/scripts/release/tests/build_info_test.py b/scripts/release/tests/build_info_test.py index bef97173d..24674a381 100644 --- a/scripts/release/tests/build_info_test.py +++ b/scripts/release/tests/build_info_test.py @@ -104,7 +104,7 @@ def test_load_build_info_development(git_repo: Repo): }, binaries={ "kubectl-mongodb": BinaryInfo( - s3_store="s3://kubectl-mongodb/dev", + s3_store="mongodb-kubernetes-dev", platforms=["linux/amd64"], version=version, sign=False, @@ -217,7 +217,7 @@ def test_load_build_info_patch(git_repo: Repo): }, binaries={ "kubectl-mongodb": BinaryInfo( - s3_store="s3://kubectl-mongodb/dev", + s3_store="mongodb-kubernetes-dev", platforms=["linux/amd64"], version=patch_id, sign=False, @@ -331,10 +331,10 @@ def test_load_build_info_staging(git_repo: Repo): }, binaries={ "kubectl-mongodb": BinaryInfo( - s3_store="s3://kubectl-mongodb/staging", + s3_store="mongodb-kubernetes-staging", platforms=["darwin/amd64", "darwin/arm64", "linux/amd64", "linux/arm64"], version=expected_commit_sha, - sign=True, + sign=False, ) }, helm_charts={ @@ -425,7 +425,7 @@ def test_load_build_info_release( }, binaries={ "kubectl-mongodb": BinaryInfo( - s3_store="s3://kubectl-mongodb/prod", + s3_store="mongodb-kubernetes-release", platforms=["darwin/amd64", "darwin/arm64", "linux/amd64", "linux/arm64"], version=version, sign=True,