diff --git a/.evergreen/config.yml b/.evergreen/config.yml index a297c49162..d83a5620df 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -25,6 +25,7 @@ timeout: binary: ls -la include: + - filename: .evergreen/generated_configs/functions.yml - filename: .evergreen/generated_configs/tasks.yml - filename: .evergreen/generated_configs/variants.yml @@ -52,147 +53,6 @@ functions: params: file: src/expansion.yml - "upload coverage" : - - command: ec2.assume_role - params: - role_arn: ${assume_role_arn} - - command: s3.put - params: - aws_key: ${AWS_ACCESS_KEY_ID} - aws_secret: ${AWS_SECRET_ACCESS_KEY} - aws_session_token: ${AWS_SESSION_TOKEN} - local_file: src/.coverage - optional: true - # Upload the coverage report for all tasks in a single build to the same directory. - remote_file: coverage/${revision}/${version_id}/coverage/coverage.${build_variant}.${task_name} - bucket: ${bucket_name} - permissions: public-read - content_type: text/html - display_name: "Raw Coverage Report" - - "download and merge coverage" : - - command: ec2.assume_role - params: - role_arn: ${assume_role_arn} - - command: subprocess.exec - params: - silent: true - binary: bash - working_dir: "src" - include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] - args: - - .evergreen/scripts/download-and-merge-coverage.sh - - ${bucket_name} - - ${revision} - - ${version_id} - - command: subprocess.exec - params: - working_dir: "src" - binary: bash - args: - - .evergreen/combine-coverage.sh - # Upload the resulting html coverage report. - - command: subprocess.exec - params: - silent: true - binary: bash - working_dir: "src" - include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] - args: - - .evergreen/scripts/upload-coverage-report.sh - - ${bucket_name} - - ${revision} - - ${version_id} - # Attach the index.html with s3.put so it shows up in the Evergreen UI. - - command: s3.put - params: - aws_key: ${AWS_ACCESS_KEY_ID} - aws_secret: ${AWS_SECRET_ACCESS_KEY} - aws_session_token: ${AWS_SESSION_TOKEN} - local_file: src/htmlcov/index.html - remote_file: coverage/${revision}/${version_id}/htmlcov/index.html - bucket: ${bucket_name} - permissions: public-read - content_type: text/html - display_name: "Coverage Report HTML" - - "upload mo artifacts": - - command: ec2.assume_role - params: - role_arn: ${assume_role_arn} - - command: archive.targz_pack - params: - target: "mongo-coredumps.tgz" - source_dir: "./" - include: - - "./**.core" - - "./**.mdmp" # Windows: minidumps - - command: s3.put - params: - aws_key: ${AWS_ACCESS_KEY_ID} - aws_secret: ${AWS_SECRET_ACCESS_KEY} - aws_session_token: ${AWS_SESSION_TOKEN} - local_file: mongo-coredumps.tgz - remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/coredumps/${task_id}-${execution}-mongodb-coredumps.tar.gz - bucket: ${bucket_name} - permissions: public-read - content_type: ${content_type|application/gzip} - display_name: Core Dumps - Execution - optional: true - - command: s3.put - params: - aws_key: ${AWS_ACCESS_KEY_ID} - aws_secret: ${AWS_SECRET_ACCESS_KEY} - aws_session_token: ${AWS_SESSION_TOKEN} - local_file: ${DRIVERS_TOOLS}/.evergreen/test_logs.tar.gz - remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-drivers-tools-logs.tar.gz - bucket: ${bucket_name} - permissions: public-read - content_type: ${content_type|application/x-gzip} - display_name: "drivers-tools-logs.tar.gz" - - "upload working dir": - - command: ec2.assume_role - params: - role_arn: ${assume_role_arn} - - command: archive.targz_pack - params: - target: "working-dir.tar.gz" - source_dir: ${PROJECT_DIRECTORY}/ - include: - - "./**" - - command: s3.put - params: - aws_key: ${AWS_ACCESS_KEY_ID} - aws_secret: ${AWS_SECRET_ACCESS_KEY} - aws_session_token: ${AWS_SESSION_TOKEN} - local_file: working-dir.tar.gz - remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/artifacts/${task_id}-${execution}-working-dir.tar.gz - bucket: ${bucket_name} - permissions: public-read - content_type: ${content_type|application/x-gzip} - display_name: "working-dir.tar.gz" - - command: archive.targz_pack - params: - target: "drivers-dir.tar.gz" - source_dir: ${DRIVERS_TOOLS} - include: - - "./**" - exclude_files: - # Windows cannot read the mongod *.lock files because they are locked. - - "*.lock" - - command: s3.put - params: - aws_key: ${AWS_ACCESS_KEY_ID} - aws_secret: ${AWS_SECRET_ACCESS_KEY} - aws_session_token: ${AWS_SESSION_TOKEN} - local_file: drivers-dir.tar.gz - remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/artifacts/${task_id}-${execution}-drivers-dir.tar.gz - bucket: ${bucket_name} - permissions: public-read - content_type: ${content_type|application/x-gzip} - display_name: "drivers-dir.tar.gz" - "upload test results": - command: attach.results params: diff --git a/.evergreen/generated_configs/functions.yml b/.evergreen/generated_configs/functions.yml new file mode 100644 index 0000000000..afd7f11374 --- /dev/null +++ b/.evergreen/generated_configs/functions.yml @@ -0,0 +1,117 @@ +functions: + # Download and merge coverage + download and merge coverage: + - command: ec2.assume_role + params: + role_arn: ${assume_role_arn} + type: setup + - command: subprocess.exec + params: + binary: bash + args: + - .evergreen/scripts/download-and-merge-coverage.sh + - ${bucket_name} + - ${revision} + - ${version_id} + working_dir: src + silent: true + include_expansions_in_env: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN + type: test + - command: subprocess.exec + params: + binary: bash + args: + - .evergreen/combine-coverage.sh + working_dir: src + type: test + - command: subprocess.exec + params: + binary: bash + args: + - .evergreen/scripts/upload-coverage-report.sh + - ${bucket_name} + - ${revision} + - ${version_id} + working_dir: src + silent: true + include_expansions_in_env: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN + type: test + - command: s3.put + params: + remote_file: coverage/${revision}/${version_id}/htmlcov/index.html + aws_key: ${AWS_ACCESS_KEY_ID} + aws_secret: ${AWS_SECRET_ACCESS_KEY} + aws_session_token: ${AWS_SESSION_TOKEN} + bucket: ${bucket_name} + local_file: src/htmlcov/index.html + permissions: public-read + content_type: text/html + display_name: Coverage Report HTML + optional: "true" + type: setup + + # Upload coverage + upload coverage: + - command: ec2.assume_role + params: + role_arn: ${assume_role_arn} + type: setup + - command: s3.put + params: + remote_file: coverage/${revision}/${version_id}/coverage/coverage.${build_variant}.${task_name} + aws_key: ${AWS_ACCESS_KEY_ID} + aws_secret: ${AWS_SECRET_ACCESS_KEY} + aws_session_token: ${AWS_SESSION_TOKEN} + bucket: ${bucket_name} + local_file: src/.coverage + permissions: public-read + content_type: text/html + display_name: Raw Coverage Report + optional: "true" + type: setup + + # Upload mo artifacts + upload mo artifacts: + - command: ec2.assume_role + params: + role_arn: ${assume_role_arn} + type: setup + - command: archive.targz_pack + params: + target: mongo-coredumps.tgz + source_dir: ./ + include: + - ./**.core + - ./**.mdmp + - command: s3.put + params: + remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/coredumps/${task_id}-${execution}-mongodb-coredumps.tar.gz + aws_key: ${AWS_ACCESS_KEY_ID} + aws_secret: ${AWS_SECRET_ACCESS_KEY} + aws_session_token: ${AWS_SESSION_TOKEN} + bucket: ${bucket_name} + local_file: mongo-coredumps.tgz + permissions: public-read + content_type: ${content_type|application/x-gzip} + display_name: Core Dumps - Execution + optional: "true" + type: setup + - command: s3.put + params: + remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-drivers-tools-logs.tar.gz + aws_key: ${AWS_ACCESS_KEY_ID} + aws_secret: ${AWS_SECRET_ACCESS_KEY} + aws_session_token: ${AWS_SESSION_TOKEN} + bucket: ${bucket_name} + local_file: ${DRIVERS_TOOLS}/.evergreen/test_logs.tar.gz + permissions: public-read + content_type: ${content_type|application/x-gzip} + display_name: drivers-tools-logs.tar.gz + optional: "true" + type: setup diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 723ef6ba31..5a5f6e93db 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -9,7 +9,14 @@ from typing import Any from shrub.v3.evg_build_variant import BuildVariant -from shrub.v3.evg_command import EvgCommandType, FunctionCall, subprocess_exec +from shrub.v3.evg_command import ( + EvgCommandType, + FunctionCall, + archive_targz_pack, + ec2_assume_role, + s3_put, + subprocess_exec, +) from shrub.v3.evg_project import EvgProject from shrub.v3.evg_task import EvgTask, EvgTaskDependency, EvgTaskRef from shrub.v3.shrub_service import ShrubService @@ -233,6 +240,12 @@ def handle_c_ext(c_ext, expansions) -> None: expansions["NO_EXT"] = "1" +def get_assume_role(**kwargs): + kwargs.setdefault("command_type", EvgCommandType.SETUP) + kwargs.setdefault("role_arn", "${assume_role_arn}") + return ec2_assume_role(**kwargs) + + def get_subprocess_exec(**kwargs): kwargs.setdefault("binary", "bash") kwargs.setdefault("working_dir", "src") @@ -240,6 +253,18 @@ def get_subprocess_exec(**kwargs): return subprocess_exec(**kwargs) +def get_s3_put(**kwargs): + kwargs["aws_key"] = "${AWS_ACCESS_KEY_ID}" + kwargs["aws_secret"] = "${AWS_SECRET_ACCESS_KEY}" # noqa:S105 + kwargs["aws_session_token"] = "${AWS_SESSION_TOKEN}" # noqa:S105 + kwargs["bucket"] = "${bucket_name}" + kwargs.setdefault("optional", "true") + kwargs.setdefault("permissions", "public-read") + kwargs.setdefault("content_type", "${content_type|application/x-gzip}") + kwargs.setdefault("command_type", EvgCommandType.SETUP) + return s3_put(**kwargs) + + def generate_yaml(tasks=None, variants=None): """Generate the yaml for a given set of tasks and variants.""" project = EvgProject(tasks=tasks, buildvariants=variants) @@ -1193,6 +1218,79 @@ def create_serverless_tasks(): return [EvgTask(name=task_name, tags=tags, commands=[test_func])] +############## +# Functions +############## + + +def create_upload_coverage_func(): + # Upload the coverage report for all tasks in a single build to the same directory. + remote_file = ( + "coverage/${revision}/${version_id}/coverage/coverage.${build_variant}.${task_name}" + ) + display_name = "Raw Coverage Report" + cmd = get_s3_put( + local_file="src/.coverage", + remote_file=remote_file, + display_name=display_name, + content_type="text/html", + ) + return "upload coverage", [get_assume_role(), cmd] + + +def create_download_and_merge_coverage_func(): + include_expansions = ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + args = [ + ".evergreen/scripts/download-and-merge-coverage.sh", + "${bucket_name}", + "${revision}", + "${version_id}", + ] + merge_cmd = get_subprocess_exec( + silent=True, include_expansions_in_env=include_expansions, args=args + ) + combine_cmd = get_subprocess_exec(args=[".evergreen/combine-coverage.sh"]) + # Upload the resulting html coverage report. + args = [ + ".evergreen/scripts/upload-coverage-report.sh", + "${bucket_name}", + "${revision}", + "${version_id}", + ] + upload_cmd = get_subprocess_exec( + silent=True, include_expansions_in_env=include_expansions, args=args + ) + display_name = "Coverage Report HTML" + remote_file = "coverage/${revision}/${version_id}/htmlcov/index.html" + put_cmd = get_s3_put( + local_file="src/htmlcov/index.html", + remote_file=remote_file, + display_name=display_name, + content_type="text/html", + ) + cmds = [get_assume_role(), merge_cmd, combine_cmd, upload_cmd, put_cmd] + return "download and merge coverage", cmds + + +def create_upload_mo_artifacts_func(): + include = ["./**.core", "./**.mdmp"] # Windows: minidumps + archive_cmd = archive_targz_pack(target="mongo-coredumps.tgz", source_dir="./", include=include) + display_name = "Core Dumps - Execution" + remote_file = "${build_variant}/${revision}/${version_id}/${build_id}/coredumps/${task_id}-${execution}-mongodb-coredumps.tar.gz" + s3_dumps = get_s3_put( + local_file="mongo-coredumps.tgz", remote_file=remote_file, display_name=display_name + ) + display_name = "drivers-tools-logs.tar.gz" + remote_file = "${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-drivers-tools-logs.tar.gz" + s3_logs = get_s3_put( + local_file="${DRIVERS_TOOLS}/.evergreen/test_logs.tar.gz", + remote_file=remote_file, + display_name=display_name, + ) + cmds = [get_assume_role(), archive_cmd, s3_dumps, s3_logs] + return "upload mo artifacts", cmds + + ################## # Generate Config ################## @@ -1258,5 +1356,40 @@ def write_tasks_to_file(): fid.write(f"{line}\n") +def write_functions_to_file(): + mod = sys.modules[__name__] + here = Path(__file__).absolute().parent + target = here.parent / "generated_configs" / "functions.yml" + if target.exists(): + target.unlink() + with target.open("w") as fid: + fid.write("functions:\n") + + functions = dict() + for name, func in sorted(getmembers(mod, isfunction)): + if name.startswith("_") or not name.endswith("_func"): + continue + if not name.startswith("create_"): + raise ValueError("Function creators must start with create_") + title = name.replace("create_", "").replace("_func", "").replace("_", " ").capitalize() + func_name, cmds = func() + functions = dict() + functions[func_name] = cmds + project = EvgProject(functions=functions, tasks=None, buildvariants=None) + out = ShrubService.generate_yaml(project).splitlines() + with target.open("a") as fid: + fid.write(f" # {title}\n") + for line in out[1:]: + fid.write(f"{line}\n") + fid.write("\n") + + # Remove extra trailing newline: + data = target.read_text().splitlines() + with target.open("w") as fid: + for line in data[:-1]: + fid.write(f"{line}\n") + + write_variants_to_file() write_tasks_to_file() +write_functions_to_file() diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index deab0724a4..a570e55ad1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -121,4 +121,4 @@ repos: entry: .evergreen/scripts/generate-config.sh language: python require_serial: true - additional_dependencies: ["shrub.py>=3.8.0", "pyyaml>=6.0.2"] + additional_dependencies: ["shrub.py>=3.9.0", "pyyaml>=6.0.2"]