Bump python to 3.13 (#4083)

# Summary
This patch bumps python version we use **locally and in CI** from 3.9 to
3.13. No customer-facing changes here.

Changes:
- unified python version in `PYTHON_VERSION` env var defined in
root-context
- exception is for race variant where the distro used doesn't contain
python 3.13 yet (3.12 is used)
- new script: `scripts/dev/recreate_python_venv.sh` which removes and
creates new, properly initialized python venv
- this script is also used in evergreen so we always create venv in one
way
- simplified referencing of python venv in evergreen.yaml to one
`python_venv` task and simplified usages so that the most tasks just
references `setup_building_host` which also sets up venv
- modified dev tooling installation script `scripts/dev/install.sh` to
create venv as well (calls `recreate_python_venv.sh`)

## Proof of Work

[EVG](https://spruce.mongodb.com/version/67a89e9412bd170007fb3be9/tasks?sorts=STATUS%3AASC%3BBASE_STATUS%3ADESC)
is green (minus one task failing also on master)

logged python version [on the evg
host](https://parsley.mongodb.com/evergreen/ops_manager_kubernetes_e2e_mdb_kind_ubi_cloudqa_e2e_replica_set_config_map_patch_46f54a608a7ce7e973d45c63aceb0efbdaf26707_67a89e9412bd170007fb3be9_25_02_09_12_24_53/0/task?bookmarks=0%2C1481&selectedLineRange=L473&shareLine=473)
```
[2025/02/09 13:37:34.373] Current python path: /data/mci/ee887af178b064cce3f8cebe0b52bb31/src/github.com/10gen/ops-manager-kubernetes/venv/bin/python

[2025/02/09 13:37:34.374] Python 3.13.2
```

and logged version when we start the tests [in the e2e test
pod](https://parsley.mongodb.com/evergreen/ops_manager_kubernetes_e2e_mdb_kind_ubi_cloudqa_e2e_replica_set_config_map_patch_46f54a608a7ce7e973d45c63aceb0efbdaf26707_67a89e9412bd170007fb3be9_25_02_09_12_24_53/0/task?bookmarks=0%2C1481&selectedLineRange=L1098&shareLine=1098):
```
[2025/02/09 13:39:32.922] platform linux -- Python 3.13.2, pytest-7.4.3, pluggy-1.5.0 -- /venv/bin/python3
```

## Checklist
- [ ] Have you linked a jira ticket and/or is the ticket in the title?
- [ ] Have you checked whether your jira ticket required DOCSP changes?
- [ ] Have you checked for release_note changes?

## Reminder (Please remove this when merging)
- Please try to Approve or Reject Changes the PR, keep PRs in review as
short as possible
- Our Short Guide for PRs:
[Link](REDACTED)
- Remember the following Communication Standards - use comment prefixes
for clarity:
  * **blocking**: Must be addressed before approval.
  * **follow-up**: Can be addressed in a later PR or ticket.
  * **q**: Clarifying question.
  * **nit**: Non-blocking suggestions.
  * **note**: Side-note, non-actionable. Example: Praise
  * --> no prefix is considered a question

Author: lsierant

.evergreen-functions.yml

@@ -1,30 +1,44 @@
 variables:
   - &e2e_include_expansions_in_env
     include_expansions_in_env:
+      - ARTIFACTORY_PASSWORD
+      - ARTIFACTORY_USERNAME
+      - GRS_PASSWORD
+      - GRS_USERNAME
+      - OVERRIDE_VERSION_ID
+      - PKCS11_URI
+      - SILK_CLIENT_ID
+      - SILK_CLIENT_SECRET
+      - branch_name
+      - build_id
+      - build_variant
+      - distro
+      - e2e_cloud_qa_apikey_owner_ubi_cloudqa
+      - e2e_cloud_qa_orgid_owner_ubi_cloudqa
+      - e2e_cloud_qa_user_owner_ubi_cloudqa
       - ecr_registry
       - ecr_registry_needs_auth
-      - openshift_url
-      - openshift_token
-      - workdir
+      - execution
+      - github_commit
+      - image_name
+      - include_tags
+      - is_patch
       - mms_eng_test_aws_access_key
-      - mms_eng_test_aws_secret
       - mms_eng_test_aws_region
-      - OVERRIDE_VERSION_ID
-      - version_id
-      - task_name
-      - e2e_cloud_qa_user_owner_ubi_cloudqa
-      - e2e_cloud_qa_apikey_owner_ubi_cloudqa
-      - e2e_cloud_qa_orgid_owner_ubi_cloudqa
-      - otel_trace_id
-      - otel_parent_id
+      - mms_eng_test_aws_secret
+      - openshift_token
+      - openshift_url
       - otel_collector_endpoint
-      - branch_name
+      - otel_parent_id
+      - otel_trace_id
+      - pin_tag_at
+      - registry
       - requester
-      - is_patch
-      - github_commit
-      - build_variant
-      - execution
-      - build_id
+      - skip_tags
+      - task_name
+      - triggered_by_git_tag
+      - version_id
+      - workdir
 
 functions:
 
@@ -59,17 +73,11 @@ functions:
         echo "Finished switching context"
 
   python_venv: &python_venv
-    command: subprocess.exec
-    type: setup
-    params:
-      command: /opt/python/3.9/bin/python3 -m venv --upgrade-deps ./venv
-
-  python_requirements: &python_requirements
     command: subprocess.exec
     type: setup
     params:
       working_dir: src/github.com/10gen/ops-manager-kubernetes
-      command: scripts/evergreen/run_python.sh -m pip install -r requirements.txt --quiet --no-warn-script-location
+      command: scripts/dev/recreate_python_venv.sh
 
   "clone":
     - command: subprocess.exec
@@ -167,13 +175,6 @@ functions:
         add_to_path:
           - ${workdir}/bin
         binary: scripts/evergreen/setup_preflight.sh
-    # The python binary is in a different location for RHEL machines so we can't use the same process
-    # for creating the virtual environment.
-    - command: subprocess.exec
-      type: setup
-      params:
-        command: python3 -m venv --upgrade-deps ./venv
-    - *python_requirements
 
   setup_prepare_openshift_bundles:
     - command: subprocess.exec
@@ -227,8 +228,6 @@ functions:
       binary: scripts/dev/configure_docker_auth.sh
 
   lint_repo:
-    - *python_venv
-    - *python_requirements
     - command: subprocess.exec
       type: setup
       params:
@@ -251,6 +250,7 @@ functions:
     - *setup_aws
     - *configure_docker_auth
     - *setup_docker_datadir
+    - *python_venv
 
   prune_docker_resources:
     - command: subprocess.exec
@@ -369,8 +369,6 @@ functions:
       command: scripts/evergreen/prepare_aws.sh
 
   build-dockerfiles:
-    - *python_venv
-    - *python_requirements
     - command: subprocess.exec
       type: setup
       params:
@@ -470,8 +468,6 @@ functions:
           - ${workdir}
 
   pipeline:
-    - *python_venv
-    - *python_requirements
     - *switch_context
     - command: shell.exec
       type: setup
@@ -487,32 +483,11 @@ functions:
       type: setup
       params:
         shell: bash
-        include_expansions_in_env:
-          - version_id
-          - registry
-          - image_name
-          - skip_tags
-          - include_tags
-          - distro
-          - is_patch
-          - GRS_USERNAME
-          - GRS_PASSWORD
-          - PKCS11_URI
-          - ARTIFACTORY_USERNAME
-          - ARTIFACTORY_PASSWORD
-          - triggered_by_git_tag
-          - pin_tag_at
-          - SILK_CLIENT_ID
-          - SILK_CLIENT_SECRET
-          - otel_trace_id
-          - otel_parent_id
-          - otel_collector_endpoint
+        <<: *e2e_include_expansions_in_env
         working_dir: src/github.com/10gen/ops-manager-kubernetes
         binary: scripts/evergreen/run_python.sh pipeline.py --include ${image_name} --parallel --sign
 
   teardown_cloud_qa_all:
-    - *python_venv
-    - *python_requirements
     - *switch_context
     - command: shell.exec
       type: setup
@@ -639,8 +614,6 @@ functions:
         files: [ "src/github.com/10gen/ops-manager-kubernetes/*.suite", "src/github.com/10gen/ops-manager-kubernetes/public/tools/multicluster/*.suite", "src/github.com/10gen/ops-manager-kubernetes/docker/mongodb-enterprise-init-ops-manager/mmsconfiguration/*.suite" ]
 
   test_python_unit:
-    - *python_venv
-    - *python_requirements
     - command: shell.exec
       type: test
       params:
@@ -651,8 +624,6 @@ functions:
           make python-tests
 
   test_sboms:
-    - *python_venv
-    - *python_requirements
     - command: shell.exec
       type: test
       params:
@@ -664,8 +635,6 @@ functions:
 
   generate_perf_tests_tasks:
     - *switch_context
-    - *python_venv
-    - *python_requirements
     - command: shell.exec
       type: setup
       params:

.evergreen.yml

@@ -539,6 +539,7 @@ task_groups:
     setup_group_can_fail_task: true
     setup_group:
       - func: clone
+      - func: python_venv
       - func: download_kube_tools
       - func: setup_shellcheck
     tasks:

.githooks/pre-commit

@@ -3,8 +3,8 @@
 set -Eeou pipefail
 
 source scripts/dev/set_env_context.sh
-if [ -f "${workdir}"/venv/bin/activate ]; then
-  source "${workdir}"/venv/bin/activate
+if [ -f "${PROJECT_DIR}"/venv/bin/activate ]; then
+  source "${PROJECT_DIR}"/venv/bin/activate
 fi
 
 if [[ -z "${EVERGREEN_MODE:-}" ]]; then

constraints.txt

@@ -6,7 +6,9 @@
 #
 # Ref: https://cryptography.io/en/latest/installation/#building-cryptography-on-linux
 #
-FROM --platform=linux/amd64 public.ecr.aws/docker/library/python:3.9-slim as builder
+ARG PYTHON_VERSION
+
+FROM --platform=linux/amd64 public.ecr.aws/docker/library/python:${PYTHON_VERSION}-slim as builder
 
 
 RUN apt-get -qq update \
@@ -18,7 +20,7 @@ COPY requirements.txt requirements.txt
 RUN python3 -m venv /venv && . /venv/bin/activate && python3 -m pip install -r requirements.txt
 
 
-FROM --platform=linux/amd64 public.ecr.aws/docker/library/python:3.9-slim
+FROM --platform=linux/amd64 public.ecr.aws/docker/library/python:${PYTHON_VERSION}-slim
 
 RUN apt-get -qq update \
     && apt-get -y -qq install \

docker/mongodb-enterprise-tests/Dockerfile

@@ -67,19 +67,7 @@ have done this, you can proceed to complete the Python installation.
 
 ### Installing Python and Dependencies ###
 
-You should have Python 3.6 installed, create a virtualenv and install
-the dependencies on the `requirements.txt` file. The following is an
-example on how to achieve this:
-
-
-* First time install only: Create a virtualenv
-``` bash
-python3.6 -m pip install venv --user
-python3.6 -m venv venv
-
-source venv/bin/activate
-PIP_CONSTRAINT=constraints.txt python -m pip install -r requirements.txt
-```
+Run `scripts/dev/install.sh` or `scripts/dev/recreate_python_venv.sh` to install necessary tools and create python virtualenv. 
 
 * After the first run, when coming back to the project it should be
 required to `activate` your virtual environment once again.

docker/mongodb-enterprise-tests/README.md

@@ -8,6 +8,8 @@ images:
   - name: build
     task_type: docker_build
     dockerfile: Dockerfile
+    buildargs:
+      PYTHON_VERSION: $(inputs.params.python_version)
     output:
     - registry: $(inputs.params.registry)/mongodb-enterprise-tests
       tag: latest

inventories/test.yaml

@@ -17,7 +17,6 @@
 from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
 from dataclasses import dataclass
 from datetime import datetime, timedelta
-from distutils.dir_util import copy_tree
 from queue import Queue
 from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple, Union
 
@@ -459,12 +458,18 @@ def build_tests_image(build_configuration: BuildConfiguration):
     shutil.rmtree(public_dest, ignore_errors=True)
 
     # Copy directories and files (recursive copy)
-    copy_tree(helm_src, helm_dest)
-    copy_tree(public_src, public_dest)
+    shutil.copytree(helm_src, helm_dest)
+    shutil.copytree(public_src, public_dest)
     shutil.copyfile("release.json", "docker/mongodb-enterprise-tests/release.json")
     shutil.copyfile("requirements.txt", requirements_dest)
 
-    sonar_build_image(image_name, build_configuration, {}, "inventories/test.yaml")
+    python_version = os.getenv("PYTHON_VERSION", "")
+    if python_version == "":
+        raise Exception("Missing PYTHON_VERSION environment variable")
+
+    buildargs = dict({"python_version": python_version})
+
+    sonar_build_image(image_name, build_configuration, buildargs, "inventories/test.yaml")
 
 
 def build_operator_image(build_configuration: BuildConfiguration):

pipeline.py

@@ -12,7 +12,7 @@ kubernetes==17.17.0
 pymongo==4.6.3
 pytest==7.4.3
 pytest-asyncio==0.14.0
-PyYAML==5.4.1
+PyYAML==6.0.2
 urllib3==1.26.19
 cryptography==43.0.1
 python-dateutil==2.9.0
@@ -40,7 +40,7 @@ autopep8==1.5.7
 flake8-isort==4.0.0
 mypy==1.14.1
 types-freezegun==0.1.4
-types-PyYAML==5.4.3
+types-PyYAML==6.0.2
 types-pytz==2021.1.0
 types-python-dateutil==0.1.4
 pipupgrade==1.12.0

requirements.txt

@@ -16,3 +16,6 @@ export TEST_POD_CLUSTER=kind-e2e-cluster-1
 export test_pod_cluster=kind-e2e-cluster-1
 export ops_manager_version="cloud_qa"
 export BUILD_WITH_RACE_DETECTION=true
+
+# there is no python 3.13 in ubuntu1804-xlarge
+export PYTHON_VERSION=3.12