PYTHON-4879 Use shrub.py for compressor tests

Author: blink1073

.evergreen/config.yml

@@ -2112,23 +2112,6 @@ axes:
            AUTH: "noauth"
            SSL: "nossl"
 
-  # Choice of wire protocol compression support
-  - id: compression
-    display_name: Compression
-    values:
-      - id: snappy
-        display_name: snappy compression
-        variables:
-            COMPRESSORS: "snappy"
-      - id: zlib
-        display_name: zlib compression
-        variables:
-            COMPRESSORS: "zlib"
-      - id: zstd
-        display_name: zstd compression
-        variables:
-            COMPRESSORS: "zstd"
-
   # Choice of MongoDB server version
   - id: mongodb-version
     display_name: "MongoDB"
@@ -3125,6 +3108,257 @@ buildvariants:
     PYTHON_BINARY: C:/python/Python313/python.exe
   tags: [encryption_tag]
 
+# Compressor tests.
+- name: snappy-compression-rhel8-py3.9-no-c
+  tasks:
+    - name: .standalone
+  display_name: snappy compression RHEL8 py3.9 No C
+  run_on:
+    - rhel87-small
+  expansions:
+    COMPRESSORS: snappy
+    NO_EXT: "1"
+    PYTHON_BINARY: /opt/python/3.9/bin/python3
+- name: snappy-compression-rhel8-py3.9
+  tasks:
+    - name: .standalone
+  display_name: snappy compression RHEL8 py3.9
+  run_on:
+    - rhel87-small
+  expansions:
+    COMPRESSORS: snappy
+    PYTHON_BINARY: /opt/python/3.9/bin/python3
+- name: snappy-compression-rhel8-py3.13-no-c
+  tasks:
+    - name: .standalone
+  display_name: snappy compression RHEL8 py3.13 No C
+  run_on:
+    - rhel87-small
+  expansions:
+    COMPRESSORS: snappy
+    NO_EXT: "1"
+    PYTHON_BINARY: /opt/python/3.13/bin/python3
+- name: snappy-compression-rhel8-py3.13
+  tasks:
+    - name: .standalone
+  display_name: snappy compression RHEL8 py3.13
+  run_on:
+    - rhel87-small
+  expansions:
+    COMPRESSORS: snappy
+    PYTHON_BINARY: /opt/python/3.13/bin/python3
+- name: zlib-compression-rhel8-py3.9-no-c
+  tasks:
+    - name: .standalone
+  display_name: zlib compression RHEL8 py3.9 No C
+  run_on:
+    - rhel87-small
+  expansions:
+    COMPRESSORS: zlib
+    NO_EXT: "1"
+    PYTHON_BINARY: /opt/python/3.9/bin/python3
+- name: zlib-compression-rhel8-py3.9
+  tasks:
+    - name: .standalone
+  display_name: zlib compression RHEL8 py3.9
+  run_on:
+    - rhel87-small
+  expansions:
+    COMPRESSORS: zlib
+    PYTHON_BINARY: /opt/python/3.9/bin/python3
+- name: zlib-compression-rhel8-py3.13-no-c
+  tasks:
+    - name: .standalone
+  display_name: zlib compression RHEL8 py3.13 No C
+  run_on:
+    - rhel87-small
+  expansions:
+    COMPRESSORS: zlib
+    NO_EXT: "1"
+    PYTHON_BINARY: /opt/python/3.13/bin/python3
+- name: zlib-compression-rhel8-py3.13
+  tasks:
+    - name: .standalone
+  display_name: zlib compression RHEL8 py3.13
+  run_on:
+    - rhel87-small
+  expansions:
+    COMPRESSORS: zlib
+    PYTHON_BINARY: /opt/python/3.13/bin/python3
+- name: zstd-compression-rhel8-py3.9-no-c
+  tasks:
+    - name: .standalone !.4.0
+  display_name: zstd compression RHEL8 py3.9 No C
+  run_on:
+    - rhel87-small
+  expansions:
+    COMPRESSORS: zstd
+    NO_EXT: "1"
+    PYTHON_BINARY: /opt/python/3.9/bin/python3
+- name: zstd-compression-rhel8-py3.9
+  tasks:
+    - name: .standalone !.4.0
+  display_name: zstd compression RHEL8 py3.9
+  run_on:
+    - rhel87-small
+  expansions:
+    COMPRESSORS: zstd
+    PYTHON_BINARY: /opt/python/3.9/bin/python3
+- name: zstd-compression-rhel8-py3.13-no-c
+  tasks:
+    - name: .standalone !.4.0
+  display_name: zstd compression RHEL8 py3.13 No C
+  run_on:
+    - rhel87-small
+  expansions:
+    COMPRESSORS: zstd
+    NO_EXT: "1"
+    PYTHON_BINARY: /opt/python/3.13/bin/python3
+- name: zstd-compression-rhel8-py3.13
+  tasks:
+    - name: .standalone !.4.0
+  display_name: zstd compression RHEL8 py3.13
+  run_on:
+    - rhel87-small
+  expansions:
+    COMPRESSORS: zstd
+    PYTHON_BINARY: /opt/python/3.13/bin/python3
+- name: snappy-compression-rhel8-py3.10
+  tasks:
+    - name: .standalone
+  display_name: snappy compression RHEL8 py3.10
+  run_on:
+    - rhel87-small
+  expansions:
+    COMPRESSORS: snappy
+    PYTHON_BINARY: /opt/python/3.10/bin/python3
+- name: snappy-compression-rhel8-py3.11
+  tasks:
+    - name: .standalone
+  display_name: snappy compression RHEL8 py3.11
+  run_on:
+    - rhel87-small
+  expansions:
+    COMPRESSORS: snappy
+    PYTHON_BINARY: /opt/python/3.11/bin/python3
+- name: snappy-compression-rhel8-py3.12
+  tasks:
+    - name: .standalone
+  display_name: snappy compression RHEL8 py3.12
+  run_on:
+    - rhel87-small
+  expansions:
+    COMPRESSORS: snappy
+    PYTHON_BINARY: /opt/python/3.12/bin/python3
+- name: snappy-compression-rhel8-pypy3.9
+  tasks:
+    - name: .standalone
+  display_name: snappy compression RHEL8 pypy3.9
+  run_on:
+    - rhel87-small
+  expansions:
+    COMPRESSORS: snappy
+    PYTHON_BINARY: /opt/python/pypy3.9/bin/python3
+- name: snappy-compression-rhel8-pypy3.10
+  tasks:
+    - name: .standalone
+  display_name: snappy compression RHEL8 pypy3.10
+  run_on:
+    - rhel87-small
+  expansions:
+    COMPRESSORS: snappy
+    PYTHON_BINARY: /opt/python/pypy3.10/bin/python3
+- name: zlib-compression-rhel8-py3.10
+  tasks:
+    - name: .standalone
+  display_name: zlib compression RHEL8 py3.10
+  run_on:
+    - rhel87-small
+  expansions:
+    COMPRESSORS: zlib
+    PYTHON_BINARY: /opt/python/3.10/bin/python3
+- name: zlib-compression-rhel8-py3.11
+  tasks:
+    - name: .standalone
+  display_name: zlib compression RHEL8 py3.11
+  run_on:
+    - rhel87-small
+  expansions:
+    COMPRESSORS: zlib
+    PYTHON_BINARY: /opt/python/3.11/bin/python3
+- name: zlib-compression-rhel8-py3.12
+  tasks:
+    - name: .standalone
+  display_name: zlib compression RHEL8 py3.12
+  run_on:
+    - rhel87-small
+  expansions:
+    COMPRESSORS: zlib
+    PYTHON_BINARY: /opt/python/3.12/bin/python3
+- name: zlib-compression-rhel8-pypy3.9
+  tasks:
+    - name: .standalone
+  display_name: zlib compression RHEL8 pypy3.9
+  run_on:
+    - rhel87-small
+  expansions:
+    COMPRESSORS: zlib
+    PYTHON_BINARY: /opt/python/pypy3.9/bin/python3
+- name: zlib-compression-rhel8-pypy3.10
+  tasks:
+    - name: .standalone
+  display_name: zlib compression RHEL8 pypy3.10
+  run_on:
+    - rhel87-small
+  expansions:
+    COMPRESSORS: zlib
+    PYTHON_BINARY: /opt/python/pypy3.10/bin/python3
+- name: zstd-compression-rhel8-py3.10
+  tasks:
+    - name: .standalone !.4.0
+  display_name: zstd compression RHEL8 py3.10
+  run_on:
+    - rhel87-small
+  expansions:
+    COMPRESSORS: zstd
+    PYTHON_BINARY: /opt/python/3.10/bin/python3
+- name: zstd-compression-rhel8-py3.11
+  tasks:
+    - name: .standalone !.4.0
+  display_name: zstd compression RHEL8 py3.11
+  run_on:
+    - rhel87-small
+  expansions:
+    COMPRESSORS: zstd
+    PYTHON_BINARY: /opt/python/3.11/bin/python3
+- name: zstd-compression-rhel8-py3.12
+  tasks:
+    - name: .standalone !.4.0
+  display_name: zstd compression RHEL8 py3.12
+  run_on:
+    - rhel87-small
+  expansions:
+    COMPRESSORS: zstd
+    PYTHON_BINARY: /opt/python/3.12/bin/python3
+- name: zstd-compression-rhel8-pypy3.9
+  tasks:
+    - name: .standalone !.4.0
+  display_name: zstd compression RHEL8 pypy3.9
+  run_on:
+    - rhel87-small
+  expansions:
+    COMPRESSORS: zstd
+    PYTHON_BINARY: /opt/python/pypy3.9/bin/python3
+- name: zstd-compression-rhel8-pypy3.10
+  tasks:
+    - name: .standalone !.4.0
+  display_name: zstd compression RHEL8 pypy3.10
+  run_on:
+    - rhel87-small
+  expansions:
+    COMPRESSORS: zstd
+    PYTHON_BINARY: /opt/python/pypy3.10/bin/python3
+
 - matrix_name: "tests-fips"
   matrix_spec:
     platform:
@@ -3214,38 +3448,6 @@ buildvariants:
     - ".4.2"
     - ".4.0"
 
-- matrix_name: "tests-python-version-rhel8-compression"
-  matrix_spec:
-    platform: rhel8
-    python-version: "*"
-    c-extensions: "*"
-    compression: "*"
-  exclude_spec:
-   # These interpreters are always tested without extensions.
-   - platform: rhel8
-     python-version: ["pypy3.9", "pypy3.10"]
-     c-extensions: "with-c-extensions"
-     compression: "*"
-  display_name: "${compression} ${c-extensions} ${python-version} ${platform}"
-  tasks:
-    - "test-latest-standalone"
-    - "test-8.0-standalone"
-    - "test-7.0-standalone"
-    - "test-6.0-standalone"
-    - "test-5.0-standalone"
-    - "test-4.4-standalone"
-    - "test-4.2-standalone"
-    - "test-4.0-standalone"
-  rules:
-    # Server version 4.0 supports snappy and zlib but not zstd.
-    - if:
-        python-version: "*"
-        c-extensions: "*"
-        compression: ["zstd"]
-      then:
-        remove_tasks:
-          - "test-4.0-standalone"
-
 - matrix_name: "tests-python-version-green-framework-rhel8"
   matrix_spec:
     platform: rhel8

.evergreen/scripts/generate_config.py

@@ -30,12 +30,14 @@
 BATCHTIME_WEEK = 10080
 AUTH_SSLS = [("auth", "ssl"), ("noauth", "ssl"), ("noauth", "nossl")]
 TOPOLOGIES = ["standalone", "replica_set", "sharded_cluster"]
+C_EXTS = ["with_ext", "without_ext"]
 SYNCS = ["sync", "async"]
 DISPLAY_LOOKUP = dict(
     ssl=dict(ssl="SSL", nossl="NoSSL"),
     auth=dict(auth="Auth", noauth="NoAuth"),
     test_suites=dict(default="Sync", default_async="Async"),
     coverage=dict(coverage="cov"),
+    no_ext={"1": "No C"},
 )
 HOSTS = dict()
 
@@ -137,6 +139,12 @@ def zip_cycle(*iterables, empty_default=None):
         yield tuple(next(i, empty_default) for i in cycles)
 
 
+def handle_c_ext(c_ext, expansions):
+    """Handle c extension option."""
+    if c_ext == C_EXTS[0]:
+        expansions["NO_EXT"] = "1"
+
+
 def generate_yaml(tasks=None, variants=None):
     """Generate the yaml for a given set of tasks and variants."""
     project = EvgProject(tasks=tasks, buildvariants=variants)
@@ -337,8 +345,48 @@ def create_load_balancer_variants():
     return variants
 
 
+def create_compression_variants():
+    # Compression tests - standalone versions of each server, across python versions, with and without c extensions.
+    # PyPy interpreters are always tested without extensions.
+    host = "rhel8"
+    task_names = dict(snappy=[".standalone"], zlib=[".standalone"], zstd=[".standalone !.4.0"])
+    variants = []
+    for compressor, python, c_ext in product(["snappy", "zlib", "zstd"], MIN_MAX_PYTHON, C_EXTS):
+        expansions = dict(COMPRESSORS=compressor)
+        handle_c_ext(c_ext, expansions)
+        base_name = f"{compressor} compression"
+        display_name = get_display_name(base_name, host, python=python, **expansions)
+        variant = create_variant(
+            task_names[compressor],
+            display_name,
+            python=python,
+            host=host,
+            expansions=expansions,
+        )
+        variants.append(variant)
+
+    other_pythons = [p for p in CPYTHONS if p not in MIN_MAX_PYTHON] + PYPYS
+    for compressor, python in product(["snappy", "zlib", "zstd"], other_pythons):
+        expansions = dict(COMPRESSORS=compressor)
+        handle_c_ext(c_ext, expansions)
+        base_name = f"{compressor} compression"
+        display_name = get_display_name(base_name, host, python=python, **expansions)
+        variant = create_variant(
+            task_names[compressor],
+            display_name,
+            python=python,
+            host=host,
+            expansions=expansions,
+        )
+        variants.append(variant)
+
+    return variants
+
+
 ##################
 # Generate Config
 ##################
 
-generate_yaml(variants=create_load_balancer_variants())
+variants = create_compression_variants()
+# print(len(variants))
+generate_yaml(variants=variants)