Exclude running some tests for specific packages (#163)

* Exclude running some tests for specific packages

* Cache test mapping

* Take platforms into account

* Adjust alts to send skipped tests for display

* Refactor test steps execution

* Minor corrections

Author: Kwaizer

README.md

@@ -61,6 +61,25 @@ For testing purposes, you can add the `--reload` argument to the scheduler launc
 
 Both Celery worker and scheduler REST API services need YAML-based configs to function. Config examples are provided in the `configs` folder.
 
+During the task execution some tests may be skipped if it specified in a particular `.json` file which can be found here: https://git.almalinux.org/almalinux/alts-exclusions/src/branch/main/skipped_tests.json
+
+Example of a file content:
+```
+{
+    "almalinux-8": {
+        "libxml2": ["run_package_integrity_tests"],
+        "curl": ["run_package_integrity_tests"],
+        "bzip2": ["run_package_integrity_tests"]
+    },
+    "almalinux-9": {
+        "libxml2": ["run_package_integrity_tests"],
+        "curl": ["run_package_integrity_tests"]
+    },
+    "ubuntu-24.04": {
+        "bzip2": ["run_package_integrity_tests"]
+    }
+}
+```
 
 Filling options in the config file
 --

alts/scheduler/scheduling.py

@@ -4,7 +4,8 @@
 import time
 import urllib.parse
 import uuid
-from typing import List
+from typing import List, Optional
+from cachetools import TTLCache
 
 import requests
 
@@ -28,6 +29,10 @@ def __init__(
         self.__graceful_terminate = graceful_terminate
         self.__celery = celery_app
         self.__get_result_timeout = get_result_timeout
+        self.__cached_config = TTLCache(
+            maxsize=CONFIG.excluded_tests_cache_size,
+            ttl=CONFIG.excluded_tests_cache_update_interval,
+        )
         self.logger = logging.getLogger(__file__)
 
     def get_available_test_tasks(self) -> List[dict]:
@@ -49,6 +54,18 @@ def get_available_test_tasks(self) -> List[dict]:
             self.logger.exception('Cannot get available test tasks:')
         return response_as_json
 
+    def get_excluded_packages(self, distro: str) -> Optional[dict]:
+        if 'excluded_packages' not in self.__cached_config:
+            uri = f'{CONFIG.excluded_tests_url}'
+            try:
+                response = requests.get(uri, timeout=10)
+                response.raise_for_status()
+                self.__cached_config['excluded_packages'] = response.json()
+            except requests.RequestException:
+                return {}
+        pkgs = self.__cached_config.get('excluded_packages', {})
+        return pkgs.get(distro, {})
+
     def schedule_test_task(self, payload: TaskRequestPayload):
         """
         Schedules new tasks in Test System.
@@ -128,9 +145,13 @@ def schedule_test_task(self, payload: TaskRequestPayload):
         task_params['task_id'] = task_id
         task_params['runner_type'] = runner_type
         task_params['repositories'] = repositories
+        distro = f"{task_params['dist_name']}-{task_params['dist_version']}"
         try:
             run_tests.apply_async(
-                (task_params,),
+                (
+                    task_params,
+                    self.get_excluded_packages(distro),
+                ),
                 task_id=task_id,
                 queue=queue_name,
             )

alts/shared/constants.py

@@ -15,6 +15,7 @@
     'DEFAULT_SSH_AUTH_METHODS',
     'X32_ARCHITECTURES',
     'X64_ARCHITECTURES',
+    'TESTS_MAPPING',
 ]
 
 
@@ -67,3 +68,10 @@
     'hostbased',
     'publickey',
 ]
+
+TESTS_MAPPING = {
+    'run_system_info_commands': 'system_info',
+    'install_package': 'install_package',
+    'run_package_integrity_tests': 'package_integrity_tests',
+    'uninstall_package': 'uninstall_package',
+}

alts/shared/models.py

@@ -389,3 +389,6 @@ class SchedulerConfig(CeleryConfig):
     working_directory: str = '/srv/alts/scheduler'
     jwt_secret: str
     hashing_algorithm: str = 'HS256'
+    excluded_tests_url: str = 'https://git.almalinux.org/almalinux/alts-exclusions/raw/branch/main/skipped_tests.json'
+    excluded_tests_cache_size: str = 1
+    excluded_tests_cache_update_interval: int = 600

alts/worker/tasks.py

@@ -3,7 +3,6 @@
 # created: 2021-04-13
 
 """AlmaLinux Test System package testing tasks running."""
-
 import logging
 import traceback
 import random
@@ -25,7 +24,7 @@
 from urllib3 import Retry
 from urllib3.exceptions import TimeoutError
 
-from alts.shared.constants import API_VERSION, DEFAULT_REQUEST_TIMEOUT
+from alts.shared.constants import API_VERSION, DEFAULT_REQUEST_TIMEOUT, TESTS_MAPPING
 from alts.shared.exceptions import (
     InstallPackageError,
     PackageIntegrityTestsError,
@@ -95,21 +94,24 @@ class RetryableTask(AbortableTask):
 
 
 @celery_app.task(bind=True, base=RetryableTask)
-def run_tests(self, task_params: dict):
+def run_tests(self, task_params: dict, tests_to_skip: dict):
     """
     Executes a package test in a specified environment.
 
     Parameters
     ----------
     task_params : dict
         Task parameters.
+    tests_to_skip : dict
+        Tests mapping.
 
     Returns
     -------
     dict
         Result summary of a test execution.
     """
     aborted = False
+    summary = defaultdict(dict)
 
     def is_success(stage_data_: dict):
         tap_result = are_tap_tests_success(stage_data_.get('stdout', ''))
@@ -120,6 +122,12 @@ def is_success(stage_data_: dict):
             return False
         return stage_data_['exit_code'] == 0
 
+    def should_skip_test(package, test_name, skipped_tests_map):
+        for pkg_pattern, tests in skipped_tests_map.items():
+            if package.startswith(pkg_pattern) and test_name in tests:
+                return True
+        return False
+
     def set_artifacts_when_stage_has_unexpected_exception(
         _artifacts: dict,
         error_message: str,
@@ -179,9 +187,19 @@ def set_artifacts_when_stage_has_unexpected_exception(
         # Wait a bit to not spawn all environments at once when
         # a lot of tasks are coming to the machine
         time.sleep(random.randint(5, 10))
+        summary['skipped_tests'] = []
+
+        def run_or_skip(test_name, func, *args, **kwargs):
+            if should_skip_test(package_name, test_name, tests_to_skip):
+                summary['skipped_tests'].append(TESTS_MAPPING[test_name])
+            else:
+                func(*args, **kwargs)
+
         runner.setup()
         runner.run_system_info_commands()
-        runner.install_package(
+        run_or_skip(
+            "install_package",
+            runner.install_package,
             package_name,
             package_version=package_version,
             package_epoch=package_epoch,
@@ -191,13 +209,16 @@ def set_artifacts_when_stage_has_unexpected_exception(
             semi_verbose=True,
         )
         if CONFIG.enable_integrity_tests:
-            runner.run_package_integrity_tests(package_name, package_version)
-        runner.run_third_party_tests(
+            run_or_skip("run_package_integrity_tests", runner.run_package_integrity_tests, package_name,
+                        package_version)
+        run_or_skip(
+            "run_third_party_tests",
+            runner.run_third_party_tests,
             package_name,
             package_version=package_version,
             package_epoch=package_epoch,
         )
-        runner.uninstall_package(package_name)
+        run_or_skip("uninstall_package", runner.uninstall_package, package_name)
     except VMImageNotFound as exc:
         logging.exception('Cannot find VM image: %s', exc)
     except WorkDirPreparationError:
@@ -238,7 +259,6 @@ def set_artifacts_when_stage_has_unexpected_exception(
         )
     finally:
         runner.teardown()
-        summary = defaultdict(dict)
         if aborted:
             summary['revoked'] = True
         else:

requirements/scheduler.txt

@@ -5,3 +5,4 @@ databases[sqlite]==0.8.0
 fastapi==0.115.0
 pyjwt==2.9.0
 uvicorn==0.30.6
+cachetools==6.1.0