Exclude projects from node (#181)

* Exclude projects from nodes (AlmaLinux/build-system#365)

* Black and isort formatting

Author: bklvsky

build_node/build_node_builder.py

@@ -41,12 +41,7 @@ class BuildNodeBuilder(threading.Thread):
     """Build thread."""
 
     def __init__(
-        self,
-        config,
-        thread_num,
-        terminated_event,
-        graceful_terminated_event,
-        task_queue,
+        self, config, thread_num, terminated_event, graceful_terminated_event
     ):
         """
         Build thread initialization.
@@ -62,9 +57,7 @@ def __init__(
         graceful_terminated_event : threading.Event
             Shows, if process got "kill -10" signal.
         """
-        super(BuildNodeBuilder, self).__init__(
-            name='Builder-{0}'.format(thread_num)
-        )
+        super(BuildNodeBuilder, self).__init__(name=f'Builder-{thread_num}')
         self.__config = config
         self.__working_dir = os.path.join(
             config.working_dir, 'builder-{0}'.format(thread_num)
@@ -151,12 +144,8 @@ def run(self):
                 )
                 capture_exception(e)
             finally:
-                only_logs = not (
-                    bool(
-                        filter_files(
-                            artifacts_dir, lambda f: f.endswith('.rpm')
-                        )
-                    )
+                only_logs = not bool(
+                    filter_files(artifacts_dir, lambda f: f.endswith('.rpm'))
                 )
                 if success is False:
                     only_logs = True

build_node/build_node_config.py

@@ -32,12 +32,16 @@
 DEFAULT_REQUEST_TIMEOUT = 60  # 1 minute
 DEFAULT_PULP_TIMEOUT = 120  # 2 minutes
 DEFAULT_JWT_TOKEN = 'test_jwt'
+DEFAULT_CACHE_UPDATE_INTERVAL = 600
+DEFAULT_EXCLUSIONS_URL = (
+    'https://git.almalinux.org/almalinux/build-node-exclusions/raw/branch/main'
+)
+DEFAULT_CACHE_SIZE = 1
 
 __all__ = ['BuildNodeConfig']
 
 
 class BuildNodeConfig(BaseConfig):
-
     """
     Build node configuration storage.
 
@@ -125,6 +129,10 @@ def __init__(self, config_file=None, **cmd_args):
             'immudb_public_key_file': None,
             'request_timeout': DEFAULT_REQUEST_TIMEOUT,
             'jwt_token': DEFAULT_JWT_TOKEN,
+            'cache_update_interval': DEFAULT_CACHE_UPDATE_INTERVAL,
+            'exclusions_url': DEFAULT_EXCLUSIONS_URL,
+            'build_node_name': self.get_node_name(),
+            'cache_size': DEFAULT_CACHE_SIZE,
         }
         schema = {
             'development_mode': {'type': 'boolean', 'default': False},
@@ -136,7 +144,11 @@ def __init__(self, config_file=None, **cmd_args):
             'working_dir': {'type': 'string', 'required': True},
             'git_cache_locks_dir': {'type': 'string', 'required': True},
             'git_repos_cache_dir': {'type': 'string', 'required': True},
-            'git_extra_options': {'type': 'list', 'required': False, 'nullable': True},
+            'git_extra_options': {
+                'type': 'list',
+                'required': False,
+                'nullable': True,
+            },
             'native_support': {'type': 'boolean', 'default': True},
             'arm64_support': {'type': 'boolean', 'default': False},
             'arm32_support': {'type': 'boolean', 'default': False},
@@ -159,12 +171,20 @@ def __init__(self, config_file=None, **cmd_args):
             'immudb_public_key_file': {'type': 'string', 'nullable': True},
             'base_arch': {'type': 'string', 'nullable': False},
             'build_src': {'type': 'boolean', 'nullable': False},
-            'pulp_timeout': {'type': 'integer', 'min': DEFAULT_PULP_TIMEOUT,
-                             'required': True},
+            'pulp_timeout': {
+                'type': 'integer',
+                'min': DEFAULT_PULP_TIMEOUT,
+                'required': True,
+            },
             'request_timeout': {'type': 'integer', 'required': True},
+            'cache_update_interval': {'type': 'integer', 'required': True},
+            'exclusions_url': {'type': 'string', 'required': True},
+            'build_node_name': {'type': 'string', 'required': True},
+            'cache_size': {'type': 'integer', 'required': True},
         }
-        super(BuildNodeConfig, self).__init__(default_config, config_file,
-                                              schema, **cmd_args)
+        super(BuildNodeConfig, self).__init__(
+            default_config, config_file, schema, **cmd_args
+        )
 
     @property
     def codenotary_enabled(self) -> bool:

build_node/build_node_supervisor.py

@@ -6,8 +6,10 @@
 import requests
 import requests.adapters
 from urllib3 import Retry
+from cachetools import TTLCache
 
 from build_node import constants
+from build_node.utils.file_utils import file_url_exists
 
 
 class BuilderSupervisor(threading.Thread):
@@ -24,6 +26,10 @@ def __init__(
         self.terminated_event = terminated_event
         self.__session = None
         self.__task_queue = task_queue
+        self.__cached_config = TTLCache(
+            maxsize=config.cache_size,
+            ttl=config.cache_update_interval,
+            )
         super(BuilderSupervisor, self).__init__(name='BuildersSupervisor')
 
     def __generate_request_session(self):
@@ -45,6 +51,7 @@ def __generate_request_session(self):
     def __request_build_task(self):
         if not self.__task_queue.full():
             supported_arches = [self.config.base_arch]
+            excluded_packages = self.get_excluded_packages()
             if self.config.base_arch == 'x86_64':
                 supported_arches.append('i686')
             if self.config.build_src:
@@ -54,7 +61,7 @@ def __request_build_task(self):
             )
             data = {
                 'supported_arches': supported_arches,
-                'excluded_packages': [],
+                'excluded_packages': excluded_packages,
             }
             try:
                 response = self.__session.post(
@@ -90,6 +97,15 @@ def __report_active_tasks(self):
                 traceback.format_exc(),
             )
 
+    def get_excluded_packages(self):
+        if 'excluded_packages' not in self.__cached_config:
+            uri = f'{self.__config.exclusions_url}/{self.__config.build_node_name}'
+            if file_url_exists(uri):
+                response = requests.get(uri).text
+                self.__cached_config['excluded_packages'] = response.splitlines()
+
+        return self.__cached_config.get('excluded_packages', [])
+
     def run(self):
         self.__generate_request_session()
         while not self.terminated_event.is_set():

build_node/utils/config.py

@@ -19,7 +19,6 @@
 
 
 class ConfigValidator(cerberus.Validator):
-
     """
     Custom validator for CloudLinux Build System configuration objects.
     """
@@ -70,20 +69,21 @@ def locate_config_file(component, config_path=None):
     if config_path:
         config_path = normalize_path(config_path)
         if not os.path.exists(config_path):
-            raise ValueError('configuration file {0} is not found'.
-                             format(config_path))
+            raise ValueError(
+                'configuration file {0} is not found'.format(config_path)
+            )
         return config_path
     config_path = normalize_path('~/.config/castor/{0}.yml'.format(component))
     if os.path.exists(config_path):
         return config_path
 
 
 class BaseConfig(object):
-
     """Base configuration object for Build System processes."""
 
-    def __init__(self, default_config, config_path=None, schema=None,
-                 **cmd_args):
+    def __init__(
+        self, default_config, config_path=None, schema=None, **cmd_args
+    ):
         """
         Configuration object initialization.
 
@@ -128,6 +128,11 @@ def generate_node_id(postfix=''):
         """
         return '{0}.{1}{2}'.format(platform.node(), os.getpid(), postfix)
 
+    @staticmethod
+    def get_node_name():
+        host_name = platform.node()
+        return host_name.rsplit('.', 2)[0]
+
     def __dir__(self):
         return list(self.__config.keys())
 
@@ -145,7 +150,9 @@ def __parse_config_file(self, config_path):
     def __validate_config(self, schema):
         validator = ConfigValidator(schema or {})
         if not validator.validate(self.__config):
-            error_list = ['{0}: {1}'.format(k, ', '.join(v))
-                          for k, v in validator.errors.items()]
+            error_list = [
+                '{0}: {1}'.format(k, ', '.join(v))
+                for k, v in validator.errors.items()
+            ]
             raise ValueError('. '.join(error_list))
         self.__config = validator.document