Move common logic from Build node to library (#205)

* Move common logic from Build node to library (#185)

* Fix WrappedGitRepo class does not have archive method
Update build library to new 0.1.1 version

Fixes: AlmaLinux/build-system#417

* Fix tests

Author: maccelf

Dockerfile

@@ -7,7 +7,7 @@ RUN <<EOT
   dnf upgrade -y
   dnf install -y \
     gcc gcc-c++ make cmake git mock mock-rpmautospec keyrings-filesystem sudo \
-    libicu libicu-devel kernel-rpm-macros createrepo_c cpio \
+    libicu libicu-devel kernel-rpm-macros createrepo_c cpio e2fsprogs \
     python3-devel python3-lxml python3-createrepo_c python3-libmodulemd \
     centpkg \
     fedpkg

almalinux_build_node.py

@@ -10,16 +10,16 @@
 from threading import Event
 
 import sentry_sdk
+from albs_build_lib.builder.mock.mock_environment import MockError
+from albs_common_lib.errors import BuildError, BuildExcluded
+from albs_common_lib.utils.file_utils import chown_recursive, rm_sudo
+from albs_common_lib.utils.spec_parser import SpecParseError
 
 import build_node.build_node_globals as node_globals
 from build_node.build_node_builder import BuildNodeBuilder
 from build_node.build_node_config import BuildNodeConfig
-from build_node.build_node_errors import BuildError, BuildExcluded
 from build_node.build_node_supervisor import BuilderSupervisor
-from build_node.mock.mock_environment import MockError
 from build_node.utils.config import locate_config_file
-from build_node.utils.file_utils import chown_recursive, rm_sudo
-from build_node.utils.spec_parser import SpecParseError
 
 running = True
 
@@ -34,7 +34,7 @@ def init_args_parser():
     """
     parser = argparse.ArgumentParser(
         prog='castor_build_node',
-        description='CloudLinux Build System build node',
+        description='AlmaLinux Build System build node',
     )
     parser.add_argument('-c', '--config', help='configuration file path')
     parser.add_argument('-i', '--id', help='build node unique identifier')

build_node/build_node_builder.py

@@ -1,9 +1,5 @@
-# -*- mode:python; coding:utf-8; -*-
-# author: Eugene Zamriy <[email protected]>
-# created: 2017-10-19
-
 """
-CloudLinux Build System build thread implementation.
+AlmaLinux Build System build thread implementation.
 """
 
 import datetime
@@ -13,31 +9,30 @@
 import platform
 import pprint
 import random
-import threading
 import typing
 import urllib.parse
 
 import requests
 import requests.adapters
+from albs_build_lib.builder.base_builder import measure_stage
+from albs_build_lib.builder.base_thread_slave_builder import BaseSlaveBuilder
+from albs_build_lib.builder.models import Task
+from albs_common_lib.errors import BuildError, BuildExcluded
+from albs_common_lib.utils.file_utils import (
+    filter_files,
+    rm_sudo,
+)
 from immudb_wrapper import ImmudbWrapper
 from requests.packages.urllib3.util.retry import Retry
 from sentry_sdk import capture_exception
 
 from build_node import constants
-from build_node.build_node_errors import BuildError, BuildExcluded
-from build_node.builders.base_builder import measure_stage
 from build_node.builders.base_rpm_builder import BaseRPMBuilder
-from build_node.models import Task
 from build_node.uploaders.pulp import PulpRpmUploader
 from build_node.utils.codenotary import notarize_build_artifacts
-from build_node.utils.file_utils import (
-    clean_dir,
-    filter_files,
-    rm_sudo,
-)
 
 
-class BuildNodeBuilder(threading.Thread):
+class BuildNodeBuilder(BaseSlaveBuilder):
     """Build thread."""
 
     def __init__(
@@ -62,19 +57,23 @@ def __init__(
         graceful_terminated_event : threading.Event
             Shows, if process got "kill -10" signal.
         """
-        super(BuildNodeBuilder, self).__init__(name=f'Builder-{thread_num}')
+        super().__init__(
+            thread_num=thread_num,
+        )
         self.__config = config
+        # current task processing start timestamp
+        self.__start_ts = None
         self.__working_dir = os.path.join(
             config.working_dir, 'builder-{0}'.format(thread_num)
         )
         self.init_working_dir(self.__working_dir)
-        self.__logger = None
-        self.__current_task_id = None
-        # current task processing start timestamp
-        self.__start_ts = None
+        self.__terminated_event = terminated_event
+        self.__graceful_terminated_event = graceful_terminated_event
         # current task builder object
         self.__builder = None
         self.__session = None
+        self.__logger = None
+        self.__current_task_id = None
         self._immudb_wrapper = None
         self._codenotary_enabled = self.__config.codenotary_enabled
         self._build_stats: typing.Optional[
@@ -87,9 +86,6 @@ def __init__(
             self.__config.pulp_chunk_size,
             self.__config.pulp_uploader_max_workers,
         )
-
-        self.__terminated_event = terminated_event
-        self.__graceful_terminated_event = graceful_terminated_event
         self.__hostname = platform.node()
         self.__task_queue = task_queue
 
@@ -149,8 +145,12 @@ def run(self):
                 )
                 capture_exception(e)
             finally:
-                only_logs = not bool(
-                    filter_files(artifacts_dir, lambda f: f.endswith('.rpm'))
+                only_logs = not (
+                    bool(
+                        filter_files(
+                            artifacts_dir, lambda f: f.endswith('.rpm')
+                        )
+                    )
                 )
                 if success is False:
                     only_logs = True
@@ -220,7 +220,9 @@ def run(self):
                         self.__report_excluded_task(task, build_artifacts)
                     else:
                         self.__report_done_task(
-                            task, success=success, artifacts=build_artifacts
+                            task,
+                            success=success,
+                            artifacts=build_artifacts,
                         )
                 except Exception:
                     self.__logger.exception(
@@ -258,7 +260,7 @@ def __cas_notarize_artifacts(
 
     def __build_packages(self, task, task_dir, artifacts_dir):
         """
-        Creates a suitable builder instance and builds RPM or Debian packages.
+        Creates a suitable builder instance and builds RPM packages.
 
         Parameters
         ----------
@@ -309,7 +311,12 @@ def __report_excluded_task(self, task, artifacts):
             **kwargs,
         )
 
-    def __report_done_task(self, task, success=True, artifacts=None):
+    def __report_done_task(
+        self,
+        task,
+        success=True,
+        artifacts=None,
+    ):
         if not artifacts:
             artifacts = []
         kwargs = {
@@ -350,15 +357,18 @@ def __call_master(
         **parameters,
     ):
         full_url = urllib.parse.urljoin(
-            self.__config.master_url, f'build_node/{endpoint}'
+            self.__config.master_url,
+            f'build_node/{endpoint}',
         )
         if endpoint in ('build_done', 'get_task'):
             session_method = self.__session.post
         else:
             session_method = self.__session.get
         try:
             response = session_method(
-                full_url, json=parameters, timeout=self.__config.request_timeout
+                full_url,
+                json=parameters,
+                timeout=self.__config.request_timeout,
             )
             # Special case when build was already done
             if response.status_code == requests.codes.conflict:
@@ -370,19 +380,6 @@ def __call_master(
         except Exception:
             self.__logger.exception('%s', err_msg)
 
-    @staticmethod
-    def init_working_dir(working_dir):
-        """
-        Creates a non-existent working directory or cleans it up from previous
-        builds.
-        """
-        if os.path.exists(working_dir):
-            logging.debug('cleaning the %s working directory', working_dir)
-            clean_dir(working_dir)
-        else:
-            logging.debug('creating the %s working directory', working_dir)
-            os.makedirs(working_dir, 0o750)
-
     def __init_task_logger(self, log_file):
         """
         Task logger initialization, configures a build thread logger to write
@@ -423,35 +420,6 @@ def __close_task_logger(self, task_handler):
         task_handler.close()
         self.__logger.handlers.remove(task_handler)
 
-    @staticmethod
-    def init_thread_logger(log_file):
-        """
-        Build thread logger initialization.
-
-        Parameters
-        ----------
-        log_file : str
-            Log file path.
-
-        Returns
-        -------
-        logging.Logger
-            Build thread logger.
-        """
-        logger = logging.getLogger(
-            'bt-{0}-logger'.format(threading.current_thread().name)
-        )
-        logger.handlers = []
-        logger.setLevel(logging.DEBUG)
-        formatter = logging.Formatter(
-            "%(asctime)s %(levelname)-8s: %(message)s", "%H:%M:%S %d.%m.%y"
-        )
-        handler = logging.FileHandler(log_file)
-        handler.setLevel(logging.DEBUG)
-        handler.setFormatter(formatter)
-        logger.addHandler(handler)
-        return logger
-
     @property
     def current_task_id(self):
         return self.__current_task_id

build_node/build_node_config.py

@@ -1,9 +1,5 @@
-# -*- mode:python; coding:utf-8; -*-
-# author: Eugene Zamriy <[email protected]>
-# created: 2017-10-19
-
 """
-CloudLinux Build System build node configuration storage.
+AlmaLinux Build System build node configuration storage.
 """
 
 import os

build_node/build_node_errors.py

@@ -1,10 +1,6 @@
-# -*- mode:python; coding:utf-8; -*-
-# author: Eugene Zamriy <[email protected]>
-# created: 2019-04-19
+"""AlmaLinux Build System node global variables."""
 
-"""CloudLinux Build System node global variables."""
-
-from build_node.mock.supervisor import MockSupervisor
+from albs_build_lib.builder.mock.supervisor import MockSupervisor
 
 __all__ = ['init_supervisors', 'MOCK_SUPERVISOR']
 
@@ -23,4 +19,7 @@ def init_supervisors(config):
         Build node configuration file.
     """
     global MOCK_SUPERVISOR
-    MOCK_SUPERVISOR = MockSupervisor(config.mock_configs_storage_dir, config.base_arch)
+    MOCK_SUPERVISOR = MockSupervisor(
+        storage_dir=config.mock_configs_storage_dir,
+        host_arch=config.base_arch,
+    )

build_node/build_node_globals.py

@@ -1,36 +1,31 @@
 import logging
-import threading
 import traceback
 import urllib.parse
 
 import requests
 import requests.adapters
-from urllib3 import Retry
 from cachetools import TTLCache
+from albs_build_lib.builder.base_supervisor import BaseSupervisor
+from albs_common_lib.utils.file_utils import file_url_exists
+from urllib3 import Retry
 
 from build_node import constants
-from build_node.utils.file_utils import file_url_exists
 
 
-class BuilderSupervisor(threading.Thread):
+class BuilderSupervisor(BaseSupervisor):
 
-    def __init__(
-        self,
-        config,
-        builders,
-        terminated_event,
-        task_queue,
-    ):
-        self.config = config
-        self.builders = builders
-        self.terminated_event = terminated_event
+    def __init__(self, config, builders, terminated_event, task_queue):
         self.__session = None
         self.__task_queue = task_queue
         self.__cached_config = TTLCache(
             maxsize=config.cache_size,
             ttl=config.cache_update_interval,
-            )
-        super(BuilderSupervisor, self).__init__(name='BuildersSupervisor')
+        )
+        super().__init__(
+            config=config,
+            builders=builders,
+            terminated_event=terminated_event,
+        )
 
     def __generate_request_session(self):
         retry_strategy = Retry(
@@ -75,11 +70,6 @@ def __request_build_task(self):
                     traceback.format_exc(),
                 )
 
-    def get_active_tasks(self):
-        return set([b.current_task_id for b in self.builders]) - set([
-            None,
-        ])
-
     def __report_active_tasks(self):
         active_tasks = self.get_active_tasks()
         logging.debug('Sending active tasks: {}'.format(active_tasks))