diff --git a/repos/system_upgrade/cloudlinux/actors/clearpackageconflicts/actor.py b/repos/system_upgrade/cloudlinux/actors/clearpackageconflicts/actor.py index fc642d5029..0bed04ed27 100644 --- a/repos/system_upgrade/cloudlinux/actors/clearpackageconflicts/actor.py +++ b/repos/system_upgrade/cloudlinux/actors/clearpackageconflicts/actor.py @@ -1,102 +1,28 @@ -import os -import errno -import shutil - from leapp.actors import Actor from leapp.models import InstalledRPM from leapp.tags import DownloadPhaseTag, IPUWorkflowTag from leapp.libraries.common.cllaunch import run_on_cloudlinux +from leapp.libraries.actor import clearpackageconflicts class ClearPackageConflicts(Actor): """ - Remove several python package files manually to resolve conflicts between versions of packages to be upgraded. + Remove several Python package files manually to resolve conflicts + between versions of packages to be upgraded. + + When the corresponding packages are detected, + the conflicting files are removed to allow for an upgrade to the new package versions. + + While most packages are handled automatically by the package manager, + some specific packages require direct intervention to resolve conflicts + between their own versions on different OS releases. """ name = "clear_package_conflicts" consumes = (InstalledRPM,) produces = () tags = (DownloadPhaseTag.Before, IPUWorkflowTag) - rpm_lookup = None - - def has_package(self, name): - """ - Check whether the package is installed. - Looks only for the package name, nothing else. - """ - if self.rpm_lookup: - return name in self.rpm_lookup - - def problem_packages_installed(self, problem_packages): - """ - Check whether any of the problem packages are present in the system. - """ - for pkg in problem_packages: - if self.has_package(pkg): - self.log.debug("Conflicting package {} detected".format(pkg)) - return True - return False - - def clear_problem_files(self, problem_files, problem_dirs): - """ - Go over the list of problem files and directories and remove them if they exist. - They'll be replaced by the new packages. - """ - for p_dir in problem_dirs: - try: - if os.path.isdir(p_dir): - shutil.rmtree(p_dir) - self.log.debug("Conflicting directory {} removed".format(p_dir)) - except OSError as e: - if e.errno != errno.ENOENT: - raise - - for p_file in problem_files: - try: - if os.path.isfile(p_file): - os.remove(p_file) - self.log.debug("Conflicting file {} removed".format(p_file)) - except OSError as e: - if e.errno != errno.ENOENT: - raise - - def alt_python37_handle(self): - """ - These alt-python37 packages are conflicting with their own builds for EL8. - """ - problem_packages = [ - "alt-python37-six", - "alt-python37-pytz", - ] - problem_files = [] - problem_dirs = [ - "/opt/alt/python37/lib/python3.7/site-packages/six-1.15.0-py3.7.egg-info", - "/opt/alt/python37/lib/python3.7/site-packages/pytz-2017.2-py3.7.egg-info", - ] - - if self.problem_packages_installed(problem_packages): - self.clear_problem_files(problem_files, problem_dirs) - - def lua_cjson_handle(self): - """ - lua-cjson package is conflicting with the incoming lua-cjson package for EL8. - """ - problem_packages = [ - "lua-cjson" - ] - problem_files = [ - "/usr/lib64/lua/5.1/cjson.so", - "/usr/share/lua/5.1/cjson/tests/bench.lua", - "/usr/share/lua/5.1/cjson/tests/genutf8.pl", - "/usr/share/lua/5.1/cjson/tests/test.lua", - ] - problem_dirs = [] - - if self.problem_packages_installed(problem_packages): - self.clear_problem_files(problem_files, problem_dirs) @run_on_cloudlinux def process(self): - # todo: (CLOS-3205) investigate why set is needed here - self.rpm_lookup = [rpm for rpm in self.consume(InstalledRPM)] - self.alt_python37_handle() + clearpackageconflicts.process() diff --git a/repos/system_upgrade/cloudlinux/actors/clearpackageconflicts/libraries/clearpackageconflicts.py b/repos/system_upgrade/cloudlinux/actors/clearpackageconflicts/libraries/clearpackageconflicts.py new file mode 100644 index 0000000000..f872c18947 --- /dev/null +++ b/repos/system_upgrade/cloudlinux/actors/clearpackageconflicts/libraries/clearpackageconflicts.py @@ -0,0 +1,90 @@ +import os +import errno +import shutil + +from leapp.libraries.stdlib import api +from leapp.models import InstalledRPM + + +def problem_packages_installed(problem_packages, lookup): + """ + Check whether any of the problem packages are present in the system. + """ + for pkg in problem_packages: + if pkg in lookup: + api.current_logger().debug("Conflicting package {} detected".format(pkg)) + return True + return False + + +def clear_problem_files(problem_files, problem_dirs): + """ + Go over the list of problem files and directories and remove them if they exist. + They'll be replaced by the new packages. + """ + for p_dir in problem_dirs: + try: + if os.path.isdir(p_dir): + shutil.rmtree(p_dir) + api.current_logger().debug("Conflicting directory {} removed".format(p_dir)) + except OSError as e: + if e.errno != errno.ENOENT: + raise + + for p_file in problem_files: + try: + if os.path.isfile(p_file): + os.remove(p_file) + api.current_logger().debug("Conflicting file {} removed".format(p_file)) + except OSError as e: + if e.errno != errno.ENOENT: + raise + + +def alt_python37_handle(package_lookup): + """ + These alt-python37 packages are conflicting with their own builds for EL8. + """ + problem_packages = [ + "alt-python37-six", + "alt-python37-pytz", + ] + problem_files = [] + problem_dirs = [ + "/opt/alt/python37/lib/python3.7/site-packages/six-1.15.0-py3.7.egg-info", + "/opt/alt/python37/lib/python3.7/site-packages/pytz-2017.2-py3.7.egg-info", + ] + + if problem_packages_installed(problem_packages, package_lookup): + clear_problem_files(problem_files, problem_dirs) + + +def lua_cjson_handle(package_lookup): + """ + lua-cjson package is conflicting with the incoming lua-cjson package for EL8. + """ + problem_packages = [ + "lua-cjson" + ] + problem_files = [ + "/usr/lib64/lua/5.1/cjson.so", + "/usr/share/lua/5.1/cjson/tests/bench.lua", + "/usr/share/lua/5.1/cjson/tests/genutf8.pl", + "/usr/share/lua/5.1/cjson/tests/test.lua", + ] + problem_dirs = [] + + if problem_packages_installed(problem_packages, package_lookup): + clear_problem_files(problem_files, problem_dirs) + + +def process(): + rpm_lookup = set() + # Each InstalledRPM is a list of RPM objects. + # There's a bunch of other fields, but all that we're interested in here is their names. + installed_rpm_messages = api.consume(InstalledRPM) + for rpm_list in installed_rpm_messages: + rpm_names = [item.name for item in rpm_list.items] + rpm_lookup.update(rpm_names) + + alt_python37_handle(rpm_lookup) diff --git a/repos/system_upgrade/cloudlinux/actors/clearpackageconflicts/tests/unit_test_clearpackageconflicts.py b/repos/system_upgrade/cloudlinux/actors/clearpackageconflicts/tests/unit_test_clearpackageconflicts.py new file mode 100644 index 0000000000..efdcbee9ea --- /dev/null +++ b/repos/system_upgrade/cloudlinux/actors/clearpackageconflicts/tests/unit_test_clearpackageconflicts.py @@ -0,0 +1,17 @@ +import pytest + +# from leapp import reporting +from leapp.libraries.actor import clearpackageconflicts + + +@pytest.mark.parametrize( + "problem_pkgs,lookup,expected_res", + ( + (["cagefs"], {"cagefs", "dnf"}, True), + (["lve-utils"], {"lve-utils", "dnf"}, True), + (["nonexistent-pkg"], {"cagefs", "dnf"}, False), + (["cagefs"], {"lve-utils", "dnf"}, False), + ), +) +def test_problem_packages_installed(problem_pkgs, lookup, expected_res): + assert expected_res == clearpackageconflicts.problem_packages_installed(problem_pkgs, lookup) diff --git a/repos/system_upgrade/cloudlinux/actors/refreshepel/actor.py b/repos/system_upgrade/cloudlinux/actors/refreshepel/actor.py new file mode 100644 index 0000000000..a0fd77fc8d --- /dev/null +++ b/repos/system_upgrade/cloudlinux/actors/refreshepel/actor.py @@ -0,0 +1,68 @@ +from __future__ import print_function +from operator import is_ +import os + +from leapp.actors import Actor +from leapp.tags import ApplicationsPhaseTag, IPUWorkflowTag +from leapp.libraries.common.cllaunch import run_on_cloudlinux +from leapp.libraries.common.backup import backup_and_remove +from leapp.libraries.common.config.version import get_target_major_version + +REPO_DIR = '/etc/yum.repos.d' + + +class RefreshEPEL(Actor): + """ + Check that the EPEL repositories are correctly configured after the upgrade. + + Depending on how the upgrade went, the EPEL repositories might still be targeting the old OS version. + This actor checks that the EPEL repositories are correctly configured and if not, it will install the + correct EPEL release package and refresh the repositories. + """ + + name = 'refresh_epel' + # We can't depend on InstalledRPM message because by this point + # the system is upgraded and the RPMs are not the same as when the data was collected. + consumes = () + produces = () + tags = (ApplicationsPhaseTag.After, IPUWorkflowTag) + + def clear_epel_repo_files(self): + for repofile in os.listdir(REPO_DIR): + if repofile.startswith('epel'): + epel_file = os.path.join(REPO_DIR, repofile) + backup_and_remove(epel_file) + + def install_epel_release_package(self, target_url): + os.system('dnf install {}'.format(target_url)) + self.log.info('EPEL release package installed: {}'.format(target_url)) + + @run_on_cloudlinux + def process(self): + epel_install_url = 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-{}.noarch.rpm'.format(get_target_major_version()) + + target_version = int(get_target_major_version()) + target_epel_release = epel_install_url.format(target_version) + + # EPEL release package name is 'epel-release' and the version should match the target OS version + epel_release_package = 'epel-release' + + is_epel_installed = os.system('rpm -q {}'.format(epel_release_package)) == 0 + is_correct_version = os.system('rpm -q {}-{}'.format(epel_release_package, target_version)) == 0 + epel_files_verified = os.system('rpm -V {}'.format(epel_release_package)) == 0 + + # It's possible (although unusual) that the correct EPEL release package is installed during the upgrade, + # but the EPEL repository files still point to the old OS version. + # This was observed on client machines before. + + if (is_epel_installed and not is_correct_version) or not epel_files_verified: + # If the EPEL release package is installed but not the correct version, remove it + # Same if the files from the package were modified + os.system('rpm -e {}'.format(epel_release_package)) + if not is_epel_installed or not is_correct_version or not epel_files_verified: + # Clear the EPEL repository files + self.clear_epel_repo_files() + # Install the correct EPEL release package + self.install_epel_release_package(target_epel_release) + # Logging for clarity + self.log.info('EPEL release package installation invoked for: {}'.format(target_epel_release)) diff --git a/repos/system_upgrade/cloudlinux/actors/replacerpmnewconfigs/actor.py b/repos/system_upgrade/cloudlinux/actors/replacerpmnewconfigs/actor.py index a454ed381a..e1a4c009fd 100644 --- a/repos/system_upgrade/cloudlinux/actors/replacerpmnewconfigs/actor.py +++ b/repos/system_upgrade/cloudlinux/actors/replacerpmnewconfigs/actor.py @@ -7,12 +7,15 @@ from leapp import reporting from leapp.reporting import Report from leapp.libraries.common.cllaunch import run_on_cloudlinux +from leapp.libraries.common.backup import backup_and_remove, LEAPP_BACKUP_SUFFIX REPO_DIR = '/etc/yum.repos.d' -REPO_DELETE_MARKERS = ['cloudlinux', 'imunify', 'epel'] +# These markers are used to identify which repository files should be directly replaced with new versions. +REPO_DELETE_MARKERS = ['cloudlinux', 'imunify'] +# These markers are used to identify which repository files should be replaced with new versions and backed up. REPO_BACKUP_MARKERS = [] +# This suffix is used to identify .rpmnew files that appear after package upgrade. RPMNEW = '.rpmnew' -LEAPP_BACKUP_SUFFIX = '.leapp-backup' class ReplaceRpmnewConfigs(Actor): @@ -30,32 +33,31 @@ def process(self): deleted_repofiles = [] renamed_repofiles = [] - for reponame in os.listdir(REPO_DIR): - if any(mark in reponame for mark in REPO_DELETE_MARKERS) and RPMNEW in reponame: - base_reponame = reponame[:-len(RPMNEW)] - base_path = os.path.join(REPO_DIR, base_reponame) - new_file_path = os.path.join(REPO_DIR, reponame) + for rpmnew_filename in os.listdir(REPO_DIR): + if any(mark in rpmnew_filename for mark in REPO_DELETE_MARKERS) and rpmnew_filename.endswith(RPMNEW): + main_reponame = rpmnew_filename[:-len(RPMNEW)] + main_file_path = os.path.join(REPO_DIR, main_reponame) + rpmnew_file_path = os.path.join(REPO_DIR, rpmnew_filename) - os.unlink(base_path) - os.rename(new_file_path, base_path) - deleted_repofiles.append(base_reponame) - self.log.debug('Yum repofile replaced: {}'.format(base_path)) + os.unlink(main_file_path) + os.rename(rpmnew_file_path, main_file_path) + deleted_repofiles.append(main_reponame) + self.log.debug('Yum repofile replaced: {}'.format(main_file_path)) - if any(mark in reponame for mark in REPO_BACKUP_MARKERS) and RPMNEW in reponame: - base_reponame = reponame[:-len(RPMNEW)] - base_path = os.path.join(REPO_DIR, base_reponame) - new_file_path = os.path.join(REPO_DIR, reponame) - backup_path = os.path.join(REPO_DIR, base_reponame + LEAPP_BACKUP_SUFFIX) + if any(mark in rpmnew_filename for mark in REPO_BACKUP_MARKERS) and rpmnew_filename.endswith(RPMNEW): + main_reponame = rpmnew_filename[:-len(RPMNEW)] + main_file_path = os.path.join(REPO_DIR, main_reponame) + rpmnew_file_path = os.path.join(REPO_DIR, rpmnew_filename) - os.rename(base_path, backup_path) - os.rename(new_file_path, base_path) - renamed_repofiles.append(base_reponame) - self.log.debug('Yum repofile replaced with backup: {}'.format(base_path)) + backup_and_remove(main_file_path) + os.rename(rpmnew_file_path, main_file_path) + renamed_repofiles.append(main_reponame) + self.log.debug('Yum repofile replaced with backup: {}'.format(main_file_path)) # Disable any old repositories. - for reponame in os.listdir(REPO_DIR): - if LEAPP_BACKUP_SUFFIX in reponame: - repofile_path = os.path.join(REPO_DIR, reponame) + for repofile_name in os.listdir(REPO_DIR): + if LEAPP_BACKUP_SUFFIX in repofile_name: + repofile_path = os.path.join(REPO_DIR, repofile_name) for line in fileinput.input(repofile_path, inplace=True): if line.startswith('enabled'): print("enabled = 0") @@ -66,7 +68,7 @@ def process(self): deleted_string = '\n'.join(['{}'.format(repofile_name) for repofile_name in deleted_repofiles]) replaced_string = '\n'.join(['{}'.format(repofile_name) for repofile_name in renamed_repofiles]) reporting.create_report([ - reporting.Title('CloudLinux repository config files replaced by updated versions'), + reporting.Title('Repository config files replaced by updated versions'), reporting.Summary( 'One or more RPM repository configuration files ' 'have been replaced with new versions provided by the upgraded packages. ' diff --git a/repos/system_upgrade/cloudlinux/libraries/backup.py b/repos/system_upgrade/cloudlinux/libraries/backup.py index 9002f569cb..7a40448bfd 100644 --- a/repos/system_upgrade/cloudlinux/libraries/backup.py +++ b/repos/system_upgrade/cloudlinux/libraries/backup.py @@ -1,3 +1,17 @@ +""" +Backup functionality for CloudLinux system upgrade process. + +This module provides utilities for backing up and restoring system configuration files +during the CloudLinux upgrade process. It includes functions for: +- Backing up files to a specified backup directory +- Creating in-place backups with .leapp-backup suffix +- Backing up and removing files +- Restoring files from backups + +Typically used in other CloudLinux upgrade actors to ensure that some specific configuration files +are preserved and can be restored in case of issues during the upgrade process. +""" + import os import shutil from leapp.libraries.stdlib import api @@ -10,6 +24,28 @@ ] BACKUP_DIR = "/var/lib/leapp/cl_backup" +LEAPP_BACKUP_SUFFIX = ".leapp-backup" + + +def backup_and_remove(path): + # type: (str) -> None + """ + Backup the file in-place and remove the original file. + + :param path: Path of the file to backup and remove. + """ + backup_file_in_place(path) + os.unlink(path) + + +def backup_file_in_place(path): + # type: (str) -> None + """ + Backup file in place, creating a copy of it with the same name and .leapp-backup suffix. + + :param path: Path of the file to backup. + """ + backup_file(path, path + LEAPP_BACKUP_SUFFIX) def backup_file(source, destination, backup_directory=""): @@ -19,14 +55,18 @@ def backup_file(source, destination, backup_directory=""): :param source: Path of the file to backup. :param destination: Destination name of a file in the backup directory. - :param dir: Backup directory override, defaults to None + If an absolute path is provided, it will be used as the destination path. + :param backup_directory: Backup directory override, defaults to None """ - if not backup_directory: - backup_directory = BACKUP_DIR - if not os.path.isdir(backup_directory): - os.makedirs(backup_directory) - - dest_path = os.path.join(backup_directory, destination) + # If destination is an absolute path, use it as the destination path + if os.path.isabs(destination): + dest_path = destination + else: + if not backup_directory: + backup_directory = BACKUP_DIR + if not os.path.isdir(backup_directory): + os.makedirs(backup_directory) + dest_path = os.path.join(backup_directory, destination) api.current_logger().debug('Backing up file: {} to {}'.format(source, dest_path)) shutil.copy(source, dest_path)