CLOS-2759: The elevation process failed during stage 3 - Cache-only enabled but no cache for 'cloudlinux-x86_64-server-8'

* Pin CLN mirror before downloading packages to cache
* Unpin after first boot
* Fix error handling in CLN cache only flag handling

Author: Grigory Ponomarenko

repos/system_upgrade/cloudlinux/actors/setclncacheonlyflag/actor.py

@@ -1,7 +1,7 @@
 from leapp.actors import Actor
 from leapp.tags import PreparationPhaseTag, IPUWorkflowTag
 from leapp.libraries.common.cllaunch import run_on_cloudlinux
-
+from leapp.libraries.common.cln_switch import get_cln_cacheonly_flag_path
 
 class SetClnCacheOnlyFlag(Actor):
     """
@@ -20,6 +20,7 @@ class SetClnCacheOnlyFlag(Actor):
     @run_on_cloudlinux
     def process(self):
         # TODO: Use a more reliable method to detect if we're running from the isolated userspace
-        # TODO: Replace hardcoded path with a constant (from target_userspace_creator.constants?)
-        with open('/var/lib/leapp/el8userspace/etc/cln_leapp_in_progress', 'w') as file:
+        # Currently we're directly placing the file into the userspace directory '/var/lib/leapp/el{}userspace'
+        # There should be better options
+        with open(get_cln_cacheonly_flag_path(), 'w') as file:
             file.write('1')

repos/system_upgrade/cloudlinux/actors/switchclnchanneldownload/actor.py

@@ -1,13 +1,21 @@
+import os
+import json
+
 from leapp.actors import Actor
 from leapp.libraries.stdlib import api
 from leapp.tags import DownloadPhaseTag, IPUWorkflowTag
 from leapp.libraries.stdlib import CalledProcessError
 from leapp.libraries.common.cllaunch import run_on_cloudlinux
-from leapp.libraries.common.cln_switch import cln_switch
+from leapp.libraries.common.cln_switch import cln_switch, get_target_userspace_path
 from leapp import reporting
 from leapp.reporting import Report
 
 
+
+CLN_REPO_ID = "cloudlinux-x86_64-server-8"
+DEFAULT_CLN_MIRROR = "https://xmlrpc.cln.cloudlinux.com/XMLRPC/"
+
+
 class SwitchClnChannelDownload(Actor):
     """
     Switch CLN channel from 7 to 8 to be able to download upgrade packages.
@@ -48,3 +56,45 @@ def process(self):
             api.current_logger().error(
                 "Could not call RHN command: Message: %s", str(e), exc_info=True
             )
+
+        self._pin_cln_mirror()
+
+    def _pin_cln_mirror(self):
+        """Pin CLN mirror"""
+        target_userspace = get_target_userspace_path()
+        api.current_logger().info("Pin CLN mirror: target userspace=%s", target_userspace)
+
+        # load last mirror URL from dnf spacewalk plugin cache
+        spacewalk_settings = {}
+
+        # find the mirror used in the last transaction
+        # (expecting to find the one used in dnf_package_download actor)
+        spacewalk_json_path = os.path.join(target_userspace, 'var/lib/dnf/_spacewalk.json')
+        try:
+            with open(spacewalk_json_path) as file:
+                spacewalk_settings = json.load(file)
+        except (OSError, IOError, ValueError):
+            api.current_logger().error(
+                "No spacewalk settings found in %s - can't identify the last used CLN mirror",
+                spacewalk_json_path,
+            )
+
+        mirror_url = spacewalk_settings.get(CLN_REPO_ID, {}).get("url", [DEFAULT_CLN_MIRROR])[0]
+
+        # pin mirror
+        for mirrorlist_path in [
+            '/etc/mirrorlist',
+            os.path.join(target_userspace, 'etc/mirrorlist'),
+        ]:
+            with open(mirrorlist_path, 'w') as file:
+                file.write(mirror_url + '\n')
+            api.current_logger().info("Pin CLN mirror %s in %s", mirror_url, mirrorlist_path)
+
+        for up2date_path in [
+            '/etc/sysconfig/rhn/up2date',
+            os.path.join(target_userspace, 'etc/sysconfig/rhn/up2date'),
+        ]:
+            # At some point up2date in `target_userspace` might be overwritten by a default one
+            with open(up2date_path, 'a+') as file:
+                file.write('\nmirrorURL[comment]=Set mirror URL to /etc/mirrorlist\nmirrorURL=file:///etc/mirrorlist\n')
+            api.current_logger().info("Updated up2date_path %s", up2date_path)

repos/system_upgrade/cloudlinux/actors/unpinclnmirror/actor.py

@@ -0,0 +1,47 @@
+import os
+
+from leapp.actors import Actor
+from leapp.libraries.common.cllaunch import run_on_cloudlinux
+from leapp.libraries.common.cln_switch import get_target_userspace_path
+from leapp.tags import FirstBootPhaseTag, IPUWorkflowTag
+
+class UnpinClnMirror(Actor):
+    """
+    Remove the pinned CLN mirror.
+    See the pin_cln_mirror actor for more details.
+    """
+
+    name = 'unpin_cln_mirror'
+    consumes = ()
+    produces = ()
+    tags = (IPUWorkflowTag, FirstBootPhaseTag)
+
+    CLN_REPO_ID = "cloudlinux-x86_64-server-8"
+    DEFAULT_CLN_MIRROR = "https://xmlrpc.cln.cloudlinux.com/XMLRPC/"
+
+    @run_on_cloudlinux
+    def process(self):
+        target_userspace = get_target_userspace_path()
+
+        for mirrorlist_path in [
+            '/etc/mirrorlist',
+            os.path.join(target_userspace, 'etc/mirrorlist'),
+        ]:
+            try:
+                os.remove(mirrorlist_path)
+            except OSError:
+                self.log.info('Can\'t remove %s, file does not exist, doing nothing', mirrorlist_path)
+
+        for up2date_path in [
+            '/etc/sysconfig/rhn/up2date',
+            os.path.join(target_userspace, 'etc/sysconfig/rhn/up2date'),
+        ]:
+            try:
+                with open(up2date_path, 'r') as file:
+                    lines = [
+                        line for line in file.readlines() if 'etc/mirrorlist' not in line
+                    ]
+                with open(up2date_path, 'w') as file:
+                    file.writelines(lines)
+            except (OSError, IOError, ValueError):
+                self.log.info('Can update %s file, doing nothing', up2date_path)

repos/system_upgrade/cloudlinux/actors/unsetclncacheonlyflag/actor.py

@@ -1,6 +1,7 @@
 from leapp.actors import Actor
 from leapp.tags import FirstBootPhaseTag, IPUWorkflowTag
 from leapp.libraries.common.cllaunch import run_on_cloudlinux
+from leapp.libraries.common.cln_switch import get_cln_cacheonly_flag_path
 
 import os
 
@@ -17,6 +18,6 @@ class UnsetClnCacheOnlyFlag(Actor):
     @run_on_cloudlinux
     def process(self):
         try:
-            os.remove('/var/lib/leapp/el8userspace/etc/cln_leapp_in_progress')
-        except FileNotFoundError:
+            os.remove(get_cln_cacheonly_flag_path())
+        except OSError:
             self.log.info('CLN cache file marker does not exist, doing nothing.')

repos/system_upgrade/cloudlinux/libraries/cln_switch.py

@@ -1,9 +1,46 @@
+import os
+
 from leapp.libraries.stdlib import api
 from leapp.libraries.stdlib import run
+from leapp.libraries.common.config.version import get_target_major_version
 
 SWITCH_BIN = "/usr/sbin/cln-switch-channel"
+TARGET_USERSPACE = '/var/lib/leapp/el{}userspace'
+CLN_CACHEONLY_MARKER = '/etc/cln_leapp_in_progress'
+
+def get_target_userspace_path():
+    """
+    Returns the path to the target OS userspace directory.
+
+    Used as a root dir for Leapp-related package operations.
+    Modifications performed in this directory are not visible to the host OS.
+    """
+    return TARGET_USERSPACE.format(get_target_major_version())
+
+def get_cln_cacheonly_flag_path():
+    """
+    Get the path to the flag file used to prevent the dnf-spacewalk-plugin
+    from contacting the CLN server during transaction.
+
+    Effectively forces the plugin to act as if network connectivity was disabled,
+    (no matter if it actually is or not), making it use the local cache only.
+
+    If this flag isn't present during the upgrade,
+    the plugin would attempt to contact the CLN server and fail due to the lack
+    of network connectivity, disrupting the upgrade.
+
+    DNF plugin runs in the target OS userspace, so the flag must be placed there.
+    """
+    return os.path.join(get_target_userspace_path(), CLN_CACHEONLY_MARKER.lstrip('/'))
 
 def cln_switch(target):
+    """
+    Switch the CloudLinux Network channel to the specified target OS.
+
+    Target OS is stored server-side, so the switch is permanent unless changed again.
+    For a CL7 to CL8 upgrade, we need to switch to the CL8 channel to
+    get served the correct packages.
+    """
     switch_cmd = [SWITCH_BIN, "-t", str(target), "-o", "-f"]
     yum_clean_cmd = ["yum", "clean", "all"]
     res = run(switch_cmd)

repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py

@@ -299,6 +299,13 @@ def _prep_repository_access(context, target_userspace):
         with open(os.path.join(target_etc, 'dnf/plugins/spacewalk.conf'), 'w') as f:
             f.writelines(new_lines)
 
+    if os.path.isfile('/etc/mirrorlist'):
+        try:
+            os.remove(os.path.join(target_etc, 'mirrorlist'))
+        except OSError:
+            pass
+        context.copy_from('/etc/mirrorlist', os.path.join(target_etc, 'mirrorlist'))
+
     # NOTE: we cannot just remove the original target yum.repos.d dir
     # as e.g. in case of RHUI a special RHUI repofiles are installed by a pkg
     # when the target userspace container is created. Removing these files we loose