Fix Alma finder to handle new vault HTML format

The top page of the AlmaLinux vault has changed from plain XML to a full
HTML page. Update the Alma finder to parse the new format correctly.

Drive-by fixes:
 - Add a CLI for AlmaLinux finder
 - recompile requirements
 - allow redirections for HEAD requests when testing URLs

 Issue: #68

Author: juledwar

pyproject.toml

@@ -39,6 +39,7 @@ dependencies = [
     "requests-oauthlib>=1.3.0",
     "lxml>=4.6.3",
     "dogpile.cache>=1.1.5",
+    "beautifulsoup4",
 ]
 
 [project.optional-dependencies]

requirements/requirements-bootstrap.txt

@@ -97,6 +97,29 @@ def centos(
         )
         return cls.find(centos_finder)
 
+    @classmethod
+    def almalinux(
+        cls,
+        name,
+        version,
+        repos=None,
+        source_repos=None,
+        binary_repos=None,
+        timeout=None,
+    ):
+        alma_finder = finder.factory(
+            "almalinux",
+            name=name,
+            version=version,
+            s_type=finder.SourceType.os,
+            source_repos=source_repos,
+            binary_repos=binary_repos,
+            cache_backend="dogpile.cache.memory",
+            cache_args=dict(cache_dict=LRU_CACHE),
+            timeout=timeout,
+        )
+        return cls.find(alma_finder)
+
     @classmethod
     def alpine(cls, name, version, aports_dir, timeout=None):
         alpine_finder = finder.factory(
@@ -243,7 +266,7 @@ def make_archive_from_discovery_source(disc_src, fname):
     "--repo",
     default=(),
     multiple=True,
-    help="For CentOS, name of repo to use instead of defaults. "
+    help="For CentOS/Almalinux, name of repo to use instead of defaults. "
     "Use 'optimal' to use an extended optimal set. May be repeated.",
 )
 @click.option(

requirements/requirements-cli.txt

@@ -246,7 +246,9 @@ def test_url(self, url, **kwargs):
 
     def _head_url(self, url, **kwargs):
         def inner():
-            response = requests.head(url, timeout=self.timeout, **kwargs)
+            response = requests.head(
+                url, timeout=self.timeout, allow_redirects=True, **kwargs
+            )
             if response.status_code == requests.codes.not_allowed:
                 # HEAD not available; we can try to download it instead and
                 # abort before starting the stream.

requirements/requirements-test.txt

@@ -3,7 +3,7 @@
 
 import re
 
-from lxml import html
+from bs4 import BeautifulSoup
 
 import soufi.finders.yum as yum_finder
 from soufi import finder
@@ -25,21 +25,30 @@ class AlmaLinuxFinder(yum_finder.YumFinder):
 
     distro = finder.Distro.almalinux.value
 
-    def _get_dirs(self):
-        """Get all the possible Vault dirs that could match."""
-        content = self.get_url(VAULT).content
-        tree = html.fromstring(content)
-        # Ignore beta releases; we may want to make this a switchable behavior
-        retval = tree.xpath("//a/text()[not(contains(.,'-beta'))]")
+    def _get_vault_repo_versions(self):
+        response = self.get_url(VAULT)
+        soup = BeautifulSoup(response.text, "html.parser")
+        versions = []
         # AlmaLinux Vault is fond of symlinking the current point release to a
         # directory with just the major version number, e.g., `6.10/`->`6/`.
         # This means that such directories are inherently unstable and their
         # contents are subject to change without notice, so we'll ignore
         # them in favour of the "full" names.
-        dirs = [dir.rstrip('/') for dir in retval if re.match(r'\d+\.\d', dir)]
+        for link in soup.find_all("a", href=re.compile(r"^./\d+\.\d")):
+            version = link.get("href").rstrip("/")
+            if '-beta' in version:
+                # Ignore beta releases; we may want to make this a switchable
+                # behaviour.
+                continue
+            if re.match(r"^./\d+\.\d+", version):
+                versions.append(version[2:])
+        return sorted(versions, reverse=True)  # Sort to prioritize newer
 
-        # Walk the tree backwards, so that newer releases get searched first
-        return reversed(dirs)
+    def _get_dirs(self):
+        """Get all the possible Vault dirs that could match."""
+        versions = self._get_vault_repo_versions()
+        # Dir names are just the versions.
+        return versions
 
     def get_source_repos(self):
         """Determine which source search paths are valid.

requirements/requirements.txt

@@ -44,7 +44,8 @@ def get_source_url(self):
             f'{self.name}-{self.version}-sources.jar'
         )
         found = self.test_url(
-            MAVEN_REPO_URL, params=params, allow_redirects=True
+            MAVEN_REPO_URL,
+            params=params,
         )
         if not found:
             raise exceptions.SourceNotFound

soufi/cli.py

@@ -94,17 +94,23 @@ def extend_side_effects(self, mock_obj, extra_value):
         effects.append(extra_value)
         mock_obj.side_effect = effects
 
-    def patch_get_with_response(self, response_code, data=None, json=None):
+    def patch_get_with_response(
+        self, response_code, data=None, json=None, as_text=False
+    ):
         """Patch `requests.get` with the provided values.
 
         :param response_code: A requests.codes value, to mimic an HTTP status
         :param data: A string-like object to mimic Response.content
         :param json: A dict or list, to mimic what Response.json() would return
+        :param as_text: If True, set Response.text instead of Response.content
         :return: The created MagicMock, to add side-effects, etc.
         """
         fake_response = mock.MagicMock()
         fake_response.return_value.status_code = response_code
-        fake_response.return_value.content = data
+        if as_text:
+            fake_response.return_value.text = data
+        else:
+            fake_response.return_value.content = data
         fake_response.return_value.json.return_value = json
         return self.patch(requests, 'get', fake_response)