feature: add PulpProject class for Pulp-based package repository support

Add a `PulpProject` subclass of `GitbuilderProject` so teuthology can use
Pulp-hosted package repos when `config.use_artifacts` is `pulp`, alongside
the existing `Gitbuilder` and `Shaman` artifact backends.

Signed-off-by: Vaibhav Mahajan <vamahaja@redhat.com>

Author: vamahaja

docs/detailed_test_config.rst

@@ -300,10 +300,24 @@ Shaman options
 ==============
 
 Shaman is a helper class which could be used to build the uri for specified
-packages based the 'shaman_host': 'shaman.ceph.com'.
+packages based the 'artifacts_host' option.
 
 Options::
 
-    use_shaman: True # Enable to use Shaman, False as default
+    use_artifacts: 'shaman'
+    artifacts_host: 'shaman.ceph.com'
     shaman:
       force_noarch: True # Force to use "noarch" to build the uri
+
+Pulp options
+============
+
+Pulp is a package manager (https://pulpproject.org/) that could be used to
+build the uri for specified packages based the 'artifacts_host' option.
+
+Options::
+
+    use_artifacts: 'pulp'
+    artifacts_host: 'pulp.example.com'
+    pulp:
+      force_noarch: True # Force to use "noarch" to build the uri

docs/siteconfig.rst

@@ -269,3 +269,12 @@ Here is a sample configuration with many of the options set and documented::
     # Do not allow more than that many jobs in a single run by default.
     # To disable this check use 0.
     job_threshold: 500
+
+    # Setting for artifacts manager (shaman or pulp)
+    use_artifacts: 'pulp'
+    artifacts_host: 'pulp.example.com'
+
+    # Settings for pulp package manager (https://pulpproject.org/)
+    pulp:
+      username: '<username>'
+      password: '<password>'

teuthology/config.py

@@ -181,8 +181,8 @@ class TeuthologyConfig(YamlConfig):
         'kojiroot_url': 'https://kojipkgs.fedoraproject.org/packages',
         'koji_task_url': 'https://kojipkgs.fedoraproject.org/work/',
         'baseurl_template': 'http://{host}/{proj}-{pkg_type}-{dist}-{arch}-{flavor}/{uri}',
-        'use_shaman': True,
-        'shaman_host': 'shaman.ceph.com',
+        'use_artifacts': 'shaman',
+        'artifacts_host': 'shaman.ceph.com',
         'teuthology_path': None,
         'suite_verify_ceph_hash': True,
         'suite_allow_missing_packages': False,

teuthology/openstack/setup-openstack.sh

@@ -54,7 +54,7 @@ function create_config() {
 
     cat > ~/.teuthology.yaml <<EOF
 $archive_upload
-use_shaman: false
+use_artifacts: 'shaman'
 archive_upload_key: teuthology/openstack/archive-key
 lock_server: http://localhost:8080/
 results_server: http://localhost:8080/

teuthology/packaging.py

@@ -847,7 +847,7 @@ def _remove_deb_repo(self):
 class ShamanProject(GitbuilderProject):
     def __init__(self, project, job_config, ctx=None, remote=None):
         super(ShamanProject, self).__init__(project, job_config, ctx, remote)
-        self.query_url = 'https://%s/api/' % config.shaman_host
+        self.query_url = 'https://%s/api/' % config.artifacts_host
 
         # Force to use the "noarch" instead to build the uri.
         self.force_noarch = self.job_config.get("shaman", {}).get("force_noarch", False)
@@ -1054,13 +1054,113 @@ def _remove_rpm_repo(self):
         )
 
 
+class PulpProject(GitbuilderProject):
+    def __init__(self, project, job_config, ctx=None, remote=None):
+        super(PulpProject, self).__init__(project, job_config, ctx, remote)
+
+        # Set the url for the pulp server.
+        self.pulp_server_url = f'http://{config.artifacts_host}'
+        self.pulp_username = config.pulp.get("username")
+        self.pulp_password = config.pulp.get("password")
+
+        if not self.pulp_username or not self.pulp_password:
+            raise ValueError("Pulp username and password are required")
+
+        # Force to use the "noarch" instead to build the uri.
+        self.force_noarch = self.job_config.get('pulp', {}).get('force_noarch', False)
+
+    @property
+    def _search_uri(self):
+        """Build the search url"""
+        tail = 'rpm' if self.pkg_type == 'rpm' else 'apt'
+        path = f'pulp/api/v3/distributions/{self.pkg_type}/{tail}'
+        return urljoin(self.pulp_server_url, path)
+
+    @property
+    def _result(self):
+        """Get the results from the pulp api"""
+        if getattr(self, '_result_obj', None) is None:
+            # Get the results from the pulp api.
+            self._result_obj = self._search().get('results', [])
+
+            # Check if there is exactly one result.
+            if not len(self._result_obj):
+                log.error(f'No results found for {self._search_uri}')
+                raise VersionNotFoundError(f'No results found for {self._search_uri}')
+            elif len(self._result_obj) > 1:
+                log.error(f'Multiple results found for {self._search_uri}')
+                raise VersionNotFoundError(f'Multiple results found for {self._search_uri}')
+
+        return self._result_obj[0]
+
+    @property
+    def repo_url(self):
+        """Get the repo url from the pulp api"""
+        return urljoin(self.pulp_server_url, self._result.get('base_url', ''))
+
+    def _get_base_url(self):
+        """Get the base url from the pulp api"""
+        return urljoin(
+            self.pulp_server_url,
+            "/".join(self._result.get('base_url', '').split('/')[:-2])
+        )
+
+    def _search(self):
+        """Search for the package in the pulp api"""
+        # Build the search parameters.
+        labels = f'project:{self.project},'
+        labels += f'flavor:{self.flavor},'
+        labels += f'distro:{self.os_type},'
+        labels += f'distro_version:{self.os_version},'
+
+        # Add the architecture to the search parameters.
+        arch = 'noarch' if self.force_noarch else self.arch
+        labels += f'arch:{arch},'
+
+        # Add the reference to the search parameters.
+        ref_name, ref_val = list(self._choose_reference().items())[0]
+        labels += f'{ref_name}:{ref_val}'
+
+        resp = requests.get(
+            self._search_uri,
+            params={'pulp_label_select': labels},
+            auth=(self.pulp_username, self.pulp_password)
+        )
+        if not resp.ok:
+            log.error(f'Failed to get packages with labels: {labels}')
+            raise VersionNotFoundError(f'Failed to get packages with labels: {labels}')
+
+        return resp.json()
+
+    @classmethod
+    def _get_distro(cls, distro=None, version=None, codename=None):
+        if distro in ('centos', 'rhel'):
+            distro = 'centos'
+            version = cls._parse_version(version)
+        if distro in ('alma', 'rocky'):
+            version = cls._parse_version(version)
+        if distro in ('ubuntu', 'debian'):
+            version = codename or version
+        return f'{distro}/{version}'
+
+    def _get_package_sha1(self):
+        """Get the package sha1 from the pulp api"""
+        return self._result.get('pulp_labels', {}).get('sha1', None)
+
+    def _get_package_version(self):
+        """Get the package version from the pulp api"""
+        return self._result.get('pulp_labels', {}).get('version', None)
+
+
 def get_builder_project():
     """
-    Depending on whether config.use_shaman is True or False, return
-    GitbuilderProject or ShamanProject (the class, not an instance).
+    Depending on whether config.use_artifacts is 'shaman' or 'pulp', return
+    GitbuilderProject, ShamanProject or PulpProject (the class, not an instance).
     """
-    if config.use_shaman is True:
+    if config.use_artifacts == 'shaman':
         builder_class = ShamanProject
+    elif config.use_artifacts == 'pulp':
+        builder_class = PulpProject
     else:
         builder_class = GitbuilderProject
     return builder_class

teuthology/run.py

@@ -390,10 +390,10 @@ def main(args):
     # fetches the tasks and returns a new suite_path if needed
     config["suite_path"] = fetch_tasks_if_needed(config)
 
-    # If the job has a 'use_shaman' key, use that value to override the global
-    # config's value.
-    if config.get('use_shaman') is not None:
-        teuth_config.use_shaman = config['use_shaman']
+    # If the job has a 'use_artifacts' key, use that value to override the global
+    # config's 'use_artifacts' value.
+    if config.get('use_artifacts') is not None:
+        teuth_config.use_artifacts = config['use_artifacts']
 
     #could be refactored for setting and unsetting in hackish way
     if interactive_on_error:

teuthology/task/kernel.py

@@ -374,7 +374,7 @@ def download_kernel(ctx, config):
                 ctx=ctx,
                 remote=role_remote,
             )
-            if teuth_config.use_shaman:
+            if teuth_config.use_artifacts == 'shaman':
                 if role_remote.os.package_type == 'rpm':
                     arch = builder.arch
                     baseurl = urljoin(
@@ -1442,7 +1442,7 @@ def process_role(ctx, config, timeout, role, role_config):
         ctx.summary['{role}-kernel-sha1'.format(role=role)] = sha1
 
         if need_to_install(ctx, role, sha1):
-            if teuth_config.use_shaman:
+            if teuth_config.use_artifacts == 'shaman':
                 version = builder.scm_version
             else:
                 version = builder.version