diff --git a/docs/detailed_test_config.rst b/docs/detailed_test_config.rst index 2a46c4d77..86656f0a6 100644 --- a/docs/detailed_test_config.rst +++ b/docs/detailed_test_config.rst @@ -300,10 +300,24 @@ Shaman options ============== Shaman is a helper class which could be used to build the uri for specified -packages based the 'shaman_host': 'shaman.ceph.com'. +packages based the 'artifacts_host' option. Options:: - use_shaman: True # Enable to use Shaman, False as default + use_artifacts: 'shaman' + artifacts_host: 'shaman.ceph.com' shaman: force_noarch: True # Force to use "noarch" to build the uri + +Pulp options +============ + +Pulp is a package manager (https://pulpproject.org/) that could be used to +build the uri for specified packages based the 'artifacts_host' option. + +Options:: + + use_artifacts: 'pulp' + artifacts_host: 'pulp.example.com' + pulp: + force_noarch: True # Force to use "noarch" to build the uri \ No newline at end of file diff --git a/docs/siteconfig.rst b/docs/siteconfig.rst index ee5d18fa5..02cbe044e 100644 --- a/docs/siteconfig.rst +++ b/docs/siteconfig.rst @@ -269,3 +269,12 @@ Here is a sample configuration with many of the options set and documented:: # Do not allow more than that many jobs in a single run by default. # To disable this check use 0. job_threshold: 500 + + # Setting for artifacts manager (shaman or pulp) + use_artifacts: 'pulp' + artifacts_host: 'pulp.example.com' + + # Settings for pulp package manager (https://pulpproject.org/) + pulp: + username: '' + password: '' \ No newline at end of file diff --git a/teuthology/config.py b/teuthology/config.py index 3400959d8..f72a3e743 100644 --- a/teuthology/config.py +++ b/teuthology/config.py @@ -181,8 +181,8 @@ class TeuthologyConfig(YamlConfig): 'kojiroot_url': 'https://kojipkgs.fedoraproject.org/packages', 'koji_task_url': 'https://kojipkgs.fedoraproject.org/work/', 'baseurl_template': 'http://{host}/{proj}-{pkg_type}-{dist}-{arch}-{flavor}/{uri}', - 'use_shaman': True, - 'shaman_host': 'shaman.ceph.com', + 'use_artifacts': 'shaman', + 'artifacts_host': 'shaman.ceph.com', 'teuthology_path': None, 'suite_verify_ceph_hash': True, 'suite_allow_missing_packages': False, diff --git a/teuthology/openstack/setup-openstack.sh b/teuthology/openstack/setup-openstack.sh index 526ba9813..a47fb3905 100755 --- a/teuthology/openstack/setup-openstack.sh +++ b/teuthology/openstack/setup-openstack.sh @@ -54,7 +54,7 @@ function create_config() { cat > ~/.teuthology.yaml < 1: + log.error(f'Multiple results found for {self._search_uri}') + raise VersionNotFoundError(f'Multiple results found for {self._search_uri}') + + return self._result_obj[0] + + @property + def repo_url(self): + """Get the repo url from the pulp api""" + return urljoin(self.pulp_server_url, self._result.get('base_url', '')) + + def _get_base_url(self): + """Get the base url from the pulp api""" + return urljoin( + self.pulp_server_url, + "/".join(self._result.get('base_url', '').split('/')[:-2]) + ) + + def _search(self): + """Search for the package in the pulp api""" + # Build the search parameters. + labels = f'project:{self.project},' + labels += f'flavor:{self.flavor},' + labels += f'distro:{self.os_type},' + labels += f'distro_version:{self.os_version},' + + # Add the architecture to the search parameters. + arch = 'noarch' if self.force_noarch else self.arch + labels += f'arch:{arch},' + + # Add the reference to the search parameters. + ref_name, ref_val = list(self._choose_reference().items())[0] + labels += f'{ref_name}:{ref_val}' + + resp = requests.get( + self._search_uri, + params={'pulp_label_select': labels}, + auth=(self.pulp_username, self.pulp_password) + ) + if not resp.ok: + log.error(f'Failed to get packages with labels: {labels}') + raise VersionNotFoundError(f'Failed to get packages with labels: {labels}') + + return resp.json() + + @classmethod + def _get_distro(cls, distro=None, version=None, codename=None): + if distro in ('centos', 'rhel'): + distro = 'centos' + version = cls._parse_version(version) + if distro in ('alma', 'rocky'): + version = cls._parse_version(version) + if distro in ('ubuntu', 'debian'): + version = codename or version + return f'{distro}/{version}' + + def _get_package_sha1(self): + """Get the package sha1 from the pulp api""" + return self._result.get('pulp_labels', {}).get('sha1', None) + + def _get_package_version(self): + """Get the package version from the pulp api""" + return self._result.get('pulp_labels', {}).get('version', None) + + def get_builder_project(): """ - Depending on whether config.use_shaman is True or False, return - GitbuilderProject or ShamanProject (the class, not an instance). + Depending on whether config.use_artifacts is 'shaman' or 'pulp', return + GitbuilderProject, ShamanProject or PulpProject (the class, not an instance). """ - if config.use_shaman is True: + if config.use_artifacts == 'shaman': builder_class = ShamanProject + elif config.use_artifacts == 'pulp': + builder_class = PulpProject else: builder_class = GitbuilderProject return builder_class diff --git a/teuthology/run.py b/teuthology/run.py index be5c2029a..44fdaa46a 100644 --- a/teuthology/run.py +++ b/teuthology/run.py @@ -390,10 +390,10 @@ def main(args): # fetches the tasks and returns a new suite_path if needed config["suite_path"] = fetch_tasks_if_needed(config) - # If the job has a 'use_shaman' key, use that value to override the global - # config's value. - if config.get('use_shaman') is not None: - teuth_config.use_shaman = config['use_shaman'] + # If the job has a 'use_artifacts' key, use that value to override the global + # config's 'use_artifacts' value. + if config.get('use_artifacts') is not None: + teuth_config.use_artifacts = config['use_artifacts'] #could be refactored for setting and unsetting in hackish way if interactive_on_error: diff --git a/teuthology/task/kernel.py b/teuthology/task/kernel.py index a96f2ce80..0cd586fb4 100644 --- a/teuthology/task/kernel.py +++ b/teuthology/task/kernel.py @@ -374,7 +374,7 @@ def download_kernel(ctx, config): ctx=ctx, remote=role_remote, ) - if teuth_config.use_shaman: + if teuth_config.use_artifacts == 'shaman': if role_remote.os.package_type == 'rpm': arch = builder.arch baseurl = urljoin( @@ -1442,7 +1442,7 @@ def process_role(ctx, config, timeout, role, role_config): ctx.summary['{role}-kernel-sha1'.format(role=role)] = sha1 if need_to_install(ctx, role, sha1): - if teuth_config.use_shaman: + if teuth_config.use_artifacts == 'shaman': version = builder.scm_version else: version = builder.version diff --git a/teuthology/test/test_packaging.py b/teuthology/test/test_packaging.py index 0ab4a733b..a5fb7b788 100644 --- a/teuthology/test/test_packaging.py +++ b/teuthology/test/test_packaging.py @@ -302,6 +302,11 @@ def test_get_response_failed_no_wait(self, m_get): packaging._get_response("google.com", sleep=1, tries=2) assert m_get.call_count == 1 + @patch('teuthology.packaging.config') + def test_get_builder_project_pulp(self, m_config): + m_config.use_artifacts = 'pulp' + assert packaging.get_builder_project() is packaging.PulpProject + class TestBuilderProject(object): klass = None @@ -533,8 +538,8 @@ class TestShamanProject(TestBuilderProject): def setup_method(self): self.p_config = patch('teuthology.packaging.config') self.m_config = self.p_config.start() - self.m_config.use_shaman = True - self.m_config.shaman_host = 'shaman.ceph.com' + self.m_config.use_artifacts = 'shaman' + self.m_config.artifacts_host = 'shaman.ceph.com' self.p_get_config_value = \ patch('teuthology.packaging._get_config_value_for_remote') self.m_get_config_value = self.p_get_config_value.start() @@ -810,3 +815,195 @@ def test_get_distro_remote(self, matrix_index): ) def test_get_distro_config(self, matrix_index): super().test_get_distro_config(matrix_index) + + +class TestPulpProject(TestBuilderProject): + klass = packaging.PulpProject + + def setup_method(self): + self.p_config = patch('teuthology.packaging.config') + self.m_config = self.p_config.start() + self.m_config.use_artifacts = 'pulp' + self.m_config.artifacts_host = 'pulp.example.com' + self.m_config.pulp = {'username': 'u', 'password': 'p'} + self.p_get_config_value = \ + patch('teuthology.packaging._get_config_value_for_remote') + self.m_get_config_value = self.p_get_config_value.start() + self.m_get_config_value.return_value = None + self.p_get = patch('requests.get') + self.m_get = self.p_get.start() + + resp = Mock() + resp.ok = True + resp.json.return_value = { + 'results': [{ + 'base_url': 'dist/ceph/main/sha/centos/8/flavors/default/', + 'pulp_labels': { + 'sha1': 'the_sha1', + 'version': '0.90.0', + }, + }], + } + self.m_get.return_value = resp + + def teardown_method(self): + self.p_config.stop() + self.p_get_config_value.stop() + self.p_get.stop() + + def test_init_requires_pulp_credentials(self): + self.m_config.pulp = {'username': '', 'password': 'p'} + with pytest.raises(ValueError, match='Pulp username and password are required'): + packaging.PulpProject('ceph', {}, ctx=dict(foo='bar'), + remote=self._get_remote()) + + self.m_config.pulp = {'username': 'u', 'password': ''} + with pytest.raises(ValueError, match='Pulp username and password are required'): + packaging.PulpProject('ceph', {}, ctx=dict(foo='bar'), + remote=self._get_remote()) + + def test_init_from_remote_base_url(self): + def m_get_base_url(obj): + obj._search() + return self.m_get.call_args_list[0][0][0] + with patch( + 'teuthology.packaging.PulpProject._get_base_url', + new=m_get_base_url, + ): + super(TestPulpProject, self).test_init_from_remote_base_url( + 'http://pulp.example.com/pulp/api/v3/distributions/deb/apt', + ) + + def test_init_from_remote_base_url_debian(self): + def m_get_base_url(obj): + obj._search() + return self.m_get.call_args_list[0][0][0] + with patch( + 'teuthology.packaging.PulpProject._get_base_url', + new=m_get_base_url, + ): + super(TestPulpProject, self).test_init_from_remote_base_url_debian( + 'http://pulp.example.com/pulp/api/v3/distributions/deb/apt', + ) + + def test_init_from_config_base_url(self): + def m_get_base_url(obj): + obj._search() + return self.m_get.call_args_list[0][0][0] + with patch( + 'teuthology.packaging.PulpProject._get_base_url', + new=m_get_base_url, + ): + super(TestPulpProject, self).test_init_from_config_base_url( + 'http://pulp.example.com/pulp/api/v3/distributions/deb/apt', + ) + + @patch('teuthology.packaging.PulpProject._get_package_sha1') + def test_init_from_config_tag_ref(self, m_get_package_sha1): + m_get_package_sha1.return_value = 'the_sha1' + super(TestPulpProject, self).test_init_from_config_tag_ref() + + def test_init_from_config_tag_overrides_branch_ref(self, caplog): + obj = super(TestPulpProject, self)\ + .test_init_from_config_tag_overrides_branch_ref(caplog) + obj._search() + labels = self.m_get.call_args[1]['params']['pulp_label_select'] + assert 'tag:v10.0.1' in labels + assert 'branch:jewel' not in labels + + def test_init_from_config_branch_overrides_sha1(self, caplog): + obj = super(TestPulpProject, self)\ + .test_init_from_config_branch_overrides_sha1(caplog) + obj._search() + labels = self.m_get.call_args[1]['params']['pulp_label_select'] + assert 'branch:jewel' in labels + assert 'sha1:sha1' not in labels + + def test_get_package_version_found(self): + resp = Mock() + resp.ok = True + resp.json.return_value = { + 'results': [{ + 'base_url': 'a/b/c/d', + 'pulp_labels': {'version': '0.90.0'}, + }], + } + self.m_get.return_value = resp + super(TestPulpProject, self).test_get_package_version_found() + + def test_get_package_version_not_found(self): + rem = self._get_remote() + ctx = dict(foo="bar") + resp = Mock() + resp.ok = False + self.m_get.return_value = resp + gp = self.klass("ceph", {}, ctx=ctx, remote=rem) + with pytest.raises(VersionNotFoundError): + gp.version + + def test_get_package_sha1_fetched_found(self): + resp = Mock() + resp.ok = True + resp.json.return_value = { + 'results': [{'base_url': 'a/b/c/d', 'pulp_labels': {'sha1': 'the_sha1'}}], + } + self.m_get.return_value = resp + super(TestPulpProject, self).test_get_package_sha1_fetched_found() + + def test_get_package_sha1_fetched_not_found(self): + resp = Mock() + resp.ok = True + resp.json.return_value = { + 'results': [{'base_url': 'a/b/c/d', 'pulp_labels': {}}], + } + self.m_get.return_value = resp + super(TestPulpProject, self).test_get_package_sha1_fetched_not_found() + + DISTRO_MATRIX = [ + ('rhel', '7.0', None, 'centos/7'), + ('alma', '9.7', None, 'alma/9'), + ('rocky', '10.1', None, 'rocky/10'), + ('centos', '8.1', None, 'centos/8'), + ('fedora', '20', None, 'fedora/20'), + ('ubuntu', '20.04', None, 'ubuntu/20.04'), + ] + + @pytest.mark.parametrize( + "matrix_index", + range(len(DISTRO_MATRIX)), + ) + def test_get_distro_remote(self, matrix_index): + super().test_get_distro_remote(matrix_index) + + DISTRO_MATRIX_NOVER = [ + ('rhel', None, None, 'centos/8'), + ('centos', None, None, 'centos/9'), + ('fedora', None, None, 'fedora/25'), + ('ubuntu', None, None, 'ubuntu/jammy'), + ('alma', None, None, 'alma/9'), + ('rocky', None, None, 'rocky/9'), + ] + + DISTRO_MATRIX_CONFIG = [ + ('rhel', '7.0', None, 'centos/7'), + ('alma', '9.7', None, 'alma/9'), + ('rocky', '10.1', None, 'rocky/10'), + ('centos', '8.1', None, 'centos/8'), + ('fedora', '20', None, 'fedora/20'), + ('ubuntu', '20.04', None, 'ubuntu/focal'), + ] + + @pytest.mark.parametrize( + "matrix_index", + range(len(DISTRO_MATRIX_CONFIG) + len(DISTRO_MATRIX_NOVER)), + ) + def test_get_distro_config(self, matrix_index): + (distro, version, _, expected) = ( + self.DISTRO_MATRIX_CONFIG + self.DISTRO_MATRIX_NOVER + )[matrix_index] + config = dict( + os_type=distro, + os_version=version + ) + gp = self.klass("ceph", config) + assert gp.distro == expected