Adding check for shaman and chacra build validations

teuthology/suite/run.py

@@ -5,6 +5,7 @@
 import pwd
 import yaml
 import re
+import requests
 import time
 
 from pathlib import Path
@@ -99,6 +100,7 @@ def create_initial_config(self):
         ceph_hash = self.choose_ceph_hash()
         suite_branch = self.choose_suite_branch()
         suite_hash = self.choose_suite_hash(suite_branch)
+        self.verify_sha_id(ceph_hash, suite_branch)
         if self.args.suite_dir:
             self.suite_repo_path = self.args.suite_dir
         else:
@@ -235,6 +237,49 @@ def choose_ceph_hash(self):
         log.info("ceph sha1: {hash}".format(hash=ceph_hash))
         return ceph_hash
 
+    def verify_sha_id(self, ceph_hash, ceph_branch):
+        """
+        Verify if a given Ceph version (SHA1) exists in either Chacra or Shaman
+        repositories for the specified branch.
+        """
+        def fetch_url(url, expect_json=False):
+            """Fetch content from a URL, optionally parse JSON."""
+            try:
+                response = requests.get(url, timeout=10)
+                response.raise_for_status()
+            except requests.exceptions.HTTPError as e:
+                if e.response.status_code == 404:
+                    log.debug(f"Branch not found in repository: {url}")
+                else:
+                    log.error(f"Failed to fetch {url}: {e}")
+                return None
+            except requests.exceptions.RequestException as e:
+                log.error(f"Failed to fetch {url}: {e}")
+                return None
+
+            if expect_json:
+                try:
+                    return response.json()
+                except ValueError:
+                    log.error(f"Invalid JSON from {url}")
+                    return None
+            return response.text
+
+        chacra_url = f"https://1.chacra.ceph.com/repos/ceph/{ceph_branch}/"
+        chacra_data = fetch_url(chacra_url, expect_json=True)
+        if chacra_data:
+            if ceph_hash not in chacra_data:
+                msg = f"Not found in Chacra for branch {ceph_branch}."
+                util.schedule_fail(msg, self.name, dry_run=self.args.dry_run)
+
+        shaman_url = f"https://shaman.ceph.com/builds/ceph/{ceph_branch}/"
+        shaman_html = fetch_url(shaman_url)
+        if shaman_html:
+            sha1s = set(re.findall(r"[0-9a-f]{40}", shaman_html))
+            if ceph_hash not in sha1s:
+                msg = f"Not found in Shaman for branch {ceph_branch}."
+                util.schedule_fail(msg, self.name, dry_run=self.args.dry_run)
+
     def choose_teuthology_branch(self):
         """Select teuthology branch, check if it is present in repo and return
         tuple (branch, hash) where hash is commit sha1 corresponding

teuthology/suite/test/test_init.py

@@ -143,11 +143,14 @@ def test_expand_short_repo_name(orig, shorthand, result):
 
 
 class TestSuiteMain(object):
-    def test_main(self):
+    @patch('teuthology.suite.run.Run.verify_sha_id')
+    def test_main(self, m_verify_sha_id):
         suite_name = 'SUITE'
         throttle = '3'
         machine_type = 'burnupi'
 
+        m_verify_sha_id.return_value = None
+
         def prepare_and_schedule(obj):
             assert obj.base_config.suite == suite_name
             assert obj.args.throttle == throttle
@@ -211,12 +214,15 @@ def test_machine_type_none_error(self, m_smtp):
         assert str(exc.value) == "Scheduling failed: Must specify a machine_type"
         m_smtp.assert_not_called()
 
-    def test_schedule_suite_noverify(self):
+    @patch('teuthology.suite.run.Run.verify_sha_id')
+    def test_schedule_suite_noverify(self, m_verify_sha_id):
         suite_name = 'noop'
         suite_dir = os.path.dirname(__file__)
         throttle = '3'
         machine_type = 'burnupi'
 
+        m_verify_sha_id.return_value = None
+
         with patch.multiple(
             'teuthology.suite.util',
             fetch_repos=DEFAULT,
@@ -239,12 +245,15 @@ def test_schedule_suite_noverify(self):
             m_sleep.assert_called_with(int(throttle))
             m['get_gitbuilder_hash'].assert_not_called()
 
-    def test_schedule_suite(self):
+    @patch('teuthology.suite.run.Run.verify_sha_id')
+    def test_schedule_suite(self, m_verify_sha_id):
         suite_name = 'noop'
         suite_dir = os.path.dirname(__file__)
         throttle = '3'
         machine_type = 'burnupi'
 
+        m_verify_sha_id.return_value = None
+
         with patch.multiple(
             'teuthology.suite.util',
             fetch_repos=DEFAULT,

teuthology/suite/test/test_run_.py

@@ -37,11 +37,14 @@ def setup_method(self):
         )
         self.args = YamlConfig.from_dict(self.args_dict)
 
+    @patch('teuthology.suite.run.Run.verify_sha_id')
     @patch('teuthology.suite.run.util.fetch_repos')
     @patch('teuthology.suite.run.util.git_ls_remote')
     @patch('teuthology.suite.run.util.git_validate_sha1')
     def test_email_addr(self, m_git_validate_sha1,
-                        m_git_ls_remote, _):
+                        m_git_ls_remote, _, m_verify_sha_id):
+        m_verify_sha_id.return_value = None
+
         # neuter choose_X_branch
         m_git_validate_sha1.return_value = self.args_dict['ceph_sha1']
         self.args_dict['teuthology_branch'] = 'main'
@@ -100,6 +103,7 @@ def test_branch_nonexistent(
             ["7d", timedelta(days=-14), False],
         ]
     )
+    @patch('teuthology.suite.run.Run.verify_sha_id')
     @patch('teuthology.repo_utils.fetch_repo')
     @patch('teuthology.suite.run.util.git_branch_exists')
     @patch('teuthology.suite.run.util.package_version_for_hash')
@@ -110,10 +114,13 @@ def test_get_expiration(
         m_package_version_for_hash,
         m_git_branch_exists,
         m_fetch_repo,
+        m_verify_sha_id,
         expire,
         delta,
         result,
     ):
+        m_verify_sha_id.return_value = None
+
         m_git_ls_remote.side_effect = 'hash'
         m_package_version_for_hash.return_value = 'a_version'
         m_git_branch_exists.return_value = True
@@ -129,6 +136,7 @@ def test_get_expiration(
             assert (now < expires_result) is result
             assert obj.base_config['expire']
 
+    @patch('teuthology.suite.run.Run.verify_sha_id')
     @patch('teuthology.suite.run.util.fetch_repos')
     @patch('requests.head')
     @patch('teuthology.suite.run.util.git_branch_exists')
@@ -141,7 +149,10 @@ def test_sha1_exists(
         m_git_branch_exists,
         m_requests_head,
         m_fetch_repos,
+        m_verify_sha_id,
     ):
+        m_verify_sha_id.return_value = None
+
         config.gitbuilder_host = 'example.com'
         m_package_version_for_hash.return_value = 'ceph_hash'
         m_git_branch_exists.return_value = True
@@ -195,10 +206,13 @@ def test_teuthology_branch_nonexistent(
             self.klass(self.args)
         m_smtp.assert_not_called()
 
+    @patch('teuthology.suite.run.Run.verify_sha_id')
     @patch('teuthology.suite.run.util.fetch_repos')
     @patch('teuthology.suite.util.git_ls_remote')
     @patch('teuthology.suite.run.util.package_version_for_hash')
-    def test_os_type(self, m_pvfh, m_git_ls_remote, m_fetch_repos):
+    def test_os_type(self, m_pvfh, m_git_ls_remote, m_fetch_repos, m_verify_sha_id):
+        m_verify_sha_id.return_value = None
+
         m_git_ls_remote.return_value = "sha1"
         del self.args['distro']
         run_ = run.Run(self.args)
@@ -214,10 +228,13 @@ def test_os_type(self, m_pvfh, m_git_ls_remote, m_fetch_repos):
         assert to_schedule[1]['yaml']['os_type'] == "ubuntu"
         assert to_schedule[1]['yaml']['os_version'] == "24.0"
 
+    @patch('teuthology.suite.run.Run.verify_sha_id')
     @patch('teuthology.suite.run.util.fetch_repos')
     @patch('teuthology.suite.util.git_ls_remote')
     @patch('teuthology.suite.run.util.package_version_for_hash')
-    def test_sha1(self, m_pvfh, m_git_ls_remote, m_fetch_repos):
+    def test_sha1(self, m_pvfh, m_git_ls_remote, m_fetch_repos, m_verify_sha_id):
+        m_verify_sha_id.return_value = None
+
         m_git_ls_remote.return_value = "sha1"
         del self.args['distro']
         run_ = run.Run(self.args)
@@ -261,6 +278,7 @@ def setup_method(self):
         )
         self.args = YamlConfig.from_dict(self.args_dict)
 
+    @patch('teuthology.suite.run.Run.verify_sha_id')
     @patch('teuthology.suite.run.Run.schedule_jobs')
     @patch('teuthology.suite.run.Run.write_rerun_memo')
     @patch('teuthology.suite.util.get_install_task_flavor')
@@ -281,7 +299,10 @@ def test_successful_schedule(
         m_get_install_task_flavor,
         m_write_rerun_memo,
         m_schedule_jobs,
+        m_verify_sha_id,
     ):
+        m_verify_sha_id.return_value = None
+
         m_get_arch.return_value = 'x86_64'
         m_git_validate_sha1.return_value = self.args.ceph_sha1
         m_package_version_for_hash.return_value = 'ceph_version'
@@ -372,6 +393,7 @@ def test_successful_schedule(
             assert k in stdin_yaml
         m_write_rerun_memo.assert_called_once_with()
 
+    @patch('teuthology.suite.run.requests.get')
     @patch('teuthology.suite.util.find_git_parents')
     @patch('teuthology.suite.run.Run.schedule_jobs')
     @patch('teuthology.suite.util.get_install_task_flavor')
@@ -392,7 +414,20 @@ def test_newest_failure(
         m_get_install_task_flavor,
         m_schedule_jobs,
         m_find_git_parents,
+        m_requests_get,
     ):
+        def mock_get(url, **kwargs):
+            response = requests.Response()
+            response.status_code = 200
+            response.raise_for_status = lambda: None
+            if 'chacra' in url:
+                response.json = lambda: {self.args.ceph_sha1: {}}
+                response.headers['Content-Type'] = 'application/json'
+            elif 'shaman' in url:
+                raise requests.exceptions.RequestException("Mock Shaman failure")
+            return response
+        m_requests_get.side_effect = mock_get
+
         m_get_arch.return_value = 'x86_64'
         m_git_validate_sha1.return_value = self.args.ceph_sha1
         m_package_version_for_hash.return_value = None
@@ -418,6 +453,7 @@ def test_newest_failure(
             [call('ceph', 'ceph_sha1', 10)]
         )
 
+    @patch('teuthology.suite.run.requests.get')
     @patch('teuthology.suite.util.find_git_parents')
     @patch('teuthology.suite.run.Run.schedule_jobs')
     @patch('teuthology.suite.run.Run.write_rerun_memo')
@@ -440,13 +476,30 @@ def test_newest_success_same_branch_same_repo(
         m_write_rerun_memo,
         m_schedule_jobs,
         m_find_git_parents,
+        m_requests_get,
     ):
         """
         Test that we can successfully schedule a job with newest
         backtracking when the ceph and suite branches are the same
         and the ceph_sha1 is not supplied. We should expect that the
         ceph_hash and suite_hash will be updated to the working sha1
         """
+        def mock_get(url, **kwargs):
+            response = requests.Response()
+            response.status_code = 200
+            response.raise_for_status = lambda: None
+            if 'chacra' in url:
+                # Include both the original hash and all backtracked hashes
+                response.json = lambda: {
+                    self.args.ceph_sha1: {},
+                    **{f"ceph_sha1_{i}": {} for i in range(5)}
+                }
+                response.headers['Content-Type'] = 'application/json'
+            elif 'shaman' in url:
+                raise requests.exceptions.RequestException("Mock Shaman failure")
+            return response
+        m_requests_get.side_effect = mock_get
+
         m_get_arch.return_value = 'x86_64'
         # rig has_packages_for_distro to fail this many times, so
         # everything will run NUM_FAILS+1 times
@@ -526,6 +579,7 @@ def test_newest_success_same_branch_same_repo(
                 assert job_yaml.get('sha1') == working_sha1
                 assert job_yaml.get('suite_sha1') == working_sha1
 
+    @patch('teuthology.suite.run.requests.get')
     @patch('teuthology.suite.util.find_git_parents')
     @patch('teuthology.suite.run.Run.schedule_jobs')
     @patch('teuthology.suite.run.Run.write_rerun_memo')
@@ -548,6 +602,7 @@ def test_newest_success_diff_branch_diff_repo(
         m_write_rerun_memo,
         m_schedule_jobs,
         m_find_git_parents,
+        m_requests_get,
     ):
         """
         Test that we can successfully schedule a job with newest
@@ -556,6 +611,22 @@ def test_newest_success_diff_branch_diff_repo(
         ceph_hash will be updated to the working sha1,
         but the suite_hash will remain the original suite_sha1.
         """
+        def mock_get(url, **kwargs):
+            response = requests.Response()
+            response.status_code = 200
+            response.raise_for_status = lambda: None
+            if 'chacra' in url:
+                # Include both the original hash and all backtracked hashes
+                response.json = lambda: {
+                    self.args.ceph_sha1: {},
+                    **{f"ceph_sha1_{i}": {} for i in range(5)}
+                }
+                response.headers['Content-Type'] = 'application/json'
+            elif 'shaman' in url:
+                raise requests.exceptions.RequestException("Mock Shaman failure")
+            return response
+        m_requests_get.side_effect = mock_get
+
         m_get_arch.return_value = 'x86_64'
         # Set different branches
         self.args.ceph_branch = 'ceph_different_branch'