Use datastore OssFuzzProject instead of github for cc-groups cronjob (#5166)

The cronjob is getting throttled by the GitHub API when trying to query
for the oss-fuzz projects info (probably since the project-setup cron is
already heavily querying it).

This change uses the `OssFuzzProject` datastore entity, which is synced
with oss-fuzz github by the project-setup cron, in order to retrieve all
the projects and their CCs.

Bug: b/477964128

Author: ViniciustCosta

src/clusterfuzz/_internal/cron/oss_fuzz_cc_groups.py

@@ -14,15 +14,16 @@
 """Cron to sync OSS-Fuzz projects groups used as CC in the issue tracker."""
 
 from clusterfuzz._internal.base import utils
-from clusterfuzz._internal.cron import project_setup
+from clusterfuzz._internal.datastore import data_types
+from clusterfuzz._internal.datastore import ndb_utils
 from clusterfuzz._internal.google_cloud_utils import google_groups
 from clusterfuzz._internal.metrics import logs
 
 _CC_GROUP_SUFFIX = '-ccs@oss-fuzz.com'
 _CC_GROUP_DESC = 'External CCs in OSS-Fuzz issue tracker for project'
 
 
-def sync_project_cc_group(project_name, info):
+def sync_project_cc_group(project_name: str, ccs: list[str]):
   """Sync the project's google group used for CCing in the issue tracker."""
   group_name = f'{project_name}{_CC_GROUP_SUFFIX}'
 
@@ -54,13 +55,13 @@ def sync_project_cc_group(project_name, info):
       logs.warning(f'Failed to allow external members for {group_name}')
       return
 
-  ccs = set(project_setup.ccs_from_info(info))
+  ccs_set = set(ccs)
 
-  to_add = ccs - group_memberships.keys()
+  to_add = ccs_set - group_memberships.keys()
   for member in to_add:
     google_groups.add_member_to_group(group_id, member)
 
-  to_delete = group_memberships.keys() - ccs
+  to_delete = group_memberships.keys() - ccs_set
   for member in to_delete:
     # Ignore the SA that created the group from members to delete.
     if utils.is_service_account(member):
@@ -73,10 +74,11 @@ def sync_project_cc_group(project_name, info):
 def main():
   """Sync OSS-Fuzz projects groups used to CC owners in the issue tracker."""
   logs.info('OSS-Fuzz CC groups sync started.')
-  projects = project_setup.get_oss_fuzz_projects()
 
-  for project, info in projects:
-    sync_project_cc_group(project, info)
+  for project in ndb_utils.get_all_from_model(data_types.OssFuzzProject):
+    project_name = project.name
+    ccs = project.ccs
+    sync_project_cc_group(project_name, ccs)
 
   logs.info('OSS-Fuzz CC groups sync succeeded.')
   return True

src/clusterfuzz/_internal/tests/appengine/handlers/cron/oss_fuzz_cc_groups_test.py

@@ -16,16 +16,17 @@
 import unittest
 
 from clusterfuzz._internal.cron import oss_fuzz_cc_groups
+from clusterfuzz._internal.datastore import data_types
 from clusterfuzz._internal.tests.test_libs import helpers as test_helpers
+from clusterfuzz._internal.tests.test_libs import test_utils
 
 
+@test_utils.with_cloud_emulators('datastore')
 class OssFuzzCcGroupsTest(unittest.TestCase):
   """Tests for oss_fuzz_cc_groups."""
 
   def setUp(self):
     test_helpers.patch(self, [
-        'clusterfuzz._internal.cron.project_setup.get_oss_fuzz_projects',
-        'clusterfuzz._internal.cron.project_setup.ccs_from_info',
         'clusterfuzz._internal.google_cloud_utils.google_groups.get_group_id',
         'clusterfuzz._internal.google_cloud_utils.google_groups.create_google_group',
         'clusterfuzz._internal.google_cloud_utils.google_groups.get_google_group_memberships',
@@ -37,14 +38,12 @@ def setUp(self):
 
   def test_main(self):
     """Test main execution for creating groups and syncing project ccs."""
-    self.mock.get_oss_fuzz_projects.return_value = [
-        ('project1', {
-            'info': 1
-        }),
-        ('project2', {
-            'info': 2
-        }),
-    ]
+    project1 = data_types.OssFuzzProject(
+        name='project1', ccs=['member2@example.com'])
+    project2 = data_types.OssFuzzProject(
+        name='project2', ccs=['member2@example.com', 'member3@example.com'])
+    project1.put()
+    project2.put()
 
     # project1 group does not exist, so create it.
     # project2 group exists, only sync members.
@@ -55,10 +54,6 @@ def test_main(self):
         'member1@example.com': 'membership1',
         'member2@example.com': 'membership2',
     }
-    self.mock.ccs_from_info.return_value = [
-        'member2@example.com',
-        'member3@example.com',
-    ]
     self.mock.is_service_account.return_value = False
 
     self.assertTrue(oss_fuzz_cc_groups.main())
@@ -77,13 +72,14 @@ def test_main(self):
 
   def test_main_exec_for_new_group(self):
     """Test main execution for a newly created group."""
-    self.mock.get_oss_fuzz_projects.return_value = [('project1', {'info': 1})]
+    project1 = data_types.OssFuzzProject(
+        name='project1', ccs=['member2@example.com'])
+    project1.put()
     self.mock.get_group_id.return_value = '1'
     self.mock.get_google_group_memberships.return_value = {
         'member1@gserviceaccount.com': 'membership1'
     }
     self.mock.set_oss_fuzz_access_settings.return_value = True
-    self.mock.ccs_from_info.return_value = ['member2@example.com']
     self.mock.is_service_account.return_value = True
 
     self.assertTrue(oss_fuzz_cc_groups.main())
@@ -97,7 +93,9 @@ def test_main_exec_for_new_group(self):
 
   def test_create_fail(self):
     """Test group creation failure."""
-    self.mock.get_oss_fuzz_projects.return_value = [('project1', {})]
+    project1 = data_types.OssFuzzProject(
+        name='project1', ccs=['member2@example.com'])
+    project1.put()
     self.mock.get_group_id.return_value = None
     self.mock.create_google_group.return_value = False
 
@@ -106,21 +104,23 @@ def test_create_fail(self):
 
   def test_get_memberships_fail(self):
     """Test get memberships failure."""
-    self.mock.get_oss_fuzz_projects.return_value = [('project1', {})]
+    project1 = data_types.OssFuzzProject(name='project1')
+    project1.put()
     self.mock.get_group_id.return_value = 'group1_id'
     self.mock.get_google_group_memberships.return_value = None
 
     self.assertTrue(oss_fuzz_cc_groups.main())
-    self.mock.ccs_from_info.assert_not_called()
+    self.mock.add_member_to_group.assert_not_called()
+    self.mock.delete_google_group_membership.assert_not_called()
 
   def test_skip_sa_deletion(self):
     """Test that service accounts are not deleted from group."""
-    self.mock.get_oss_fuzz_projects.return_value = [('project1', {})]
+    project1 = data_types.OssFuzzProject(name='project1')
+    project1.put()
     self.mock.get_group_id.return_value = 'group1_id'
     self.mock.get_google_group_memberships.return_value = {
         'sa@serviceaccount.com': 'membership_sa',
     }
-    self.mock.ccs_from_info.return_value = []
     self.mock.is_service_account.return_value = True
 
     self.assertTrue(oss_fuzz_cc_groups.main())