Fix GitHub rate limit in project_setup by using recursive tree fetch (#5107)

## Description
This PR addresses the GitHub API rate limiting issues encountered during
the `project_setup` cron job execution for OSS-Fuzz projects.

### Problem
The previous implementation of `get_oss_fuzz_projects` made multiple API
calls per project (fetching directory trees individually), resulting in
O(N) requests where N is the number of projects. This frequently
triggered GitHub's rate limits (403 Forbidden).

### Solution
Optimized the project fetching logic to use the `recursive=1` parameter
when fetching the OSS-Fuzz repository tree.
- Fetches the entire directory tree in a single API call (O(1)).
- Uses SHA from the tree to cache `project.yaml` contents, skipping
unchanged files.
- Disables retries specifically for 403 errors to prevent retry storms.
- Parses the flattened tree structure to identify projects with
`project.yaml` and `Dockerfile`.
- Significantly reduces the API request count, mitigating rate limit
issues.

### Verification
- **Regression Testing**: Ran existing tests in
`src/clusterfuzz/_internal/tests/appengine/handlers/cron/project_setup_test.py`.
All passed.
- **New Tests**: Added `test_get_oss_fuzz_projects_api_error` to verify
error handling when the API call fails.
- **Mock Data**: Updated `url_results.txt` to include a mock response
for the recursive tree fetch, covering edge cases like:
    - Invalid/Malformed YAML.
    - Nested files (ignored).
    - Files at root (ignored).
    - Projects without YAML (ignored).

### Impact
- **Performance**: Drastic reduction in GitHub API calls.
- **Reliability**: Reduced likelihood of cron job failures due to rate
limiting.
- **Scope**: Changes are isolated to `project_setup.py` and only affect
the OSS-Fuzz project source flow.

---------

Co-authored-by: André Nogueira Ribeiro <94117783+decoNR@users.noreply.github.com>

Author: hunsche

src/clusterfuzz/_internal/cron/project_setup.py

@@ -23,6 +23,7 @@
 import requests
 import yaml
 
+from clusterfuzz._internal.base import retry
 from clusterfuzz._internal.base import tasks
 from clusterfuzz._internal.base import untrusted
 from clusterfuzz._internal.base import utils
@@ -80,6 +81,14 @@ class ProjectSetupError(Exception):
   """Exception."""
 
 
+class GitHubGenericError(Exception):
+  """GitHub generic error."""
+
+
+class GitHubRateLimitError(Exception):
+  """GitHub rate limit error."""
+
+
 class JobInfo:
   """Job information."""
 
@@ -218,6 +227,11 @@ def _to_experimental_job(job_info):
   return job_info
 
 
+@retry.wrap(
+    retries=3,
+    delay=2,
+    function='cron.project_setup.get_github_url',
+    exception_types=[GitHubGenericError])
 def get_github_url(url):
   """Return contents of URL."""
   github_credentials = db_config.get_value('github_credentials')
@@ -227,54 +241,79 @@ def get_github_url(url):
   client_id, client_secret = github_credentials.strip().split(';')
   response = requests.get(
       url, auth=(client_id, client_secret), timeout=HTTP_TIMEOUT_SECONDS)
+
+  if response.status_code == 403:
+    raise GitHubRateLimitError(f'GitHub rate limit exceeded for {url}.')
+
   if response.status_code != 200:
     logs.error(
         f'Failed to get github url: {url}.', status_code=response.status_code)
-    response.raise_for_status()
+    raise GitHubGenericError(f'Failed to get github url: {url}.')
 
   return json.loads(response.text)
 
 
-def find_github_item_url(github_json, name):
-  """Get url of a blob/tree from a github json response."""
-  for item in github_json['tree']:
-    if item['path'] == name:
-      return item['url']
-
-  return None
-
-
 def get_oss_fuzz_projects():
   """Return list of projects for oss-fuzz."""
   ossfuzz_tree_url = ('https://api.github.com/repos/google/oss-fuzz/'
-                      'git/trees/master')
+                      'git/trees/master?recursive=1')
   tree = get_github_url(ossfuzz_tree_url)
-  projects = []
 
-  projects_url = find_github_item_url(tree, 'projects')
-  if not projects_url:
-    logs.error('No projects found.')
-    return []
+  projects = []
+  project_map = {}
 
-  tree = get_github_url(projects_url)
   for item in tree['tree']:
-    if item['type'] != 'tree':
+    path = item['path']
+    if not path.startswith('projects/'):
       continue
 
-    item_json = get_github_url(item['url'])
-    project_yaml_url = find_github_item_url(item_json, 'project.yaml')
-    if not project_yaml_url:
+    parts = path.split('/')
+    if len(parts) != 3:
       continue
 
-    projects_yaml = get_github_url(project_yaml_url)
-    info = yaml.safe_load(base64.b64decode(projects_yaml['content']))
+    project_name = parts[1]
+    filename = parts[2]
+
+    if project_name not in project_map:
+      project_map[project_name] = {
+          'yaml_url': None,
+          'has_dockerfile': False,
+          'yaml_sha': None
+      }
 
-    has_dockerfile = (
-        find_github_item_url(item_json, 'Dockerfile') or 'dockerfile' in info)
+    if filename == 'project.yaml':
+      project_map[project_name]['yaml_url'] = item['url']
+      project_map[project_name]['yaml_sha'] = item['sha']
+    elif filename == 'Dockerfile':
+      project_map[project_name]['has_dockerfile'] = True
+
+  # Get all existing projects to check for cache hits.
+  existing_projects = {p.name: p for p in data_types.OssFuzzProject.query()}
+
+  for project_name, details in project_map.items():
+    if not details['yaml_url']:
+      continue
+
+    # Check if we have a cached version of project.yaml.
+    existing_project = existing_projects.get(project_name)
+    if (existing_project and existing_project.project_yaml_sha and
+        existing_project.project_yaml_sha == details['yaml_sha']):
+      # Cache hit.
+      continue
+
+    try:
+      projects_yaml = get_github_url(details['yaml_url'])
+      content = base64.b64decode(projects_yaml['content'])
+      info = yaml.safe_load(content)
+    except Exception as e:
+      logs.error(f'Failed to parse project.yaml for {project_name}: {e}')
+      continue
+
+    has_dockerfile = (details['has_dockerfile'] or 'dockerfile' in info)
     if not has_dockerfile:
       continue
 
-    projects.append((item['path'], info))
+    projects.append((project_name, info, details['yaml_sha']))
 
   return projects
 
@@ -286,7 +325,7 @@ def get_projects_from_gcs(gcs_url):
   except json.decoder.JSONDecodeError as e:
     raise ProjectSetupError(f'Error loading json file from {gcs_url}: {e}')
 
-  return [(project['name'], project) for project in data['projects']]
+  return [(project['name'], project, None) for project in data['projects']]
 
 
 def _process_sanitizers_field(sanitizers):
@@ -533,7 +572,7 @@ def cleanup_old_projects_settings(project_names):
     ndb_utils.delete_multi(to_delete)
 
 
-def create_project_settings(project, info, service_account):
+def create_project_settings(project, info, project_yaml_sha, service_account):
   """Setup settings for ClusterFuzz (such as CPU distribution)."""
   key = ndb.Key(data_types.OssFuzzProject, project)
   oss_fuzz_project = key.get()
@@ -561,6 +600,10 @@ def create_project_settings(project, info, service_account):
     if oss_fuzz_project.base_os_version != base_os_version:
       oss_fuzz_project.base_os_version = base_os_version
       oss_fuzz_project.put()
+
+    if oss_fuzz_project.project_yaml_sha != project_yaml_sha:
+      oss_fuzz_project.project_yaml_sha = project_yaml_sha
+      oss_fuzz_project.put()
   else:
     if language in MEMORY_SAFE_LANGUAGES:
       cpu_weight = OSS_FUZZ_MEMORY_SAFE_LANGUAGE_PROJECT_WEIGHT
@@ -574,7 +617,8 @@ def create_project_settings(project, info, service_account):
         cpu_weight=cpu_weight,
         service_account=service_account['email'],
         ccs=ccs,
-        base_os_version=base_os_version).put()
+        base_os_version=base_os_version,
+        project_yaml_sha=project_yaml_sha).put()
 
 
 def _create_pubsub_topic(name, client):
@@ -1010,7 +1054,7 @@ def set_up(self, projects):
     """Do project setup. Return a list of all the project names that were set
     up."""
     job_names = []
-    for project, info in projects:
+    for project, info, project_yaml_sha in projects:
       logs.info(f'Syncing configs for {project}.')
 
       backup_bucket_name = None
@@ -1038,11 +1082,12 @@ def set_up(self, projects):
 
         # Set up projects settings (such as CPU distribution settings).
         if not info.get('disabled', False):
-          create_project_settings(project, info, service_account)
+          create_project_settings(project, info, project_yaml_sha,
+                                  service_account)
 
     # Delete old/disabled project settings.
     enabled_projects = [
-        project for project, info in projects
+        project for project, info, _ in projects
         if not info.get('disabled', False)
     ]
     return SetupResult(enabled_projects, job_names)

src/clusterfuzz/_internal/datastore/data_types.py

@@ -1476,6 +1476,9 @@ class OssFuzzProject(Model):
   # Base OS version for the project.
   base_os_version = ndb.StringProperty()
 
+  # SHA of the project.yaml file.
+  project_yaml_sha = ndb.StringProperty()
+
 
 class OssFuzzProjectInfo(Model):
   """Set up information for a project (cpu allocation, instance groups, service

src/clusterfuzz/_internal/tests/appengine/handlers/cron/project_setup_data/url_results.txt

@@ -240,5 +240,76 @@
   "content": "aG9tZXBhZ2U6ICJodHRwczovL3d3dy5mcmVldHlwZS5vcmcvIgo=\\n",
   "encoding": "base64"
 }
+""",
+  'https://api.github.com/repos/google/oss-fuzz/git/trees/master?recursive=1': """
+{
+  "sha": "recursive_sha",
+  "url": "https://api.github.com/repos/google/oss-fuzz/git/trees/master?recursive=1",
+  "tree": [
+    {
+      "path": "projects/boringssl/project.yaml",
+      "mode": "100644",
+      "type": "blob",
+      "sha": "e57f1846ff0fdbb0fe08e98ca38b6235008b41be",
+      "url": "https://api.github.com/repos/google/oss-fuzz/git/blobs/e57f1846ff0fdbb0fe08e98ca38b6235008b41be"
+    },
+    {
+      "path": "projects/boringssl/Dockerfile",
+      "mode": "100644",
+      "type": "blob",
+      "sha": "0368f8166f92043678183473f6cc225a5b316e75",
+      "url": "https://api.github.com/repos/google/oss-fuzz/git/blobs/0368f8166f92043678183473f6cc225a5b316e75"
+    },
+    {
+      "path": "projects/curl/project.yaml",
+      "mode": "100644",
+      "type": "blob",
+      "sha": "30580bab5896f92de90e904cda76ecdb41c6397a",
+      "url": "https://api.github.com/repos/google/oss-fuzz/git/blobs/30580bab5896f92de90e904cda76ecdb41c6397a"
+    },
+    {
+      "path": "projects/freetype2/project.yaml",
+      "mode": "100644",
+      "type": "blob",
+      "sha": "46400ddfc8f2db662f70d6f09190dadc31244a58",
+      "url": "https://api.github.com/repos/google/oss-fuzz/git/blobs/46400ddfc8f2db662f70d6f09190dadc31244a58"
+    },
+    {
+      "path": "projects/bad_yaml/project.yaml",
+      "mode": "100644",
+      "type": "blob",
+      "sha": "bad_yaml_sha",
+      "url": "https://api.github.com/repos/google/oss-fuzz/git/blobs/bad_yaml_sha"
+    },
+    {
+      "path": "projects/nested/subdir/file",
+      "mode": "100644",
+      "type": "blob",
+      "url": "https://api.github.com/repos/google/oss-fuzz/git/blobs/nested_sha"
+    },
+    {
+      "path": "projects/root_file",
+      "mode": "100644",
+      "type": "blob",
+      "url": "https://api.github.com/repos/google/oss-fuzz/git/blobs/root_sha"
+    },
+    {
+      "path": "projects/only_docker/Dockerfile",
+      "mode": "100644",
+      "type": "blob",
+      "url": "https://api.github.com/repos/google/oss-fuzz/git/blobs/docker_sha"
+    }
+  ],
+  "truncated": false
+}
+""",
+  'https://api.github.com/repos/google/oss-fuzz/git/blobs/bad_yaml_sha': """
+{
+  "sha": "bad_yaml_sha",
+  "size": 10,
+  "url": "https://api.github.com/repos/google/oss-fuzz/git/blobs/bad_yaml_sha",
+  "content": "aW52YWxpZCB5YW1sOiA6OiB3aGF0Cg==\\n",
+  "encoding": "base64"
+}
 """
 }