Check GitHub ratelimit correctly

Just removing the ratelimit threshold will cause Anitya to fail on
KeyError as instead of 403 when the ratelimit is reached GitHub returns
200 and json with error structure, which is completely different then
standard response. To fix this we need to check the response header for
specific parameters regarding rate limit.

This change is checking X-RateLimit-Remaining and X-RateLimit-Reset
parameters in response header and raising RateLimitException when the
limit is reached.

Signed-off-by: Michal Konecny <mkonecny@redhat.com>

Author: Zlopez

anitya/lib/backends/github.py

@@ -1,16 +1,17 @@
 # -*- coding: utf-8 -*-
 
 """
-(c) 2014-2020 - Copyright Red Hat Inc
+(c) 2014-2025 - Copyright Red Hat Inc
 
 Authors:
   Pierre-Yves Chibon <pingou@pingoured.fr>
   Michal Konecny <mkonecny@redhat.com>
 
 """
-
 import logging
 
+import arrow
+
 from anitya.config import config
 from anitya.lib import utilities
 from anitya.lib.backends import REQUEST_HEADERS, BaseBackend, http_session
@@ -20,13 +21,6 @@
 
 _log = logging.getLogger(__name__)
 
-"""
-Reset time that is currently set for GitHub backend.
-Used when GitHub starts returning HTTP status code 403,
-which doesn't contains this information anymore.
-"""
-reset_time = "1970-01-01T00:00:00Z"
-
 
 class GithubBackend(BaseBackend):
     """The custom class for projects hosted on github.com.
@@ -95,10 +89,17 @@ def _retrieve_versions(cls, owner, repo, project):
             ) from err
 
         if resp.ok:
+            if int(resp.headers["X-RateLimit-Remaining"]) == 0:
+                _log.info("Github API ratelimit reached.")
+                reset_time = arrow.Arrow.utcfromtimestamp(
+                    resp.headers["X-RateLimit-Reset"]
+                )
+                raise RateLimitException(reset_time.isoformat())
             json = resp.json()
         elif resp.status_code == 403:
             _log.info("Github API ratelimit reached.")
-            raise RateLimitException(reset_time)
+            reset_time = arrow.Arrow.utcfromtimestamp(resp.headers["X-RateLimit-Reset"])
+            raise RateLimitException(reset_time.isoformat())
         else:
             raise AnityaPluginException(
                 f"{project.name}: Server responded with status "
@@ -200,23 +201,6 @@ def parse_json(json, project):
             when rate limit threshold is reached.
 
     """
-    global reset_time  # pylint: disable=W0603
-    # We need to check limit first,
-    # because exceeding the limit will also return error
-    try:
-        remaining = json["data"]["rateLimit"]["remaining"]
-        reset_time = json["data"]["rateLimit"]["resetAt"]
-        _log.debug(
-            "Github API ratelimit remains %s, will reset at %s UTC",
-            remaining,
-            reset_time,
-        )
-
-        if remaining <= 0:
-            raise RateLimitException(reset_time)
-    except KeyError:
-        _log.info("Github API ratelimit key is missing. Checking for errors.")
-
     if "errors" in json:
         error_str = ""
         for error in json["errors"]:

anitya/tests/lib/backends/test_github.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright © 2018  Red Hat, Inc.
+# Copyright © 2018-2025  Red Hat, Inc.
 #
 # This copyrighted material is made available to anyone wishing to use,
 # modify, copy, or redistribute it subject to the terms and conditions
@@ -25,6 +25,7 @@
 
 import unittest
 
+import arrow
 import mock
 
 import anitya.lib.backends.github as backend
@@ -414,6 +415,20 @@ def test_get_versions_no_version_retrieved(self):
             AnityaPluginException, backend.GithubBackend.get_versions, project
         )
 
+    @mock.patch.dict("anitya.config.config", {"GITHUB_ACCESS_TOKEN": "foobar"})
+    @mock.patch("anitya.lib.backends.http_session.post")
+    def test_get_versions_ratelimit_reached(self, mock_post):
+        """Test the get_versions function of the github when ratelimit is reached."""
+        mock_resp = mock.MagicMock()
+        mock_resp.ok = True
+        mock_resp.headers = {"X-RateLimit-Remaining": "0", "X-RateLimit-Reset": "0"}
+        mock_post.return_value = mock_resp
+        project = self.projects["valid_without_version_url"]
+        with self.assertRaises(RateLimitException) as test:
+            backend.GithubBackend.get_versions(project)
+
+        self.assertEqual(test.exception.reset_time, arrow.get("1970-01-01T00:00:00Z"))
+
     @mock.patch.dict("anitya.config.config", {"GITHUB_ACCESS_TOKEN": "foobar"})
     @mock.patch("anitya.lib.backends.http_session.post")
     def test_get_versions_403(self, mock_post):
@@ -423,13 +438,13 @@ def test_get_versions_403(self, mock_post):
         mock_resp = mock.MagicMock()
         mock_resp.status_code = 403
         mock_resp.ok = False
+        mock_resp.headers = {"X-RateLimit-Remaining": "0", "X-RateLimit-Reset": "0"}
         mock_post.return_value = mock_resp
         project = self.projects["valid_without_version_url"]
-        backend.reset_time = "1970-01-01T00:00:00Z"
-        self.assertRaises(
-            RateLimitException, backend.GithubBackend.get_versions, project
-        )
-        self.assertEqual(backend.reset_time, "1970-01-01T00:00:00Z")
+        with self.assertRaises(RateLimitException) as test:
+            backend.GithubBackend.get_versions(project)
+
+        self.assertEqual(test.exception.reset_time, arrow.get("1970-01-01T00:00:00Z"))
 
     @mock.patch.dict("anitya.config.config", {"GITHUB_ACCESS_TOKEN": "foobar"})
     def test_plexus_utils(self):
@@ -610,23 +625,6 @@ def test_parse_json_with_errors(self):
 
         self.assertIn('"FOO": "BAR"', str(excinfo.exception))
 
-    def test_parse_json_threshold_exceeded(self):
-        """Test behavior when rate limit threshold is exceeded."""
-        # Limit reached
-        json = {
-            "data": {
-                "repository": {"refs": {"totalCount": 0}},
-                "rateLimit": {
-                    "limit": 5000,
-                    "remaining": 0,
-                    "resetAt": "2008-09-03T20:56:35.450686",
-                },
-            }
-        }
-        with self.assertRaises(RateLimitException):
-            backend.parse_json(json, self.project)
-        self.assertEqual(backend.reset_time, "2008-09-03T20:56:35.450686")
-
     def test_parse_json_tag_missing(self):
         """Test parsing a JSON skips releases where tag is missing."""
         project = models.Project(

anitya/tests/request-data/anitya.tests.lib.backends.test_github.GithubBackendtests.test_gargoyle

@@ -49,7 +49,7 @@ interactions:
       x-oauth-scopes: ['repo:status']
       x-ratelimit-limit: ['5000']
       x-ratelimit-remaining: ['5000']
-      x-ratelimit-reset: ['1539938803']
+      x-ratelimit-reset: ['2147483647']
       x-xss-protection: [1; mode=block]
     status: {code: 200, message: OK}
 version: 1

anitya/tests/request-data/anitya.tests.lib.backends.test_github.GithubBackendtests.test_get_version_invalid_unknown_project

@@ -72,7 +72,7 @@ interactions:
       X-RateLimit-Remaining:
       - '4999'
       X-RateLimit-Reset:
-      - '1576605150'
+      - '2147483647'
       X-XSS-Protection:
       - 1; mode=block
       content-length:

anitya/tests/request-data/anitya.tests.lib.backends.test_github.GithubBackendtests.test_get_version_valid_with_version_url

@@ -71,7 +71,7 @@ interactions:
       X-RateLimit-Remaining:
       - '4996'
       X-RateLimit-Reset:
-      - '1576605295'
+      - '2147483647'
       X-XSS-Protection:
       - 1; mode=block
       content-length:

anitya/tests/request-data/anitya.tests.lib.backends.test_github.GithubBackendtests.test_get_version_valid_without_version_url

@@ -71,7 +71,7 @@ interactions:
       X-RateLimit-Remaining:
       - '4994'
       X-RateLimit-Reset:
-      - '1576605418'
+      - '2147483647'
       X-XSS-Protection:
       - 1; mode=block
       content-length:

anitya/tests/request-data/anitya.tests.lib.backends.test_github.GithubBackendtests.test_get_versions_filter

@@ -89,7 +89,7 @@ interactions:
       X-RateLimit-Remaining:
       - '4999'
       X-RateLimit-Reset:
-      - '1615217132'
+      - '2147483647'
       X-RateLimit-Used:
       - '1'
       X-XSS-Protection:

anitya/tests/request-data/anitya.tests.lib.backends.test_github.GithubBackendtests.test_get_versions_invalid_unknown_project

@@ -74,7 +74,7 @@ interactions:
       X-RateLimit-Remaining:
       - '4990'
       X-RateLimit-Reset:
-      - '1576682771'
+      - '2147483647'
       X-XSS-Protection:
       - 1; mode=block
       content-length:

anitya/tests/request-data/anitya.tests.lib.backends.test_github.GithubBackendtests.test_get_versions_no_token

@@ -37,7 +37,7 @@ interactions:
       x-github-request-id: ['BF56:23A8:3340315:62F7CF2:5B7FEF90']
       x-ratelimit-limit: ['0']
       x-ratelimit-remaining: ['0']
-      x-ratelimit-reset: ['1535114656']
+      x-ratelimit-reset: ['2147483647']
       x-runtime-rack: ['0.011707']
       x-xss-protection: [1; mode=block]
     status: {code: 401, message: Unauthorized}

anitya/tests/request-data/anitya.tests.lib.backends.test_github.GithubBackendtests.test_get_versions_no_version_retrieved

@@ -44,7 +44,7 @@ interactions:
       x-oauth-scopes: ['repo:status']
       x-ratelimit-limit: ['5000']
       x-ratelimit-remaining: ['5000']
-      x-ratelimit-reset: ['1537438156']
+      x-ratelimit-reset: ['2147483647']
       x-runtime-rack: ['0.107819']
       x-xss-protection: [1; mode=block]
     status: {code: 200, message: OK}