Add rate limiting to address #1081 (#1085)

* Add rate limiting to address #1081

Workaround for issue #1081.  Limits max concurrent connections for downloading the CVE tarballs to two at a time. Moves non-response related code out of the async block to avoid holding a semaphore permit unnecessarily.

* style: format with black

Signed-off-by: John Andersen <[email protected]>

Co-authored-by: John Andersen <[email protected]>

Author: nisamson

cve_bin_tool/cvedb.py

@@ -35,6 +35,8 @@
 DISK_LOCATION_DEFAULT = os.path.join(os.path.expanduser("~"), ".cache", "cve-bin-tool")
 DBNAME = "cve.db"
 OLD_CACHE_DIR = os.path.join(os.path.expanduser("~"), ".cache", "cvedb")
+# Workaround for issue #1081
+RATE_LIMITER = asyncio.BoundedSemaphore(2)
 
 
 class CVEDB:
@@ -130,12 +132,13 @@ async def cache_update(self, session, url, sha, chunk_size=16 * 1024):
                 self.LOGGER.debug(f"Correct SHA for {filename}")
                 return
         self.LOGGER.debug(f"Updating CVE cache for {filename}")
-        async with session.get(url) as response:
-            gzip_data = await response.read()
-            json_data = gzip.decompress(gzip_data)
-            gotsha = hashlib.sha256(json_data).hexdigest().upper()
-            async with FileIO(filepath, "wb") as filepath_handle:
-                await filepath_handle.write(gzip_data)
+        async with RATE_LIMITER:
+            async with session.get(url) as response:
+                gzip_data = await response.read()
+        json_data = gzip.decompress(gzip_data)
+        gotsha = hashlib.sha256(json_data).hexdigest().upper()
+        async with FileIO(filepath, "wb") as filepath_handle:
+            await filepath_handle.write(gzip_data)
         # Raise error if there was an issue with the sha
         if gotsha != sha:
             # Remove the file if there was an issue