limiting tcp connections to nvd (#1093)

Author: imsahil007

cve_bin_tool/cvedb.py

@@ -35,8 +35,6 @@
 DISK_LOCATION_DEFAULT = os.path.join(os.path.expanduser("~"), ".cache", "cve-bin-tool")
 DBNAME = "cve.db"
 OLD_CACHE_DIR = os.path.join(os.path.expanduser("~"), ".cache", "cvedb")
-# Workaround for issue #1081
-RATE_LIMITER = asyncio.BoundedSemaphore(2)
 
 
 class CVEDB:
@@ -132,9 +130,9 @@ async def cache_update(self, session, url, sha, chunk_size=16 * 1024):
                 self.LOGGER.debug(f"Correct SHA for {filename}")
                 return
         self.LOGGER.debug(f"Updating CVE cache for {filename}")
-        async with RATE_LIMITER:
-            async with session.get(url) as response:
-                gzip_data = await response.read()
+
+        async with session.get(url) as response:
+            gzip_data = await response.read()
         json_data = gzip.decompress(gzip_data)
         gotsha = hashlib.sha256(json_data).hexdigest().upper()
         async with FileIO(filepath, "wb") as filepath_handle:
@@ -193,7 +191,8 @@ async def refresh(self):
             self.LOGGER.info("Checking if there is a newer version.")
             check_latest_version()
         if not self.session:
-            self.session = aiohttp.ClientSession(trust_env=True)
+            connector = aiohttp.TCPConnector(limit_per_host=19)
+            self.session = aiohttp.ClientSession(connector=connector, trust_env=True)
         self.LOGGER.info("Downloading CVE data...")
         nvd_metadata, curl_metadata = await asyncio.gather(
             self.nist_scrape(self.session), self.get_curl_versions(self.session)

test/test_cvedb.py

@@ -22,7 +22,10 @@ def teardown_class(cls):
 
     @pytest.mark.asyncio
     async def test_00_getmeta(self):
-        async with aiohttp.ClientSession(trust_env=True) as session:
+        connector = aiohttp.TCPConnector(limit_per_host=19)
+        async with aiohttp.ClientSession(
+            connector=connector, trust_env=True
+        ) as session:
             _jsonurl, meta = await self.cvedb.getmeta(
                 session,
                 "https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-modified.meta",