cvedb: Store NVD JSON files gzip compressed (#637)

Signed-off-by: John Andersen <[email protected]>

Author: John Andersen

cve_bin_tool/cvedb.py

@@ -91,7 +91,7 @@ def cache_update(cachedir, url, sha, chunk_size=16 * 1024, logger=LOGGER):
     """
     Update the cache for a single year of NVD data.
     """
-    filename = url.split("/")[-1].replace(".gz", "")
+    filename = url.split("/")[-1]
     # Ensure we only write to files within the cachedir
     filepath = os.path.abspath(os.path.join(cachedir, filename))
     if not filepath.startswith(os.path.abspath(cachedir)):
@@ -101,7 +101,7 @@ def cache_update(cachedir, url, sha, chunk_size=16 * 1024, logger=LOGGER):
         # Validate the sha and write out
         sha = sha.upper()
         calculate = hashlib.sha256()
-        with open(filepath, "rb") as handle:
+        with gzip.open(filepath, "rb") as handle:
             chunk = handle.read(chunk_size)
             while chunk:
                 calculate.update(chunk)
@@ -127,7 +127,7 @@ def cache_update(cachedir, url, sha, chunk_size=16 * 1024, logger=LOGGER):
             sha = sha.upper()
             calculate = hashlib.sha256()
             # Copy the contents while updating the sha
-            with open(filepath, "wb") as filepath_handle:
+            with gzip.open(filepath, "wb") as filepath_handle:
                 chunk = jsondata_fileobj.read(chunk_size)
                 while chunk:
                     calculate.update(chunk)
@@ -149,7 +149,7 @@ class CVEDB(object):
     CACHEDIR = DISK_LOCATION_DEFAULT
     FEED = "https://nvd.nist.gov/vuln/data-feeds"
     LOGGER = LOGGER.getChild("CVEDB")
-    NVDCVE_FILENAME_TEMPLATE = "nvdcve-1.1-{}.json"
+    NVDCVE_FILENAME_TEMPLATE = "nvdcve-1.1-{}.json.gz"
     META_REGEX = re.compile(r"https:\/\/.*\/json\/.*-[0-9]*\.[0-9]*-[0-9]*\.meta")
     RANGE_UNSET = ""
 
@@ -558,7 +558,7 @@ def year(self, year):
         if not os.path.isfile(filename):
             raise CVEDataForYearNotInCache(year)
         # Open the file and load the JSON data, log the number of CVEs loaded
-        with open(filename, "rb") as fileobj:
+        with gzip.open(filename, "rb") as fileobj:
             cves_for_year = json.load(fileobj)
             self.LOGGER.debug(
                 f'Year {year} has {len(cves_for_year["CVE_Items"])} CVEs in dataset'
@@ -571,9 +571,9 @@ def years(self):
         """
         return sorted(
             [
-                int(filename.split(".")[-2].split("-")[-1])
+                int(filename.split(".")[-3].split("-")[-1])
                 for filename in glob.glob(
-                    os.path.join(self.cachedir, "nvdcve-1.1-*.json")
+                    os.path.join(self.cachedir, "nvdcve-1.1-*.json.gz")
                 )
             ]
         )

test/test_cvedb.py

@@ -46,6 +46,6 @@ def test_04_verify_false(self):
         with self.cvedb:
             self.assertTrue(
                 os.path.isfile(
-                    os.path.join(self.cvedb.cachedir, "nvdcve-1.1-2015.json")
+                    os.path.join(self.cvedb.cachedir, "nvdcve-1.1-2015.json.gz")
                 )
             )

test/test_json.py

@@ -6,6 +6,7 @@
 """
 import hashlib
 import json
+import gzip
 import os
 import unittest
 import datetime
@@ -24,7 +25,7 @@
 NVD_SCHEMA = "https://scap.nist.gov/schema/nvd/feed/1.1/nvd_cve_feed_json_1.1.schema"
 
 # NVD feeds from "https://nvd.nist.gov/vuln/data-feeds#JSON_FEED" but stored locally
-NVD_FILE_TEMPLATE = "nvdcve-1.1-{}.json"
+NVD_FILE_TEMPLATE = "nvdcve-1.1-{}.json.gz"
 
 
 class TestJSON(unittest.TestCase):
@@ -39,11 +40,11 @@ def test_json_validation(self):
         years = list(range(2002, datetime.datetime.now().year + 1))
         # Open the latest nvd file on disk
         for year in years:
-            with open(
-                os.path.join(DISK_LOCATION_DEFAULT, f"nvdcve-1.1-{year}.json"), "rb",
+            with gzip.open(
+                os.path.join(DISK_LOCATION_DEFAULT, f"nvdcve-1.1-{year}.json.gz"), "rb",
             ) as json_file:
                 nvd_json = json.loads(json_file.read())
-                print(f"Loaded json for year {year}: nvdcve-1.1-{year}.json")
+                print(f"Loaded json for year {year}: nvdcve-1.1-{year}.json.gz")
 
                 # Validate -- will raise a ValidationError if not valid
                 try: