Ingest npm data through github api

Signed-off-by: Tushar Goel <[email protected]>

Author: TG1999

CHANGELOG.rst

@@ -10,6 +10,7 @@ Version v31.1.0
 - We are now handling purl fragments in package search. For example:
   you can now serch using queries in the UI like this : `[email protected]`,
   `cherrypy` or `pkg:pypi`.
+- We are now ingesting npm advisories data through GitHub API.
 
 
 Version v31.0.0

requirements.txt

@@ -54,7 +54,7 @@ MarkupSafe==2.1.1
 matplotlib-inline==0.1.3
 multidict==6.0.2
 mypy-extensions==0.4.3
-packageurl-python==0.10.3
+packageurl-python==0.10.5rc1
 packaging==21.3
 paramiko==2.10.3
 parso==0.8.3

setup.cfg

@@ -69,7 +69,7 @@ install_requires =
     coreapi>=2.3.3
 
     #essentials
-    packageurl-python>=0.9.4
+    packageurl-python>=0.10.4
     univers>=30.9.0
     license-expression>=21.6.14
 

vulnerabilities/importers/github.py

@@ -114,6 +114,7 @@
     "COMPOSER": "composer",
     "PIP": "pypi",
     "RUBYGEMS": "gem",
+    "NPM": "npm",
     # "GO": "golang",
 }
 
@@ -122,8 +123,9 @@
 }
 
 # TODO: We will try to gather more info from GH API
+# Check https://github.com/nexB/vulnerablecode/issues/1039#issuecomment-1366458885
 # Check https://github.com/nexB/vulnerablecode/issues/645
-# set of all possible values of first '%s' = {'MAVEN','COMPOSER', 'NUGET', 'RUBYGEMS', 'PYPI'}
+# set of all possible values of first '%s' = {'MAVEN','COMPOSER', 'NUGET', 'RUBYGEMS', 'PYPI', 'NPM'}
 # second '%s' is interesting, it will have the value '' for the first request,
 GRAPHQL_QUERY_TEMPLATE = """
 query{
@@ -202,13 +204,13 @@ def get_purl(pkg_type: str, github_name: str) -> Optional[PackageURL]:
         ns, _, name = github_name.partition(":")
         return PackageURL(type=pkg_type, namespace=ns, name=name)
 
-    if pkg_type == "composer":
+    if pkg_type in ("composer", "npm"):
         if "/" not in github_name:
             return PackageURL(type=pkg_type, name=github_name)
         vendor, _, name = github_name.partition("/")
         return PackageURL(type=pkg_type, namespace=vendor, name=name)
 
-    if pkg_type in ("nuget", "pypi", "gem", "golang"):
+    if pkg_type in ("nuget", "pypi", "gem", "golang", "npm"):
         return PackageURL(type=pkg_type, name=github_name)
 
     logger.error(f"get_purl: Unknown package type {pkg_type}")

vulnerabilities/package_managers.py

@@ -266,7 +266,8 @@ class NpmVersionAPI(VersionAPI):
     package_type = "npm"
 
     def fetch(self, pkg):
-        url = f"https://registry.npmjs.org/{pkg}"
+        lower_pkg = pkg.lower()
+        url = f"https://registry.npmjs.org/{lower_pkg}"
         response = get_response(url=url, content_type="json")
         if not response:
             logger.error(f"Failed to fetch {url}")