feat: Centos package list parser (#1203)

Fixes: #1195 

Co-authored-by: Bread Genie <[email protected]>

Author: BreadGenie

README.md

@@ -83,9 +83,13 @@ You can also use `-m` or `--merge` along with `-f --format` and `-o --output-fil
 
 > Note: For backward compatibility, we still support `csv2cve` command for producing CVEs from csv but we recommend using new `--input-file` command instead.
 
-`-L` or `--package-list` option runs a CVE scan on installed packages listed in a package list. It takes a python package list (requirements.txt) or a package list of packages of an Ubuntu system as an input for the scan. This option is much faster and detects more CVEs than the default method of scanning binaries.
+`-L` or `--package-list` option runs a CVE scan on installed packages listed in a package list. It takes a python package list (requirements.txt) or a package list of packages of an Ubuntu or CentOS system as an input for the scan. This option is much faster and detects more CVEs than the default method of scanning binaries.
 
-> You can get a package list of all installed packages in an Ubuntu system by running `dpkg-query -W -f '${binary:Package}\n' > pkg-list` in the terminal and provide it as an input for a full package scan.
+You can get a package list of all installed packages in 
+  - an Ubuntu system by running `dpkg-query -W -f '${binary:Package}\n' > pkg-list` 
+  - a CentOS system by running `rpm -qa --queryformat '%{NAME}\n' > pkg-list` 
+  
+in the terminal and provide it as an input by running `cve-bin-tool -L pkg-list` for a full package scan.
 
 You can use `--config` option to provide configuration file for the tool. You can still override options specified in config file with command line arguments. See our sample config files in the
 [test/config](https://github.com/intel/cve-bin-tool/blob/main/test/config/)

cve_bin_tool/package_list_parser/__init__.py

@@ -1,14 +1,14 @@
 # Copyright (C) 2021 Intel Corporation
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-import csv
 import json
 import re
 from collections import defaultdict
 from logging import Logger
 from os.path import dirname, getsize, isfile, join
 from subprocess import PIPE, run
-from sys import platform
+
+import distro
 
 from cve_bin_tool.error_handler import (
     EmptyTxtError,
@@ -23,6 +23,7 @@
 
 ROOT_PATH = join(dirname(__file__), "..")
 PYPI_CSV = join(ROOT_PATH, "package_list_parser", "pypi_list.csv")
+SUPPORTED_DISTROS = ["ubuntu", "centos"]
 
 
 class PackageListParser:
@@ -42,15 +43,17 @@ def parse_list(self):
         self.check_file()
 
         if not input_file.endswith("requirements.txt"):
-            if platform != "linux":
-                LOGGER.warning("Package list support only available on Linux!")
+            if distro.id() not in SUPPORTED_DISTROS:
+                LOGGER.warning(
+                    f"Package list support only available on {','.join(SUPPORTED_DISTROS)}!"
+                )
                 return {}
 
             system_packages = []
-            linux_distribution = run(["lsb_release", "-si"], stdout=PIPE)
 
-            if "Ubuntu" in linux_distribution.stdout.decode("utf-8"):
-                LOGGER.info("Scanning ubuntu package list.")
+            LOGGER.info(f"Scanning {distro.id().capitalize()} package list.")
+
+            if "ubuntu" in distro.id():
                 installed_packages = run(
                     [
                         "dpkg-query",
@@ -59,6 +62,17 @@ def parse_list(self):
                     ],
                     stdout=PIPE,
                 )
+            elif "centos" in distro.id():
+                installed_packages = run(
+                    [
+                        "rpm",
+                        "--query",
+                        "--all",
+                        "--queryformat",
+                        '{"name": "%{NAME}", "version": "%{VERSION}"\\}, ',
+                    ],
+                    stdout=PIPE,
+                )
             installed_packages = json.loads(
                 f"[{installed_packages.stdout.decode('utf-8')[0:-2]}]"
             )
@@ -144,15 +158,37 @@ def check_file(self):
                 raise EmptyTxtError(input_file)
 
         if not input_file.endswith("requirements.txt"):
-            # Simulate installation on Ubuntu using apt-get to check if the file is valid
-            output = run(
-                [f"xargs", "-a", input_file, "apt-get", "install", "-s"],
-                stderr=PIPE,
-                stdout=PIPE,
-            )
+            if "ubuntu" in distro.id():
+                # Simulate installation on Ubuntu using apt-get to check if the file is valid
+                output = run(
+                    [f"xargs", "-a", input_file, "apt-get", "install", "-s"],
+                    stderr=PIPE,
+                    stdout=PIPE,
+                )
 
-            if output.returncode != 0:
+                if output.returncode != 0:
+                    with ErrorHandler(mode=error_mode):
+                        raise InvalidListError(
+                            f"Invalid Package list\n{output.stderr.decode('utf-8')}"
+                        )
+            elif "centos" in distro.id():
+                output = run(
+                    [f"xargs", "-a", input_file, "rpm", "-qi"],
+                    stderr=PIPE,
+                    stdout=PIPE,
+                )
+
+                not_installed_packages = re.findall(
+                    r"package (.+) is not installed", output.stdout.decode("utf-8")
+                )
+                if not_installed_packages:
+                    with ErrorHandler(mode=error_mode):
+                        raise InvalidListError(
+                            f"The packages {','.join(not_installed_packages)} seems to be not installed.\nIt is either an invalid package or not installed.\nUse `sudo yum install $(cat package-list)` to install all packages"
+                        )
+
+            else:
+                # TODO: Replace below error handling with a proper pip install dry run
+                # See: https://github.com/pypa/pip/issues/53
                 with ErrorHandler(mode=error_mode):
-                    raise InvalidListError(
-                        f"Invalid Package list\n{output.stderr.decode('utf-8')}"
-                    )
+                    raise InvalidListError("Invalid Python Package list")

doc/MANUAL.md

@@ -337,7 +337,7 @@ The output will look like following:
 
 This option runs a CVE scan on installed packages listed in a package list. It takes a python package list (requirements.txt) or a package list of packages of an Ubuntu system as an input for the scan. This option is much faster and detects more CVEs than the default method of scanning binaries.
 
-An example of the package list for Ubuntu systems:
+An example of the package list for Linux systems:
 
 ```
 bash
@@ -347,7 +347,10 @@ sed
 python3
 ```
 
-> Note: The packages in the package list should be installed in the system before the scan. Run `pip install -r requirements.txt` to install python packages and `sudo apt-get install $(package-list)` for packages in an Ubuntu system.
+> Note: The packages in the package list should be installed in the system before the scan. Run 
+  - `pip install -r requirements.txt` to install python packages
+  - `sudo apt-get install $(cat package-list)` for packages in an Ubuntu system
+  - `sudo yum install $(cat package-list)`for packages in a CentOS system
 
 > Note: Don't use invalid package names in the package list, as it will throw errors.
 
@@ -356,7 +359,11 @@ You can test it using our [test package list](https://github.com/intel/cve-bin-t
 ```console
 cve-bin-tool -L test/txt/test_ubuntu_list.txt
 ```
-> You could get a package list of all installed packages in an Ubuntu system by running `dpkg-query -W -f '${binary:Package}\n' > pkg-list` in the terminal and provide it as an input for a full installed packages scan.
+You can get a package list of all installed packages in 
+  - an Ubuntu system by running `dpkg-query -W -f '${binary:Package}\n' > pkg-list` 
+  - a CentOS system by running `rpm -qa --queryformat '%{NAME}\n' > pkg-list` 
+  
+in the terminal and provide it as an input by running `cve-bin-tool -L pkg-list` for a full package scan.
 
 ### -C CONFIG, --config CONFIG
 

requirements.csv

@@ -16,3 +16,4 @@ jsonschema_not_in_db,jsonschema
 python_not_in_db,py
 srossross_not_in_db,rpmfile
 indygreg_not_in_db,zstandard
+nir0s_not_in_db,distro

requirements.txt

@@ -15,3 +15,4 @@ pytest-asyncio
 rpmfile>=1.0.6
 zstandard; python_version >= "3.4"
 reportlab
+distro

test/test_package_list_parser.py

@@ -6,10 +6,12 @@
 from os.path import dirname, join
 from sys import platform
 
+import distro
 import pytest
 
 from cve_bin_tool.error_handler import ErrorMode
 from cve_bin_tool.package_list_parser import (
+    SUPPORTED_DISTROS,
     EmptyTxtError,
     InvalidListError,
     PackageListParser,
@@ -21,12 +23,6 @@
 class TestPackageListParser:
     TXT_PATH = join(dirname(__file__), "txt")
 
-    DISTRO = (
-        subprocess.run(["lsb_release", "-sd"], stdout=subprocess.PIPE)
-        if platform == "linux"
-        else ""
-    )
-
     REQ_PARSED_TRIAGE_DATA = {
         ProductInfo(vendor="httplib2_project*", product="httplib2", version="0.18.1"): {
             "default": {"remarks": Remarks.Unexplored, "comments": "", "severity": ""},
@@ -91,19 +87,23 @@ def test_valid_requirements(self, filepath, parsed_data):
         # Update the packages back to latest
         subprocess.run(["pip", "install", "httplib2", "requests", "html5lib", "-U"])
 
+    @pytest.mark.skipif(
+        distro.id() not in SUPPORTED_DISTROS,
+        reason=f"Test for {','.join(SUPPORTED_DISTROS)} systems",
+    )
     @pytest.mark.parametrize(
         "filepath, exception",
-        [(join(TXT_PATH, "test_broken_ubuntu_list.txt"), InvalidListError)],
+        [(join(TXT_PATH, "test_broken_linux_list.txt"), InvalidListError)],
     )
-    def test_invalid_ubuntu_list(self, filepath, exception):
+    def test_invalid_linux_list(self, filepath, exception):
         package_list = PackageListParser(filepath, error_mode=ErrorMode.FullTrace)
         with pytest.raises(exception):
             package_list.parse_list()
 
     @pytest.mark.skipif(
         "ACTIONS" not in environ
         or not platform == "linux"
-        or "Ubuntu 20.04" not in DISTRO.stdout.decode(),
+        or ("ubuntu" not in distro.id() and "20.04" not in distro.version()),
         reason="Running locally requires root permission",
     )
     @pytest.mark.parametrize(