Merge pull request #197 from nokia/check-url-download-location

The option `--strict-url-check` now checks the PackageDownloadLocation instead of the PackageHomePage

Author: vargenau

tools/openchain_telco_sbom_validator/README.md

@@ -5,7 +5,8 @@ the [OpenChain Telco SBOM Guide](https://github.com/OpenChain-Project/Telco-WG/b
 
 What is new in version 0.3.1:
 * new option `--noassertion` will list fields that have value NOASSERTION,
-* implement the strict mode for tool name and version (presence of "-").
+* implement the strict mode for tool name and version (presence of "-"),
+* the option `--strict-url-check` now checks the PackageDownloadLocation instead of the PackageHomePage.
 
 What is new in version 0.3.0:
 * you can validate recursively SBOMs linked by SPDX Relationships,

tools/openchain_telco_sbom_validator/src/openchain_telco_sbom_validator/cli.py

@@ -67,7 +67,8 @@ def main():
                              args.guide_version,
                              args.strict,
                              args.noassertion,
-                             args.strict_purl_check)
+                             args.strict_purl_check,
+                             args.strict_url_check)
         sys.exit(exitCode)
     except KeyboardInterrupt:
         print(" Ctrl-C pressed. Terminating...")
@@ -117,7 +118,7 @@ def parseArguments(additionalArguments: AdditionalArguments = AdditionalArgument
                         ' run a non-strict purl check meaning that it is not checked if the'
                         ' purl is translating to a downloadable URL.')
     parser.add_argument('--strict-url-check', action="store_true",
-                        help='Runs a strict check on the URLs of the PackageHomepages. Strict check'
+                        help='Runs a strict check on the URLs of the PackageDowloadLocation. Strict check'
                         ' means that the validator checks also if the given URL can be accessed.'
                         ' The default behaviour is to run a non-strict URL check, meaning that'
                         ' it is not checked if the URL points to a valid page. Strict URL check'
@@ -162,7 +163,7 @@ def parseArguments(additionalArguments: AdditionalArguments = AdditionalArgument
     if args.strict_purl_check:
         logger.info("Running strict checks for purls, what means that it is tested if the purls can be translated to a downloadable url.")
     if args.strict_url_check:
-        logger.info("Running strict checks for URL, what means that it is tested if the PackageHomePage fields are pointing to real pages.")
+        logger.info("Running strict checks for URL, what means that it is tested if the PackageDowloadLocation fields are pointing to real pages.")
 
     if args.guide_version:
         logger.info(f"Checking for the {args.guide_version} version of the OpenChain Telco Guide")

tools/openchain_telco_sbom_validator/src/openchain_telco_sbom_validator/reporter.py

@@ -13,7 +13,7 @@
 logger = logging.getLogger(__name__)
 logger.propagate = True
 
-def reportCli(result, problems, nr_of_errors, input, guide_version, strict, noassertion, strict_purl_check):
+def reportCli(result, problems, nr_of_errors, input, guide_version, strict, noassertion, strict_purl_check, strict_url_check):
     if len(problems):
 
         errors = problems.get_errors()
@@ -38,6 +38,11 @@ def reportCli(result, problems, nr_of_errors, input, guide_version, strict, noas
                 print("Fields with purl that cannot be converted to a downloadable URL:")
                 printTable(warnings, problems.print_file)
 
+        if strict_url_check:
+            if len(warnings):
+                print("PackageDownloadLocation field points to a nonexisting page:")
+                printTable(warnings, problems.print_file)
+
     if not result:
         if len(problems.checked_files) == 1:
             if strict:

tools/openchain_telco_sbom_validator/src/openchain_telco_sbom_validator/validator.py

@@ -183,7 +183,7 @@ def validate(self,
         """ Validates, returns a status and a list of problems.
             filePath: path to the SPDX file to validate.
             strict_purl_check: not only checks the syntax of the PURL, but also checks if the package can be downloaded.
-            strict_url_check: checks if the given URLs in PackageHomepages can be accessed.
+            strict_url_check: checks if the given URLs in PackageDownloadLocation can be accessed.
             strict: checks for both MANDATORY and RECOMMENDED fields.
             noassertion: lists fields with value NOASSERTION.
             functionRegistry: is an optionsl functionRegistry class to inject custom checks.
@@ -529,27 +529,27 @@ def validate(self,
                                 "ExternalRef field is missing (no Package URL)",
                                 Problem.SCOPE_OPEN_CHAIN,
                                 Problem.SEVERITY_ERROR, file)
-            if isinstance(package.homepage, type(None)):
-                logger.debug("Package homepage is missing")
+            if isinstance(package.download_location, type(None)):
+                logger.debug("PackageDownloadLocation is missing")
             else:
-                logger.debug(f"Package homepage is ({package.homepage})")
-                if not validators.url(package.homepage):
-                    logger.debug("Package homepage is not a valid URL")
+                logger.debug(f"PackageDownloadLocation is ({package.download_location})")
+                if not validators.url(package.download_location):
+                    logger.debug("PackageDownloadLocation not a valid URL")
                     # Adding this to the problem list is not needed as the SPDX validator also adds it
-                    # problems.append(["Invalid field in Package", package.spdx_id, package.name, f"PackageHomePage is not a valid URL ({package.homepage})"])
+                    # problems.append(["Invalid field in Package", package.spdx_id, package.name, f"PackageDownloadLocation a valid URL ({package.download_location})"])
                 else:
                     if strict_url_check:
                         try:
-                            logger.debug("Checking package homepage")
-                            page = requests.get(package.homepage)
+                            logger.debug("Checking PackageDownloadLocation")
+                            page = requests.get(package.download_location)
                         except Exception as err:
                             logger.debug(f"Exception received ({format(err)})")
                             problems.append("Invalid field in Package",
                                             package.spdx_id,
                                             package.name,
-                                            f"PackageHomePage field points to a nonexisting page ({package.homepage})",
+                                            f"PackageDownloadLocation field points to a nonexisting page ({package.download_location})",
                                             Problem.SCOPE_OPEN_CHAIN,
-                                            Problem.SEVERITY_ERROR,
+                                            Problem.SEVERITY_WARNING,
                                             file)
             # Version specifics
             match guide_version:

tools/openchain_telco_sbom_validator/unittests/sboms/unittest-sbom-12.spdx

@@ -45,7 +45,7 @@ PackageVersion: 2.4.57+dfsg-3+deb11u1
 PackageOriginator: Person: Debian OpenLDAP Maintainers <pkg-openldap-devel@lists.alioth.debian.org>
 PackageSupplier: Person: Jane Doe (jane.doe@example.com)
 PackageChecksum: SHA256: e325881d738b1091d0105778523ad60eb61e62557b9dc15624e08144b3991d08
-PackageDownloadLocation: NOASSERTION
+PackageDownloadLocation: https://www.not-openldap.org/
 FilesAnalyzed: true
 PackageHomePage: https://www.not-openldap.org/
 PackageVerificationCode: 8360ca10f484b118d367165552d0b934835d128a
@@ -98,4 +98,4 @@ ExtractedText: Autoconf
 
 
 Relationship: SPDXRef-DOCUMENT DESCRIBES SPDXRef-Package-deb-nopurl-libldap-2.4-2-796a192b709a2a2b
-Relationship: SPDXRef-DOCUMENT DESCRIBES SPDXRef-Package-deb-badpurl-libldap-2.4-2-796a192b709a2a2b
+Relationship: SPDXRef-DOCUMENT DESCRIBES SPDXRef-Package-deb-badpurl-libldap-2.4-2-796a192b709a2a2b

tools/openchain_telco_sbom_validator/unittests/test_validator_package.py

@@ -18,6 +18,6 @@ def test_nok_purls():
     assert result == False
     assert len(problems) == 1
     assert problems[0].ErrorType == "Invalid field in Package"
-    assert problems[0].Reason == "PackageHomePage field points to a nonexisting page (https://www.not-openldap.org/)"
+    assert problems[0].Reason == "PackageDownloadLocation field points to a nonexisting page (https://www.not-openldap.org/)"
     assert problems[0].SPDX_ID == "SPDXRef-Package-deb-badpurl-libldap-2.4-2-796a192b709a2a2b"
     assert problems[0].PackageName == "badpurl-libldap-2.4-2"