Fixed #511 - Improve check performance

Signed-off-by: Chin Yeung Li <[email protected]>

Author: chinyeungli

CHANGELOG.rst

@@ -1,10 +1,11 @@
 ==============================
 Changelog
 ==============================
-2022-08-23
-    Release 7.0.3
+2022-09-21
+    Release 7.1.0
 
-    * Fixed version mismatch in v7.0.2 (https://github.com/nexB/aboutcode-toolkit/issues/510)
+    * Fixed version mismatch (https://github.com/nexB/aboutcode-toolkit/issues/510)
+    * Improve `check` performance (https://github.com/nexB/aboutcode-toolkit/issues/511)
 
 
 2022-03-21

about.ABOUT

@@ -1,6 +1,6 @@
 about_resource: .
 name: AboutCode-toolkit
-version: 7.0.3
+version: 7.1.0
 author: Jillian Daguil, Chin Yeung Li, Philippe Ombredanne, Thomas Druez
 copyright: Copyright (c) nexB Inc.
 description: |

src/attributecode/__init__.py

@@ -20,7 +20,7 @@
 
 import saneyaml
 
-__version__ = '7.0.3'
+__version__ = '7.1.0'
 
 __about_spec_version__ = '3.2.3'
 

src/attributecode/cmd.py

@@ -351,7 +351,7 @@ def gen_license(location, output, djc, scancode, verbose):
         if not key in lic_dict_output:
             lic_filename = license_dict[key][1]
             lic_context = license_dict[key][2]
-            lic_dict_output[lic_filename] = lic_context 
+            lic_dict_output[lic_filename] = lic_context
 
     write_errors = write_licenses(lic_dict_output, output)
     if write_errors:
@@ -419,7 +419,7 @@ def validate_template(ctx, param, value):
 @click.option('--reference',
     metavar='DIR',
     type=click.Path(exists=True, file_okay=False, readable=True, resolve_path=True),
-    help='Path to a directory with reference files where "license_file" and/or "notice_file"' 
+    help='Path to a directory with reference files where "license_file" and/or "notice_file"'
         ' located.')
 
 @click.option('--template',
@@ -710,7 +710,8 @@ def check(location, djc, log, verbose):
     errors, abouts = collect_inventory(location)
 
     # Validate license_expression
-    _key_text_dict, errs = pre_process_and_fetch_license_dict(abouts, api_url, api_key)
+    from_check = True
+    _key_text_dict, errs = pre_process_and_fetch_license_dict(abouts, from_check, api_url, api_key)
     for e in errs:
         errors.append(e)
 

src/attributecode/model.py

@@ -1643,7 +1643,7 @@ def save_as_excel(location, about_dicts):
     formatted_list = util.format_about_dict_output(about_dicts)
     write_excel(location, formatted_list)
 
-def pre_process_and_fetch_license_dict(abouts, api_url=None, api_key=None, scancode=False, reference=None):
+def pre_process_and_fetch_license_dict(abouts, from_check=False, api_url=None, api_key=None, scancode=False, reference=None):
     """
     Return a dictionary containing the license information (key, name, text, url)
     fetched from the ScanCode LicenseDB or DejaCode API.
@@ -1710,9 +1710,13 @@ def pre_process_and_fetch_license_dict(abouts, api_url=None, api_key=None, scanc
                                 if msg == "Invalid '--api_url'. License generation is skipped.":
                                     errors.extend(errs)
                                     return key_text_dict, errors
-                            for severity, message in errs: 
+                            for severity, message in errs:
                                 msg = (about.about_file_path + ": " + message)
                                 errors.append(Error(severity, msg))
+                            # We don't want to actually get the license information from the
+                            # check utility
+                            if from_check:
+                                continue
                             if not license_data:
                                 continue
                             license_name = license_data.get('short_name', '')
@@ -1725,6 +1729,10 @@ def pre_process_and_fetch_license_dict(abouts, api_url=None, api_key=None, scanc
                             license_text_url = url + lic_key + '.LICENSE'
                             try:
                                 json_url = urlopen(license_url)
+                                # We don't want to actually get the license information from the
+                                # check utility
+                                if from_check:
+                                    continue
                                 data = json.loads(json_url.read())
                                 license_name = data['short_name']
                                 license_text = urllib.request.urlopen(license_text_url).read().decode('utf-8')
@@ -1738,12 +1746,13 @@ def pre_process_and_fetch_license_dict(abouts, api_url=None, api_key=None, scanc
                                     msg = u"Invalid 'license': " + lic_key
                                 errors.append(Error(ERROR, msg))
                                 continue
-                        detail_list.append(license_name)
-                        detail_list.append(license_filename)
-                        detail_list.append(license_text)
-                        detail_list.append(lic_url)
-                        detail_list.append(spdx_license_key)
-                        key_text_dict[lic_key] = detail_list
+                        if not from_check:
+                            detail_list.append(license_name)
+                            detail_list.append(license_filename)
+                            detail_list.append(license_text)
+                            detail_list.append(lic_url)
+                            detail_list.append(spdx_license_key)
+                            key_text_dict[lic_key] = detail_list
                 if not about.license_key.value:
                     about.license_key.value = lic_list