Fixed #487 - Add spdx_license_key support

 * Update changelog/tests

Signed-off-by: Chin Yeung Li <[email protected]>

Author: chinyeungli

CHANGELOG.rst

@@ -14,6 +14,7 @@
     * Add new option to choose extract license from ScanCode LicenseDB or DJC License Library
     * Add ability to transform Excel formatted file 
     * Support Excel file format for `inventory`, `gen` and `attrib`
+    * Add 'spdx_license_key' support
 
 2021-04-02
     Release 6.0.0

src/attributecode/api.py

@@ -99,7 +99,4 @@ def get_license_details_from_api(api_url, api_key, license_key):
     Missing values are provided as empty strings.
     """
     license_data, errors = request_license_data(api_url, api_key, license_key)
-    license_name = license_data.get('short_name', '')
-    license_text = license_data.get('full_text', '')
-    license_key = license_data.get('key', '')
-    return license_name, license_key, license_text, errors
+    return license_data, errors

src/attributecode/gen.py

@@ -333,22 +333,27 @@ def generate(location, base_dir, android=None, reference_dir=None, fetch_license
             licenses_dict = {}
             if gen_license:
                 # Write generated LICENSE file
-                license_key_name_context_url_list = about.dump_lic(dump_loc, license_dict)
+                license_key_name_context_url_list, err = about.dump_lic(dump_loc, license_dict)
+                if err:
+                    errors.append(err)
                 if license_key_name_context_url_list:
-                    for lic_key, lic_name, lic_filename, lic_context, lic_url in license_key_name_context_url_list:
-                        licenses_dict[lic_key] = [lic_name, lic_filename, lic_context, lic_url]
+                    for lic_key, lic_name, lic_filename, lic_context, lic_url, spdx_lic_key in license_key_name_context_url_list:
+                        licenses_dict[lic_key] = [lic_name, lic_filename, lic_context, lic_url, spdx_lic_key]
                         if not lic_name in about.license_name.value:
                             about.license_name.value.append(lic_name)
                         about.license_file.value[lic_filename] = lic_filename
                         if not lic_url in about.license_url.value:
                             about.license_url.value.append(lic_url)
-
+                        if not spdx_lic_key in about.spdx_license_key.value:
+                            about.spdx_license_key.value.append(spdx_lic_key)
                         if about.license_name.value:
                             about.license_name.present = True
                         if about.license_file.value:
                             about.license_file.present = True
                         if about.license_url.value:
                             about.license_url.present = True
+                        if about.spdx_license_key.value:
+                            about.spdx_license_key.present = True
 
             about.dump(dump_loc, licenses_dict)
 

src/attributecode/model.py

@@ -777,6 +777,7 @@ def set_standard_fields(self):
             ('license_name', ListField()),
             ('license_file', FileTextField()),
             ('license_url', UrlListField()),
+            ('spdx_license_key', ListField()),
             ('copyright', StringField()),
             ('notice_file', FileTextField()),
             ('notice_url', UrlField()),
@@ -1035,7 +1036,7 @@ def load_dict(self, fields_dict, base_dir, scancode=False, from_attrib=False, ru
                 continue
             if key == u'licenses':
                 # FIXME: use a license object instead
-                lic_key, lic_name, lic_file, lic_url, lic_score = ungroup_licenses(value)
+                lic_key, lic_name, lic_file, lic_url, spdx_lic_key, lic_score = ungroup_licenses(value)
                 if lic_key:
                     fields.append(('license_key', lic_key))
                 if lic_name:
@@ -1044,6 +1045,10 @@ def load_dict(self, fields_dict, base_dir, scancode=False, from_attrib=False, ru
                     fields.append(('license_file', lic_file))
                 if lic_url:
                     fields.append(('license_url', lic_url))
+                if lic_url:
+                    fields.append(('license_url', lic_url))
+                if spdx_lic_key:
+                    fields.append(('spdx_license_key', spdx_lic_key))
                 # The license score is a key from scancode license scan
                 if lic_score:
                     fields.append(('license_score', lic_score))
@@ -1085,6 +1090,7 @@ def dumps(self, licenses_dict=None):
         license_name = []
         license_file = []
         license_url = []
+        spdx_license_key = []
         bool_fields = ['redistribute', 'attribute', 'track_changes', 'modified', 'internal_use_only']
         for field in self.all_fields():
             if not field.value and not field.name in bool_fields:
@@ -1119,6 +1125,8 @@ def dumps(self, licenses_dict=None):
                     license_file = list(field.value.keys())
             elif field.name == 'license_url' and field.value:
                 license_url = field.value
+            elif field.name == 'spdx_license_key' and field.value:
+                spdx_license_key = field.value
             elif field.name in file_fields and field.value:
                 data[field.name] = field.original_value
             elif field.name in bool_fields and not field.value == None:
@@ -1136,14 +1144,12 @@ def dumps(self, licenses_dict=None):
             lic_dict = {}
             if licenses_dict and lic_key in licenses_dict:
                 lic_dict['key'] = lic_key
-                lic_name, lic_filename, lic_context, lic_url = licenses_dict[lic_key]
-                #lic_name = licenses_dict[lic_key][0]
-                #lic_url = licenses_dict[lic_key][2]
-                #lic_file = lic_key + '.LICENSE'
+                lic_name, lic_filename, lic_context, lic_url, spdx_lic_key = licenses_dict[lic_key]
 
                 lic_dict['name'] = lic_name
                 lic_dict['file'] = lic_filename
                 lic_dict['url'] = lic_url
+                lic_dict['spdx_license_key'] = spdx_lic_key
 
                 # Remove the license information if it has been handled
                 lic_key_copy.remove(lic_key)
@@ -1156,7 +1162,7 @@ def dumps(self, licenses_dict=None):
                 lic_dict_list.append(lic_dict)
 
         # Handle license information that have not been handled.
-        license_group = list(zip_longest(lic_key_copy, license_name, license_file, license_url))
+        license_group = list(zip_longest(lic_key_copy, license_name, license_file, license_url, spdx_license_key))
         for lic_group in license_group:
             lic_dict = {}
             if lic_group[0]:
@@ -1171,6 +1177,8 @@ def dumps(self, licenses_dict=None):
                 lic_dict['file'] = lic_group[2]
             if lic_group[3]:
                 lic_dict['url'] = lic_group[3]
+            if lic_group[4]:
+                lic_dict['spdx_license_key'] = lic_group[4]
             lic_dict_list.append(lic_dict)
 
         # Format the license information in the same order of the license expression
@@ -1267,6 +1275,7 @@ def dump_lic(self, location, license_dict):
         loc = util.to_posix(location)
         parent = posixpath.dirname(loc)
         license_key_name_context_url = []
+        err = ''
 
         if not posixpath.exists(parent):
             os.makedirs(add_unc(parent))
@@ -1282,16 +1291,15 @@ def dump_lic(self, location, license_dict):
                             license_path = posixpath.join(parent, lic_key)
                             license_path += u'.LICENSE'
                             license_path = add_unc(license_path)
-                            license_name, license_filename, license_context, license_url = license_dict[lic_key]
-                            license_info = (lic_key, license_name, license_filename, license_context, license_url)
+                            license_name, license_filename, license_context, license_url, spdx_license_key = license_dict[lic_key]
+                            license_info = (lic_key, license_name, license_filename, license_context, license_url, spdx_license_key)
                             license_key_name_context_url.append(license_info)
                             with io.open(license_path, mode='w', encoding='utf-8', newline='\n', errors='replace') as lic:
                                 lic.write(license_context)
                     except Exception as e:
-                        # TODO: it should return error if exception caught
-                        pass
+                        err = str(e)
 
-        return license_key_name_context_url
+        return license_key_name_context_url, err
 
 
 def collect_inventory(location):
@@ -1593,9 +1601,14 @@ def pre_process_and_fetch_license_dict(abouts, api_url=None, api_key=None, scanc
                         license_name = ''
                         license_filename = ''
                         license_text = ''
+                        spdx_license_key = ''
                         detail_list = []
                         if api_key:
-                            license_name, _license_key, license_text, errs = api.get_license_details_from_api(url, api_key, lic_key)
+                            license_data, errs = api.get_license_details_from_api(url, api_key, lic_key)
+                            license_name = license_data.get('short_name', '')
+                            license_text = license_data.get('full_text', '')
+                            license_key = license_data.get('key', '')
+                            spdx_license_key = license_data.get('spdx_license_key', '')
                             for severity, message in errs: 
                                 msg = (about.about_file_path + ": " + message)
                                 errors.append(Error(severity, msg))
@@ -1611,6 +1624,7 @@ def pre_process_and_fetch_license_dict(abouts, api_url=None, api_key=None, scanc
                                 license_text = urllib.request.urlopen(license_text_url).read().decode('utf-8')
                                 license_filename = data['key'] + '.LICENSE'
                                 lic_url = url + license_filename
+                                spdx_license_key = data['spdx_license_key']
                             except:
                                 msg = about.about_file_path + u" : Invalid 'license': " + lic_key
                                 errors.append(Error(ERROR, msg))
@@ -1619,6 +1633,7 @@ def pre_process_and_fetch_license_dict(abouts, api_url=None, api_key=None, scanc
                         detail_list.append(license_filename)
                         detail_list.append(license_text)
                         detail_list.append(lic_url)
+                        detail_list.append(spdx_license_key)
                         key_text_dict[lic_key] = detail_list
                 if not about.license_key.value:
                     about.license_key.value = lic_list

src/attributecode/util.py

@@ -470,6 +470,7 @@ def ungroup_licenses(licenses):
     lic_name = []
     lic_file = []
     lic_url = []
+    spdx_lic_key = []
     lic_score = []
     for lic in licenses:
         if 'key' in lic:
@@ -480,9 +481,11 @@ def ungroup_licenses(licenses):
             lic_file.append(lic['file'])
         if 'url' in lic:
             lic_url.append(lic['url'])
+        if 'spdx_license_key' in lic:
+            spdx_lic_key.append(lic['spdx_license_key'])
         if 'score' in lic:
             lic_score.append(lic['score'])
-    return lic_key, lic_name, lic_file, lic_url, lic_score
+    return lic_key, lic_name, lic_file, lic_url, spdx_lic_key, lic_score
 
 
 # FIXME: add docstring

tests/test_api.py

@@ -44,11 +44,7 @@ def test_api_get_license_details_from_api(self, request_license_data):
         errors = []
         request_license_data.return_value = license_data, errors
 
-        expected = (
-            'Apache 2.0',
-            'apache-2.0',
-            'Apache License Version 2.0 ...',
-            [])
+        expected = ({'short_name': 'Apache 2.0', 'full_text': 'Apache License Version 2.0 ...', 'key': 'apache-2.0'}, [])
         result = api.get_license_details_from_api(
             api_url='api_url', api_key='api_key', license_key='license_key')
         assert expected == result

tests/test_gen.py

@@ -301,7 +301,8 @@ def test_generate_license_key_with_custom_file_450_with_fetch(self):
         lic_dict = {u'public-domain': [u'Public Domain',
                                        u'public-domain.LICENSE',
                                        u'This component is released to the public domain by the author.',
-                                       u'https://enterprise.dejacode.com/urn/?urn=urn:dje:license:public-domain'
+                                       u'https://enterprise.dejacode.com/urn/?urn=urn:dje:license:public-domain',
+                                       u''
                                        ]}
         a = abouts[0]
         a.license_key.value.append('public-domain')
@@ -316,6 +317,7 @@ def test_generate_license_key_with_custom_file_450_with_fetch(self):
     name: Public Domain
     file: public-domain.LICENSE
     url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:public-domain
+    spdx_license_key:
   - key: custom
     name: custom
     file: custom.txt
@@ -332,7 +334,8 @@ def test_generate_license_key_with_custom_file_450_with_fetch_with_order(self):
         lic_dict = {u'public-domain': [u'Public Domain',
                                        u'public-domain.LICENSE',
                                        u'This component is released to the public domain by the author.',
-                                       u'https://enterprise.dejacode.com/urn/?urn=urn:dje:license:public-domain'
+                                       u'https://enterprise.dejacode.com/urn/?urn=urn:dje:license:public-domain',
+                                       u''
                                        ]}
         # The first row from the test file
         a = abouts[0]
@@ -354,6 +357,7 @@ def test_generate_license_key_with_custom_file_450_with_fetch_with_order(self):
     name: Public Domain
     file: public-domain.LICENSE
     url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:public-domain
+    spdx_license_key:
   - key: custom
     name: custom
     file: custom.txt
@@ -372,6 +376,7 @@ def test_generate_license_key_with_custom_file_450_with_fetch_with_order(self):
     name: Public Domain
     file: public-domain.LICENSE
     url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:public-domain
+    spdx_license_key:
 '''
         )
         assert expected1 == result1

tests/test_model.py

@@ -690,8 +690,8 @@ def test_load_dict_issue_433(self):
             'license_expression': 'license1 AND license2',
             'notice_file': 'package1.zip.NOTICE',
             'licenses': [
-                {'key': 'license1', 'name': 'License1', 'file': 'license1.LICENSE', 'url': 'some_url'},
-                {'key': 'license2', 'name': 'License2', 'file': 'license2.LICENSE', 'url': 'some_url'},
+                {'key': 'license1', 'name': 'License1', 'file': 'license1.LICENSE', 'url': 'some_url', 'spdx_license_key': 'key'},
+                {'key': 'license2', 'name': 'License2', 'file': 'license2.LICENSE', 'url': 'some_url', 'spdx_license_key': 'key'},
             ],
         }
         about = model.About()
@@ -708,12 +708,14 @@ def test_load_dict_issue_433(self):
     name: License1
     file: license1.LICENSE
     url: some_url
+    spdx_license_key: key
   - key: license2
     name: License2
     file: license2.LICENSE
     url: some_url
+    spdx_license_key: key
 '''
-        lic_dict = {u'license1': [u'License1', u'license1.LICENSE',u'', u'some_url'], u'license2' : [u'License2', u'license2.LICENSE', u'', u'some_url']}
+        lic_dict = {u'license1': [u'License1', u'license1.LICENSE',u'', u'some_url', 'key'], u'license2' : [u'License2', u'license2.LICENSE', u'', u'some_url', 'key']}
         assert about.dumps(lic_dict) == expected
 
 

tests/test_util.py

@@ -556,24 +556,28 @@ def test_ungroup_licenses(self):
                 (u'key', u'mit'),
                 (u'name', u'MIT License'),
                 (u'file', u'mit.LICENSE'),
-                (u'url', u'https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit')]),
+                (u'url', u'https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit'),
+                (u'spdx_license_key', u'MIT')]),
             dict([
                 (u'key', u'bsd-new'),
                 (u'name', u'BSD-3-Clause'),
                 (u'file', u'bsd-new.LICENSE'),
-                (u'url', u'https://enterprise.dejacode.com/urn/?urn=urn:dje:license:bsd-new')])
+                (u'url', u'https://enterprise.dejacode.com/urn/?urn=urn:dje:license:bsd-new'),
+                (u'spdx_license_key', u'BSD-3-Clause')])
         ]
         expected_lic_key = [u'mit', u'bsd-new']
         expected_lic_name = [u'MIT License', u'BSD-3-Clause']
         expected_lic_file = [u'mit.LICENSE', u'bsd-new.LICENSE']
         expected_lic_url = [
             u'https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit',
             u'https://enterprise.dejacode.com/urn/?urn=urn:dje:license:bsd-new']
-        lic_key, lic_name, lic_file, lic_url, lic_score = util.ungroup_licenses(about)
+        expected_spdx = [u'MIT', u'BSD-3-Clause']
+        lic_key, lic_name, lic_file, lic_url, spdx_lic_key, lic_score = util.ungroup_licenses(about)
         assert expected_lic_key == lic_key
         assert expected_lic_name == lic_name
         assert expected_lic_file == lic_file
         assert expected_lic_url == lic_url
+        assert expected_spdx == spdx_lic_key
 
     def test_unique_does_deduplicate_and_keep_ordering(self):
         items = ['a', 'b', 'd', 'b', 'c', 'a']