Merge pull request #2528 from nexB/2507-license-bugs

Update license detection

Author: pombredanne

etc/scripts/licenses/synclic.py

@@ -546,10 +546,13 @@ def build_license(self, mapping, scancode_licenses):
         # instead each part of the combo
         dejacode_special_composites = set([
               'intel-bsd-special',
+              #'newlib-subdirectory',
             ])
-        is_combo = key in dejacode_special_composites
+        is_component_license = mapping.get('is_component_license') or False
+
+        is_combo = is_component_license or key in dejacode_special_composites
         if is_combo:
-            if TRACE: print('Skipping DejaCode combo license', key)
+            if TRACE: print('Skipping DejaCode combo/component license', key)
             return
 
         deprecated = not mapping.get('is_active')
@@ -560,6 +563,13 @@ def build_license(self, mapping, scancode_licenses):
         standard_notice = mapping.get('standard_notice') or ''
         standard_notice = clean_text(standard_notice)
 
+        spdx_license_key = mapping.get('spdx_license_key') or None
+        if deprecated:
+            spdx_license_key = None
+        else:
+            if not spdx_license_key:
+                spdx_license_key = f'LicenseRef-scancode-{key}'
+
         lic = License(
             key=key,
             src_dir=self.original_dir,
@@ -572,7 +582,7 @@ def build_license(self, mapping, scancode_licenses):
 
             # FIXME: we may not want to carry notes over???
             # lic.notes = mapping.notes
-            spdx_license_key=mapping['spdx_license_key'],
+            spdx_license_key=spdx_license_key,
             text_urls=mapping['text_urls'].splitlines(False),
             osi_url=mapping['osi_url'],
             faq_url=mapping['faq_url'],
@@ -805,8 +815,12 @@ def license_to_dict(lico):
 }
 
 
-def merge_licenses(scancode_license, external_license, updatable_attributes,
-                   from_spdx=False):
+def merge_licenses(
+    scancode_license, 
+    external_license, 
+    updatable_attributes,
+    from_spdx=False,
+):
     """
     Compare and update two License objects in-place given a sequence of
     `updatable_attributes`.
@@ -839,20 +853,21 @@ def update_external(_attrib, _sc_val, _ext_val):
         external_key = external_license.spdx_license_key
         if scancode_key != external_key:
             raise Exception(
-                f'Non mergeable licenses with different SPDX keys: scancode_license.spdx_license_key {scancode_key} <>  external_license.spdx_license_key {external_key}'
+                f'Non mergeable licenses with different SPDX keys: '
+                'scancode_license.spdx_license_key {scancode_key} <>  '
+                'external_license.spdx_license_key {external_key}'
             )
     else:
         scancode_key = scancode_license.key
         external_key = external_license.key
         if scancode_key != external_key:
-            raise Exception('Non mergeable licenses with different keys: %(scancode_key)s <> %(external_key)s' % locals())
+            raise Exception('Non mergeable licenses with different keys: '
+                            '%(scancode_key)s <> %(external_key)s' % locals())
 
-#         if scancode_license.spdx_license_key != external_license.spdx_license_key:
-#             pass
-#         else:
-#             if TRACE:
-#                 print('Merging licenses with different keys, but same SPDX key: %(scancode_key)s <> %(external_key)s' % locals())
-#             update_external('key', scancode_key, external_key)
+        if scancode_license.spdx_license_key != external_license.spdx_license_key:
+            if TRACE:
+                print(f'Updating external SPDX key: from {external_license.spdx_license_key} to {scancode_license.spdx_license_key}')
+            external_license.spdx_license_key = scancode_license.spdx_license_key
 
     for attrib in updatable_attributes:
         scancode_value = getattr(scancode_license, attrib)
@@ -932,7 +947,11 @@ def update_external(_attrib, _sc_val, _ext_val):
 
         # on difference, the other license wins
         if scancode_value != external_value:
-            update_scancode(attrib, scancode_value, external_value)
+            # unless we have SPDX ids
+            if attrib== 'spdx_license_key' and external_value.startswith('LicenseRef-scancode'):
+                update_external(attrib, scancode_value, external_value)
+            else:
+                update_scancode(attrib, scancode_value, external_value)
             continue
 
     return updated_scancode_attributes, updated_external_attributes
@@ -1065,16 +1084,20 @@ def synchronize_licenses(scancode_licenses, external_source, use_spdx_key=False,
                 print('External license with different key:', matching_key, 'and text matched to ScanCode key:', matched_key)
 
             if matched_key:
-                print('\nExternal license with different key and matched text to ScanCode:', matching_key, ' matched to:', matched_key)
+                print('External license with different key:', matching_key, 'and text matched to ScanCode key:', matched_key)
                 if matched_key in unmatched_scancode_by_key:
                     del unmatched_scancode_by_key[matched_key]
 
                 scancode_license = scancodes_by_key.get(matched_key)
+                if TRACE:
+                    print('scancode_license:', matching_key, scancode_license)
 
                 scancode_updated, external_updated = merge_licenses(
-                    scancode_license, external_license,
-                    external_source.updatable_attributes,
-                    from_spdx=use_spdx_key)
+                    scancode_license=scancode_license,
+                    external_license=external_license,
+                    updatable_attributes=external_source.updatable_attributes,
+                    from_spdx=use_spdx_key,
+                )
 
                 if not scancode_updated and not external_updated:
                     if TRACE_DEEP: print('License attributes are identical:', matching_key)

src/licensedcode/data/licenses/acdl-1.0.yml

@@ -4,10 +4,16 @@ name: Apple Common Documentation License v1.0
 category: Copyleft Limited
 owner: Apple
 homepage_url: http://fedoraproject.org/wiki/Licensing/Common_Documentation_License
-spdx_license_key: LicenseRef-scancode-acdl-1.0
+spdx_license_key: CDL-1.0
+other_spdx_license_keys:
+    - LicenseRef-scancode-acdl-1.0
 text_urls:
-    - http://fedoraproject.org/wiki/Licensing/Common_Documentation_License
+    - http://www.opensource.apple.com/cdl/
 faq_url: http://fedoraproject.org/wiki/Licensing/Common_Documentation_License
+other_urls:
+    - http://fedoraproject.org/wiki/Licensing/Common_Documentation_License
+    - https://fedoraproject.org/wiki/Licensing/Common_Documentation_License
+    - https://www.gnu.org/licenses/license-list.html#ACDL
 ignorable_copyrights:
     - Copyright (c) 2001 Apple Computer, Inc.
 ignorable_holders:

src/licensedcode/data/licenses/apl-1.1.yml

@@ -3,12 +3,12 @@ short_name: APL-1.1
 name: Academic Public License version 1.1
 category: Permissive
 owner: OMNeT++
-spdx_license_key: LicenseRef-scancode-apl-1.1
 homepage_url: https://github.com/omnetpp/omnetpp/blob/master/doc/License
-other_urls: 
-     - https://opencarp.org/download/license
+spdx_license_key: LicenseRef-scancode-apl-1.1
+other_urls:
+    - https://opencarp.org/download/license
 ignorable_copyrights:
-    -  Copyright (c) 2003, 2010, 2015 Andras Varga
+    - Copyright (c) 2003, 2010, 2015 Andras Varga
 ignorable_holders:
     - Andras Varga
 ignorable_urls:

src/licensedcode/data/licenses/apple-mfi-license.LICENSE

@@ -0,0 +1,43 @@
+Disclaimer: IMPORTANT: This Apple software is supplied to you, by Apple Inc. ("Apple"), in your
+capacity as a current, and in good standing, Licensee in the MFi Licensing Program. Use of this
+Apple software is governed by and subject to the terms and conditions of your MFi License,
+including, but not limited to, the restrictions specified in the provision entitled ”Public
+Software”, and is further subject to your agreement to the following additional terms, and your
+agreement that the use, installation, modification or redistribution of this Apple software
+constitutes acceptance of these additional terms. If you do not agree with these additional terms,
+please do not use, install, modify or redistribute this Apple software.
+
+Subject to all of these terms and in consideration of your agreement to abide by them, Apple grants
+you, for as long as you are a current and in good-standing MFi Licensee, a personal, non-exclusive
+license, under Apple's copyrights in this original Apple software (the "Apple Software"), to use,
+reproduce, and modify the Apple Software in source form, and to use, reproduce, modify, and
+redistribute the Apple Software, with or without modifications, in binary form. While you may not
+redistribute the Apple Software in source form, should you redistribute the Apple Software in binary
+form, you must retain this notice and the following text and disclaimers in all such redistributions
+of the Apple Software. Neither the name, trademarks, service marks, or logos of Apple Inc. may be
+used to endorse or promote products derived from the Apple Software without specific prior written
+permission from Apple. Except as expressly stated in this notice, no other rights or licenses,
+express or implied, are granted by Apple herein, including but not limited to any patent rights that
+may be infringed by your derivative works or by other works in which the Apple Software may be
+incorporated.
+
+Unless you explicitly state otherwise, if you provide any ideas, suggestions, recommendations, bug
+fixes or enhancements to Apple in connection with this software (“Feedback”), you hereby grant to
+Apple a non-exclusive, fully paid-up, perpetual, irrevocable, worldwide license to make, use,
+reproduce, incorporate, modify, display, perform, sell, make or have made derivative works of,
+distribute (directly or indirectly) and sublicense, such Feedback in connection with Apple products
+and services. Providing this Feedback is voluntary, but if you do provide Feedback to Apple, you
+acknowledge and agree that Apple may exercise the license granted above without the payment of
+royalties or further consideration to Participant.
+
+The Apple Software is provided by Apple on an "AS IS" basis. APPLE MAKES NO WARRANTIES, EXPRESS OR
+IMPLIED, INCLUDING WITHOUT LIMITATION THE IMPLIED WARRANTIES OF NON-INFRINGEMENT, MERCHANTABILITY
+AND FITNESS FOR A PARTICULAR PURPOSE, REGARDING THE APPLE SOFTWARE OR ITS USE AND OPERATION ALONE OR
+IN COMBINATION WITH YOUR PRODUCTS.
+
+IN NO EVENT SHALL APPLE BE LIABLE FOR ANY SPECIAL, INDIRECT, INCIDENTAL OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) ARISING IN ANY WAY OUT OF THE USE, REPRODUCTION, MODIFICATION
+AND/OR DISTRIBUTION OF THE APPLE SOFTWARE, HOWEVER CAUSED AND WHETHER UNDER THEORY OF CONTRACT, TORT
+(INCLUDING NEGLIGENCE), STRICT LIABILITY OR OTHERWISE, EVEN IF APPLE HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.

src/licensedcode/data/licenses/apple-mfi-license.yml

@@ -0,0 +1,6 @@
+key: apple-mfi-license
+short_name: Apple MFi License
+name: Apple MFi License
+category: Proprietary Free
+owner: Apple
+spdx_license_key: LicenseRef-scancode-apple-mfi-license

src/licensedcode/data/licenses/artistic-dist-1.0.yml

@@ -8,8 +8,8 @@ notes: |
  This is a variant Artistic license still used in today's Perl and that 
  comes with the "dist" tool included in the standard Perl distrbution.
  See also https://github.com/pexip/os-perl/blob/d7a993d1118c65177c97ae41577b188653845a8a/regen-configure/U/README#L158
- 
 spdx_license_key: LicenseRef-scancode-artistic-1988-1.0
 text_urls:
     - https://raw.githubusercontent.com/rmanfredi/dist/2cec35331a912b165e2dd135d22de81f34bbc83f/Artistic
-
+other_urls:
+    - https://github.com/pexip/os-perl/blob/d7a993d1118c65177c97ae41577b188653845a8a/regen-configure/U/README#L158

src/licensedcode/data/licenses/broadcom-dual.yml

@@ -1,7 +1,7 @@
 key: broadcom-dual
+is_deprecated: yes
 short_name: Broadcom Dual GPL-Commercial
 name: Broadcom Dual GPL-Commercial
 category: Copyleft
 owner: Broadcom
 notes: composite
-is_deprecated: yes

src/licensedcode/data/licenses/bsd-3-clause-no-military.yml

@@ -3,8 +3,10 @@ short_name: BSD-3-Clause-No-Military
 name: BSD-3-Clause-No-Military
 category: Free Restricted
 owner: Unspecified
+spdx_license_key: BSD-3-Clause-No-Military-License
+other_spdx_license_keys:
+    - LicenseRef-scancode-bsd-3-clause-no-military
 other_urls:
-    - https://github.com/greymass/swift-eosio/blob/master/LICENSE
     - https://gitlab.syncad.com/hive/dhive/-/blob/master/LICENSE
+    - https://github.com/greymass/swift-eosio/blob/master/LICENSE
 minimum_coverage: 90
-spdx_license_key: LicenseRef-scancode-bsd-3-clause-no-military

src/licensedcode/data/rules/bsd-original_57.RULE renamed to src/licensedcode/data/licenses/bsd-original-voices.LICENSE

@@ -8,7 +8,7 @@
  *    documentation and/or other materials provided with the distribution.
  * 3. All advertising materials mentioning features or use of this software
  *    must display the following acknowledgement:
- *    This product includes software developed by Bill Paul.
+ *	This product includes software developed by Bill Paul.
  * 4. Neither the name of the author nor the names of any co-contributors
  *    may be used to endorse or promote products derived from this software
  *    without specific prior written permission.
@@ -23,5 +23,4 @@
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
- * THE POSSIBILITY OF SUCH DAMAGE.
- */
+ * THE POSSIBILITY OF SUCH DAMAGE.

src/licensedcode/data/licenses/bsd-original-voices.yml

@@ -0,0 +1,10 @@
+key: bsd-original-voices
+short_name: BSD-4-Clause with Voices
+name: BSD-4-Clause with Voices in Head
+category: Permissive
+owner: Bill Paul
+homepage_url: https://github.com/freebsd/freebsd-src/blob/098dbd7ff7f3da9dda03802cdb2d8755f816eada/sys/dev/an/if_an_isa.c
+spdx_license_key: LicenseRef-scancode-bsd-original-voices
+ignorable_authors:
+  - Bill Paul
+