Support equivalent words in license detection #4190

Handle similar words in license detection by allowing multiple
"legalese words" to have the same token id.

Regenerate the tokens ids accordingly.

Convert Index.tokens_by_tid to a computed property, available on demand.
Convert tokens_by_tid to a dictionary from a list.
Ensure that all code relying on the tokens_by_tid is updated as needed.
All locations were used only for testing and debugging.

Deprecate all rules that are duplicated under this new regime, where
tokens like "license" and "licence" are not treated as identical.

Update test suite to test the detection of all deprecated licenses and
rules as a sanity check. A rule with "relevance" set to 0 is not tested
if deprecated, as some rules are deprecated because they are false
positive and should no longer be detected. Also improved the validation
and loading of rules relevance, including the case for zero relevance.

Update ambiguous or conflicting rules as needed.
In particular ensure that all rules in the style of "MIT or GPL"
without a GPL version are now reported consistently as:
"mit or gpl-1.0-plus"

Add new rules as needed to resolve failing tests and improve accuracy.

Improve deprecated support for rules and licenses, adding a new
"replaced_by" list attribute that lists the new expressions that must be
detected from scanning the deprecated license or rule text.

Reference: #4190
Signed-off-by: Philippe Ombredanne <[email protected]>

Author: pombredanne

etc/scripts/licenses/buildrules.py

@@ -149,7 +149,7 @@ def rule_exists(text):
 
 def all_rule_by_tokens():
     """
-    Return a mapping of {tuples of tokens: rule id}, with one item for each
+    Return a mapping of {(tuple of token id): rule id}, with one item for each
     existing and added rules. Used to avoid duplicates.
     """
     rule_tokens = {}
@@ -159,7 +159,7 @@ def all_rule_by_tokens():
         except Exception as e:
             rf = f"  file://{rule.rule_file()}"
             raise Exception(
-                f"Failed to to get tokens from rule:: {rule.identifier}\n" f"{rf}"
+                f"Failed to get tokens from rule:: {rule.identifier}\n" f"{rf}"
             ) from e
     return rule_tokens
 

src/formattedcode/output_cyclonedx.py

@@ -20,9 +20,9 @@
 from typing import List
 
 import attr
+from lxml import etree
 from commoncode.cliutils import OUTPUT_GROUP
 from commoncode.cliutils import PluggableCommandLineOption
-from lxml import etree
 from plugincode.output import OutputPlugin
 from plugincode.output import output_impl
 

src/licensedcode/cache.py

@@ -276,7 +276,7 @@ def build_licensing(licenses_db=None):
     from licensedcode.models import load_licenses
 
     licenses_db = licenses_db or load_licenses()
-    return Licensing((LicenseSymbolLike(lic) for lic in licenses_db.values()))
+    return Licensing(symbols=(LicenseSymbolLike(lic) for lic in licenses_db.values()))
 
 
 def build_spdx_symbols(licenses_db=None):
@@ -316,7 +316,6 @@ def get_licenses_by_spdx_key(
 
     Optionally include deprecated if ``include_deprecated`` is True.
 
-
     Optionally make the keys lowercase if ``lowercase_keys`` is True.
 
     Optionally include the license "other_spdx_license_keys" if present and

src/licensedcode/data/licenses/agpl-3.0-bacula.LICENSE

@@ -1,6 +1,14 @@
 ---
 key: agpl-3.0-bacula
 is_deprecated: yes
+replaced_by:
+    - bacula-exception
+    - bsd-simplified
+    - bsd-simplified
+    - bsd-simplified
+    - agpl-3.0-plus
+    - agpl-3.0-plus
+    - agpl-3.0
 short_name: AGPL 3.0 with Bacula exception
 name: AGPL 3.0 with Bacula exception
 category: Copyleft

src/licensedcode/data/licenses/agpl-3.0-linking-exception.LICENSE

@@ -1,13 +1,15 @@
 ---
 key: agpl-3.0-linking-exception
+is_deprecated: yes
+replaced_by:
+    - linking-exception-agpl-3.0
 short_name: AGPL 3.0 linking exception
 name: AGPL 3.0 linking exception
 category: Copyleft Limited
 owner: Unspecified
-is_exception: yes
 homepage_url: http://mo.morsi.org/blog/2009/08/13/lesser_affero_gplv3/
 notes: renamed to linking-exception-agpl-3.0
-is_deprecated: yes
+is_exception: yes
 ---
 
 Additional permission under the GNU Affero GPL version 3 section 7:

src/licensedcode/data/licenses/agpl-3.0-openssl.LICENSE

@@ -1,15 +1,16 @@
 ---
 key: agpl-3.0-openssl
+is_deprecated: yes
+replaced_by:
+    - openssl-exception-agpl-3.0
 short_name: AGPL 3.0 with OpenSSL exception
 name: AGPL 3.0 with OpenSSL exception
 category: Copyleft
 owner: MongoDB
-is_exception: yes
-is_deprecated: yes
 notes: replaced by openssl-exception-agpl-3.0
+is_exception: yes
 ---
 
-
 As a special exception, the copyright holders give permission to link the
 code of portions of this program with the OpenSSL library under certain
 conditions as described in each individual source file and distribute

src/licensedcode/data/licenses/aladdin-md5.LICENSE

@@ -1,6 +1,8 @@
 ---
 key: aladdin-md5
 is_deprecated: yes
+replaced_by:
+    - zlib
 short_name: Aladdin MD5 License
 name: Aladdin MD5 License
 category: Permissive

src/licensedcode/data/licenses/aop-pd.LICENSE

@@ -1,8 +1,10 @@
 ---
 key: aop-pd
+is_deprecated: yes
+replaced_by:
+    - cc-pd
 short_name: AOP-PD
 name: AOP Public Domain License
-is_deprecated: yes
 category: Public Domain
 owner: AOP Alliance Project
 ---

src/licensedcode/data/licenses/apache-2.0-linking-exception.LICENSE

@@ -1,12 +1,14 @@
 ---
 key: apache-2.0-linking-exception
+is_deprecated: yes
+replaced_by:
+    - compuphase-linking-exception
 short_name: Apache 2.0 with Linking Exception
 name: Apache 2.0 with Linking Exception
 category: Permissive
 owner: compuphase
 homepage_url: https://github.com/compuphase/minIni/blob/master/LICENSE
 is_exception: yes
-is_deprecated: yes
 ---
 
 EXCEPTION TO THE APACHE 2.0 LICENSE

src/licensedcode/data/licenses/apache-2.0-runtime-library-exception.LICENSE

@@ -1,5 +1,8 @@
 ---
 key: apache-2.0-runtime-library-exception
+is_deprecated: yes
+replaced_by:
+    - apple-runtime-library-exception
 short_name: Apache 2.0 with Runtime Library Exception
 name: Apache 2.0 with Runtime Library Exception
 category: Permissive
@@ -8,7 +11,6 @@ homepage_url: https://github.com/apple/swift/blob/master/LICENSE.txt#L205
 is_exception: yes
 other_urls:
     - https://swift.org/
-is_deprecated: yes
 ---
 
 ## Runtime Library Exception to the Apache 2.0 License: ##