Merge pull request #3924 from aboutcode-org/update-rules-with-required-phrases-auto

Update rules with required phrases automatically

Author: pombredanne

docs/source/how-to-guides/add_new_license_detection_rule.rst

@@ -73,7 +73,7 @@ More (advanced) rules options:
   be present in the result license detections. These just have the license text and a
   `is_false_positive` flag set to True.
 
-- you can specify key phrases by surrounding one or more words between the `{{`
+- you can specify required phrases by surrounding one or more words between the `{{`
   and `}}` tags. Key phrases are words that **must** be matched/present in order
   for a RULE to be considered a match.
 

etc/scripts/licenses/buildrules.py

@@ -16,6 +16,7 @@
 from licensedcode import models
 from licensedcode import match_hash
 from licensedcode import frontmatter
+from licensedcode.models import get_rule_id_for_text
 from license_expression import Licensing
 
 """
@@ -130,23 +131,6 @@ def load_data(location="00-new-licenses.txt"):
     return rules
 
 
-def rule_exists(text):
-    """
-    Return the matched rule identifier if the text is an existing rule matched
-    exactly, False otherwise.
-    """
-    idx = cache.get_index()
-
-    matches = idx.match(query_string=text)
-    if not matches:
-        return False
-    if len(matches) > 1:
-        return False
-    match = matches[0]
-    if match.matcher == match_hash.MATCH_HASH and match.score() == 100:
-        return match.rule.identifier
-
-
 def all_rule_by_tokens():
     """
     Return a mapping of {tuples of tokens: rule id}, with one item for each
@@ -347,7 +331,7 @@ def cli(licenses_file, dump_to_file_on_errors=False):
 
         text = rule.text
 
-        existing_rule = rule_exists(text)
+        existing_rule = get_rule_id_for_text(text)
         skinny_text = " ".join(text[:80].split()).replace("{", " ").replace("}", " ")
 
         existing_msg = (

etc/scripts/licenses/report_license_rules.py

@@ -62,6 +62,8 @@
     "is_license_reference",
     "is_license_intro",
     "is_license_clue",
+    "is_required_phrase",
+    "skip_for_required_phrase_generation",
     "is_deprecated",
     "has_unknown",
     "only_known_words",

respective.txt

@@ -0,0 +1,3 @@
+ This software is (C) by the respective authors, and licensed under the GPL
+ License.
+

setup-mini.cfg

@@ -158,6 +158,7 @@ console_scripts =
     scancode-reindex-licenses = licensedcode.reindex:reindex_licenses
     scancode-license-data = licensedcode.license_db:dump_scancode_license_data
     regen-package-docs = packagedcode.regen_package_docs:regen_package_docs
+    add-required-phrases = licensedcode.required_phrases:add_required_phrases
 
 # These are configurations for ScanCode plugins as setuptools entry points.
 # Each plugin entry hast this form:

setup.cfg

@@ -158,6 +158,8 @@ console_scripts =
     scancode-reindex-licenses = licensedcode.reindex:reindex_licenses
     scancode-license-data = licensedcode.license_db:dump_scancode_license_data
     regen-package-docs = packagedcode.regen_package_docs:regen_package_docs
+    add-required-phrases = licensedcode.required_phrases:add_required_phrases
+	gen-new-required-phrases-rules = licensedcode.required_phrases:gen_required_phrases_rules
 
 # These are configurations for ScanCode plugins as setuptools entry points.
 # Each plugin entry hast this form:

src/licensedcode/cache.py

@@ -9,6 +9,7 @@
 
 import os
 import pickle
+
 from shutil import rmtree
 
 from commoncode.fileutils import create_dir
@@ -185,10 +186,7 @@ def load_or_build(
                     additional_license_plugins=plugin_directories,
                 )
 
-                # save the cache as pickle new tree checksum
-                with open(cache_file, 'wb') as fn:
-                    pickle.dump(license_cache, fn, protocol=PICKLE_PROTOCOL)
-
+                license_cache.dump(cache_file)
                 return license_cache
 
         except lockfile.LockTimeout:
@@ -201,6 +199,13 @@ def has_additional_licenses(self):
         if cache.additional_license_directory or cache.additional_license_plugins:
             return True
 
+    def dump(self, cache_file):
+        """
+        Dump this license cache on disk at ``cache_file``.
+        """
+        with open(cache_file, 'wb') as fn:
+            pickle.dump(self, fn, protocol=PICKLE_PROTOCOL)
+
 
 def build_index(
     licenses_db=None,
@@ -235,9 +240,12 @@ def build_index(
     if not licenses_db:
         # combine the licenses in these additional directories with the licenses in the original DB
         additional_license_dirs = get_license_dirs(additional_dirs=additional_directories)
-        combined_license_directories = [licenses_data_dir] + additional_license_dirs
         # generate a single combined license db with all licenses
-        licenses_db = load_licenses_from_multiple_dirs(license_dirs=combined_license_directories)
+        licenses_db = load_licenses_from_multiple_dirs(
+            builtin_license_data_dir=licenses_data_dir,
+            additional_license_data_dirs=additional_license_dirs,
+            with_deprecated=False,
+        )
 
     # if we have additional directories, extract the rules from them
     additional_rule_dirs = get_rule_dirs(additional_dirs=additional_directories)
@@ -393,7 +401,7 @@ def get_cache(
     Return a LicenseCache either rebuilt, cached or loaded from disk.
 
     If ``index_all_languages`` is True, include texts in all languages when
-    building the license index. Otherwise, only include the English license \
+    building the license index. Otherwise, only include the English license
     texts and rules (the default)
     """
     return populate_cache(
@@ -531,7 +539,7 @@ def validate_spdx_license_keys(license_expression, licensing):
         if not type(key) == str:
             msg = f"Invalid license key: {key} of type {type(key)}, license key should be a string"
             messages.append(msg)
-    
+
         lic = license_db.get(key, None)
         if not lic:
             licenses = load_licenses(with_deprecated=True)

src/licensedcode/data/licenses/any-osi-perl-modules.LICENSE

@@ -11,6 +11,7 @@ other_urls:
     - https://metacpan.org/pod/Net::MQTT::Simple#LICENSE
 ignorable_urls:
     - http://www.opensource.org/licenses/alphabetical
+minimum_coverage: 90
 ---
 
 This software may be redistributed under the terms of the GPL, LGPL,
@@ -23,4 +24,4 @@ When using a packaged version, please refer to the package metadata to see
 under which license terms it was distributed. Alternatively, a distributor
 may choose to replace the LICENSE section of the documentation and/or
 include a LICENSE file to reflect the license(s) they chose to redistribute
-under.
+under.

src/licensedcode/data/licenses/dco-1.0.LICENSE

@@ -0,0 +1,39 @@
+---
+key: dco-1.0
+short_name: DCO 1.0
+name: Developer Certificate of Origin 1.0
+category: CLA
+owner: Linux Foundation
+homepage_url: https://developercertificate.org/
+spdx_license_key: LicenseRef-scancode-dco-1.0
+text_urls:
+    - https://developercertificate.org/
+other_urls:
+    - https://github.com/nexB/scancode-toolkit/issues/3038#issuecomment-1317511139
+minimum_coverage: 90
+---
+
+Developer's Certificate of Origin 1.0
+
+By making a contribution to this project, I certify that:
+
+1. The contribution was created in whole or in part by me and I
+   have the right to submit it under the open source license
+   indicated in the file LICENSE; or
+
+2. The contribution is based upon previous work that, to the best
+   of my knowledge, is covered under an appropriate open source
+   license and I have the right under that license to submit that
+   work with modifications, whether created in whole or in part
+   by me, under the same open source license (unless I am
+   permitted to submit under a different license), as indicated
+   in the file LICENSE; or
+
+3. The contribution was provided directly to me by some other
+   person who certified (1), (2) or (3) and I have not modified
+   it.
+
+4. I understand and agree that this project and the contribution
+   are public and that a record of the contribution (including all
+   personal information I submit with it, including my sign-off) is
+   maintained indefinitely and may be redistributed consistent

src/licensedcode/data/licenses/samba-dco-1.0.LICENSE

@@ -0,0 +1,51 @@
+---
+key: samba-dco-1.0
+short_name: Samba Developer's Declaration, Version 1.0
+name: Samba Developer's Declaration, Version 1.0
+category: CLA
+owner: Samba
+homepage_url: https://github.com/samba-team/samba/blob/master/README.contributing
+spdx_license_key: LicenseRef-scancode-samba-dco-1.0
+text_urls:
+    - https://github.com/samba-team/samba/blob/master/README.contributing
+minimum_coverage: 80
+ignorable_urls:
+    - http://www.gnu.org/licenses/gpl-3.0.html
+    - http://www.gnu.org/licenses/lgpl-3.0.html
+---
+
+Samba Developer's Declaration, Version 1.0
+
+By making a contribution to this project, I certify that:
+
+(a) The contribution was created in whole or in part by me and I
+    have the right to submit it under the appropriate
+    version of the GNU General Public License; or
+
+(b) The contribution is based upon previous work that, to the best
+    of my knowledge, is covered under an appropriate open source
+    license and I have the right under that license to submit that
+    work with modifications, whether created in whole or in part
+    by me, under the GNU General Public License, in the
+    appropriate version; or
+
+(c) The contribution was provided directly to me by some other
+    person who certified (a) or (b) and I have not modified
+    it.
+
+(d) I understand and agree that this project and the
+    contribution are public and that a record of the
+    contribution (including all metadata and personal
+    information I submit with it, including my sign-off) is
+    maintained indefinitely and may be redistributed
+    consistent with the Samba Team's policies and the
+    requirements of the GNU GPL where they are relevant.
+
+(e) I am granting this work to this project under the terms of both
+    the GNU General Public License and the GNU Lesser General Public
+    License as published by the Free Software Foundation; either version
+    3 of these Licenses, or (at the option of the project) any later
+    version.
+
+    http://www.gnu.org/licenses/gpl-3.0.html
+    http://www.gnu.org/licenses/lgpl-3.0.html