Merge branch 'develop' into update-license-rules

Author: AyanSinhaMahapatra

.github/ISSUE_TEMPLATE/--doc-report.md

@@ -1,8 +1,8 @@
 ---
-name: "Documentation Improvement"
+name: Documentation Improvement
 about: Your suggestions for improving the scancode-toolkit Documentation.
 title: ''
-labels: 'documentation'
+labels: documentation
 assignees: ''
 
 ---

.github/ISSUE_TEMPLATE/new-license-request.md

@@ -0,0 +1,20 @@
+---
+name: New License Request
+about: Request a new license for scancode detection purposes
+title: 'New license request: '
+labels: new-license
+assignees: DennisClark
+
+---
+
+**License Name**
+The full name of the license, preferably as provided by the original authors.
+
+**License Homepage URL**
+The primary and/or most reliable URL that contains the text of this license and/or information about it. This is ideally a web page dedicated to the license, but it may also be a source file that contains the license text.
+
+**Reference URLs**
+Optionally provide URLs that contain references to the license.
+ 
+**License Notes**
+Optionally describe any aspects of the license that deserve special attention.

.github/workflows/about-files-ci.yml

@@ -7,16 +7,16 @@ permissions:
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
 
     strategy:
       max-parallel: 4
       matrix:
-        python-version: [3.9]
+        python-version: [3.11]
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4

azure-pipelines.yml

@@ -14,7 +14,7 @@ jobs:
     - template: etc/ci/azure-posix.yml
       parameters:
           job_name: core_tests
-          image_name: ubuntu-20.04
+          image_name: ubuntu-24.04
           python_versions: ['3.10']
           test_suites:
               misc_and_scancode: |
@@ -197,8 +197,8 @@ jobs:
 
     - template: etc/ci/azure-posix.yml
       parameters:
-          job_name: ubuntu20_test_all_supported_click_versions
-          image_name: ubuntu-20.04
+          job_name: ubuntu22_test_all_supported_click_versions
+          image_name: ubuntu-22.04
           python_versions: ['3.9', '3.10', '3.11', '3.12']
           python_architecture: x64
           test_suites:
@@ -225,8 +225,8 @@ jobs:
 
     - template: etc/ci/azure-posix.yml
       parameters:
-          job_name: ubuntu20_cpython_latest_from_pip
-          image_name: ubuntu-20.04
+          job_name: ubuntu24_cpython_latest_from_pip
+          image_name: ubuntu-24.04
           python_versions: ['3.9', '3.10', '3.11', '3.12']
           test_suites:
               all: venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[testing] && venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py

docs/source/how-to-guides/add_new_license_detection_rule.rst

@@ -73,7 +73,7 @@ More (advanced) rules options:
   be present in the result license detections. These just have the license text and a
   `is_false_positive` flag set to True.
 
-- you can specify key phrases by surrounding one or more words between the `{{`
+- you can specify required phrases by surrounding one or more words between the `{{`
   and `}}` tags. Key phrases are words that **must** be matched/present in order
   for a RULE to be considered a match.
 

etc/scripts/licenses/buildrules.py

@@ -16,6 +16,7 @@
 from licensedcode import models
 from licensedcode import match_hash
 from licensedcode import frontmatter
+from licensedcode.models import get_rule_id_for_text
 from license_expression import Licensing
 
 """
@@ -130,23 +131,6 @@ def load_data(location="00-new-licenses.txt"):
     return rules
 
 
-def rule_exists(text):
-    """
-    Return the matched rule identifier if the text is an existing rule matched
-    exactly, False otherwise.
-    """
-    idx = cache.get_index()
-
-    matches = idx.match(query_string=text)
-    if not matches:
-        return False
-    if len(matches) > 1:
-        return False
-    match = matches[0]
-    if match.matcher == match_hash.MATCH_HASH and match.score() == 100:
-        return match.rule.identifier
-
-
 def all_rule_by_tokens():
     """
     Return a mapping of {tuples of tokens: rule id}, with one item for each
@@ -347,7 +331,7 @@ def cli(licenses_file, dump_to_file_on_errors=False):
 
         text = rule.text
 
-        existing_rule = rule_exists(text)
+        existing_rule = get_rule_id_for_text(text)
         skinny_text = " ".join(text[:80].split()).replace("{", " ").replace("}", " ")
 
         existing_msg = (

etc/scripts/licenses/report_license_rules.py

@@ -62,6 +62,8 @@
     "is_license_reference",
     "is_license_intro",
     "is_license_clue",
+    "is_required_phrase",
+    "skip_for_required_phrase_generation",
     "is_deprecated",
     "has_unknown",
     "only_known_words",

respective.txt

@@ -0,0 +1,3 @@
+ This software is (C) by the respective authors, and licensed under the GPL
+ License.
+

setup-mini.cfg

@@ -158,6 +158,7 @@ console_scripts =
     scancode-reindex-licenses = licensedcode.reindex:reindex_licenses
     scancode-license-data = licensedcode.license_db:dump_scancode_license_data
     regen-package-docs = packagedcode.regen_package_docs:regen_package_docs
+    add-required-phrases = licensedcode.required_phrases:add_required_phrases
 
 # These are configurations for ScanCode plugins as setuptools entry points.
 # Each plugin entry hast this form:

setup.cfg

@@ -158,6 +158,8 @@ console_scripts =
     scancode-reindex-licenses = licensedcode.reindex:reindex_licenses
     scancode-license-data = licensedcode.license_db:dump_scancode_license_data
     regen-package-docs = packagedcode.regen_package_docs:regen_package_docs
+    add-required-phrases = licensedcode.required_phrases:add_required_phrases
+	gen-new-required-phrases-rules = licensedcode.required_phrases:gen_required_phrases_rules
 
 # These are configurations for ScanCode plugins as setuptools entry points.
 # Each plugin entry hast this form: