Merge latest develop

Signed-off-by: Philippe Ombredanne <[email protected]>

Author: pombredanne

CHANGELOG.rst

@@ -46,6 +46,11 @@ v33.0.0 (next next, roadmap)
     license/exception texts added, and also 1 license was deprecated.
     For more details see https://github.com/aboutcode-org/scancode-toolkit/pull/3897
 
+- New and improved copyright detection with many false positive removed
+  and refined detection added.
+
+- Fix Python ``SyntaxWarning`` in textcode module.
+
 v32.2.1 - 2024-07-02
 ---------------------
 
@@ -881,6 +886,8 @@ Package detection:
 
 - For Pypi packages, python_requires is treated as a package dependency.
 
+- Update JavaScript package handlers to handle aliases in npm and yarn manifests.
+
 
 License Clarity Scoring Update:
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

azure-pipelines.yml

@@ -75,6 +75,33 @@ jobs:
                   tests/licensedcode/test_detection_validate.py \
                   -k TestValidateLicenseExtended5
 
+
+              license_validate_ignorables_1: |
+                venv/bin/pytest -n 3 -vvs --test-suite=validate \
+                  tests/licensedcode/test_detection_validate.py \
+                  -k TestValidateLicenseIgnorableClues1
+
+              license_validate_ignorables_2: |
+                venv/bin/pytest -n 3 -vvs --test-suite=validate \
+                  tests/licensedcode/test_detection_validate.py \
+                  -k TestValidateLicenseIgnorableClues2
+
+              license_validate_ignorables_3: |
+                venv/bin/pytest -n 3 -vvs --test-suite=validate \
+                  tests/licensedcode/test_detection_validate.py \
+                  -k TestValidateLicenseIgnorableClues3
+
+              license_validate_ignorables_4: |
+                venv/bin/pytest -n 3 -vvs --test-suite=validate \
+                  tests/licensedcode/test_detection_validate.py \
+                  -k TestValidateLicenseIgnorableClues4
+
+              license_validate_ignorables_5: |
+                venv/bin/pytest -n 3 -vvs --test-suite=validate \
+                  tests/licensedcode/test_detection_validate.py \
+                  -k TestValidateLicenseIgnorableClues5
+
+
               license_cache: |
                 venv/bin/pytest -n 3 -vvs --test-suite=all \
                   tests/licensedcode/test_zzzz_cache.py --reruns 2

etc/scripts/gen_copyright_tests.py

@@ -93,7 +93,7 @@ def create_copyright_tests(
                 content = url
 
             if end_line != 0:
-                content = "".join(content.strip().splitlines()[start_line:end_line])
+                content = "\n".join(content.strip().splitlines()[start_line:end_line])
 
             with open(name, "w") as out:
                 out.write(content)

etc/scripts/licenses/buildrules.py

@@ -73,6 +73,7 @@ def __attrs_post_init__(self, *args, **kwargs):
             print(rdat)
             print("########################################################")
             raise
+        self.data = {k: v for k, v in self.data.items() if v is not None or (v is None and k == "license_expression")}
 
 
 def load_data(location="00-new-licenses.txt"):

requirements.txt

@@ -10,7 +10,7 @@ chardet==5.0.0
 charset-normalizer==2.1.0
 click==8.1.7
 colorama==0.4.5
-commoncode==31.2.1
+commoncode==32.0.0
 construct==2.10.68
 container-inspector==31.1.0
 cryptography==42.0.5
@@ -54,7 +54,7 @@ ply==3.11
 publicsuffix2==2.20191221
 pyahocorasick==2.1.0
 pycparser==2.21
-pygmars==0.7.0
+pygmars==0.9.0
 Pygments==2.13.0
 pymaven-patch==0.3.2
 pyparsing==3.0.9

setup-mini.cfg

@@ -70,7 +70,7 @@ install_requires =
     chardet >= 3.0.0
     click >= 6.7, !=7.0
     colorama >= 0.3.9
-    commoncode >= 31.0.3
+    commoncode >= 32.0.0
     container-inspector >= 31.0.0
     debian-inspector >= 31.1.0
     dparse2 >= 0.7.0
@@ -100,7 +100,7 @@ install_requires =
     plugincode >= 32.0.0
     publicsuffix2
     pyahocorasick >= 2.0.0
-    pygmars >= 0.7.0
+    pygmars >= 0.9.0
     pygments
     pymaven_patch >= 0.2.8
     requests >= 2.7.0
@@ -112,7 +112,7 @@ install_requires =
     xmltodict >= 0.11.0
     zipp >= 3.0.0; python_version < "3.9"
     typecode >= 30.0.1
-#    typecode[full] >= 30.0.0
+#    typecode[full] >= 30.0.1
 #    extractcode[full] >= 31.0.0
 
 
@@ -200,6 +200,7 @@ scancode_post_scan =
     filter-clues = cluecode.plugin_filter_clues:RedundantCluesFilter
     consolidate = summarycode.plugin_consolidate:Consolidator
     license-references = licensedcode.licenses_reference:LicenseReference
+    todo = summarycode.todo:AmbiguousDetectionsToDoPlugin
     classify = summarycode.classify_plugin:FileClassifier
 
 

setup.cfg

@@ -70,7 +70,7 @@ install_requires =
     chardet >= 3.0.0
     click >= 6.7, !=7.0
     colorama >= 0.3.9
-    commoncode >= 31.0.3
+    commoncode >= 32.0.0
     container-inspector >= 31.0.0
     debian-inspector >= 31.1.0
     dparse2 >= 0.7.0
@@ -100,7 +100,7 @@ install_requires =
     plugincode >= 32.0.0
     publicsuffix2
     pyahocorasick >= 2.0.0
-    pygmars >= 0.7.0
+    pygmars >= 0.9.0
     pygments
     pymaven_patch >= 0.2.8
     requests >= 2.7.0
@@ -138,9 +138,9 @@ testing =
 
 docs =
     Sphinx == 5.1.0
-    sphinx-rtd-theme >= 0.5.0
-    doc8 >= 0.8.1
+    sphinx_rtd_theme >= 0.5.1
     sphinx-reredirects >= 0.1.2
+    doc8 >= 0.8.1
     sphinx-autobuild
     sphinx-rtd-dark-mode>=1.3.0
     sphinx-copybutton