Improve license rules and license tests

Signed-off-by: Philippe Ombredanne <[email protected]>

Author: pombredanne

src/licensedcode/data/rules/gfdl-1.1_36.RULE

@@ -20,7 +20,7 @@ This program is free software: you can redistribute it and/or modify
     You should have received a copy of the {{GNU General Public License}}
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-On Debian GNU/Linux systems, the complete text of the {{GNU General Public License}}
+On Debian GNU/Linux systems, the complete text of the GNU General Public License
 version 2 can be found in `/usr/share/common-licenses/GPL-2'.  the
 complete text of the {{GNU General Public License version 3 can be found
-in `/usr/share/common-licenses/GPL-3}}'.
+in `/usr/share/common-licenses/GPL-3}}'.

src/licensedcode/data/rules/gpl-3.0-plus_417.RULE

@@ -26,4 +26,4 @@ notes: See in LIBTASN1
  On Debian GNU/Linux systems, the complete text of the {{GNU Lesser
  General Public License}} can be found in
  `/usr/share/common-licenses/LGPL'; the text of the earliest applying version
- of the license (2.1) can be found in `/usr/share/common-licenses/LGPL-2.1'.
+ of the license (2.1) can be found in {{ /usr/share/common-licenses/LGPL-2.1 }}

src/licensedcode/data/rules/lgpl-2.1-plus_312.RULE

@@ -29,10 +29,10 @@ headers:
       system_environment:
         operating_system: linux
         cpu_architecture: 64
-        platform: Linux-5.15.0-119-generic-x86_64-with-glibc2.35
-        platform_version: '#129-Ubuntu SMP Fri Aug 2 19:25:20 UTC 2024'
-        python_version: 3.10.12 (main, Jul 29 2024, 16:56:48) [GCC 11.4.0]
-      spdx_license_list_version: '3.24'
+        platform: Linux-6.8.0-45-generic-x86_64-with-glibc2.35
+        platform_version: '#45~22.04.1-Ubuntu SMP PREEMPT_DYNAMIC Wed Sep 11 15:25:05 UTC 2'
+        python_version: 3.11.8 (main, Feb 12 2024, 14:25:06) [GCC 11.4.0]
+      spdx_license_list_version: '3.25'
       files_count: 4
 summary:
   declared_license_expression: apache-2.0
@@ -856,7 +856,7 @@ license_rule_references:
     is_license_tag: yes
     is_license_intro: no
     is_license_clue: no
-    is_required_phrase: no
+    is_required_phrase: yes
     skip_for_required_phrase_generation: no
     is_continuous: no
     is_builtin: yes
@@ -872,7 +872,7 @@ license_rule_references:
     ignorable_authors: []
     ignorable_urls: []
     ignorable_emails: []
-    text: 'license: Apache-2.0'
+    text: license="Apache-2.0
   - license_expression: apache-2.0
     identifier: apache-2.0_73.RULE
     language: en
@@ -982,7 +982,7 @@ license_rule_references:
     is_license_tag: no
     is_license_intro: no
     is_license_clue: no
-    is_required_phrase: no
+    is_required_phrase: yes
     skip_for_required_phrase_generation: no
     is_continuous: no
     is_builtin: yes
@@ -998,7 +998,7 @@ license_rule_references:
     ignorable_authors: []
     ignorable_urls: []
     ignorable_emails: []
-    text: apache-2.0
+    text: Apache-2.0
 files:
   - path: package-and-licenses
     type: directory

tests/formattedcode/data/yaml/package-and-licenses-expected.yaml

@@ -4,23 +4,23 @@ declared_license_expression_spdx: LGPL-2.1-or-later AND GPL-2.0-only
 other_license_expression:
 other_license_expression_spdx:
 license_detections:
-  - license_expression: lgpl-2.1-plus AND gpl-2.0
-    license_expression_spdx: LGPL-2.1-or-later AND GPL-2.0-only
+  - license_expression: lgpl-2.1-plus
+    license_expression_spdx: LGPL-2.1-or-later
     matches:
       - license_expression: lgpl-2.1-plus
         spdx_license_expression: LGPL-2.1-or-later
         from_file:
         start_line: 16
-        end_line: 41
-        matcher: 3-seq
-        score: '89.63'
-        matched_length: 147
-        match_coverage: '89.63'
+        end_line: 31
+        matcher: 2-aho
+        score: '100.0'
+        matched_length: 141
+        match_coverage: '100.0'
         rule_relevance: 100
-        rule_identifier: lgpl-2.1-plus_312.RULE
-        rule_url: https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/lgpl-2.1-plus_312.RULE
+        rule_identifier: lgpl-2.1-plus_298.RULE
+        rule_url: https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/lgpl-2.1-plus_298.RULE
         matched_text: |
-          library is free software; you can redistribute it and/or
+          This library is free software; you can redistribute it and/or
               modify it under the terms of the GNU Lesser General Public
               License as published by the Free Software Foundation; either
               version 2.1 of the License, or (at your option) any later version.
@@ -36,16 +36,10 @@ license_detections:
 
           On Debian GNU/Linux systems, the complete text of the Lesser GNU General
           Public License can be found in `/usr/share/common-licenses/LGPL'.
-
-           This package is maintained by Manoj Srivastava <[email protected]>.
-
-           The Debian specific changes are © 2005-2008, Manoj Srivastava
-           <[email protected]>, and distributed under the terms of the GNU
-           General Public License, version 2.
-
-
-          On Debian GNU/Linux systems, the complete text of the GNU General
-          Public License can be found in `/usr/share/common-licenses/
+    identifier: lgpl_2_1_plus-8a5e2990-4794-788b-8d4c-f58a9743e092
+  - license_expression: gpl-2.0
+    license_expression_spdx: GPL-2.0-only
+    matches:
       - license_expression: gpl-2.0
         spdx_license_expression: GPL-2.0-only
         from_file:
@@ -70,7 +64,7 @@ license_detections:
               <URL:http://www.gnu.org/copyleft/gpl.html>.  You may also obtain
               it by writing to the Free Software Foundation, Inc., 51 Franklin
               St, Fifth Floor, Boston, MA 02110-1301 USA
-    identifier: lgpl_2_1_plus_and_gpl_2_0-b692f769-146f-b44e-952a-1c81d2c7cf5e
+    identifier: gpl_2_0-e6238f93-b011-923d-bb0a-6fe716f20b50
 other_license_detections: []
 copyright: |
   Copyright (c) 2003, 2004 Stephen Smalley <[email protected]>

tests/packagedcode/data/debian/copyright/debian-slim-2021-04-07/usr/share/doc/libsepol1/copyright-detailed.expected.yml

@@ -69,14 +69,14 @@ license_detections:
         spdx_license_expression: GPL-3.0-or-later
         from_file:
         start_line: 50
-        end_line: 70
-        matcher: 3-seq
-        score: '85.91'
+        end_line: 66
+        matcher: 2-aho
+        score: '100.0'
         matched_length: 128
-        match_coverage: '85.91'
+        match_coverage: '100.0'
         rule_relevance: 100
-        rule_identifier: gpl-3.0-plus_417.RULE
-        rule_url: https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/gpl-3.0-plus_417.RULE
+        rule_identifier: gpl-3.0-plus_288.RULE
+        rule_url: https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/gpl-3.0-plus_288.RULE
         matched_text: |
           This program is free software: you can redistribute it and/or modify
            * it under the terms of the GNU General Public License as published by
@@ -95,10 +95,6 @@ license_detections:
 
           On Debian GNU/Linux systems, the complete text of the GNU General Public
           License version 3 can be found in /usr/share/common-licenses/GPL-3.
-
-
-          The documentation is distributed under the terms of the GNU Free
-          Documentation License (
       - license_expression: gfdl-1.3
         spdx_license_expression: GFDL-1.3-only
         from_file:
@@ -137,7 +133,7 @@ license_detections:
 
           On Debian systems a copy of the complete text of the GNU FDL 1.3
           can be found in /usr/share/common-licenses/GFDL-1.3.
-    identifier: gpl_3_0_plus_and_gfdl_1_3_and_gfdl_1_3_plus-20aaaa92-d02e-2c2d-0615-d69acc5b00d5
+    identifier: gpl_3_0_plus_and_gfdl_1_3_and_gfdl_1_3_plus-ca4df800-141f-5c15-72b1-19b8df01217e
 other_license_detections: []
 copyright: |
   Copyright (c) 2000-2020 Free Software Foundation, Inc.

tests/packagedcode/data/debian/copyright/debian-slim-2021-04-07/usr/share/doc/libtasn1-6/copyright-detailed.expected.yml

@@ -60,16 +60,16 @@
             "start_line": 1,
             "end_line": 1,
             "matcher": "1-hash",
-            "score": 90.0,
+            "score": 100.0,
             "matched_length": 3,
             "match_coverage": 100.0,
-            "rule_relevance": 90,
+            "rule_relevance": 100,
             "rule_identifier": "bsd-new_708.RULE",
             "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/bsd-new_708.RULE",
             "matched_text": "revised BSD license"
           }
         ],
-        "identifier": "bsd_new-9d4149ba-60ed-9e49-34a8-20576feb3295"
+        "identifier": "bsd_new-98c737b4-e505-9919-75a7-94b39533f450"
       },
       {
         "license_expression": "bsd-new",

tests/packagedcode/data/pypi/setup.py/boolean2_py_setup.py-expected.json

@@ -71,16 +71,16 @@
             "start_line": 1,
             "end_line": 1,
             "matcher": "2-aho",
-            "score": 95.0,
+            "score": 100.0,
             "matched_length": 3,
             "match_coverage": 100.0,
-            "rule_relevance": 95,
+            "rule_relevance": 100,
             "rule_identifier": "spdx_license_id_cc0-1.0_for_cc0-1.0.RULE",
             "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/spdx_license_id_cc0-1.0_for_cc0-1.0.RULE",
             "matched_text": "Apache-2.0 with ScanCode acknowledgment and CC0-1.0 and others"
           }
         ],
-        "identifier": "apache_2_0_and_scancode_acknowledgment__and_cc0_1_0-d8559b0f-151a-7966-fe46-c91dd29c027e"
+        "identifier": "apache_2_0_and_scancode_acknowledgment__and_cc0_1_0-15c44782-c134-8767-6ef8-10e8ab046877"
       },
       {
         "license_expression": "apache-2.0",