Improve copyright detection more

Signed-off-by: Philippe Ombredanne <[email protected]>

Author: pombredanne

requirements.txt

@@ -54,7 +54,7 @@ ply==3.11
 publicsuffix2==2.20191221
 pyahocorasick==2.1.0
 pycparser==2.21
-pygmars==0.7.0
+pygmars==0.9.0
 Pygments==2.13.0
 pymaven-patch==0.3.2
 pyparsing==3.0.9

setup-mini.cfg

@@ -100,7 +100,7 @@ install_requires =
     plugincode >= 32.0.0
     publicsuffix2
     pyahocorasick >= 2.0.0
-    pygmars >= 0.7.0
+    pygmars >= 0.9.0
     pygments
     pymaven_patch >= 0.2.8
     requests >= 2.7.0
@@ -112,7 +112,7 @@ install_requires =
     xmltodict >= 0.11.0
     zipp >= 3.0.0; python_version < "3.9"
     typecode >= 30.0.1
-#    typecode[full] >= 30.0.0
+#    typecode[full] >= 30.0.1
 #    extractcode[full] >= 31.0.0
 
 
@@ -199,6 +199,7 @@ scancode_post_scan =
     filter-clues = cluecode.plugin_filter_clues:RedundantCluesFilter
     consolidate = summarycode.plugin_consolidate:Consolidator
     license-references = licensedcode.licenses_reference:LicenseReference
+    todo = summarycode.todo:AmbiguousDetectionsToDoPlugin
     classify = summarycode.classify_plugin:FileClassifier
 
 

setup.cfg

@@ -100,7 +100,7 @@ install_requires =
     plugincode >= 32.0.0
     publicsuffix2
     pyahocorasick >= 2.0.0
-    pygmars >= 0.7.0
+    pygmars >= 0.9.0
     pygments
     pymaven_patch >= 0.2.8
     requests >= 2.7.0
@@ -138,9 +138,9 @@ testing =
 
 docs =
     Sphinx == 5.1.0
-    sphinx-rtd-theme >= 0.5.0
-    doc8 >= 0.8.1
+    sphinx_rtd_theme >= 0.5.1
     sphinx-reredirects >= 0.1.2
+    doc8 >= 0.8.1
     sphinx-autobuild
     sphinx-rtd-dark-mode>=1.3.0
     sphinx-copybutton

src/cluecode/copyrights.py

@@ -869,6 +869,9 @@ def build_detection_from_node(
     (r'^Implementation-Vendor$', 'JUNK'),
     (r'^dnl$', 'JUNK'),
 
+    (r'^as$', 'NN'),
+    (r'^[Vv]isit$', 'JUNK'),
+
     (r'^rem$', 'JUNK'),
     (r'^REM$', 'JUNK'),
     (r'^Supports$', 'JUNK'),
@@ -995,6 +998,11 @@ def build_detection_from_node(
     (r'^Privacy$', 'JUNK'),
     (r'^within$', 'JUNK'),
 
+    (r'^official$', 'JUNK'),
+    (r'^duties$', 'JUNK'),
+    (r'^civil$', 'JUNK'),
+    (r'^servants?$', 'JUNK'),
+
     # various trailing words that are junk
     (r'^Copyleft$', 'JUNK'),
     (r'^LegalCopyright$', 'JUNK'),
@@ -1060,6 +1068,7 @@ def build_detection_from_node(
     # FIXME: may be lowercase instead?
     (r'^Title:?$', 'JUNK'),
     (r'^Debianized-By:?$', 'JUNK'),
+    (r'^[Dd]ebianized$', 'JUNK'),
     (r'^Upstream-Maintainer:?$', 'JUNK'),
     (r'^Content', 'JUNK'),
     (r'^Upstream-Author:?$', 'JUNK'),
@@ -1307,7 +1316,7 @@ def build_detection_from_node(
     (r'^DISCLAIMED', 'NN'),
     (r'^Docs?$', 'NN'),
     (r'^DOCUMENTATION', 'NN'),
-    (r'^Download', 'NN'),
+    (r'^Download', 'JUNK'),
     (r'^DOM$', 'NN'),
     (r'^Do$', 'NN'),
     (r'^DoubleClick$', 'NN'),
@@ -1341,6 +1350,8 @@ def build_detection_from_node(
     (r'^Except', 'NN'),
     (r'^When$', 'NN'),
     # (r'^Owner$', 'NN'),
+    (r'^Specifications?$', 'NN'),
+    (r'^Final$', 'NN'),
     (r'^Holds$', 'NN'),
     (r'^Image', 'NN'),
     (r'^Supplier', 'NN'),
@@ -1369,7 +1380,7 @@ def build_detection_from_node(
     (r'^GnuPG$', 'NN'),
     (r'^Government.', 'NNP'),
     (r'^OProfile$', 'NNP'),
-    (r'^Government', 'NN'),
+    (r'^Government$', 'COMP'),
     (r'^Grants?\.?,?$', 'NN'),
     (r'^Header', 'NN'),
     (r'^HylaFAX$', 'NN'),
@@ -1513,7 +1524,7 @@ def build_detection_from_node(
     (r'^Those$', 'NN'),
     (r'^Timer', 'NN'),
     (r'^TODO$', 'NN'),
-    (r'^Tool.?$', 'NN'),
+    (r'^Tools?.?$', 'NN'),
     (r'^Trademarks?$', 'NN'),
     (r'^True$', 'NN'),
     (r'^TRUE$', 'NN'),
@@ -1563,19 +1574,6 @@ def build_detection_from_node(
     (r'^AM$', 'NN'),
     (r'^PM$', 'NN'),
 
-    (r'^January$', 'NN'),
-    (r'^February$', 'NN'),
-    (r'^March$', 'NN'),
-    (r'^April$', 'NN'),
-    (r'^May$', 'NN'),
-    (r'^June$', 'NN'),
-    (r'^July$', 'NN'),
-    (r'^August$', 'NN'),
-    (r'^September$', 'NN'),
-    (r'^October$', 'NN'),
-    (r'^November$', 'NN'),
-    (r'^December$', 'NN'),
-
     (r'^Name[\.,]?$', 'NN'),
     (r'^Co-Author[\.,]?$', 'NN'),
     (r'^Author\'s$', 'NN'),
@@ -1584,10 +1582,12 @@ def build_detection_from_node(
     (r'^Convention[\.,]?$', 'NN'),
     (r'^Paris[\.,]?$', 'NN'),
 
-    # we do not include Jan and Jun that are common enough first names
-    (r'^(Feb|Mar|Apr|May|Jul|Aug|Sep|Oct|Nov|Dec)$', 'NN'),
-    (r'^(Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday)$', 'NN'),
-    (r'^(Mon|Tue|Wed|Thu|Fri|Sat|Sun)$', 'NN'),
+    (r'^([Jj]anuary|[Ff]ebruary|[Mm]arch|[Aa]pril|[Jj]uly|[Aa]ugust|[Ss]eptember|[Oo]ctober|[Nn]ovember|[Dd]ecember)$', 'NN'),
+    # we do not include May, Jan and Jun that are common enough first names
+    (r'^(Feb|Mar|Apr|Jul|Aug|Sep|Oct|Nov|Dec),?$', 'MONTH'),
+
+    (r'^([Mm]onday|[Tt]uesday|[Ww]ednesday|[Tt]hursday|[Ff]riday|[Ss]aturday|[Ss]unday),?$', 'DAY'),
+    (r'^(Mon|Tue|Wed|Thu|Fri|Sat|Sun|May),?$', 'NN'),
 
     # misc words that are not NNs
     # lowercase verbs ending in "ing"
@@ -1862,10 +1862,11 @@ def build_detection_from_node(
     ############################################################################
 
     # "authors" or "contributors" is interesting, and so a tag of its own
+    (r'^[Aa]uthors,$', 'AUTHDOT'),
     (r'^[Aa]uthor$', 'AUTH'),
     (r'^[Aa]uthor\.$', 'AUTHDOT'),
     (r'^[Aa]uthors?\.$', 'AUTHDOT'),
-    (r'^[Aa]uthors|author\'$', 'AUTHS'),
+    (r'^([Aa]uthors|author\')$', 'AUTHS'),
     (r'^[Aa]uthor\(s\)$', 'AUTHS'),
     (r'^[Aa]uthor\(s\)\.?$', 'AUTHDOT'),
     # as javadoc
@@ -1978,7 +1979,7 @@ def build_detection_from_node(
     ############################################################################
 
     # rare cases of trailing + signon years
-    (r'^20[0-1][0-9]\+$', 'YR-PLUS'),
+    (r'^20[0-3][0-9]\+$', 'YR-PLUS'),
 
     # year or year ranges
     # plain year with various leading and trailing punct
@@ -2007,6 +2008,9 @@ def build_detection_from_node(
 
     (r'^(' + _YEAR_DASH_PRESENT + ')+$', 'YR'),
 
+    # ISO dates as in 2024-12-09
+    (r'^' + _YEAR + '-(0?[1-9]|1[012])-(0?[1-9]|[12][0-9]|3[01])$', 'YR'),
+
     # 88, 93, 94, 95, 96: this is a pattern mostly used in FSF copyrights
     (r'^[8-9][0-9],$', 'YR'),
 
@@ -2176,12 +2180,7 @@ def build_detection_from_node(
 # End of line commenst are rules descriptions.
 # One rule per line.
 
-USE_MAIN_BRANCH = False or os.environ.get('SCANCODE_COPYRIGHT_USE_MAIN_BRANCH', False)
-
-if USE_MAIN_BRANCH:
-    from cluecode.copyrightorig import grammar as GRAMMAR
-else:
-    GRAMMAR = """
+GRAMMAR = """
 
 #######################################
 # YEARS
@@ -3111,7 +3110,7 @@ def build_detection_from_node(
     COPYRIGHT: {<COPYRIGHT>  <MAINT> }       #83020
 
     # copyright its authors
-    COPYRIGHT: {<COPY> <NN> <AUTHS>}       #83030
+    COPYRIGHT: {<COPY> <NN> <AUTHDOT>}       #83030
 
     # Copyright: 2004-2007 by Internet Systems Consortium, Inc. ("ISC")
     #            1995-2003 by Internet Software Consortium
@@ -3234,7 +3233,7 @@ def build_detection_from_node(
     COPYRIGHT: {<COPY> <BY>? <AUTHOR>+  <YR-RANGE>*}        #2800-1
 
     COPYRIGHT: {<AUTHOR> <COPYRIGHT2>}        #2820
-    COPYRIGHT: {<AUTHOR> <YR-RANGE>}        #2830
+
     # copyrighted by MIT
     COPYRIGHT: {<COPY> <BY> <MIT>} #2840
 
@@ -3556,11 +3555,14 @@ def is_junk_copyryright(s, patterns=COPYRIGHTS_JUNK_PATTERN_MATCHERS):
         'author\'',
         'authors,',
         'authorship',
+        'maintainer',
+        'co-maintainer',
         'or',
         'spdx-filecontributor',
         '</b>',
         'mailto:',
         "name'",
+        "a",
     ])
 ))
 

src/licensedcode/data/licenses/d-fsl-1.0-en.LICENSE

@@ -18,7 +18,7 @@ ignorable_copyrights:
 ignorable_holders:
     - Ministry of Science and Research, State of North-Rhine Westphalia
 ignorable_authors:
-    - Axel Metzger and Till Jaeger, Institut
+    - Axel Metzger and Till Jaeger, Institut fur Rechtsfragen der Freien
 ignorable_urls:
     - http://www.d-fsl.org/
     - http://www.fsf.org/licenses/gpl
@@ -462,4 +462,4 @@ This Program may be used by anyone in accordance
 with the terms of the German Free Software 
 License
 
-The License may be obtained under <http://www.d-fsl.org>."
+The License may be obtained under <http://www.d-fsl.org>."

src/licensedcode/data/licenses/ietf-trust.LICENSE

@@ -21,7 +21,6 @@ ignorable_holders:
     - IETF Trust
     - IETF Trust and the persons identified as the document authors
 ignorable_authors:
-    - IETF Trust
     - the IETF Trust
 ignorable_urls:
     - http://opensource.org/licenses/bsd-license.php
@@ -328,4 +327,4 @@ licensing administrator for the IRTF Document Stream and that these Legal Provis
 applied to documents submitted and published in the IRTF Document Stream following 
 December 28, 2009.  Section 4 of these Legal Provisions shall not apply to documents in the 
 IRTF Document Stream, and all references to Section 4 hereof shall be disregarded with respect 
-to documents in the IRTF Document Stream.
+to documents in the IRTF Document Stream.

src/licensedcode/data/licenses/minpack.LICENSE

@@ -17,7 +17,7 @@ ignorable_copyrights:
 ignorable_holders:
     - University of Chicago
 ignorable_authors:
-    - the University of Chicago
+    - the University of Chicago, as Operator of Argonne National Laboratory
 ---
 
 Minpack Copyright Notice (1999) University of Chicago.  All rights reserved
@@ -70,4 +70,4 @@ PROFITS OR LOSS OF DATA, FOR ANY REASON WHATSOEVER, WHETHER
 SUCH LIABILITY IS ASSERTED ON THE BASIS OF CONTRACT, TORT
 (INCLUDING NEGLIGENCE OR STRICT LIABILITY), OR OTHERWISE,
 EVEN IF ANY OF SAID PARTIES HAS BEEN WARNED OF THE
-POSSIBILITY OF SUCH LOSS OR DAMAGES.
+POSSIBILITY OF SUCH LOSS OR DAMAGES.

src/licensedcode/data/licenses/qpopper.LICENSE

@@ -15,7 +15,7 @@ ignorable_holders:
     - QUALCOMM Incorporated
     - The Regents of the University of California
 ignorable_authors:
-    - Austin Shelton
+    - Austin Shelton, Information Systems and Technology, University of California, Berkeley
 ---
 
 Qpopper(tm) is licensed by QUALCOMM Incorporated under the following
@@ -158,4 +158,4 @@ Qpopper(tm) is licensed by QUALCOMM Incorporated under the following
     OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE
     POPPER SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND THE
     UNIVERSITY OF CALIFORNIA HAS NO OBLIGATIONS TO PROVIDE
-    MAINTENANCE, SUPPORT, UPDATES, ENCHANCEMENTS, OR MODIFICATIONS.
+    MAINTENANCE, SUPPORT, UPDATES, ENCHANCEMENTS, OR MODIFICATIONS.

src/licensedcode/data/licenses/sparky.LICENSE

@@ -13,7 +13,7 @@ ignorable_copyrights:
 ignorable_holders:
     - The Regents of the University of California
 ignorable_authors:
-    - the UCSF Resource
+    - the UCSF Resource for Biocomputing, Visualization, and Informatics
 ---
 
 Copyright, License, and Disclaimer
@@ -48,4 +48,4 @@ OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
 BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
 OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
-EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

src/licensedcode/data/licenses/us-govt-public-domain.LICENSE

@@ -13,9 +13,13 @@ other_urls:
     - https://www.law.cornell.edu/uscode/text/17/105
     - https://en.wikipedia.org/wiki/Copyright_status_of_works_by_the_federal_government_of_the_United_States
     - https://en.wikipedia.org/wiki/Wikipedia:Public_domain#U.S._government_works
+ignorable_copyrights:
+  - copyright United States Government
+ignorable_holders:
+  - United States Government
 ---
 
 17 U.S. Code $ 105. Subject matter of copyright: United States Government works
 
 Copyright protection under this title is not available for any work of the
-United States Government.
+United States Government.