Improve copyright detection

Signed-off-by: Philippe Ombredanne <[email protected]>

Author: pombredanne

src/cluecode/copyrights.py

@@ -215,8 +215,16 @@ class CopyrightDetector(object):
     """
 
     def __init__(self):
-        self.lexer = lex.Lexer(patterns)
-        self.parser = parse.Parser(grammar, trace=TRACE_DEEP, validate=VALIDATE)
+        """
+        Initialize this detector with a lexer and a parser.
+        """
+        self.lexer = lex.Lexer(matchers=PATTERNS)
+        self.parser = parse.Parser(
+            grammar=GRAMMAR,
+            loop=1,
+            trace=TRACE_DEEP,
+            validate=VALIDATE,
+        )
 
     def detect(self,
         numbered_lines,
@@ -591,7 +599,7 @@ def build_detection_from_node(
 _YEAR_THEN_YEAR_SHORT = fr'({_YEAR_OR_YEAR_YEAR_WITH_PUNCT}({_YEAR_SHORT_PUNCT})*)'
 _YEAR_DASH_PRESENT = _YEAR + r'[\-~]? ?[Pp]resent\.?,?'
 
-patterns = [
+PATTERNS = [
     ############################################################################
     # COPYRIGHT
     ############################################################################
@@ -877,6 +885,7 @@ def build_detection_from_node(
     (r'^DISCLAIMS?$', 'JUNK'),
     (r'^SPECIFICALLY$', 'JUNK'),
 
+    (r'^identifying', 'JUNK'),
     (r'^IDENTIFICATION$', 'JUNK'),
     (r'^WARRANTIE?S?$', 'JUNK'),
     (r'^WARRANTS?$', 'JUNK'),
@@ -1204,6 +1213,7 @@ def build_detection_from_node(
     # NN often used in conjunction with copyright
     (r'^[Ss]tatements?.?$', 'JUNK'),
     (r'^issues?.?$', 'JUNK'),
+    (r'^retain?.?$', 'JUNK'),
 
     ############################################################################
     # Nouns and proper Nouns
@@ -1267,6 +1277,7 @@ def build_detection_from_node(
     (r'^But$', 'NN'),
     (r'^Builders?\.?$', 'NN'),
     (r'^Cacute$', 'NN'),
+    (r'^CD$', 'JUNK'),
     (r'^Cell.$', 'NN'),
     (r'^Change\.?[lL]og$', 'NN'),
     (r'^CHANGElogger$', 'NN'),
@@ -1304,17 +1315,18 @@ def build_detection_from_node(
     (r'^Education$', 'NN'),
     (r'^Extended', 'NN'),
     (r'^Every$', 'NN'),
-    (r'^Exhibit$', 'NN'),
+    (r'^EXHIBIT$', 'JUNK'),
+    (r'^Exhibit$', 'JUNK'),
     (r'^Digitized', 'NN'),
     (r'^[Ds]istributed?.?$', 'NN'),
+    (r'^Distributions?', 'NN'),
     (r'^Multiply$', 'NN'),
     (r'^Convert$', 'NN'),
     (r'^Compute$', 'NN'),
     (r'^Case$', 'NN'),
     (r'^Hessian$', 'NN'),
     (r'^Include', 'NN'),
     (r'^Downstream', 'NN'),
-    (r'^Distributions?', 'NN'),
     (r'^Volumes?', 'NN'),
     (r'^Manuals?.?', 'NN'),
     (r'^Update.?', 'NN'),
@@ -1435,8 +1447,8 @@ def build_detection_from_node(
     (r'^Packaging$', 'NN'),
     (r'^Patent', 'NN'),
     (r'^Pentium$', 'NN'),
-    (r'^[Pp]ermission', 'NN'),
-    (r'^PERMISSIONS?', 'NN'),
+    (r'^[Pp]ermission', 'JUNK'),
+    (r'^PERMISSIONS?', 'JUNK'),
     (r'^PGP$', 'NN'),
     (r'^Phrase', 'NN'),
     (r'^Plugin', 'NN'),
@@ -1871,15 +1883,16 @@ def build_detection_from_node(
 
     # same for developed, etc...
     (r'^[Cc]oded$', 'AUTH2'),
-    (r'^[Rr]ecoded$', 'AUTH2'),
-    (r'^[Mm]odified$', 'AUTH2'),
-    (r'^[Cc]reated$', 'AUTH2'),
+    (r'^\(?[Rr]ecoded$', 'AUTH2'),
+    (r'^\(?[Mm]odified$', 'AUTH2'),
+    (r'^\(?[Cc]reated$', 'AUTH2'),
     # written is often mispelled
-    (r'^[Ww]ritt?e[dn]$', 'AUTH2'),
+    (r'^\(?[Ww]ritt?e[dn]$', 'AUTH2'),
     # rewritten is often mispelled
-    (r'^[Rr]ewritt?e[dn]$', 'AUTH2'),
-    (r'^[Mm]aintained$', 'AUTH2'),
-    (r'^[Dd]eveloped$', 'AUTH2'),
+    (r'^\(?[Rr]ewritt?e[dn]$', 'AUTH2'),
+    (r'^\(?[Mm]aintained$', 'AUTH2'),
+    (r'^\(?[Dd]eveloped$', 'AUTH2'),
+    (r'^\(?[Au]thored$', 'AUTH2'),
 
     # commiters is interesting, and so a tag of its own
     (r'[Cc]ommitters\.?,?', 'COMMIT'),
@@ -2162,7 +2175,13 @@ def build_detection_from_node(
 # Comments in the Grammar are lines that start with #
 # End of line commenst are rules descriptions.
 # One rule per line.
-grammar = """
+
+USE_MAIN_BRANCH = False or os.environ.get('SCANCODE_COPYRIGHT_USE_MAIN_BRANCH', False)
+
+if USE_MAIN_BRANCH:
+    from cluecode.copyrightorig import grammar as GRAMMAR
+else:
+    GRAMMAR = """
 
 #######################################
 # YEARS
@@ -2173,7 +2192,7 @@ def build_detection_from_node(
     YR-RANGE: {<CD|BARE-YR>? <YR> <BARE-YR>?}        #40
     YR-RANGE: {<YR>+ <BARE-YR>? }        #50
     YR-AND: {<CC>? <YR>+ <CC>+ <YR>}        #60
-    YR-RANGE: {<YR-AND>+}        #70|
+    YR-RANGE: {<YR-AND>+}        #70
     YR-RANGE: {<YR-RANGE>+ <DASH|TO> <YR-RANGE>+}        #71
     YR-RANGE: {<YR-RANGE>+ <DASH>?}        #72
 
@@ -2336,7 +2355,7 @@ def build_detection_from_node(
     NAME: {<NNP|PN>+ <NNP>+}        #351
 
     # Distributed Management Task Force
-    # NAME: {<NN> <NNP>{3}} #881111
+    NAME: {<NN> <NNP>{3}} #881111
 
     # @author <a href="mailto:[email protected]">Stephane Hillion</a>
     NAME: { <NN>? <NN>? <EMAIL> <NAME> } #351.1
@@ -2407,10 +2426,10 @@ def build_detection_from_node(
     NAME-YEAR: {<YR-RANGE> <NNP>+ <CAPS>?} #5612
 
     #Academy of Motion Picture Arts and Sciences
-    NAME: {<NAME> <CC> <NNP>} # 561
+    NAME: {<NAME> <CC> <NNP>} #561
 
     # Adam Weinberger and the GNOME Foundation
-    NAME: {<CC> <NN> <COMPANY>} # 565
+    ANDCO: {<CC> <NN> <COMPANY>} #565
 
     # (c) 1991-1992, Thomas G. Lane , Part of the Independent JPEG Group's
     NAME: {<PORTIONS> <OF> <NN> <NAME>+} #566
@@ -2497,6 +2516,8 @@ def build_detection_from_node(
     # <s>Timothy Terriberry</s>, <s>CSIRO</s>, and other contributors
     ANDCO: {<CC> <CAPS|COMPANY|NAME|NAME-EMAIL|NAME-YEAR>+}          #960
 
+    COMPANY: {<COMPANY|NAME|NAME-EMAIL|NAME-YEAR> <ANDCO>+}     #970
+
     # Copyright © 1998-2009 Bill Spitzak ([email protected] ) and others,
     COMPANY: {<COMPANY|NAME|NAME-EMAIL|NAME-YEAR> <PARENS>? <ANDCO>+}     #970
 
@@ -2596,7 +2617,7 @@ def build_detection_from_node(
     # Copyright 2015 The Error Prone Authors.
     NAME: {<NN> <NAME> <CONTRIBUTORS|AUTHS>}        #196023
 
-    # Copyright (C) <s>Suresh P <[email protected]></s> 
+    # Copyright (C) <s>Suresh P <[email protected]></s>
     NAME: {<NNP> <PN> <EMAIL>} #19601.1
 
     # Copyright or Copr. Mines Paristech, France - Mark NOBLE, Alexandrine GESRET
@@ -2611,9 +2632,14 @@ def build_detection_from_node(
     #   Copyright (C) 1998-2001 VideoLAN ( Johan Bilien <[email protected]> and Gildas Bazin <[email protected]> )
     NAME: {<PARENS> <NAME> <PARENS>} #19653
 
+    #  by the Initial Developer
+    INITIALDEV:  {<BY>?  <NN>  <NN>  <MAINT>} #19663
+
+    # UNIVERTSITY OF CHICAGO
+    NAME: {<UNI>  <OF>  <CAPS>} #19673
 
-#################################    #COPYRIGHT: {<COPY> <COPY> <MIT>}        #1802
-######
+
+#######################################
 # VARIOUS FORMS OF COPYRIGHT
 #######################################
 
@@ -2646,12 +2672,13 @@ def build_detection_from_node(
     # Bart Hanssens from FedICT
     COPYRIGHT: {<COPY>+ <NAME-YEAR> <NN> <CAPS> <NN> <OF> <COMPANY> <NAME>}  #83005
 
+    # Gracenote, Inc., copyright © 2000-2008 Gracenote.
+    # Gracenote Software, copyright © 2000-2008 Gracenote.
+    # COPYRIGHT: {<COMPANY> <COPY>{1,2} <NAME-YEAR>}        #157999.12
+
     # Copyright (c) Ian F. Darwin 1986, 1987, 1989, 1990, 1991, 1992, 1994, 1995.
     COPYRIGHT: {<COPY>+ <NAME|NAME-EMAIL|NAME-YEAR>+ <YR-RANGE>*}        #157999
 
-    # portions copyright The Internet Society, Tom Tromey and Red Hat, Inc.
-    COPYRIGHT: {<PORTIONS>  <COPY>  <NN>  <NAME>}        #157998
-
     COPYRIGHT: {<COPY>+ <CAPS|NNP>+ <CC> <NN> <COPY> <YR-RANGE>?}        #1590
 
     # // (c) (C) → ©
@@ -2740,8 +2767,8 @@ def build_detection_from_node(
     # (c) Copyright 1985-1999 SOME TECHNOLOGY SYSTEMS
     COPYRIGHT2: {<COPY> <COPY> <YR-RANGE> <CAPS> <CAPS> <CAPS>? <CAPS>?} #2271
 
-    # Minpack Copyright Notice (1999) University of Chicago
-    COPYRIGHT: {<COPY>  <NOTICE>  <NAME-YEAR>}  #2273.1
+    # Copyright Notice (1999) University of Chicago. All rights reserved
+    COPYRIGHT: {<COPY>  <NOTICE>  <NAME-YEAR>  <ALLRIGHTRESERVED>? } #2271.1
 
     # NAME-COPY is a name with a trailing copyright
     # Daisy (c) 1998
@@ -2789,7 +2816,6 @@ def build_detection_from_node(
     # Copyright (C) 2006 XStream committers.
     # Copyright (c) 2019-2021, Open source contributors.
     # Copyright 2007 ZXing authors
-    # Copyright (c) 2002 the Initial Developer
     # Copyright (c) 2024 bgme <[email protected]>.
     COPYRIGHT: {<COPY>+  <YR-RANGE>  <NN>+ <CONTRIBUTORS|COMMIT|AUTHS|MAINT>? <EMAIL>? <ALLRIGHTRESERVED>?} #22793.3
 
@@ -3081,9 +3107,6 @@ def build_detection_from_node(
     # Copyright OProfile authors
     COPYRIGHT: {<COPY> <NN>?<NNP>+ <AUTHS>}         #83004
 
-    # (C) Distributed Management Task Force (Distributed is an NN)
-    # COPYRIGHT: {<COPY> <NN> <NAME>}         #83010
-
     # Copyright (c) 2014 The Rust Project Developers
     COPYRIGHT: {<COPYRIGHT>  <MAINT> }       #83020
 
@@ -3092,15 +3115,45 @@ def build_detection_from_node(
 
     # Copyright: 2004-2007 by Internet Systems Consortium, Inc. ("ISC")
     #            1995-2003 by Internet Software Consortium
-    COPYRIGHT: {<YR-RANGE>  <BY>  <COMPANY> } #1615
+    COPYRIGHT: {<COPYRIGHT> <NN> <YR-RANGE>  <BY>  <COMPANY> } #1615
+
+    # Russ Dill <[email protected]> 2001-2003
+    # Rewrited by Vladimir Oleynik <[email protected]> (C) 2003
+    COPYRIGHT: {<NAME-EMAIL>  <YR-RANGE>  <AUTH2>  <BY>  <NAME-EMAIL>  <COPY>  <YR-RANGE>} #22793.5
+
+    # portions copyright The Internet Society, Tom Tromey and Red Hat, Inc.
+    COPYRIGHT: {<PORTIONS>  <COPY>  <NN>  <NAME>}        #157998
+
+    # Minpack Copyright Notice (1999) University of Chicago
+    COPYRIGHT: {<COPY>  <NOTICE>  <NAME-YEAR>}  #2273.1
+
+    # Portions created by the Initial Developer are Copyright (C)
+    # the Initial Developer. All Rights Reserved.
+    COPYRIGHT: {<PORTIONS>  <AUTH2>  <INITIALDEV>  <IS>  <COPY|COPYRIGHT2>+  <YR-RANGE>? <INITIALDEV>} #2609.1
+
+    # Portions created by the Initial Developer are Copyright (C) 
+    # the Initial Developer. All Rights Reserved.
+    # and
+    # Portions created by the Initial Developer are Copyright (C) 2002
+    # the Initial Developer. All Rights Reserved.
+    COPYRIGHT: {<COPYRIGHT|COPYRIGHT2>  <INITIALDEV>  <ALLRIGHTRESERVED>?}  #2609.2
+
+    # Copyright (C) the Initial Developer.
+    COPYRIGHT: {<COPY>+  <INITIALDEV>} #35012
+
+    # (C) Distributed Management Task Force (Distributed is an NN)
+    # COPYRIGHT: {<COPY> <NN>? <NAME>}         #83010
+
+    # Gracenote, Inc., copyright © 2000-2008 Gracenote.
+    # Gracenote Software, copyright © 2000-2008 Gracenote.
+    COPYRIGHT: {<COMPANY> <COPY>{1,2} <NAME-YEAR>}        #157999.12
 
 #######################################
 # Copyright is held by ....
 #######################################
     # Copyright is held by ....
     COPYRIGHT: {<COPY> <IS> <HELD> <BY> <NNP|NAME|COMPANYNAME-EMAIL>+ }         #10989898
 
-
 #######################################
 # Authors
 #######################################
@@ -3120,6 +3173,9 @@ def build_detection_from_node(
     # @author [email protected] (Anatol Pomazau)
     AUTHOR: {<AUTH|CONTRIBUTORS|AUTHS>+ <NN>? <COMPANY|NAME|YR-RANGE>* <BY>? <EMAIL>+ <NAME>?}        #2650
 
+    # developed by the National Center for Supercomputing Applications at the University of Illinois at Urbana-Champaign
+    AUTHOR: {<AUTH|CONTRIBUTORS|AUTHS>+ <NN>? <COMPANY|NAME|NAME-EMAIL|NAME-YEAR>+ <NN>? <COMPANY|NAME|NAME-EMAIL|NAME-YEAR>+ <YR-RANGE>*}       #2660
+
     AUTHOR: {<AUTH|CONTRIBUTORS|AUTHS>+ <NN>? <COMPANY|NAME|NAME-EMAIL|NAME-YEAR>+ <YR-RANGE>*}       #2660
 
     # developed by the Center for Information
@@ -3145,7 +3201,7 @@ def build_detection_from_node(
     AUTHOR: {<AUTH|AUTHS|AUTH2> <BY>? <NNP> <CC> <PN>} #2761
 
     # developed by the National Center for Supercomputing Applications at the University of Illinois at Urbana-Champaign
-    AUTHOR: {<AUTHOR> <NN> <NAME> <NAME>} #2762
+    AUTHOR: {<AUTHOR> <NN> <NAME|COMPANY>+ } #2762
 
     # created by Axel Metzger and Till Jaeger, Institut fur Rechtsfragen der Freien und Open Source Software
     AUTHOR: {<AUTH2> <CC> <AUTHOR> <NN> <NAME> <NN> <NN> <NNP>} #2645-4
@@ -3194,7 +3250,8 @@ def build_detection_from_node(
     # Copyright (c) 2015 Jon Schlinkert, contributors.
     COPYRIGHT: { <COPYRIGHT>  <CONTRIBUTORS>} #420121
 
-
+    # J. Schoenwaelder, Copyright (c) 1999
+    # COPYRIGHT: {<NAME> <COPYRIGHT>} #22793.7
 
 #######################################
 # Last resort catch all ending with ALLRIGHTRESERVED
@@ -3246,6 +3303,19 @@ def refine_copyright(c):
     return c.strip()
 
 
+def remove_dupe_holder(h):
+    """
+    Remove duplicated holders
+    """
+    dupes_holders = {
+        "the Initial Developer the Initial Developer": "the Initial Developer",
+    }
+    for src, tgt in dupes_holders.items():
+        if src in h:
+            h = h.replace(src, tgt)
+    return h
+
+
 def refine_holder(h):
     """
     Refine a detected holder.
@@ -3278,6 +3348,7 @@ def refine_holder(h):
     h = h.strip('+- ')
     h = strip_trailing_period(h)
     h = h.strip('+- ')
+    h = remove_dupe_holder(h)
     h = ' '.join(h.split())
     if h and h.lower() not in HOLDERS_JUNK:
         return h
@@ -3694,7 +3765,9 @@ def remove_some_extra_words_and_punct(c):
     c = c.replace(".net'", ".net")
     c = c.replace("mailto:", "")
     c = c.replace("@see", "")
-    return c
+    if c.endswith('as represented by'):
+        c, _, _ = c.partition('as represented by')
+    return c.strip()
 
 
 def strip_prefixes(s, prefixes=()):

src/licensedcode/data/licenses/cups.LICENSE

@@ -16,7 +16,6 @@ ignorable_holders:
     - Jelmer Vernooij
 ignorable_authors:
     - Apple Inc.
-    - permission from Apple Inc.
 ignorable_urls:
     - http://www.cups.org/
 ---
@@ -74,4 +73,4 @@ Altered source versions must be plainly marked as such, and must not be misrepre
 This notice may not be removed or altered from any source distribution.
 Trademarks
 
-CUPS and the CUPS logo (the "CUPS Marks") are trademarks of Apple Inc. Apple grants you a non-exclusive and non-transferable right to use the CUPS Marks in any direct port or binary distribution incorporating CUPS software and in any promotional material therefor. You agree that your products will meet the highest levels of quality and integrity for similar goods, not be unlawful, and be developed, manufactured, and distributed in compliance with this license. You will not interfere with Apple's rights in the CUPS Marks, and all use of the CUPS Marks shall inure to the benefit of Apple. This license does not apply to use of the CUPS Marks in a derivative products, which requires prior written permission from Apple Inc.
+CUPS and the CUPS logo (the "CUPS Marks") are trademarks of Apple Inc. Apple grants you a non-exclusive and non-transferable right to use the CUPS Marks in any direct port or binary distribution incorporating CUPS software and in any promotional material therefor. You agree that your products will meet the highest levels of quality and integrity for similar goods, not be unlawful, and be developed, manufactured, and distributed in compliance with this license. You will not interfere with Apple's rights in the CUPS Marks, and all use of the CUPS Marks shall inure to the benefit of Apple. This license does not apply to use of the CUPS Marks in a derivative products, which requires prior written permission from Apple Inc.

src/licensedcode/data/licenses/efsl-1.0.LICENSE

@@ -13,13 +13,11 @@ other_urls:
     - https://www.eclipse.org/legal/epl-2.0
     - https://www.opensource.org/licenses/EPL-2.0
 ignorable_copyrights:
-    - COPYRIGHT HOLDERS AND THE ECLIPSE FOUNDATION
     - Copyright (c) Eclipse Foundation
     - Copyright (c) Eclipse Foundation, Inc.
 ignorable_holders:
     - Eclipse Foundation
     - Eclipse Foundation, Inc.
-    - THE ECLIPSE FOUNDATION
 ---
 
 Eclipse Foundation Specification License - v1.0
@@ -73,4 +71,4 @@ DOCUMENT OR THE PERFORMANCE OR IMPLEMENTATION OF THE CONTENTS THEREOF.
 The name and trademarks of the copyright holders or the Eclipse Foundation may
 NOT be used in advertising or publicity pertaining to this document or its
 contents without specific, written prior permission. Title to copyright in this
-document will at all times remain with copyright holders.
+document will at all times remain with copyright holders.