Improve copyright detection

- Start detecting "is held by"
- Do not include some trailing junk

Signed-off-by: Philippe Ombredanne <[email protected]>

Author: pombredanne

src/cluecode/copyrights.py

@@ -279,18 +279,22 @@ def detect(self,
             'YR-RANGE', 'YR-AND', 'YR', 'YR-PLUS', 'BARE-YR',
             'EMAIL', 'URL',
             'HOLDER', 'AUTHOR',
+            'IS', 'HELD',
+            
         ])
 
         non_holder_labels_mini = frozenset([
             'COPY',
             'YR-RANGE', 'YR-AND', 'YR', 'YR-PLUS', 'BARE-YR',
             'HOLDER', 'AUTHOR',
+            'IS', 'HELD',
         ])
 
         non_authors_labels = frozenset([
             'COPY',
             'YR-RANGE', 'YR-AND', 'YR', 'YR-PLUS', 'BARE-YR',
             'HOLDER', 'AUTHOR',
+            'IS', 'HELD',
         ])
 
         # then walk the parse parse_tree, collecting copyrights, years and authors
@@ -703,6 +707,11 @@ def build_detection_from_node(
     (r'^[Rr]éservés[\.,]*$', 'RESERVED'),
     (r'^[Rr]eserves[\.,]*$', 'RESERVED'),
 
+    # used to detect "copyright is held by..." 
+    (r'^is$', 'IS'),
+    (r'^are$', 'IS'),
+    (r'^held$', 'HELD'),
+
     # TODO: in Dutch Alle rechten voorbehouden.
     # TODO: in Spanish Reservados todos los derechos
 
@@ -736,6 +745,9 @@ def build_detection_from_node(
     # JUNK proper
     ############################################################################
 
+    # all lower case with dashes "enforce-trailing-newline" at least 3 times
+    (r'^((\w+-){3,}\w+)$', 'JUNK'),
+ 
     # path with trailing year-like are NOT a year as in
     # Landroid/icu/impl/IDNA2003 : treat as JUNK
     (r'^[^\\/]+[\\/][^\\/]+[\\/].*$', 'JUNK'),
@@ -825,7 +837,6 @@ def build_detection_from_node(
     (r'^Idata$', 'JUNK'),
     (r'^[Cc]ontributed?$', 'JUNK'),
     (r'^[Ff]unctions?$', 'JUNK'),
-    (r'^[Nn]otices?$', 'JUNK'),
     (r'^[Mm]ust$', 'JUNK'),
     (r'^ISUPPER?$', 'JUNK'),
     (r'^ISLOWER$', 'JUNK'),
@@ -891,8 +902,6 @@ def build_detection_from_node(
 
     (r'^providing$', 'JUNK'),
     (r'^Execute$', 'JUNK'),
-    (r'^NOTICE[.,]*$', 'JUNK'),
-    (r'^[Nn]otice[.,]*$', 'JUNK'),
     (r'^passes$', 'JUNK'),
     (r'^Should$', 'JUNK'),
     (r'^[Ll]icensing\@?$', 'JUNK'),
@@ -964,7 +973,6 @@ def build_detection_from_node(
     (r'^Much$', 'JUNK'),
     (r'^remains?,?$', 'JUNK'),
     (r'^earlier$', 'JUNK'),
-    (r'^is$', 'JUNK'),
     (r'^[lL]aws?$', 'JUNK'),
     (r'^Insert$', 'JUNK'),
     (r'^url$', 'JUNK'),
@@ -986,7 +994,6 @@ def build_detection_from_node(
     (r'^interfaces?,?$', 'JUNK'),
     (r'^than$', 'JUNK'),
     (r'^whom$', 'JUNK'),
-    (r'^are$', 'JUNK'),
     (r'^However,?$', 'JUNK'),
     (r'^[Cc]ollectively$', 'JUNK'),
     (r'^following$', 'JUNK'),
@@ -1365,7 +1372,8 @@ def build_detection_from_node(
     (r'^Neither$', 'NN'),
     (r'^Norwegian$', 'NN'),
     (r'^Notes?$', 'NN'),
-    (r'^NOTICE', 'NN'),
+    (r'^NOTICE[\.\,]?$', 'NN'),
+    (r'^[Nn]otices?[\.,]?$', 'NN'),
     (r'^NOT$', 'NN'),
     (r'^NULL$', 'NN'),
     (r'^Objects?$', 'NN'),
@@ -2764,8 +2772,8 @@ def build_detection_from_node(
     # portions copyright
     COPYRIGHT: {<PORTIONS> <COPYRIGHT|COPYRIGHT2>}        #2610
 
-    #copyright notice (3dfx Interactive, Inc. 1999), (notice is JUNK)
-    COPYRIGHT: {<COPY> <JUNK> <COMPANY> <YR-RANGE>}       #2620
+    #copyright notice (3dfx Interactive, Inc. 1999),
+    COPYRIGHT: {<COPY> <NN> <COMPANY> <YR-RANGE>}       #2620
 
     # Copyright (C) <2013>, GENIVI Alliance, Inc.
     COPYRIGHT: {<COPYRIGHT2> <ANDCO>}       #2625
@@ -2977,7 +2985,7 @@ def build_detection_from_node(
     COPYRIGHT: {<COPY> <HOLDER> <NAME>}       #83000
 
     #holder is Tim Hudson ([email protected]).
-    COPYRIGHT: {<HOLDER> <JUNK> <NAME-EMAIL>}       #83001
+    COPYRIGHT: {<HOLDER> <IS> <NAME-EMAIL>}       #83001
 
     # Copyright lowRISC contributors.
     COPYRIGHT: {<COPY> <NN> <CONTRIBUTORS>}        #83002
@@ -2991,14 +2999,19 @@ def build_detection_from_node(
     # Copyright OProfile authors
     COPYRIGHT: {<COPY> <NN>?<NNP>+ <AUTHS>}         #83004
 
+#######################################
+# Copyright is held by ....
+#######################################
+    # Copyright is held by ....
+    COPYRIGHT: {<COPY> <IS> <HELD> <BY> <NNP|NAME|COMPANYNAME-EMAIL>+ }         #10989898
 
 
 #######################################
 # Authors
 #######################################
 
     # SPDX-FileContributor special case
-    AUTHOR: {<SPDX-CONTRIB> <CCOMPANY|NAME|NAME-EMAIL|NAME-YEAR|EMAIL> <COMPANY|NAME|NAME-EMAIL|NAME-YEAR|EMAIL|NN>? }        #264000
+    AUTHOR: {<SPDX-CONTRIB> <COMPANY|NAME|NAME-EMAIL|NAME-YEAR|EMAIL> <COMPANY|NAME|NAME-EMAIL|NAME-YEAR|EMAIL|NN>? }        #264000
 
     # developed by Project Mayo.
     AUTHOR: {<AUTH2>+ <BY> <COMPANY> <NNP>}        #2645-1
@@ -3635,11 +3648,11 @@ def strip_trailing_period(s):
 
     is_single_word = len(s.split()) == 1
 
-        # U.S.A., e.V., M.I.T. and similar
+    # U.S.A., e.V., M.I.T. and similar
     if s[-2].isupper() and not is_single_word:
         return s
 
-        # S.A., e.v., b.v. and other
+    # S.A., e.v., b.v. and other
     if s[-3] == '.':
         return s
 

tests/cluecode/data/copyright_fossology/testdata118_raw

@@ -334,7 +334,7 @@ For files in the libpng directory:
      Willem van Schaik</s>
 
   libpng versions 0.89, June 1996, through 0.96, May 1997, are
-  <s>Copyright (c) 1996, 1997 Andreas Dilger Distributed</s> according to the
+  <s>Copyright (c) 1996, 1997 Andreas Dilger</s> Distributed according to the
   same disclaimer and license as libpng-0.88, with the following
   individuals added to the list of Contributing <t>Authors:
 

tests/cluecode/data/copyright_fossology/testdata119_raw

@@ -334,7 +334,7 @@ For files in the libpng directory:
      Willem van Schaik</s>
 
   libpng versions 0.89, June 1996, through 0.96, May 1997, are
-  <s>Copyright (c) 1996, 1997 Andreas Dilger Distributed</s> according to the
+  <s>Copyright (c) 1996, 1997 Andreas Dilger</s> Distributed according to the
   same disclaimer and license as libpng-0.88, with the following
   individuals added to the list of Contributing <s>Authors:
 

tests/cluecode/data/copyright_fossology/testdata128_raw

@@ -334,7 +334,7 @@ For files in the libpng directory:
      Willem van Schaik</t>
 
   libpng versions 0.89, June 1996, through 0.96, May 1997, are
-  <s>Copyright (c) 1996, 1997 Andreas Dilger Distributed</s> according to the
+  <s>Copyright (c) 1996, 1997 Andreas Dilger</s> Distributed according to the
   same disclaimer and license as libpng-0.88, with the following
   individuals added to the list of Contributing Authors:<t>
 

tests/cluecode/data/copyright_fossology/testdata23_raw

@@ -229,8 +229,8 @@
 </div>
 <table xmlns:rev="http://www.cs.rpi.edu/~gregod/boost/tools/doc/revision" width="100%"><tr>
 <td align="left"></td>
-<td align="right"><div class="copyright-footer"><s>Copyright (c) 2003 - 2008 Christopher M. Kohlhoff
-        Distributed</s> <p>under the Boost Software License, Version 1.0. (See accompanying
+<td align="right"><div class="copyright-footer"><s>Copyright (c) 2003 - 2008 Christopher M. Kohlhoff</s>
+        Distributed <p>under the Boost Software License, Version 1.0. (See accompanying
         file LICENSE_1_0.txt or copy at <a href="http://www.boost.org/LICENSE_1_0.txt" target="_top">http://www.boost.org/LICENSE_1_0.txt</a>)
       </p>
 </div></td>

tests/cluecode/data/copyrights/with_trailing_words.js.yml

@@ -7,7 +7,6 @@ copyrights:
   - Copyright 2009-2015 Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors
   - Copyright 2009-2015 Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors
   - Copyright 2009-2015 Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors
-    Distributed
   - (c) Varun Malhotra 2013 Source Code https://github.com/softvar/json2html
   - Copyright 2015, Mycompany
   - Copyright 2015, Mycompany
@@ -16,18 +15,16 @@ holders:
   - The Dojo Foundation
   - Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors
   - Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors
-  - Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors Distributed
+  - Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors
   - Varun Malhotra Source Code
   - Mycompany
   - Mycompany
   - Mycompany
 holders_summary:
+  - value: Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors
+    count: 3
   - value: Mycompany
     count: 3
-  - value: Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors
-    count: 2
-  - value: Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors Distributed
-    count: 1
   - value: The Dojo Foundation
     count: 1
   - value: Varun Malhotra Source Code

tests/cluecode/data/generated/copyright_49.txt.yml

@@ -4,7 +4,5 @@ what:
   - authors
 copyrights:
   - Copyright (c) 2006, Industrial Light & Magic
-  - copyright held by others as indicated
 holders:
   - Industrial Light & Magic
-  - others as indicated

tests/cluecode/data/ics/chromium-chrome-browser-resources/about_credits.html.yml

@@ -60,7 +60,7 @@ copyrights:
   - Copyright (c) 2004, 2006-2009 Glenn Randers-Pehrson
   - Copyright (c) 2000-2002 Glenn Randers-Pehrson
   - Copyright (c) 1998, 1999 Glenn Randers-Pehrson
-  - Copyright (c) 1996, 1997 Andreas Dilger Distributed
+  - Copyright (c) 1996, 1997 Andreas Dilger
   - Copyright (c) 1995, 1996 Guy Eric Schalnat, Group 42, Inc.
   - Copyright (c) 2001-2006 Cisco Systems, Inc.
   - Copyright (c) 2010, Google Inc.
@@ -165,7 +165,7 @@ holders:
   - Glenn Randers-Pehrson
   - Glenn Randers-Pehrson
   - Glenn Randers-Pehrson
-  - Andreas Dilger Distributed
+  - Andreas Dilger
   - Guy Eric Schalnat, Group 42, Inc.
   - Cisco Systems, Inc.
   - Google Inc.
@@ -247,7 +247,7 @@ holders_summary:
     count: 1
   - value: Analog Devices Inc.
     count: 1
-  - value: Andreas Dilger Distributed
+  - value: Andreas Dilger
     count: 1
   - value: Andrew Tridgell
     count: 1

tests/cluecode/data/ics/chromium-chrome-common-extensions-docs-examples-extensions-oauth_contacts/NOTICE.yml

@@ -6,13 +6,13 @@ copyrights:
   - copyright unitedHeroes.net
   - Copyright (c) 2009, unitedHeroes.net
   - Copyright Paul Johnston 2000 - 2002. Other contributors Greg Holt, Andrew Kepert, Ydnar,
-    Lostinet Distributed
+    Lostinet
 holders:
   - unitedHeroes.net
   - unitedHeroes.net
-  - Paul Johnston - Other contributors Greg Holt, Andrew Kepert, Ydnar, Lostinet Distributed
+  - Paul Johnston - Other contributors Greg Holt, Andrew Kepert, Ydnar, Lostinet
 holders_summary:
   - value: unitedHeroes.net
     count: 2
-  - value: Paul Johnston - Other contributors Greg Holt, Andrew Kepert, Ydnar, Lostinet Distributed
+  - value: Paul Johnston - Other contributors Greg Holt, Andrew Kepert, Ydnar, Lostinet
     count: 1

tests/cluecode/data/ics/qemu-distrib-libpng-1.2.19/png.h.yml

@@ -9,7 +9,7 @@ copyrights:
   - Copyright (c) 2004, 2006-2007 Glenn Randers-Pehrson
   - Copyright (c) 2000-2002 Glenn Randers-Pehrson
   - Copyright (c) 1998, 1999, 2000 Glenn Randers-Pehrson
-  - Copyright (c) 1996, 1997 Andreas Dilger Distributed
+  - Copyright (c) 1996, 1997 Andreas Dilger
   - Copyright (c) 1995, 1996 Guy Eric Schalnat, Group 42, Inc.
 holders:
   - Glenn Randers-Pehrson
@@ -18,14 +18,12 @@ holders:
   - Glenn Randers-Pehrson
   - Glenn Randers-Pehrson
   - Glenn Randers-Pehrson
-  - Andreas Dilger Distributed
+  - Andreas Dilger
   - Guy Eric Schalnat, Group 42, Inc.
 holders_summary:
   - value: Glenn Randers-Pehrson
     count: 4
+  - value: Andreas Dilger
+    count: 2
   - value: Guy Eric Schalnat, Group 42, Inc.
     count: 2
-  - value: Andreas Dilger
-    count: 1
-  - value: Andreas Dilger Distributed
-    count: 1