Improve misc. copyright detections

Spotted in some common python libraries such as numpy and scipy

Signed-off-by: Philippe Ombredanne <[email protected]>

Author: pombredanne

src/cluecode/copyrights.py

@@ -1140,6 +1140,16 @@ def build_detection_from_node(
     (r'^[MmNn]odules?[,\.]?$', 'JUNK'),
     (r'^[Rr]eturned$', 'JUNK'),
 
+    # misc junk
+    (r'^False.?$', 'JUNK'),
+    (r'^True.?$', 'JUNK'),
+
+    (r'^imports?$', 'JUNK'),
+    (r'^[Ww]arnings?$', 'JUNK'),
+    (r'^[Ww]hether$', 'JUNK'),
+    (r'^[Bb]oth$', 'JUNK'),
+    (r'^[Cc]aller$', 'JUNK'),
+
     # tags
     (r'^E-?[Mm]ail:?$', 'JUNK'),
     (r'^URL:?$', 'JUNK'),
@@ -1252,6 +1262,12 @@ def build_detection_from_node(
     (r'^Every$', 'NN'),
     (r'^Digitized', 'NN'),
     (r'^[Ds]istributed?.?$', 'NN'),
+
+    (r'^Multiply$', 'NN'),
+    (r'^Convert$', 'NN'),
+    (r'^Compute$', 'NN'),
+    (r'^Case$', 'NN'),
+    
     (r'^END$', 'NN'),
     (r'^Entity$', 'NN'),
     (r'^Example', 'NN'),
@@ -3262,7 +3278,7 @@ def refine_names(s, prefixes):
     r'^copyright \(c\)$',
     r'^\(c\) by$',
 
-    r"\(c\) [A-Z][a-z] \(c\)",
+    r"\(c\) [a-z][a-z] \(c\)",
     r"^copyright holder or simply",
     r"^copyright notice\.",
     r"^copyright of uc berkeley's berkeley software distribution",
@@ -3326,6 +3342,8 @@ def refine_names(s, prefixes):
     r'^u\.s\. copyright act',
     r'^\(c\) Object c$',
     r'^copyright headers?',
+    r'Copyright \(c\) 2021 Dot',
+    r'^\(c\) \(c\) B$'
 ]
 
 # a collection of junk junk matcher callables
@@ -3551,7 +3569,7 @@ def remove_dupe_copyright_words(c):
 
 def remove_some_extra_words_and_punct(c):
     """
-    Remove misc junk includein some punctuations
+    Remove misc junk including some punctuations
     """
     c = c.replace('<p>', ' ')
     c = c.replace('<a href', ' ')
@@ -4135,6 +4153,8 @@ def prepare_text_line(line, dedeb=True, to_ascii=True):
         # backticks ` and "
         .replace('`', "'")
         .replace('"', "'")
+        # u nicode prefix in Python strings
+        .replace(" u'", " '")        
         # see https://github.com/nexB/scancode-toolkit/issues/3667
         .replace('§', " ")
     )

tests/cluecode/data/copyrights/copyrights-to-fix.txt

@@ -0,0 +1,68 @@
+copyright u'2017-2018, NumPy Developers
+
+
+
+Copyright (c) 2017
+(c) Convert Chebyshev
+(c) Multiply a Hermite
+(c) Multiply a Laguerre
+(c) Multiply a Legendre
+(c) Multiply a Chebyshev
+
+(c), True, True, False, False
+
+(c), False, False, False, True
+(c), False, False, True, False
+
+(c), (c)
+(c) . B' Both
+(c) B Whether
+
+
+
+Copyright 2001
+
+Copyright 2003
+
+Copyright 2008
+
+
+
+(c) 2003, C. Bond
+
+(c) Case 2 Caller
+
+(c) Compute Hessian H
+
+# Author: Jake Vanderplas  -- <[email protected]>
+# License: BSD 3 clause (C) 2011
+import warnings
+
+
+# Copyright (c) 2011, 2012
+# Authors: Pietro Berkes,
+
+
+
+# Author: Jake Vanderplas  -- <[email protected]>
+# License: BSD 3 clause (C) 2011
+
+import numpy as np
+
+
+
+# extra quote
+
+copyright = u"2010-2020, Benjamin Peterson"
+
+
+(c) KOKOKOKOKKuKyKxKxKxKxKxKxKxKxKxKxKxKxKxKxKxKxKxKxKxKxKxKxKxKxKxKxKxKxKxKyK KyKxKzKzKzKzKzKzKzKzKzKzKzKzKyKxKzKzKzKzKzKzKzKzKzKzKzKzKzKzKzKzKzKzKzKzKzKzKzKzK K K KzK K
+
+
+
+# junk in Pyparsing
+Don't get excited!
+I said "Don't get excited!"
+Copyright © 2021
+Dot ⟶ ˙
+

tests/cluecode/data/copyrights/copyrights-to-fix.txt.yml

@@ -0,0 +1,18 @@
+what:
+  - copyrights
+  - holders
+copyrights:
+  - copyright 2017-2018, NumPy Developers
+  - Copyright (c) 2017
+  - Copyright 2001
+  - Copyright 2003
+  - Copyright 2008
+  - (c) 2003, C. Bond
+  - (c) 2011
+  - Copyright (c) 2011, 2012
+  - (c) 2011
+  - copyright 2010-2020, Benjamin Peterson
+holders:
+  - NumPy Developers
+  - C. Bond
+  - Benjamin Peterson