Improve copyright detection in markup

Signed-off-by: Philippe Ombredanne <[email protected]>

Author: pombredanne

src/cluecode/copyrights.py

@@ -107,7 +107,7 @@ def detect_copyrights(
     numbered_lines = numbered_text_lines(location, demarkup=demarkup)
     numbered_lines = list(numbered_lines)
 
-    if TRACE:
+    if TRACE or TRACE_TOK:
         logger_debug('detect_copyrights: numbered_lines')
         for nl in numbered_lines:
             logger_debug('  numbered_line:', repr(nl))
@@ -167,7 +167,7 @@ def detect_copyrights_from_lines(
 
     candidate_lines_groups = candidate_lines(numbered_lines)
 
-    if TRACE:
+    if TRACE or TRACE_TOK:
         candidate_lines_groups = list(candidate_lines_groups)
         logger_debug(
             f'detect_copyrights_from_lines: ALL groups of candidate '
@@ -245,7 +245,7 @@ def detect(self,
         if not numbered_lines:
             return
 
-        if TRACE:
+        if TRACE or TRACE_TOK:
             logger_debug(f'CopyrightDetector: numbered_lines: {numbered_lines}')
 
         tokens = list(get_tokens(numbered_lines))
@@ -633,10 +633,12 @@ def build_detection_from_node(
     # (c)opyright and (c)opyleft, we ignore case
     (r'^(?i:\(c\)opy(rights?|righted|left))$', 'COPY'),
 
-    # opyright and opyleft, we ignore case
+    # truncated opyright and opyleft, we ignore case
     (r'^(?i:opy(rights?|righted|left|lefted)[\.\,]?)$', 'COPY'),
     (r'^//opylefted$', 'COPY'),
     (r"^c'opylefted$", 'COPY'),
+    # typo in cppyright
+    (r'^[Cc]ppyright[\.\,]?$', 'COPY'),
 
     # with a trailing comma
     (r'^Copyright,$', 'COPY'),
@@ -782,6 +784,7 @@ def build_detection_from_node(
     (r'^[Pp]rocedures?$', 'JUNK'),
     (r'^You$', 'JUNK'),
     (r'^Everyone$', 'JUNK'),
+    (r'^[Ff]unded$', 'JUNK'),
     (r'^Unless$', 'JUNK'),
     (r'^rant$', 'JUNK'),
     (r'^Subject$', 'JUNK'),
@@ -1244,6 +1247,7 @@ def build_detection_from_node(
     (r'^Each$', 'NN'),
     (r'^Education$', 'NN'),
     (r'^Extended', 'NN'),
+    (r'^Every$', 'NN'),
     (r'^Digitized', 'NN'),
     (r'^END$', 'NN'),
     (r'^Entity$', 'NN'),
@@ -3490,10 +3494,14 @@ def remove_dupe_copyright_words(c):
     c = c.replace('SPDX-FileCopyrightText', 'Copyright')
     c = c.replace('SPDX-SnippetCopyrightText', 'Copyright')
     c = c.replace('Bundle-Copyright', 'Copyright')
-    # from .net assemblies
-    c = c.replace('AssemblyCopyright', 'Copyright')
-    c = c.replace('AppCopyright', 'Copyright')
-
+    c = (
+        # from .net assemblies
+        c.replace('AssemblyCopyright', 'Copyright')
+        .replace('AppCopyright', 'Copyright')
+        # typos
+        .replace('Cppyright', 'Copyright')
+        .replace('cppyright', 'Copyright')
+    )
     # various prefix to the word copyright seen in binaries
     # TODO use a regex instead
     c = c.replace('BCopyright', 'Copyright')
@@ -3858,6 +3866,11 @@ def candidate_lines(numbered_lines):
     # used as a state and line counter
     in_copyright = 0
 
+    if TRACE_TOK:
+        numbered_lines = list(numbered_lines)
+        logger_debug(
+            f'candidate_lines: numbered_lines: {numbered_lines!r}')
+
     # the previous line (chars only)
     previous_chars = None
     for numbered_line in numbered_lines:
@@ -4024,21 +4037,29 @@ def prepare_text_line(line, dedeb=True, to_ascii=True):
 
     If ``to_ascii`` convert the text to ASCII characters.
     """
+    if TRACE_TOK:
+        logger_debug('  prepare_text_line: initial: ' + repr(line))
+
     # remove some junk in man pages: \(co
     line = (line
         .replace('\\\\ co', ' ')
         .replace('\\ co', ' ')
         .replace('(co ', ' ')
     )
     line = remove_printf_format_codes(' ', line)
+    if TRACE_TOK:
+        logger_debug('  prepare_text_line: after remove_printf_format_codes: ' + repr(line))
+
 
     # less common comment line prefixes
     line = remove_comment_markers(' ', line)
+    if TRACE_TOK:
+        logger_debug('  prepare_text_line: after remove_comment_markers: ' + repr(line))
 
     line = remove_man_comment_markers(' ', line)
 
     if TRACE_TOK:
-        logger_debug('  get_tokens: WIP line1: ' + repr(line))
+        logger_debug('  prepare_text_line: after remove_man_comment_markers: ' + repr(line))
 
     line = (line
         # C and C++ style comment markers
@@ -4103,6 +4124,10 @@ def prepare_text_line(line, dedeb=True, to_ascii=True):
         .replace('`', u"'")
         .replace('"', u"'")
     )
+
+    if TRACE_TOK:
+        logger_debug('  prepare_text_line: after replacements: ' + repr(line))
+
     # keep only one quote
     line = fold_consecutive_quotes(u"'", line)
 

src/licensedcode/data/licenses/bsl-1.0.LICENSE

@@ -11,9 +11,9 @@ other_urls:
     - https://mariadb.com/bsl-faq-adopting
     - https://mariadb.com/products/mariadb-enterprise
 ignorable_copyrights:
-    - (c) 2016 MariaDB Corporation
+    - (c) 2016 MariaDB Corporation Ab
 ignorable_holders:
-    - MariaDB Corporation
+    - MariaDB Corporation Ab
 ignorable_urls:
     - https://mariadb.com/products/mariadb-enterprise
 ---

src/licensedcode/data/rules/other-permissive_341.RULE

@@ -3,9 +3,9 @@ license_expression: other-permissive
 is_license_text: yes
 notes: https://github.com/DataJuggler/DataTier.Net/blob/cdd3493436b4c081db19829e7db3bed9389bf4b5/DataTier.Net/ProjectTemplates/DataTier.Net6.DataTemplate/Working/templates/DataTier.Net6.ClassLibrary/License/License.txt
 ignorable_copyrights:
-    - Copyright (c) 2022 - Data Juggler. Do
+    - Copyright (c) 2022 - Data Juggler
 ignorable_holders:
-    - Data Juggler. Do
+    - Data Juggler
 ---
 
 DataJuggler Do What You Want License

src/textcode/markup.py

@@ -9,7 +9,6 @@
 #
 
 from collections import Counter
-import logging
 import os
 import re
 
@@ -20,7 +19,25 @@
 Extract text from HTML, XML and related angular markup-like files.
 """
 
-logger = logging.getLogger(__name__)
+# Tracing flags
+TRACE = False or os.environ.get('SCANCODE_DEBUG_TEXT_ANALYSIS', False)
+
+
+# Tracing flags
+def logger_debug(*args):
+    pass
+
+
+if TRACE:
+    import logging
+    import sys
+
+    logger = logging.getLogger(__name__)
+    logging.basicConfig(stream=sys.stdout)
+    logger.setLevel(logging.DEBUG)
+
+    def logger_debug(*args):
+        return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args))
 
 bin_dir = os.path.join(os.path.dirname(__file__), 'bin')
 
@@ -46,7 +63,7 @@ def is_markup(location):
 
     with open(location, 'rb') as f:
         start = as_unicode(f.read(1024))
-    
+
     return is_markup_text(start)
 
 
@@ -86,9 +103,14 @@ def demarkup(location):
     from textcode.analysis import unicode_text_lines
 
     for line in unicode_text_lines(location):
+        if TRACE:
+            logger_debug(f'demarkup: {line} : demarked: {demarkup_text(line)}')
         yield demarkup_text(line)
 
 
+get_tags_and_entities = re.compile(r'(</?[^\s></]+(?:>|\s)?|&[^\s&]+;|href|[\'"]?\/\>)', re.IGNORECASE).split
+
+
 def demarkup_text(text):
     """
     Return text lightly stripped from markup. The whitespaces are collapsed to
@@ -100,18 +122,22 @@ def demarkup_text(text):
     kept_tags = (
         'lic', 'copy', 'www', 'http', 'auth', 'contr', 'leg', 'inc', '@',
         '<s>', '</s>', '169', 'a9'
-        )
+    )
 
     # find start and closing tags or the first white space whichever comes first
-    # or entities
-    # this regex is such that ' '.join(tags.split(a))==a
+    # or entities. This regex is such that ' '.join(tags.split(a))==a
 
-    tags_ents = re.compile(r'(</?[^\s></]+(?:>|\s)?|&[^\s&]+;|href|[\'"]?\/\>)', re.IGNORECASE).split
+    tags_and_ents = get_tags_and_entities(text)
+    if TRACE:
+        logger_debug(f'demarkup_text: {text!r}')
+        logger_debug(f'demarkup_text: tags_and_ents: {tags_and_ents}')
 
     cleaned = []
-    for token in tags_ents(text):
-        if token.lower().startswith(('<', '&', 'href', '=')) and not any(k in token.lower() for k in kept_tags):
+    cleaned_append = cleaned.append
+    for token in tags_and_ents:
+        tlow = token.lower()
+        if tlow.startswith(('<', '&', 'href',)) and not any(k in tlow for k in kept_tags):
             continue
         else:
-            cleaned.append(token)
+            cleaned_append(token)
     return u' '.join(cleaned)

tests/cluecode/data/copyrights/babkin_txt.txt.yml

@@ -4,7 +4,7 @@ what:
   - holders_summary
 copyrights:
   - Copyright (c) North
-  - Copyright (c) South Copyright (c)
+  - Copyright (c) South
   - Copyright (c) 2001 by the TTF2PT1 project
   - Copyright (c) 2001 by Sergey Babkin
 holders:

tests/cluecode/data/copyrights/index.html.yml

@@ -6,7 +6,7 @@ copyrights:
   - Copyright (c) 2002-2004 James W. Newkirk, Michael C. Two, Alexei A. Vorontsov
   - Copyright (c) 2000-2002 Philip A. Craig
   - Copyright (c) 2003-2008, Terence Parr
-  - Copyright (c) 1994-2011 Lua.org
+  - Copyright (c) 1994-2011 Lua.org, PUC-Rio
   - Copyright (c) 2000,2001,2002,2003,2004,2006,2007 Keith Packard
   - Copyright (c) 2005 Patrick Lam
   - Copyright (c) 2009 Roozbeh Pournader
@@ -29,7 +29,7 @@ holders:
   - James W. Newkirk, Michael C. Two, Alexei A. Vorontsov
   - Philip A. Craig
   - Terence Parr
-  - Lua.org
+  - Lua.org, PUC-Rio
   - Keith Packard
   - Patrick Lam
   - Roozbeh Pournader

tests/cluecode/data/copyrights/index.txt.yml

@@ -6,7 +6,7 @@ copyrights:
   - Copyright (c) 2002-2004 James W. Newkirk, Michael C. Two, Alexei A. Vorontsov
   - Copyright (c) 2000-2002 Philip A. Craig
   - Copyright (c) 2003-2008, Terence Parr
-  - Copyright (c) 1994-2011 Lua.org
+  - Copyright (c) 1994-2011 Lua.org, PUC-Rio
   - Copyright (c) 2000,2001,2002,2003,2004,2006,2007 Keith Packard
   - Copyright (c) 2005 Patrick Lam
   - Copyright (c) 2009 Roozbeh Pournader
@@ -29,7 +29,7 @@ holders:
   - James W. Newkirk, Michael C. Two, Alexei A. Vorontsov
   - Philip A. Craig
   - Terence Parr
-  - Lua.org
+  - Lua.org, PUC-Rio
   - Keith Packard
   - Patrick Lam
   - Roozbeh Pournader

tests/cluecode/data/copyrights/openoffice_org_report_builder_bin.copyright.yml

@@ -18,8 +18,8 @@ copyrights:
   - Copyright (c) 2002 France Telecom
   - Copyright (c) 1990-2003 Sleepycat Software
   - Copyright (c) 1990, 1993, 1994, 1995 The Regents of the University of California
-  - Copyright (c) 2003 by Bitstream, Inc.
-  - Cppyright Copyright (c) 2006 by Tavmjong Bah
+  - Copyright (c) 2003 by Bitstream, Inc. Cppyright
+  - Copyright (c) 2006 by Tavmjong Bah
   - Copyright (c) 2007 Red Hat, Inc
   - Copyright (c) 2007 Red Hat, Inc.
   - Copyright 2000-2003 Beman Dawes

tests/cluecode/data/ics/bluetooth-glib-gio-xdgmime/xdgmimealias.h.yml

@@ -9,8 +9,7 @@ holders:
   - Red Hat, Inc.
   - Matthias Clasen
 holders_summary:
-  - value: Red Hat
-    count: 1
   - value: Matthias Clasen
     count: 1
-    
+  - value: Red Hat
+    count: 1

tests/cluecode/test_copyrights_basic.py

@@ -442,7 +442,7 @@ def test_copyright_lines_babkin_txt(self):
         test_file = self.get_test_loc('copyrights_basic/babkin_txt.txt')
         expected = [
             copyrights.CopyrightDetection('Copyright (c) North', 1, 1),
-            copyrights.CopyrightDetection('Copyright (c) South Copyright (c)', 2, 3),
+            copyrights.CopyrightDetection('Copyright (c) South', 2, 3),
             copyrights.CopyrightDetection('Copyright (c) 2001 by the TTF2PT1 project', 4, 4),
             copyrights.CopyrightDetection('Copyright (c) 2001 by Sergey Babkin', 5, 5),
         ]