Merge pull request #3662 from nexB/misc-updates

Add Misc updates

Author: AyanSinhaMahapatra

src/cluecode/copyrights.py

@@ -2517,7 +2517,8 @@ def build_detection_from_node(
     COMPANY: {<COMPANY> <MAINT>} #19603
 
 
-#######################################
+#################################    #COPYRIGHT: {<COPY> <COPY> <MIT>}        #1802
+######
 # VARIOUS FORMS OF COPYRIGHT
 #######################################
 
@@ -2572,6 +2573,8 @@ def build_detection_from_node(
 
     COPYRIGHT: {<COPY> <COPY> <COMP>+}        #1690
 
+    COPYRIGHT: {<COPY> <COPY> <MIT>}        #1802
+
     COPYRIGHT: {<COPY> <COPY> <NN>+ <COMPANY|NAME|NAME-EMAIL>+}        #1710
 
     COPYRIGHT: {<COPY>  <COPY>  <NN>  <NNP>  <NN>  <COMPANY>}        #1711
@@ -4125,8 +4128,10 @@ def prepare_text_line(line, dedeb=True, to_ascii=True):
 
         # normalize (possibly repeated) quotes to unique single quote '
         # backticks ` and "
-        .replace('`', u"'")
-        .replace('"', u"'")
+        .replace('`', "'")
+        .replace('"', "'")
+        # see https://github.com/nexB/scancode-toolkit/issues/3667
+        .replace('§', " ")
     )
 
     if TRACE_TOK:

src/packagedcode/gemfile_lock.py

@@ -340,8 +340,8 @@ def get_option(s):
     '%(NAME_VERSION)s'
     '$' % locals()).match
 
-PLATS = re.compile('^  (?P<platform>.*)$').match
-BUNDLED_WITH = re.compile('^\s+(?P<version>(?:\d+.)+\d+)\s*$').match
+PLATS = re.compile(r'^  (?P<platform>.*)$').match
+BUNDLED_WITH = re.compile(r'^\s+(?P<version>(?:\d+.)+\d+)\s*$').match
 
 
 class GemfileLockParser:

src/packagedcode/recognize.py

@@ -16,7 +16,7 @@
 from packagedcode import ALL_DATAFILE_HANDLERS
 from packagedcode import models
 
-TRACE = False or os.environ.get('SCANCODE_DEBUG_PACKAGE_API', False)
+TRACE = os.environ.get('SCANCODE_DEBUG_PACKAGE_API', False)
 
 
 def logger_debug(*args):
@@ -62,7 +62,6 @@ def recognize_package_data(
         datafile_handlers = APPLICATION_PACKAGE_DATAFILE_HANDLERS
     elif system:
         datafile_handlers = SYSTEM_PACKAGE_DATAFILE_HANDLERS
-
     return list(_parse(location, datafile_handlers=datafile_handlers))
 
 
@@ -78,6 +77,9 @@ def _parse(
     """
 
     for handler in datafile_handlers:
+        if TRACE:
+            logger_debug(f'_parse:.is_datafile: {handler}')
+
         if not handler.is_datafile(location):
             continue
 

src/packagedcode/rpm.py

@@ -233,6 +233,9 @@ class RpmInstalledNdbDatabaseHandler(BaseRpmInstalledDatabaseHandler):
 # TODO: add dependencies!!!
 class RpmInstalledSqliteDatabaseHandler(BaseRpmInstalledDatabaseHandler):
     # used by newer RHEL/CentOS/Fedora/CoreOS
+    # Filetype: SQLite 3.x database, ...
+    # Mimetype: application/vnd.sqlite3
+
     datasource_id = 'rpm_installed_database_sqlite'
     path_patterns = ('*rpm/rpmdb.sqlite',)
     default_package_type = 'rpm'

src/summarycode/generated.py

@@ -176,6 +176,17 @@ def generated_scanner(location, **kwargs):
     # protoc
     'generated by the protocol buffer compiler',
     'generated by the protocol buffer compiler. do not edit!',
+    'generated by the protocol buffer compiler.  do not edit!',
+    'code generated by protoc-gen-go. do not edit.',
+    'code generated by generate-types. do not edit.',
+    'code generated by generate-protos. do not edit.',
+    'do not edit -- your changes will be discarded when the file is',  # next line:  * regenerated.
+    'do not edit -- generated code',
+    'generated code -- do not edit!',
+    'autogenerated by method_dump.sh. do not edit by hand.',
+    'this file is generated from the .proto files for the well-known',  # next line: types. Do not edit!
+    'this file was generated by upbc (the upb compiler) from the input',
+    'do not edit -- your changes will be discarded when the file is',  # next line: regenerated
 
     # autotools
     'makefile.in generated by automake',
@@ -191,6 +202,7 @@ def generated_scanner(location, **kwargs):
     'generated by:javacc: do not edit this line',
     'generated by:jjtree: do not edit this line',
     'generated code (do not edit this line)',
+    
 ))
 
 

src/textcode/strings.py

@@ -180,33 +180,33 @@ def is_shared_object(s):
     return so(s)
 
 
+# TODO: implement me
+_posix = re.compile('^/[\\w_\\-].*$', re.IGNORECASE).match
 def is_posix_path(s):
     """
     Return True if s looks like a posix path.
     Example: /usr/lib/librt.so.1 or /usr/lib
     """
-    # TODO: implement me
-    posix = re.compile('^/[\\w_\\-].*$', re.IGNORECASE).match
-    posix(s)
-    return False
+    return _posix(s)
 
 
+# TODO: implement me
+_relative = re.compile('^(?:([^/]|\\.\\.)[\\w_\\-]+/.*$)', re.IGNORECASE).match
 def is_relative_path(s):
     """
     Return True if s looks like a relative posix path.
     Example: usr/lib/librt.so.1 or ../usr/lib
     """
-    relative = re.compile('^(?:([^/]|\\.\\.)[\\w_\\-]+/.*$)', re.IGNORECASE).match
-    return relative(s)
+    return bool(_relative(s))
 
 
+_winpath = re.compile('^[\\w_\\-]+\\.so\\.[0-9]+\\.*.[0-9]*$', re.IGNORECASE).match
 def is_win_path(s):
     """
     Return True if s looks like a win path.
     Example: c:\\usr\\lib\\librt.so.1.
     """
-    winpath = re.compile('^[\\w_\\-]+\\.so\\.[0-9]+\\.*.[0-9]*$', re.IGNORECASE).match
-    return winpath(s)
+    return _winpath(s)
 
 
 def is_c_source(s):

src/textcode/strings2.py

@@ -23,8 +23,8 @@
 import re
 
 ASCII_BYTE = (
-    " !\"#\$%&\'\(\)\*\+,-\./0123456789:;<=>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ"
-    "\[\]\^_`abcdefghijklmnopqrstuvwxyz\{\|\}\\\~\t"
+    r" !\"#\$%&\'\(\)\*\+,-\./0123456789:;<=>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+    r"\[\]\^_`abcdefghijklmnopqrstuvwxyz\{\|\}\\\~\t"
 )
 
 

tests/cluecode/data/copyrights/misco2/pom.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+     Copyright (C) 2016§ The Android Open Source Project
+-->

tests/cluecode/data/copyrights/misco2/pom.xml.yml

@@ -0,0 +1,4 @@
+what:
+  - copyrights
+copyrights:
+  - Copyright (c) 2016 The Android Open Source Project

tests/packagedcode/test_gemfile_lock.py

@@ -274,7 +274,7 @@ def test_Gem_as_nv_tree(self):
                 }
             }
         }
-        self.assertEqual(expected, a.as_nv_tree())
+        assert a.as_nv_tree() == expected
 
     def test_Gem_flatten(self):
         Gem = gemfile_lock.Gem
@@ -320,7 +320,6 @@ def test_Gem_as_nv_tree_with_no_deps(self):
         results = a.as_nv_tree()
         assert results == expected
 
-
     def test_Gem_to_dict(self):
         Gem = gemfile_lock.Gem
         a = Gem('a', '1')