Merge latest develop

Signed-off-by: Philippe Ombredanne <[email protected]>

Author: pombredanne

etc/scripts/buildrules.py

@@ -157,14 +157,67 @@ def find_rule_base_loc(license_expression):
         idx += 1
 
 
+def validate_license_rules(rules_data, licensing):
+    """
+    Checks all rules and return a list of errors
+    """
+    errors = []
+    for rule in rules_data:
+        is_negative = rule.data.get('is_negative')
+        is_false_positive = rule.data.get('is_false_positive')
+        fp_flags = [f for f in [is_negative, is_false_positive] if f]
+        if len(fp_flags) > 1:
+            msg = 'Invalid rule with mutually exclusive false positive/negative flags: {}'.format(rule)
+            errors.append(msg)
+        if is_negative:
+            continue
+
+        relevance = rule.data.get('relevance', 0) or 0
+        relevance = float(relevance)
+        if relevance < 0 or relevance > 100:
+            msg = 'Invalid rule relevance: {}'.format(rule)
+            errors.append(msg)
+
+        minimum_coverage = rule.data.get('minimum_coverage', 0) or 0
+        minimum_coverage = float(minimum_coverage)
+        if minimum_coverage < 0 or minimum_coverage > 100:
+            msg = 'Invalid rule minimum_coverage: {}'.format(rule)
+            errors.append(msg)
+
+        is_license_notice = rule.data.get('is_license_notice')
+        is_license_text = rule.data.get('is_license_text')
+        is_license_reference = rule.data.get('is_license_reference')
+        is_license_tag = rule.data.get('is_license_tag')
+
+        type_flags = [f for f in [is_license_notice, is_license_text, is_license_tag, is_license_reference] if f]
+        if len(type_flags) != 1:
+            msg = 'Invalid rule is_license_* flags. Only one allowed. At least one needed: {}'.format(rule)
+            errors.append(msg)
+
+        license_expression = rule.data.get('license_expression')
+        if not license_expression:
+            msg = 'Missing license_expression for rule: {}'.format(rule)
+            errors.append(msg)
+
+        try:
+            licensing.parse(license_expression, validate=True, simple=True)
+        except Exception as e:
+            msg = 'Invalid license_expression for rule: {}\n{}'.format(rule, str(e))
+            errors.append(msg)
+
+    return errors
+
+
 @click.command()
 @click.argument('licenses_file', type=click.Path(), metavar='FILE')
 @click.help_option('-h', '--help')
 def cli(licenses_file):
     """
-    Create rules from a structured text file
+    Create rules from a text file with delimited blocks of metadata and texts.
+
+    As an example a file would contains one of more blocks such as this:
 
-    For instance:
+\b
         ----------------------------------------
         license_expression: lgpl-2.1
         relevance: 100
@@ -182,6 +235,16 @@ def cli(licenses_file):
     licenses = cache.get_licenses_db()
     licensing = Licensing(licenses.values())
 
+    print()
+    errors = validate_license_rules(rules_data, licensing)
+    if errors:
+        print('Invalid rules: exiting....')
+        for error in errors:
+            print(error)
+            print()
+
+        raise Exception('Invalid rules: exiting....')
+
     print()
     for rule in rules_data:
         is_negative = rule.data.get('is_negative')
@@ -190,14 +253,12 @@ def cli(licenses_file):
         if existing and not is_negative:
             print('Skipping existing non-negative rule:', existing, 'with text:\n', rule.text[:50].strip(), '...')
             continue
-        
+
         if is_negative:
             base_name = 'not-a-license'
         else:
             license_expression = rule.data.get('license_expression')
-            if not license_expression:
-                raise Exception('Missing license_expression for text:', rule)
-            licensing.parse(license_expression, validate=True, simple=True)
+            license_expression = str(licensing.parse(license_expression, validate=True, simple=True))
             base_name = license_expression
             if is_false_positive:
                 base_name = 'false-positive_' + base_name

etc/scripts/synclic.py

@@ -411,7 +411,7 @@ def build_license(self, mapping, scancode_licenses):
                 'lgpl-2.0', 'lgpl-2.1', 'lgpl-3.0',
                 'agpl-1.0', 'agpl-2.0', 'agpl-3.0',
                 'gfdl-1.1', 'gfdl-1.2', 'gfdl-1.3',
-                'nokia-qt-exception-1.1', ]):
+                'nokia-qt-exception-1.1', 'bzip2-1.0.5']):
             return
 
         deprecated = mapping.get('isDeprecatedLicenseId', False)
@@ -425,6 +425,8 @@ def build_license(self, mapping, scancode_licenses):
                 # 'agpl-3.0+'
                 deprecated = False
 
+        # TODO: handle other_spdx_license_keys in license yaml files.
+
         other_urls = mapping.get('seeAlso', [])
         other_urls = (o for o in other_urls if o)
         other_urls = (o.strip() for o in other_urls)

src/cluecode/copyrights.py

@@ -451,6 +451,9 @@ def as_str(cls, node, ignores=frozenset(), include_allrights=False):
     # such as in (1)(ii)(OCT
     (r'^.*\(.*\).*\(.*\).*$', 'JUNK'),
 
+    # parens such as (1) or (a) is a sign of junk but of course NOT (c)
+    (r'^\(([abdefghi\d]|ii|iii)\)$', 'JUNK'),
+
     # found in crypto certificates and LDAP
     (r'^O=$', 'JUNK'),
     (r'^OU=?$', 'JUNK'),
@@ -502,7 +505,8 @@ def as_str(cls, node, ignores=frozenset(), include_allrights=False):
     (r'^[Rr]eleased?$', 'JUNK'),
     (r'^[Cc]opyrighting$', 'JUNK'),
     (r'^Authori.*$', 'JUNK'),
-
+    (r'^such$', 'JUNK'),
+    (r'^[Aa]ssignments?[.,]?$', 'JUNK'),
     (r'^[Bb]uild$', 'JUNK'),
     (r'^[Ss]tring$', 'JUNK'),
     (r'^Implementation-Vendor$', 'JUNK'),
@@ -618,6 +622,7 @@ def as_str(cls, node, ignores=frozenset(), include_allrights=False):
     (r'^Updates$', 'JUNK'),
     (r'^Record-keeping$', 'JUNK'),
     (r'^Privacy$', 'JUNK'),
+    (r'^within$', 'JUNK'),
 
     # various trailing words that are junk
     (r'^Copyleft$', 'JUNK'),
@@ -666,6 +671,7 @@ def as_str(cls, node, ignores=frozenset(), include_allrights=False):
     (r'^However,?$', 'JUNK'),
     (r'^[Cc]ollectively$', 'JUNK'),
     (r'^following$', 'JUNK'),
+    (r'^file\.$', 'JUNK'),
 
     # junk when HOLDER(S): typically used in disclaimers instead
     (r'^HOLDER\(S\)$', 'JUNK'),
@@ -739,6 +745,8 @@ def as_str(cls, node, ignores=frozenset(), include_allrights=False):
     (r'^GA$', 'JUNK'),
     (r'^unzip$', 'JUNK'),
     (r'^EULA', 'JUNK'),
+    (r'^Terms?[.,]?$', 'JUNK'),
+    (r'^Non-Assertion$', 'JUNK'),
 
     # this is not Copr.
     (r'^Coproduct,?[,\.]?$$', 'JUNK'),
@@ -747,6 +755,7 @@ def as_str(cls, node, ignores=frozenset(), include_allrights=False):
     (r'^CONTRIBUTORS?[,\.]?$', 'JUNK'),
     (r'^OTHERS?[,\.]?$', 'JUNK'),
     (r'^Contributors?\:[,\.]?$', 'JUNK'),
+    (r'^Version$', 'JUNK'),
 
     ############################################################################
     # Nouns and proper Nouns
@@ -846,6 +855,7 @@ def as_str(cls, node, ignores=frozenset(), include_allrights=False):
     (r'^GPL\'d', 'NN'),
     (r'^Gnome$', 'NN'),
     (r'^GnuPG$', 'NN'),
+    (r'^Government.', 'NNP'),
     (r'^Government', 'NN'),
     (r'^Grants?\.?,?$', 'NN'),
     (r'^Header', 'NN'),
@@ -946,6 +956,7 @@ def as_str(cls, node, ignores=frozenset(), include_allrights=False):
     (r'^Section', 'NN'),
     (r'^Send$', 'NN'),
     (r'^Separa', 'NN'),
+    (r'^Service$', 'NN'),
     (r'^Several$', 'NN'),
     (r'^SIGN$', 'NN'),
     (r'^Site\.?$', 'NN'),
@@ -983,7 +994,6 @@ def as_str(cls, node, ignores=frozenset(), include_allrights=False):
     (r'^VALUE$', 'NN'),
     (r'^Various', 'NN'),
     (r'^Vendor', 'NN'),
-    (r'^Version', 'NN'),
     (r'^VIEW$', 'NN'),
     (r'^Visit', 'NN'),
     (r'^Website', 'NN'),
@@ -993,8 +1003,11 @@ def as_str(cls, node, ignores=frozenset(), include_allrights=False):
     (r'^WPA$', 'NN'),
     (r'^Xalan$', 'NN'),
     (r'^YOUR', 'NN'),
+    (r'^Your', 'NN'),
     (r'^DateTime', 'NN'),
     (r'^Create$', 'NN'),
+    (r'^Engine\.$', 'NN'),
+    (r'^While$', 'NN'),
 
     # Hours/Date/Day/Month text references
     (r'^am$', 'NN'),
@@ -1091,10 +1104,15 @@ def as_str(cls, node, ignores=frozenset(), include_allrights=False):
     (r'werken$', 'NNP'),
     (r'various\.?$', 'NNP'),
 
+    # treat Attributable as proper noun as it is seen in Author tags such as in:
+    # @author not attributable
+    (r'^[Aa]ttributable$', 'NNP'),
+
     # rarer caps
     # EPFL-LRC/ICA
     (r'^[A-Z]{3,6}-[A-Z]{3,6}/[A-Z]{3,6}', 'NNP'),
 
+
     ############################################################################
     # Named entities: companies, groups, universities, etc
     ############################################################################
@@ -1221,6 +1239,9 @@ def as_str(cls, node, ignores=frozenset(), include_allrights=False):
     (r'^[Cc]ontribut(ors|ing)\.?$', 'CONTRIBUTORS'),
     (r'^contributors,$', 'CONTRIBUTORS'),
 
+    (r'^Contributor[,.]?$', 'NN'),
+    (r'^Licensor[,.]?$', 'NN'),
+
     # same for developed, etc...
     (r'^[Cc]oded$', 'AUTH2'),
     (r'^[Rr]ecoded$', 'AUTH2'),
@@ -1416,6 +1437,8 @@ def as_str(cls, node, ignores=frozenset(), include_allrights=False):
     (r'^<([a-zA-Z]+[a-zA-Z\.]){3,}$', 'EMAIL_START'),
     (r'^[a-zA-Z\.]{2,}>$', 'EMAIL_END'),
 
+    # a .sh shell scripts is NOT an email.
+    (r'^.*\.sh\.?$', 'JUNK'),
     # email eventually in parens or brackets with some trailing punct.
     (r'^[\<\(]?[a-zA-Z0-9]+[a-zA-Z0-9\+_\-\.\%]*(@|at)[a-zA-Z0-9][a-zA-Z0-9\+_\-\.\%]+\.[a-zA-Z]{2,5}?[\>\)\.\,]*$', 'EMAIL'),
 
@@ -1704,7 +1727,7 @@ def as_str(cls, node, ignores=frozenset(), include_allrights=False):
     # and other contributors
     NAME: {<CC> <NN>? <CONTRIBUTORS>}        #644
 
-    NAME: {<NNP|CAPS>+ <AUTHS|CONTRIBUTORS>}        #660
+    NAME: {<NNP|CAPS>+ <AUTHS|AUTHDOT|CONTRIBUTORS>}        #660
 
     NAME: {<VAN|OF> <NAME>}        #680
     NAME: {<NAME-YEAR> <COMP|COMPANY>}        #690
@@ -1970,7 +1993,17 @@ def as_str(cls, node, ignores=frozenset(), include_allrights=False):
     COPYRIGHT: { <COPY> <COPY> <YR-RANGE> <CONTRIBUTORS> <OTH> } #2276
 
     # copyrighted by Object Computing, Inc., St. Louis Missouri, Copyright (C) 2002, all rights reserved.
-    COPYRIGHT: {<COPYRIGHT> <COPY>+  <YR-RANGE> <ALLRIGHTRESERVED>} #2290
+    COPYRIGHT: {<COPYRIGHT> <COPY>+  <YR-RANGE> <ALLRIGHTRESERVED>} #2278
+
+    # copyrighted by Object Computing, Inc., St. Louis Missouri, Copyright (C) 2002, all rights reserved.
+    COPYRIGHT: {<COPYRIGHT> <COPY>+  <YR-RANGE> <ALLRIGHTRESERVED>} #2279
+
+    # Copyright (c) 2004, The Codehaus
+    COPYRIGHT: {<COPY>  <COPY>  <YR-RANGE>  <NN>  <NNP>} #22790
+
+    # Copyright (c) 2017 odahcam
+    COPYRIGHT: {<COPY>  <COPY>  <YR-RANGE>  <NN> <ALLRIGHTRESERVED>} #22791
+    COPYRIGHT: {<COPY>  <COPY>  <YR-RANGE>  <NN>} #22792
 
     COPYRIGHT2: {<COPY>+ <NN|CAPS>? <YR-RANGE>+ <PN>*}        #2280
 
@@ -2036,6 +2069,7 @@ def as_str(cls, node, ignores=frozenset(), include_allrights=False):
     # Copyright 2008 TJ <[email protected]>
     COPYRIGHT: {<COPYRIGHT2> <EMAIL>}       #2636
 
+    # Copyright RUSS DILL Russ <[email protected]>
     COPYRIGHT: {<COPYRIGHT> <CAPS> <NAME-EMAIL>}       #2637
 
     # maintainer Norbert Tretkowski <[email protected]> 2005-04-16
@@ -2137,7 +2171,7 @@ def as_str(cls, node, ignores=frozenset(), include_allrights=False):
     COPYRIGHT: {<COMPANY> <ALLRIGHTRESERVED> <COPYRIGHT2>} #3030
 
     # Copyright (c) 2000 United States Government as represented by the Secretary of the Navy. All rights reserved.
-    COPYRIGHT: {<COPYRIGHT> <NN> <NN> <NN> <BY> <NN> <NAME> <ALLRIGHTRESERVED>} #3035
+    COPYRIGHT: {<COPYRIGHT> <NN> <NN> <NN|NNP> <BY> <NN> <NAME> <ALLRIGHTRESERVED>} #3035
 
     # Copyright (c) 2007-2008, Y Giridhar Appaji Nag <[email protected]>
     COPYRIGHT: {<COPYRIGHT> <COMPANY|NAME|NAME-EMAIL|NAME-YEAR>+} #3040
@@ -2209,6 +2243,9 @@ def as_str(cls, node, ignores=frozenset(), include_allrights=False):
     # Copyright (C) 2005-2006  dann frazier <[email protected]>
     COPYRIGHT: {<COPYRIGHT2>  <NN>  <NN>  <EMAIL>} #999991
 
+    # Copyright (c) 2008 Intel Corporation / Qualcomm Inc.
+    COPYRIGHT: {<COPYRIGHT>  <DASH>  <COMPANY>} #copydash-co
+
 #######################################
 # Authors
 #######################################
@@ -2252,13 +2289,22 @@ def as_str(cls, node, ignores=frozenset(), include_allrights=False):
     # developed by the XML DB Initiative http//www.xmldb.org
     AUTHOR: {<AUTH2> <COMPANY>} #2645-7
 
+    # Author not attributable
+    AUTHOR: {<AUTH>  <NN>  <NNP>} #not attributable
+
+    # author (Panagiotis Tsirigotis)
+    AUTHOR: {<AUTH>  <NNP><NNP>+} #author Foo Bar
+
+
 #######################################
-# Mixed AUTHORS and COPYRIGHTS
+# Mixed AUTHOR and COPYRIGHT
 #######################################
 
     # Compounded statements usings authors
-    # found in some rare cases with a long list of authors.
-    COPYRIGHT: {<COPY> <BY> <AUTHOR>+ <YR-RANGE>*}        #2800
+
+    # Copyright by Daniel K. Gebhart
+    # Also found in some rare cases with a long list of authors.
+    COPYRIGHT: {<COPY> <BY>? <AUTHOR>+  <YR-RANGE>*}        #2800-1
 
     COPYRIGHT: {<AUTHOR> <COPYRIGHT2>}        #2820
     COPYRIGHT: {<AUTHOR> <YR-RANGE>}        #2830
@@ -2312,6 +2358,7 @@ def refine_copyright(c):
     c = strip_balanced_edge_parens(c)
     c = strip_suffixes(c, suffixes=COPYRIGHTS_SUFFIXES)
     c = strip_trailing_period(c)
+    c = c.strip("'")
     return c.strip()
 
 
@@ -2334,8 +2381,6 @@ def refine_holder(h):
     h = h.strip()
     h = strip_trailing_period(h)
     h = h.strip()
-    h = strip_balanced_edge_parens(h)
-    h = h.strip()
     if h and h.lower() not in HOLDERS_JUNK:
         return h
 
@@ -2351,6 +2396,10 @@ def refine_author(a):
     a = a.strip()
     a = strip_trailing_period(a)
     a = a.strip()
+    a = strip_balanced_edge_parens(a)
+    a = a.strip()
+    a = refine_names(a, prefixes=AUTHORS_PREFIXES)
+    a = a.strip()
     if a and a.lower() not in AUTHORS_JUNK:
         return a
 
@@ -2365,6 +2414,8 @@ def refine_names(s, prefixes):
     s = strip_all_unbalanced_parens(s)
     s = strip_some_punct(s)
     s = s.strip()
+    s = strip_balanced_edge_parens(s)
+    s = s.strip()
     s = strip_prefixes(s, prefixes)
     s = s.strip()
     return s
@@ -2543,6 +2594,7 @@ def refine_names(s, prefixes):
         'author\'',
         'authors,',
         'authorship',
+        'or',
     ])
 ))
 
@@ -2561,6 +2613,11 @@ def refine_names(s, prefixes):
     'company',
     'contributing project',
     'its author',
+    'gnomovision',
+    'would',
+    'may',
+    'attributions',
+    'the',
 ])
 
 ################################################################################

src/licensedcode/data/licenses/996-icu-1.0.yml

@@ -1,9 +1,9 @@
 key: 996-icu-1.0
 short_name: Anti 996 License 1.0
 name: Anti 996 License Version 1.0 (Draft)
-category: Proprietary Free
+category: Free Restricted
 owner: 996icu
 homepage_url: https://github.com/996icu/996.ICU
+notes: this is based on the still draft text of 2019-04-17
 text_urls:
     - https://github.com/996icu/996.ICU/blob/dd185162b9d56b629e52c5726995cd7505326b06/LICENSE
-notes: this is based on the still draft text of 2019-04-17