Merge changed from develop branch

Signed-off-by: Chin Yeung Li <[email protected]>

Author: chinyeungli

.gitignore

@@ -44,4 +44,5 @@ docs/_build
 .ve
 *.bak
 /.cache/
-/.settings/
+/.settings/
+/tcl/

USAGE.rst

@@ -45,8 +45,9 @@ attrib
                                 names - mapping.config
     --mapping-file              Use a custom mapping file with mapping between input
                                 keys and ABOUT field names.
-    --verbose                   Show all the errors and warning.
     --template PATH             Path to a custom attribution template.
+    --vartext TEXT              Variable texts to the attribution template
+    --verbose                   Show all the errors and warning.
     -q, --quiet                 Do not print any error/warning.
     -h, --help                  Show this message and exit.
 
@@ -55,7 +56,7 @@ Purpose
 Generate an attribution file which contains the all license information
 from the LOCATION along with the license text.
 
-Assuming the follow:
+Assume the following:
 
 ::
 
@@ -90,11 +91,6 @@ Options
 
     $ about attrib --mapping-file CUSTOM_MAPPING_FILE_PATH LOCATION OUTPUT
 
-    --verbose
-
-        This option tells the tool to show all errors found.
-        The default behavior will only show 'CRITICAL', 'ERROR', and 'WARNING'
-
     --template
 
         This option allows you to use your own template for attribution generation.
@@ -103,6 +99,21 @@ Options
 
     $ about attrib --template /home/custom_template/template.html LOCATION OUTPUT
 
+    --vartext
+
+        This option allow you to pass variable texts to the attribution template
+
+    $ about attrib --vartext title="Attribution Notice" --vartext header="Product 101" LOCATION OUTPUT
+
+        Users can use the following in the template to get the vartext:
+        {{ vartext_dict['title'] }}
+        {{ vartext_dict['header'] }} 
+
+    --verbose
+
+        This option tells the tool to show all errors found.
+        The default behavior will only show 'CRITICAL', 'ERROR', and 'WARNING'
+
 
 The following data are passed to jinja2 and, therefore, can be used for a custom template:
  * about object: the about objects
@@ -251,10 +262,13 @@ inventory
 
 ::
 
+    --filter TEXT               Filter for the output inventory.
     -f, --format [json|csv]     Set OUTPUT file format.  [default: csv]
     --mapping                   Use file mapping.config to collect the defined not supported fields in ABOUT files.
     --mapping-file              Use a custom mapping file with mapping between input
                                 keys and ABOUT field names.
+    --mapping-output FILE       Use a custom mapping file with mapping between
+                                ABOUT field names and output keys
     --verbose                   Show all the errors and warning.
     -q, --quiet                 Do not print any error/warning.
     -h, --help                  Show this message and exit.
@@ -268,6 +282,14 @@ Options
 
 ::
 
+    -filter TEXT
+ 
+        Filter for the output inventory.
+
+    $ about inventory --filter "license_expression=gpl-2.0" LOCATION OUTPUT
+
+    The above command will only inventory the ABOUT files which have the "license_expression: gpl-2.0"
+
     -f, --format [json|csv]
  
         Set OUTPUT file format.  [default: csv]
@@ -282,7 +304,19 @@ Options
 
         Same behavior as `--mapping` but with custom mapping file
 
-    $ about attrib --mapping-file CUSTOM_MAPPING_FILE_PATH LOCATION OUTPUT
+    $ about inventory --mapping-file CUSTOM_MAPPING_FILE_PATH LOCATION OUTPUT
+
+    --mapping-output
+
+        Same behavior as `--mapping-file` but with custom mapping file
+        In the custom mapping file, the left side is the custom key name where
+        the right side is the ABOUT field name. For instance,
+        Component: name
+        
+        The "Component" is a custom field name for the output
+        The "name" is one of the defaul ABOUT field name that user want to convert
+
+    $ about inventory --mapping-output CUSTOM_MAPPING_FILE_PATH LOCATION OUTPUT
 
     --verbose
 

about

@@ -16,4 +16,4 @@ fi
 
 source $ABOUT_ROOT_DIR/bin/activate
 
-$ABOUT_ROOT_DIR/bin/about-code "$@"
+$ABOUT_ROOT_DIR/bin/about "$@"

about.bat

@@ -29,7 +29,7 @@ goto about
 
 :about
 call %ABOUT_ROOT_DIR%\Scripts\activate
-echo %ABOUT_ROOT_DIR%\bin\about-code %CMD_LINE_ARGS%
-%ABOUT_ROOT_DIR%\bin\about-code %CMD_LINE_ARGS%
+echo %ABOUT_ROOT_DIR%\bin\about %CMD_LINE_ARGS%
+%ABOUT_ROOT_DIR%\bin\about %CMD_LINE_ARGS%
 
 :EOS

docs/CHANGELOG.rst

@@ -1,3 +1,13 @@
+2018-09-19
+
+    Release 3.1.2
+
+    * New `--vartext` option for `attrib`
+    * Add support for `checksum_sha256` and `author_file`
+    * `check` command will not count INFO message as error when `--verbose` is set
+    * Update `track_change` to `track_changes`
+    * New `--filter` and `--mapping-output` options for `inventory`
+
 2018-6-25
 
     Release 3.1.1

setup.py

@@ -24,7 +24,7 @@ def read(*names, **kwargs):
 
 setup(
     name='aboutcode-toolkit',
-    version='3.1.1',
+    version='3.1.2',
     license='Apache-2.0',
     description=(
         'AboutCode-toolkit is a tool to document the provenance (origin and license) of '

src/attributecode/__init__.py

@@ -27,7 +27,7 @@
     basestring = str  # Python 3
 
 
-__version__ = '3.1.1'
+__version__ = '3.1.2'
 
 
 __about_spec_version__ = '3.0.0'

src/attributecode/attrib.py

@@ -37,7 +37,7 @@
 from attributecode.util import add_unc
 
 
-def generate(abouts, template_string=None):
+def generate(abouts, template_string=None, vartext_dict=None):
     """
     Generate and return attribution text from a list of About objects and a
     template string.
@@ -110,7 +110,7 @@ def generate(abouts, template_string=None):
                                    license_file_name_and_key=license_file_name_and_key,
                                    license_key_to_license_name=license_key_to_license_name,
                                    license_name_to_license_key=license_name_to_license_key,
-                                   utcnow=utcnow)
+                                   utcnow=utcnow, vartext_dict=vartext_dict)
     except Exception as e:
         line = getattr(e, 'lineno', None)
         ln_msg = ' at line: %r' % line if line else ''
@@ -134,7 +134,7 @@ def check_template(template_string):
 default_template = join(os.path.dirname(os.path.realpath(__file__)),
                                 'templates', 'default_html.template')
 
-def generate_from_file(abouts, template_loc=None):
+def generate_from_file(abouts, template_loc=None, vartext_dict=None):
     """
     Generate and return attribution string from a list of About objects and a
     template location.
@@ -144,11 +144,11 @@ def generate_from_file(abouts, template_loc=None):
     template_loc = add_unc(template_loc)
     with codecs.open(template_loc, 'rb', encoding='utf-8') as tplf:
         tpls = tplf.read()
-    return generate(abouts, template_string=tpls)
+    return generate(abouts, template_string=tpls, vartext_dict=vartext_dict)
 
 
 def generate_and_save(abouts, output_location, use_mapping=False, mapping_file=None,
-                      template_loc=None, inventory_location=None):
+                      template_loc=None, inventory_location=None, vartext=None):
     """
     Generate attribution file using the `abouts` list of About object
     at `output_location`.
@@ -167,6 +167,7 @@ def generate_and_save(abouts, output_location, use_mapping=False, mapping_file=N
     afp_list = []
     not_match_path = []
     errors = []
+    vartext_dict = {}
 
     if not inventory_location:
         updated_abouts = abouts
@@ -239,7 +240,14 @@ def generate_and_save(abouts, output_location, use_mapping=False, mapping_file=N
             else:
                 about.license.value = lic_list
 
-    rendered = generate_from_file(updated_abouts, template_loc=template_loc)
+    # Parse the vartext and save to the vartext dictionary
+    if vartext:
+        for var in vartext:
+            key = var.partition('=')[0]
+            value = var.partition('=')[2]
+            vartext_dict[key] = value
+
+    rendered = generate_from_file(updated_abouts, template_loc=template_loc, vartext_dict=vartext_dict)
 
     if rendered:
         output_location = add_unc(output_location)

src/attributecode/cmd.py

@@ -37,6 +37,7 @@
 from attributecode.util import extract_zip
 from attributecode.util import update_severity_level_about_resource_path_not_exist_error
 from attributecode.util import to_posix
+from attributecode.util import inventory_filter
 
 
 __copyright__ = """
@@ -90,7 +91,7 @@ def cli():
 
 Read, write and collect provenance and license inventories from .ABOUT files to and from JSON or CSV files.
 
-Use about-code <command> --help for help on a command.
+Use about <command> --help for help on a command.
     """
 
 
@@ -108,6 +109,9 @@ def cli():
 @click.argument('output', nargs=1, required=True,
     type=click.Path(exists=False, dir_okay=False, resolve_path=True))
 
+@click.option('--filter', nargs=1, multiple=True, 
+    help='Filter for the output inventory.')
+
 @click.option('-f', '--format', is_flag=False, default='csv', show_default=True,
     type=click.Choice(['json', 'csv']),
     help='Set OUTPUT inventory file format.')
@@ -119,6 +123,10 @@ def cli():
     type=click.Path(exists=True, dir_okay=True, readable=True, resolve_path=True),
     help='Use a custom mapping file with mapping between input keys and ABOUT field names.')
 
+@click.option('--mapping-output', metavar='FILE', nargs=1,
+    type=click.Path(exists=True, dir_okay=True, readable=True, resolve_path=True),
+    help='Use a custom mapping file with mapping between ABOUT field names and output keys')
+
 @click.option('--verbose', is_flag=True, default=False,
     help='Show all errors and warnings. '
         'By default, the tool only prints these '
@@ -132,7 +140,7 @@ def cli():
 
 @click.help_option('-h', '--help')
 
-def inventory(location, output, mapping, mapping_file, quiet, format, verbose):
+def inventory(location, output, mapping, mapping_file, mapping_output, filter, quiet, format, verbose):
     """
 Collect a JSON or CSV inventory of components from .ABOUT files.
 
@@ -157,6 +165,22 @@ def inventory(location, output, mapping, mapping_file, quiet, format, verbose):
 
     errors, abouts = model.collect_inventory(location, use_mapping=mapping, mapping_file=mapping_file)
 
+    updated_abouts = []
+    if filter:
+        filter_dict = {}
+        # Parse the filter and save to the filter dictionary with a list of value
+        for element in filter:
+            key = element.partition('=')[0]
+            value = element.partition('=')[2]
+            if key in filter_dict:
+                filter_dict[key].append(value)
+            else:
+                value_list = [value]
+                filter_dict[key] = value_list
+        updated_abouts = inventory_filter(abouts, filter_dict)
+    else:
+        updated_abouts = abouts
+
     # Do not write the output if one of the ABOUT files has duplicated key names
     dup_error_msg = u'Duplicated key name(s)'
     halt_output = False
@@ -166,7 +190,7 @@ def inventory(location, output, mapping, mapping_file, quiet, format, verbose):
             break
 
     if not halt_output:
-        write_errors = model.write_output(abouts, output, format)
+        write_errors = model.write_output(updated_abouts, output, format, mapping_output)
         for err in write_errors:
             errors.append(err)
     else:
@@ -290,6 +314,12 @@ def gen(location, output, mapping, mapping_file, license_notice_text_location, f
     type=click.Path(exists=True, dir_okay=True, readable=True, resolve_path=True),
     help='Use a custom mapping file with mapping between input keys and ABOUT field names.')
 
+@click.option('--template', type=click.Path(exists=True), nargs=1,
+    help='Path to an optional custom attribution template used for generation.')
+
+@click.option('--vartext', nargs=1, multiple=True, 
+    help='Variable texts to the attribution template.')
+
 @click.option('--verbose', is_flag=True, default=False,
     help='Show all errors and warnings. '
         'By default, the tool only prints these '
@@ -298,15 +328,12 @@ def gen(location, output, mapping, mapping_file, license_notice_text_location, f
         'for any level.'
 )
 
-@click.option('--template', type=click.Path(exists=True), nargs=1,
-    help='Path to an optional custom attribution template used for generation.')
-
 @click.option('-q', '--quiet', is_flag=True,
     help='Do not print error or warning messages.')
 
 @click.help_option('-h', '--help')
 
-def attrib(location, output, template, mapping, mapping_file, inventory, quiet, verbose):
+def attrib(location, output, template, mapping, mapping_file, inventory, vartext, quiet, verbose):
     """
 Generate an attribution document at OUTPUT using .ABOUT files at LOCATION.
 
@@ -325,7 +352,7 @@ def attrib(location, output, template, mapping, mapping_file, inventory, quiet,
     no_match_errors = attrib_generate_and_save(
         abouts=abouts, output_location=output,
         use_mapping=mapping, mapping_file=mapping_file, template_loc=template,
-        inventory_location=inventory)
+        inventory_location=inventory, vartext=vartext)
 
     if not no_match_errors:
         # Check for template error
@@ -378,21 +405,24 @@ def check(location, verbose):
 
     msg_format = '%(sever)s: %(message)s'
     print_errors = []
+    number_of_errors = 0
     for severity, message in errors:
         sever = severities[severity]
+        # Only problematic_errors should be counted.
+        # Others such as INFO should not be counted as error.
+        if sever in problematic_errors:
+            number_of_errors = number_of_errors + 1
         if verbose:
             print_errors.append(msg_format % locals())
         elif sever in problematic_errors:
             print_errors.append(msg_format % locals())
 
-    number_of_errors = len(print_errors)
-
     for err in print_errors:
         print(err)
 
     if print_errors:
         click.echo('Found {} errors.'.format(number_of_errors))
-        # FIXME: not sure this is the right way to exit with a retrun code
+        # FIXME: not sure this is the right way to exit with a return code
         sys.exit(1)
     else:
         click.echo('No error found.')