diff --git a/src/formattedcode/__init__.py b/src/formattedcode/__init__.py index 4241e9f005..51ce6bd899 100644 --- a/src/formattedcode/__init__.py +++ b/src/formattedcode/__init__.py @@ -25,9 +25,62 @@ def convert(self, value, param, ctx): if value in known_opts: self.fail( 'Illegal file name conflicting with an option name: ' - f'{ os.fsdecode(value)}. ' + f'{os.fsdecode(value)!r}. ' 'Use the special "-" file name to print results on screen/stdout.', param, ctx, ) + + try: + validate_output_file_path(location=value) + except Exception as e: + self.fail(str(e), param, ctx) + return click.File.convert(self, value, param, ctx) + + +class InvalidScanCodeOutputFileError(Exception): + pass + + +def validate_output_file_path(location): + """ + Raise an InvalidScanCodeOutputFileError if the output file is invalid. + """ + if location != "-": + from pathlib import Path + from commoncode.filetype import is_writable + + path = Path(location) + + if path.is_dir(): + raise InvalidScanCodeOutputFileError( + f'output file is a directory, not a file: {os.fsdecode(location)!r}', + ) + + if path.is_fifo() or path.is_socket() or path.is_block_device() or path.is_char_device(): + raise InvalidScanCodeOutputFileError( + f'output file cannot be a special/char/device/fifo/pipe file: {os.fsdecode(location)!r}', + ) + + if path.exists(): + if not path.is_file(): + raise InvalidScanCodeOutputFileError( + f'output file exists and is not a file: {os.fsdecode(location)!r}', + ) + if not is_writable(location): + raise InvalidScanCodeOutputFileError( + f'output file exists and is not writable: {os.fsdecode(location)!r}', + ) + + else: + parent = path.parent + if not parent.exists() or not parent.is_dir(): + raise InvalidScanCodeOutputFileError( + f'output file parent is not a directory or does not exists: {os.fsdecode(location)!r}', + ) + + if not is_writable(str(parent)): + raise InvalidScanCodeOutputFileError( + f'output file parent is not a writable directory: {os.fsdecode(location)!r}', + ) diff --git a/src/licensedcode/plugin_license_policy.py b/src/licensedcode/plugin_license_policy.py index b004e66fe9..f2f30f5760 100644 --- a/src/licensedcode/plugin_license_policy.py +++ b/src/licensedcode/plugin_license_policy.py @@ -7,20 +7,24 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import os +import logging + +from collections import defaultdict from os.path import exists from os.path import isdir import attr -import os -import logging +import click import saneyaml -from plugincode.post_scan import PostScanPlugin -from plugincode.post_scan import post_scan_impl from commoncode.cliutils import PluggableCommandLineOption from commoncode.cliutils import POST_SCAN_GROUP +from commoncode.filetype import is_file +from commoncode.filetype import is_readable from licensedcode.detection import get_license_keys_from_detections - +from plugincode.post_scan import PostScanPlugin +from plugincode.post_scan import post_scan_impl TRACE = os.environ.get('SCANCODE_DEBUG_LICENSE_POLICY', False) @@ -42,6 +46,21 @@ def logger_debug(*args): return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args)) +def validate_policy_path(ctx, param, value): + """ + Validate the ``value`` of the policy file path + """ + policy = value + if policy: + if not is_file(location=value, follow_symlinks=True): + raise click.BadParameter(f"policy file is not a regular file: {value!r}") + + if not is_readable(location=value): + raise click.BadParameter(f"policy file is not readable: {value!r}") + policy = load_license_policy(value) + return policy + + @post_scan_impl class LicensePolicy(PostScanPlugin): """ @@ -57,10 +76,12 @@ class LicensePolicy(PostScanPlugin): options = [ PluggableCommandLineOption(('--license-policy',), multiple=False, + callback=validate_policy_path, metavar='FILE', help='Load a License Policy file and apply it to the scan at the ' 'Resource level.', - help_group=POST_SCAN_GROUP) + help_group=POST_SCAN_GROUP, + ) ] def is_enabled(self, license_policy, **kwargs): @@ -74,12 +95,19 @@ def process_codebase(self, codebase, license_policy, **kwargs): if not self.is_enabled(license_policy): return - if has_policy_duplicates(license_policy): - codebase.errors.append('ERROR: License Policy file contains duplicate entries.\n') + # license_policy has been validated through a callback and contains data + # loaded from YAML + policies = license_policy.get('license_policies', []) + if not policies: + codebase.errors.append(f'ERROR: License Policy file is empty') return # get a list of unique license policies from the license_policy file - policies = load_license_policy(license_policy).get('license_policies', []) + dupes = get_duplicate_policies(policies) + if dupes: + dupes = '\n'.join(repr(d) for d in dupes.items()) + codebase.errors.append(f'ERROR: License Policy file contains duplicate entries:\n{dupes}') + return # apply policy to Resources if they contain an offending license for resource in codebase.walk(topdown=True): @@ -106,37 +134,46 @@ def process_codebase(self, codebase, license_policy, **kwargs): codebase.save_resource(resource) -def has_policy_duplicates(license_policy_location): +def get_duplicate_policies(policies): """ - Returns True if the policy file contains duplicate entries for a specific license - key. Returns False otherwise. + Return a list of duplicated policy mappings based on the license key. + Return an empty list if there are no duplicates. """ - policies = load_license_policy(license_policy_location).get('license_policies', []) - - unique_policies = {} - - if policies == []: - return False + if not policies: + return [] + policies_by_license = defaultdict(list) for policy in policies: license_key = policy.get('license_key') - - if license_key in unique_policies.keys(): - return True - else: - unique_policies[license_key] = policy - - return False + policies_by_license[license_key].append(policy) + return {key: pols for key, pols in policies_by_license.items() if len(pols) > 1} def load_license_policy(license_policy_location): """ - Return a license_policy dictionary loaded from a license policy file. + Return a license policy mapping loaded from a license policy file. """ - if not license_policy_location or not exists(license_policy_location): - return {} - elif isdir(license_policy_location): + if not license_policy_location: return {} - with open(license_policy_location, 'r') as conf: - conf_content = conf.read() - return saneyaml.load(conf_content) + + if not exists(license_policy_location): + raise click.BadParameter(f"policy file does not exists: {license_policy_location!r} ") + + if isdir(license_policy_location): + raise click.BadParameter(f"policy file is a directory: {license_policy_location!r} ") + + try: + with open(license_policy_location, 'r') as conf: + conf_content = conf.read() + policy = saneyaml.load(conf_content) + if not policy: + raise click.BadParameter(f"policy file is empty: {license_policy_location!r}") + if "license_policies" not in policy: + raise click.BadParameter(f"policy file is missing a 'license_policies' attribute: {license_policy_location!r} ") + except Exception as e: + if isinstance(e, click.BadParameter): + raise e + else: + raise click.BadParameter(f"policy file is not a well formed or readable YAML file: {license_policy_location!r} {e!r}") from e + return policy + diff --git a/src/scancode/cli.py b/src/scancode/cli.py index 1e6bc0dc51..1e189c8964 100644 --- a/src/scancode/cli.py +++ b/src/scancode/cli.py @@ -13,6 +13,7 @@ # Import early because of the side effects import scancode_config +import json import logging import os import platform @@ -42,6 +43,9 @@ class WindowsError(Exception): from commoncode.cliutils import path_progress_message from commoncode.cliutils import progressmanager from commoncode.cliutils import PluggableCommandLineOption +from commoncode.filetype import is_dir +from commoncode.filetype import is_file +from commoncode.filetype import is_readable from commoncode.fileutils import as_posixpath from commoncode.timeutils import time2tstamp from commoncode.resource import Codebase @@ -68,7 +72,6 @@ class WindowsError(Exception): from scancode.interrupt import interruptible from scancode.pool import ScanCodeTimeoutError - # Tracing flags TRACE = False TRACE_DEEP = False @@ -173,6 +176,32 @@ def validate_depth(ctx, param, value): return value +def validate_input_path(ctx, param, value): + """ + Validate a ``value`` list of inputs path strings + """ + options = ctx.params + from_json = options.get("--from-json", False) + for inp in value: + if not (is_file(location=inp, follow_symlinks=True) or is_dir(location=inp, follow_symlinks=True)): + raise click.BadParameter(f"input: {inp!r} is not a regular file or a directory") + + if not is_readable(location=inp): + raise click.BadParameter(f"input: {inp!r} is not readable") + + if from_json and not is_file(location=inp, follow_symlinks=True): + # extra JSON validation + raise click.BadParameter(f"JSON input: {inp!r} is not a file") + if not inp.lower().endswith(".json"): + raise click.BadParameter(f"JSON input: {inp!r} is not a JSON file with a .json extension") + with open(inp) as js: + start = js.read(100).strip() + if not start.startswith("{"): + raise click.BadParameter(f"JSON input: {inp!r} is not a well formed JSON file") + + return value + + @click.command(name='scancode', epilog=epilog_text, cls=ScancodeCommand, @@ -182,6 +211,7 @@ def validate_depth(ctx, param, value): @click.argument('input', metavar=' ...', nargs=-1, + callback=validate_input_path, type=click.Path(exists=True, readable=True, path_type=str)) @click.option('--strip-root', @@ -850,9 +880,12 @@ def echo_func(*_args, **_kwargs): max_in_memory=max_in_memory, max_depth=max_depth, ) - except: - msg = 'ERROR: failed to collect codebase at: %(input)r' % locals() - raise ScancodeError(msg + '\n' + traceback.format_exc()) + except Exception as e: + if from_json and isinstance(e, (json.decoder.JSONDecodeError, UnicodeDecodeError)): + raise click.BadParameter(f"Input JSON scan file(s) is not valid JSON: {input!r} : {e!r}") + else: + msg = f'Failed to process codebase at: {input!r}' + raise ScancodeError(msg + '\n' + traceback.format_exc()) # update headers cle = codebase.get_or_create_current_header() diff --git a/tests/formattedcode/test_formattedcode.py b/tests/formattedcode/test_formattedcode.py new file mode 100644 index 0000000000..361f95294a --- /dev/null +++ b/tests/formattedcode/test_formattedcode.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import os +import subprocess + +import pytest + +from commoncode import fileutils +from commoncode.system import on_windows +from commoncode.system import on_linux +from commoncode.testcase import FileDrivenTesting +from formattedcode import validate_output_file_path +from commoncode.testcase import make_non_writable +from commoncode.filetype import is_writable + +test_env = FileDrivenTesting() + + +def test_validate_output_file_path_with_non_existing_file_in_writable_path(): + test_path = test_env.get_temp_dir('test_dir') + test_file = os.path.join(test_path, 'some-non-existing-path') + validate_output_file_path(test_file) + + +def test_validate_output_file_path_with_existing_writable_file(): + test_path = test_env.get_temp_dir('test_dir') + test_file = os.path.join(test_path, 'some-existing-path') + with open(test_file, 'w') as tp: + tp.write("foo") + validate_output_file_path(test_file) + + +def test_validate_output_file_path_with_existing_writable_dir_raise_exception(): + test_path = test_env.get_temp_dir('test_dir') + test_file = os.path.join(test_path, 'non-existing-parent', 'non-existing-path') + with pytest.raises(Exception): + validate_output_file_path(test_file) + + +def test_validate_output_file_path_with_existing_writable_parent_dir_raise_exception(): + test_path = test_env.get_temp_dir('test_dir') + with pytest.raises(Exception): + validate_output_file_path(test_path) + + +@pytest.mark.skipif(not on_linux, reason='This special file is only on Linux') +def test_validate_output_file_path_with_existing_system_device_file_raise_exception(): + test_file = '/dev/ppp' + with pytest.raises(Exception): + validate_output_file_path(test_file) + + +@pytest.mark.skipif(not on_linux, reason='Special files are easier to test on Linux') +def test_validate_output_file_path_with_existing_fifo_pipe_special_file_raise_exception(): + from uuid import uuid4 + test_file = f"/tmp/scancode-test-{uuid4().hex}" + try: + # p create a FIFO + subprocess.check_call(["mknod", test_file, "p"]) + with pytest.raises(Exception): + validate_output_file_path(test_file) + finally: + fileutils.delete(location=test_file) + + +@pytest.mark.skipif(on_windows, reason='It is hard to have non-readable files on Windows') +def test_validate_output_file_path_with_non_existing_file_in_non_writable_path_raise_exception(): + test_dir = test_env.get_temp_dir('test_dir') + # make dir non writable + test_file = os.path.join(test_dir, 'some-non-existing-path') + try: + # make dir non writable + make_non_writable(test_dir) + assert not is_writable(test_dir) + with pytest.raises(Exception): + validate_output_file_path(test_file) + finally: + fileutils.chmod(test_dir, fileutils.RW, recurse=True) + +@pytest.mark.skipif(on_windows, reason='It is hard to have non-readable/writable files on Windows') +def test_validate_output_file_path_with_existing_non_writable_file_raise_exception(): + test_dir = test_env.get_temp_dir('test_dir') + test_file = os.path.join(test_dir, 'some-existing-path') + with open(test_file, 'w') as tp: + tp.write("foo") + try: + # make file non writable + make_non_writable(test_file) + assert not is_writable(test_file) + with pytest.raises(Exception): + validate_output_file_path(test_file) + finally: + fileutils.chmod(test_dir, fileutils.RW, recurse=True) diff --git a/tests/licensedcode/data/plugin_license_policy/has_policy_duplicates_empty.yml b/tests/licensedcode/data/plugin_license_policy/get_duplicate_policies_empty.yml similarity index 100% rename from tests/licensedcode/data/plugin_license_policy/has_policy_duplicates_empty.yml rename to tests/licensedcode/data/plugin_license_policy/get_duplicate_policies_empty.yml diff --git a/tests/licensedcode/data/plugin_license_policy/has_policy_duplicates_invalid_dupes.yml b/tests/licensedcode/data/plugin_license_policy/get_duplicate_policies_invalid_dupes.yml similarity index 100% rename from tests/licensedcode/data/plugin_license_policy/has_policy_duplicates_invalid_dupes.yml rename to tests/licensedcode/data/plugin_license_policy/get_duplicate_policies_invalid_dupes.yml diff --git a/tests/licensedcode/data/plugin_license_policy/has_policy_duplicates_invalid_no_dupes.yml b/tests/licensedcode/data/plugin_license_policy/get_duplicate_policies_invalid_no_dupes.yml similarity index 100% rename from tests/licensedcode/data/plugin_license_policy/has_policy_duplicates_invalid_no_dupes.yml rename to tests/licensedcode/data/plugin_license_policy/get_duplicate_policies_invalid_no_dupes.yml diff --git a/tests/licensedcode/data/plugin_license_policy/has_policy_duplicates_valid.yml b/tests/licensedcode/data/plugin_license_policy/get_duplicate_policies_valid.yml similarity index 100% rename from tests/licensedcode/data/plugin_license_policy/has_policy_duplicates_valid.yml rename to tests/licensedcode/data/plugin_license_policy/get_duplicate_policies_valid.yml diff --git a/tests/licensedcode/data/plugin_license_policy/various-inputs/not-a-json-but-png.json b/tests/licensedcode/data/plugin_license_policy/various-inputs/not-a-json-but-png.json new file mode 100644 index 0000000000..9b222e65e0 Binary files /dev/null and b/tests/licensedcode/data/plugin_license_policy/various-inputs/not-a-json-but-png.json differ diff --git a/tests/licensedcode/data/plugin_license_policy/various-inputs/not-a-yaml-but-png.yaml b/tests/licensedcode/data/plugin_license_policy/various-inputs/not-a-yaml-but-png.yaml new file mode 100644 index 0000000000..9b222e65e0 Binary files /dev/null and b/tests/licensedcode/data/plugin_license_policy/various-inputs/not-a-yaml-but-png.yaml differ diff --git a/tests/licensedcode/data/plugin_license_policy/various-inputs/png.png b/tests/licensedcode/data/plugin_license_policy/various-inputs/png.png new file mode 100644 index 0000000000..9b222e65e0 Binary files /dev/null and b/tests/licensedcode/data/plugin_license_policy/various-inputs/png.png differ diff --git a/tests/licensedcode/data/plugin_license_policy/various-inputs/true-json.json b/tests/licensedcode/data/plugin_license_policy/various-inputs/true-json.json new file mode 100644 index 0000000000..18d7acf586 --- /dev/null +++ b/tests/licensedcode/data/plugin_license_policy/various-inputs/true-json.json @@ -0,0 +1 @@ +{"foo": "bar"} \ No newline at end of file diff --git a/tests/licensedcode/data/plugin_license_policy/various-inputs/true-scan-json.json b/tests/licensedcode/data/plugin_license_policy/various-inputs/true-scan-json.json new file mode 100644 index 0000000000..ec354f3308 --- /dev/null +++ b/tests/licensedcode/data/plugin_license_policy/various-inputs/true-scan-json.json @@ -0,0 +1,9 @@ +{ + "headers": [{"tool_name": "scancode-toolkit"}], + "files": [ + { + "path": "apache-2.0.LICENSE", + "type": "file" + } + ] +} \ No newline at end of file diff --git a/tests/licensedcode/data/plugin_license_policy/various-inputs/true-yaml-license-policy-yaml.yml b/tests/licensedcode/data/plugin_license_policy/various-inputs/true-yaml-license-policy-yaml.yml new file mode 100644 index 0000000000..6b4b384200 --- /dev/null +++ b/tests/licensedcode/data/plugin_license_policy/various-inputs/true-yaml-license-policy-yaml.yml @@ -0,0 +1,5 @@ +license_policies: +- license_key: bsd-1988 + label: Approved License + color_code: '#008000' + icon: icon-ok-circle diff --git a/tests/licensedcode/data/plugin_license_policy/various-inputs/true-yaml.yaml b/tests/licensedcode/data/plugin_license_policy/various-inputs/true-yaml.yaml new file mode 100644 index 0000000000..7db363eb6d --- /dev/null +++ b/tests/licensedcode/data/plugin_license_policy/various-inputs/true-yaml.yaml @@ -0,0 +1,2 @@ +foo: + - bar \ No newline at end of file diff --git a/tests/licensedcode/data/plugin_license_policy/various-inputs/true-yaml.yml b/tests/licensedcode/data/plugin_license_policy/various-inputs/true-yaml.yml new file mode 100644 index 0000000000..7db363eb6d --- /dev/null +++ b/tests/licensedcode/data/plugin_license_policy/various-inputs/true-yaml.yml @@ -0,0 +1,2 @@ +foo: + - bar \ No newline at end of file diff --git a/tests/licensedcode/test_plugin_license_policy.py b/tests/licensedcode/test_plugin_license_policy.py index 07cae146fc..6e4000fb8c 100644 --- a/tests/licensedcode/test_plugin_license_policy.py +++ b/tests/licensedcode/test_plugin_license_policy.py @@ -7,297 +7,346 @@ # See https://aboutcode.org for more information about nexB OSS projects. # -from os.path import dirname -from os.path import join +import os + +import pytest from commoncode.testcase import FileDrivenTesting -from licensedcode.plugin_license_policy import has_policy_duplicates from licensedcode.plugin_license_policy import load_license_policy +from licensedcode.plugin_license_policy import get_duplicate_policies from scancode.cli_test_utils import load_json_result from scancode.cli_test_utils import run_scan_click from scancode.cli_test_utils import check_json_scan from scancode_config import REGEN_TEST_FIXTURES -class TestLicensePolicy(FileDrivenTesting): - - test_data_dir = join(dirname(__file__), 'data') - - def test_end_to_end_scan_with_license_policy(self): - test_dir = self.extract_test_tar('plugin_license_policy/policy-codebase.tgz') - policy_file = self.get_test_loc('plugin_license_policy/process_codebase_info_license_valid_policy_file.yml') - result_file = self.get_temp_file('json') - args = [ - '--info', - '--license', - '--license-policy', - policy_file, - test_dir, - '--json-pp', - result_file - ] - run_scan_click(args) - test_loc = self.get_test_loc('plugin_license_policy/policy-codebase.expected.json') - check_json_scan(test_loc, result_file, regen=REGEN_TEST_FIXTURES, remove_file_date=True) - - def test_end_to_end_scan_with_license_policy_multiple_text(self): - test_dir = self.get_test_loc('plugin_license_policy/file_with_multiple_licenses.txt') - policy_file = self.get_test_loc('plugin_license_policy/sample_valid_policy_file.yml') - result_file = self.get_temp_file('json') - args = [ - '--info', - '--license', - '--license-policy', - policy_file, - test_dir, - '--json-pp', - result_file - ] - run_scan_click(args) - test_loc = self.get_test_loc('plugin_license_policy/file_with_multiple_licenses.expected.json') - check_json_scan(test_loc, result_file, regen=REGEN_TEST_FIXTURES, remove_file_date=True) - - def test_process_codebase_info_license_duplicate_key_policy_file(self): - test_dir = self.extract_test_tar('plugin_license_policy/policy-codebase.tgz') - policy_file = self.get_test_loc('plugin_license_policy/process_codebase_info_license_duplicate_key_policy_file.yml') - - result_file = self.get_temp_file('json') - - run_scan_click(['--info', '--license', '--license-policy', policy_file, test_dir, '--json-pp', result_file]) - - scan_result = load_json_result(result_file) - - for result in scan_result['files']: - assert 'license_policy' in result.keys() - assert result['license_policy'] == [] - - def test_process_codebase_info_license_valid_policy_file(self): - test_dir = self.extract_test_tar('plugin_license_policy/policy-codebase.tgz') - policy_file = self.get_test_loc('plugin_license_policy/process_codebase_info_license_valid_policy_file.yml') - - result_file = self.get_temp_file('json') - - run_scan_click(['--info', '--license', '--license-policy', policy_file, test_dir, '--json-pp', result_file]) +test_env = FileDrivenTesting() +test_env.test_data_dir = os.path.join(os.path.dirname(__file__), 'data') - scan_result = load_json_result(result_file) - for result in scan_result['files']: - assert 'license_policy' in result.keys() +def test_end_to_end_scan_with_license_policy(): + test_dir = test_env.extract_test_tar('plugin_license_policy/policy-codebase.tgz') + policy_file = test_env.get_test_loc('plugin_license_policy/process_codebase_info_license_valid_policy_file.yml') + result_file = test_env.get_temp_file('json') + args = [ + '--info', + '--license', + '--license-policy', + policy_file, + test_dir, + '--json-pp', + result_file + ] + run_scan_click(args) + test_loc = test_env.get_test_loc('plugin_license_policy/policy-codebase.expected.json') + check_json_scan(test_loc, result_file, regen=REGEN_TEST_FIXTURES, remove_file_date=True) - approved, restricted = 0, 0 - for result in scan_result['files']: - if result.get('license_policy') != []: - if result.get('license_policy')[0].get('label') == "Approved License": - approved += 1 - if result.get('license_policy')[0].get('label') == "Restricted License": - restricted += 1 - assert approved == 1 - assert restricted == 4 +def test_end_to_end_scan_with_license_policy_multiple_text(): + test_dir = test_env.get_test_loc('plugin_license_policy/file_with_multiple_licenses.txt') + policy_file = test_env.get_test_loc('plugin_license_policy/sample_valid_policy_file.yml') + result_file = test_env.get_temp_file('json') + args = [ + '--info', + '--license', + '--license-policy', + policy_file, + test_dir, + '--json-pp', + result_file + ] + run_scan_click(args) + test_loc = test_env.get_test_loc('plugin_license_policy/file_with_multiple_licenses.expected.json') + check_json_scan(test_loc, result_file, regen=REGEN_TEST_FIXTURES, remove_file_date=True) - def test_process_codebase_license_only_valid_policy_file(self): - test_dir = self.extract_test_tar('plugin_license_policy/policy-codebase.tgz') - policy_file = self.get_test_loc('plugin_license_policy/process_codebase_license_only_valid_policy_file.yml') - result_file = self.get_temp_file('json') +def test_process_codebase_info_license_duplicate_key_policy_file(): + test_dir = test_env.extract_test_tar('plugin_license_policy/policy-codebase.tgz') + policy_file = test_env.get_test_loc('plugin_license_policy/process_codebase_info_license_duplicate_key_policy_file.yml') - run_scan_click(['--license', '--license-policy', policy_file, test_dir, '--json-pp', result_file]) + result_file = test_env.get_temp_file('json') - scan_result = load_json_result(result_file) + run_scan_click(['--info', '--license', '--license-policy', policy_file, test_dir, '--json-pp', result_file]) - for result in scan_result['files']: - assert 'license_policy' in result.keys() + scan_result = load_json_result(result_file) - approved, restricted = 0, 0 - for result in scan_result['files']: - if result.get('license_policy') != []: - if result.get('license_policy')[0].get('label') == "Approved License": - approved += 1 - if result.get('license_policy')[0].get('label') == "Restricted License": - restricted += 1 + for result in scan_result['files']: + assert 'license_policy' in result.keys() + assert result['license_policy'] == [] - assert approved == 1 - assert restricted == 4 - def test_process_codebase_info_only_valid_policy_file(self): - test_dir = self.extract_test_tar('plugin_license_policy/policy-codebase.tgz') - policy_file = self.get_test_loc('plugin_license_policy/process_codebase_info_only_valid_policy_file.yml') +def test_process_codebase_info_license_valid_policy_file(): + test_dir = test_env.extract_test_tar('plugin_license_policy/policy-codebase.tgz') + policy_file = test_env.get_test_loc('plugin_license_policy/process_codebase_info_license_valid_policy_file.yml') - result_file = self.get_temp_file('json') + result_file = test_env.get_temp_file('json') - run_scan_click(['--info', '--license-policy', policy_file, test_dir, '--json-pp', result_file]) + run_scan_click(['--info', '--license', '--license-policy', policy_file, test_dir, '--json-pp', result_file]) - scan_result = load_json_result(result_file) + scan_result = load_json_result(result_file) - for result in scan_result['files']: - assert 'license_policy' in result.keys() + for result in scan_result['files']: + assert 'license_policy' in result.keys() - for result in scan_result['files']: - assert result.get('license_policy') == [] + approved, restricted = 0, 0 + for result in scan_result['files']: + if result.get('license_policy') != []: + if result.get('license_policy')[0].get('label') == "Approved License": + approved += 1 + if result.get('license_policy')[0].get('label') == "Restricted License": + restricted += 1 - def test_process_codebase_empty_policy_file(self): - test_dir = self.extract_test_tar('plugin_license_policy/policy-codebase.tgz') - policy_file = self.get_test_loc('plugin_license_policy/process_codebase_empty_policy_file.yml') + assert approved == 1 + assert restricted == 4 - result_file = self.get_temp_file('json') - run_scan_click(['--license', '--license-policy', policy_file, test_dir, '--json-pp', result_file]) +def test_process_codebase_license_only_valid_policy_file(): + test_dir = test_env.extract_test_tar('plugin_license_policy/policy-codebase.tgz') + policy_file = test_env.get_test_loc('plugin_license_policy/process_codebase_license_only_valid_policy_file.yml') - scan_result = load_json_result(result_file) + result_file = test_env.get_temp_file('json') - for result in scan_result['files']: - assert 'license_policy' in result.keys() + run_scan_click(['--license', '--license-policy', policy_file, test_dir, '--json-pp', result_file]) - for result in scan_result['files']: - assert result.get('license_policy') == [] + scan_result = load_json_result(result_file) - def test_process_codebase_invalid_policy_file(self): - test_dir = self.extract_test_tar('plugin_license_policy/policy-codebase.tgz') - policy_file = self.get_test_loc('plugin_license_policy/process_codebase_invalid_policy_file.yml') + for result in scan_result['files']: + assert 'license_policy' in result.keys() - result_file = self.get_temp_file('json') + approved, restricted = 0, 0 + for result in scan_result['files']: + if result.get('license_policy') != []: + if result.get('license_policy')[0].get('label') == "Approved License": + approved += 1 + if result.get('license_policy')[0].get('label') == "Restricted License": + restricted += 1 - run_scan_click(['--license', '--license-policy', policy_file, test_dir, '--json-pp', result_file]) + assert approved == 1 + assert restricted == 4 - scan_result = load_json_result(result_file) - for result in scan_result['files']: - assert 'license_policy' in result.keys() +def test_process_codebase_info_only_valid_policy_file(): + test_dir = test_env.extract_test_tar('plugin_license_policy/policy-codebase.tgz') + policy_file = test_env.get_test_loc('plugin_license_policy/process_codebase_info_only_valid_policy_file.yml') + + result_file = test_env.get_temp_file('json') + + run_scan_click(['--info', '--license-policy', policy_file, test_dir, '--json-pp', result_file]) + + scan_result = load_json_result(result_file) + + for result in scan_result['files']: + assert 'license_policy' in result.keys() + + for result in scan_result['files']: + assert result.get('license_policy') == [] + + +def test_process_codebase_empty_policy_file(): + test_dir = test_env.extract_test_tar('plugin_license_policy/policy-codebase.tgz') + policy_file = test_env.get_test_loc('plugin_license_policy/process_codebase_empty_policy_file.yml') + + result_file = test_env.get_temp_file('json') + + run_scan_click(['--license', '--license-policy', policy_file, test_dir, '--json-pp', result_file]) + + scan_result = load_json_result(result_file) - for result in scan_result['files']: - assert result.get('license_policy') == [] + for result in scan_result['files']: + assert 'license_policy' in result.keys() - def test_has_policy_duplcates_invalid_dupes(self): - test_file = self.get_test_loc('plugin_license_policy/has_policy_duplicates_invalid_dupes.yml') + for result in scan_result['files']: + assert result.get('license_policy') == [] - result = has_policy_duplicates(test_file) - assert result == True +def test_process_codebase_invalid_policy_file(): + test_dir = test_env.extract_test_tar('plugin_license_policy/policy-codebase.tgz') + policy_file = test_env.get_test_loc('plugin_license_policy/process_codebase_invalid_policy_file.yml') + result_file = test_env.get_temp_file('json') + run_scan_click(['--license', '--license-policy', policy_file, test_dir, '--json-pp', result_file], expected_rc=2) - def test_has_policy_duplcates_valid(self): - test_file = self.get_test_loc('plugin_license_policy/has_policy_duplicates_valid.yml') - result = has_policy_duplicates(test_file) - - assert result == False - - def test_has_policy_duplicates_empty(self): - test_file = self.get_test_loc('plugin_license_policy/has_policy_duplicates_empty.yml') - - result = has_policy_duplicates(test_file) +def test_get_duplicate_policies_with_dupes(): + test_file = test_env.get_test_loc('plugin_license_policy/get_duplicate_policies_invalid_dupes.yml') + result = load_license_policy(test_file) + policies = result.get('license_policies', []) + expected = { + 'broadcom-commercial': [ + {'color_code': '#FFcc33', + 'icon': 'icon-warning-sign', + 'label': 'Restricted License', + 'license_key': 'broadcom-commercial'}, + {'color_code': '#008000', + 'icon': 'icon-ok-circle', + 'label': 'Approved License', + 'license_key': 'broadcom-commercial'}, + ], + } + assert get_duplicate_policies(policies) == expected + + +def test_get_duplicate_policies_with_no_dupes(): + test_file = test_env.get_test_loc('plugin_license_policy/get_duplicate_policies_valid.yml') + result = load_license_policy(test_file) + policies = result.get('license_policies', []) + assert get_duplicate_policies(policies) == {} + + +def test_get_duplicate_policies_empty(): + test_file = test_env.get_test_loc('plugin_license_policy/get_duplicate_policies_empty.yml') + result = load_license_policy(test_file) + policies = result.get('license_policies', []) + assert get_duplicate_policies(policies) == [] + + +def test_load_license_policy_with_duplicate_keys(): + test_file = test_env.get_test_loc('plugin_license_policy/load_license_policy_duplicate_keys.yml') + + expected = dict([ + ('license_policies', [ + dict([ + ('license_key', 'broadcom-commercial'), + ('label', 'Restricted License'), + ('color_code', '#FFcc33'), + ('icon', 'icon-warning-sign'), + ]), + dict([ + ('license_key', 'bsd-1988'), + ('label', 'Approved License'), + ('color_code', '#008000'), + ('icon', 'icon-ok-circle'), + ]), + dict([ + ('license_key', 'esri-devkit'), + ('label', 'Restricted License'), + ('color_code', '#FFcc33'), + ('icon', 'icon-warning-sign'), + ]), + dict([ + ('license_key', 'oracle-java-ee-sdk-2010'), + ('label', 'Restricted License'), + ('color_code', '#FFcc33'), + ('icon', 'icon-warning-sign'), + ]), + dict([ + ('license_key', 'rh-eula'), + ('label', 'Restricted License'), + ('color_code', '#FFcc33'), + ('icon', 'icon-warning-sign'), + ]), + dict([ + ('license_key', 'broadcom-commercial'), + ('label', 'Approved License'), + ('color_code', '#008000'), + ('icon', 'icon-ok-circle'), + ]), + ]) + ]) + + result = load_license_policy(test_file) + assert result == expected + + +def test_load_license_policy_simple(): + test_file = test_env.get_test_loc('plugin_license_policy/load_license_policy_valid.yml') + + expected = dict([ + ('license_policies', [ + dict([ + ('license_key', 'broadcom-commercial'), + ('label', 'Restricted License'), + ('color_code', '#FFcc33'), + ('icon', 'icon-warning-sign'), + ]), + dict([ + ('license_key', 'bsd-1988'), + ('label', 'Approved License'), + ('color_code', '#008000'), + ('icon', 'icon-ok-circle'), + ]), + dict([ + ('license_key', 'esri-devkit'), + ('label', 'Restricted License'), + ('color_code', '#FFcc33'), + ('icon', 'icon-warning-sign'), + ]), + dict([ + ('license_key', 'oracle-java-ee-sdk-2010'), + ('label', 'Restricted License'), + ('color_code', '#FFcc33'), + ('icon', 'icon-warning-sign'), + ]), + dict([ + ('license_key', 'rh-eula'), + ('label', 'Restricted License'), + ('color_code', '#FFcc33'), + ('icon', 'icon-warning-sign'), + ]), + ]) + ]) - assert result == False + result = load_license_policy(test_file) - def test_has_policy_duplicates_invalid_no_dupes(self): - test_file = self.get_test_loc('plugin_license_policy/has_policy_duplicates_invalid_no_dupes.yml') + assert result == expected - result = has_policy_duplicates(test_file) - assert result == False +def test_load_license_policy_empty(): + test_file = test_env.get_test_loc('plugin_license_policy/load_license_policy_empty.yml') + result = load_license_policy(test_file) + assert result == {'license_policies': []} - def test_load_license_policy_duplicate_keys(self): - test_file = self.get_test_loc('plugin_license_policy/load_license_policy_duplicate_keys.yml') - expected = dict([ - ('license_policies', [ - dict([ - ('license_key', 'broadcom-commercial'), - ('label', 'Restricted License'), - ('color_code', '#FFcc33'), - ('icon', 'icon-warning-sign'), - ]), - dict([ - ('license_key', 'bsd-1988'), - ('label', 'Approved License'), - ('color_code', '#008000'), - ('icon', 'icon-ok-circle'), - ]), - dict([ - ('license_key', 'esri-devkit'), - ('label', 'Restricted License'), - ('color_code', '#FFcc33'), - ('icon', 'icon-warning-sign'), - ]), - dict([ - ('license_key', 'oracle-java-ee-sdk-2010'), - ('label', 'Restricted License'), - ('color_code', '#FFcc33'), - ('icon', 'icon-warning-sign'), - ]), - dict([ - ('license_key', 'rh-eula'), - ('label', 'Restricted License'), - ('color_code', '#FFcc33'), - ('icon', 'icon-warning-sign'), - ]), - dict([ - ('license_key', 'broadcom-commercial'), - ('label', 'Approved License'), - ('color_code', '#008000'), - ('icon', 'icon-ok-circle'), - ]), - ]) - ]) +def test_load_license_policy_invalid(): + test_file = test_env.get_test_loc('plugin_license_policy/load_license_policy_invalid.yml') + with pytest.raises(Exception): + load_license_policy(test_file) - result = load_license_policy(test_file) - - assert result == expected - - def test_load_license_policy_valid(self): - test_file = self.get_test_loc('plugin_license_policy/load_license_policy_valid.yml') - - expected = dict([ - ('license_policies', [ - dict([ - ('license_key', 'broadcom-commercial'), - ('label', 'Restricted License'), - ('color_code', '#FFcc33'), - ('icon', 'icon-warning-sign'), - ]), - dict([ - ('license_key', 'bsd-1988'), - ('label', 'Approved License'), - ('color_code', '#008000'), - ('icon', 'icon-ok-circle'), - ]), - dict([ - ('license_key', 'esri-devkit'), - ('label', 'Restricted License'), - ('color_code', '#FFcc33'), - ('icon', 'icon-warning-sign'), - ]), - dict([ - ('license_key', 'oracle-java-ee-sdk-2010'), - ('label', 'Restricted License'), - ('color_code', '#FFcc33'), - ('icon', 'icon-warning-sign'), - ]), - dict([ - ('license_key', 'rh-eula'), - ('label', 'Restricted License'), - ('color_code', '#FFcc33'), - ('icon', 'icon-warning-sign'), - ]), - ]) - ]) - result = load_license_policy(test_file) +def test_load_license_policy_invalid2(): + test_file = test_env.get_test_loc('plugin_license_policy/get_duplicate_policies_invalid_no_dupes.yml') + with pytest.raises(Exception): + load_license_policy(test_file) - assert result == expected - def test_load_license_policy_empty(self): - test_file = self.get_test_loc('plugin_license_policy/load_license_policy_empty.yml') +faulty_policy_yaml = [ + ('plugin_license_policy/various-inputs/not-a-json-but-png.json', 2, "Error: Invalid value for '--license-policy': policy file is not a well formed or readable YAML file:"), + ('plugin_license_policy/various-inputs/not-a-yaml-but-png.yaml', 2, "Error: Invalid value for '--license-policy': policy file is not a well formed or readable YAML file:"), + ('plugin_license_policy/various-inputs/png.png', 2, "Error: Invalid value for '--license-policy': policy file is not a well formed or readable YAML file:"), + ('plugin_license_policy/various-inputs/true-json.json', 2, "Error: Invalid value for '--license-policy': policy file is missing a 'license_policies' attribute"), + ('plugin_license_policy/various-inputs/true-yaml.yaml', 2, "Error: Invalid value for '--license-policy': policy file is missing a 'license_policies' attribute"), + ('plugin_license_policy/various-inputs/true-yaml.yml', 2, "Error: Invalid value for '--license-policy': policy file is missing a 'license_policies' attribute"), +] - expected = dict([ - (u'license_policies', []) - ]) - result = load_license_policy(test_file) - - assert result == expected - - def test_load_license_policy_invalid(self): - test_file = self.get_test_loc('plugin_license_policy/load_license_policy_invalid.yml') - - result = load_license_policy(test_file) +@pytest.mark.parametrize('test_policy, expected_rc, expected_message', faulty_policy_yaml) +def test_scan_does_validate_input_and_fails_on_faulty_policy_input(test_policy, expected_rc, expected_message): + test_input = test_env.get_test_loc('plugin_license_policy/various-inputs/true-scan-json.json') + test_policy = test_env.get_test_loc(test_policy) + result = run_scan_click([ + '--from-json', + test_input, + '--json-pp', + '-', + '--license-policy', + test_policy, + ], + expected_rc=expected_rc, + retry=False, + ) + assert expected_message in result.output + + +def test_scan_does_validate_input_and_does_not_fail_on_valid_policy_input(): + test_input = test_env.get_test_loc('plugin_license_policy/various-inputs/true-scan-json.json') + test_policy = test_env.get_test_loc('plugin_license_policy/various-inputs/true-yaml-license-policy-yaml.yml') + run_scan_click( + [ + '--from-json', + test_input, + '--json-pp', + '-', + '--license-policy', + test_policy, + ], + retry=False, + ) - assert result == {} diff --git a/tests/scancode/data/various-inputs/not-a-json-but-png.json b/tests/scancode/data/various-inputs/not-a-json-but-png.json new file mode 100644 index 0000000000..9b222e65e0 Binary files /dev/null and b/tests/scancode/data/various-inputs/not-a-json-but-png.json differ diff --git a/tests/scancode/data/various-inputs/not-a-yaml-but-png.yaml b/tests/scancode/data/various-inputs/not-a-yaml-but-png.yaml new file mode 100644 index 0000000000..9b222e65e0 Binary files /dev/null and b/tests/scancode/data/various-inputs/not-a-yaml-but-png.yaml differ diff --git a/tests/scancode/data/various-inputs/png.png b/tests/scancode/data/various-inputs/png.png new file mode 100644 index 0000000000..9b222e65e0 Binary files /dev/null and b/tests/scancode/data/various-inputs/png.png differ diff --git a/tests/scancode/data/various-inputs/true-json.json b/tests/scancode/data/various-inputs/true-json.json new file mode 100644 index 0000000000..18d7acf586 --- /dev/null +++ b/tests/scancode/data/various-inputs/true-json.json @@ -0,0 +1 @@ +{"foo": "bar"} \ No newline at end of file diff --git a/tests/scancode/data/various-inputs/true-scan-json.json b/tests/scancode/data/various-inputs/true-scan-json.json new file mode 100644 index 0000000000..ec354f3308 --- /dev/null +++ b/tests/scancode/data/various-inputs/true-scan-json.json @@ -0,0 +1,9 @@ +{ + "headers": [{"tool_name": "scancode-toolkit"}], + "files": [ + { + "path": "apache-2.0.LICENSE", + "type": "file" + } + ] +} \ No newline at end of file diff --git a/tests/scancode/data/various-inputs/true-yaml-license-policy-yaml.yml b/tests/scancode/data/various-inputs/true-yaml-license-policy-yaml.yml new file mode 100644 index 0000000000..6b4b384200 --- /dev/null +++ b/tests/scancode/data/various-inputs/true-yaml-license-policy-yaml.yml @@ -0,0 +1,5 @@ +license_policies: +- license_key: bsd-1988 + label: Approved License + color_code: '#008000' + icon: icon-ok-circle diff --git a/tests/scancode/data/various-inputs/true-yaml.yaml b/tests/scancode/data/various-inputs/true-yaml.yaml new file mode 100644 index 0000000000..7db363eb6d --- /dev/null +++ b/tests/scancode/data/various-inputs/true-yaml.yaml @@ -0,0 +1,2 @@ +foo: + - bar \ No newline at end of file diff --git a/tests/scancode/data/various-inputs/true-yaml.yml b/tests/scancode/data/various-inputs/true-yaml.yml new file mode 100644 index 0000000000..7db363eb6d --- /dev/null +++ b/tests/scancode/data/various-inputs/true-yaml.yml @@ -0,0 +1,2 @@ +foo: + - bar \ No newline at end of file diff --git a/tests/scancode/test_cli.py b/tests/scancode/test_cli.py index 693a873b32..746da18051 100644 --- a/tests/scancode/test_cli.py +++ b/tests/scancode/test_cli.py @@ -709,7 +709,7 @@ def test_scan_to_json_without_FILE_does_not_write_to_next_option(): result = run_scan_click(args, expected_rc=2) assert ( 'Error: Invalid value for "--json": Illegal file name ' - 'conflicting with an option name: --info.' + 'conflicting with an option name: "--info".' ).replace("'", '"') in result.output.replace("'", '"') @@ -966,3 +966,35 @@ def test_getting_version_returns_valid_yaml(): args = ['-V'] result = run_scan_click(args) assert saneyaml.load(result.output) == test_version + + +faulty_json = [ + ('various-inputs/not-a-json-but-png.json', 2, 'Error: Invalid value: Input JSON scan file(s) is not valid JSON'), + ('various-inputs/not-a-yaml-but-png.yaml', 2, 'Error: Invalid value: Input JSON scan file(s) is not valid JSON'), + ('various-inputs/png.png', 2, 'Error: Invalid value: Input JSON scan file(s) is not valid JSON'), + ('various-inputs/true-json.json', 2, 'Error: Invalid value: Failed to process codebase'), + ('various-inputs/true-yaml.yaml', 2, 'Error: Invalid value: Input JSON scan file(s) is not valid JSON'), + ('various-inputs/true-yaml-license-policy-yaml.yml', 2, 'Error: Invalid value: Input JSON scan file(s) is not valid JSON'), + ('various-inputs/true-yaml.yml', 2, 'Error: Invalid value: Input JSON scan file(s) is not valid JSON'), +] + + +@pytest.mark.parametrize('test_file, expected_rc, expected_message', faulty_json) +def test_scan_does_validate_input_and_fails_on_faulty_json_input(test_file, expected_rc, expected_message): + test_file = test_env.get_test_loc(test_file) + result = run_scan_click( + [ + '--from-json', + test_file, + '--json-pp', + '-', + ], + expected_rc=expected_rc, + retry=False, + ) + assert expected_message in result.output + + +def test_scan_does_validate_input_and_does_not_fail_on_valid_json_input(): + test_file = test_env.get_test_loc('various-inputs/true-scan-json.json') + run_scan_click(['--from-json', test_file, '--json-pp', '-'], retry=False)