From e3be1308a4949617adbf8961af339cec09fc5923 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Gera?= Date: Thu, 1 Jun 2017 13:50:04 +0200 Subject: [PATCH 01/23] CTU basic implementation --- libscanbuild/analyze.py | 167 +++++++++++++++++++++++++++++++++++--- libscanbuild/arguments.py | 35 ++++++++ 2 files changed, 191 insertions(+), 11 deletions(-) diff --git a/libscanbuild/analyze.py b/libscanbuild/analyze.py index d923115..0f3be09 100644 --- a/libscanbuild/analyze.py +++ b/libscanbuild/analyze.py @@ -23,6 +23,8 @@ import platform import contextlib import datetime +import shutil +import glob from libscanbuild import command_entry_point, wrapper_entry_point, \ wrapper_environment, run_build, run_command @@ -40,6 +42,9 @@ COMPILER_WRAPPER_CXX = 'analyze-c++' ENVIRONMENT_KEY = 'ANALYZE_BUILD' +CTU_FUNCTION_MAP_FILENAME = 'externalFnMap.txt' +CTU_TEMP_FNMAP_FOLDER = 'tmpExternalFnMaps' + @command_entry_point def scan_build(): @@ -156,23 +161,69 @@ def direct_args(args): 'output_failures': args.output_failures, 'direct_args': direct_args(args), 'force_debug': args.force_debug, - 'excludes': args.excludes + 'excludes': args.excludes, + 'ctu_collect': + args.ctu_collect if hasattr(args, 'ctu_collect') else False, + 'ctu_analyze': + args.ctu_analyze if hasattr(args, 'ctu_analyze') else False, + 'ctu_dir': + os.path.abspath(args.ctu_dir) if hasattr(args, 'ctu_dir') else '' } +def merge_ctu_func_maps(ctudir): + """ Merge individual function maps into a global one. """ + + fnmap_dir = os.path.join(ctudir, CTU_TEMP_FNMAP_FOLDER) + files = glob.glob(os.path.join(fnmap_dir, '*')) + extern_fns_map_file = os.path.join(ctudir, CTU_FUNCTION_MAP_FILENAME) + mangled_to_asts = {} + for filename in files: + with open(filename, 'rb') as in_file: + for line in in_file: + mangled_name, ast_file = line.strip().split(' ') + if mangled_name not in mangled_to_asts: + mangled_to_asts[mangled_name] = {ast_file} + else: + mangled_to_asts[mangled_name].add(ast_file) + with open(extern_fns_map_file, 'wb') as out_file: + for mangled_name, ast_files in mangled_to_asts.iteritems(): + if len(ast_files) == 1: + out_file.write('%s %s\n' % (mangled_name, ast_files.pop())) + shutil.rmtree(fnmap_dir, ignore_errors=True) + + def run_analyzer_parallel(compilations, args): """ Runs the analyzer against the given compilations. """ + def run_parallel_with_consts(compilations, consts): + """ Run one phase of an analyzer run. """ + + parameters = (dict(compilation.as_dict(), **consts) + for compilation in compilations) + # when verbose output requested execute sequentially + pool = multiprocessing.Pool(1 if args.verbose > 2 else None) + for current in pool.imap_unordered(run, parameters): + logging_analyzer_output(current) + pool.close() + pool.join() + logging.debug('run analyzer against compilation database') consts = analyze_parameters(args) - parameters = (dict(compilation.as_dict(), **consts) - for compilation in compilations) - # when verbose output requested execute sequentially - pool = multiprocessing.Pool(1 if args.verbose > 2 else None) - for current in pool.imap_unordered(run, parameters): - logging_analyzer_output(current) - pool.close() - pool.join() + if consts['ctu_collect']: + shutil.rmtree(consts['ctu_dir'], ignore_errors=True) + if consts['ctu_collect'] and consts['ctu_analyze']: + consts['ctu_analyze'] = False + run_parallel_with_consts(compilations, consts) + merge_ctu_func_maps(consts['ctu_dir']) + consts['ctu_collect'] = False + consts['ctu_analyze'] = True + run_parallel_with_consts(compilations, consts) + shutil.rmtree(consts['ctu_dir'], ignore_errors=True) + else: + run_parallel_with_consts(compilations, consts) + if consts['ctu_collect']: + merge_ctu_func_maps(consts['ctu_dir']) def setup_environment(args): @@ -275,7 +326,8 @@ def wrapper(*args, **kwargs): 'force_debug', # kill non debug macros 'output_dir', # where generated report files shall go 'output_format', # it's 'plist' or 'html' or both - 'output_failures']) # generate crash reports or not + 'output_failures', # generate crash reports or not + 'ctu_collect', 'ctu_analyze', 'ctu_dir']) # ctu control options def run(opts): """ Entry point to run (or not) static analyzer against a single entry of the compilation database. @@ -395,8 +447,101 @@ def target(): return result +@require(['clang', 'directory', 'flags', 'direct_args', 'source', 'ctu_dir']) +def ctu_collect_phase(opts): + """ Preprocess source by generating all data needed by CTU analysis. """ + + def get_triple_arch(): + """Returns the architecture part of the target triple for the current + compilation command. """ + + cwd = opts['directory'] + cmd = get_arguments([opts['clang'], '--analyze'] + + opts['direct_args'] + opts['flags'] + + opts['source'], + cwd) + arch = "" + i = 0 + while i < len(cmd) and cmd[i] != "-triple": + i += 1 + if i < (len(cmd) - 1): + arch = cmd[i + 1].split("-")[0] + return arch + + def generate_ast(triple_arch): + """ Generates ASTs for the current compilation command. """ + + args = opts['direct_args'] + opts['flags'] + ast_joined_path = os.path.join(opts['ctu_dir'], 'ast', triple_arch, + os.path.realpath(opts['source'])[1:] + + '.ast') + ast_path = os.path.abspath(ast_joined_path) + try: + os.makedirs(os.path.dirname(ast_path)) + except OSError: + if os.path.isdir(os.path.dirname(ast_path)): + pass + else: + raise + ast_command = [opts['clang'], '-emit-ast'] + ast_command.extend(args) + ast_command.append('-w') + ast_command.append(opts['source']) + ast_command.append('-o') + ast_command.append(ast_path) + logging.debug('Generating AST using %s', ' '.join(ast_command)) + subprocess.call(ast_command, cwd=opts['directory'], shell=False) + + def map_functions(triple_arch): + """ Generate function map file for the current source. """ + + args = opts['direct_args'] + opts['flags'] + funcmap_command = [os.path.join(os.path.dirname(opts['clang']), + 'clang-func-mapping')] + funcmap_command.append(opts['source']) + funcmap_command.append('--') + funcmap_command.extend(args) + logging.debug("Generating function map using %s", + ' '.join(funcmap_command)) + fn_out = subprocess.check_output(funcmap_command, + cwd=opts['directory'], + shell=False) + output = [] + fn_list = fn_out.splitlines() + for fn_txt in fn_list: + dpos = fn_txt.find(" ") + mangled_name = fn_txt[0:dpos] + path = fn_txt[dpos + 1:] + ast_path = os.path.join("ast", triple_arch, path[1:] + ".ast") + output.append(mangled_name + "@" + triple_arch + " " + ast_path) + extern_fns_map_folder = os.path.join(opts['ctu_dir'], + CTU_TEMP_FNMAP_FOLDER) + if output: + with tempfile.NamedTemporaryFile(dir=extern_fns_map_folder, + delete=False) as out_file: + out_file.write("\n".join(output) + "\n") + + triple_arch = get_triple_arch() + generate_ast(triple_arch) + map_functions(triple_arch) + + +@require(['ctu_collect', 'ctu_analyze', 'ctu_dir']) +def dispatch_ctu(opts, continuation=run_analyzer): + """ Execute only one phase of 2 phases of CTU if needed. """ + + if opts['ctu_collect'] or opts['ctu_analyze']: + if opts['ctu_collect']: + return ctu_collect_phase(opts) + if opts['ctu_analyze']: + opts['direct_args'].append('ctu-dir=' + opts['ctu_dir']) + opts['direct_args'].append('reanalyze-ctu-visited=true') + + return continuation(opts) + + @require(['flags', 'force_debug']) -def filter_debug_flags(opts, continuation=run_analyzer): +def filter_debug_flags(opts, continuation=dispatch_ctu): """ Filter out nondebug macros when requested. """ if opts.pop('force_debug'): diff --git a/libscanbuild/arguments.py b/libscanbuild/arguments.py index 8171335..0c0c414 100644 --- a/libscanbuild/arguments.py +++ b/libscanbuild/arguments.py @@ -98,6 +98,11 @@ def normalize_args_for_analyze(args, from_build_command): # add cdb parameter invisibly to make report module working. args.cdb = 'compile_commands.json' + if not from_build_command: + if args.ctu and not args.ctu_collect and not args.ctu_analyze: + args.ctu_collect = True + args.ctu_analyze = True + def validate_args_for_analyze(parser, args, from_build_command): """ Command line parsing is done by the argparse module, but semantic @@ -121,6 +126,10 @@ def validate_args_for_analyze(parser, args, from_build_command): parser.error(message='missing build command') elif not from_build_command and not os.path.exists(args.cdb): parser.error(message='compilation database is missing') + if not from_build_command: + if args.ctu_analyze and not args.ctu_collect: + if not os.path.exists(args.ctu_dir): + parser.error(message='missing CTU directory') def create_intercept_parser(): @@ -331,6 +340,32 @@ def create_analyze_parser(from_build_command): if from_build_command: parser.add_argument( dest='build', nargs=argparse.REMAINDER, help="""Command to run.""") + else: + ctu = parser.add_argument_group('cross translation unit analysis') + ctu.add_argument( + '--ctu', + action='store_true', + help="""Perform ctu analysis (both collect and analyze phases) + using default for temporary output. + At the end of the analysis, the temporary directory is removed.""") + ctu.add_argument( + '--ctu-dir', + metavar='', + default='ctu-dir', + help="""Defines the temporary directory used between ctu + phases.""") + ctu.add_argument( + '--ctu-collect-only', + action='store_true', + dest='ctu_collect', + help="""Do not perform the analyis phase, only the 1st collect + phase. Keep for further use.""") + ctu.add_argument( + '--ctu-analyze-only', + action='store_true', + dest='ctu_analyze', + help="""Perform only the 2nd analysis phase. should be + present and will not be removed after analysis.""") return parser From 665b2050e3b2466a7e3ee74101e380b540c6ac48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Gera?= Date: Thu, 1 Jun 2017 14:13:28 +0200 Subject: [PATCH 02/23] Handle filenames with spaces --- libscanbuild/analyze.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libscanbuild/analyze.py b/libscanbuild/analyze.py index 0f3be09..1d7049b 100644 --- a/libscanbuild/analyze.py +++ b/libscanbuild/analyze.py @@ -181,7 +181,7 @@ def merge_ctu_func_maps(ctudir): for filename in files: with open(filename, 'rb') as in_file: for line in in_file: - mangled_name, ast_file = line.strip().split(' ') + mangled_name, ast_file = line.strip().split(' ', 1) if mangled_name not in mangled_to_asts: mangled_to_asts[mangled_name] = {ast_file} else: From 4bcb040a246e3b4391cde83b955d2b5e5e50a0ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Gera?= Date: Thu, 1 Jun 2017 16:45:02 +0200 Subject: [PATCH 03/23] Fixing parameter related bugs --- libscanbuild/analyze.py | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/libscanbuild/analyze.py b/libscanbuild/analyze.py index 1d7049b..599a24b 100644 --- a/libscanbuild/analyze.py +++ b/libscanbuild/analyze.py @@ -102,20 +102,21 @@ def need_analyzer(args): return len(args) and not re.search('configure|autogen', args[0]) +def prefix_with(constant, pieces): + """ From a sequence create another sequence where every second element + is from the original sequence and the odd elements are the prefix. + + eg.: prefix_with(0, [1,2,3]) creates [0, 1, 0, 2, 0, 3] """ + + return [elem for piece in pieces for elem in [constant, piece]] + + def analyze_parameters(args): """ Mapping between the command line parameters and the analyzer run method. The run method works with a plain dictionary, while the command line parameters are in a named tuple. The keys are very similar, and some values are preprocessed. """ - def prefix_with(constant, pieces): - """ From a sequence create another sequence where every second element - is from the original sequence and the odd elements are the prefix. - - eg.: prefix_with(0, [1,2,3]) creates [0, 1, 0, 2, 0, 3] """ - - return [elem for piece in pieces for elem in [constant, piece]] - def direct_args(args): """ A group of command line arguments can mapped to command line arguments of the analyzer. """ @@ -212,13 +213,15 @@ def run_parallel_with_consts(compilations, consts): consts = analyze_parameters(args) if consts['ctu_collect']: shutil.rmtree(consts['ctu_dir'], ignore_errors=True) + os.makedirs(os.path.join(consts['ctu_dir'], CTU_TEMP_FNMAP_FOLDER)) if consts['ctu_collect'] and consts['ctu_analyze']: + compilation_list = list(compilations) consts['ctu_analyze'] = False - run_parallel_with_consts(compilations, consts) + run_parallel_with_consts(compilation_list, consts) merge_ctu_func_maps(consts['ctu_dir']) consts['ctu_collect'] = False consts['ctu_analyze'] = True - run_parallel_with_consts(compilations, consts) + run_parallel_with_consts(compilation_list, consts) shutil.rmtree(consts['ctu_dir'], ignore_errors=True) else: run_parallel_with_consts(compilations, consts) @@ -458,7 +461,7 @@ def get_triple_arch(): cwd = opts['directory'] cmd = get_arguments([opts['clang'], '--analyze'] + opts['direct_args'] + opts['flags'] + - opts['source'], + [opts['source']], cwd) arch = "" i = 0 @@ -489,7 +492,7 @@ def generate_ast(triple_arch): ast_command.append(opts['source']) ast_command.append('-o') ast_command.append(ast_path) - logging.debug('Generating AST using %s', ' '.join(ast_command)) + logging.debug("Generating AST using '%s'", ast_command) subprocess.call(ast_command, cwd=opts['directory'], shell=False) def map_functions(triple_arch): @@ -501,8 +504,7 @@ def map_functions(triple_arch): funcmap_command.append(opts['source']) funcmap_command.append('--') funcmap_command.extend(args) - logging.debug("Generating function map using %s", - ' '.join(funcmap_command)) + logging.debug("Generating function map using '%s'", funcmap_command) fn_out = subprocess.check_output(funcmap_command, cwd=opts['directory'], shell=False) @@ -534,8 +536,11 @@ def dispatch_ctu(opts, continuation=run_analyzer): if opts['ctu_collect']: return ctu_collect_phase(opts) if opts['ctu_analyze']: - opts['direct_args'].append('ctu-dir=' + opts['ctu_dir']) - opts['direct_args'].append('reanalyze-ctu-visited=true') + ctu_options = ['ctu-dir=' + opts['ctu_dir'], + 'reanalyze-ctu-visited=true'] + analyzer_options = prefix_with('-analyzer-config', ctu_options) + direct_options = prefix_with('-Xclang', analyzer_options) + opts['direct_args'].extend(direct_options) return continuation(opts) From 7a10e08d5aedc7453768763428fb9e8acdf090b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Gera?= Date: Thu, 1 Jun 2017 19:41:17 +0200 Subject: [PATCH 04/23] Better parameter for adressing analyzer directly --- libscanbuild/analyze.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libscanbuild/analyze.py b/libscanbuild/analyze.py index 599a24b..4aa292c 100644 --- a/libscanbuild/analyze.py +++ b/libscanbuild/analyze.py @@ -539,7 +539,7 @@ def dispatch_ctu(opts, continuation=run_analyzer): ctu_options = ['ctu-dir=' + opts['ctu_dir'], 'reanalyze-ctu-visited=true'] analyzer_options = prefix_with('-analyzer-config', ctu_options) - direct_options = prefix_with('-Xclang', analyzer_options) + direct_options = prefix_with('-Xanalyzer', analyzer_options) opts['direct_args'].extend(direct_options) return continuation(opts) From 41302f02112441491f80574e670b7c48c2cdff5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Gera?= Date: Thu, 1 Jun 2017 19:57:37 +0200 Subject: [PATCH 05/23] Add plist-multi-file output format --- libscanbuild/analyze.py | 6 ++++-- libscanbuild/arguments.py | 9 +++++++++ tests/functional/cases/report/report_file_format.fts | 1 + 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/libscanbuild/analyze.py b/libscanbuild/analyze.py index 4aa292c..33d00af 100644 --- a/libscanbuild/analyze.py +++ b/libscanbuild/analyze.py @@ -328,7 +328,7 @@ def wrapper(*args, **kwargs): 'excludes', # list of directories 'force_debug', # kill non debug macros 'output_dir', # where generated report files shall go - 'output_format', # it's 'plist' or 'html' or both + 'output_format', # it's 'plist', 'html', both or plist-multi-file 'output_failures', # generate crash reports or not 'ctu_collect', 'ctu_analyze', 'ctu_dir']) # ctu control options def run(opts): @@ -422,7 +422,9 @@ def run_analyzer(opts, continuation=report_failure): def target(): """ Creates output file name for reports. """ - if opts['output_format'] in {'plist', 'plist-html'}: + if opts['output_format'] in {'plist', + 'plist-html', + 'plist-multi-file'}: (handle, name) = tempfile.mkstemp(prefix='report-', suffix='.plist', dir=opts['output_dir']) diff --git a/libscanbuild/arguments.py b/libscanbuild/arguments.py index 0c0c414..fdcd0e1 100644 --- a/libscanbuild/arguments.py +++ b/libscanbuild/arguments.py @@ -226,6 +226,15 @@ def create_analyze_parser(from_build_command): default='html', action='store_const', help="""Cause the results as a set of .html and .plist files.""") + format_group.add_argument( + '--plist-multi-file', + '-plist-multi-file', + dest='output_format', + const='plist-multi-file', + default='html', + action='store_const', + help="""Cause the results as a set of .plist files with extra + information on related files.""") advanced = parser.add_argument_group('advanced options') advanced.add_argument( diff --git a/tests/functional/cases/report/report_file_format.fts b/tests/functional/cases/report/report_file_format.fts index c937ff0..e8928e5 100644 --- a/tests/functional/cases/report/report_file_format.fts +++ b/tests/functional/cases/report/report_file_format.fts @@ -5,6 +5,7 @@ # RUN: cd %T/report_file_format; %{scan-build} --output . --keep-empty --plist ./run.sh | ./check_plist.sh # RUN: cd %T/report_file_format; %{scan-build} --output . --keep-empty --plist-html ./run.sh | ./check_html.sh # RUN: cd %T/report_file_format; %{scan-build} --output . --keep-empty --plist-html ./run.sh | ./check_plist.sh +# RUN: cd %T/report_file_format; %{scan-build} --output . --keep-empty --plist-multi-file ./run.sh | ./check_plist.sh set -o errexit set -o nounset From fffd8ec4e1dda54723324ce5448b27607b2598c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Gera?= Date: Fri, 2 Jun 2017 20:28:54 +0200 Subject: [PATCH 06/23] Add CTU test case --- .../functional/cases/analyze/analyze_ctu.fts | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 tests/functional/cases/analyze/analyze_ctu.fts diff --git a/tests/functional/cases/analyze/analyze_ctu.fts b/tests/functional/cases/analyze/analyze_ctu.fts new file mode 100644 index 0000000..679fc8c --- /dev/null +++ b/tests/functional/cases/analyze/analyze_ctu.fts @@ -0,0 +1,87 @@ +#!/usr/bin/env bash + +# RUN: bash %s %T/ctu +# RUN: cd %T/ctu; %{analyze-build} -o . --cdb buildlog.json --ctu --plist-multi-file | ./check.sh + +set -o errexit +set -o nounset +set -o xtrace + +# the test creates a subdirectory inside output dir. +# +# ${root_dir} +# ├── buildlog.json +# ├── check.sh +# └── src +# ├── lib.c +# └── main.c + +root_dir=$1 +mkdir -p "${root_dir}/src" + +cat > "${root_dir}/src/lib.c" << EOF +int bad_guy(int * i) +{ + *i = 9; + return *i; +} +EOF + +cat > "${root_dir}/src/main.c" << EOF +int bad_guy(int * i); + +void bad_guy_test() +{ + int * ptr = 0; + bad_guy(ptr); +} +EOF + +cat > "${root_dir}/buildlog.json" << EOF +[ + { + "directory": "${root_dir}", + "file": "${root_dir}/src/lib.c", + "command": "cc -c ./src/lib.c -o ./src/lib.o" + }, + { + "directory": "${root_dir}", + "file": "${root_dir}/src/main.c", + "command": "cc -c ./src/main.c -o ./src/main.o" + } +] +EOF + +checker_file="${root_dir}/check.sh" +cat > ${checker_file} << EOF +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o xtrace + +out=\$(sort | uniq) +runs=\$(echo "\$out" | grep "exec command" | sort | uniq) +analyze_out=\$(echo "\$out" | grep "logging_analyzer_output" | sort | uniq) + +assert_present() { + local pattern="\$1"; + local message="\$2"; + + if [ \$(echo "\$runs" | grep -- "\$pattern" | wc -l) -eq 0 ]; then + echo "\$message" && false; + fi +} + +assert_present "ctu-dir=" "using CTU mode" +assert_present "reanalyze-ctu-visited=true" "using CTU reanalyze mode" + +if [ \$(echo "\$analyze_out" | grep -- "Dereference of null pointer" | wc -l) -ne 1 ]; then + echo "cross translation unit problem found" && false; +fi + +if [ \$(find ${root_dir} -type d -name "scan-build-*" | wc -l) -ne 1 ]; then + echo "CTU report was generated" && false; +fi +EOF +chmod +x ${checker_file} From 0bc1a81ba574d1f884cb052952e5d39f7811f052 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Gera?= Date: Wed, 14 Jun 2017 15:03:35 +0200 Subject: [PATCH 07/23] Make it python 3 compatible --- libscanbuild/analyze.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/libscanbuild/analyze.py b/libscanbuild/analyze.py index 9b3843e..4e4068e 100644 --- a/libscanbuild/analyze.py +++ b/libscanbuild/analyze.py @@ -180,15 +180,15 @@ def merge_ctu_func_maps(ctudir): extern_fns_map_file = os.path.join(ctudir, CTU_FUNCTION_MAP_FILENAME) mangled_to_asts = {} for filename in files: - with open(filename, 'rb') as in_file: + with open(filename, 'r') as in_file: for line in in_file: mangled_name, ast_file = line.strip().split(' ', 1) if mangled_name not in mangled_to_asts: mangled_to_asts[mangled_name] = {ast_file} else: mangled_to_asts[mangled_name].add(ast_file) - with open(extern_fns_map_file, 'wb') as out_file: - for mangled_name, ast_files in mangled_to_asts.iteritems(): + with open(extern_fns_map_file, 'w') as out_file: + for mangled_name, ast_files in mangled_to_asts.items(): if len(ast_files) == 1: out_file.write('%s %s\n' % (mangled_name, ast_files.pop())) shutil.rmtree(fnmap_dir, ignore_errors=True) @@ -425,8 +425,7 @@ def target(): if opts['output_format'] in { 'plist', 'plist-html', - 'plist-multi-file' - }: + 'plist-multi-file'}: (handle, name) = tempfile.mkstemp(prefix='report-', suffix='.plist', dir=opts['output_dir']) @@ -511,7 +510,8 @@ def map_functions(triple_arch): logging.debug("Generating function map using '%s'", funcmap_command) fn_out = subprocess.check_output(funcmap_command, cwd=opts['directory'], - shell=False) + shell=False, + universal_newlines=True) output = [] fn_list = fn_out.splitlines() for fn_txt in fn_list: @@ -523,7 +523,8 @@ def map_functions(triple_arch): extern_fns_map_folder = os.path.join(opts['ctu_dir'], CTU_TEMP_FNMAP_FOLDER) if output: - with tempfile.NamedTemporaryFile(dir=extern_fns_map_folder, + with tempfile.NamedTemporaryFile(mode='w', + dir=extern_fns_map_folder, delete=False) as out_file: out_file.write("\n".join(output) + "\n") From f5cc7667f898a4fb10216e2bf956f61cf6b26146 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Gera?= Date: Fri, 16 Jun 2017 13:54:28 +0200 Subject: [PATCH 08/23] More robust help and options --- libscanbuild/arguments.py | 42 +++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/libscanbuild/arguments.py b/libscanbuild/arguments.py index fdcd0e1..b1e78d0 100644 --- a/libscanbuild/arguments.py +++ b/libscanbuild/arguments.py @@ -98,11 +98,6 @@ def normalize_args_for_analyze(args, from_build_command): # add cdb parameter invisibly to make report module working. args.cdb = 'compile_commands.json' - if not from_build_command: - if args.ctu and not args.ctu_collect and not args.ctu_analyze: - args.ctu_collect = True - args.ctu_analyze = True - def validate_args_for_analyze(parser, args, from_build_command): """ Command line parsing is done by the argparse module, but semantic @@ -126,10 +121,11 @@ def validate_args_for_analyze(parser, args, from_build_command): parser.error(message='missing build command') elif not from_build_command and not os.path.exists(args.cdb): parser.error(message='compilation database is missing') - if not from_build_command: - if args.ctu_analyze and not args.ctu_collect: - if not os.path.exists(args.ctu_dir): - parser.error(message='missing CTU directory') + + # If it is CTU analyze_only, the input directory should exist + if not from_build_command and args.ctu[1] and not args.ctu[0] \ + and not os.path.exists(args.ctu_dir): + parser.error(message='missing CTU directory') def create_intercept_parser(): @@ -351,11 +347,13 @@ def create_analyze_parser(from_build_command): dest='build', nargs=argparse.REMAINDER, help="""Command to run.""") else: ctu = parser.add_argument_group('cross translation unit analysis') - ctu.add_argument( + ctu_mutex_group = ctu.add_mutually_exclusive_group() + ctu_mutex_group.add_argument( '--ctu', - action='store_true', - help="""Perform ctu analysis (both collect and analyze phases) - using default for temporary output. + action='store_const', const=(True, True), + dest='ctu', + help="""Perform cross translation unit (ctu) analysis (both collect + and analyze phases) using default for temporary output. At the end of the analysis, the temporary directory is removed.""") ctu.add_argument( '--ctu-dir', @@ -363,17 +361,17 @@ def create_analyze_parser(from_build_command): default='ctu-dir', help="""Defines the temporary directory used between ctu phases.""") - ctu.add_argument( + ctu_mutex_group.add_argument( '--ctu-collect-only', - action='store_true', - dest='ctu_collect', - help="""Do not perform the analyis phase, only the 1st collect - phase. Keep for further use.""") - ctu.add_argument( + action='store_const', const=(True, False), + dest='ctu', + help="""Perform only the collect phase of ctu. + Keep for further use.""") + ctu_mutex_group.add_argument( '--ctu-analyze-only', - action='store_true', - dest='ctu_analyze', - help="""Perform only the 2nd analysis phase. should be + action='store_const', const=(False, True), + dest='ctu', + help="""Perform only the analyze phase of ctu. should be present and will not be removed after analysis.""") return parser From 8475aea9721341c22b125aa48304064a62b44731 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Gera?= Date: Fri, 16 Jun 2017 14:05:09 +0200 Subject: [PATCH 09/23] Better naming for argparse variable --- libscanbuild/arguments.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/libscanbuild/arguments.py b/libscanbuild/arguments.py index b1e78d0..e3860ed 100644 --- a/libscanbuild/arguments.py +++ b/libscanbuild/arguments.py @@ -123,7 +123,8 @@ def validate_args_for_analyze(parser, args, from_build_command): parser.error(message='compilation database is missing') # If it is CTU analyze_only, the input directory should exist - if not from_build_command and args.ctu[1] and not args.ctu[0] \ + if not from_build_command \ + and args.ctu_phases[1] and not args.ctu_phases[0] \ and not os.path.exists(args.ctu_dir): parser.error(message='missing CTU directory') @@ -351,7 +352,7 @@ def create_analyze_parser(from_build_command): ctu_mutex_group.add_argument( '--ctu', action='store_const', const=(True, True), - dest='ctu', + dest='ctu_phases', help="""Perform cross translation unit (ctu) analysis (both collect and analyze phases) using default for temporary output. At the end of the analysis, the temporary directory is removed.""") @@ -364,13 +365,13 @@ def create_analyze_parser(from_build_command): ctu_mutex_group.add_argument( '--ctu-collect-only', action='store_const', const=(True, False), - dest='ctu', + dest='ctu_phases', help="""Perform only the collect phase of ctu. Keep for further use.""") ctu_mutex_group.add_argument( '--ctu-analyze-only', action='store_const', const=(False, True), - dest='ctu', + dest='ctu_phases', help="""Perform only the analyze phase of ctu. should be present and will not be removed after analysis.""") return parser From ef7ea795ecc03c09b848cecf473d9b9857e19886 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Gera?= Date: Fri, 16 Jun 2017 17:14:20 +0200 Subject: [PATCH 10/23] Use namedtuple for ctu configuration --- libscanbuild/__init__.py | 2 ++ libscanbuild/analyze.py | 72 ++++++++++++++++++++++++--------------- libscanbuild/arguments.py | 20 ++++++----- 3 files changed, 59 insertions(+), 35 deletions(-) diff --git a/libscanbuild/__init__.py b/libscanbuild/__init__.py index 97f3d32..1a7e2be 100644 --- a/libscanbuild/__init__.py +++ b/libscanbuild/__init__.py @@ -19,6 +19,8 @@ Execution = collections.namedtuple('Execution', ['pid', 'cwd', 'cmd']) +CtuConfig = collections.namedtuple('CtuConfig', ['collect', 'analyze', 'dir']) + def shell_split(string): """ Takes a command string and returns as a list. """ diff --git a/libscanbuild/analyze.py b/libscanbuild/analyze.py index 4e4068e..928b90f 100644 --- a/libscanbuild/analyze.py +++ b/libscanbuild/analyze.py @@ -27,7 +27,7 @@ import glob from libscanbuild import command_entry_point, wrapper_entry_point, \ - wrapper_environment, run_build, run_command + wrapper_environment, run_build, run_command, CtuConfig from libscanbuild.arguments import parse_args_for_scan_build, \ parse_args_for_analyze_build from libscanbuild.intercept import capture @@ -155,6 +155,16 @@ def direct_args(args): return prefix_with('-Xclang', result) + def get_ctu_config(args): + """ CTU configuration is created from the chosen phases and dir """ + + return ( + CtuConfig(collect=args.ctu_phases.collect, + analyze=args.ctu_phases.analyze, + dir=args.ctu_dir) + if hasattr(args, 'ctu_phases') and hasattr(args.ctu_phases, 'dir') + else CtuConfig(collect=False, analyze=False, dir='')) + return { 'clang': args.clang, 'output_dir': args.output, @@ -163,12 +173,7 @@ def direct_args(args): 'direct_args': direct_args(args), 'force_debug': args.force_debug, 'excludes': args.excludes, - 'ctu_collect': - args.ctu_collect if hasattr(args, 'ctu_collect') else False, - 'ctu_analyze': - args.ctu_analyze if hasattr(args, 'ctu_analyze') else False, - 'ctu_dir': - os.path.abspath(args.ctu_dir) if hasattr(args, 'ctu_dir') else '' + 'ctu': get_ctu_config(args) } @@ -211,22 +216,26 @@ def run_parallel_with_consts(compilations, consts): logging.debug('run analyzer against compilation database') consts = analyze_parameters(args) - if consts['ctu_collect']: - shutil.rmtree(consts['ctu_dir'], ignore_errors=True) - os.makedirs(os.path.join(consts['ctu_dir'], CTU_TEMP_FNMAP_FOLDER)) - if consts['ctu_collect'] and consts['ctu_analyze']: + ctu_config = consts['ctu'] + if ctu_config.collect: + shutil.rmtree(ctu_config.dir, ignore_errors=True) + os.makedirs(os.path.join(ctu_config.dir, CTU_TEMP_FNMAP_FOLDER)) + if ctu_config.collect and ctu_config.analyze: compilation_list = list(compilations) - consts['ctu_analyze'] = False + consts['ctu'] = CtuConfig(collect=True, + analyze=False, + dir=ctu_config.dir) run_parallel_with_consts(compilation_list, consts) - merge_ctu_func_maps(consts['ctu_dir']) - consts['ctu_collect'] = False - consts['ctu_analyze'] = True + merge_ctu_func_maps(ctu_config.dir) + consts['ctu'] = CtuConfig(collect=False, + analyze=True, + dir=ctu_config.dir) run_parallel_with_consts(compilation_list, consts) - shutil.rmtree(consts['ctu_dir'], ignore_errors=True) + shutil.rmtree(ctu_config.dir, ignore_errors=True) else: run_parallel_with_consts(compilations, consts) - if consts['ctu_collect']: - merge_ctu_func_maps(consts['ctu_dir']) + if ctu_config.collect: + merge_ctu_func_maps(ctu_config.dir) def setup_environment(args): @@ -330,7 +339,7 @@ def wrapper(*args, **kwargs): 'output_dir', # where generated report files shall go 'output_format', # it's 'plist', 'html', both or plist-multi-file 'output_failures', # generate crash reports or not - 'ctu_collect', 'ctu_analyze', 'ctu_dir']) # ctu control options + 'ctu']) # ctu control options def run(opts): """ Entry point to run (or not) static analyzer against a single entry of the compilation database. @@ -453,7 +462,7 @@ def target(): return result -@require(['clang', 'directory', 'flags', 'direct_args', 'source', 'ctu_dir']) +@require(['clang', 'directory', 'flags', 'direct_args', 'source', 'ctu']) def ctu_collect_phase(opts): """ Preprocess source by generating all data needed by CTU analysis. """ @@ -478,7 +487,7 @@ def generate_ast(triple_arch): """ Generates ASTs for the current compilation command. """ args = opts['direct_args'] + opts['flags'] - ast_joined_path = os.path.join(opts['ctu_dir'], 'ast', triple_arch, + ast_joined_path = os.path.join(opts['ctu'].dir, 'ast', triple_arch, os.path.realpath(opts['source'])[1:] + '.ast') ast_path = os.path.abspath(ast_joined_path) @@ -520,7 +529,7 @@ def map_functions(triple_arch): path = fn_txt[dpos + 1:] ast_path = os.path.join("ast", triple_arch, path[1:] + ".ast") output.append(mangled_name + "@" + triple_arch + " " + ast_path) - extern_fns_map_folder = os.path.join(opts['ctu_dir'], + extern_fns_map_folder = os.path.join(opts['ctu'].dir, CTU_TEMP_FNMAP_FOLDER) if output: with tempfile.NamedTemporaryFile(mode='w', @@ -533,15 +542,24 @@ def map_functions(triple_arch): map_functions(triple_arch) -@require(['ctu_collect', 'ctu_analyze', 'ctu_dir']) +@require(['ctu']) def dispatch_ctu(opts, continuation=run_analyzer): """ Execute only one phase of 2 phases of CTU if needed. """ - if opts['ctu_collect'] or opts['ctu_analyze']: - if opts['ctu_collect']: + ctu_config = opts['ctu'] + # Recover namedtuple from json when coming from analyze_cc + if not hasattr(ctu_config, 'collect'): + ctu_config = CtuConfig(collect=ctu_config[0], + analyze=ctu_config[1], + dir=ctu_config[2]) + opts['ctu'] = ctu_config + + if ctu_config.collect or ctu_config.analyze: + assert ctu_config.collect != ctu_config.analyze + if ctu_config.collect: return ctu_collect_phase(opts) - if opts['ctu_analyze']: - ctu_options = ['ctu-dir=' + opts['ctu_dir'], + if ctu_config.analyze: + ctu_options = ['ctu-dir=' + ctu_config.dir, 'reanalyze-ctu-visited=true'] analyzer_options = prefix_with('-analyzer-config', ctu_options) direct_options = prefix_with('-Xanalyzer', analyzer_options) diff --git a/libscanbuild/arguments.py b/libscanbuild/arguments.py index e3860ed..c6c0230 100644 --- a/libscanbuild/arguments.py +++ b/libscanbuild/arguments.py @@ -18,7 +18,7 @@ import argparse import logging import tempfile -from libscanbuild import reconfigure_logging +from libscanbuild import reconfigure_logging, CtuConfig from libscanbuild.clang import get_checkers __all__ = ['parse_args_for_intercept_build', 'parse_args_for_analyze_build', @@ -123,10 +123,11 @@ def validate_args_for_analyze(parser, args, from_build_command): parser.error(message='compilation database is missing') # If it is CTU analyze_only, the input directory should exist - if not from_build_command \ - and args.ctu_phases[1] and not args.ctu_phases[0] \ - and not os.path.exists(args.ctu_dir): - parser.error(message='missing CTU directory') + if not from_build_command and hasattr(args, 'ctu_phases') \ + and hasattr(args.ctu_phases, 'dir'): + if args.ctu_phases.analyze and not args.ctu_phases.collect \ + and not os.path.exists(args.ctu_dir): + parser.error(message='missing CTU directory') def create_intercept_parser(): @@ -351,7 +352,8 @@ def create_analyze_parser(from_build_command): ctu_mutex_group = ctu.add_mutually_exclusive_group() ctu_mutex_group.add_argument( '--ctu', - action='store_const', const=(True, True), + action='store_const', + const=CtuConfig(collect=True, analyze=True, dir=''), dest='ctu_phases', help="""Perform cross translation unit (ctu) analysis (both collect and analyze phases) using default for temporary output. @@ -364,13 +366,15 @@ def create_analyze_parser(from_build_command): phases.""") ctu_mutex_group.add_argument( '--ctu-collect-only', - action='store_const', const=(True, False), + action='store_const', + const=CtuConfig(collect=True, analyze=False, dir=''), dest='ctu_phases', help="""Perform only the collect phase of ctu. Keep for further use.""") ctu_mutex_group.add_argument( '--ctu-analyze-only', - action='store_const', const=(False, True), + action='store_const', + const=CtuConfig(collect=False, analyze=True, dir=''), dest='ctu_phases', help="""Perform only the analyze phase of ctu. should be present and will not be removed after analysis.""") From 9a81c51c407b267316c6225fd7a61c991bf710ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Gera?= Date: Fri, 16 Jun 2017 18:05:15 +0200 Subject: [PATCH 11/23] prefix_with unittest --- tests/unit/test_analyze.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/unit/test_analyze.py b/tests/unit/test_analyze.py index 919da07..7c09243 100644 --- a/tests/unit/test_analyze.py +++ b/tests/unit/test_analyze.py @@ -337,3 +337,14 @@ def test_directory_name_comparison(self): sut.report_directory(tmp_dir, False) as report_dir3: self.assertLess(report_dir1, report_dir2) self.assertLess(report_dir2, report_dir3) + + +class PrefixWithTest(unittest.TestCase): + + def test_gives_empty_on_empty(self): + res = sut.prefix_with(0, []) + self.assertFalse(res) + + def test_interleaves_prefix(self): + res = sut.prefix_with(0, [1, 2, 3]) + self.assertListEqual([0, 1, 0, 2, 0, 3], res) From 105da910411998dd5d89e275437c2e53e783fb2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Gera?= Date: Fri, 16 Jun 2017 20:07:03 +0200 Subject: [PATCH 12/23] Refactor function map merging --- libscanbuild/analyze.py | 72 ++++++++++++++++++++++++++++++++--------- 1 file changed, 56 insertions(+), 16 deletions(-) diff --git a/libscanbuild/analyze.py b/libscanbuild/analyze.py index 928b90f..65e44fa 100644 --- a/libscanbuild/analyze.py +++ b/libscanbuild/analyze.py @@ -177,25 +177,65 @@ def get_ctu_config(args): } +def create_global_ctu_function_map(func_map_lines): + """ Takes iterator of individual function maps and creates a global map + keeping only unique names. We leave conflicting names out of CTU. + A function map contains the id of a function (mangled name) and the + originating source (the corresponding AST file) name.""" + + mangled_to_asts = {} + + for line in func_map_lines: + mangled_name, ast_file = line.strip().split(' ', 1) + # We collect all occurences of a function name into a list + if mangled_name not in mangled_to_asts: + mangled_to_asts[mangled_name] = {ast_file} + else: + mangled_to_asts[mangled_name].add(ast_file) + + mangled_ast_pairs = [] + + for mangled_name, ast_files in mangled_to_asts.items(): + if len(ast_files) == 1: + mangled_ast_pairs.append((mangled_name, ast_files.pop())) + + return mangled_ast_pairs + + def merge_ctu_func_maps(ctudir): - """ Merge individual function maps into a global one. """ + """ Merge individual function maps into a global one. + + As the collect phase runs parallel on multiple threads, all compilation + units are separately mapped into a temporary file in CTU_TEMP_FNMAP_FOLDER. + These function maps contain the mangled names of functions and the source + (AST generated from the source) which had them. + These files should be merged at the end into a global map file: + CTU_FUNCTION_MAP_FILENAME.""" + + def generate_func_map_lines(fnmap_dir): + """ Iterate over all lines of input files in random order. """ + + files = glob.glob(os.path.join(fnmap_dir, '*')) + for filename in files: + with open(filename, 'r') as in_file: + for line in in_file: + yield line + + def write_global_map(ctudir, mangled_ast_pairs): + """ Write (mangled function name, ast file) pairs into final file. """ + + extern_fns_map_file = os.path.join(ctudir, CTU_FUNCTION_MAP_FILENAME) + with open(extern_fns_map_file, 'w') as out_file: + for mangled_name, ast_file in mangled_ast_pairs: + out_file.write('%s %s\n' % (mangled_name, ast_file)) fnmap_dir = os.path.join(ctudir, CTU_TEMP_FNMAP_FOLDER) - files = glob.glob(os.path.join(fnmap_dir, '*')) - extern_fns_map_file = os.path.join(ctudir, CTU_FUNCTION_MAP_FILENAME) - mangled_to_asts = {} - for filename in files: - with open(filename, 'r') as in_file: - for line in in_file: - mangled_name, ast_file = line.strip().split(' ', 1) - if mangled_name not in mangled_to_asts: - mangled_to_asts[mangled_name] = {ast_file} - else: - mangled_to_asts[mangled_name].add(ast_file) - with open(extern_fns_map_file, 'w') as out_file: - for mangled_name, ast_files in mangled_to_asts.items(): - if len(ast_files) == 1: - out_file.write('%s %s\n' % (mangled_name, ast_files.pop())) + + func_map_lines = generate_func_map_lines(fnmap_dir) + mangled_ast_pairs = create_global_ctu_function_map(func_map_lines) + write_global_map(ctudir, mangled_ast_pairs) + + # Remove all temporary files shutil.rmtree(fnmap_dir, ignore_errors=True) From de399628fa73fdab00e3d8b1f8c00c2f6df48840 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Gera?= Date: Mon, 19 Jun 2017 10:24:20 +0200 Subject: [PATCH 13/23] Unit tests for merging CTU maps --- tests/unit/test_analyze.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/unit/test_analyze.py b/tests/unit/test_analyze.py index 7c09243..a9f7750 100644 --- a/tests/unit/test_analyze.py +++ b/tests/unit/test_analyze.py @@ -348,3 +348,39 @@ def test_gives_empty_on_empty(self): def test_interleaves_prefix(self): res = sut.prefix_with(0, [1, 2, 3]) self.assertListEqual([0, 1, 0, 2, 0, 3], res) + + +class MergeCtuMapTest(unittest.TestCase): + + def test_no_map_gives_empty(self): + pairs = sut.create_global_ctu_function_map([]) + self.assertListEqual([], pairs) + + def test_multiple_maps_merged(self): + concat_map = ['_Z1fun1i@x86_64 ast/x86_64/fun1.c.ast', + '_Z1fun2i@x86_64 ast/x86_64/fun2.c.ast', + '_Z1fun3i@x86_64 ast/x86_64/fun3.c.ast'] + pairs = sut.create_global_ctu_function_map(concat_map) + self.assertTrue(('_Z1fun1i@x86_64', 'ast/x86_64/fun1.c.ast') in pairs) + self.assertTrue(('_Z1fun2i@x86_64', 'ast/x86_64/fun2.c.ast') in pairs) + self.assertTrue(('_Z1fun3i@x86_64', 'ast/x86_64/fun3.c.ast') in pairs) + self.assertEqual(3, len(pairs)) + + def test_not_unique_func_left_out(self): + concat_map = ['_Z1fun1i@x86_64 ast/x86_64/fun1.c.ast', + '_Z1fun2i@x86_64 ast/x86_64/fun2.c.ast', + '_Z1fun1i@x86_64 ast/x86_64/fun7.c.ast'] + pairs = sut.create_global_ctu_function_map(concat_map) + self.assertFalse(('_Z1fun1i@x86_64', 'ast/x86_64/fun1.c.ast') in pairs) + self.assertFalse(('_Z1fun1i@x86_64', 'ast/x86_64/fun7.c.ast') in pairs) + self.assertTrue(('_Z1fun2i@x86_64', 'ast/x86_64/fun2.c.ast') in pairs) + self.assertEqual(1, len(pairs)) + + def test_duplicates_are_kept(self): + concat_map = ['_Z1fun1i@x86_64 ast/x86_64/fun1.c.ast', + '_Z1fun2i@x86_64 ast/x86_64/fun2.c.ast', + '_Z1fun1i@x86_64 ast/x86_64/fun1.c.ast'] + pairs = sut.create_global_ctu_function_map(concat_map) + self.assertTrue(('_Z1fun1i@x86_64', 'ast/x86_64/fun1.c.ast') in pairs) + self.assertTrue(('_Z1fun2i@x86_64', 'ast/x86_64/fun2.c.ast') in pairs) + self.assertEqual(2, len(pairs)) From b2312e7781151e007e52ed70463b1ad73b631237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Gera?= Date: Mon, 19 Jun 2017 10:55:32 +0200 Subject: [PATCH 14/23] Refactor run_analyzer_parallel to have ctu logic separated --- libscanbuild/analyze.py | 68 ++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/libscanbuild/analyze.py b/libscanbuild/analyze.py index 65e44fa..c208393 100644 --- a/libscanbuild/analyze.py +++ b/libscanbuild/analyze.py @@ -62,7 +62,7 @@ def scan_build(): exit_code, compilations = capture(args) if need_analyzer(args.build): # run the analyzer against the captured commands - run_analyzer_parallel(compilations, args) + run_analyzer_with_ctu(compilations, args) else: # run build command and analyzer with compiler wrappers environment = setup_environment(args) @@ -82,7 +82,7 @@ def analyze_build(): with report_directory(args.output, args.keep_empty) as args.output: # run the analyzer against a compilation db compilations = CompilationDatabase.load(args.cdb) - run_analyzer_parallel(compilations, args) + run_analyzer_with_ctu(compilations, args) # cover report generation and bug counting number_of_bugs = document(args) # set exit status as it was requested @@ -111,6 +111,17 @@ def prefix_with(constant, pieces): return [elem for piece in pieces for elem in [constant, piece]] +def get_ctu_config(args): + """ CTU configuration is created from the chosen phases and dir """ + + return ( + CtuConfig(collect=args.ctu_phases.collect, + analyze=args.ctu_phases.analyze, + dir=args.ctu_dir) + if hasattr(args, 'ctu_phases') and hasattr(args.ctu_phases, 'dir') + else CtuConfig(collect=False, analyze=False, dir='')) + + def analyze_parameters(args): """ Mapping between the command line parameters and the analyzer run method. The run method works with a plain dictionary, while the command @@ -155,16 +166,6 @@ def direct_args(args): return prefix_with('-Xclang', result) - def get_ctu_config(args): - """ CTU configuration is created from the chosen phases and dir """ - - return ( - CtuConfig(collect=args.ctu_phases.collect, - analyze=args.ctu_phases.analyze, - dir=args.ctu_dir) - if hasattr(args, 'ctu_phases') and hasattr(args.ctu_phases, 'dir') - else CtuConfig(collect=False, analyze=False, dir='')) - return { 'clang': args.clang, 'output_dir': args.output, @@ -242,38 +243,37 @@ def write_global_map(ctudir, mangled_ast_pairs): def run_analyzer_parallel(compilations, args): """ Runs the analyzer against the given compilations. """ - def run_parallel_with_consts(compilations, consts): - """ Run one phase of an analyzer run. """ - - parameters = (dict(compilation.as_dict(), **consts) - for compilation in compilations) - # when verbose output requested execute sequentially - pool = multiprocessing.Pool(1 if args.verbose > 2 else None) - for current in pool.imap_unordered(run, parameters): - logging_analyzer_output(current) - pool.close() - pool.join() - logging.debug('run analyzer against compilation database') consts = analyze_parameters(args) - ctu_config = consts['ctu'] + parameters = (dict(compilation.as_dict(), **consts) + for compilation in compilations) + # when verbose output requested execute sequentially + pool = multiprocessing.Pool(1 if args.verbose > 2 else None) + for current in pool.imap_unordered(run, parameters): + logging_analyzer_output(current) + pool.close() + pool.join() + + +def run_analyzer_with_ctu(compilations, args): + """ Governs multiple runs in CTU mode or runs once in normal mode. """ + + ctu_config = get_ctu_config(args) if ctu_config.collect: shutil.rmtree(ctu_config.dir, ignore_errors=True) os.makedirs(os.path.join(ctu_config.dir, CTU_TEMP_FNMAP_FOLDER)) if ctu_config.collect and ctu_config.analyze: + # compilations is a generator but we want to do 2 CTU rounds compilation_list = list(compilations) - consts['ctu'] = CtuConfig(collect=True, - analyze=False, - dir=ctu_config.dir) - run_parallel_with_consts(compilation_list, consts) + # CTU folder is coming from args.ctu_dir, so we can leave it empty + args.ctu_phases = CtuConfig(collect=True, analyze=False, dir='') + run_analyzer_parallel(compilation_list, args) merge_ctu_func_maps(ctu_config.dir) - consts['ctu'] = CtuConfig(collect=False, - analyze=True, - dir=ctu_config.dir) - run_parallel_with_consts(compilation_list, consts) + args.ctu_phases = CtuConfig(collect=False, analyze=True, dir='') + run_analyzer_parallel(compilation_list, args) shutil.rmtree(ctu_config.dir, ignore_errors=True) else: - run_parallel_with_consts(compilations, consts) + run_analyzer_parallel(compilations, args) if ctu_config.collect: merge_ctu_func_maps(ctu_config.dir) From 453c451a110e23b014cd7fbf9221969db757fc49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Gera?= Date: Mon, 19 Jun 2017 12:08:27 +0200 Subject: [PATCH 15/23] Move triple arch extraction into clang module --- libscanbuild/analyze.py | 24 +++++------------------- libscanbuild/clang.py | 16 +++++++++++++++- tests/unit/test_clang.py | 6 ++++++ 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/libscanbuild/analyze.py b/libscanbuild/analyze.py index c208393..69da942 100644 --- a/libscanbuild/analyze.py +++ b/libscanbuild/analyze.py @@ -34,7 +34,7 @@ from libscanbuild.report import document from libscanbuild.compilation import Compilation, classify_source, \ CompilationDatabase -from libscanbuild.clang import get_version, get_arguments +from libscanbuild.clang import get_version, get_arguments, get_triple_arch __all__ = ['scan_build', 'analyze_build', 'analyze_compiler_wrapper'] @@ -506,23 +506,6 @@ def target(): def ctu_collect_phase(opts): """ Preprocess source by generating all data needed by CTU analysis. """ - def get_triple_arch(): - """Returns the architecture part of the target triple for the current - compilation command. """ - - cwd = opts['directory'] - cmd = get_arguments([opts['clang'], '--analyze'] + - opts['direct_args'] + opts['flags'] + - [opts['source']], - cwd) - arch = "" - i = 0 - while i < len(cmd) and cmd[i] != "-triple": - i += 1 - if i < (len(cmd) - 1): - arch = cmd[i + 1].split("-")[0] - return arch - def generate_ast(triple_arch): """ Generates ASTs for the current compilation command. """ @@ -577,7 +560,10 @@ def map_functions(triple_arch): delete=False) as out_file: out_file.write("\n".join(output) + "\n") - triple_arch = get_triple_arch() + cwd = opts['directory'] + cmd = [opts['clang'], '--analyze'] + opts['direct_args'] + opts['flags'] \ + + [opts['source']] + triple_arch = get_triple_arch(cmd, cwd) generate_ast(triple_arch) map_functions(triple_arch) diff --git a/libscanbuild/clang.py b/libscanbuild/clang.py index 86c6c4d..fbee82b 100644 --- a/libscanbuild/clang.py +++ b/libscanbuild/clang.py @@ -11,7 +11,7 @@ import re from libscanbuild import shell_split, run_command -__all__ = ['get_version', 'get_arguments', 'get_checkers'] +__all__ = ['get_version', 'get_arguments', 'get_checkers', 'get_triple_arch'] # regex for activated checker ACTIVE_CHECKER_PATTERN = re.compile(r'^-analyzer-checker=(.*)$') @@ -151,3 +151,17 @@ def get_checkers(clang, plugins): raise Exception('Could not query Clang for available checkers.') return checkers + + +def get_triple_arch(command, cwd): + """Returns the architecture part of the target triple for the given + compilation command. """ + + cmd = get_arguments(command, cwd) + arch = "" + i = 0 + while i < len(cmd) and cmd[i] != "-triple": + i += 1 + if i < (len(cmd) - 1): + arch = cmd[i + 1].split("-")[0] + return arch diff --git a/tests/unit/test_clang.py b/tests/unit/test_clang.py index d9ec7af..9925d1a 100644 --- a/tests/unit/test_clang.py +++ b/tests/unit/test_clang.py @@ -90,3 +90,9 @@ def test_parse_checkers(self): self.assertEqual('Checker One description', result.get('checker.one')) self.assertTrue('checker.two' in result) self.assertEqual('Checker Two description', result.get('checker.two')) + + +class ClangGetTripleArchTest(unittest.TestCase): + def test_arch_is_not_empty(self): + arch = sut.get_triple_arch(['clang', '-E', '-'], '.') + self.assertTrue(len(arch) > 0) From 580a8234933cc9dfa579dc8ccdbd00ee2e037900 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Gera?= Date: Mon, 19 Jun 2017 12:23:34 +0200 Subject: [PATCH 16/23] Use built-in encoding --- libscanbuild/analyze.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/libscanbuild/analyze.py b/libscanbuild/analyze.py index 69da942..4cd2c70 100644 --- a/libscanbuild/analyze.py +++ b/libscanbuild/analyze.py @@ -514,13 +514,9 @@ def generate_ast(triple_arch): os.path.realpath(opts['source'])[1:] + '.ast') ast_path = os.path.abspath(ast_joined_path) - try: - os.makedirs(os.path.dirname(ast_path)) - except OSError: - if os.path.isdir(os.path.dirname(ast_path)): - pass - else: - raise + ast_dir = os.path.dirname(ast_path) + if not os.path.isdir(ast_dir): + os.makedirs(ast_dir) ast_command = [opts['clang'], '-emit-ast'] ast_command.extend(args) ast_command.append('-w') @@ -528,7 +524,7 @@ def generate_ast(triple_arch): ast_command.append('-o') ast_command.append(ast_path) logging.debug("Generating AST using '%s'", ast_command) - subprocess.call(ast_command, cwd=opts['directory'], shell=False) + run_command(ast_command, cwd=opts['directory']) def map_functions(triple_arch): """ Generate function map file for the current source. """ @@ -540,12 +536,8 @@ def map_functions(triple_arch): funcmap_command.append('--') funcmap_command.extend(args) logging.debug("Generating function map using '%s'", funcmap_command) - fn_out = subprocess.check_output(funcmap_command, - cwd=opts['directory'], - shell=False, - universal_newlines=True) + fn_list = run_command(funcmap_command, cwd=opts['directory']) output = [] - fn_list = fn_out.splitlines() for fn_txt in fn_list: dpos = fn_txt.find(" ") mangled_name = fn_txt[0:dpos] From b459743c49c685ccd502536ef73034c01952f353 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Gera?= Date: Mon, 19 Jun 2017 13:07:32 +0200 Subject: [PATCH 17/23] Add extra test for spaces in filenames --- tests/unit/test_analyze.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/unit/test_analyze.py b/tests/unit/test_analyze.py index a9f7750..1c2074f 100644 --- a/tests/unit/test_analyze.py +++ b/tests/unit/test_analyze.py @@ -384,3 +384,9 @@ def test_duplicates_are_kept(self): self.assertTrue(('_Z1fun1i@x86_64', 'ast/x86_64/fun1.c.ast') in pairs) self.assertTrue(('_Z1fun2i@x86_64', 'ast/x86_64/fun2.c.ast') in pairs) self.assertEqual(2, len(pairs)) + + def test_space_handled_in_source(self): + concat_map = ['_Z1fun1i@x86_64 ast/x86_64/f un.c.ast'] + pairs = sut.create_global_ctu_function_map(concat_map) + self.assertTrue(('_Z1fun1i@x86_64', 'ast/x86_64/f un.c.ast') in pairs) + self.assertEqual(1, len(pairs)) From db9bc4036ae0cf95f87096f9872dfb4a25e74a7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Gera?= Date: Mon, 19 Jun 2017 13:44:29 +0200 Subject: [PATCH 18/23] Function map generation refactor and testing --- libscanbuild/analyze.py | 28 ++++++++++++++++++---------- tests/unit/test_analyze.py | 23 ++++++++++++++++++++++- 2 files changed, 40 insertions(+), 11 deletions(-) diff --git a/libscanbuild/analyze.py b/libscanbuild/analyze.py index 4cd2c70..53b8399 100644 --- a/libscanbuild/analyze.py +++ b/libscanbuild/analyze.py @@ -502,6 +502,20 @@ def target(): return result +def func_map_list_src_to_ast(func_src_list, triple_arch): + """ Turns textual function map list with source files into a + function map list with ast files. """ + + func_ast_list = [] + for fn_src_txt in func_src_list: + dpos = fn_src_txt.find(" ") + mangled_name = fn_src_txt[0:dpos] + path = fn_src_txt[dpos + 1:] + ast_path = os.path.join("ast", triple_arch, path[1:] + ".ast") + func_ast_list.append(mangled_name + "@" + triple_arch + " " + ast_path) + return func_ast_list + + @require(['clang', 'directory', 'flags', 'direct_args', 'source', 'ctu']) def ctu_collect_phase(opts): """ Preprocess source by generating all data needed by CTU analysis. """ @@ -536,21 +550,15 @@ def map_functions(triple_arch): funcmap_command.append('--') funcmap_command.extend(args) logging.debug("Generating function map using '%s'", funcmap_command) - fn_list = run_command(funcmap_command, cwd=opts['directory']) - output = [] - for fn_txt in fn_list: - dpos = fn_txt.find(" ") - mangled_name = fn_txt[0:dpos] - path = fn_txt[dpos + 1:] - ast_path = os.path.join("ast", triple_arch, path[1:] + ".ast") - output.append(mangled_name + "@" + triple_arch + " " + ast_path) + func_src_list = run_command(funcmap_command, cwd=opts['directory']) + func_ast_list = func_map_list_src_to_ast(func_src_list, triple_arch) extern_fns_map_folder = os.path.join(opts['ctu'].dir, CTU_TEMP_FNMAP_FOLDER) - if output: + if func_ast_list: with tempfile.NamedTemporaryFile(mode='w', dir=extern_fns_map_folder, delete=False) as out_file: - out_file.write("\n".join(output) + "\n") + out_file.write("\n".join(func_ast_list) + "\n") cwd = opts['directory'] cmd = [opts['clang'], '--analyze'] + opts['direct_args'] + opts['flags'] \ diff --git a/tests/unit/test_analyze.py b/tests/unit/test_analyze.py index 1c2074f..25e2823 100644 --- a/tests/unit/test_analyze.py +++ b/tests/unit/test_analyze.py @@ -354,7 +354,7 @@ class MergeCtuMapTest(unittest.TestCase): def test_no_map_gives_empty(self): pairs = sut.create_global_ctu_function_map([]) - self.assertListEqual([], pairs) + self.assertFalse(pairs) def test_multiple_maps_merged(self): concat_map = ['_Z1fun1i@x86_64 ast/x86_64/fun1.c.ast', @@ -390,3 +390,24 @@ def test_space_handled_in_source(self): pairs = sut.create_global_ctu_function_map(concat_map) self.assertTrue(('_Z1fun1i@x86_64', 'ast/x86_64/f un.c.ast') in pairs) self.assertEqual(1, len(pairs)) + + +class FuncMapSrcToAstTest(unittest.TestCase): + + def test_empty_gives_empty(self): + fun_ast_lst = sut.func_map_list_src_to_ast([], 'armv7') + self.assertFalse(fun_ast_lst) + + def test_sources_to_asts(self): + fun_src_lst = ['_Z1f1i /path/f1.c', + '_Z1f2i /path/f2.c'] + fun_ast_lst = sut.func_map_list_src_to_ast(fun_src_lst, 'armv7') + self.assertTrue('_Z1f1i@armv7 ast/armv7/path/f1.c.ast' in fun_ast_lst) + self.assertTrue('_Z1f2i@armv7 ast/armv7/path/f2.c.ast' in fun_ast_lst) + self.assertEqual(2, len(fun_ast_lst)) + + def test_spaces_handled(self): + fun_src_lst = ['_Z1f1i /path/f 1.c'] + fun_ast_lst = sut.func_map_list_src_to_ast(fun_src_lst, 'armv7') + self.assertTrue('_Z1f1i@armv7 ast/armv7/path/f 1.c.ast' in fun_ast_lst) + self.assertEqual(1, len(fun_ast_lst)) From 13ebac821d78c4c720b5cc4f4660af2692a03618 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Gera?= Date: Fri, 23 Jun 2017 21:27:08 +0200 Subject: [PATCH 19/23] Add function map generator detection --- libscanbuild/__init__.py | 3 ++- libscanbuild/analyze.py | 20 ++++++++++++-------- libscanbuild/arguments.py | 28 +++++++++++++++++++++++----- libscanbuild/clang.py | 16 +++++++++++++++- tests/unit/test_clang.py | 7 +++++++ 5 files changed, 59 insertions(+), 15 deletions(-) diff --git a/libscanbuild/__init__.py b/libscanbuild/__init__.py index 1a7e2be..7fffd61 100644 --- a/libscanbuild/__init__.py +++ b/libscanbuild/__init__.py @@ -19,7 +19,8 @@ Execution = collections.namedtuple('Execution', ['pid', 'cwd', 'cmd']) -CtuConfig = collections.namedtuple('CtuConfig', ['collect', 'analyze', 'dir']) +CtuConfig = collections.namedtuple('CtuConfig', ['collect', 'analyze', 'dir', + 'func_map_cmd']) def shell_split(string): diff --git a/libscanbuild/analyze.py b/libscanbuild/analyze.py index 53b8399..3347889 100644 --- a/libscanbuild/analyze.py +++ b/libscanbuild/analyze.py @@ -117,9 +117,10 @@ def get_ctu_config(args): return ( CtuConfig(collect=args.ctu_phases.collect, analyze=args.ctu_phases.analyze, - dir=args.ctu_dir) + dir=args.ctu_dir, + func_map_cmd=args.func_map_cmd) if hasattr(args, 'ctu_phases') and hasattr(args.ctu_phases, 'dir') - else CtuConfig(collect=False, analyze=False, dir='')) + else CtuConfig(collect=False, analyze=False, dir='', func_map_cmd='')) def analyze_parameters(args): @@ -265,11 +266,14 @@ def run_analyzer_with_ctu(compilations, args): if ctu_config.collect and ctu_config.analyze: # compilations is a generator but we want to do 2 CTU rounds compilation_list = list(compilations) - # CTU folder is coming from args.ctu_dir, so we can leave it empty - args.ctu_phases = CtuConfig(collect=True, analyze=False, dir='') + # CTU strings are coming from args.ctu_dir and func_map_cmd, + # so we can leave it empty + args.ctu_phases = CtuConfig(collect=True, analyze=False, + dir='', func_map_cmd='') run_analyzer_parallel(compilation_list, args) merge_ctu_func_maps(ctu_config.dir) - args.ctu_phases = CtuConfig(collect=False, analyze=True, dir='') + args.ctu_phases = CtuConfig(collect=False, analyze=True, + dir='', func_map_cmd='') run_analyzer_parallel(compilation_list, args) shutil.rmtree(ctu_config.dir, ignore_errors=True) else: @@ -544,8 +548,7 @@ def map_functions(triple_arch): """ Generate function map file for the current source. """ args = opts['direct_args'] + opts['flags'] - funcmap_command = [os.path.join(os.path.dirname(opts['clang']), - 'clang-func-mapping')] + funcmap_command = [opts['ctu'].func_map_cmd] funcmap_command.append(opts['source']) funcmap_command.append('--') funcmap_command.extend(args) @@ -577,7 +580,8 @@ def dispatch_ctu(opts, continuation=run_analyzer): if not hasattr(ctu_config, 'collect'): ctu_config = CtuConfig(collect=ctu_config[0], analyze=ctu_config[1], - dir=ctu_config[2]) + dir=ctu_config[2], + func_map_cmd=ctu_config[3]) opts['ctu'] = ctu_config if ctu_config.collect or ctu_config.analyze: diff --git a/libscanbuild/arguments.py b/libscanbuild/arguments.py index c6c0230..20ad3bf 100644 --- a/libscanbuild/arguments.py +++ b/libscanbuild/arguments.py @@ -19,7 +19,7 @@ import logging import tempfile from libscanbuild import reconfigure_logging, CtuConfig -from libscanbuild.clang import get_checkers +from libscanbuild.clang import get_checkers, is_ctu_capable __all__ = ['parse_args_for_intercept_build', 'parse_args_for_analyze_build', 'parse_args_for_scan_build'] @@ -122,12 +122,17 @@ def validate_args_for_analyze(parser, args, from_build_command): elif not from_build_command and not os.path.exists(args.cdb): parser.error(message='compilation database is missing') - # If it is CTU analyze_only, the input directory should exist + # If the user wants CTU mode if not from_build_command and hasattr(args, 'ctu_phases') \ and hasattr(args.ctu_phases, 'dir'): + # If CTU analyze_only, the input directory should exist if args.ctu_phases.analyze and not args.ctu_phases.collect \ and not os.path.exists(args.ctu_dir): parser.error(message='missing CTU directory') + # Check CTU capability via checking clang-func-mapping + if not is_ctu_capable(args.clang, args.func_map_cmd): + parser.error(message="""This version of clang does not support CTU + functionality or clang-func-mapping command not found.""") def create_intercept_parser(): @@ -353,7 +358,8 @@ def create_analyze_parser(from_build_command): ctu_mutex_group.add_argument( '--ctu', action='store_const', - const=CtuConfig(collect=True, analyze=True, dir=''), + const=CtuConfig(collect=True, analyze=True, + dir='', func_map_cmd=''), dest='ctu_phases', help="""Perform cross translation unit (ctu) analysis (both collect and analyze phases) using default for temporary output. @@ -367,17 +373,29 @@ def create_analyze_parser(from_build_command): ctu_mutex_group.add_argument( '--ctu-collect-only', action='store_const', - const=CtuConfig(collect=True, analyze=False, dir=''), + const=CtuConfig(collect=True, analyze=False, + dir='', func_map_cmd=''), dest='ctu_phases', help="""Perform only the collect phase of ctu. Keep for further use.""") ctu_mutex_group.add_argument( '--ctu-analyze-only', action='store_const', - const=CtuConfig(collect=False, analyze=True, dir=''), + const=CtuConfig(collect=False, analyze=True, + dir='', func_map_cmd=''), dest='ctu_phases', help="""Perform only the analyze phase of ctu. should be present and will not be removed after analysis.""") + ctu.add_argument( + '--use-func-map-cmd', + metavar='', + dest='func_map_cmd', + default='clang-func-mapping', + help="""'%(prog)s' uses the 'clang-func-mapping' executable + relative to itself for generating function maps for static + analysis. One can override this behavior with this option by using + the 'clang-func-mapping' packaged with Xcode (on OS X) or from the + PATH.""") return parser diff --git a/libscanbuild/clang.py b/libscanbuild/clang.py index fbee82b..08edcbe 100644 --- a/libscanbuild/clang.py +++ b/libscanbuild/clang.py @@ -8,10 +8,12 @@ Since Clang command line interface is so rich, but this project is using only a subset of that, it makes sense to create a function specific wrapper. """ +import subprocess import re from libscanbuild import shell_split, run_command -__all__ = ['get_version', 'get_arguments', 'get_checkers', 'get_triple_arch'] +__all__ = ['get_version', 'get_arguments', 'get_checkers', 'is_ctu_capable', + 'get_triple_arch'] # regex for activated checker ACTIVE_CHECKER_PATTERN = re.compile(r'^-analyzer-checker=(.*)$') @@ -153,6 +155,18 @@ def get_checkers(clang, plugins): return checkers +def is_ctu_capable(clang_cmd, func_map_cmd): + """ Detects if the current (or given) clang and function mapping + executables are CTU compatible. """ + + try: + run_command([func_map_cmd, '-version']) + run_command([clang_cmd, '--version']) + except (OSError, subprocess.CalledProcessError): + return False + return True + + def get_triple_arch(command, cwd): """Returns the architecture part of the target triple for the given compilation command. """ diff --git a/tests/unit/test_clang.py b/tests/unit/test_clang.py index 9925d1a..8ad8e99 100644 --- a/tests/unit/test_clang.py +++ b/tests/unit/test_clang.py @@ -92,6 +92,13 @@ def test_parse_checkers(self): self.assertEqual('Checker Two description', result.get('checker.two')) +class ClangIsCtuCapableTest(unittest.TestCase): + def test_ctu_not_found(self): + is_ctu = sut.is_ctu_capable('not-found-clang', + 'not-found-clang-func-mapping') + self.assertFalse(is_ctu) + + class ClangGetTripleArchTest(unittest.TestCase): def test_arch_is_not_empty(self): arch = sut.get_triple_arch(['clang', '-E', '-'], '.') From 89f5571e1892364f57d5a0f7d97531f1e7574026 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Gera?= Date: Mon, 26 Jun 2017 14:06:32 +0200 Subject: [PATCH 20/23] Functional CTU test passes on CTU not supported --- tests/functional/cases/analyze/analyze_ctu.fts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/functional/cases/analyze/analyze_ctu.fts b/tests/functional/cases/analyze/analyze_ctu.fts index 679fc8c..debec13 100644 --- a/tests/functional/cases/analyze/analyze_ctu.fts +++ b/tests/functional/cases/analyze/analyze_ctu.fts @@ -1,7 +1,7 @@ #!/usr/bin/env bash # RUN: bash %s %T/ctu -# RUN: cd %T/ctu; %{analyze-build} -o . --cdb buildlog.json --ctu --plist-multi-file | ./check.sh +# RUN: cd %T/ctu; ./go_with_ctu.sh %{analyze-build} -o . --cdb buildlog.json --ctu --plist-multi-file set -o errexit set -o nounset @@ -19,6 +19,20 @@ set -o xtrace root_dir=$1 mkdir -p "${root_dir}/src" +go_with_ctu="${root_dir}/go_with_ctu.sh" +cat > ${go_with_ctu} << EOF +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o xtrace + +if command -v clang-func-mapping; then + \$* | ./check.sh; +fi +EOF +chmod +x ${go_with_ctu} + cat > "${root_dir}/src/lib.c" << EOF int bad_guy(int * i) { From 367614fb80b30826eaa4af237bc342a5562d97c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Gera?= Date: Wed, 28 Jun 2017 14:25:50 +0200 Subject: [PATCH 21/23] Make path handling windows compatible --- libscanbuild/analyze.py | 6 +++++- tests/unit/test_analyze.py | 22 ++++++++++++++-------- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/libscanbuild/analyze.py b/libscanbuild/analyze.py index 3347889..b9605dc 100644 --- a/libscanbuild/analyze.py +++ b/libscanbuild/analyze.py @@ -515,7 +515,11 @@ def func_map_list_src_to_ast(func_src_list, triple_arch): dpos = fn_src_txt.find(" ") mangled_name = fn_src_txt[0:dpos] path = fn_src_txt[dpos + 1:] - ast_path = os.path.join("ast", triple_arch, path[1:] + ".ast") + # Normalize path on windows as well + path = os.path.splitdrive(path)[1] + # Make relative path out of absolute + path = path[1:] if path[0] == os.sep else path + ast_path = os.path.join("ast", triple_arch, path + ".ast") func_ast_list.append(mangled_name + "@" + triple_arch + " " + ast_path) return func_ast_list diff --git a/tests/unit/test_analyze.py b/tests/unit/test_analyze.py index 25e2823..7808765 100644 --- a/tests/unit/test_analyze.py +++ b/tests/unit/test_analyze.py @@ -4,13 +4,13 @@ # This file is distributed under the University of Illinois Open Source # License. See LICENSE.TXT for details. -import libear -import libscanbuild.analyze as sut import unittest import os import os.path import glob import platform +import libear +import libscanbuild.analyze as sut IS_WINDOWS = os.getenv('windows') @@ -399,15 +399,21 @@ def test_empty_gives_empty(self): self.assertFalse(fun_ast_lst) def test_sources_to_asts(self): - fun_src_lst = ['_Z1f1i /path/f1.c', - '_Z1f2i /path/f2.c'] + fun_src_lst = ['_Z1f1i ' + os.path.join(os.sep + 'path', 'f1.c'), + '_Z1f2i ' + os.path.join(os.sep + 'path', 'f2.c')] fun_ast_lst = sut.func_map_list_src_to_ast(fun_src_lst, 'armv7') - self.assertTrue('_Z1f1i@armv7 ast/armv7/path/f1.c.ast' in fun_ast_lst) - self.assertTrue('_Z1f2i@armv7 ast/armv7/path/f2.c.ast' in fun_ast_lst) + self.assertTrue('_Z1f1i@armv7 ' + + os.path.join('ast', 'armv7', 'path', 'f1.c.ast') + in fun_ast_lst) + self.assertTrue('_Z1f2i@armv7 ' + + os.path.join('ast', 'armv7', 'path', 'f2.c.ast') + in fun_ast_lst) self.assertEqual(2, len(fun_ast_lst)) def test_spaces_handled(self): - fun_src_lst = ['_Z1f1i /path/f 1.c'] + fun_src_lst = ['_Z1f1i ' + os.path.join(os.sep + 'path', 'f 1.c')] fun_ast_lst = sut.func_map_list_src_to_ast(fun_src_lst, 'armv7') - self.assertTrue('_Z1f1i@armv7 ast/armv7/path/f 1.c.ast' in fun_ast_lst) + self.assertTrue('_Z1f1i@armv7 ' + + os.path.join('ast', 'armv7', 'path', 'f 1.c.ast') + in fun_ast_lst) self.assertEqual(1, len(fun_ast_lst)) From 3ab8c19336dd82a131531f3d920e67797722829f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Gera?= Date: Fri, 30 Jun 2017 14:13:31 +0200 Subject: [PATCH 22/23] Use abspath for ctu-dir on dir hopping projects --- libscanbuild/arguments.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libscanbuild/arguments.py b/libscanbuild/arguments.py index 20ad3bf..a38eb72 100644 --- a/libscanbuild/arguments.py +++ b/libscanbuild/arguments.py @@ -98,6 +98,11 @@ def normalize_args_for_analyze(args, from_build_command): # add cdb parameter invisibly to make report module working. args.cdb = 'compile_commands.json' + # Make ctu_dir an abspath as it is needed inside clang + if not from_build_command and hasattr(args, 'ctu_phases') \ + and hasattr(args.ctu_phases, 'dir'): + args.ctu_dir = os.path.abspath(args.ctu_dir) + def validate_args_for_analyze(parser, args, from_build_command): """ Command line parsing is done by the argparse module, but semantic From 52071a27fd8e522ecdb2b77ec16afdb291018f65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Gera?= Date: Fri, 30 Jun 2017 15:19:23 +0200 Subject: [PATCH 23/23] Protect parser libs from bogus reports --- libscanbuild/report.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libscanbuild/report.py b/libscanbuild/report.py index 98e38d3..1e94fbe 100644 --- a/libscanbuild/report.py +++ b/libscanbuild/report.py @@ -266,6 +266,14 @@ def read_bugs(output_dir, html): duplicate = duplicate_check( lambda bug: '{bug_line}.{bug_path_length}:{bug_file}'.format(**bug)) + # Protect parsers from bogus report files coming from clang crashes + for filename in glob.iglob(os.path.join(output_dir, pattern)): + if os.stat(filename).st_size == 0: + try: + os.remove(filename) + except OSError: + pass + bugs = itertools.chain.from_iterable( # parser creates a bug generator not the bug itself parser(filename)