diff --git a/clang-tidy-diff.py b/clang-tidy-diff.py index a67ac433b3d41..6f157c64613d8 100644 --- a/clang-tidy-diff.py +++ b/clang-tidy-diff.py @@ -1,79 +1,424 @@ + +#!/usr/bin/env python3 +# +# ===- clang-tidy-diff.py - ClangTidy Diff Checker -----------*- python -*--===# +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ===-----------------------------------------------------------------------===# + +r""" +ClangTidy Diff Checker +====================== + +This script reads input from a unified diff, runs clang-tidy on all changed +files and outputs clang-tidy warnings in changed lines only. This is useful to +detect clang-tidy regressions in the lines touched by a specific patch. +Example usage for git/svn users: + + git diff -U0 HEAD^ | clang-tidy-diff.py -p1 + svn diff --diff-cmd=diff -x-U0 | \ + clang-tidy-diff.py -fix -checks=-*,modernize-use-override + +""" + +import argparse +import glob +import json +import multiprocessing +import os +import re +import shutil import subprocess -import requests -import yaml import sys -import difflib -from termcolor import colored - -# Function to load configuration from the YAML file -def load_config(): - with open("config.yaml") as f: - return yaml.safe_load(f)["project"] - -# Function to fetch the diff from the GitHub API -def fetch_diff(owner, repo, pr_number): - url = f"https://api.github.com/repos/{owner}/{repo}/pulls/{pr_number}.diff" - headers = {"Accept": "application/vnd.github.v3.diff"} - print(f"๐Ÿ“ฅ Fetching diff from {url}") - resp = requests.get(url, headers=headers) - if resp.status_code != 200: - print(f"โŒ Failed to fetch diff: {resp.status_code}") - sys.exit(1) - return resp.text - -# Function to extract only formatted lines (those with +) -def extract_formatted_lines(diff): - formatted_lines = [] - for line in diff.splitlines(): - if line.startswith('+'): # Lines added or modified (formatted) - formatted_lines.append(line[1:]) # Remove the "+" sign - return formatted_lines - -# Function to run clang-tidy on specific files/lines -def run_clang_tidy_on_formatted_lines(files): - for file in files: - print(f"๐ŸŽฏ Running clang-tidy on {file}") +import tempfile +import threading +import traceback +from pathlib import Path + +try: + import yaml +except ImportError: + yaml = None + +is_py2 = sys.version[0] == "2" + +if is_py2: + import Queue as queue +else: + import queue as queue + + +def run_tidy(task_queue, lock, timeout, failed_files): + watchdog = None + while True: + command = task_queue.get() try: - result = subprocess.run( - ["clang-tidy", file, "--checks='*'"], # Run clang-tidy with all checks - capture_output=True, - check=True + proc = subprocess.Popen( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) - print("๐Ÿ“ค clang-tidy Output:") - print(result.stdout.decode()) - except subprocess.CalledProcessError as e: - print("โŒ clang-tidy failed!") - print("๐Ÿ“ค STDOUT:") - print(e.stdout.decode()) - print("๐Ÿ“ฅ STDERR:") - print(e.stderr.decode()) - sys.exit(1) - -# Main function to load configuration, fetch diff, and run clang-tidy on formatted lines + + if timeout is not None: + watchdog = threading.Timer(timeout, proc.kill) + watchdog.start() + + stdout, stderr = proc.communicate() + if proc.returncode != 0: + if proc.returncode < 0: + msg = "Terminated by signal %d : %s\n" % ( + -proc.returncode, + " ".join(command), + ) + stderr += msg.encode("utf-8") + failed_files.append(command) + + with lock: + sys.stdout.write(stdout.decode("utf-8") + "\n") + sys.stdout.flush() + if stderr: + sys.stderr.write(stderr.decode("utf-8") + "\n") + sys.stderr.flush() + except Exception as e: + with lock: + sys.stderr.write("Failed: " + str(e) + ": ".join(command) + "\n") + finally: + with lock: + if not (timeout is None or watchdog is None): + if not watchdog.is_alive(): + sys.stderr.write( + "Terminated by timeout: " + " ".join(command) + "\n" + ) + watchdog.cancel() + task_queue.task_done() + + +def start_workers(max_tasks, tidy_caller, arguments): + for _ in range(max_tasks): + t = threading.Thread(target=tidy_caller, args=arguments) + t.daemon = True + t.start() + + +def merge_replacement_files(tmpdir, mergefile): + """Merge all replacement files in a directory into a single file""" + # The fixes suggested by clang-tidy >= 4.0.0 are given under + # the top level key 'Diagnostics' in the output yaml files + mergekey = "Diagnostics" + merged = [] + for replacefile in glob.iglob(os.path.join(tmpdir, "*.yaml")): + content = yaml.safe_load(open(replacefile, "r")) + if not content: + continue # Skip empty files. + merged.extend(content.get(mergekey, [])) + + if merged: + # MainSourceFile: The key is required by the definition inside + # include/clang/Tooling/ReplacementsYaml.h, but the value + # is actually never used inside clang-apply-replacements, + # so we set it to '' here. + output = {"MainSourceFile": "", mergekey: merged} + with open(mergefile, "w") as out: + yaml.safe_dump(output, out) + else: + # Empty the file: + open(mergefile, "w").close() + + +def get_compiling_files(args): + """Read a compile_commands.json database and return a set of file paths""" + current_dir = Path.cwd() + compile_commands_json = ( + (current_dir / args.build_path) if args.build_path else current_dir + ) + compile_commands_json = compile_commands_json / "compile_commands.json" + files = set() + with open(compile_commands_json) as db_file: + db_json = json.load(db_file) + for entry in db_json: + if "file" not in entry: + continue + files.add(Path(entry["file"])) + return files + + def main(): - config = load_config() - diff_text = fetch_diff(config["owner"], config["repo"], config["pr_number"]) - - # Extract only the lines that were formatted (added/modified) - formatted_lines = extract_formatted_lines(diff_text) - - if not formatted_lines: - print("โœ… No formatted lines found.") + parser = argparse.ArgumentParser( + description="Run clang-tidy against changed files, and " + "output diagnostics only for modified " + "lines." + ) + parser.add_argument( + "-clang-tidy-binary", + metavar="PATH", + default="clang-tidy", + help="path to clang-tidy binary", + ) + parser.add_argument( + "-p", + metavar="NUM", + default=0, + help="strip the smallest prefix containing P slashes", + ) + parser.add_argument( + "-regex", + metavar="PATTERN", + default=None, + help="custom pattern selecting file paths to check " + "(case sensitive, overrides -iregex)", + ) + parser.add_argument( + "-iregex", + metavar="PATTERN", + default=r".*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc)", + help="custom pattern selecting file paths to check " + "(case insensitive, overridden by -regex)", + ) + parser.add_argument( + "-j", + type=int, + default=1, + help="number of tidy instances to be run in parallel.", + ) + parser.add_argument( + "-timeout", type=int, default=None, help="timeout per each file in seconds." + ) + parser.add_argument( + "-fix", action="store_true", default=False, help="apply suggested fixes" + ) + parser.add_argument( + "-checks", + help="checks filter, when not specified, use clang-tidy " "default", + default="", + ) + parser.add_argument( + "-config-file", + dest="config_file", + help="Specify the path of .clang-tidy or custom config file", + default="", + ) + parser.add_argument("-use-color", action="store_true", help="Use colors in output") + parser.add_argument( + "-path", dest="build_path", help="Path used to read a compile command database." + ) + if yaml: + parser.add_argument( + "-export-fixes", + metavar="FILE_OR_DIRECTORY", + dest="export_fixes", + help="A directory or a yaml file to store suggested fixes in, " + "which can be applied with clang-apply-replacements. If the " + "parameter is a directory, the fixes of each compilation unit are " + "stored in individual yaml files in the directory.", + ) else: - print("\n๐Ÿงผ Suggested clang-tidy changes on formatted lines:\n") - # Display formatted lines to user - for line in formatted_lines: - print(colored(line, 'green')) # Display formatted lines in green - - # Run clang-tidy on the files with formatted lines - files_to_check = [] # Populate this list with the relevant files - for line in formatted_lines: - # Here you can add logic to identify the file and the line number (from the diff) - file = "" # Extract file path from the diff - files_to_check.append(file) - - # Run clang-tidy on the formatted files - run_clang_tidy_on_formatted_lines(files_to_check) - + parser.add_argument( + "-export-fixes", + metavar="DIRECTORY", + dest="export_fixes", + help="A directory to store suggested fixes in, which can be applied " + "with clang-apply-replacements. The fixes of each compilation unit are " + "stored in individual yaml files in the directory.", + ) + parser.add_argument( + "-extra-arg", + dest="extra_arg", + action="append", + default=[], + help="Additional argument to append to the compiler " "command line.", + ) + parser.add_argument( + "-extra-arg-before", + dest="extra_arg_before", + action="append", + default=[], + help="Additional argument to prepend to the compiler " "command line.", + ) + parser.add_argument( + "-quiet", + action="store_true", + default=False, + help="Run clang-tidy in quiet mode", + ) + parser.add_argument( + "-load", + dest="plugins", + action="append", + default=[], + help="Load the specified plugin in clang-tidy.", + ) + parser.add_argument( + "-allow-no-checks", + action="store_true", + help="Allow empty enabled checks.", + ) + parser.add_argument( + "-only-check-in-db", + dest="skip_non_compiling", + default=False, + action="store_true", + help="Only check files in the compilation database", + ) + + clang_tidy_args = [] + argv = sys.argv[1:] + if "--" in argv: + clang_tidy_args.extend(argv[argv.index("--") :]) + argv = argv[: argv.index("--")] + + args = parser.parse_args(argv) + + compiling_files = get_compiling_files(args) if args.skip_non_compiling else None + + # Extract changed lines for each file. + filename = None + lines_by_file = {} + for line in sys.stdin: + match = re.search(r'^\+\+\+\ "?(.*?/){%s}([^ \t\n"]*)' % args.p, line) + if match: + filename = match.group(2) + if filename is None: + continue + + if args.regex is not None: + if not re.match("^%s$" % args.regex, filename): + continue + else: + if not re.match("^%s$" % args.iregex, filename, re.IGNORECASE): + continue + + # Skip any files not in the compiling list + if ( + compiling_files is not None + and (Path.cwd() / filename) not in compiling_files + ): + continue + + match = re.search(r"^@@.*\+(\d+)(,(\d+))?", line) + if match: + start_line = int(match.group(1)) + line_count = 1 + if match.group(3): + line_count = int(match.group(3)) + if line_count == 0: + continue + end_line = start_line + line_count - 1 + lines_by_file.setdefault(filename, []).append([start_line, end_line]) + + if not any(lines_by_file): + print("No relevant changes found.") + sys.exit(0) + + max_task_count = args.j + if max_task_count == 0: + max_task_count = multiprocessing.cpu_count() + max_task_count = min(len(lines_by_file), max_task_count) + + combine_fixes = False + export_fixes_dir = None + delete_fixes_dir = False + if args.export_fixes is not None: + # if a directory is given, create it if it does not exist + if args.export_fixes.endswith(os.path.sep) and not os.path.isdir( + args.export_fixes + ): + os.makedirs(args.export_fixes) + + if not os.path.isdir(args.export_fixes): + if not yaml: + raise RuntimeError( + "Cannot combine fixes in one yaml file. Either install PyYAML or specify an output directory." + ) + + combine_fixes = True + + if os.path.isdir(args.export_fixes): + export_fixes_dir = args.export_fixes + + if combine_fixes: + export_fixes_dir = tempfile.mkdtemp() + delete_fixes_dir = True + + # Tasks for clang-tidy. + task_queue = queue.Queue(max_task_count) + # A lock for console output. + lock = threading.Lock() + + # List of files with a non-zero return code. + failed_files = [] + + # Run a pool of clang-tidy workers. + start_workers( + max_task_count, run_tidy, (task_queue, lock, args.timeout, failed_files) + ) + + # Form the common args list. + common_clang_tidy_args = [] + if args.fix: + common_clang_tidy_args.append("-fix") + if args.checks != "": + common_clang_tidy_args.append("-checks=" + args.checks) + if args.config_file != "": + common_clang_tidy_args.append("-config-file=" + args.config_file) + if args.quiet: + common_clang_tidy_args.append("-quiet") + if args.build_path is not None: + common_clang_tidy_args.append("-p=%s" % args.build_path) + if args.use_color: + common_clang_tidy_args.append("--use-color") + if args.allow_no_checks: + common_clang_tidy_args.append("--allow-no-checks") + for arg in args.extra_arg: + common_clang_tidy_args.append("-extra-arg=%s" % arg) + for arg in args.extra_arg_before: + common_clang_tidy_args.append("-extra-arg-before=%s" % arg) + for plugin in args.plugins: + common_clang_tidy_args.append("-load=%s" % plugin) + + for name in lines_by_file: + line_filter_json = json.dumps( + [{"name": name, "lines": lines_by_file[name]}], separators=(",", ":") + ) + + # Run clang-tidy on files containing changes. + command = [args.clang_tidy_binary] + command.append("-line-filter=" + line_filter_json) + if args.export_fixes is not None: + # Get a temporary file. We immediately close the handle so clang-tidy can + # overwrite it. + (handle, tmp_name) = tempfile.mkstemp(suffix=".yaml", dir=export_fixes_dir) + os.close(handle) + command.append("-export-fixes=" + tmp_name) + command.extend(common_clang_tidy_args) + command.append(name) + command.extend(clang_tidy_args) + + task_queue.put(command) + + # Application return code + return_code = 0 + + # Wait for all threads to be done. + task_queue.join() + # Application return code + return_code = 0 + if failed_files: + return_code = 1 + + if combine_fixes: + print("Writing fixes to " + args.export_fixes + " ...") + try: + merge_replacement_files(export_fixes_dir, args.export_fixes) + except: + sys.stderr.write("Error exporting fixes.\n") + traceback.print_exc() + return_code = 1 + + if delete_fixes_dir: + shutil.rmtree(export_fixes_dir) + sys.exit(return_code) + + if __name__ == "__main__": main() diff --git a/clang/lib/AST/ASTStructuralEquivalence.cpp b/clang/lib/AST/ASTStructuralEquivalence.cpp index 499854a75abc6..00e81d3f069ee 100644 --- a/clang/lib/AST/ASTStructuralEquivalence.cpp +++ b/clang/lib/AST/ASTStructuralEquivalence.cpp @@ -250,23 +250,27 @@ class StmtComparer { return E1->isExact() == E2->isExact() && E1->getValue() == E2->getValue(); } - bool IsStmtEquivalent(const GenericSelectionExpr *E1, - const GenericSelectionExpr *E2) { - for (auto Pair : zip_longest(E1->getAssocTypeSourceInfos(), - E2->getAssocTypeSourceInfos())) { - std::optional Child1 = std::get<0>(Pair); - std::optional Child2 = std::get<1>(Pair); - // Skip this case if there are a different number of associated types. - if (!Child1 || !Child2) - return false; + bool IsStmtEquivalent ( const GenericSelectionExpr * E1, + const GenericSelectionExpr * E2 ) +{ +for ( auto Pair : zip_longest( E1->getAssocTypeSourceInfos(), + E2->getAssocTypeSourceInfos() ) ) +{ +std::optional< TypeSourceInfo * > Child1 = std::get< 0 >( Pair ); +std::optional< TypeSourceInfo * > Child2 = std::get< 1 >( Pair ); + +// Skip this case if there are a different number of associated types. +if ( !Child1 || !Child2 ) +return false; + +if ( !IsStructurallyEquivalent( Context, ( *Child1 )->getType(), + ( *Child2 )->getType() ) ) +return false; +} - if (!IsStructurallyEquivalent(Context, (*Child1)->getType(), - (*Child2)->getType())) - return false; - } +return true; +} - return true; - } bool IsStmtEquivalent(const ImplicitCastExpr *CastE1, const ImplicitCastExpr *CastE2) { diff --git a/remote_clang_format.py b/remote_clang_format.py index 4d1f9aa3c0ddb..740e33f9b7e10 100644 --- a/remote_clang_format.py +++ b/remote_clang_format.py @@ -3,14 +3,12 @@ import yaml import sys import difflib -from termcolor import colored # Import termcolor to use colored output +from termcolor import colored -# Function to load configuration from the YAML file def load_config(): with open("config.yaml") as f: return yaml.safe_load(f)["project"] -# Function to fetch the diff from the GitHub API def fetch_diff(owner, repo, pr_number): url = f"https://api.github.com/repos/{owner}/{repo}/pulls/{pr_number}.diff" headers = {"Accept": "application/vnd.github.v3.diff"} @@ -21,75 +19,54 @@ def fetch_diff(owner, repo, pr_number): sys.exit(1) return resp.text -# Function to run the clang-format-diff.py and get the formatted output -def run_clang_format_diff(diff): - print("๐ŸŽฏ Running clang-format-diff.py on diff...") +def run_git_clang_format(base_commit): + print(f"๐ŸŽฏ Running git-clang-format against base {base_commit}...") try: result = subprocess.run( - ["python3", "./clang-format-diff.py", "-p1", "-binary", "/ptmp/jay/new/llvm-project-checks/build/bin/clang-format"], - input=diff.encode("utf-8"), + [ + "python3", + "/ptmp/jay/new/llvm-project-checks/clang/tools/clang-format/git-clang-format", + "--diff", + base_commit, + "--binary", + "/ptmp/jay/new/llvm-project-checks/build/bin/clang-format" # replace with your actual path + ], capture_output=True, - check=True + check=True, ) return result.stdout.decode() except subprocess.CalledProcessError as e: - print("โŒ clang-format-diff.py failed!") - print("๐Ÿ“ค STDOUT:") - print(e.stdout.decode()) - print("๐Ÿ“ฅ STDERR:") - print(e.stderr.decode()) + print("โŒ git-clang-format failed!") + print("๐Ÿ“ค STDOUT:\n", e.stdout.decode()) + print("๐Ÿ“ฅ STDERR:\n", e.stderr.decode()) sys.exit(1) -# Function to display before and after code with green coloring for the corrected code -def display_changes(before, after): - """ - This function will compare the before and after code and display the difference to the user. - The actual corrected code will be printed in green. - """ - print("\n===================================") - print(colored(" BEFORE FORMATTING ", 'yellow', attrs=['bold'])) - print("===================================") - print(colored(before, 'red')) # Display original code in red - print("\n===================================") - print(colored(" AFTER FORMATTING ", 'yellow', attrs=['bold'])) - print("===================================") - - diff = difflib.unified_diff(before.splitlines(), after.splitlines(), lineterm='') +def display_diff_output(diff_output): print("\n๐Ÿงผ Suggested clang-format changes:\n") - - for line in diff: - if line.startswith("+"): # Lines that were added or corrected - print(colored(line, 'green')) # Print the corrected lines in green - elif line.startswith("-"): # Lines that were removed or incorrect - print(colored(line, 'red')) # Print the removed lines in red + for line in diff_output.splitlines(): + if line.startswith("+"): + print(colored(line, 'green')) + elif line.startswith("-"): + print(colored(line, 'red')) else: - print(line) # Print unchanged lines + print(line) -# Main function to load configuration, fetch diff, and run clang-format-diff.py def main(): config = load_config() - diff_text = fetch_diff(config["owner"], config["repo"], config["pr_number"]) + owner = config["owner"] + repo = config["repo"] + pr_number = config["pr_number"] + base_commit = config.get("base_commit", "origin/main") - # Simulate before and after formatting - formatted_output = run_clang_format_diff(diff_text) + _ = fetch_diff(owner, repo, pr_number) # Just logging, not used further - if not formatted_output.strip(): + diff_output = run_git_clang_format(base_commit) + + if "no modified files to format" in diff_output or not diff_output.strip(): print("โœ… No formatting issues found.") else: - # Get the "before" code from the diff (those with + or - sign) - before_code = "" - after_code = "" - - # Extract the diff lines and display before/after comparison - for line in diff_text.splitlines(): - if line.startswith('-'): - before_code += line[1:] + "\n" # Before formatting (lines with '-') - elif line.startswith('+'): - after_code += line[1:] + "\n" # After formatting (lines with '+') - - # Now display the changes with green color for the corrected code - display_changes(before_code, formatted_output) + display_diff_output(diff_output) if __name__ == "__main__": main() diff --git a/remote_clang_tidy.py b/remote_clang_tidy.py index 026bb125c0a44..02c988267451e 100644 --- a/remote_clang_tidy.py +++ b/remote_clang_tidy.py @@ -29,7 +29,7 @@ # === Run clang-tidy-diff.py on diff from stdin === print("๐Ÿงผ Running clang-tidy-diff.py on PR diff...") # Adjust if clang-tidy-diff.py is in a different path -clang_tidy_diff_path = "clang-tidy-diff.py" +clang_tidy_diff_path = "./clang-tidy-diff.py" result = subprocess.run( ["python3", clang_tidy_diff_path, "-p1"], input=diff_text,