diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a863033edcc..72ed71d27af 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -85,6 +85,11 @@ jobs: - name: Run lintrunner working-directory: ${{ env.working_directory }} run: | + tools/apt-install-things.sh & + tools/pip-install-things.sh & + wait + source tools/setup-env.sh + # Install lintrunner pip install lintrunner diff --git a/.lintrunner.toml b/.lintrunner.toml index f10037e39bb..1995f1989a7 100644 --- a/.lintrunner.toml +++ b/.lintrunner.toml @@ -36,16 +36,9 @@ exclude_patterns = [ command = [ 'python3', 'tools/linter/adapters/clangformat_linter.py', - '--binary=~/.local/bin/clang-format', '--', '@{{PATHSFILE}}' ] -init_command = [ - 'python3', - 'tools/linter/adapters/pip_init.py', - '--dry-run={{DRYRUN}}', - 'clang-format==19.1.7', -] is_formatter = true [[linter]] @@ -174,17 +167,9 @@ exclude_patterns = [ 'third_party/**', 'tools/examples/**', ] - -init_command = [ - 'python3', - 'tools/linter/adapters/pip_init.py', - '--dry-run={{DRYRUN}}', - 'clang-tidy==19.1.0.1', -] command = [ 'python3', 'tools/linter/adapters/clangtidy_linter.py', - '--binary=~/.local/bin/clang-tidy', '--build_dir=./python/build', '--', '@{{PATHSFILE}}' diff --git a/tools/apt-install-things.sh b/tools/apt-install-things.sh index fe52c69456d..4eb88626a25 100755 --- a/tools/apt-install-things.sh +++ b/tools/apt-install-things.sh @@ -4,17 +4,25 @@ set -e # Install cuda keyring wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb -sudo dpkg -i cuda-keyring_1.1-1_all.deb +sudo dpkg -i cuda-keyring_1.1-1_all.deb && rm cuda-keyring_1.1-1_all.deb sudo apt-get update # Remove some old toolchains. By default, the github action comes with multiple versions of gcc and clang installed. # Having many versions of gcc and clang installed interfers with each other, causing weird build and clang-tidy errors. # We only keep one version of gcc and clang in the system, and remove the rest. -sudo apt-get -y remove gcc-13 libstdc++-13-dev gcc-12 libstdc++-12-dev +sudo apt-get -y remove llvm-18 gcc-13 libstdc++-13-dev gcc-12 libstdc++-12-dev # Install the latest version of clang and gcc. -sudo apt-get -y install clang-19 gcc-14 nlohmann-json3-dev ninja-build +sudo apt-get -y install clang-19 clang-tidy-19 clang-format-19 gcc-14 nlohmann-json3-dev ninja-build +# llvm provides llvm-config, which downstream scripts use to locate binaries. +sudo apt-get -y install llvm-19 + +# llvm-dev are for host IR compilation, which uses LLVM JIT. +sudo apt-get -y install llvm-19-dev + +# Should we use llvm-config to locate clang? +# # Ensure clang-19 and clang++-19 are available and properly linked # Create symlinks if they don't exist to handle runner environment variations if [ ! -x "/usr/bin/clang-19" ] && [ -x "/usr/bin/clang" ]; then @@ -28,11 +36,18 @@ if ! command -v clang-19 >/dev/null 2>&1; then ls -la /usr/bin/clang* || true fi +# Verify llvm-config-19 installation +if command -v llvm-config-19 >/dev/null 2>&1; then + # Set up llvm-config so lintrunner can use it to locate clang-tidy and + # clang-format without hardcoding which version + sudo ln -sf "$(command -v llvm-config-19)" /usr/bin/llvm-config +else + echo "Warning: llvm-config-19 not found in PATH after installation" + ls -la /usr/bin/llvm-config* || true +fi + # Install minimal cuda toolkit. sudo apt-get -y install cuda-compiler-13-0 cuda-command-line-tools-13-0 cuda-libraries-dev-13-0 libnccl-dev -# llvm-dev are for host IR compilation, which uses LLVM JIT. -sudo apt-get -y install llvm-19-dev - # PyTorch now relies on packages sudo apt-get -y install libfmt-dev diff --git a/tools/linter/adapters/clangformat_linter.py b/tools/linter/adapters/clangformat_linter.py index 44ba1d54e0d..b83d6acb1e4 100644 --- a/tools/linter/adapters/clangformat_linter.py +++ b/tools/linter/adapters/clangformat_linter.py @@ -11,9 +11,6 @@ from typing import Any, List, NamedTuple, Optional -IS_WINDOWS: bool = os.name == "nt" - - def eprint(*args: Any, **kwargs: Any) -> None: print(*args, file=sys.stderr, flush=True, **kwargs) @@ -37,10 +34,6 @@ class LintMessage(NamedTuple): description: Optional[str] -def as_posix(name: str) -> str: - return name.replace("\\", "/") if IS_WINDOWS else name - - def _run_command( args: List[str], *, @@ -51,9 +44,7 @@ def _run_command( try: return subprocess.run( args, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=IS_WINDOWS, # So batch scripts are found. + capture_output=True, timeout=timeout, check=True, ) @@ -138,7 +129,7 @@ def check_file( "STDOUT\n{stdout}" ).format( returncode=err.returncode, - command=" ".join(as_posix(x) for x in err.cmd), + command=" ".join(err.cmd), stderr=err.stderr.decode("utf-8").strip() or "(empty)", stdout=err.stdout.decode("utf-8").strip() or "(empty)", ) @@ -170,11 +161,6 @@ def main() -> None: description="Format files with clang-format.", fromfile_prefix_chars="@", ) - parser.add_argument( - "--binary", - required=True, - help="clang-format binary path", - ) parser.add_argument( "--retries", default=3, @@ -201,16 +187,36 @@ def main() -> None: logging.basicConfig( format="<%(threadName)s:%(levelname)s> %(message)s", - level=logging.NOTSET - if args.verbose - else logging.DEBUG - if len(args.filenames) < 1000 - else logging.INFO, + level=( + logging.NOTSET + if args.verbose + else logging.DEBUG + if len(args.filenames) < 1000 + else logging.INFO + ), stream=sys.stderr, ) - binary = os.path.normpath(args.binary) if IS_WINDOWS else args.binary - binary = os.path.expanduser(binary) + try: + llvm_bindir = subprocess.check_output( + ["llvm-config", "--bindir"], text=True + ).strip() + except FileNotFoundError: + lint_message = LintMessage( + path=None, + line=None, + char=None, + code="CLANGFORMAT", + severity=LintSeverity.ERROR, + name="init-error", + original=None, + replacement=None, + description="llvm-config doesn't exist (is LLVM installed?).", + ) + print(json.dumps(lint_message._asdict()), flush=True) + sys.exit(0) + + binary = os.path.join(llvm_bindir, "clang-format") if not Path(binary).exists(): lint_message = LintMessage( path=None, @@ -222,8 +228,7 @@ def main() -> None: original=None, replacement=None, description=( - f"Could not find clang-format binary at {binary}, " - "did you forget to run `lintrunner init`?" + f"Could not find clang-format binary at {binary} (is LLVM installed?)", ), ) print(json.dumps(lint_message._asdict()), flush=True) diff --git a/tools/linter/adapters/clangtidy_linter.py b/tools/linter/adapters/clangtidy_linter.py index 1e4d98896c4..303fa44bd1b 100644 --- a/tools/linter/adapters/clangtidy_linter.py +++ b/tools/linter/adapters/clangtidy_linter.py @@ -130,11 +130,15 @@ def clang_search_dirs() -> List[str]: return search_paths -include_args = [] +# NVFUSER_USE_PCH=ON somehow requires the CUDA include dir e.g. +# "/usr/local/cuda-13.0/targets/x86_64-linux/include". I'll try to add that in +# a future PR. include_dir = [ get_python_include_dir(), os.path.join(NVFUSER_ROOT, "third_party/pybind11/include"), ] + clang_search_dirs() + +include_args = [] for dir in include_dir: include_args += ["--extra-arg", f"-I{dir}"] @@ -199,11 +203,6 @@ def main() -> None: description="clang-tidy wrapper linter.", fromfile_prefix_chars="@", ) - parser.add_argument( - "--binary", - required=True, - help="clang-tidy binary path", - ) parser.add_argument( "--build-dir", "--build_dir", @@ -235,8 +234,11 @@ def main() -> None: stream=sys.stderr, ) - args.binary = os.path.expanduser(args.binary) - if not os.path.exists(args.binary): + try: + llvm_bindir = subprocess.check_output( + ["llvm-config", "--bindir"], text=True + ).strip() + except FileNotFoundError: err_msg = LintMessage( path="", line=None, @@ -246,23 +248,28 @@ def main() -> None: name="command-failed", original=None, replacement=None, - description=( - f"Could not find clang-tidy binary at {args.binary}," - " you may need to run `lintrunner init`." - ), + description="llvm-config doesn't exist (is LLVM installed?).", ) print(json.dumps(err_msg._asdict()), flush=True) exit(0) - abs_build_dir = Path(args.build_dir).resolve() + binary_path = os.path.join(llvm_bindir, "clang-tidy") + if not os.path.exists(binary_path): + err_msg = LintMessage( + path="", + line=None, + char=None, + code="CLANGTIDY", + severity=LintSeverity.ERROR, + name="command-failed", + original=None, + replacement=None, + description=f"Could not find clang-tidy binary at {binary_path} (is LLVM installed?)", + ) + print(json.dumps(err_msg._asdict()), flush=True) + exit(0) - # Get the absolute path to clang-tidy and use this instead of the relative - # path such as .lintbin/clang-tidy. The problem here is that os.chdir is - # per process, and the linter uses it to move between the current directory - # and the build folder. And there is no .lintbin directory in the latter. - # When it happens in a race condition, the linter command will fails with - # the following no such file or directory error: '.lintbin/clang-tidy' - binary_path = os.path.abspath(args.binary) + abs_build_dir = Path(args.build_dir).resolve() with concurrent.futures.ThreadPoolExecutor( max_workers=os.cpu_count(),