diff --git a/.clang-format b/.clang-format index 8b58306..d25daf9 100644 --- a/.clang-format +++ b/.clang-format @@ -6,11 +6,11 @@ # The basic usage is, # clang-format -i -style=file PATH/TO/SOURCE/CODE # -# The -style=file implicit use ".clang-format" file located in one of -# parent directory. +# The -style=file implicit use ".clang-format" file located in one of +# parent directory. # The -i means inplace change. # -# The document of clang-format is +# The document of clang-format is # http://clang.llvm.org/docs/ClangFormat.html # http://clang.llvm.org/docs/ClangFormatStyleOptions.html --- @@ -20,7 +20,7 @@ IndentWidth: 2 TabWidth: 2 ContinuationIndentWidth: 4 AccessModifierOffset: -1 # The private/protected/public has no indent in class -Standard: Cpp11 +Standard: Cpp11 AllowAllParametersOfDeclarationOnNextLine: true BinPackParameters: false BinPackArguments: false diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..1f28e73 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,26 @@ +# EditorConfig is a cross-editor configuration file +# that helps to unify code styles for multiple +# developers collaborative projects. +# See more at https://editorconfig.org/ + +root = true + +[*] +indent_style = space +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true + +[*.{c,cc,cxx,cpp,cu,cuh,h,hpp,hxx,kps,yml,yaml}] +indent_size = 2 + +[*.{py,pyi,java,r,toml}] +indent_size = 4 + +[Dockerfile.*] +indent_size = 4 + +[*.go] +indent_style = tab +indent_size = 4 diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 634f362..0000000 --- a/.flake8 +++ /dev/null @@ -1,22 +0,0 @@ -[flake8] -select = C,E,F,W -exclude = - ./build, -ignore = - # E, see https://pycodestyle.pycqa.org/en/latest/intro.html#error-codes - E203, - E401,E402, - E501, - E721,E722,E731,E741, - - # F, see https://flake8.pycqa.org/en/latest/user/error-codes.html - F405, - F811,F841, - - # W, see https://pycodestyle.pycqa.org/en/latest/intro.html#error-codes - W503 -per-file-ignores = - # Ignore unused imports in __init__.py - __init__.py: F401 - # Ignore undefined variables in CMake config and some dygraph_to_static tests - .cmake-format.py: F821 diff --git a/.github/workflows/check-bypass.yml b/.github/workflows/check-bypass.yml new file mode 100644 index 0000000..75521fc --- /dev/null +++ b/.github/workflows/check-bypass.yml @@ -0,0 +1,47 @@ +on: + workflow_call: + inputs: + workflow-name: + required: true + type: string + secrets: + github-token: + required: true + outputs: + can-skip: + description: "Whether the workflow can be skipped." + value: ${{ jobs.check-bypass.outputs.can-skip }} + +jobs: + check-bypass: + name: Check bypass + runs-on: ubuntu-latest + permissions: + contents: read + env: + CI_TEAM_MEMBERS: '["SigureMo", "BingooYang", "yongqiangma"]' + outputs: + can-skip: ${{ steps.check-bypass.outputs.can-skip }} + steps: + - id: check-bypass + name: Check Bypass + uses: PFCCLab/ci-bypass@v2 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + non-pull-request-event-strategy: "never-skipped" + type: "composite" + composite-rule: | + { + "any": [ + { + "type": "labeled", + "label": ["skip-ci: ${{ inputs.workflow-name }}", "skip-ci: all"], + "username": ${{ env.CI_TEAM_MEMBERS }} + }, + { + "type": "commented", + "comment-pattern": [".*/skip-ci ${{ inputs.workflow-name }}.*", ".*/skip-ci all.*"], + "username": ${{ env.CI_TEAM_MEMBERS }} + } + ] + } diff --git a/.github/workflows/codestyle-check.yml b/.github/workflows/codestyle-check.yml new file mode 100644 index 0000000..39e61ab --- /dev/null +++ b/.github/workflows/codestyle-check.yml @@ -0,0 +1,55 @@ +name: codestyle-check + +on: + pull_request: + branches: ["master"] + +jobs: + check-bypass: + name: Check bypass + uses: ./.github/workflows/check-bypass.yml + with: + workflow-name: "codestyle" + secrets: + github-token: ${{ secrets.GITHUB_TOKEN }} + + pre-commit: + name: Run pre-commit checks + needs: check-bypass + if: ${{ github.repository_owner == 'PFCCLab' && needs.check-bypass.outputs.can-skip != 'true' }} + runs-on: ubuntu-latest + env: + PR_ID: ${{ github.event.pull_request.number }} + + steps: + - name: Checkout base repo + uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.base.ref }} + fetch-depth: 200 + + - name: Merge PR to test branch + run: | + git fetch origin pull/${PR_ID}/merge + git checkout -b test FETCH_HEAD + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + python-version: "3.14" + enable-cache: true + + - name: Install dependencies + run: | + uv venv -p 3.14 + source .venv/bin/activate + uv pip install cpplint==1.6.0 clang-format==13.0.0 + + - name: Install prek + run: | + uv tool install prek + + - name: Run prek + run: | + source .venv/bin/activate + prek run --all-files diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 08e0ce4..c1f0299 100755 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,90 +1,77 @@ repos: -# Common hooks -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.1.0 + # Common hooks + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 hooks: - - id: check-added-large-files - - id: check-merge-conflict - - id: check-symlinks - - id: detect-private-key - - id: end-of-file-fixer - - id: trailing-whitespace - files: (.*\.(py|bzl|md|rst|c|cc|cxx|cpp|cu|h|hpp|hxx|xpu|kps|cmake|yaml|yml|hook|scpp)|BUILD|.*\.BUILD|WORKSPACE|CMakeLists\.txt)$ -- repo: https://github.com/Lucas-C/pre-commit-hooks.git - rev: v1.1.14 + - id: check-added-large-files + - id: check-merge-conflict + - id: check-symlinks + - id: detect-private-key + - id: end-of-file-fixer + - id: sort-simple-yaml + files: (ops|backward|op_[a-z_]+)\.yaml$ + - id: trailing-whitespace + - repo: https://github.com/Lucas-C/pre-commit-hooks.git + rev: v1.5.1 hooks: - - id: remove-crlf - - id: remove-tabs + - id: remove-crlf + - id: remove-tabs name: Tabs remover (C++) - files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|xpu|kps)$ + files: \.(c|cc|cxx|cpp|cu|h|cuh|hpp|hxx|xpu|kps)$ args: [--whitespaces-count, '2'] - - id: remove-tabs + - id: remove-tabs name: Tabs remover (Python) files: (.*\.(py|bzl)|BUILD|.*\.BUILD|WORKSPACE)$ args: [--whitespaces-count, '4'] -# For Python files -- repo: https://github.com/psf/black.git - rev: 22.8.0 + # For Python files + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.14.4 hooks: - - id: black - files: (.*\.(py|pyi|bzl)|BUILD|.*\.BUILD|WORKSPACE)$ -- repo: https://github.com/PyCQA/flake8 - rev: 4.0.1 + - id: ruff-check + args: [--fix, --exit-non-zero-on-fix, --no-cache] + - id: ruff-format + # For C++ files + - repo: local hooks: - - id: flake8 -- repo: https://github.com/PyCQA/autoflake - rev: v1.7.7 - hooks: - - id: autoflake - args: - - --in-place - - --remove-all-unused-imports - - --ignore-pass-after-docstring - - --ignore-init-module-imports - - --exclude=python/paddle/fluid/[!t]**,python/paddle/fluid/tra** -- repo: local - hooks: - - id: pylint-doc-string - name: pylint - description: Check python docstring style using docstring_checker. - entry: bash ./tools/codestyle/pylint_pre_commit.hook - language: system - files: \.(py)$ -# For C++ files -- repo: local - hooks: - - id: clang-format + - id: clang-format name: clang-format description: Format files with ClangFormat. - entry: bash ./tools/codestyle/clang_format.hook -i + entry: bash ./tools/codestyle/clang_format.sh -i language: system - files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|xpu|kps|mm|m|scpp)$ -- repo: local + files: \.(c|cc|cxx|cpp|cu|h|cuh|hpp|hxx|xpu|kps)$ + - repo: local hooks: - - id: cpplint-cpp-source + - id: cpplint-cpp-source name: cpplint description: Check C++ code style using cpplint.py. - entry: bash ./tools/codestyle/cpplint_pre_commit.hook + entry: bash ./tools/codestyle/cpplint_pre_commit.sh language: system - files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|scpp)$ + files: \.(cc|cxx|cpp|cu|h|hpp|hxx)$ args: - - --extensions=c,cc,cxx,cpp,cu,cuh,h,hpp,hxx,kps - - --filter=-readability/fn_size,-build/include_what_you_use,-build/c++11,-whitespace/parens,-legal/copyright - - --quiet -# For CMake files -- repo: local - hooks: - - id: auto-generate-cmakelists - name: auto-generate-cmakelists - entry: bash ./tools/gen_ut_cmakelists.hook - language: system - files: testslist.csv$ -- repo: https://github.com/cheshirekow/cmake-format-precommit + - --extensions=cc,cxx,cpp,cu,cuh,h,hpp,hxx,kps + - --filter=-readability/fn_size,-build/include_what_you_use,-build/c++11,-whitespace/parens,-legal/copyright + - --quiet + # For CMake files + - repo: https://github.com/cheshirekow/cmake-format-precommit rev: v0.6.13 hooks: - - id: cmake-format -- repo: https://github.com/cmake-lint/cmake-lint - rev: 1.4.2 + - id: cmake-format + - repo: https://github.com/PFCCLab/cmake-lint-paddle + rev: v1.5.1 hooks: - - id: cmakelint + - id: cmakelint args: [--config=./tools/codestyle/.cmakelintrc] + # For YAML files + - repo: https://github.com/PFCCLab/yamlfmt-pre-commit-mirror.git + rev: v0.16.0 + hooks: + - id: yamlfmt + files: | + (?x)^( + \.github/.+\.(yaml|yml)| + \.pre-commit-config\.yaml| + \.yamlfmt| + sgconfig\.yml| + ci/rules/.+\.yml| + ci/rule-tests/.+\.yml + ) diff --git a/.style.yapf b/.style.yapf deleted file mode 100644 index 4741fb4..0000000 --- a/.style.yapf +++ /dev/null @@ -1,3 +0,0 @@ -[style] -based_on_style = pep8 -column_limit = 80 diff --git a/.yamlfmt b/.yamlfmt new file mode 100644 index 0000000..01761f6 --- /dev/null +++ b/.yamlfmt @@ -0,0 +1,5 @@ +formatter: + indent: 2 + type: basic + retain_line_breaks: true + scan_folded_as_literal: true diff --git a/README.md b/README.md index 3fa5570..933309e 100644 --- a/README.md +++ b/README.md @@ -4,14 +4,16 @@ 一个用于验证 PaddlePaddle 和 PyTorch API 兼容性的 C++ 测试框架项目，依托 API 单元测试和持续守护流程，降低第三方库从 PyTorch 迁移到 PaddlePaddle 的技术门槛。 - ## 依赖要求 + ### 系统要求 + - CMake >= 3.18 - C++17 - Python 3.x (用于检测 PaddlePaddle) ### 第三方库依赖 + - PaddlePaddle (通过 Python 包自动检测) - PyTorch (libtorch，默认路径: /usr/lib/libtorch/) - Google Test (源码依赖，项目自动下载和构建) @@ -19,18 +21,21 @@ ## 快速开始 ### 1. 克隆项目 + ```bash git clone cd PaddleCPPAPITest ``` ### 2. 配置构建环境 + ```bash mkdir build && cd build cmake ../PaddleCPPAPITest -DTORCH_DIR= -G Ninja ``` ### 3. 编译项目 + ```bash ninja ``` @@ -38,16 +43,19 @@ ninja ### 4. 运行测试 #### 运行 PaddlePaddle 测试 + ```bash ./paddle/paddle_TensorTest ``` #### 运行 PyTorch 测试 + ```bash ./torch/torch_TensorTest ``` #### 运行所有测试 + ```bash ctest ``` @@ -55,6 +63,7 @@ ctest ## 代码风格 项目已配置以下代码风格工具： + - **clang-format**: C++ 代码格式化 -- **flake8**: Python 代码检查 +- **Ruff**: Python 代码检查 - **pre-commit**: Git 提交前检查 diff --git a/coverage/coverage_analysis.py b/coverage/coverage_analysis.py index 8705fc1..6f46c48 100644 --- a/coverage/coverage_analysis.py +++ b/coverage/coverage_analysis.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import os -import sys import re +import sys def parse_coverage_info(info_path): @@ -114,7 +114,11 @@ def get_cpp_functions(header_file): line_num = idx + 1 line = line.strip() # 跳过注释 - if line.startswith("//") or line.startswith("/*") or line.startswith("*"): + if ( + line.startswith("//") + or line.startswith("/*") + or line.startswith("*") + ): continue # 跳过宏定义 if line.startswith("#"): @@ -262,7 +266,11 @@ def main(): print(f"Scanning headers in {header_folder}...") for root, dirs, files in os.walk(header_folder): for file in files: - if file.endswith(".h") or file.endswith(".hpp") or file.endswith(".cuh"): + if ( + file.endswith(".h") + or file.endswith(".hpp") + or file.endswith(".cuh") + ): full_path = os.path.join(root, file) # 提取函数 funcs = get_cpp_functions(full_path) diff --git a/coverage/rename_func.py b/coverage/rename_func.py index e4523b8..16124d2 100644 --- a/coverage/rename_func.py +++ b/coverage/rename_func.py @@ -4,15 +4,19 @@ """ import re -import sys import subprocess +import sys def demangle_cpp_name(mangled_name): """使用 c++filt 解析 C++ 函数名""" try: result = subprocess.run( - ["c++filt"], input=mangled_name, capture_output=True, text=True, check=True + ["c++filt"], + input=mangled_name, + capture_output=True, + text=True, + check=True, ) return result.stdout.strip() except: @@ -63,21 +67,23 @@ def main(): ) try: - with open(input_file, "r", encoding="utf-8") as f_in: - with open(output_file, "w", encoding="utf-8") as f_out: - line_count = 0 - processed_count = 0 - - for line in f_in: - line_count += 1 - - if "FNA:" in line.upper(): - processed_line = process_fna_line(line) - if processed_line != line: - processed_count += 1 - f_out.write(processed_line) - else: - f_out.write(line) + with ( + open(input_file, "r", encoding="utf-8") as f_in, + open(output_file, "w", encoding="utf-8") as f_out, + ): + line_count = 0 + processed_count = 0 + + for line in f_in: + line_count += 1 + + if "FNA:" in line.upper(): + processed_line = process_fna_line(line) + if processed_line != line: + processed_count += 1 + f_out.write(processed_line) + else: + f_out.write(line) print(f"输入文件: {input_file}") print(f"输出文件: {output_file}") diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..bf96aaf --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,146 @@ +[tool.ruff] +exclude = ["./build", "third_party"] +line-length = 80 +target-version = "py310" + +[tool.ruff.format] +# Prevent change to double quotes by some users use ruff format +docstring-code-format = true +docstring-code-line-length = 120 + +[tool.ruff.lint] +select = [ + # Pycodestyle + "E", + "W", + + # Pyflakes + "F", + + # Isort + "I", + + # Comprehensions + "C4", + + # Debugger + "T100", + + # Pyupgrade + "UP", + + # Flake8-pyi + "PYI", + + # NumPy-specific rules + "NPY001", + "NPY003", + "NPY201", + + # Bugbear + "B002", + "B003", + "B004", + "B005", + "B009", + "B010", + "B011", + "B012", + "B013", + "B014", + "B015", + "B016", + "B017", + "B018", + "B019", + "B020", + "B021", + "B022", + "B025", + "B029", + "B032", + + # Pylint + "PLE", + "PLC3002", + "PLR0206", + "PLR0402", + "PLR1711", + "PLR1722", + "PLW3301", + + # Flake8-simplify + "SIM101", + "SIM117", + + # Pygrep-hooks + "PGH004", + + # Flake8-type-checking + "TC", + + # Ruff-specific rules + "RUF005", + "RUF008", + "RUF009", + "RUF010", + "RUF013", + "RUF015", + "RUF016", + "RUF017", + "RUF018", + "RUF019", + "RUF020", + "RUF024", + "RUF026", + "RUF100", + + # Flake8-raise + "RSE", + + # Flake8-quotes + "Q003", + "Q004", + + # Refurb + "FURB", + + # Flake8-future-annotations + "FA", +] +unfixable = ["NPY001"] +ignore = [ + # Whitespace before ‘,’, ‘;’, or ‘:’, it is not compatible with ruff format + "E203", + # Module level import not at top of file + "E402", + # Line too long (82 > 79 characters) + "E501", + # Do not compare types, use `isinstance()` + "E721", + # Do not use bare except, specify exception instead + "E722", + # Do not assign a lambda expression, use a def + "E731", + # Do not use variables named ‘l’, ‘O’, or ‘I’ + "E741", + # `name` may be undefined, or defined from star imports: `module` + "F405", + # Local variable name is assigned to but never used + "F841", + # It not met the "Explicit is better than implicit" rule + "UP015", + # collections.namedtuple can be quickly created a inlined class + "PYI024", + # `__all__.append` is a common pattern in Paddle + "PYI056", +] + +[tool.ruff.lint.isort] +combine-as-imports = true + +[tool.ruff.lint.per-file-ignores] +# Ignore for re-export in __init__ files +"__init__.py" = ["PLC0414"] +# Ignore undefined variables in CMake config and some dygraph_to_static tests +".cmake-format.py" = ["F821"] diff --git a/tools/codestyle/clang_format.hook b/tools/codestyle/clang_format.sh similarity index 100% rename from tools/codestyle/clang_format.hook rename to tools/codestyle/clang_format.sh diff --git a/tools/codestyle/cpplint_pre_commit.hook b/tools/codestyle/cpplint_pre_commit.sh similarity index 100% rename from tools/codestyle/cpplint_pre_commit.hook rename to tools/codestyle/cpplint_pre_commit.sh diff --git a/tools/codestyle/pre_commit.sh b/tools/codestyle/pre_commit.sh deleted file mode 100755 index fc71188..0000000 --- a/tools/codestyle/pre_commit.sh +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -x - -# use pre-commit 2.17 -if ! [[ $(pre-commit --version) == *"2.17.0"* ]]; then - pip install pre-commit==2.17.0 1>nul -fi - -# Install clang-format before git commit to avoid repeat installation due to -# pre-commit multi-thread running. -readonly VERSION="13.0.0" -version=$(clang-format -version) -if ! [[ $(python -V 2>&1 | awk '{print $2}' | awk -F '.' '{print $1$2}') -ge 36 ]]; then - echo "clang-format installation by pip need python version great equal 3.6, - please change the default python to higher version." - exit 1 -fi - -diff_files=$(git diff --numstat develop ':(exclude)Paddle' | awk '{print $NF}') -num_diff_files=$(echo "$diff_files" | wc -l) -echo -e "diff files between pr and develop:\n${diff_files}" - -echo "Checking code style by pre-commit ..." -pre-commit run --files ${diff_files};check_error=$? - -if test ! -z "$(git diff)"; then - echo -e '\n************************************************************************************' - echo -e "These files have been formatted by code format hook. You should use pre-commit to \ -format them before git push." - echo -e '************************************************************************************\n' - git diff 2>&1 -fi - -echo -e '\n************************************************************************************' -if [ ${check_error} != 0 ];then - echo "Your PR code style check failed." - echo "Please install pre-commit locally and set up git hook scripts:" - echo "" - echo " pip install pre-commit==2.17.0" - echo " pre-commit install" - echo "" - if [[ $num_diff_files -le 100 ]];then - echo "Then, run pre-commit to check codestyle issues in your PR:" - echo "" - echo " pre-commit run --files" $(echo ${diff_files} | tr "\n" " ") - echo "" - fi - echo "For more information, please refer to our codestyle check guide:" - echo "https://www.paddlepaddle.org.cn/documentation/docs/zh/develop/dev_guides/git_guides/codestyle_check_guide_cn.html" -else - echo "Your PR code style check passed." -fi -echo -e '************************************************************************************\n' - -exit ${check_error} diff --git a/tools/codestyle/pylint_pre_commit.hook b/tools/codestyle/pylint_pre_commit.hook deleted file mode 100755 index 2a01f53..0000000 --- a/tools/codestyle/pylint_pre_commit.hook +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -TOTAL_ERRORS=0 - - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -export PYTHONPATH=$DIR:$PYTHONPATH - -readonly VERSION="2.12.0" -version=$(pylint --version | grep 'pylint') - -if ! [[ $version == *"$VERSION"* ]]; then - pip install pylint==2.12.0 -fi - -# The trick to remove deleted files: https://stackoverflow.com/a/2413151 -for file in $(git diff --name-status | awk '$1 != "D" {print $2}'); do - # skip submodule of Paddle check - if ! [[ $file == "Paddle" ]]; then - pylint --disable=all --load-plugins=docstring_checker \ - --enable=doc-string-one-line,doc-string-end-with,doc-string-with-all-args,doc-string-triple-quotes,doc-string-missing,doc-string-indent-error,doc-string-with-returns,doc-string-with-raises $file; - TOTAL_ERRORS=$(expr $TOTAL_ERRORS + $?); - fi -done - -exit $TOTAL_ERRORS -#For now, just warning: -#exit 0