diff --git a/.github/actions/run_benchmark/action.yml b/.github/actions/run_benchmark/action.yml new file mode 100644 index 000000000..ca02babd3 --- /dev/null +++ b/.github/actions/run_benchmark/action.yml @@ -0,0 +1,41 @@ +name: 'Run Benchmark' +description: 'Run pytest with benchmarking and process results' + +inputs: + test_path: + description: 'Path to test files to run' + required: true + timeout: + description: 'Timeout in minutes' + required: false + default: '10' + suite_name: + description: 'Name of the test suite' + required: false + default: ${{ github.job }} + extra_args: + description: 'Additional pytest arguments' + required: false + default: '' + codecov_token: + description: 'Token for uploading to codecov' + required: false + codspeed_token: + description: 'Token for uploading to codspeed' + required: false + +runs: + using: 'composite' + steps: + - name: Make benchmark directory + shell: bash + run: mkdir -p build/test-results/test + - name: Run benchmarks + uses: CodSpeedHQ/action@v3 + with: + token: ${{ inputs.codspeed_token }} + run: bash -el -c "uv run --frozen pytest ${{ inputs.test_path }} ${{ inputs.extra_args }} -o junit_suite_name=${{ inputs.suite_name }}--codspeed" + - uses: ./.github/actions/report + with: + flag: no-flag + codecov_token: ${{ inputs.codecov_token }} diff --git a/.github/actions/run_pytest/action.yml b/.github/actions/run_pytest/action.yml new file mode 100644 index 000000000..c840afcdc --- /dev/null +++ b/.github/actions/run_pytest/action.yml @@ -0,0 +1,43 @@ +name: 'Run Pytest' +description: 'Run pytest with configurable parameters' + +inputs: + test_path: + description: 'Path to test files to run' + required: true + timeout: + description: 'Timeout in minutes' + required: false + default: '10' + suite_name: + description: 'Name of the test suite' + required: false + default: ${{ github.job }} + extra_args: + description: 'Additional pytest arguments' + required: false + default: '' + codecov_token: + description: 'Token for uploading to codecov' + required: false + +runs: + using: 'composite' + steps: + - name: Test with pytest + shell: bash + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + run: | + uv run pytest \ + -o junit_suite_name="${{ inputs.suite_name || github.job }}" \ + -n=auto \ + --timeout=$((${{ inputs.timeout }} * 60)) \ + ${{ inputs.extra_args }} \ + ${{ inputs.test_path }} + + - uses: ./.github/actions/report + if: inputs.codecov_token != '' + with: + flag: no-flag + codecov_token: ${{ inputs.codecov_token }} diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 1a7a6ddb1..066df6679 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -11,6 +11,20 @@ on: workflow_dispatch: jobs: + benchmark-tests: + runs-on: ubuntu-latest-16 + steps: + - uses: actions/checkout@v4 + - name: Setup backend + uses: ./.github/actions/setup-backend + + - name: Run benchmark tests + uses: ./.github/actions/run_benchmark + with: + test_path: tests/benchmark + timeout: 5 + codecov_token: ${{ secrets.CODECOV_TOKEN }} + codspeed_token: ${{ secrets.CODSPEED_TOKEN }} unit-tests: # changing the following value will significantly affect github's cost. Be careful and consult with the team before changing it. runs-on: ubuntu-latest-32 @@ -28,6 +42,7 @@ jobs: codecov_token: ${{ secrets.CODECOV_TOKEN }} collect_args: "--timeout 5" codecov_flags: unit-tests + codemod-tests: # changing the following value will significantly affect github's cost. Be careful and consult with the team before changing it. runs-on: ubuntu-latest-32 @@ -62,34 +77,6 @@ jobs: env: GITHUB_WORKSPACE: $GITHUB_WORKSPACE - # test_verified_codemods: - # # changing the following value will significantly affect github's cost. Be careful and consult with the team before changing it. - # if: false - # runs-on: ubuntu-latest-32 - # environment: testing - # concurrency: - # group: ${{ github.workflow }}-${{github.ref}}-${{github.event_name == 'push'&& github.sha}} - # cancel-in-progress: true - # name: "Verified Codemod tests: Sync Graph=false" - # steps: - # - uses: actions/checkout@v4 - # - name: Setup backend - # uses: ./.github/actions/setup-backend - # - name: Run ATS and Tests - # uses: ./.github/actions/run_ats - # with: - # default_tests: "tests/integration/codemod/test_verified_codemods.py" - # codecov_static_token: ${{ secrets.CODECOV_STATIC_TOKEN }} - # codecov_token: ${{ secrets.CODECOV_TOKEN }} - # collect_args: "--cli-api-key ${{ secrets.PROD_CLI_API_KEY }} --token ${{ secrets.CODEGEN_BOT_GHE_TOKEN }}" - # ats_collect_args: "--cli-api-key=${{ secrets.PROD_CLI_API_KEY }},--token=${{ secrets.CODEGEN_BOT_GHE_TOKEN }}," - # base_sha: ${{github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before}} - # job_name: ${{ github.job }} - # codecov_flags: smart-tests-verified-codemod-tests - # env: - # CODEGEN_BOT_GHE_TOKEN: ${{ secrets.CODEGEN_BOT_GHE_TOKEN }} - # GITHUB_WORKSPACE: $GITHUB_WORKSPACE - parse-tests: # changing the following value will significantly affect github's cost. Be careful and consult with the team before changing it. runs-on: ubuntu-latest-32 @@ -99,28 +86,20 @@ jobs: - uses: actions/checkout@v4 - name: Setup backend uses: ./.github/actions/setup-backend - - name: Cache oss-repos uses: ./.github/actions/setup-oss-repos - - name: Install yarn and pnpm run: | npm install -g yarn & npm install -g pnpm - - - name: Test with pytest - timeout-minutes: 15 - env: - GITHUB_WORKSPACE: $GITHUB_WORKSPACE - run: | - uv run pytest \ - -n auto \ - -o junit_suite_name="${{github.job}}" \ - tests/integration/codemod/test_parse.py - - uses: ./.github/actions/report + - name: Run benchmark tests + uses: ./.github/actions/run_benchmark with: - flag: no-flag + test_path: tests/integration/codemod/test_parse.py + extra_args: "--codspeed-max-rounds=1" + timeout: 15 codecov_token: ${{ secrets.CODECOV_TOKEN }} + codspeed_token: ${{ secrets.CODSPEED_TOKEN }} - name: Notify parse tests failure uses: slackapi/slack-github-action@v2.0.0 if: failure() && github.event_name == 'push' && false @@ -156,70 +135,16 @@ jobs: } ] } - # test_codemod_diffs: - # # changing the following value will significantly affect github's cost. Be careful and consult with the team before changing it. - # runs-on: ubuntu-latest-16 - # - # steps: - # - uses: actions/checkout@v4 - # - name: Setup backend - # uses: ./.github/actions/setup-backend - # - name: Cache oss-repos - # uses: ./.github/actions/setup-oss-repos - # with: - # CODEGEN_BOT_GHE_TOKEN: ${{ secrets.CODEGEN_BOT_GHE_TOKEN }} - # - name: Test with pytest - # timeout-minutes: 10 - # run: | - # ENV=local \ - # uv run pytest \ - # -n auto \ - # -vv \ - # --token $CODEGEN_BOT_GHE_TOKEN \ - # tests/codemod/test_diffs.py - # env: - # CODEGEN_BOT_GHE_TOKEN: ${{ secrets.CODEGEN_BOT_GHE_TOKEN }} - # GITHUB_WORKSPACE: $GITHUB_WORKSPACE - # - # - name: Publish Test Report (Verify diffs) - # uses: mikepenz/action-junit-report@v4 - # if: (success() || failure()) # always publish report even if the tests fail - # continue-on-error: true - # with: - # report_paths: "**/build/test-results/test/TEST.xml" - # detailed_summary: true - # annotate_only: true - # # Codecov is a required check but won't pass without a coverage report - # # Wwhen there are no changes in the backend (ex: frontend only change) we do an empty upload to force the check to pass - # - name: Upload empty coverage report to Codecov - # if: env.skip == '0' - # continue-on-error: true - # env: - # CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - # run: | - # cd codegen-backend - # pip install codecov-cli - # codecovcli create-commit -C ${{ github.event.pull_request.head.sha }} - # codecovcli create-report -C ${{ github.event.pull_request.head.sha }} - # codecovcli do-upload --disable-search --file empty_coverage.xml -C ${{ github.event.pull_request.head.sha }} - # codecovcli empty-upload --force -C ${{ github.event.pull_request.head.sha }} integration-tests: runs-on: ubuntu-latest-16 steps: - uses: actions/checkout@v4 - name: Setup backend uses: ./.github/actions/setup-backend - - name: Test with pytest - timeout-minutes: 5 - env: - GITHUB_WORKSPACE: $GITHUB_WORKSPACE - run: | - uv run pytest \ - -n auto \ - -o junit_suite_name="${{github.job}}" \ - tests/integration/codegen - - uses: ./.github/actions/report + - name: Run pytest + uses: ./.github/actions/run_pytest with: - flag: integration-tests + test_path: tests/integration/codegen + timeout: 5 codecov_token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.gitignore b/.gitignore index 1f54b61cc..6397ce6bb 100644 --- a/.gitignore +++ b/.gitignore @@ -65,3 +65,4 @@ graph-sitter-types/typings/** coverage.json tests/integration/verified_codemods/codemod_data/repo_commits.json .benchmarks/* +.codspeed/* diff --git a/pyproject.toml b/pyproject.toml index 2ed2133b5..2ff0cbcce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -140,8 +140,8 @@ dev-dependencies = [ "black>=24.8.0", "isort>=5.13.2", "emoji>=2.14.0", - "pytest-benchmark[histogram]>=5.1.0", "loguru>=0.7.3", + "pytest-codspeed>=3.2.0", ] keyring-provider = "subprocess" #extra-index-url = ["https://aws@codegen-922078275900.d.codeartifact.us-east-1.amazonaws.com/pypi/codegen/simple/"] diff --git a/tests/unit/codegen/sdk/benchmark/codebase/test_codebase_reset.py b/tests/benchmark/codegen/sdk/codebase/test_reset.py similarity index 95% rename from tests/unit/codegen/sdk/benchmark/codebase/test_codebase_reset.py rename to tests/benchmark/codegen/sdk/codebase/test_reset.py index 317b3b2c8..7324e0ecd 100644 --- a/tests/unit/codegen/sdk/benchmark/codebase/test_codebase_reset.py +++ b/tests/benchmark/codegen/sdk/codebase/test_reset.py @@ -31,9 +31,9 @@ def reset_codebase(codebase: Codebase): def test_codebase_reset_stress_test(extension: str, tmp_path, benchmark): def setup(): codebase, _ = setup_codebase(NUM_FILES, extension, tmp_path) - return ((codebase,), {}) + return codebase - benchmark.pedantic(reset_codebase, setup=setup) + benchmark(lambda: reset_codebase(setup())) @pytest.mark.skip("Skipping this test for now") diff --git a/tests/integration/codemod/test_parse.py b/tests/integration/codemod/test_parse.py index 0f5edb72f..55a394e82 100644 --- a/tests/integration/codemod/test_parse.py +++ b/tests/integration/codemod/test_parse.py @@ -4,6 +4,7 @@ import psutil import pytest +from pytest_codspeed import BenchmarkFixture from codegen.git.repo_operator.repo_operator import RepoOperator from codegen.sdk.codebase.config import CodebaseConfig, DefaultFlags, ProjectConfig @@ -18,7 +19,7 @@ @pytest.mark.timeout(60 * 12, func_only=True) -def test_codemods_parse(repo: Repo, op: RepoOperator, request) -> None: +def test_codemods_parse(repo: Repo, op: RepoOperator, request, codspeed_benchmark: BenchmarkFixture) -> None: # Setup Feature Flags if repo.feature_flags is not None: feature_flags = repo.feature_flags @@ -39,9 +40,9 @@ def test_codemods_parse(repo: Repo, op: RepoOperator, request) -> None: # Setup Codebase config = CodebaseConfig(feature_flags=feature_flags) projects = [ProjectConfig(repo_operator=op, programming_language=repo.language, subdirectories=repo.subdirectories)] - codebase = Codebase(projects=projects, config=config) - process = psutil.Process(os.getpid()) - memory_used = process.memory_info().rss + codebase = codspeed_benchmark(lambda: Codebase(projects=projects, config=config)) + memory_used = psutil.Process(os.getpid()).memory_info().rss + codspeed_benchmark.extra_info["memory_used"] = memory_used logger.info(f"Using {memory_used / BYTES_IN_GIGABYTE} GB of memory.") assert memory_used <= BYTES_IN_GIGABYTE * MAX_ALLOWED_GIGABYTES, "Graph is using too much memory!" validation_res = post_init_validation(codebase) diff --git a/uv.lock b/uv.lock index 4eb7025db..67ab138de 100644 --- a/uv.lock +++ b/uv.lock @@ -453,7 +453,7 @@ dev = [ { name = "pre-commit" }, { name = "pre-commit-uv" }, { name = "pytest" }, - { name = "pytest-benchmark", extra = ["histogram"] }, + { name = "pytest-codspeed" }, { name = "pytest-cov" }, { name = "pytest-mock" }, { name = "pytest-timeout" }, @@ -547,7 +547,7 @@ dev = [ { name = "pre-commit", specifier = ">=4.0.1" }, { name = "pre-commit-uv", specifier = ">=4.1.4" }, { name = "pytest", specifier = ">=8.3.3" }, - { name = "pytest-benchmark", extras = ["histogram"], specifier = ">=5.1.0" }, + { name = "pytest-codspeed", specifier = ">=3.2.0" }, { name = "pytest-cov", specifier = ">=6.0.0,<6.0.1" }, { name = "pytest-mock", specifier = ">=3.14.0,<4.0.0" }, { name = "pytest-timeout", specifier = ">=2.3.1" }, @@ -1082,18 +1082,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442 }, ] -[[package]] -name = "importlib-metadata" -version = "8.6.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "zipp" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/33/08/c1395a292bb23fd03bdf572a1357c5a733d3eecbab877641ceacab23db6e/importlib_metadata-8.6.1.tar.gz", hash = "sha256:310b41d755445d74569f993ccfc22838295d9fe005425094fad953d7f15c8580", size = 55767 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/79/9d/0fb148dc4d6fa4a7dd1d8378168d9b4cd8d4560a6fbf6f0121c5fc34eb68/importlib_metadata-8.6.1-py3-none-any.whl", hash = "sha256:02a89390c1e15fdfdc0d7c6b25cb3e62650d0494005c97d6f148bf5b9787525e", size = 26971 }, -] - [[package]] name = "inflect" version = "5.6.2" @@ -1708,15 +1696,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7b/d7/7831438e6c3ebbfa6e01a927127a6cb42ad3ab844247f3c5b96bea25d73d/psutil-6.1.1-cp37-abi3-win_amd64.whl", hash = "sha256:f35cfccb065fff93529d2afb4a2e89e363fe63ca1e4a5da22b603a85833c2649", size = 254444 }, ] -[[package]] -name = "py-cpuinfo" -version = "9.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335 }, -] - [[package]] name = "pycparser" version = "2.22" @@ -1793,27 +1772,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d4/d7/f1b7db88d8e4417c5d47adad627a93547f44bdc9028372dbd2313f34a855/pyflakes-3.2.0-py2.py3-none-any.whl", hash = "sha256:84b5be138a2dfbb40689ca07e2152deb896a65c3a3e24c251c5c62489568074a", size = 62725 }, ] -[[package]] -name = "pygal" -version = "3.0.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "importlib-metadata" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/49/7b/8f50821a0f1585881ef40ae13ecb7603b0d81ef99fedf992ec35e6b6f7d5/pygal-3.0.5.tar.gz", hash = "sha256:c0a0f34e5bc1c01975c2bfb8342ad521e293ad42e525699dd00c4d7a52c14b71", size = 80489 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/22/7d/b5d656dbeb73f488ce7409a75108a775f6cf8e20624ed8025a9476cbc1bb/pygal-3.0.5-py3-none-any.whl", hash = "sha256:a3268a5667b470c8fbbb0eca7e987561a7321caeba589d40e4c1bc16dbe71393", size = 129548 }, -] - -[[package]] -name = "pygaljs" -version = "1.0.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/75/19/3a53f34232a9e6ddad665e71c83693c5db9a31f71785105905c5bc9fbbba/pygaljs-1.0.2.tar.gz", hash = "sha256:0b71ee32495dcba5fbb4a0476ddbba07658ad65f5675e4ad409baf154dec5111", size = 89711 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/49/6f/07dab31ca496feda35cf3455b9e9380c43b5c685bb54ad890831c790da38/pygaljs-1.0.2-py2.py3-none-any.whl", hash = "sha256:d75e18cb21cc2cda40c45c3ee690771e5e3d4652bf57206f20137cf475c0dbe8", size = 91111 }, -] - [[package]] name = "pygit2" version = "1.17.0" @@ -2013,23 +1971,21 @@ wheels = [ ] [[package]] -name = "pytest-benchmark" -version = "5.1.0" +name = "pytest-codspeed" +version = "3.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "py-cpuinfo" }, + { name = "cffi" }, { name = "pytest" }, + { name = "rich" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/39/d0/a8bd08d641b393db3be3819b03e2d9bb8760ca8479080a26a5f6e540e99c/pytest-benchmark-5.1.0.tar.gz", hash = "sha256:9ea661cdc292e8231f7cd4c10b0319e56a2118e2c09d9f50e1b3d150d2aca105", size = 337810 } +sdist = { url = "https://files.pythonhosted.org/packages/03/98/16fe3895b1b8a6d537a89eecb120b97358df8f0002c6ecd11555d6304dc8/pytest_codspeed-3.2.0.tar.gz", hash = "sha256:f9d1b1a3b2c69cdc0490a1e8b1ced44bffbd0e8e21d81a7160cfdd923f6e8155", size = 18409 } wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/d6/b41653199ea09d5969d4e385df9bbfd9a100f28ca7e824ce7c0a016e3053/pytest_benchmark-5.1.0-py3-none-any.whl", hash = "sha256:922de2dfa3033c227c96da942d1878191afa135a29485fb942e85dff1c592c89", size = 44259 }, -] - -[package.optional-dependencies] -histogram = [ - { name = "pygal" }, - { name = "pygaljs" }, - { name = "setuptools" }, + { url = "https://files.pythonhosted.org/packages/0b/8b/9e95472589d17bb68960f2a09cfa8f02c4d43c82de55b73302bbe0fa4350/pytest_codspeed-3.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:46a1afaaa1ac4c2ca5b0700d31ac46d80a27612961d031067d73c6ccbd8d3c2b", size = 27182 }, + { url = "https://files.pythonhosted.org/packages/2a/18/82aaed8095e84d829f30dda3ac49fce4e69685d769aae463614a8d864cdd/pytest_codspeed-3.2.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c48ce3af3dfa78413ed3d69d1924043aa1519048dbff46edccf8f35a25dab3c2", size = 25933 }, + { url = "https://files.pythonhosted.org/packages/e2/15/60b18d40da66e7aa2ce4c4c66d5a17de20a2ae4a89ac09a58baa7a5bc535/pytest_codspeed-3.2.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:66692506d33453df48b36a84703448cb8b22953eea51f03fbb2eb758dc2bdc4f", size = 27180 }, + { url = "https://files.pythonhosted.org/packages/51/bd/6b164d4ae07d8bea5d02ad664a9762bdb63f83c0805a3c8fe7dc6ec38407/pytest_codspeed-3.2.0-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:479774f80d0bdfafa16112700df4dbd31bf2a6757fac74795fd79c0a7b3c389b", size = 25923 }, + { url = "https://files.pythonhosted.org/packages/f1/9b/952c70bd1fae9baa58077272e7f191f377c86d812263c21b361195e125e6/pytest_codspeed-3.2.0-py3-none-any.whl", hash = "sha256:54b5c2e986d6a28e7b0af11d610ea57bd5531cec8326abe486f1b55b09d91c39", size = 15007 }, ] [[package]] @@ -3006,12 +2962,3 @@ sdist = { url = "https://files.pythonhosted.org/packages/50/05/51dcca9a9bf5e1bce wheels = [ { url = "https://files.pythonhosted.org/packages/d6/45/fc303eb433e8a2a271739c98e953728422fa61a3c1f36077a49e395c972e/xmltodict-0.14.2-py2.py3-none-any.whl", hash = "sha256:20cc7d723ed729276e808f26fb6b3599f786cbc37e06c65e192ba77c40f20aac", size = 9981 }, ] - -[[package]] -name = "zipp" -version = "3.21.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3f/50/bad581df71744867e9468ebd0bcd6505de3b275e06f202c2cb016e3ff56f/zipp-3.21.0.tar.gz", hash = "sha256:2c9958f6430a2040341a52eb608ed6dd93ef4392e02ffe219417c1b28b5dd1f4", size = 24545 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/1a/7e4798e9339adc931158c9d69ecc34f5e6791489d469f5e50ec15e35f458/zipp-3.21.0-py3-none-any.whl", hash = "sha256:ac1bbe05fd2991f160ebce24ffbac5f6d11d83dc90891255885223d42b3cd931", size = 9630 }, -]