diff --git a/.circleci/config.yml b/.circleci/config.yml index c9316ec91..af5bfacf6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -46,18 +46,26 @@ commands: command: | curl -LsSf https://astral.sh/uv/install.sh | sh source $HOME/.local/bin/env - cache-extra-repos: + clone-repos: + parameters: + extra_repos: + type: boolean + default: true steps: - restore_cache: keys: - - extra-repos + - repos-<> - run: command: | - uv run --frozen gs codemod clone-repos --extra-repos --token ${CODEGEN_BOT_GHE_TOKEN} --clean-cache + EXTRA_REPOS_ARG="" + if [ "<>" = "true" ]; then + EXTRA_REPOS_ARG="--extra-repos" + fi + uv run --frozen gs codemod clone-repos ${EXTRA_REPOS_ARG} --token ${CODEGEN_BOT_GHE_TOKEN} --clean-cache - save_cache: paths: - $GITHUB_WORKSPACE - key: extra-repos + key: repos-<> fetch-verified-codemods: steps: - run: @@ -242,6 +250,33 @@ jobs: branch_pattern: "develop" channel: "alerts-codemod-tests" template: basic_fail_1 + oss-codemod-tests: + parameters: + sync_graph: + type: boolean + default: true + size: + type: string + default: "small" + parallelism: 2 + executor: default_vm + resource_class: xlarge + steps: + - install-uv + - setup-uv + - clone-repos: + extra_repos: false + - run_ats: + default_tests: "tests/codemod/test_codemods.py" + codecov_flags: "smart-tests-codemod-oss" + collect_args: --size=<> --sync-graph=<> --token ${CODEGEN_BOT_GHE_TOKEN} + ats_collect_args: --size=<>,--sync-graph=<>,--token=${CODEGEN_BOT_GHE_TOKEN}, + split_tests: false + - slack/notify: + event: fail + branch_pattern: "develop" + channel: "alerts-codemod-tests" + template: basic_fail_1 parse-tests: parameters: extra_repos: @@ -257,10 +292,15 @@ jobs: install-pnpm: true install-yarn: true use-nvm-cache: true - - cache-extra-repos + - clone-repos: + extra_repos: <> - run: command: | - PYTEST_ARGS='--extra-repos=<> --token ${CODEGEN_BOT_GHE_TOKEN} -o junit_suite_name="${CIRCLE_JOB}" tests/codemod/test_parse.py' + EXTRA_REPOS_ARG="" + if [ "<>" = "true" ]; then + EXTRA_REPOS_ARG="--extra-repos" + fi + PYTEST_ARGS="${EXTRA_REPOS_ARG} --token ${CODEGEN_BOT_GHE_TOKEN} -o junit_suite_name=\"${CIRCLE_JOB}\" tests/codemod/test_parse.py" echo "Running tests with args: $PYTEST_ARGS" TESTS_TO_RUN=$(PYTEST_ARGS=${PYTEST_ARGS} ./.circleci/collect.sh) echo $TESTS_TO_RUN | circleci tests run --command "ulimit -s unlimited; xargs uv run --frozen pytest -n auto ${PYTEST_ARGS}" @@ -370,26 +410,76 @@ workflows: - GHE requires: - hold-parse-tests + matrix: + parameters: + extra_repos: [true, false] + - parse-tests: + context: + - GHE + - slack + filters: + branches: + only: develop + extra_repos: true - parse-tests: + name: oss-parse-tests + context: + - GHE + - slack + filters: + branches: + only: develop + extra_repos: false + - oss-codemod-tests: + name: oss-codemod-tests context: - GHE - slack filters: branches: only: develop + matrix: + parameters: + sync_graph: [true, false] + size: [small, large] pr_checks: jobs: - - unit-tests: - context: - - Codecov + - unit-tests + - hold-codemod-tests: + type: approval + - hold-large-oss-codemod-tests: + type: approval - codemod-tests: context: - - Codecov - GHE - Codemod - slack + requires: + - hold-codemod-tests - pre-commit - doctests + - oss-codemod-tests: + filters: + branches: + ignore: develop + context: + - GHE + - slack + matrix: + parameters: + sync_graph: [true, false] + size: [small] + - oss-codemod-tests: + name: large-oss-codemod-tests + context: + - GHE + - slack + matrix: + parameters: + sync_graph: [true, false] + size: [large] + requires: + - hold-large-oss-codemod-tests publish-packages: jobs: - linux-wheels: diff --git a/.github/actions/report/action.yml b/.github/actions/report/action.yml deleted file mode 100644 index 7a9978c55..000000000 --- a/.github/actions/report/action.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: report -description: "Publish Junit Report" - -runs: - using: "composite" - steps: - - name: Upload Test Report - uses: actions/upload-artifact@v4 - with: - name: junit-test-results-${{github.run_id}}-${{github.job}}-${{ join(matrix.*, ' - ') }} - path: '**/build/test-results/test/TEST.xml' - retention-days: 1 - - name: Download Test Report - uses: actions/download-artifact@v4 - with: - pattern: junit-test-results-${{github.run_id}}* - - name: Publish Test Report - uses: EnricoMi/publish-unit-test-result-action@v2 - with: - files: "**/build/test-results/test/TEST.xml" - deduplicate_classes_by_file_name: true \ No newline at end of file diff --git a/.github/actions/setup-oss-repos/action.yml b/.github/actions/setup-oss-repos/action.yml deleted file mode 100644 index 30dca0a87..000000000 --- a/.github/actions/setup-oss-repos/action.yml +++ /dev/null @@ -1,27 +0,0 @@ -# yaml-language-server: $schema=https://json.schemastore.org/github-action.json -name: "Setup OSS repos" -description: "Setup OSS repos" -inputs: - CODEGEN_BOT_GHE_TOKEN: - required: true - description: "Codegen bot PAT for readonly GHE access" -runs: - using: "composite" - steps: - - name: Cache oss-repos - id: cache-oss-repos - uses: actions/cache@v4 - with: - path: oss_repos - key: ${{ runner.os }}-repo-cache-2-${{hashFiles('codegen-backend/codegen_tests/graph_sitter/codemod/repos/open_source/*.json')}} - - name: Populate oss-repos if the cache is empty - if: steps.cache-oss-repos.outputs.cache-hit != 'true' - shell: bash - run: | - uv run gs codemod clone-repos --clean-cache --token $CODEGEN_BOT_GHE_TOKEN - env: - GITHUB_WORKSPACE: $GITHUB_WORKSPACE - CODEGEN_BOT_GHE_TOKEN: ${{ inputs.CODEGEN_BOT_GHE_TOKEN }} - - name: Verify cache contents - shell: bash - run: ls -la $GITHUB_WORKSPACE/oss_repos/ diff --git a/.github/codecov.yml b/.github/codecov.yml index 634b84ff9..183deb54e 100644 --- a/.github/codecov.yml +++ b/.github/codecov.yml @@ -54,6 +54,12 @@ flag_management: statuses: - type: 'project' - type: 'patch' + - name: smart-tests-codemod-oss + carryforward: true + carryforward_mode: 'labels' + statuses: + - type: 'project' + - type: 'patch' comment: layout: "condensed_header, condensed_files" hide_project_coverage: true diff --git a/.github/workflows/cache-warm-up.yml b/.github/workflows/cache-warm-up.yml deleted file mode 100644 index d7b8f0aaf..000000000 --- a/.github/workflows/cache-warm-up.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: Cache Warm-Up - -on: - workflow_dispatch: - push: - branches: - - "develop" - -jobs: - - warm-up-cache-uv: - # changing the following value will significantly affect github's billing. Be careful and consult with the team before changing it. - strategy: - matrix: - os: [ ubuntu-latest, macos-latest ] - runs-on: ${{ matrix.os }} - - steps: - - name: Check if on develop branch - run: | - if [ "${{ github.ref }}" != "refs/heads/develop" ]; then - echo "Not on develop branch. Exiting." - exit 1 - fi - - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: develop # Ensure we're operating on the 'develop' branch - - - name: Setup backend - uses: ./.github/actions/setup-backend - warm-up-cache: - # changing the following value will significantly affect github's billing. Be careful and consult with the team before changing it. - runs-on: ubuntu-latest - needs: [warm-up-cache-uv] - steps: - - name: Check if on develop branch - run: | - if [ "${{ github.ref }}" != "refs/heads/develop" ]; then - echo "Not on develop branch. Exiting." - exit 1 - fi - - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: develop # Ensure we're operating on the 'develop' branch - - - name: Setup backend - uses: ./.github/actions/setup-backend - - name: Cache oss-repos - uses: ./.github/actions/setup-oss-repos - with: - CODEGEN_BOT_GHE_TOKEN: ${{ secrets.CODEGEN_BOT_GHE_TOKEN }} - - name: Setup-pre-commit - run: uv tool install pre-commit --with pre-commit-uv --force-reinstall - - uses: actions/cache@v4 - with: - path: ~/.cache/pre-commit - key: pre-commit-3|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} - - run: pre-commit install-hooks - shell: bash diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml deleted file mode 100644 index 04f10774e..000000000 --- a/.github/workflows/unit-tests.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: Unit Tests - -on: - push: - branches: - - "develop" - pull_request: - types: [ opened, synchronize, reopened, labeled ] - branches: - - "develop" - workflow_dispatch: - -jobs: - test_codemods: - # changing the following value will significantly affect github's billing. Be careful and consult with the team before changing it. - runs-on: ubuntu-latest-32 - strategy: - matrix: - sync_graph: [ true, false ] - size: [ small, large ] - exclude: - # Exclude large codemod tests when not needed - - size: ${{(contains(github.event.pull_request.labels.*.name, 'big-codemod-tests') || github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'kevin' || 'large'}} - - size: large - sync_graph: true - concurrency: - group: ${{ github.workflow }}-${{github.ref}}-${{matrix.sync_graph}}-${{matrix.size}}-${{github.event_name == 'push'&& github.sha}} - cancel-in-progress: true - name: "Codemod tests ${{matrix.size}}: Sync Graph=${{matrix.sync_graph}}" - steps: - - uses: actions/checkout@v4 - with: - ssh-key: ${{secrets.CODEGEN_BOT_SSH_KEY}} - - run: | - git lfs install - git lfs pull - - name: Setup backend - uses: ./.github/actions/setup-backend - - name: Cache oss-repos - uses: ./.github/actions/setup-oss-repos - with: - CODEGEN_BOT_GHE_TOKEN: ${{ secrets.CODEGEN_BOT_GHE_TOKEN }} - - name: Test with pytest - timeout-minutes: 12 - run: | - uv run pytest \ - -n auto \ - --size=${{matrix.size}} \ - --sync-graph=${{matrix.sync_graph}} \ - --token $CODEGEN_BOT_GHE_TOKEN \ - -o junit_suite_name="${{github.job}}" \ - tests/codemod/test_codemods.py::test_codemods_cloned_repos - env: - CODEGEN_BOT_GHE_TOKEN: ${{ secrets.CODEGEN_BOT_GHE_TOKEN }} - GITHUB_WORKSPACE: $GITHUB_WORKSPACE - - name: Install pycoverage - run: | - pip install coverage - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: build/test-results/test/TEST.xml diff --git a/uv.lock b/uv.lock index 2119ca4a2..ec89f17ec 100644 --- a/uv.lock +++ b/uv.lock @@ -716,7 +716,6 @@ dependencies = [ { name = "tree-sitter-javascript" }, { name = "tree-sitter-python" }, { name = "tree-sitter-typescript" }, - { name = "typer" }, { name = "typing-extensions" }, { name = "watchfiles" }, { name = "wrapt" }, @@ -802,7 +801,6 @@ requires-dist = [ { name = "tree-sitter-javascript", specifier = ">=0.23.1" }, { name = "tree-sitter-python", specifier = ">=0.23.4" }, { name = "tree-sitter-typescript", specifier = ">=0.23.2" }, - { name = "typer", specifier = ">=0.12.5,<1.0.0" }, { name = "types-networkx", marker = "extra == 'types'", specifier = ">=3.2.1.20240918" }, { name = "types-tabulate", marker = "extra == 'types'", specifier = ">=0.9.0.20240106" }, { name = "typing-extensions", specifier = ">=4.12.2" },