From 1843211e9a998d9d7eedf0e88df3c2eb8e9aedb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gonzalo=20Pe=C3=B1a-Castellanos?= Date: Wed, 11 Jun 2025 17:15:34 -0500 Subject: [PATCH 1/2] Use astral uv on CI testing for faster installs --- .github/workflows/code-style.yml | 8 ++- .github/workflows/docs.yml | 73 ++++++++++++++++++-------- .github/workflows/hvd-tests.yml | 33 +++++------- .github/workflows/install_docs_deps.sh | 11 ++-- .github/workflows/mps-tests.yml | 20 +++++-- .github/workflows/unit-tests.yml | 39 ++++++-------- docs/requirements.txt | 2 +- 7 files changed, 111 insertions(+), 75 deletions(-) diff --git a/.github/workflows/code-style.yml b/.github/workflows/code-style.yml index c7a74d699a12..d8ebbcc4d269 100644 --- a/.github/workflows/code-style.yml +++ b/.github/workflows/code-style.yml @@ -19,9 +19,15 @@ jobs: steps: - if: github.event_name == 'push' uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: astral-sh/setup-uv@v6 with: + version: "latest" python-version: "3.11" + activate-environment: true + enable-cache: true + cache-dependency-glob: | + **/requirements-dev.txt + **/pyproject.toml - run: | pip install -r requirements-dev.txt pre-commit run -a diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index b9f95c3cc7d2..456a99353f8e 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -17,18 +17,28 @@ jobs: permissions: contents: write if: (github.ref == 'refs/heads/master' && github.event_name == 'push') || github.event_name == 'release' - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} + strategy: + max-parallel: 10 + fail-fast: false + matrix: + os: ["ubuntu-latest"] + python-version: ["3.10"] steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: astral-sh/setup-uv@v6 with: - python-version: "3.10" + version: "latest" + python-version: "${{ matrix.python-version }}" + activate-environment: true + enable-cache: true + cache-suffix: "${{ hashFiles('requirements-dev.txt') }}-${{ hashFiles('docs/requirements.txt') }}" + cache-dependency-glob: | + **/docs/requirements.txt + **/requirements-dev.txt + **/pyproject.toml - run: sudo npm install katex -g - - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: pip-${{ hashFiles('requirements-dev.txt') }}-${{ hashFiles('docs/requirements.txt') }} - name: Install docs deps run: bash .github/workflows/install_docs_deps.sh @@ -47,18 +57,27 @@ jobs: linkcheck: if: github.event_name == 'pull_request' || github.event_name == 'push' - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} timeout-minutes: 10 + strategy: + max-parallel: 10 + fail-fast: false + matrix: + os: ["ubuntu-latest"] + python-version: ["3.10"] steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - uses: actions/cache@v4 + - uses: astral-sh/setup-uv@v6 with: - path: ~/.cache/pip - key: pip-${{ hashFiles('requirements-dev.txt') }}-${{ hashFiles('docs/requirements.txt') }} + version: "latest" + python-version: ${{ matrix.python-version }} + activate-environment: true + enable-cache: true + cache-suffix: "${{ hashFiles('requirements-dev.txt') }}-${{ hashFiles('docs/requirements.txt') }}" + cache-dependency-glob: | + **/docs/requirements.txt + **/requirements-dev.txt + **/pyproject.toml - name: Install docs deps run: bash .github/workflows/install_docs_deps.sh @@ -69,18 +88,28 @@ jobs: doctest: if: github.event_name == 'pull_request' || github.event_name == 'push' - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} + strategy: + max-parallel: 10 + fail-fast: false + matrix: + os: ["ubuntu-latest"] + python-version: ["3.10"] steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: astral-sh/setup-uv@v6 with: - python-version: "3.10" + version: "latest" + python-version: "${{ matrix.python-version }}" + activate-environment: true + enable-cache: true + cache-suffix: "${{ hashFiles('requirements-dev.txt') }}-${{ hashFiles('docs/requirements.txt') }}" + cache-dependency-glob: | + **/docs/requirements.txt + **/requirements-dev.txt + **/pyproject.toml - run: sudo npm install katex -g - - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: pip-${{ hashFiles('requirements-dev.txt') }}-${{ hashFiles('docs/requirements.txt') }} - name: Install docs deps run: bash .github/workflows/install_docs_deps.sh diff --git a/.github/workflows/hvd-tests.yml b/.github/workflows/hvd-tests.yml index efa73425549e..6ddd2d6bfc0d 100644 --- a/.github/workflows/hvd-tests.yml +++ b/.github/workflows/hvd-tests.yml @@ -39,31 +39,24 @@ jobs: run: echo "date=$(/bin/date "+%Y-%U")" >> $GITHUB_OUTPUT shell: bash -l {0} - - name: Get pip cache dir - id: pip-cache - run: | - python3 -m pip install -U pip - echo "pip_cache=$(python3 -m pip cache dir)" >> $GITHUB_OUTPUT - shell: bash -l {0} - - - uses: actions/cache@v4 - with: - path: | - ~/conda_pkgs_dir - ${{ steps.pip-cache.outputs.pip_cache }} - key: ${{ steps.get-date.outputs.date }}-horovod-${{ hashFiles('requirements-dev.txt') }} - - - uses: actions/setup-python@v5 + - uses: astral-sh/setup-uv@v6 with: + version: "latest" python-version: ${{ matrix.python-version }} + activate-environment: true + enable-cache: true + cache-suffix: "${{ steps.get-date.outputs.date }}-pytorch-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.pytorch-channel }}" + cache-dependency-glob: | + **/requirements-dev.txt + **/pyproject.toml - name: Install dependencies shell: bash -l {0} run: | - #install other dependencies - pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu - pip install -r requirements-dev.txt + # Install other dependencies + uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu + uv pip install -r requirements-dev.txt # Install Horovod from source and apply a patch to build with recent pytorch # We can't use pip install as build-env can't find pytorch and @@ -78,7 +71,9 @@ jobs: python -c "import horovod.torch as hvd; hvd.mpi_ops.Sum" # Install ignite - pip install . + uv pip install . + + uv pip list # Download MNIST: https://github.com/pytorch/ignite/issues/1737 # to "/tmp" for cpu tests diff --git a/.github/workflows/install_docs_deps.sh b/.github/workflows/install_docs_deps.sh index 499b64211943..3fd602c9b9fb 100644 --- a/.github/workflows/install_docs_deps.sh +++ b/.github/workflows/install_docs_deps.sh @@ -1,6 +1,7 @@ # remove pkg-resources as it causes failure when installing https://github.com/pytorch-ignite/sphinxcontrib-versioning -pip uninstall -y pkg-resources setuptools && pip install --upgrade setuptools pip wheel -pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu -U -pip install -r requirements-dev.txt -pip install -r docs/requirements.txt -pip install git+https://github.com/pytorch-ignite/sphinxcontrib-versioning.git +uv pip uninstall pkg-resources setuptools +uv pip install setuptools pip wheel +uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu +uv pip install -r requirements-dev.txt +uv pip install -r docs/requirements.txt +uv pip install git+https://github.com/pytorch-ignite/sphinxcontrib-versioning.git \ No newline at end of file diff --git a/.github/workflows/mps-tests.yml b/.github/workflows/mps-tests.yml index e7255c7d633f..1adad190270d 100644 --- a/.github/workflows/mps-tests.yml +++ b/.github/workflows/mps-tests.yml @@ -68,13 +68,20 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Install uv + shell: bash -l {0} + run: | + conda shell.bash hook + conda activate $CONDA_ENV + pip install uv + - name: Install PyTorch if: ${{ matrix.pytorch-channel == 'pytorch' }} shell: bash -l {0} run: | conda shell.bash hook conda activate $CONDA_ENV - pip install -U torch torchvision + uv pip install torch torchvision - name: Install PyTorch (nightly) if: ${{ matrix.pytorch-channel == 'pytorch-nightly' }} @@ -82,21 +89,22 @@ jobs: run: | conda shell.bash hook conda activate $CONDA_ENV - pip install --pre -U torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu + uv pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu - name: Install dependencies shell: bash -l {0} working-directory: ${{ github.repository }} run: | + conda shell.bash hook conda activate $CONDA_ENV # TODO: We add set -xe to explicitly fail the CI if one of the commands is failing. # Somehow the step is passing even if a subcommand failed set -xe - pip install -r requirements-dev.txt + uv pip install -r requirements-dev.txt echo "1 returned code: $?" - pip install -e . + uv pip install . echo "2 returned code: $?" - pip list + uv pip list echo "3 returned code: $?" # Download MNIST: https://github.com/pytorch/ignite/issues/1737 @@ -115,6 +123,7 @@ jobs: shell: bash -l {0} working-directory: ${{ github.repository }} run: | + conda shell.bash hook conda activate $CONDA_ENV SKIP_DISTRIB_TESTS=${{ matrix.skip-distrib-tests }} bash tests/run_cpu_tests.sh @@ -129,5 +138,6 @@ jobs: shell: bash -l {0} working-directory: ${{ github.repository }} run: | + conda shell.bash hook conda activate $CONDA_ENV python examples/mnist/mnist.py --epochs=1 diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index aa4635d7706c..5868e0a8451b 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -59,44 +59,39 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - name: Get year & week number id: get-date run: | echo "date=$(/bin/date "+%Y-%U")" >> $GITHUB_OUTPUT - - name: Get pip cache dir - id: pip-cache - run: | - pip install -U pip || python -m pip install -U pip - echo "pip_cache=$(pip cache dir)" >> $GITHUB_OUTPUT - - - uses: actions/cache@v4 + - uses: astral-sh/setup-uv@v6 with: - path: | - ${{ steps.pip-cache.outputs.pip_cache }} - key: ${{ steps.get-date.outputs.date }}-pytorch-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.pytorch-channel }}-${{ hashFiles('requirements-dev.txt') }} - restore-keys: | - ${{ steps.get-date.outputs.date }}-pytorch-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.pytorch-channel }}- + version: "latest" + python-version: ${{ matrix.python-version }} + activate-environment: true + enable-cache: true + cache-suffix: "${{ steps.get-date.outputs.date }}-pytorch-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.pytorch-channel }}" + cache-dependency-glob: | + **/requirements-dev.txt + **/pyproject.toml - - run: pip install pip wheel setuptools -Uqq + - name: Install dev tools + run: uv pip install wheel setuptools -Uqq - name: Install PyTorch if: ${{ matrix.pytorch-channel == 'pytorch' }} - run: pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu + run: uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu - name: Install PyTorch (nightly) if: ${{ matrix.pytorch-channel == 'pytorch-nightly' }} - run: pip install torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu --pre + run: uv pip install torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu --pre - name: Install dependencies run: | - pip install -r requirements-dev.txt - pip install . - pip list + uv pip install -r requirements-dev.txt + uv pip install . + uv pip list - name: Check code formatting run: | @@ -155,7 +150,7 @@ jobs: # 3.1) mnist_with_tensorboard.py with tbX python examples/mnist/mnist_with_tensorboard.py --epochs=1 # 3.2) mnist_with_tensorboard.py with native torch tb - pip uninstall -y tensorboardX + uv pip uninstall tensorboardX python examples/mnist/mnist_with_tensorboard.py --epochs=1 - name: Run MNIST Example With Crash diff --git a/docs/requirements.txt b/docs/requirements.txt index 9a88587a0ee2..8723edbff733 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ sphinx<6 --e git+https://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme +git+https://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme sphinxcontrib-katex sphinx-copybutton==0.4.0 docutils<0.18 From 208af5761e0bd234eba98d9af8a35de887ca9b63 Mon Sep 17 00:00:00 2001 From: vfdev Date: Thu, 19 Jun 2025 09:36:32 +0200 Subject: [PATCH 2/2] Apply suggestions from code review --- .github/workflows/install_docs_deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/install_docs_deps.sh b/.github/workflows/install_docs_deps.sh index 3fd602c9b9fb..f303a7af1798 100644 --- a/.github/workflows/install_docs_deps.sh +++ b/.github/workflows/install_docs_deps.sh @@ -4,4 +4,4 @@ uv pip install setuptools pip wheel uv pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu uv pip install -r requirements-dev.txt uv pip install -r docs/requirements.txt -uv pip install git+https://github.com/pytorch-ignite/sphinxcontrib-versioning.git \ No newline at end of file +uv pip install git+https://github.com/pytorch-ignite/sphinxcontrib-versioning.git