diff --git a/.github/workflows/_legacy-checkpoints.yml b/.github/workflows/_legacy-checkpoints.yml index 8cceb5870af8d..6c37281fc04ec 100644 --- a/.github/workflows/_legacy-checkpoints.yml +++ b/.github/workflows/_legacy-checkpoints.yml @@ -57,28 +57,32 @@ jobs: steps: - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - name: Install uv and set Python version + uses: astral-sh/setup-uv@v6 with: - # Python version here needs to be supported by all PL versions listed in back-compatible-versions.txt. python-version: "3.9" + # TODO: Avoid activating environment like this + # see: https://github.com/astral-sh/setup-uv/tree/v6/?tab=readme-ov-file#activate-environment + activate-environment: true + enable-cache: true - name: Install PL from source env: PACKAGE_NAME: pytorch FREEZE_REQUIREMENTS: 1 timeout-minutes: 20 - run: pip install . --extra-index-url="${TORCH_URL}" + run: uv pip install . --extra-index-url="${TORCH_URL}" if: inputs.pl_version == '' - name: Install PL version timeout-minutes: 20 - run: pip install "pytorch-lightning==${{ inputs.pl_version }}" --extra-index-url="${TORCH_URL}" + run: uv pip install "pytorch-lightning==${{ inputs.pl_version }}" --extra-index-url="${TORCH_URL}" if: inputs.pl_version != '' - name: Adjust tests -> PL if: ${{ matrix.pkg-name != 'lightning' }} run: | - pip install -q -r .actions/requirements.txt + uv pip install -q -r .actions/requirements.txt python .actions/assistant.py copy_replace_imports --source_dir="./tests" \ --source_import="lightning.fabric,lightning.pytorch" \ --target_import="lightning_fabric,pytorch_lightning" @@ -115,7 +119,7 @@ jobs: # export to env bool if secrets.AWS_REGION is not empty run: echo "WITH_SECRETS=$([ -n '${{ secrets.AWS_REGION }}' ] && echo 1 || echo 0)" >> $GITHUB_ENV - - run: pip install -r requirements/ci.txt + - run: uv pip install -r requirements/ci.txt - name: Upload checkpoints to S3 if: ${{ env.WITH_SECRETS == '1' }} working-directory: ${{ env.LEGACY_FOLDER }} diff --git a/.github/workflows/ci-tests-pytorch.yml b/.github/workflows/ci-tests-pytorch.yml index f9cfc9a843b6c..86f0a58089481 100644 --- a/.github/workflows/ci-tests-pytorch.yml +++ b/.github/workflows/ci-tests-pytorch.yml @@ -69,52 +69,49 @@ jobs: TORCH_URL_STABLE: "https://download.pytorch.org/whl/cpu/" TORCH_URL_TEST: "https://download.pytorch.org/whl/test/cpu/" FREEZE_REQUIREMENTS: ${{ ! (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release/')) }} - PYPI_CACHE_DIR: "_pip-wheels" # TODO: Remove this - Enable running MPS tests on this platform DISABLE_MPS: ${{ matrix.os == 'macOS-14' && '1' || '0' }} steps: - uses: actions/checkout@v5 - - name: Set up Python ${{ matrix.config.python-version }} - uses: actions/setup-python@v5 + - name: Install uv and set Python version + uses: astral-sh/setup-uv@v6 with: python-version: ${{ matrix.config.python-version || '3.9' }} + # TODO: Avoid activating environment like this + # see: https://github.com/astral-sh/setup-uv/tree/v6/?tab=readme-ov-file#activate-environment + activate-environment: true + enable-cache: true - - name: basic setup - run: pip install -q -r .actions/requirements.txt + - name: Basic setup + run: uv pip install -q -r .actions/requirements.txt - name: Append Env. vars for Linux if: ${{ runner.os == 'Linux' }} run: echo "GLOO_SOCKET_IFNAME=eth0" >> $GITHUB_ENV + - name: Append Env. vars for MacOS + if: ${{ runner.os == 'macOS' }} + run: echo "GLOO_SOCKET_IFNAME=lo0" >> $GITHUB_ENV - name: Set min. dependencies if: ${{ matrix.config.requires == 'oldest' }} run: | cd requirements/pytorch - pip install -U "lightning-utilities[cli]" + uv pip install -U "lightning-utilities[cli]" python -m lightning_utilities.cli requirements set-oldest --req_files "['base.txt', 'extra.txt', 'strategies.txt', 'examples.txt', 'test.txt']" - pip install "cython<3.0" wheel - pip install "pyyaml==5.4" --no-build-isolation + uv pip install "cython<3.0" wheel + uv pip install "pyyaml==5.4" --no-build-isolation - name: Adjust PyTorch versions in requirements files if: ${{ matrix.config.requires != 'oldest' }} run: | - pip install -q -r requirements/ci.txt + uv pip install -q -r requirements/ci.txt python -m wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py for fpath in `ls requirements/**/*.txt`; do \ python ./adjust-torch-versions.py $fpath ${{ matrix.config.pytorch-version }}; \ done cat requirements/pytorch/base.txt - - name: pip wheels cache - uses: actions/cache/restore@v4 - with: - path: ${{ env.PYPI_CACHE_DIR }} - key: pypi_wheels - - run: | - mkdir -p $PYPI_CACHE_DIR - ls -lh $PYPI_CACHE_DIR - - name: Env. variables run: | # Switch PyTorch URL between stable and test/future @@ -129,20 +126,22 @@ jobs: - name: Install package & dependencies timeout-minutes: 20 run: | - pip install ".[${EXTRA_PREFIX}extra,${EXTRA_PREFIX}test,${EXTRA_PREFIX}strategies]" \ - -U --upgrade-strategy=eager --prefer-binary \ + uv pip install ".[${EXTRA_PREFIX}extra,${EXTRA_PREFIX}test,${EXTRA_PREFIX}strategies]" \ + --upgrade \ -r requirements/_integrations/accelerators.txt \ - --extra-index-url="${TORCH_URL}" \ - --find-links="${PYPI_CACHE_DIR}" \ + --find-links="${TORCH_URL}" \ --find-links="https://download.pytorch.org/whl/torch-tensorrt" - pip list + uv pip list + - name: Drop LAI from extensions if: ${{ matrix.config.pkg-name != 'lightning' }} # Lightning is dependency of Habana or other accelerators/integrations so in case we test PL we need to remove it - run: pip uninstall -y lightning + run: uv pip uninstall lightning + - name: Drop PL for LAI if: ${{ matrix.config.pkg-name == 'lightning' }} - run: pip uninstall -y pytorch-lightning + run: uv pip uninstall pytorch-lightning + - name: Dump handy wheels if: github.event_name == 'push' && github.ref == 'refs/heads/master' continue-on-error: true @@ -219,6 +218,9 @@ jobs: name: CPU-coverage fail_ci_if_error: false + - name: Minimize uv cache + run: uv cache prune --ci + pl-cpu-guardian: runs-on: ubuntu-latest needs: pl-cpu diff --git a/tests/legacy/generate_checkpoints.sh b/tests/legacy/generate_checkpoints.sh index 1d083a2a8e052..d3cfa693c2906 100644 --- a/tests/legacy/generate_checkpoints.sh +++ b/tests/legacy/generate_checkpoints.sh @@ -7,18 +7,17 @@ set -e LEGACY_FOLDER=$(cd $(dirname $0); pwd -P) -printf "LEGACY_FOLDER: $LEGACY_FOLDER" +printf "LEGACY_FOLDER: $LEGACY_FOLDER\n" TESTS_FOLDER=$(dirname $LEGACY_FOLDER) -ENV_PATH=$LEGACY_FOLDER/vEnv -printf "ENV_PATH: $ENV_PATH" +ENV_PATH=$LEGACY_FOLDER/.venv +printf "ENV_PATH: $ENV_PATH\n" export PYTHONPATH=$TESTS_FOLDER # for `import tests_pytorch` -printf "PYTHONPATH: $PYTHONPATH" +printf "PYTHONPATH: $PYTHONPATH\n" rm -rf $ENV_PATH function create_and_save_checkpoint { - python --version - python -m pip --version - python -m pip list + uv --version + uv pip list python $LEGACY_FOLDER/simple_classif_training.py $pl_ver @@ -33,10 +32,10 @@ do printf "\n\n processing version: $pl_ver\n" # Don't install/update anything before activating venv to avoid breaking any existing environment. - python -m venv $ENV_PATH + uv venv $ENV_PATH source $ENV_PATH/bin/activate - python -m pip install "pytorch_lightning==$pl_ver" \ + uv pip install "pytorch_lightning==$pl_ver" \ -r $LEGACY_FOLDER/requirements.txt \ -r "$(dirname $TESTS_FOLDER)/requirements/pytorch/test.txt" \ -f https://download.pytorch.org/whl/cpu/torch_stable.html @@ -52,7 +51,7 @@ done if [[ -z "$@" ]]; then printf "\n\n processing local version\n" - python -m pip install \ + uv pip install \ -r $LEGACY_FOLDER/requirements.txt \ -r "$(dirname $TESTS_FOLDER)/requirements/pytorch/test.txt" \ -f https://download.pytorch.org/whl/cpu/torch_stable.html