[Versioning] Better usage of pyproject.toml (#3089)

Author: vmoens

.github/workflows/nightly_build.yml

@@ -61,14 +61,14 @@ jobs:
           export PATH="/opt/python/${{ matrix.python_version[1] }}/bin:$PATH"
           python3 -mpip install --pre torch --index-url https://download.pytorch.org/whl/nightly/${{ matrix.cuda_support[1] }}
       - name: Build TorchRL Nightly
+        env:
+          TORCHRL_NIGHTLY: 1
         run: |
           rm -r dist || true
           export PATH="/opt/python/${{ matrix.python_version[1] }}/bin:$PATH"
-          python3 -mpip install wheel
-          python3 setup.py bdist_wheel \
-            --package_name torchrl-nightly \
-            --python-tag=${{ matrix.python-tag }}
-           find dist -name '*whl' -exec bash -c ' mv $0 ${0/linux/manylinux1}' {} \;
+          python3 -mpip install build wheel
+          ./build_nightly.sh
+          find dist -name '*whl' -exec bash -c ' mv $0 ${0/linux/manylinux1}' {} \;
       # pytorch/pytorch binaries are also manylinux_2_17 compliant but they
       # pretend that they're manylinux1 compliant so we do the same.
       - name: Upload wheel for the test-wheel job
@@ -112,14 +112,8 @@ jobs:
           python3 -mpip install --upgrade pip
       - name: Install tensordict
         run: |
-          if [[ "$OSTYPE" == "darwin"* ]]; then
-            brew install cmake
-          else
-            sudo apt-get update
-            sudo apt-get install -y cmake
-          fi
-          python3 -mpip install "pybind11[global]"
-          python3 -mpip install git+https://github.com/pytorch/tensordict.git
+          export PATH="/opt/python/${{ matrix.python_version[1] }}/bin:$PATH"
+          python3 -mpip install tensordict-nightly
       - name: Install test dependencies
         run: |
           export PATH="/opt/python/${{ matrix.python_version[1] }}/bin:$PATH"
@@ -141,6 +135,11 @@ jobs:
           rm -rf torchrl/
           export PATH="/opt/python/${{ matrix.python_version[1] }}/bin:$PATH"
           python3 -c "import torchrl; print(torchrl.__version__)"
+      - name: Verify nightly version format
+        if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+        run: |
+          export PATH="/opt/python/${{ matrix.python_version[1] }}/bin:$PATH"
+          python3 scripts/verify_nightly_version.py
       - name: Run tests
         run: |
           set -e
@@ -213,7 +212,7 @@ jobs:
         ]
     steps:
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python_version[1] }}
       - name: Checkout torchrl
@@ -223,13 +222,13 @@ jobs:
         run: |
           python3 -mpip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu -U
       - name: Build TorchRL nightly
+        env:
+          TORCHRL_NIGHTLY: 1
         shell: bash
         run: |
           rm -r dist || true
-          python3 -mpip install wheel
-          python3 setup.py bdist_wheel \
-            --package_name torchrl-nightly \
-            --python-tag=${{ matrix.python-tag }}
+          python3 -mpip install build wheel
+          ./build_nightly.sh
       - name: Upload wheel for the test-wheel job
         uses: actions/upload-artifact@v4
         with:
@@ -273,15 +272,7 @@ jobs:
       - name: Install tensordict
         shell: bash
         run: |
-          if ! choco -v &> /dev/null; then
-            powershell -NoProfile -InputFormat None -ExecutionPolicy Bypass -Command "iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))"
-          fi
-          # Install cmake using Chocolatey
-          choco install cmake -y
-          # Install necessary Python packages
-          python -m pip install --upgrade pip
-          python -m pip install "pybind11[global]"
-          python3 -mpip install git+https://github.com/pytorch/tensordict.git
+          python3 -mpip install tensordict-nightly
       - name: Download built wheels
         uses: actions/download-artifact@v4
         with:
@@ -297,6 +288,11 @@ jobs:
           # Avoid ambiguity of "import torchrl" by deleting the source files.
           rm -rf torchrl/
           python3 -c "import torchrl; print(torchrl.__version__)"
+      - name: Verify nightly version format
+        if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+        shell: bash
+        run: |
+          python3 scripts/verify_nightly_version.py
       - name: Run tests
         shell: bash
         run: |

MANIFEST.in

@@ -0,0 +1 @@
+include torchrl/version.py 

README.md

@@ -972,7 +972,8 @@ such as `apt`, `apt-get`, `conda` or `yum` but NOT `pip`, as well as `pip instal
 
 One can also build the wheels to distribute to co-workers using
 ```bash
-python setup.py bdist_wheel
+pip install build
+python -m build --wheel
 ```
 Your wheels will be stored there `./dist/torchrl<name>.whl` and installable via
 ```bash

benchmarks/test_compressed_storage_benchmark.py

@@ -8,6 +8,7 @@
 import pytest
 import torch
 
+
 try:
     from safetensors.torch import save
 except ImportError:

build_nightly.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+set -e
+
+# Check if we're in a nightly build
+if [ "$TORCHRL_NIGHTLY" != "1" ]; then
+    echo "Not a nightly build, exiting"
+    exit 0
+fi
+
+echo "Starting nightly build process..."
+
+# Create backups of original files
+cp pyproject.toml pyproject.toml.backup
+cp version.txt version.txt.backup
+
+# Function to restore original files
+restore_files() {
+    echo "Restoring original files..."
+    mv pyproject.toml.backup pyproject.toml
+    mv version.txt.backup version.txt
+}
+
+# Set up trap to restore files on exit (success or failure)
+trap restore_files EXIT
+
+# Modify pyproject.toml for nightly build
+echo "Modifying pyproject.toml for nightly build..."
+sed -i.bak 's/name = "torchrl"/name = "torchrl-nightly"/' pyproject.toml
+
+# Replace tensordict dependency with tensordict-nightly
+echo "Replacing tensordict with tensordict-nightly..."
+sed -i.bak 's/"tensordict[^"]*"/"tensordict-nightly"/g' pyproject.toml
+
+# Clean up sed backup files
+rm -f pyproject.toml.bak
+
+# Set nightly version (YYYY.MM.DD format)
+echo "Setting nightly version..."
+echo "$(date +%Y.%m.%d)" > version.txt
+
+# Build the package
+echo "Building nightly package..."
+echo "Python version: $(python --version)"
+echo "Platform: $(python -c 'import sys; print(sys.platform)')"
+
+# Install build dependencies
+echo "Installing build dependencies..."
+# Use setuptools 65.3.0 for Python <= 3.10 (fixes compatibility issues)
+# Use latest setuptools for Python 3.11+
+# 
+# Rationale:
+# - Python 3.9/3.10: setuptools 65.3.0 is more stable and fixes version detection issues
+#   that cause "0.0.0+unknown" version strings in nightly builds
+# - Python 3.11+: Newer setuptools versions are required for proper functionality
+#   with the latest Python features and build system changes
+PYTHON_VERSION=$(python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')")
+if python -c "import sys; exit(0 if sys.version_info < (3, 11) else 1)"; then
+    echo "Using setuptools 65.3.0 for Python $PYTHON_VERSION (compatibility mode)"
+    python -m pip install wheel setuptools==65.3.0
+else
+    echo "Using latest setuptools for Python $PYTHON_VERSION (modern mode)"
+    python -m pip install wheel setuptools
+fi
+
+python setup.py bdist_wheel
+
+echo "Nightly build completed successfully!" 

packaging/build_wheels.sh

@@ -7,10 +7,9 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 export BUILD_TYPE=wheel
 setup_env
 setup_wheel_python
-pip_install numpy pyyaml future ninja "pybind11[global]"
+pip_install numpy pyyaml future ninja "pybind11[global]" build
 pip_install --upgrade setuptools
 setup_pip_pytorch_version
-python setup.py clean
 
 # Copy binaries to be included in the wheel distribution
 if [[ "$(uname)" == Darwin || "$OSTYPE" == "msys" ]]; then
@@ -29,9 +28,9 @@ else
 fi
 
 if [[ "$OSTYPE" == "msys" ]]; then
-    IS_WHEEL=1 "$script_dir/windows/internal/vc_env_helper.bat" python setup.py bdist_wheel
+    IS_WHEEL=1 "$script_dir/windows/internal/vc_env_helper.bat" python -m build --wheel
 else
-    python setup.py bdist_wheel
+    python -m build --wheel
     if [[ "$(uname)" != Darwin ]]; then
       rename "linux_x86_64" "manylinux1_x86_64" dist/*.whl
     fi

packaging/prepare_nightly_build.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+import logging
+import os
+import re
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+def prepare_nightly_build():
+    """Prepare pyproject.toml for nightly builds by modifying dependencies."""
+    is_nightly = os.getenv("TORCHRL_NIGHTLY") == "1"
+
+    if not is_nightly:
+        logger.info("Not a nightly build, skipping pyproject.toml modification")
+        return
+
+    pyproject_path = Path("pyproject.toml")
+    if not pyproject_path.exists():
+        logger.info("pyproject.toml not found")
+        return
+
+    # Read the current pyproject.toml
+    with open(pyproject_path) as f:
+        content = f.read()
+
+    # Replace tensordict dependency with tensordict-nightly using regex
+    # This pattern matches "tensordict" followed by any version constraints
+    tensordict_pattern = r"tensordict[^,\]]*"
+    if re.search(tensordict_pattern, content):
+        content = re.sub(tensordict_pattern, "tensordict-nightly", content)
+        logger.info("Replaced tensordict with tensordict-nightly in pyproject.toml")
+    else:
+        logger.info("tensordict dependency not found in pyproject.toml")
+
+    # Write the modified content back
+    with open(pyproject_path, "w") as f:
+        f.write(content)
+
+    logger.info("pyproject.toml prepared for nightly build")
+
+
+if __name__ == "__main__":
+    prepare_nightly_build()

packaging/verify_nightly_version.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+import re
+import sys
+from datetime import date
+
+# Try to import torchrl with better error handling
+try:
+    import torchrl
+
+    print(f"Successfully imported torchrl from: {torchrl.__file__}")
+except ImportError as e:
+    print(f"Failed to import torchrl: {e}")
+    sys.exit(1)
+
+# Check if __version__ attribute exists
+if not hasattr(torchrl, "__version__"):
+    print(
+        f'Available attributes in torchrl: {[attr for attr in dir(torchrl) if not attr.startswith("_")]}'
+    )
+    raise AttributeError("torchrl module has no __version__ attribute")
+
+version = torchrl.__version__
+print(f"Checking version: {version}")
+
+# Check that version is not the major version (0.9.0)
+if re.match(r"^\d+\.\d+\.\d+$", version):
+    raise ValueError(f"Version should not be the major version: {version}")
+
+# Check that version matches date format (YYYY.M.D)
+date_pattern = r"^\d{4}\.\d{1,2}\.\d{1,2}$"
+if not re.match(date_pattern, version):
+    raise ValueError(f"Version should match date format YYYY.M.D, got: {version}")
+
+# Verify it's today's date
+today = date.today()
+expected_version = f"{today.year}.{today.month}.{today.day}"
+if version != expected_version:
+    raise ValueError(f"Version should be today date {expected_version}, got: {version}")
+
+print(f"✓ Version {version} is correctly formatted as nightly date")
+
+# Check that tensordict-nightly is installed (not stable tensordict)
+try:
+    import tensordict
+
+    # Check if it's the nightly version by looking at the version
+    tensordict_version = tensordict.__version__
+    print(f"Checking tensordict version: {tensordict_version}")
+
+    # Check if it's a nightly version (either date format or contains 'd' followed by date)
+    if re.match(date_pattern, tensordict_version) or "d2025" in tensordict_version:
+        print(f"✓ tensordict version {tensordict_version} appears to be nightly")
+    else:
+        # Check if it's a stable version that should not be used in nightly builds
+        if tensordict_version.startswith("0.9."):
+            raise ValueError(
+                f"tensordict should be nightly, not stable version: {tensordict_version}"
+            )
+        print(
+            f"⚠ tensordict version {tensordict_version} - please verify this is nightly"
+        )
+
+except ImportError:
+    raise ValueError(
+        "tensordict is not installed - nightly builds should include tensordict-nightly"
+    )