Reapply "Fix building source distributions (#6381)"

This reverts commit a7b48c8.

Author: anmyachev

.github/workflows/create_release.yml

@@ -43,30 +43,31 @@ jobs:
           tag_or_branch="${tag_or_branch#refs/heads/}"
           # replace directory separators with _ in branch name
           tag_or_branch="${tag_or_branch//\//_}"
+          if [[ ${tag_or_branch} == v* ]]; then
+            # strip trailing v from tag name
+            tag_or_branch="${tag_or_branch#v}"
+            # important: version must be fixed in setup.py
+            sed -i -e "s:^TRITON_VERSION = .*:TRITON_VERSION = '${tag_or_branch}':" setup.py || exit 1
+          fi
           echo "RELEASE_NAME=triton-$tag_or_branch" >> "$GITHUB_ENV"
-          echo "RELEASE_FILE=triton-$tag_or_branch.tar.gz" >> "$GITHUB_ENV"
       - name: Create source distribution
         run: |
-            # Create new folder with specified name so extracting the archive yields that
-            rm -rf "/tmp/$RELEASE_NAME"
-            cp -r "$PWD" "/tmp/$RELEASE_NAME"
-            mv "/tmp/$RELEASE_NAME" .
-            # Cleanup
-            find "$RELEASE_NAME" -name '.git*' -exec rm -rv {} \; || true
-            # Create archive
-            tar -czf "$RELEASE_FILE" "$RELEASE_NAME"
-            echo "Created source archive $RELEASE_FILE with content: $(ls -a "$RELEASE_NAME")"
+          pip install build || exit 1
+          python -m build -s || exit 1
+          cd dist || exit 1
+          release_file=( *.tar.gz )
+          echo "RELEASE_FILE=${release_file}" >> "$GITHUB_ENV"
       - name: Upload source distribution for release
         if: ${{ github.event_name == 'release' }}
         uses: softprops/action-gh-release@v2
         with:
-          files: ${{env.RELEASE_FILE}}
+          files: dist/${{env.RELEASE_FILE}}
       - name: Upload source distribution to GHA artifacts for release tags
         if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && contains(github.ref, 'rc') }}
         uses: actions/[email protected]
         with:
           name: ${{ env.RELEASE_FILE }}
-          path: ${{ env.RELEASE_FILE }}
+          path: dist/${{ env.RELEASE_FILE }}
       - name: Set output
         id: release_name
         run: echo "name=release_name::${{ env.RELEASE_NAME }}.tar.gz" >> "${GITHUB_OUTPUT}"

.github/workflows/wheels.yml

@@ -79,7 +79,12 @@ jobs:
           export CIBW_BUILD="cp3{9,10,11,12,13,13t}-manylinux_${{ matrix.config.arch }}"
           export CIBW_SKIP="cp{35,36,37,38}-*"
           export CIBW_FREE_THREADED_SUPPORT=1
-          python3 -m cibuildwheel python --output-dir wheelhouse
+          python3 -m cibuildwheel . --output-dir wheelhouse
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: cibw-wheels-manylinux_2_28_${{ matrix.config.arch }}-wheels-upload
+          path: ./wheelhouse/*.whl
 
       - name: Install Azure CLI
         if: ${{ steps.check-version.outputs.new_commit == 'true' }}

MANIFEST.in

@@ -0,0 +1,18 @@
+graft bin
+graft cmake
+graft docs
+graft include
+graft lib
+graft python/src
+graft python/test
+graft python/triton/backends/amd
+graft python/triton/backends/nvidia
+graft python/triton/tools/extra/cuda
+graft test
+graft third_party
+graft unittest
+include CMakeLists.txt
+include Makefile
+include python/build_helpers.py
+include python/requirements.txt
+include python/test-requirements.txt

Makefile

@@ -79,7 +79,7 @@ dev-install-torch:
 
 .PHONY: dev-install-triton
 dev-install-triton:
-	$(PYTHON) -m pip install -e python --no-build-isolation -v
+	$(PYTHON) -m pip install -e . --no-build-isolation -v
 
 .PHONY: dev-install
 .NOPARALLEL: dev-install

README.md

@@ -71,7 +71,7 @@ git clone https://github.com/triton-lang/triton.git
 cd triton
 
 pip install -r python/requirements.txt # build-time dependencies
-pip install -e python
+pip install -e .
 ```
 
 Or with a virtualenv:
@@ -84,7 +84,7 @@ python -m venv .venv --prompt triton
 source .venv/bin/activate
 
 pip install -r python/requirements.txt # build-time dependencies
-pip install -e python
+pip install -e .
 ```
 
 # Building with a custom LLVM
@@ -124,7 +124,7 @@ arbitrary LLVM version.
        $ LLVM_INCLUDE_DIRS=$LLVM_BUILD_DIR/include \
          LLVM_LIBRARY_DIR=$LLVM_BUILD_DIR/lib \
          LLVM_SYSPATH=$LLVM_BUILD_DIR \
-         pip install -e python
+         pip install -e .
 
 # Tips for building
 
@@ -139,7 +139,7 @@ arbitrary LLVM version.
   can be changed anytime.
 
 - If you're running out of memory when building Triton, specify the `MAX_JOBS`
-  environment variable (to the `pip install -e python` command) to limit the
+  environment variable (to the `pip install -e .` command) to limit the
   number of jobs.
 
 - Pass `--no-build-isolation` to `pip install` to make nop builds faster.
@@ -150,7 +150,7 @@ arbitrary LLVM version.
   (probably because, in our build, users don't invoke cmake directly, but
   instead use setup.py).  Teach vscode how to compile Triton as follows.
 
-    - Do a local build. Run command `pip install -e python`
+    - Do a local build. Run command `pip install -e .`
     - Get the full path to the `compile_commands.json` file produced by the build:
       `find python/build -name 'compile_commands.json' | xargs readlink -f`.
       You might get a full path similar to `/Users/{username}/triton/python/build/cmake.macosx-11.1-arm64-cpython-3.12/compile_commands.json`

pyproject.toml

@@ -1,5 +1,6 @@
 [build-system]
-requires = ["setuptools>=78.1.0", "wheel", "cmake>=3.18", "ninja>=1.11.1"]
+requires = ["setuptools>=78.1.0", "cmake>=3.18", "ninja>=1.11.1", "pybind11>=2.13.1"]
+build-backend = "setuptools.build_meta"
 
 [tool.yapf]
 based_on_style = "pep8"
@@ -9,6 +10,12 @@ each_dict_entry_on_separate_line=false
 split_before_named_assigns = false
 split_complex_comprehension = true
 
+# We're incrementally switching from autopep8 to ruff.
+[tool.autopep8]
+aggressive = 1
+ignore = "E501,E701,E731,W690,W503"
+max_line_length = 88
+
 [tool.ruff]
 line-length = 120
 

python/build_helpers.py

@@ -12,6 +12,6 @@ def get_cmake_dir():
     plat_name = sysconfig.get_platform()
     python_version = sysconfig.get_python_version()
     dir_name = f"cmake.{plat_name}-{sys.implementation.name}-{python_version}"
-    cmake_dir = Path(get_base_dir()) / "python" / "build" / dir_name
+    cmake_dir = Path(get_base_dir()) / "build" / dir_name
     cmake_dir.mkdir(parents=True, exist_ok=True)
     return cmake_dir

python/triton/_C/include

@@ -1,10 +1,15 @@
 import importlib
 import inspect
-import os
+import sys
 from dataclasses import dataclass
 from .driver import DriverBase
 from .compiler import BaseBackend
 
+if sys.version_info >= (3, 10):
+    from importlib.metadata import entry_points
+else:
+    from importlib_metadata import entry_points
+
 
 def _find_concrete_subclasses(module, base_class):
     ret = []
@@ -27,16 +32,11 @@ class Backend:
 
 def _discover_backends():
     backends = dict()
-    root = os.path.dirname(__file__)
-    for name in os.listdir(root):
-        if not os.path.isdir(os.path.join(root, name)):
-            continue
-        if name.startswith('__'):
-            continue
-        compiler = importlib.import_module(f"triton.backends.{name}.compiler")
-        driver = importlib.import_module(f"triton.backends.{name}.driver")
-        backends[name] = Backend(_find_concrete_subclasses(compiler, BaseBackend),
-                                 _find_concrete_subclasses(driver, DriverBase))
+    for ep in entry_points().select(group="triton.backends"):
+        compiler = importlib.import_module(f"{ep.value}.compiler")
+        driver = importlib.import_module(f"{ep.value}.driver")
+        backends[ep.name] = Backend(_find_concrete_subclasses(compiler, BaseBackend),
+                                    _find_concrete_subclasses(driver, DriverBase))
     return backends