diff --git a/.github/workflows/preview.yaml b/.github/workflows/preview.yaml index bd84ecad..d6c7d59b 100644 --- a/.github/workflows/preview.yaml +++ b/.github/workflows/preview.yaml @@ -64,7 +64,7 @@ jobs: - name: Test python run: | - bazel test --incompatible_use_python_toolchains=false --python_path=$(which python) //libs/wrappers/python:rtbot_test + bazelisk test --incompatible_use_python_toolchains=false --python_path=$(which python) //libs/wrappers/python:rtbot_test - name: Test javascript run: bazelisk test //libs/wrappers/javascript:test diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index fdfb0d16..1d548fd7 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -44,13 +44,9 @@ jobs: - name: Test finance run: | bazelisk test //libs/finance/test - # - name: Test python - # run: | - # # TODO: make sure we don't have to build the wheel and install it - # # in order to make the test to pass - # bazelisk build //libs/core/wrappers/python:rtbot_wheel - # pip install --force-reinstall dist/bin/libs/core/wrappers/python/rtbot-_VERSION_-py3-none-manylinux2014_x86_64.whl - # bazelisk test //libs/core/wrappers/python:rtbot_test + - name: Test python + run: | + bazelisk test --incompatible_use_python_toolchains=false --python_path=$(which python) //libs/wrappers/python:rtbot_test - name: Test javascript run: | bazelisk test //libs/wrappers/javascript:test @@ -93,13 +89,12 @@ jobs: path: dist/bin/libs/api/jsonschema build_wheels: - name: Build wheel on ${{ matrix.os }} + name: Build wheel on ${{ matrix.os }} Python ${{ matrix.python-version }} runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04, macOS-11, windows-2019] - # TODO: add more python versions later, when we can afford the cost - #python: [3.11] + os: [ubuntu-20.04, macOS-12] + python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v3 - uses: pnpm/action-setup@v2.2.4 @@ -109,21 +104,49 @@ jobs: - args: [--frozen-lockfile] - uses: actions/setup-python@v4 with: - python-version: 3.11 #${{ matrix.python }} + python-version: ${{ matrix.python-version }} - name: Mount bazel caches uses: actions/cache@v3 with: path: | "~/.cache/bazel" - key: bazel-cache-${{ matrix.os }} + key: bazel-cache-${{ matrix.os }}-py${{ matrix.python-version }} - uses: bazelbuild/setup-bazelisk@v2 - name: Build wheel run: | bazelisk build --stamp //libs/wrappers/python:rtbot_wheel + - name: Rename wheel with platform tag + run: | + cd dist/bin/libs/wrappers/python/ + WHEEL_FILE="rtbot.whl" + if [ -f "$WHEEL_FILE" ]; then + # Extract version from wheel metadata + VERSION=$(python -c " +import zipfile, re +with zipfile.ZipFile('$WHEEL_FILE', 'r') as z: + for name in z.namelist(): + if '.dist-info/METADATA' in name: + # Extract version from rtbot-VERSION.dist-info/METADATA + dist_info_dir = name.split('/')[0] + version = dist_info_dir.replace('rtbot-', '').replace('.dist-info', '') + print(version) + break +") + # Determine platform tag + if [ "${{ runner.os }}" = "Linux" ]; then + PLATFORM="manylinux2014_x86_64" + else + PLATFORM="macosx_10_9_x86_64" + fi + # Create properly named wheel + NEW_NAME="rtbot-${VERSION}-py3-none-${PLATFORM}.whl" + cp "$WHEEL_FILE" "$NEW_NAME" + echo "Created wheel: $NEW_NAME" + fi - uses: actions/upload-artifact@v3 with: - name: wheel-${{ matrix.os }}.tar - path: dist/bin/libs/wrappers/python/rtbot-*.whl + name: wheel-${{ matrix.os }}-py${{ matrix.python-version }} + path: dist/bin/libs/wrappers/python/rtbot-*-py3-none-*.whl publish: needs: @@ -149,18 +172,12 @@ jobs: run: | GIT_TAG=${{ github.ref_name }} echo "version=${GIT_TAG:1}" >> $GITHUB_OUTPUT - - uses: actions/download-artifact@v3 - with: - name: wheel-ubuntu-20.04.tar - path: wheel-linux - - uses: actions/download-artifact@v3 - with: - name: wheel-macOS-11.tar - path: wheel-macos - - uses: actions/download-artifact@v3 + # Download all wheel artifacts + - name: Download all wheel artifacts + uses: actions/download-artifact@v3 with: - name: wheel-windows-2019.tar - path: wheel-windows + path: wheels/ + pattern: wheel-* - uses: actions/download-artifact@v3 with: name: npm-wasm.tar @@ -173,18 +190,14 @@ jobs: with: name: jsonschema.tar path: jsonschema - - name: Patch files before publishing + - name: Organize wheels for release run: | - ls -l - cd wheel-linux - mv rtbot-_VERSION_-py3-none-manylinux2014_x86_64.whl rtbot-${{ steps.version.outputs.version }}-py3-none-manylinux2014_x86_64.whl - cd .. - cd wheel-macos - mv rtbot-_VERSION_-py3-none-macosx_10_7_x86_64.whl rtbot-${{ steps.version.outputs.version }}-py3-none-macosx_10_7_x86_64.whl - cd .. - cd wheel-windows - mv rtbot-_VERSION_-py3-none-win_amd64.whl rtbot-${{ steps.version.outputs.version }}-py3-none-win_amd64.whl - cd .. + ls -la wheels/ + # Create directories for organizing wheels + mkdir -p wheel-organized + # Copy all wheels to organized directory + find wheels/ -name "*.whl" -exec cp {} wheel-organized/ \; + ls -la wheel-organized/ # copy repo readme to main npm package cp README.md ./npm-rtbot # package npm files, which will be used on the github release @@ -208,7 +221,5 @@ jobs: files: | *.tar.gz jsonschema/* - wheel-linux/* - wheel-macos/* - wheel-windows/* + wheel-organized/* fail_on_unmatched_files: true diff --git a/BUILD.bazel b/BUILD.bazel index 521731fe..af55b747 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -2,6 +2,8 @@ load("@aspect_rules_js//js:defs.bzl", "js_library") load("@aspect_rules_ts//ts:defs.bzl", "ts_config") load("@npm//:defs.bzl", "npm_link_all_packages") load("@aspect_rules_js//npm:defs.bzl", "npm_link_package") +load("@bazel_skylib//rules:write_file.bzl", "write_file") +load("@aspect_bazel_lib//lib:expand_template.bzl", "expand_template") package(default_visibility = ["//visibility:public"]) @@ -35,3 +37,17 @@ js_library( name = "package_json", srcs = ["package.json"], ) + +# Global version computation that can be reused across all targets +write_file( + name = "version_tmpl", + out = "version.txt.tmpl", + content = ["{{RTBOT_VERSION}}"], +) + +expand_template( + name = "version", + out = "version.txt", + stamp_substitutions = {"{{RTBOT_VERSION}}": "{{RTBOT_VERSION}}"}, + template = ":version_tmpl", +) diff --git a/libs/wrappers/python/BUILD.bazel b/libs/wrappers/python/BUILD.bazel index 3302e12f..043f4a41 100644 --- a/libs/wrappers/python/BUILD.bazel +++ b/libs/wrappers/python/BUILD.bazel @@ -2,6 +2,8 @@ load("@pybind11_bazel//:build_defs.bzl", "pybind_extension") load("@rules_python//python:packaging.bzl", "py_package", "py_wheel") load("//tools/generator:generator.bzl", "rtbot_generate") load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@aspect_bazel_lib//lib:expand_template.bzl", "expand_template") +load("@bazel_skylib//rules:write_file.bzl", "write_file") load("@py_deps//:requirements.bzl", "requirement") package(default_visibility = ["//visibility:public"]) @@ -68,25 +70,28 @@ copy_to_directory( }), ) -genrule( - name = "version", - outs = ["version.txt"], - cmd = "cat bazel-out/stable-status.txt | grep '^VERSION ' | cut -d' ' -f2 > $@", - stamp = 1, -) +# Use global version from root BUILD.bazel py_wheel( - name = "rtbot_wheel", + name = "rtbot_wheel_base", distribution = "rtbot", python_tag = "py3", stamp = 1, strip_path_prefixes = [ "libs/wrappers/python", ], - version = "0.1.0", # We'll need to read from version file in setup.py + version = "{RTBOT_VERSION}", # Dynamic version from workspace status deps = [":copy"], ) +genrule( + name = "rtbot_wheel", + srcs = [":rtbot_wheel_base", "//:version"], + outs = ["rtbot.whl"], + cmd = "cp $(location :rtbot_wheel_base) $(location rtbot.whl)", + stamp = 1, +) + py_test( name = "rtbot_test", srcs = ["rtbot_test.py"], diff --git a/libs/wrappers/python/setup.py b/libs/wrappers/python/setup.py index 64dd2713..92237794 100644 --- a/libs/wrappers/python/setup.py +++ b/libs/wrappers/python/setup.py @@ -1,135 +1,99 @@ +#!/usr/bin/env python3 +""" +Minimal setup.py for RtBot Python wrapper. + +This setup.py is designed to work with pre-built artifacts from Bazel. +For development builds, use: bazel build //libs/wrappers/python:rtbot_wheel +""" + import os -import platform -import subprocess import sys -import tempfile -import shutil -from setuptools import setup, Extension -from setuptools.command.build_ext import build_ext -from setuptools.command.egg_info import egg_info - -RTBOT_REPO = "https://github.com/rtbot-dev/rtbot.git" +from setuptools import setup, find_packages def get_version(): + """Get version from git tags, with dev suffix if ahead of latest tag.""" + import subprocess + try: - output = subprocess.check_output( - ['bash', '-c', "grep '^VERSION ' dist/out/stable-status.txt | cut -d' ' -f2"], + # Get the latest git tag + latest_tag = subprocess.check_output( + ['git', 'describe', '--tags', '--abbrev=0'], stderr=subprocess.PIPE ).decode().strip() - return output if output else "0.1.0" - except: - return "0.1.0" - -class BazelExtension(Extension): - def __init__(self, name): - super().__init__(name, sources=[]) -class CustomEggInfo(egg_info): - def run(self): - os.makedirs('rtbot', exist_ok=True) - super().run() + # Remove 'v' prefix if present + if latest_tag.startswith('v'): + latest_tag = latest_tag[1:] -class BazelBuildExt(build_ext): - def run(self): - self._install_bazelisk() - repo_dir = self._get_repo_dir() - self._build_rtbot(repo_dir) - - def _get_repo_dir(self): - current_dir = os.path.abspath(os.getcwd()) - while current_dir != '/': - if os.path.exists(os.path.join(current_dir, 'WORKSPACE')): - return current_dir - current_dir = os.path.dirname(current_dir) - - tmp_dir = tempfile.mkdtemp() - self._clone_repo(tmp_dir) - return tmp_dir + # Get current commit description + git_describe = subprocess.check_output( + ['git', 'describe', '--tags', '--always'], + stderr=subprocess.PIPE + ).decode().strip() - def _install_bazelisk(self): - try: - subprocess.check_call(['bazelisk', '--version']) - return - except (subprocess.CalledProcessError, FileNotFoundError): - pass + # If we're exactly on a tag, use that version + if git_describe == f'v{latest_tag}' or git_describe == latest_tag: + return latest_tag - print("Installing bazelisk...") - system = platform.system().lower() - arch = platform.machine().lower() - - if system == 'windows': - url = f"https://github.com/bazelbuild/bazelisk/releases/latest/download/bazelisk-windows-{arch}.exe" - out = "bazelisk.exe" - else: - url = f"https://github.com/bazelbuild/bazelisk/releases/latest/download/bazelisk-{system}-{arch}" - out = "bazelisk" - - subprocess.check_call(['curl', '-L', url, '-o', out]) - if system != 'windows': - os.chmod(out, 0o755) - - os.environ['PATH'] = os.getcwd() + os.pathsep + os.environ['PATH'] + # If we're ahead of the tag, create a dev version + # Format: v0.3.8-135-gc6ee14a -> 0.3.8.dev135+gc6ee14a + parts = git_describe.split('-') + if len(parts) >= 3: + tag_part = parts[0] + if tag_part.startswith('v'): + tag_part = tag_part[1:] + commits_ahead = parts[1] + short_sha = parts[2] + return f"{tag_part}.dev{commits_ahead}+{short_sha}" - def _clone_repo(self, tmp_dir): - print("Cloning RTBot repository...") - subprocess.check_call(['git', 'clone', '--depth', '1', RTBOT_REPO, tmp_dir]) + # Fallback: just use the tag + return latest_tag - def _get_bazel_bin(self, repo_dir): - bazel_cmd = 'bazelisk' if platform.system().lower() != 'windows' else 'bazelisk.exe' + except (subprocess.CalledProcessError, FileNotFoundError, IndexError): + # Fallback to reading from Bazel build if git fails try: - bazel_bin = subprocess.check_output( - [bazel_cmd, 'info', 'bazel-bin'], - cwd=repo_dir, - text=True - ).strip() - return bazel_bin - except subprocess.CalledProcessError: - return os.path.join(repo_dir, 'dist') # Fallback to --symlink_prefix value - - def _build_rtbot(self, repo_dir): - print("Building RTBot...") - bazel_cmd = 'bazelisk' if platform.system().lower() != 'windows' else 'bazelisk.exe' - - subprocess.check_call( - [bazel_cmd, 'build', '//libs/wrappers/python:rtbotapi.so', '//libs/wrappers/python:copy'], - cwd=repo_dir - ) - - bazel_bin = self._get_bazel_bin(repo_dir) - package_dir = os.path.join(self.build_lib, 'rtbot') - os.makedirs(package_dir, exist_ok=True) + version_file = "dist/bin/libs/wrappers/python/version.txt" + if os.path.exists(version_file): + with open(version_file, 'r') as f: + return f.read().strip() + except: + pass + return "0.1.0" - # Copy files from bazel-bin - copy_dir = os.path.join(bazel_bin, 'libs/wrappers/python/rtbot') - for item in ['MANIFEST.in', 'README.md', 'operators.py', 'setup.py', '__init__.py']: - src = os.path.join(copy_dir, item) - if os.path.exists(src): - shutil.copy2(src, package_dir) +# Check if we're in development mode (rtbotapi.so exists locally) +has_extension = os.path.exists("rtbotapi.so") or os.path.exists("rtbotapi.pyd") - # Copy the extension - ext_path = os.path.join(bazel_bin, 'libs/wrappers/python/rtbotapi.so') - if platform.system().lower() == 'windows': - ext_path = ext_path.replace('.so', '.pyd') - - if os.path.exists(ext_path): - shutil.copy2(ext_path, package_dir) - else: - raise RuntimeError(f"Built extension not found at {ext_path}") +if not has_extension: + print("Error: rtbot extension not found.") + print("This package requires pre-built artifacts from Bazel.") + print("Run: bazel build //libs/wrappers/python:copy") + print("Then copy the artifacts to this directory before running setup.py") + sys.exit(1) setup( name='rtbot', version=get_version(), - description='Python bindings for RTBot framework', - author='RTBot Developers', + description='Python bindings for RtBot framework', + long_description='RtBot is a real-time data processing framework with Python bindings.', + long_description_content_type='text/plain', + author='RtBot Developers', url='https://github.com/rtbot-dev/rtbot', - ext_modules=[BazelExtension('rtbot')], - cmdclass={ - 'build_ext': BazelBuildExt, - 'egg_info': CustomEggInfo, + packages=find_packages(), + package_data={ + 'rtbot': ['*.so', '*.pyd', '*.py'], }, - packages=['rtbot'], install_requires=[ "pandas>=1.0.0" ], python_requires='>=3.10', + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', + ], ) \ No newline at end of file diff --git a/tools/bazel_workspace_status.sh b/tools/bazel_workspace_status.sh index 46755b3c..c4b39bd8 100755 --- a/tools/bazel_workspace_status.sh +++ b/tools/bazel_workspace_status.sh @@ -27,3 +27,33 @@ echo "CURRENT_TIME $(date +%s)" echo "STABLE_GIT_COMMIT $(git rev-parse HEAD)" echo "STABLE_SHORT_GIT_COMMIT $(git rev-parse HEAD | cut -c 1-8)" echo "STABLE_USER_NAME $USER" + +# Generate dynamic version for RtBot Python wheel +if git describe --tags --abbrev=0 >/dev/null 2>&1; then + LATEST_TAG=$(git describe --tags --abbrev=0) + GIT_DESCRIBE=$(git describe --tags --always) + + # Remove 'v' prefix if present + LATEST_TAG=${LATEST_TAG#v} + + # If we're exactly on a tag, use that version + if [ "$GIT_DESCRIBE" = "v$LATEST_TAG" ] || [ "$GIT_DESCRIBE" = "$LATEST_TAG" ]; then + RTBOT_VERSION="$LATEST_TAG" + else + # Parse format: v0.3.8-135-gc6ee14a -> 0.3.8.dev135+gc6ee14a + TAG_PART=$(echo "$GIT_DESCRIBE" | cut -d'-' -f1) + TAG_PART=${TAG_PART#v} + COMMITS_AHEAD=$(echo "$GIT_DESCRIBE" | cut -d'-' -f2) + SHORT_SHA=$(echo "$GIT_DESCRIBE" | cut -d'-' -f3) + + if [ -n "$TAG_PART" ] && [ -n "$COMMITS_AHEAD" ] && [ -n "$SHORT_SHA" ]; then + RTBOT_VERSION="${TAG_PART}.dev${COMMITS_AHEAD}+${SHORT_SHA}" + else + RTBOT_VERSION="$LATEST_TAG" + fi + fi +else + RTBOT_VERSION="0.1.0" +fi + +echo "RTBOT_VERSION $RTBOT_VERSION"