diff --git a/.github/workflows/release_swerex_remote.yaml b/.github/workflows/release_swerex_remote.yaml new file mode 100644 index 0000000..05350d9 --- /dev/null +++ b/.github/workflows/release_swerex_remote.yaml @@ -0,0 +1,158 @@ +name: Release swerex-remote Executable + +on: + push: + branches: [ main ] + tags: + - 'v*' # Triggers on version tags like v1.0.0, v2.1.3, etc. + pull_request: + branches: [ main ] + workflow_dispatch: # Allow manual triggers + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - arch: amd64 + name: amd64 + - arch: arm64 + name: arm64 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build executable in compatible container + run: | + # Create build script + cat > build_script.sh << 'EOF' + #!/bin/bash + set -e + + # Update package lists + apt-get update + + # Install basic dependencies + apt-get install -y curl ca-certificates build-essential + + # Install uv + curl -LsSf https://astral.sh/uv/install.sh | sh + source $HOME/.local/bin/env + + # Create virtual environment with Python 3.13 + uv venv --python 3.13 .venv + source .venv/bin/activate + + # Install dependencies + uv pip install -e '.[dev]' + uv pip install pyinstaller + + # Build executable + pyinstaller src/swerex/server.py --onefile --name swerex-remote-$ARCH_NAME + + # Make executable and copy to output + chmod +x dist/swerex-remote-$ARCH_NAME + cp dist/swerex-remote-$ARCH_NAME /output/ + EOF + + chmod +x build_script.sh + + # Create output directory + mkdir -p output + + BASE_IMAGE="ubuntu:14.04" + + docker run --rm \ + --platform linux/${{ matrix.arch }} \ + -v $(pwd):/workspace \ + -v $(pwd)/output:/output \ + -w /workspace \ + -e ARCH_NAME=${{ matrix.name }} \ + $BASE_IMAGE \ + bash build_script.sh + + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: swerex-remote-${{ matrix.name }} + path: output/swerex-remote-${{ matrix.name }} + retention-days: 30 + + release: + needs: build + runs-on: ubuntu-latest + if: github.event_name == 'push' # Only create releases on push events (not PRs) + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + + - name: Prepare release assets + run: | + mkdir -p release_assets + cp artifacts/swerex-remote-amd64/swerex-remote-amd64 release_assets/ + cp artifacts/swerex-remote-arm64/swerex-remote-arm64 release_assets/ + + # Create checksums + cd release_assets + sha256sum * > checksums.txt + cd .. + + - name: Determine release type and details + id: release_info + run: | + if [[ $GITHUB_REF == refs/tags/v* ]]; then + # This is a version tag + VERSION=${GITHUB_REF#refs/tags/} + echo "type=version" >> $GITHUB_OUTPUT + echo "tag=$VERSION" >> $GITHUB_OUTPUT + echo "name=Release $VERSION" >> $GITHUB_OUTPUT + echo "prerelease=false" >> $GITHUB_OUTPUT + echo "body=Release $VERSION of SWE-ReX remote server" >> $GITHUB_OUTPUT + else + # This is a dev build from main/master branch + DATE=$(date +'%Y%m%d-%H%M%S') + COMMIT_SHORT=$(echo $GITHUB_SHA | cut -c1-7) + echo "type=dev" >> $GITHUB_OUTPUT + echo "tag=dev-$DATE-$COMMIT_SHORT" >> $GITHUB_OUTPUT + echo "name=Development Build $DATE" >> $GITHUB_OUTPUT + echo "prerelease=true" >> $GITHUB_OUTPUT + echo "body=Development build from commit $GITHUB_SHA" >> $GITHUB_OUTPUT + fi + + - name: Create or update release + uses: softprops/action-gh-release@v1 + with: + tag_name: ${{ steps.release_info.outputs.tag }} + name: ${{ steps.release_info.outputs.name }} + body: | + ${{ steps.release_info.outputs.body }} + + ## Download Links + - **Linux AMD64**: [swerex-remote-amd64](https://github.com/${{ github.repository }}/releases/download/${{ steps.release_info.outputs.tag }}/swerex-remote-amd64) + - **Linux ARM64**: [swerex-remote-arm64](https://github.com/${{ github.repository }}/releases/download/${{ steps.release_info.outputs.tag }}/swerex-remote-arm64) + - **Checksums**: [checksums.txt](https://github.com/${{ github.repository }}/releases/download/${{ steps.release_info.outputs.tag }}/checksums.txt) + + ${{ steps.release_info.outputs.type == 'version' && '## Always Latest Links + - **Linux AMD64**: [swerex-remote-amd64](https://github.com/' || '' }}${{ steps.release_info.outputs.type == 'version' && github.repository || '' }}${{ steps.release_info.outputs.type == 'version' && '/releases/latest/download/swerex-remote-amd64) + - **Linux ARM64**: [swerex-remote-arm64](https://github.com/' || '' }}${{ steps.release_info.outputs.type == 'version' && github.repository || '' }}${{ steps.release_info.outputs.type == 'version' && '/releases/latest/download/swerex-remote-arm64)' || '' }} + files: | + release_assets/swerex-remote-amd64 + release_assets/swerex-remote-arm64 + release_assets/checksums.txt + prerelease: ${{ steps.release_info.outputs.prerelease }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8538adb..ae60670 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ repos: exclude: pyproject.toml - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.12.8 + rev: v0.12.10 hooks: # Run the linter. - id: ruff diff --git a/src/swerex/deployment/docker.py b/src/swerex/deployment/docker.py index 71195a6..617a89f 100644 --- a/src/swerex/deployment/docker.py +++ b/src/swerex/deployment/docker.py @@ -1,10 +1,15 @@ +import asyncio import logging +import os import shlex import subprocess +import tempfile import time import uuid +from pathlib import Path from typing import Any +import requests from typing_extensions import Self from swerex import PACKAGE_NAME, REMOTE_EXECUTABLE_NAME @@ -21,6 +26,8 @@ __all__ = ["DockerDeployment", "DockerDeploymentConfig"] +REMOTE_EXECUTABLE_PATH = Path("/", REMOTE_EXECUTABLE_NAME) + def _is_image_available(image: str, runtime: str = "docker") -> bool: try: @@ -119,11 +126,7 @@ def _get_token(self) -> str: def _get_swerex_start_cmd(self, token: str) -> list[str]: rex_args = f"--auth-token {token}" - pipx_install = "python3 -m pip install pipx && python3 -m pipx ensurepath" - if self._config.python_standalone_dir: - cmd = f"{self._config.python_standalone_dir}/python3.11/bin/{REMOTE_EXECUTABLE_NAME} {rex_args}" - else: - cmd = f"{REMOTE_EXECUTABLE_NAME} {rex_args} || ({pipx_install} && pipx run {PACKAGE_NAME} {rex_args})" + cmd = f"chmod +x {REMOTE_EXECUTABLE_PATH} && {REMOTE_EXECUTABLE_PATH} --port 8000 {rex_args}" # Need to wrap with /bin/sh -c to avoid having '&&' interpreted by the parent shell return [ "/bin/sh", @@ -229,7 +232,7 @@ def _build_image(self) -> str: async def start(self): """Starts the runtime.""" - self._pull_image() + asyncio.to_thread(self._pull_image) if self._config.python_standalone_dir: image_id = self._build_image() else: @@ -245,32 +248,58 @@ async def start(self): rm_arg = [] if self._config.remove_container: rm_arg = ["--rm"] - cmds = [ + + image_arch = subprocess.check_output( + self._config.container_runtime + " inspect --format '{{.Architecture}}' " + image_id, shell=True, text=True + ).strip() + assert image_arch in {"amd64", "arm64"}, f"Unsupported architecture: {image_arch}" + t0 = time.time() + + def _start_and_copy(): + with tempfile.TemporaryDirectory() as temp_dir: + # download the remote server + tmp_exec_path = Path(temp_dir) / REMOTE_EXECUTABLE_NAME + exec_url = f"https://github.com/Co1lin/SWE-ReX/releases/latest/download/swerex-remote-{image_arch}" + self.logger.info(f"Downloading remote executable from {exec_url} to {tmp_exec_path}") + r_exec = requests.get(exec_url) + r_exec.raise_for_status() + with open(tmp_exec_path, "wb") as f: + f.write(r_exec.content) + # start the container + cmds_run = [ + self._config.container_runtime, + "run", + *rm_arg, + "-p", + f"{self._config.port}:8000", + *platform_arg, + *self._config.docker_args, + "--name", + self._container_name, + "-itd", + image_id, + ] + self.logger.info( + f"Starting container {self._container_name} with image {self._config.image} serving on port {self._config.port}: {shlex.join(cmds_run)}" + ) + subprocess.check_output(cmds_run, stderr=subprocess.STDOUT) + # copy the remote server executable into the container + self._copy_to(tmp_exec_path, REMOTE_EXECUTABLE_PATH) + + await asyncio.to_thread(_start_and_copy) + # execute the remote server + cmds_exec = [ self._config.container_runtime, - "run", - *rm_arg, - "-p", - f"{self._config.port}:8000", - *platform_arg, - *self._config.docker_args, - "--name", + "exec", self._container_name, - image_id, *self._get_swerex_start_cmd(token), ] - cmd_str = shlex.join(cmds) - self.logger.info( - f"Starting container {self._container_name} with image {self._config.image} serving on port {self._config.port}" - ) - self.logger.debug(f"Command: {cmd_str!r}") - # shell=True required for && etc. - self._container_process = subprocess.Popen(cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.logger.info(f"Executing remote server in container {self._container_name}: {shlex.join(cmds_exec)}") + self._container_process = subprocess.Popen(cmds_exec, stdout=subprocess.PIPE, stderr=subprocess.PIPE) self._hooks.on_custom_step("Starting runtime") - self.logger.info(f"Starting runtime at {self._config.port}") self._runtime = RemoteRuntime.from_config( RemoteRuntimeConfig(port=self._config.port, timeout=self._runtime_timeout, auth_token=token) ) - t0 = time.time() await self._wait_until_alive(timeout=self._config.startup_timeout) self.logger.info(f"Runtime started in {time.time() - t0:.2f}s") @@ -288,6 +317,7 @@ async def stop(self): stderr=subprocess.DEVNULL, timeout=10, ) + self.logger.info(f"Killed container {self._container_name}") except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: self.logger.warning( f"Failed to kill container {self._container_name}: {e}. Will try harder.", @@ -324,3 +354,38 @@ def runtime(self) -> RemoteRuntime: if self._runtime is None: raise DeploymentNotStartedError() return self._runtime + + def _copy_to(self, src: str, dst: str) -> None: + """ + Copies a file or directory from the host to the container. + + Args: + src (str): The path to the source file or directory on the host. + dst (str): The destination path inside the container. If `dst` ends + with '/', it's treated as a directory. + """ + # Separate the destination path into directory and filename + dst_dir, dst_filename = os.path.split(dst) + + # If dst is a directory path (e.g., "/path/to/dir/"), dst_filename will be empty. + # In this case, the destination filename should be the source filename. + if not dst_filename: + dst_filename = Path(src).name + dst_path = Path(dst_dir) / dst_filename + + # Step 1: docker cp (host -> container) + subprocess.check_output(["docker", "cp", src, f"{self._container_name}:{dst_path}"], stderr=subprocess.STDOUT) + + # Step 2: fix ownership to match container user + uid = subprocess.check_output( + ["docker", "exec", self._container_name, "id", "-u"], + text=True, + ).strip() + gid = subprocess.check_output( + ["docker", "exec", self._container_name, "id", "-g"], + text=True, + ).strip() + subprocess.check_output( + ["docker", "exec", self._container_name, "chown", "-R", f"{uid}:{gid}", dst_path], + stderr=subprocess.STDOUT, + )