From e60731cbf5ed6006f6145e6c801d2b16acb8d3af Mon Sep 17 00:00:00 2001 From: Co1lin Date: Thu, 21 Aug 2025 18:17:47 -0400 Subject: [PATCH 01/10] ci: release swerex-remote executable --- .github/workflows/release_swerex_remote.yaml | 127 +++++++++++++++++++ .pre-commit-config.yaml | 2 +- 2 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/release_swerex_remote.yaml diff --git a/.github/workflows/release_swerex_remote.yaml b/.github/workflows/release_swerex_remote.yaml new file mode 100644 index 0000000..4d59977 --- /dev/null +++ b/.github/workflows/release_swerex_remote.yaml @@ -0,0 +1,127 @@ +name: Release swerex-remote Executable + +on: + push: + tags: + - 'v*' # Triggers on version tags like v1.0.0, v2.1.3, etc. + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + arch: [x86_64, aarch64] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build executable in Ubuntu 14.04 container + run: | + # Create build script + cat > build_script.sh << 'EOF' + #!/bin/bash + set -e + + # Update package lists + apt-get update + + # Install basic dependencies + apt-get install -y curl ca-certificates build-essential + + # Install uv + curl -LsSf https://astral.sh/uv/install.sh | sh + export PATH="$HOME/.cargo/bin:$PATH" + + # Create virtual environment with Python 3.11 + uv venv --python 3.13 .venv + source .venv/bin/activate + + # Install dependencies + uv pip install -e '.[dev]' + uv pip install pyinstaller + + # Build executable + pyinstaller src/swerex/server.py --onefile --name swerex-remote-$ARCH + + # Make executable and copy to output + chmod +x dist/swerex-remote-$ARCH + cp dist/swerex-remote-$ARCH /output/ + EOF + + chmod +x build_script.sh + + # Create output directory + mkdir -p output + + # Build using Ubuntu 14.04 with emulation for cross-platform builds + docker run --rm \ + --platform linux/${{ matrix.arch == 'x86_64' && 'amd64' || 'arm64' }} \ + -v $(pwd):/workspace \ + -v $(pwd)/output:/output \ + -w /workspace \ + -e ARCH=${{ matrix.arch }} \ + ubuntu:14.04 \ + bash build_script.sh + + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: swerex-remote-${{ matrix.arch }} + path: output/swerex-remote-${{ matrix.arch }} + retention-days: 30 + + release: + needs: build + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + + - name: Prepare release assets + run: | + mkdir -p release_assets + cp artifacts/swerex-remote-x86_64/swerex-remote-x86_64 release_assets/ + cp artifacts/swerex-remote-aarch64/swerex-remote-aarch64 release_assets/ + + # Create checksums + cd release_assets + sha256sum * > checksums.txt + cd .. + + - name: Extract version from tag + id: version + run: echo "version=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT + + - name: Create versioned release + uses: softprops/action-gh-release@v1 + with: + tag_name: ${{ steps.version.outputs.version }} + name: "Release ${{ steps.version.outputs.version }}" + body: | + Release ${{ steps.version.outputs.version }} of SWE-ReX remote server + + ## Download Links + - **Linux x86_64**: [swerex-remote-x86_64](https://github.com/${{ github.repository }}/releases/download/${{ steps.version.outputs.version }}/swerex-remote-x86_64) + - **Linux ARM64**: [swerex-remote-aarch64](https://github.com/${{ github.repository }}/releases/download/${{ steps.version.outputs.version }}/swerex-remote-aarch64) + - **Checksums**: [checksums.txt](https://github.com/${{ github.repository }}/releases/download/${{ steps.version.outputs.version }}/checksums.txt) + + ## Always Latest Links + - **Linux x86_64**: [swerex-remote-x86_64](https://github.com/${{ github.repository }}/releases/latest/download/swerex-remote-x86_64) + - **Linux ARM64**: [swerex-remote-aarch64](https://github.com/${{ github.repository }}/releases/latest/download/swerex-remote-aarch64) + files: | + release_assets/swerex-remote-x86_64 + release_assets/swerex-remote-aarch64 + release_assets/checksums.txt + prerelease: false + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8538adb..ae60670 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ repos: exclude: pyproject.toml - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.12.8 + rev: v0.12.10 hooks: # Run the linter. - id: ruff From 59c621eda629e2f80cf74d12eecbecd74dd0313a Mon Sep 17 00:00:00 2001 From: Co1lin Date: Thu, 21 Aug 2025 18:30:03 -0400 Subject: [PATCH 02/10] ci: release swerex-remote executable --- .github/workflows/release_swerex_remote.yaml | 54 ++++++++++++++------ 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/.github/workflows/release_swerex_remote.yaml b/.github/workflows/release_swerex_remote.yaml index 4d59977..eb78f35 100644 --- a/.github/workflows/release_swerex_remote.yaml +++ b/.github/workflows/release_swerex_remote.yaml @@ -2,8 +2,12 @@ name: Release swerex-remote Executable on: push: + branches: [ main ] tags: - 'v*' # Triggers on version tags like v1.0.0, v2.1.3, etc. + pull_request: + branches: [ main ] + workflow_dispatch: # Allow manual triggers jobs: build: @@ -36,7 +40,7 @@ jobs: curl -LsSf https://astral.sh/uv/install.sh | sh export PATH="$HOME/.cargo/bin:$PATH" - # Create virtual environment with Python 3.11 + # Create virtual environment with Python 3.13 uv venv --python 3.13 .venv source .venv/bin/activate @@ -77,6 +81,7 @@ jobs: release: needs: build runs-on: ubuntu-latest + if: github.event_name == 'push' # Only create releases on push events (not PRs) steps: - name: Checkout code @@ -98,30 +103,49 @@ jobs: sha256sum * > checksums.txt cd .. - - name: Extract version from tag - id: version - run: echo "version=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT + - name: Determine release type and details + id: release_info + run: | + if [[ $GITHUB_REF == refs/tags/v* ]]; then + # This is a version tag + VERSION=${GITHUB_REF#refs/tags/} + echo "type=version" >> $GITHUB_OUTPUT + echo "tag=$VERSION" >> $GITHUB_OUTPUT + echo "name=Release $VERSION" >> $GITHUB_OUTPUT + echo "prerelease=false" >> $GITHUB_OUTPUT + echo "body=Release $VERSION of SWE-ReX remote server" >> $GITHUB_OUTPUT + else + # This is a dev build from main/master branch + DATE=$(date +'%Y%m%d-%H%M%S') + COMMIT_SHORT=$(echo $GITHUB_SHA | cut -c1-7) + echo "type=dev" >> $GITHUB_OUTPUT + echo "tag=dev-$DATE-$COMMIT_SHORT" >> $GITHUB_OUTPUT + echo "name=Development Build $DATE" >> $GITHUB_OUTPUT + echo "prerelease=true" >> $GITHUB_OUTPUT + echo "body=Development build from commit $GITHUB_SHA" >> $GITHUB_OUTPUT + fi - - name: Create versioned release + - name: Create or update release uses: softprops/action-gh-release@v1 with: - tag_name: ${{ steps.version.outputs.version }} - name: "Release ${{ steps.version.outputs.version }}" + tag_name: ${{ steps.release_info.outputs.tag }} + name: ${{ steps.release_info.outputs.name }} body: | - Release ${{ steps.version.outputs.version }} of SWE-ReX remote server + ${{ steps.release_info.outputs.body }} ## Download Links - - **Linux x86_64**: [swerex-remote-x86_64](https://github.com/${{ github.repository }}/releases/download/${{ steps.version.outputs.version }}/swerex-remote-x86_64) - - **Linux ARM64**: [swerex-remote-aarch64](https://github.com/${{ github.repository }}/releases/download/${{ steps.version.outputs.version }}/swerex-remote-aarch64) - - **Checksums**: [checksums.txt](https://github.com/${{ github.repository }}/releases/download/${{ steps.version.outputs.version }}/checksums.txt) + - **Linux x86_64**: [swerex-remote-x86_64](https://github.com/${{ github.repository }}/releases/download/${{ steps.release_info.outputs.tag }}/swerex-remote-x86_64) + - **Linux ARM64**: [swerex-remote-aarch64](https://github.com/${{ github.repository }}/releases/download/${{ steps.release_info.outputs.tag }}/swerex-remote-aarch64) + - **Checksums**: [checksums.txt](https://github.com/${{ github.repository }}/releases/download/${{ steps.release_info.outputs.tag }}/checksums.txt) - ## Always Latest Links - - **Linux x86_64**: [swerex-remote-x86_64](https://github.com/${{ github.repository }}/releases/latest/download/swerex-remote-x86_64) - - **Linux ARM64**: [swerex-remote-aarch64](https://github.com/${{ github.repository }}/releases/latest/download/swerex-remote-aarch64) + ${{ steps.release_info.outputs.type == 'version' && '## Always Latest Links + - **Linux x86_64**: [swerex-remote-x86_64](https://github.com/' || '' }}${{ steps.release_info.outputs.type == 'version' && github.repository || '' }}${{ steps.release_info.outputs.type == 'version' && '/releases/latest/download/swerex-remote-x86_64) + - **Linux ARM64**: [swerex-remote-aarch64](https://github.com/' || '' }}${{ steps.release_info.outputs.type == 'version' && github.repository || '' }}${{ steps.release_info.outputs.type == 'version' && '/releases/latest/download/swerex-remote-aarch64)' || '' }} files: | release_assets/swerex-remote-x86_64 release_assets/swerex-remote-aarch64 release_assets/checksums.txt - prerelease: false + prerelease: ${{ steps.release_info.outputs.prerelease }} + make_latest: ${{ steps.release_info.outputs.type == 'version' }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From e12270264291906b5932c5f39f0d9910a5491210 Mon Sep 17 00:00:00 2001 From: Co1lin Date: Thu, 21 Aug 2025 18:31:34 -0400 Subject: [PATCH 03/10] fix: ci: release swerex-remote executable --- .github/workflows/release_swerex_remote.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_swerex_remote.yaml b/.github/workflows/release_swerex_remote.yaml index eb78f35..4476f66 100644 --- a/.github/workflows/release_swerex_remote.yaml +++ b/.github/workflows/release_swerex_remote.yaml @@ -38,7 +38,7 @@ jobs: # Install uv curl -LsSf https://astral.sh/uv/install.sh | sh - export PATH="$HOME/.cargo/bin:$PATH" + source $HOME/.local/bin/env # Create virtual environment with Python 3.13 uv venv --python 3.13 .venv From 0021b8975309e53881f4c19d8577a19e897f2e4d Mon Sep 17 00:00:00 2001 From: Co1lin Date: Thu, 21 Aug 2025 18:40:04 -0400 Subject: [PATCH 04/10] fix: ci: release swerex-remote executable --- .github/workflows/release_swerex_remote.yaml | 42 +++++++++++++------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/.github/workflows/release_swerex_remote.yaml b/.github/workflows/release_swerex_remote.yaml index 4476f66..74787d8 100644 --- a/.github/workflows/release_swerex_remote.yaml +++ b/.github/workflows/release_swerex_remote.yaml @@ -14,7 +14,11 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - arch: [x86_64, aarch64] + include: + - arch: amd64 + name: amd64 + - arch: arm64 + name: aarch64 steps: - name: Checkout code @@ -23,7 +27,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Build executable in Ubuntu 14.04 container + - name: Build executable in compatible container run: | # Create build script cat > build_script.sh << 'EOF' @@ -49,11 +53,11 @@ jobs: uv pip install pyinstaller # Build executable - pyinstaller src/swerex/server.py --onefile --name swerex-remote-$ARCH + pyinstaller src/swerex/server.py --onefile --name swerex-remote-$ARCH_NAME # Make executable and copy to output - chmod +x dist/swerex-remote-$ARCH - cp dist/swerex-remote-$ARCH /output/ + chmod +x dist/swerex-remote-$ARCH_NAME + cp dist/swerex-remote-$ARCH_NAME /output/ EOF chmod +x build_script.sh @@ -61,21 +65,29 @@ jobs: # Create output directory mkdir -p output - # Build using Ubuntu 14.04 with emulation for cross-platform builds + # Use different base images for different architectures to maximize compatibility + if [ "${{ matrix.arch }}" = "amd64" ]; then + # Use Ubuntu 14.04 for AMD64 (maximum compatibility with old systems) + BASE_IMAGE="ubuntu:14.04" + else + # Use Ubuntu 16.04 for ARM64 (first Ubuntu with official ARM64 support) + BASE_IMAGE="ubuntu:16.04" + fi + docker run --rm \ - --platform linux/${{ matrix.arch == 'x86_64' && 'amd64' || 'arm64' }} \ + --platform linux/${{ matrix.arch }} \ -v $(pwd):/workspace \ -v $(pwd)/output:/output \ -w /workspace \ - -e ARCH=${{ matrix.arch }} \ - ubuntu:14.04 \ + -e ARCH_NAME=${{ matrix.name }} \ + $BASE_IMAGE \ bash build_script.sh - name: Upload build artifacts uses: actions/upload-artifact@v4 with: - name: swerex-remote-${{ matrix.arch }} - path: output/swerex-remote-${{ matrix.arch }} + name: swerex-remote-${{ matrix.name }} + path: output/swerex-remote-${{ matrix.name }} retention-days: 30 release: @@ -95,7 +107,7 @@ jobs: - name: Prepare release assets run: | mkdir -p release_assets - cp artifacts/swerex-remote-x86_64/swerex-remote-x86_64 release_assets/ + cp artifacts/swerex-remote-amd64/swerex-remote-amd64 release_assets/ cp artifacts/swerex-remote-aarch64/swerex-remote-aarch64 release_assets/ # Create checksums @@ -134,15 +146,15 @@ jobs: ${{ steps.release_info.outputs.body }} ## Download Links - - **Linux x86_64**: [swerex-remote-x86_64](https://github.com/${{ github.repository }}/releases/download/${{ steps.release_info.outputs.tag }}/swerex-remote-x86_64) + - **Linux AMD64**: [swerex-remote-amd64](https://github.com/${{ github.repository }}/releases/download/${{ steps.release_info.outputs.tag }}/swerex-remote-amd64) - **Linux ARM64**: [swerex-remote-aarch64](https://github.com/${{ github.repository }}/releases/download/${{ steps.release_info.outputs.tag }}/swerex-remote-aarch64) - **Checksums**: [checksums.txt](https://github.com/${{ github.repository }}/releases/download/${{ steps.release_info.outputs.tag }}/checksums.txt) ${{ steps.release_info.outputs.type == 'version' && '## Always Latest Links - - **Linux x86_64**: [swerex-remote-x86_64](https://github.com/' || '' }}${{ steps.release_info.outputs.type == 'version' && github.repository || '' }}${{ steps.release_info.outputs.type == 'version' && '/releases/latest/download/swerex-remote-x86_64) + - **Linux AMD64**: [swerex-remote-amd64](https://github.com/' || '' }}${{ steps.release_info.outputs.type == 'version' && github.repository || '' }}${{ steps.release_info.outputs.type == 'version' && '/releases/latest/download/swerex-remote-amd64) - **Linux ARM64**: [swerex-remote-aarch64](https://github.com/' || '' }}${{ steps.release_info.outputs.type == 'version' && github.repository || '' }}${{ steps.release_info.outputs.type == 'version' && '/releases/latest/download/swerex-remote-aarch64)' || '' }} files: | - release_assets/swerex-remote-x86_64 + release_assets/swerex-remote-amd64 release_assets/swerex-remote-aarch64 release_assets/checksums.txt prerelease: ${{ steps.release_info.outputs.prerelease }} From d1e94ec717afae402de75ddf9c816883bfe8dd15 Mon Sep 17 00:00:00 2001 From: Co1lin Date: Thu, 21 Aug 2025 18:52:18 -0400 Subject: [PATCH 05/10] fix: ci: release swerex-remote executable, add qemu --- .github/workflows/release_swerex_remote.yaml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release_swerex_remote.yaml b/.github/workflows/release_swerex_remote.yaml index 74787d8..e1e8a17 100644 --- a/.github/workflows/release_swerex_remote.yaml +++ b/.github/workflows/release_swerex_remote.yaml @@ -24,6 +24,9 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -65,14 +68,7 @@ jobs: # Create output directory mkdir -p output - # Use different base images for different architectures to maximize compatibility - if [ "${{ matrix.arch }}" = "amd64" ]; then - # Use Ubuntu 14.04 for AMD64 (maximum compatibility with old systems) - BASE_IMAGE="ubuntu:14.04" - else - # Use Ubuntu 16.04 for ARM64 (first Ubuntu with official ARM64 support) - BASE_IMAGE="ubuntu:16.04" - fi + BASE_IMAGE="ubuntu:14.04" docker run --rm \ --platform linux/${{ matrix.arch }} \ From df22b424bb8889c9af174586a0755afa69188216 Mon Sep 17 00:00:00 2001 From: Co1lin Date: Thu, 21 Aug 2025 20:06:35 -0400 Subject: [PATCH 06/10] fix: ci: release_swerex_remote.yaml: remove make_latest --- .github/workflows/release_swerex_remote.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/release_swerex_remote.yaml b/.github/workflows/release_swerex_remote.yaml index e1e8a17..71a0b07 100644 --- a/.github/workflows/release_swerex_remote.yaml +++ b/.github/workflows/release_swerex_remote.yaml @@ -154,6 +154,5 @@ jobs: release_assets/swerex-remote-aarch64 release_assets/checksums.txt prerelease: ${{ steps.release_info.outputs.prerelease }} - make_latest: ${{ steps.release_info.outputs.type == 'version' }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 1848ff99e41a49bdb275c3ff8957b712fbf57f7a Mon Sep 17 00:00:00 2001 From: Co1lin Date: Thu, 21 Aug 2025 20:52:48 -0400 Subject: [PATCH 07/10] fix: ci: release_swerex_remote.yaml: use arm64 to be consistent with docker --- .github/workflows/release_swerex_remote.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release_swerex_remote.yaml b/.github/workflows/release_swerex_remote.yaml index 71a0b07..05350d9 100644 --- a/.github/workflows/release_swerex_remote.yaml +++ b/.github/workflows/release_swerex_remote.yaml @@ -18,7 +18,7 @@ jobs: - arch: amd64 name: amd64 - arch: arm64 - name: aarch64 + name: arm64 steps: - name: Checkout code @@ -104,7 +104,7 @@ jobs: run: | mkdir -p release_assets cp artifacts/swerex-remote-amd64/swerex-remote-amd64 release_assets/ - cp artifacts/swerex-remote-aarch64/swerex-remote-aarch64 release_assets/ + cp artifacts/swerex-remote-arm64/swerex-remote-arm64 release_assets/ # Create checksums cd release_assets @@ -143,15 +143,15 @@ jobs: ## Download Links - **Linux AMD64**: [swerex-remote-amd64](https://github.com/${{ github.repository }}/releases/download/${{ steps.release_info.outputs.tag }}/swerex-remote-amd64) - - **Linux ARM64**: [swerex-remote-aarch64](https://github.com/${{ github.repository }}/releases/download/${{ steps.release_info.outputs.tag }}/swerex-remote-aarch64) + - **Linux ARM64**: [swerex-remote-arm64](https://github.com/${{ github.repository }}/releases/download/${{ steps.release_info.outputs.tag }}/swerex-remote-arm64) - **Checksums**: [checksums.txt](https://github.com/${{ github.repository }}/releases/download/${{ steps.release_info.outputs.tag }}/checksums.txt) ${{ steps.release_info.outputs.type == 'version' && '## Always Latest Links - **Linux AMD64**: [swerex-remote-amd64](https://github.com/' || '' }}${{ steps.release_info.outputs.type == 'version' && github.repository || '' }}${{ steps.release_info.outputs.type == 'version' && '/releases/latest/download/swerex-remote-amd64) - - **Linux ARM64**: [swerex-remote-aarch64](https://github.com/' || '' }}${{ steps.release_info.outputs.type == 'version' && github.repository || '' }}${{ steps.release_info.outputs.type == 'version' && '/releases/latest/download/swerex-remote-aarch64)' || '' }} + - **Linux ARM64**: [swerex-remote-arm64](https://github.com/' || '' }}${{ steps.release_info.outputs.type == 'version' && github.repository || '' }}${{ steps.release_info.outputs.type == 'version' && '/releases/latest/download/swerex-remote-arm64)' || '' }} files: | release_assets/swerex-remote-amd64 - release_assets/swerex-remote-aarch64 + release_assets/swerex-remote-arm64 release_assets/checksums.txt prerelease: ${{ steps.release_info.outputs.prerelease }} env: From f139fe735c76ceb04785c50a3cf7b8777a47a9a8 Mon Sep 17 00:00:00 2001 From: Co1lin Date: Fri, 22 Aug 2025 01:04:46 -0400 Subject: [PATCH 08/10] feat: use pre-built swerex-remote in docker --- src/swerex/deployment/docker.py | 105 +++++++++++++++++++++++++------- 1 file changed, 82 insertions(+), 23 deletions(-) diff --git a/src/swerex/deployment/docker.py b/src/swerex/deployment/docker.py index 71195a6..a7b5b8b 100644 --- a/src/swerex/deployment/docker.py +++ b/src/swerex/deployment/docker.py @@ -1,10 +1,14 @@ import logging +import os import shlex import subprocess +import tempfile import time import uuid +from pathlib import Path from typing import Any +import requests from typing_extensions import Self from swerex import PACKAGE_NAME, REMOTE_EXECUTABLE_NAME @@ -21,6 +25,8 @@ __all__ = ["DockerDeployment", "DockerDeploymentConfig"] +REMOTE_EXECUTABLE_PATH = Path("/", REMOTE_EXECUTABLE_NAME) + def _is_image_available(image: str, runtime: str = "docker") -> bool: try: @@ -119,11 +125,7 @@ def _get_token(self) -> str: def _get_swerex_start_cmd(self, token: str) -> list[str]: rex_args = f"--auth-token {token}" - pipx_install = "python3 -m pip install pipx && python3 -m pipx ensurepath" - if self._config.python_standalone_dir: - cmd = f"{self._config.python_standalone_dir}/python3.11/bin/{REMOTE_EXECUTABLE_NAME} {rex_args}" - else: - cmd = f"{REMOTE_EXECUTABLE_NAME} {rex_args} || ({pipx_install} && pipx run {PACKAGE_NAME} {rex_args})" + cmd = f"chmod +x {REMOTE_EXECUTABLE_PATH} && {REMOTE_EXECUTABLE_PATH} --port 8000 {rex_args}" # Need to wrap with /bin/sh -c to avoid having '&&' interpreted by the parent shell return [ "/bin/sh", @@ -245,32 +247,53 @@ async def start(self): rm_arg = [] if self._config.remove_container: rm_arg = ["--rm"] - cmds = [ + # download the remote server + image_arch = subprocess.check_output( + self._config.container_runtime + " inspect --format '{{.Architecture}}' " + image_id, shell=True, text=True + ).strip() + assert image_arch in {"amd64", "arm64"}, f"Unsupported architecture: {image_arch}" + with tempfile.TemporaryDirectory() as temp_dir: + tmp_exec_path = Path(temp_dir) / REMOTE_EXECUTABLE_NAME + exec_url = f"https://github.com/Co1lin/SWE-ReX/releases/latest/download/swerex-remote-{image_arch}" + self.logger.info(f"Downloading remote executable from {exec_url} to {tmp_exec_path}") + r_exec = requests.get(exec_url) + r_exec.raise_for_status() + with open(tmp_exec_path, "wb") as f: + f.write(r_exec.content) + # start the container + cmds_run = [ + self._config.container_runtime, + "run", + *rm_arg, + "-p", + f"{self._config.port}:8000", + *platform_arg, + *self._config.docker_args, + "--name", + self._container_name, + "-itd", + image_id, + ] + self.logger.info( + f"Starting container {self._container_name} with image {self._config.image} serving on port {self._config.port}: {shlex.join(cmds_run)}" + ) + t0 = time.time() + subprocess.check_output(cmds_run, stderr=subprocess.STDOUT) + # copy the remote server executable into the container + self._copy_to(tmp_exec_path, REMOTE_EXECUTABLE_PATH) + # execute the remote server + cmds_exec = [ self._config.container_runtime, - "run", - *rm_arg, - "-p", - f"{self._config.port}:8000", - *platform_arg, - *self._config.docker_args, - "--name", + "exec", self._container_name, - image_id, *self._get_swerex_start_cmd(token), ] - cmd_str = shlex.join(cmds) - self.logger.info( - f"Starting container {self._container_name} with image {self._config.image} serving on port {self._config.port}" - ) - self.logger.debug(f"Command: {cmd_str!r}") - # shell=True required for && etc. - self._container_process = subprocess.Popen(cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.logger.info(f"Executing remote server in container {self._container_name}: {shlex.join(cmds_exec)}") + self._container_process = subprocess.Popen(cmds_exec, stdout=subprocess.PIPE, stderr=subprocess.PIPE) self._hooks.on_custom_step("Starting runtime") - self.logger.info(f"Starting runtime at {self._config.port}") self._runtime = RemoteRuntime.from_config( RemoteRuntimeConfig(port=self._config.port, timeout=self._runtime_timeout, auth_token=token) ) - t0 = time.time() await self._wait_until_alive(timeout=self._config.startup_timeout) self.logger.info(f"Runtime started in {time.time() - t0:.2f}s") @@ -288,6 +311,7 @@ async def stop(self): stderr=subprocess.DEVNULL, timeout=10, ) + self.logger.info(f"Killed container {self._container_name}") except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: self.logger.warning( f"Failed to kill container {self._container_name}: {e}. Will try harder.", @@ -324,3 +348,38 @@ def runtime(self) -> RemoteRuntime: if self._runtime is None: raise DeploymentNotStartedError() return self._runtime + + def _copy_to(self, src: str, dst: str) -> None: + """ + Copies a file or directory from the host to the container. + + Args: + src (str): The path to the source file or directory on the host. + dst (str): The destination path inside the container. If `dst` ends + with '/', it's treated as a directory. + """ + # Separate the destination path into directory and filename + dst_dir, dst_filename = os.path.split(dst) + + # If dst is a directory path (e.g., "/path/to/dir/"), dst_filename will be empty. + # In this case, the destination filename should be the source filename. + if not dst_filename: + dst_filename = Path(src).name + dst_path = Path(dst_dir) / dst_filename + + # Step 1: docker cp (host -> container) + subprocess.check_output(["docker", "cp", src, f"{self._container_name}:{dst_path}"], stderr=subprocess.STDOUT) + + # Step 2: fix ownership to match container user + uid = subprocess.check_output( + ["docker", "exec", self._container_name, "id", "-u"], + text=True, + ).strip() + gid = subprocess.check_output( + ["docker", "exec", self._container_name, "id", "-g"], + text=True, + ).strip() + subprocess.check_output( + ["docker", "exec", self._container_name, "chown", "-R", f"{uid}:{gid}", dst_path], + stderr=subprocess.STDOUT, + ) From e26aa8b0d7e9f92bddf2a5b3d6ee7749be7d1894 Mon Sep 17 00:00:00 2001 From: Co1lin Date: Fri, 22 Aug 2025 01:16:36 -0400 Subject: [PATCH 09/10] fix: use asyncio.to_thread in docker deployment start to avoid blocking --- src/swerex/deployment/docker.py | 65 ++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/src/swerex/deployment/docker.py b/src/swerex/deployment/docker.py index a7b5b8b..2c7deeb 100644 --- a/src/swerex/deployment/docker.py +++ b/src/swerex/deployment/docker.py @@ -1,3 +1,4 @@ +import asyncio import logging import os import shlex @@ -231,7 +232,7 @@ def _build_image(self) -> str: async def start(self): """Starts the runtime.""" - self._pull_image() + asyncio.to_thread(self._pull_image) if self._config.python_standalone_dir: image_id = self._build_image() else: @@ -252,35 +253,39 @@ async def start(self): self._config.container_runtime + " inspect --format '{{.Architecture}}' " + image_id, shell=True, text=True ).strip() assert image_arch in {"amd64", "arm64"}, f"Unsupported architecture: {image_arch}" - with tempfile.TemporaryDirectory() as temp_dir: - tmp_exec_path = Path(temp_dir) / REMOTE_EXECUTABLE_NAME - exec_url = f"https://github.com/Co1lin/SWE-ReX/releases/latest/download/swerex-remote-{image_arch}" - self.logger.info(f"Downloading remote executable from {exec_url} to {tmp_exec_path}") - r_exec = requests.get(exec_url) - r_exec.raise_for_status() - with open(tmp_exec_path, "wb") as f: - f.write(r_exec.content) - # start the container - cmds_run = [ - self._config.container_runtime, - "run", - *rm_arg, - "-p", - f"{self._config.port}:8000", - *platform_arg, - *self._config.docker_args, - "--name", - self._container_name, - "-itd", - image_id, - ] - self.logger.info( - f"Starting container {self._container_name} with image {self._config.image} serving on port {self._config.port}: {shlex.join(cmds_run)}" - ) - t0 = time.time() - subprocess.check_output(cmds_run, stderr=subprocess.STDOUT) - # copy the remote server executable into the container - self._copy_to(tmp_exec_path, REMOTE_EXECUTABLE_PATH) + t0 = time.time() + + def _start_and_copy(): + with tempfile.TemporaryDirectory() as temp_dir: + tmp_exec_path = Path(temp_dir) / REMOTE_EXECUTABLE_NAME + exec_url = f"https://github.com/Co1lin/SWE-ReX/releases/latest/download/swerex-remote-{image_arch}" + self.logger.info(f"Downloading remote executable from {exec_url} to {tmp_exec_path}") + r_exec = requests.get(exec_url) + r_exec.raise_for_status() + with open(tmp_exec_path, "wb") as f: + f.write(r_exec.content) + # start the container + cmds_run = [ + self._config.container_runtime, + "run", + *rm_arg, + "-p", + f"{self._config.port}:8000", + *platform_arg, + *self._config.docker_args, + "--name", + self._container_name, + "-itd", + image_id, + ] + self.logger.info( + f"Starting container {self._container_name} with image {self._config.image} serving on port {self._config.port}: {shlex.join(cmds_run)}" + ) + subprocess.check_output(cmds_run, stderr=subprocess.STDOUT) + # copy the remote server executable into the container + self._copy_to(tmp_exec_path, REMOTE_EXECUTABLE_PATH) + + await asyncio.to_thread(_start_and_copy) # execute the remote server cmds_exec = [ self._config.container_runtime, From f83059c8fd4ec05801381f845cea5d0ed154d2cb Mon Sep 17 00:00:00 2001 From: Co1lin Date: Fri, 22 Aug 2025 01:17:48 -0400 Subject: [PATCH 10/10] chore --- src/swerex/deployment/docker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/swerex/deployment/docker.py b/src/swerex/deployment/docker.py index 2c7deeb..617a89f 100644 --- a/src/swerex/deployment/docker.py +++ b/src/swerex/deployment/docker.py @@ -248,7 +248,7 @@ async def start(self): rm_arg = [] if self._config.remove_container: rm_arg = ["--rm"] - # download the remote server + image_arch = subprocess.check_output( self._config.container_runtime + " inspect --format '{{.Architecture}}' " + image_id, shell=True, text=True ).strip() @@ -257,6 +257,7 @@ async def start(self): def _start_and_copy(): with tempfile.TemporaryDirectory() as temp_dir: + # download the remote server tmp_exec_path = Path(temp_dir) / REMOTE_EXECUTABLE_NAME exec_url = f"https://github.com/Co1lin/SWE-ReX/releases/latest/download/swerex-remote-{image_arch}" self.logger.info(f"Downloading remote executable from {exec_url} to {tmp_exec_path}")