feat: use pre-built swerex-remote in containers

.github/workflows/release_swerex_remote.yaml

@@ -0,0 +1,158 @@
+name: Release swerex-remote Executable
+
+on:
+  push:
+    branches: [ main ]
+    tags:
+      - 'v*' # Triggers on version tags like v1.0.0, v2.1.3, etc.
+  pull_request:
+    branches: [ main ]
+  workflow_dispatch: # Allow manual triggers
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - arch: amd64
+            name: amd64
+          - arch: arm64
+            name: arm64
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Build executable in compatible container
+      run: |
+        # Create build script
+        cat > build_script.sh << 'EOF'
+        #!/bin/bash
+        set -e
+
+        # Update package lists
+        apt-get update
+
+        # Install basic dependencies
+        apt-get install -y curl ca-certificates build-essential
+
+        # Install uv
+        curl -LsSf https://astral.sh/uv/install.sh | sh
+        source $HOME/.local/bin/env
+
+        # Create virtual environment with Python 3.13
+        uv venv --python 3.13 .venv
+        source .venv/bin/activate
+
+        # Install dependencies
+        uv pip install -e '.[dev]'
+        uv pip install pyinstaller
+
+        # Build executable
+        pyinstaller src/swerex/server.py --onefile --name swerex-remote-$ARCH_NAME
+
+        # Make executable and copy to output
+        chmod +x dist/swerex-remote-$ARCH_NAME
+        cp dist/swerex-remote-$ARCH_NAME /output/
+        EOF
+
+        chmod +x build_script.sh
+
+        # Create output directory
+        mkdir -p output
+
+        BASE_IMAGE="ubuntu:14.04"
+
+        docker run --rm \
+          --platform linux/${{ matrix.arch }} \
+          -v $(pwd):/workspace \
+          -v $(pwd)/output:/output \
+          -w /workspace \
+          -e ARCH_NAME=${{ matrix.name }} \
+          $BASE_IMAGE \
+          bash build_script.sh
+
+    - name: Upload build artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: swerex-remote-${{ matrix.name }}
+        path: output/swerex-remote-${{ matrix.name }}
+        retention-days: 30
+
+  release:
+    needs: build
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' # Only create releases on push events (not PRs)
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Download all artifacts
+      uses: actions/download-artifact@v4
+      with:
+        path: artifacts
+
+    - name: Prepare release assets
+      run: |
+        mkdir -p release_assets
+        cp artifacts/swerex-remote-amd64/swerex-remote-amd64 release_assets/
+        cp artifacts/swerex-remote-arm64/swerex-remote-arm64 release_assets/
+
+        # Create checksums
+        cd release_assets
+        sha256sum * > checksums.txt
+        cd ..
+
+    - name: Determine release type and details
+      id: release_info
+      run: |
+        if [[ $GITHUB_REF == refs/tags/v* ]]; then
+          # This is a version tag
+          VERSION=${GITHUB_REF#refs/tags/}
+          echo "type=version" >> $GITHUB_OUTPUT
+          echo "tag=$VERSION" >> $GITHUB_OUTPUT
+          echo "name=Release $VERSION" >> $GITHUB_OUTPUT
+          echo "prerelease=false" >> $GITHUB_OUTPUT
+          echo "body=Release $VERSION of SWE-ReX remote server" >> $GITHUB_OUTPUT
+        else
+          # This is a dev build from main/master branch
+          DATE=$(date +'%Y%m%d-%H%M%S')
+          COMMIT_SHORT=$(echo $GITHUB_SHA | cut -c1-7)
+          echo "type=dev" >> $GITHUB_OUTPUT
+          echo "tag=dev-$DATE-$COMMIT_SHORT" >> $GITHUB_OUTPUT
+          echo "name=Development Build $DATE" >> $GITHUB_OUTPUT
+          echo "prerelease=true" >> $GITHUB_OUTPUT
+          echo "body=Development build from commit $GITHUB_SHA" >> $GITHUB_OUTPUT
+        fi
+
+    - name: Create or update release
+      uses: softprops/action-gh-release@v1
+      with:
+        tag_name: ${{ steps.release_info.outputs.tag }}
+        name: ${{ steps.release_info.outputs.name }}
+        body: |
+          ${{ steps.release_info.outputs.body }}
+
+          ## Download Links
+          - **Linux AMD64**: [swerex-remote-amd64](https://github.com/${{ github.repository }}/releases/download/${{ steps.release_info.outputs.tag }}/swerex-remote-amd64)
+          - **Linux ARM64**: [swerex-remote-arm64](https://github.com/${{ github.repository }}/releases/download/${{ steps.release_info.outputs.tag }}/swerex-remote-arm64)
+          - **Checksums**: [checksums.txt](https://github.com/${{ github.repository }}/releases/download/${{ steps.release_info.outputs.tag }}/checksums.txt)
+
+          ${{ steps.release_info.outputs.type == 'version' && '## Always Latest Links
+          - **Linux AMD64**: [swerex-remote-amd64](https://github.com/' || '' }}${{ steps.release_info.outputs.type == 'version' && github.repository || '' }}${{ steps.release_info.outputs.type == 'version' && '/releases/latest/download/swerex-remote-amd64)
+          - **Linux ARM64**: [swerex-remote-arm64](https://github.com/' || '' }}${{ steps.release_info.outputs.type == 'version' && github.repository || '' }}${{ steps.release_info.outputs.type == 'version' && '/releases/latest/download/swerex-remote-arm64)' || '' }}
+        files: |
+          release_assets/swerex-remote-amd64
+          release_assets/swerex-remote-arm64
+          release_assets/checksums.txt
+        prerelease: ${{ steps.release_info.outputs.prerelease }}
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.pre-commit-config.yaml

@@ -10,7 +10,7 @@ repos:
         exclude: pyproject.toml
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.12.8
+    rev: v0.12.10
     hooks:
       # Run the linter.
       - id: ruff

src/swerex/deployment/docker.py

@@ -1,10 +1,15 @@
+import asyncio
 import logging
+import os
 import shlex
 import subprocess
+import tempfile
 import time
 import uuid
+from pathlib import Path
 from typing import Any
 
+import requests
 from typing_extensions import Self
 
 from swerex import PACKAGE_NAME, REMOTE_EXECUTABLE_NAME
@@ -21,6 +26,8 @@
 
 __all__ = ["DockerDeployment", "DockerDeploymentConfig"]
 
+REMOTE_EXECUTABLE_PATH = Path("/", REMOTE_EXECUTABLE_NAME)
+
 
 def _is_image_available(image: str, runtime: str = "docker") -> bool:
     try:
@@ -119,11 +126,7 @@ def _get_token(self) -> str:
 
     def _get_swerex_start_cmd(self, token: str) -> list[str]:
         rex_args = f"--auth-token {token}"
-        pipx_install = "python3 -m pip install pipx && python3 -m pipx ensurepath"
-        if self._config.python_standalone_dir:
-            cmd = f"{self._config.python_standalone_dir}/python3.11/bin/{REMOTE_EXECUTABLE_NAME} {rex_args}"
-        else:
-            cmd = f"{REMOTE_EXECUTABLE_NAME} {rex_args} || ({pipx_install} && pipx run {PACKAGE_NAME} {rex_args})"
+        cmd = f"chmod +x {REMOTE_EXECUTABLE_PATH} && {REMOTE_EXECUTABLE_PATH} --port 8000 {rex_args}"
         # Need to wrap with /bin/sh -c to avoid having '&&' interpreted by the parent shell
         return [
             "/bin/sh",
@@ -229,7 +232,7 @@ def _build_image(self) -> str:
 
     async def start(self):
         """Starts the runtime."""
-        self._pull_image()
+        asyncio.to_thread(self._pull_image)
         if self._config.python_standalone_dir:
             image_id = self._build_image()
         else:
@@ -245,32 +248,58 @@ async def start(self):
         rm_arg = []
         if self._config.remove_container:
             rm_arg = ["--rm"]
-        cmds = [
+
+        image_arch = subprocess.check_output(
+            self._config.container_runtime + " inspect --format '{{.Architecture}}' " + image_id, shell=True, text=True
+        ).strip()
+        assert image_arch in {"amd64", "arm64"}, f"Unsupported architecture: {image_arch}"
+        t0 = time.time()
+
+        def _start_and_copy():
+            with tempfile.TemporaryDirectory() as temp_dir:
+                # download the remote server
+                tmp_exec_path = Path(temp_dir) / REMOTE_EXECUTABLE_NAME
+                exec_url = f"https://github.com/Co1lin/SWE-ReX/releases/latest/download/swerex-remote-{image_arch}"
+                self.logger.info(f"Downloading remote executable from {exec_url} to {tmp_exec_path}")
+                r_exec = requests.get(exec_url)
+                r_exec.raise_for_status()
+                with open(tmp_exec_path, "wb") as f:
+                    f.write(r_exec.content)
+                # start the container
+                cmds_run = [
+                    self._config.container_runtime,
+                    "run",
+                    *rm_arg,
+                    "-p",
+                    f"{self._config.port}:8000",
+                    *platform_arg,
+                    *self._config.docker_args,
+                    "--name",
+                    self._container_name,
+                    "-itd",
+                    image_id,
+                ]
+                self.logger.info(
+                    f"Starting container {self._container_name} with image {self._config.image} serving on port {self._config.port}: {shlex.join(cmds_run)}"
+                )
+                subprocess.check_output(cmds_run, stderr=subprocess.STDOUT)
+                # copy the remote server executable into the container
+                self._copy_to(tmp_exec_path, REMOTE_EXECUTABLE_PATH)
+
+        await asyncio.to_thread(_start_and_copy)
+        # execute the remote server
+        cmds_exec = [
             self._config.container_runtime,
-            "run",
-            *rm_arg,
-            "-p",
-            f"{self._config.port}:8000",
-            *platform_arg,
-            *self._config.docker_args,
-            "--name",
+            "exec",
             self._container_name,
-            image_id,
             *self._get_swerex_start_cmd(token),
         ]
-        cmd_str = shlex.join(cmds)
-        self.logger.info(
-            f"Starting container {self._container_name} with image {self._config.image} serving on port {self._config.port}"
-        )
-        self.logger.debug(f"Command: {cmd_str!r}")
-        # shell=True required for && etc.
-        self._container_process = subprocess.Popen(cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        self.logger.info(f"Executing remote server in container {self._container_name}: {shlex.join(cmds_exec)}")
+        self._container_process = subprocess.Popen(cmds_exec, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         self._hooks.on_custom_step("Starting runtime")
-        self.logger.info(f"Starting runtime at {self._config.port}")
         self._runtime = RemoteRuntime.from_config(
             RemoteRuntimeConfig(port=self._config.port, timeout=self._runtime_timeout, auth_token=token)
         )
-        t0 = time.time()
         await self._wait_until_alive(timeout=self._config.startup_timeout)
         self.logger.info(f"Runtime started in {time.time() - t0:.2f}s")
 
@@ -288,6 +317,7 @@ async def stop(self):
                     stderr=subprocess.DEVNULL,
                     timeout=10,
                 )
+                self.logger.info(f"Killed container {self._container_name}")
             except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
                 self.logger.warning(
                     f"Failed to kill container {self._container_name}: {e}. Will try harder.",
@@ -324,3 +354,38 @@ def runtime(self) -> RemoteRuntime:
         if self._runtime is None:
             raise DeploymentNotStartedError()
         return self._runtime
+
+    def _copy_to(self, src: str, dst: str) -> None:
+        """
+        Copies a file or directory from the host to the container.
+
+        Args:
+            src (str): The path to the source file or directory on the host.
+            dst (str): The destination path inside the container. If `dst` ends
+                       with '/', it's treated as a directory.
+        """
+        # Separate the destination path into directory and filename
+        dst_dir, dst_filename = os.path.split(dst)
+
+        # If dst is a directory path (e.g., "/path/to/dir/"), dst_filename will be empty.
+        # In this case, the destination filename should be the source filename.
+        if not dst_filename:
+            dst_filename = Path(src).name
+        dst_path = Path(dst_dir) / dst_filename
+
+        # Step 1: docker cp (host -> container)
+        subprocess.check_output(["docker", "cp", src, f"{self._container_name}:{dst_path}"], stderr=subprocess.STDOUT)
+
+        # Step 2: fix ownership to match container user
+        uid = subprocess.check_output(
+            ["docker", "exec", self._container_name, "id", "-u"],
+            text=True,
+        ).strip()
+        gid = subprocess.check_output(
+            ["docker", "exec", self._container_name, "id", "-g"],
+            text=True,
+        ).strip()
+        subprocess.check_output(
+            ["docker", "exec", self._container_name, "chown", "-R", f"{uid}:{gid}", dst_path],
+            stderr=subprocess.STDOUT,
+        )