diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index c9dd6a0b734..30666e13006 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -13,32 +13,32 @@ concurrency: cancel-in-progress: true jobs: - # test-qnn-wheel-packages-linux: - # name: test-qnn-wheel-packages-linux - # uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main - # permissions: - # id-token: write - # contents: read - # strategy: - # fail-fast: false - # matrix: - # python-version: [ "3.10", "3.11", "3.12" ] - # with: - # runner: linux.2xlarge - # docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk - # submodules: 'recursive' - # ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - # timeout: 180 - # script: | - # # The generic Linux job chooses to use base env, not the one setup by the image - # CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") - # conda activate "${CONDA_ENV}" + test-qnn-wheel-packages-linux: + name: test-qnn-wheel-packages-linux + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + permissions: + id-token: write + contents: read + strategy: + fail-fast: false + matrix: + python-version: [ "3.10", "3.11", "3.12" ] + with: + runner: linux.2xlarge + docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk + submodules: 'recursive' + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout: 180 + script: | + # The generic Linux job chooses to use base env, not the one setup by the image + CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") + conda activate "${CONDA_ENV}" - # # Create a clean env for each python version - # conda create -y -n test_env_${{ matrix.python-version }} python=${{ matrix.python-version }} - # conda activate test_env_${{ matrix.python-version }} + # Create a clean env for each python version + conda create -y -n test_env_${{ matrix.python-version }} python=${{ matrix.python-version }} + conda activate test_env_${{ matrix.python-version }} - # PYTHON_EXECUTABLE=python bash .ci/scripts/test_wheel_package_qnn.sh "${{ matrix.python-version }}" + PYTHON_EXECUTABLE=python bash .ci/scripts/test_wheel_package_qnn.sh "${{ matrix.python-version }}" test-setup-linux-gcc: name: test-setup-linux-gcc diff --git a/backends/qualcomm/scripts/download_qnn_sdk.py b/backends/qualcomm/scripts/download_qnn_sdk.py index 5524adf8988..ec223cb273b 100644 --- a/backends/qualcomm/scripts/download_qnn_sdk.py +++ b/backends/qualcomm/scripts/download_qnn_sdk.py @@ -117,7 +117,7 @@ def _atomic_download(url: str, dest: pathlib.Path): def _download_archive(url: str, archive_path: pathlib.Path) -> bool: - """Robust streaming download with retries.""" + """Streaming download with retry + resume support.""" logger.debug("Archive will be saved to: %s", archive_path) @@ -130,32 +130,80 @@ def _download_archive(url: str, archive_path: pathlib.Path) -> bool: ) session.mount("https://", HTTPAdapter(max_retries=retries)) + # ------------------------------------------------------------ + # 1. Detect total file size (HEAD is broken on Qualcomm) + # ------------------------------------------------------------ try: - with session.get(url, stream=True) as r: + # NOTE: + # Qualcomm's download endpoint does not return accurate metadata on HEAD requests. + # Many Qualcomm URLs first redirect to an HTML "wrapper" page (typically ~134 bytes), + # and the HEAD request reflects *that wrapper* rather than the actual ZIP archive. + # + # Example: + # HEAD -> Content-Length: 134, Content-Type: text/html + # GET -> Content-Length: 1354151797, Content-Type: application/zip + # + # Because Content-Length from HEAD is frequently incorrect, we fall back to issuing + # a GET request with stream=True to obtain the real Content-Length without downloading + # the full file. This ensures correct resume logic and size validation. + r_head = session.get(url, stream=True) + r_head.raise_for_status() + + if "content-length" not in r_head.headers: + logger.error("Server did not return content-length!") + return False + + total_size = int(r_head.headers["content-length"]) + except Exception as e: + logger.exception("Failed to determine file size: %s", e) + return False + + # ------------------------------------------------------------ + # 2. If partial file exists, resume + # ------------------------------------------------------------ + downloaded = archive_path.stat().st_size if archive_path.exists() else 0 + if downloaded > total_size: + logger.warning("Existing file is larger than expected. Removing.") + archive_path.unlink() + downloaded = 0 + + logger.info("Resuming download from %d / %d bytes", downloaded, total_size) + + headers = {} + if downloaded > 0: + headers["Range"] = f"bytes={downloaded}-" + + try: + # resume GET + with session.get(url, stream=True, headers=headers) as r: r.raise_for_status() - downloaded = 0 chunk_size = 1024 * 1024 # 1MB + mode = "ab" if downloaded > 0 else "wb" - with open(archive_path, "wb") as f: + with open(archive_path, mode) as f: for chunk in r.iter_content(chunk_size): if chunk: f.write(chunk) downloaded += len(chunk) - logger.info("Download completed!") - except Exception as e: logger.exception("Error during download: %s", e) return False - if archive_path.exists() and archive_path.stat().st_size == 0: - logger.warning("Downloaded file is empty!") - return False - elif not archive_path.exists(): - logger.error("File was not downloaded!") + # ------------------------------------------------------------ + # 3. Validate final size + # ------------------------------------------------------------ + final_size = archive_path.stat().st_size + if final_size != total_size: + logger.error( + "Download incomplete: expected %d, got %d", + total_size, + final_size, + ) return False + logger.info("Download completed successfully!") return True diff --git a/tools/cmake/preset/pybind.cmake b/tools/cmake/preset/pybind.cmake index 699a7c50358..a0d06d74d17 100644 --- a/tools/cmake/preset/pybind.cmake +++ b/tools/cmake/preset/pybind.cmake @@ -37,7 +37,7 @@ elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux") set_overridable_option(EXECUTORCH_BUILD_EXTENSION_LLM_RUNNER ON) set_overridable_option(EXECUTORCH_BUILD_EXTENSION_LLM ON) if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|amd64|i.86)$") - set_overridable_option(EXECUTORCH_BUILD_QNN OFF) + set_overridable_option(EXECUTORCH_BUILD_QNN ON) endif() elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows" OR CMAKE_SYSTEM_NAME STREQUAL "WIN32"