Skip to content

Commit 56b84d6

Browse files
jcfrjcfr
committed
feat: Add support for publishing wheels bundling s5cmd executable
This adds support for building s5cmd wheels and is based on prior work done in https://github.com/ImagingDataCommons/idc-index/ It includes: * Support building wheels for 64-bit systems (including macOS arm64 for now) and targeting Python >= 3.8. * Extraction of package version from source control by leveraging `setuptools-scm`. * Infrastructure for building sphinx-based documentation. * Pre-commit hooks for checking style * Dependabot configuration * GitHub workflows for continuous integration (`ci.yml`) and continuous development (`cd.yml`). Each time a GitHub Release is created, it triggers the build, test and publishing of wheels.
1 parent 33e5e74 commit 56b84d6

File tree

11 files changed

+204
-71
lines changed

11 files changed

+204
-71
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,3 @@ jobs:
6464
run: >-
6565
python -m pytest -ra --cov --cov-report=xml --cov-report=term
6666
--durations=20
67-
68-
- name: Upload coverage report
69-
uses: codecov/[email protected]
70-
with:
71-
token: ${{ secrets.CODECOV_TOKEN }}

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,3 +156,6 @@ Thumbs.db
156156
# Common editor files
157157
*~
158158
*.swp
159+
160+
# s5cmd-python-distributions
161+
src/s5cmd/s5cmd*

‎.pre-commit-config.yaml‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -79,11 +79,6 @@ repos:
7979
entry: PyBind|Numpy|Cmake|CCache|Github|PyTest
8080
exclude: .pre-commit-config.yaml
8181

82-
- repo: https://github.com/cheshirekow/cmake-format-precommit
83-
rev: "v0.6.13"
84-
hooks:
85-
- id: cmake-format
86-
8782
- repo: https://github.com/abravalheri/validate-pyproject
8883
rev: "v0.16"
8984
hooks:

‎CMakeLists.txt‎

Lines changed: 78 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,81 @@
11
cmake_minimum_required(VERSION 3.15...3.26)
2-
project(${SKBUILD_PROJECT_NAME} LANGUAGES CXX)
2+
project(${SKBUILD_PROJECT_NAME} LANGUAGES NONE)
33

4-
set(PYBIND11_FINDPYTHON ON)
5-
find_package(pybind11 CONFIG REQUIRED)
4+
function(_check_archive dest_file expected_sha256 output_var)
5+
get_filename_component(filename ${dest_file} NAME)
6+
message(STATUS "Checking ${filename}")
67

7-
pybind11_add_module(_core MODULE src/main.cpp)
8-
install(TARGETS _core DESTINATION ${SKBUILD_PROJECT_NAME})
8+
if(NOT EXISTS ${dest_file})
9+
message(STATUS "Checking ${filename} - nonexistent")
10+
set(${output_var} "nonexistent" PARENT_SCOPE)
11+
return()
12+
endif()
13+
14+
file(SHA256 ${dest_file} current_hash)
15+
if(NOT ${current_hash} STREQUAL ${expected_sha256})
16+
message(STATUS "Checking ${filename} - expired")
17+
set(${output_var} "expired" PARENT_SCOPE)
18+
return()
19+
endif()
20+
21+
message(STATUS "Checking ${filename} - up-to-date")
22+
set(${output_var} "ok" PARENT_SCOPE)
23+
endfunction()
24+
25+
# Set in the current scope the following variables:
26+
# - s5cmd_archive_url
27+
# - s5cmd_archive_sha256
28+
include(${CMAKE_CURRENT_SOURCE_DIR}/s5cmdUrls.cmake)
29+
30+
#
31+
# Download archive
32+
#
33+
cmake_path(GET s5cmd_archive_url FILENAME archive_filename)
34+
set(destination_file "${CMAKE_CURRENT_BINARY_DIR}/${archive_filename}")
35+
36+
_check_archive(${destination_file} ${s5cmd_archive_sha256} result)
37+
38+
if(result MATCHES "^(nonexistent|expired)$")
39+
message(STATUS "Downloading ${s5cmd_archive_url}")
40+
file(
41+
DOWNLOAD
42+
${s5cmd_archive_url}
43+
${destination_file}
44+
EXPECTED_HASH SHA256=${s5cmd_archive_sha256}
45+
)
46+
elseif(result STREQUAL "ok")
47+
# ok
48+
else()
49+
message(FATAL_ERROR "Unknown result value: ${result}")
50+
endif()
51+
52+
#
53+
# Extract archive
54+
#
55+
set(executable_name "s5cmd${CMAKE_EXECUTABLE_SUFFIX}")
56+
57+
string(MAKE_C_IDENTIFIER ${archive_filename} extract_subdir)
58+
set(extract_dir "${CMAKE_CURRENT_BINARY_DIR}/${extract_subdir}")
59+
message(STATUS "Extracting ${archive_filename} into ${extract_dir}")
60+
file(ARCHIVE_EXTRACT
61+
INPUT ${destination_file}
62+
DESTINATION ${extract_dir}
63+
PATTERNS "${executable_name}"
64+
VERBOSE
65+
)
66+
67+
set(s5cmd_package "src/s5cmd")
68+
69+
message(STATUS "Copying ${executable_name} to ${s5cmd_package}")
70+
file(COPY ${extract_dir}/${executable_name} DESTINATION ${PROJECT_SOURCE_DIR}/${s5cmd_package})
71+
72+
set(_permissions PERMISSIONS
73+
OWNER_READ OWNER_WRITE OWNER_EXECUTE
74+
GROUP_READ GROUP_EXECUTE
75+
WORLD_READ WORLD_EXECUTE
76+
)
77+
78+
message(STATUS "Changing ${executable_name} permissions to be executable")
79+
file(CHMOD ${PROJECT_SOURCE_DIR}/${s5cmd_package}/${executable_name} ${_permissions})
80+
81+
install(PROGRAMS ${extract_dir}/${executable_name} DESTINATION "${s5cmd_package}" ${_permissions})

‎README.md‎

Lines changed: 41 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,57 @@
1-
# s5cmd
1+
# s5cmd Python Distributions
22

33
[![Actions Status][actions-badge]][actions-link]
4-
[![Documentation Status][rtd-badge]][rtd-link]
54

65
[![PyPI version][pypi-version]][pypi-link]
7-
[![Conda-Forge][conda-badge]][conda-link]
86
[![PyPI platforms][pypi-platforms]][pypi-link]
97

10-
[![GitHub Discussion][github-discussions-badge]][github-discussions-link]
11-
128
<!-- SPHINX-START -->
139

10+
## Overview
11+
12+
`s5cmd` is a very fast S3 and local filesystem execution tool. It comes with
13+
support for a multitude of operations including tab completion and wildcard
14+
support for files, which can be very handy for your object storage workflow
15+
while working with large number of files.
16+
17+
This project provides the infrastructure for building the `s5cmd` Python wheels.
18+
For more information about `s5cmd`, please refer to
19+
https://github.com/peak/s5cmd.
20+
21+
The Python wheels provided here contain the official `s5cmd` executable, which
22+
is sourced from the [GitHub releases](https://github.com/peak/s5cmd/releases).
23+
Once the wheel is installed, a convenient launcher executable is automatically
24+
placed in the PATH. This launcher is created during installation by pip,
25+
leveraging the `[project.scripts]` configuration defined in the `pyproject.toml`
26+
file.
27+
28+
## Platforms
29+
30+
The following platforms are supported by the binary wheels:
31+
32+
| OS | Arch |
33+
| ------------- | -------------------------------------------------------------------------------------- |
34+
| Windows | 64-bit<br>32-bit |
35+
| Linux Intel | manylinux2010+ 64-bit<br>musllinux 64-bit<br>manylinux2010+ 32-bit<br>musllinux 32-bit |
36+
| Linux ARM | manylinux2014+ AArch64<br>musllinux AArch64 |
37+
| Linux PowerPC | manylinux2014+ ppc64le<br>musllinux ppc64le |
38+
| Linux IBM Z | manylinux2014+ s390x<br>musllinux s390x |
39+
| macOS 10.10+ | Intel |
40+
| macOS 11+ | Apple Silicon |
41+
42+
## License
43+
44+
This project is maintained by Jean-Christophe Fillion-Robin from Kitware Inc. It
45+
is covered by the OSI-approved MIT License.
46+
47+
`s5cmd` is distributed under the OSI-approved MIT License. For further details
48+
regarding `s5cmd`, please visit https://github.com/peak/s5cmd.
49+
1450
<!-- prettier-ignore-start -->
1551
[actions-badge]: https://github.com/jcfr/s5cmd-python-distributions/workflows/CI/badge.svg
1652
[actions-link]: https://github.com/jcfr/s5cmd-python-distributions/actions
17-
[conda-badge]: https://img.shields.io/conda/vn/conda-forge/s5cmd
18-
[conda-link]: https://github.com/conda-forge/s5cmd-feedstock
19-
[github-discussions-badge]: https://img.shields.io/static/v1?label=Discussions&message=Ask&color=blue&logo=github
20-
[github-discussions-link]: https://github.com/jcfr/s5cmd-python-distributions/discussions
2153
[pypi-link]: https://pypi.org/project/s5cmd/
2254
[pypi-platforms]: https://img.shields.io/pypi/pyversions/s5cmd
2355
[pypi-version]: https://img.shields.io/pypi/v/s5cmd
24-
[rtd-badge]: https://readthedocs.org/projects/s5cmd/badge/?version=latest
25-
[rtd-link]: https://s5cmd.readthedocs.io/en/latest/?badge=latest
2656

2757
<!-- prettier-ignore-end -->

‎pyproject.toml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,10 @@ Discussions = "https://github.com/jcfr/s5cmd-python-distributions/discussions"
5656
Changelog = "https://github.com/jcfr/s5cmd-python-distributions/releases"
5757

5858

59+
[project.scripts]
60+
s5cmd = "s5cmd:s5cmd"
61+
62+
5963
[tool.scikit-build]
6064
minimum-version = "0.4"
6165
build-dir = "build/{wheel_tag}"
@@ -153,7 +157,7 @@ isort.required-imports = ["from __future__ import annotations"]
153157
[tool.pylint]
154158
py-version = "3.8"
155159
ignore-paths = [".*/_version.py"]
156-
extension-pkg-allow-list = ["s5cmd._core"]
160+
extension-pkg-allow-list = []
157161
reports.output-format = "colorized"
158162
similarities.ignore-imports = "yes"
159163
messages_control.disable = [

‎s5cmdUrls.cmake‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
# Checksum copied from "s5cmd_checksums.txt" associated with the s5cmd GitHub release
2+
3+
set(version "2.2.2")
4+
5+
set(linux32_filename "s5cmd_${version}_Linux-32bit.tar.gz")
6+
set(linux32_sha256 "dc9ebe570fb5abcf5781511901d93425879022d56e73ab44dd32c45b2bfbc04b")
7+
8+
set(linux64_filename "s5cmd_${version}_Linux-64bit.tar.gz")
9+
set(linux64_sha256 "a15f83d2a6dc091e43b2a120f29f8f6c86d146c381766c0197ec75d7985af2b6")
10+
11+
set(linuxarm64_filename "s5cmd_${version}_Linux-arm64.tar.gz")
12+
set(linuxarm64_sha256 "eabf18082398c332d33c692d383a889be204b1e7716f820e014bf11474ad345b")
13+
14+
set(macos64_filename "s5cmd_${version}_macOS-64bit.tar.gz")
15+
set(macos64_sha256 "5503a3308e239f081e5238e0af57958ae618e0de8b9c71142fe80f38be77e1c7")
16+
17+
set(macosarm64_filename "s5cmd_${version}_macOS-arm64.tar.gz")
18+
set(macosarm64_sha256 "fa3ae7e093fd6ac8a5236a000d5373779eb403c57ee955fc7da9549668644e38")
19+
20+
set(win32_filename "s5cmd_${version}_Windows-32bit.zip")
21+
set(win32_sha256 "ee667eb01b955a7dda588456bd102982f8344bed393a8b63b5d4c9c325e01349")
22+
23+
set(win64_filename "s5cmd_${version}_Windows-64bit.zip")
24+
set(win64_sha256 "f7c311907c78efa56e27a25fba1f87520754c402bbe1cb4901d3522f12a75497")
25+
26+
27+
cmake_host_system_information(RESULT is_64bit QUERY IS_64BIT)
28+
29+
set(archive "linux32")
30+
if(is_64bit)
31+
set(archive "linux64")
32+
endif()
33+
34+
if(APPLE)
35+
set(archive "macos64")
36+
if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "ARM64")
37+
set(archive "macosarm64")
38+
endif()
39+
endif()
40+
41+
if(WIN32)
42+
set(archive "win32")
43+
if(is_64bit AND NOT (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "ARM64"))
44+
set(archive "win64")
45+
endif()
46+
endif()
47+
48+
if(NOT DEFINED "${archive}_filename")
49+
message(FATAL_ERROR "Failed to determine which archive to download: '${archive}_filename' variable is not defined")
50+
endif()
51+
52+
if(NOT DEFINED "${archive}_sha256")
53+
message(FATAL_ERROR "Could you make sure variable '${archive}_sha256' is defined ?")
54+
endif()
55+
56+
set(s5cmd_archive_filename "${${archive}_filename}")
57+
set(s5cmd_archive_sha256 "${${archive}_sha256}")
58+
59+
set(s5cmd_archive_url "https://github.com/peak/s5cmd/releases/download/v${version}/${s5cmd_archive_filename}")

‎src/main.cpp‎

Lines changed: 0 additions & 28 deletions
This file was deleted.

‎src/s5cmd/__init__.py‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,23 @@
77

88
from __future__ import annotations
99

10+
import subprocess
11+
import sys
12+
from pathlib import Path
13+
from typing import NoReturn
14+
1015
from ._version import version as __version__
1116

12-
__all__ = ["__version__"]
17+
__all__ = ["__version__", "s5cmd"]
18+
19+
20+
S5CMD_BIN_DIR: Path = Path(__file__).parent
21+
22+
23+
def _program(name: str, args: list[str]) -> int:
24+
return subprocess.call([S5CMD_BIN_DIR / name, *args], close_fds=False)
25+
26+
27+
def s5cmd() -> NoReturn:
28+
"""Run the s5cmd executable with arguments passed to a Python script."""
29+
raise SystemExit(_program("s5cmd", sys.argv[1:]))

‎src/s5cmd/_core.pyi‎

Lines changed: 0 additions & 4 deletions
This file was deleted.

0 commit comments

Comments
 (0)