diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index aecfdc47a..000000000 --- a/.dockerignore +++ /dev/null @@ -1,5 +0,0 @@ -# Ignore generated files -__pycache__ -/venv/ -/dist/ -/publish/ diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 8a3c6259c..f3134ba7d 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -16,7 +16,7 @@ jobs: build: strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] platform: ["ubuntu-latest", "macos-latest", "windows-latest"] # https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python # Optional - x64 or x86 architecture, defaults to x64 @@ -44,7 +44,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install . - python -m pip install -r requirements-dev.txt + python -m pip install --group dev - name: Lint with ruff run: | @@ -56,7 +56,7 @@ jobs: - name: Sort imports with usort run: | - usort diff mapillary_tools/ + usort diff mapillary_tools - name: Type check with mypy run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 085a413c5..51cee0341 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -92,7 +92,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install . - python -m pip install -r requirements-dev.txt + python -m pip install --group dev - name: Validate version run: | diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md deleted file mode 100644 index c80aef30d..000000000 --- a/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,48 +0,0 @@ - - -### Basic information -* Release version: `0.0.0` | `0.0.1` | `...` -* System: `Windows` | `Linux` | `...` -* Capture Device: `iOS` | `Android` | `Action Camera` | `...` - -### Steps to reproduce behavior - - 1. - 2. - 3. - -### Expected behavior - -... - -### Actual behavior - -... - -### Corresponding data - -* sample images -* meta data - -... - -### Additional information - -... - - - - diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 9adb79f3c..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,2 +0,0 @@ -include schema/*.json -include requirements.txt diff --git a/README.md b/README.md index 1792329cf..0072274bc 100644 --- a/README.md +++ b/README.md @@ -28,18 +28,40 @@ mapillary_tools --help - [Supported File Formats](#supported-file-formats) + - [Image Formats](#image-formats) + - [Video Formats](#video-formats) - [Installation](#installation) + - [Standalone Executable](#standalone-executable) + - [Installing via pip](#installing-via-pip) + - [Installing on Android Devices](#installing-on-android-devices) - [Usage](#usage) - [Process and Upload](#process-and-upload) - [Process](#process) - [Upload](#upload) - [Advanced Usage](#advanced-usage) - [Local Video Processing](#local-video-processing) + - [Install FFmpeg](#install-ffmpeg) + - [Video Processing](#video-processing) - [Geotagging with GPX](#geotagging-with-gpx) + - [New video geotagging features (experimental)](#new-video-geotagging-features-experimental) + - [Usage](#usage-1) + - [Examples](#examples) + - [Generic supported videos](#generic-supported-videos) + - [External GPX](#external-gpx) + - [Insta360 stitched videos](#insta360-stitched-videos) + - [Limitations of `--video_geotag_source`](#limitations-of---video_geotag_source) - [Authenticate](#authenticate) + - [Examples](#examples-1) - [Image Description](#image-description) - [Zip Images](#zip-images) - [Development](#development) + - [Setup](#setup) + - [Option 1: Using uv (Recommended)](#option-1-using-uv-recommended) + - [Option 2: Using pip with virtual environment](#option-2-using-pip-with-virtual-environment) + - [Running the code](#running-the-code) + - [Tests](#tests) + - [Code Quality](#code-quality) + - [Release and Build](#release-and-build) @@ -465,29 +487,49 @@ git clone git@github.com:mapillary/mapillary_tools.git cd mapillary_tools ``` -Set up the virtual environment. It is optional but recommended: +### Option 1: Using uv (Recommended) + +Use [uv](https://docs.astral.sh/uv/) - a fast Python package manager. + +Install the project in development mode with all dependencies: ```sh -pip install pipenv +# Install the project and development dependencies +uv sync --group dev + +# Activate the virtual environment +source .venv/bin/activate # On Windows: .venv\Scripts\activate ``` -Install dependencies: +### Option 2: Using pip with virtual environment + +Set up a virtual environment (recommended): ```sh -pipenv install -r requirements.txt -pipenv install -r requirements-dev.txt +python -m venv .venv +source .venv/bin/activate # On Windows: .venv\Scripts\activate ``` -Enter the virtualenv shell: +Install the project in development mode: ```sh -pipenv shell +# Install the project and all dependencies in editable mode +pip install -e . + +# Install development dependencies +pip install --group dev ``` +## Running the code + Run the code from the repository: ```sh -python3 -m mapillary_tools.commands --version +# If you have mapillary_tools installed in editable mode +mapillary_tools --version + +# Alternatively +python -m mapillary_tools.commands --version ``` ## Tests @@ -495,19 +537,28 @@ python3 -m mapillary_tools.commands --version Run tests: ```sh -# test all cases -python3 -m pytest -s -vv tests -# or test a single case specifically -python3 -m pytest -s -vv tests/unit/test_camm_parser.py::test_build_and_parse +# Test all cases +pytest -s -vv tests +# Or test a single case specifically +pytest -s -vv tests/unit/test_camm_parser.py::test_build_and_parse ``` -Run linting: +## Code Quality + +Run code formatting and linting: ```sh -# format code -black mapillary_tools tests -# sort imports +# Format code with ruff +ruff format mapillary_tools tests + +# Lint code with ruff +ruff check mapillary_tools tests + +# Sort imports with usort usort format mapillary_tools tests + +# Type checking with mypy +mypy mapillary_tools ``` ## Release and Build diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index d9d08530e..000000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM ubuntu:20.04 - -ENV DEBIAN_FRONTEND=noninteractive - -RUN apt update -y && apt install -y python3 python3-pip git && apt install -y --no-install-recommends ffmpeg - -RUN python3 -m pip install --upgrade git+https://github.com/mapillary/mapillary_tools diff --git a/docker/Dockerfile-dev b/docker/Dockerfile-dev deleted file mode 100644 index 1d3f532e3..000000000 --- a/docker/Dockerfile-dev +++ /dev/null @@ -1,10 +0,0 @@ -FROM ubuntu:20.04 - -ENV DEBIAN_FRONTEND=noninteractive - -RUN apt update -y && apt install -y python3 python3-pip git && apt install -y --no-install-recommends ffmpeg - -WORKDIR /mapillary_tools -ADD requirements.txt requirements-dev.txt /mapillary_tools -RUN python3 -m pip install -r requirements.txt -r requirements-dev.txt -ADD . /mapillary_tools diff --git a/mapillary_tools/geotag/base.py b/mapillary_tools/geotag/base.py index b0ed1beb9..f4384fa91 100644 --- a/mapillary_tools/geotag/base.py +++ b/mapillary_tools/geotag/base.py @@ -51,7 +51,7 @@ def to_description( ) ) - return results + error_metadatas + return T.cast(list[types.ImageMetadataOrError], results + error_metadatas) # This method is passed to multiprocessing # so it has to be classmethod or staticmethod to avoid pickling the instance @@ -117,7 +117,7 @@ def to_description( ) ) - return results + error_metadatas + return T.cast(list[types.VideoMetadataOrError], results + error_metadatas) # This method is passed to multiprocessing # so it has to be classmethod or staticmethod to avoid pickling the instance diff --git a/mapillary_tools/process_geotag_properties.py b/mapillary_tools/process_geotag_properties.py index 673831dcf..7ac842539 100644 --- a/mapillary_tools/process_geotag_properties.py +++ b/mapillary_tools/process_geotag_properties.py @@ -308,7 +308,7 @@ def _validate_metadatas( ) ) - return validated_metadatas + error_metadatas + return T.cast(list[types.MetadataOrError], validated_metadatas + error_metadatas) def process_finalize( diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index 32a4c7eb2..000000000 --- a/mypy.ini +++ /dev/null @@ -1,28 +0,0 @@ -[mypy] - -[mypy-tqdm.*] -ignore_missing_imports = True - -[mypy-piexif.*] -ignore_missing_imports = True - -[mypy-pynmea2.*] -ignore_missing_imports = True - -[mypy-gpxpy.*] -ignore_missing_imports = True - -[mypy-exifread.*] -ignore_missing_imports = True - -[mypy-construct.*] -ignore_missing_imports = True - -[mypy-jsonschema.*] -ignore_missing_imports = True - -[mypy-PIL.*] -ignore_missing_imports = True - -[mypy-py.*] -ignore_missing_imports = True diff --git a/mapillary_tools.spec b/pyinstaller/mapillary_tools.spec similarity index 95% rename from mapillary_tools.spec rename to pyinstaller/mapillary_tools.spec index b44a49589..f8e627160 100644 --- a/mapillary_tools.spec +++ b/pyinstaller/mapillary_tools.spec @@ -5,7 +5,7 @@ block_cipher = None options = [("u", None, "OPTION")] a = Analysis( - ["./pyinstaller/main.py"], + ["./main.py"], pathex=[SPECPATH], binaries=[], datas=[], diff --git a/mapillary_tools_folder.spec b/pyinstaller/mapillary_tools_folder.spec similarity index 95% rename from mapillary_tools_folder.spec rename to pyinstaller/mapillary_tools_folder.spec index 3c7e48beb..69ac5faf6 100644 --- a/mapillary_tools_folder.spec +++ b/pyinstaller/mapillary_tools_folder.spec @@ -4,7 +4,7 @@ block_cipher = None -a = Analysis(["./pyinstaller/main.py"], +a = Analysis(["./main.py"], pathex=[SPECPATH], binaries=[], datas=[], diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..63311422e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,84 @@ +# Build system configuration - tells Python packaging tools how to build this package +# This section is required for modern Python packaging with pyproject.toml +[build-system] +requires = ["setuptools>=61.0", "wheel"] # Tools needed to build the package +build-backend = "setuptools.build_meta" # Use setuptools backend that reads pyproject.toml + +[project] +name = "mapillary_tools" +dynamic = ["version"] +description = "Mapillary Image/Video Import Pipeline" +readme = "README.md" +requires-python = ">=3.9" +license = {text = "BSD"} +authors = [ + {name = "Mapillary", email = "support@mapillary.com"}, +] +keywords = ["mapillary", "gis", "computer vision", "street view"] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: End Users/Desktop", + "License :: OSI Approved :: BSD License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", +] +dependencies = [ + "appdirs>=1.4.4,<2.0.0", + "construct>=2.10.0,<3.0.0", + "exifread==2.3.2", + "gpxpy>=1.5.0,<1.6.0", + "jsonschema~=4.17.3", + "piexif==1.1.3", + "pynmea2>=1.12.0,<2.0.0", + "requests[socks]>=2.20.0,<3.0.0", + "tqdm>=4.0,<5.0", + "typing-extensions>=4.12.2", +] + +[project.urls] +Homepage = "https://github.com/mapillary/mapillary_tools" +Repository = "https://github.com/mapillary/mapillary_tools" +Issues = "https://github.com/mapillary/mapillary_tools/issues" + +[project.scripts] +mapillary_tools = "mapillary_tools.commands.__main__:main" + +[tool.setuptools.dynamic] +version = {attr = "mapillary_tools.VERSION"} + +[tool.setuptools.packages.find] +include = ["mapillary_tools*"] + +[dependency-groups] +dev = [ + "mypy", + "pyinstaller", + "pyre-check", + "pytest", + "ruff", + "types-appdirs", + "types-requests", + "types-tqdm", + "types-jsonschema", + "usort", +] + +# Mypy configuration +[tool.mypy] +# Global mypy settings + +[[tool.mypy.overrides]] +module = [ + "piexif.*", + "pynmea2.*", + "gpxpy.*", + "exifread.*", + "construct.*", + "py.*", +] +ignore_missing_imports = true \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index f7ea7715e..000000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,8 +0,0 @@ -pytest -mypy -ruff -pyinstaller -types-requests -types-appdirs -usort -pyre-check diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index bc6d1fd73..000000000 --- a/requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -appdirs>=1.4.4,<2.0.0 -construct>=2.10.0,<3.0.0 -exifread==2.3.2 -piexif==1.1.3 -gpxpy>=1.5.0,<1.6.0 -pynmea2>=1.12.0,<2.0.0 -requests[socks]>=2.20.0,<3.0.0 -tqdm>=4.0,<5.0 -typing_extensions -jsonschema~=4.17.3 diff --git a/script/build_linux b/script/build_linux index ac1d75dc4..3b1929198 100755 --- a/script/build_linux +++ b/script/build_linux @@ -7,8 +7,8 @@ OS=linux mkdir -p dist rm -rf dist/${OS} pyinstaller --version -pyinstaller --noconfirm --distpath dist/${OS} mapillary_tools.spec -pyinstaller --noconfirm --distpath dist/${OS} mapillary_tools_folder.spec +pyinstaller --noconfirm --distpath dist/${OS} pyinstaller/mapillary_tools.spec +pyinstaller --noconfirm --distpath dist/${OS} pyinstaller/mapillary_tools_folder.spec # check SOURCE=dist/${OS}/mapillary_tools diff --git a/script/build_osx b/script/build_osx index 6b2bfe7b7..35195bd58 100755 --- a/script/build_osx +++ b/script/build_osx @@ -7,8 +7,8 @@ OS=osx mkdir -p dist rm -rf dist/${OS} pyinstaller --version -pyinstaller --noconfirm --distpath dist/${OS} mapillary_tools.spec -pyinstaller --noconfirm --distpath dist/${OS} mapillary_tools_folder.spec +pyinstaller --noconfirm --distpath dist/${OS} pyinstaller/mapillary_tools.spec +pyinstaller --noconfirm --distpath dist/${OS} pyinstaller/mapillary_tools_folder.spec # check SOURCE=dist/${OS}/mapillary_tools.app/Contents/MacOS/mapillary_tools diff --git a/script/build_win.ps1 b/script/build_win.ps1 index 3eb1cdf13..cd0612188 100644 --- a/script/build_win.ps1 +++ b/script/build_win.ps1 @@ -12,8 +12,8 @@ if ($MAXSIZE32 -ceq "True") { # build mkdir -Force dist pyinstaller --version -pyinstaller --noconfirm --distpath dist\win mapillary_tools.spec -pyinstaller --noconfirm --distpath dist\win mapillary_tools_folder.spec +pyinstaller --noconfirm --distpath dist\win pyinstaller\mapillary_tools.spec +pyinstaller --noconfirm --distpath dist\win pyinstaller\mapillary_tools_folder.spec # check $SOURCE="dist\win\mapillary_tools.exe" diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index c117b1d09..000000000 --- a/setup.cfg +++ /dev/null @@ -1,3 +0,0 @@ -[tool:pytest] -testpaths = mapillary_tools -addopts = --doctest-modules \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index c67fd8830..000000000 --- a/setup.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python -import os - -from setuptools import setup - -here = os.path.abspath(os.path.dirname(__file__)) - - -def read_requirements(): - import ssl - - requires = [] - - # Workaround for python3.9 on macOS which is compiled with LibreSSL - # See https://github.com/urllib3/urllib3/issues/3020 - if not ssl.OPENSSL_VERSION.startswith("OpenSSL "): - requires.append("urllib3<2.0.0") - - with open("requirements.txt") as fp: - requires.extend([row.strip() for row in fp if row.strip()]) - - return requires - - -about = {} -with open(os.path.join(here, "mapillary_tools", "__init__.py"), "r") as f: - exec(f.read(), about) - - -def readme(): - with open("README.md") as f: - return f.read() - - -setup( - name="mapillary_tools", - version=about["VERSION"], - description="Mapillary Image/Video Import Pipeline", - long_description=readme(), - long_description_content_type="text/markdown", - url="https://github.com/mapillary/mapillary_tools", - author="Mapillary", - license="BSD", - python_requires=">=3.8", - packages=[ - "mapillary_tools", - "mapillary_tools.camm", - "mapillary_tools.commands", - "mapillary_tools.geotag", - "mapillary_tools.geotag.image_extractors", - "mapillary_tools.geotag.video_extractors", - "mapillary_tools.gpmf", - "mapillary_tools.mp4", - ], - entry_points=""" - [console_scripts] - mapillary_tools=mapillary_tools.commands.__main__:main - """, - install_requires=read_requirements(), -)