WIP: add API v4 support (#393)

* feat: add API v4 support

* do not require MAPSettingsUserKey

* fix mime

* add back MAPSettingsUserKey as required

* sort images by captured_at

* refactor

* v4 login support

* handle null cluster id

* fix config path

* fix types

* fix path in windows

* fix the vertical path in windows

* read captured_at from mapillary_image_description.json instead of EXIF

* refactor logins

* refactor login again

* default api version to v4

* delete send_videos_for_processing

* delete download commands

* bump version to 0.7

* update README

* remove master key

* format

* update the endpoints

Author: ptpt

README.md

@@ -46,12 +46,6 @@ password?" link on the Mapillary sign-in screen.
 To upload images to Mapillary, image `GPS` and `capture time` are minimally required. More
 information [here](https://help.mapillary.com/hc/en-us/articles/115001717829-Geotagging-images).
 
-### Videos
-
-To upload videos to Mapillary, videos are sampled into images and tagged with image `GPS` and `capture time`. Use
-the [`send_videos_for_processing`](#direct-upload-recommended) command for Blackvue cameras. For all other models
-use [Video Sampling and Upload](#video-sampling-and-upload)
-
 ## Installation
 
 ### Python 3
@@ -204,7 +198,6 @@ Available commands for advanced usage:
     - interpolate
     - authenticate
     - post_process
-    - download
 
 ### Geotag and Upload
 
@@ -625,66 +618,10 @@ gps data in a set of otherwise geotagged images.
 `post_process` provides functionalities to help summarize and organize the results of the `process` and/or `upload`
 commands.
 
-#### `download`
-
-There are few ways to download blurred originals of private images from Mapillary: by import path or by image key.
-
-##### Downloading by import path
-
-`download` (by import path) will download blurred originals of private images on Mapillary for a certain `import_path`.
-The import path is specified as a folder where you have the images you've uploaded to Mapillary as private imagery. They
-need to have the Mapillary image description field in EXIF (which gets added during capture with our mobile apps or
-processing with our command line tools). Matching images will be downloaded to the output folder you specify.
-
-```bash
-mapillary_tools download --advanced --import_path "path/to/images" --output_folder "path/to/output_folder"
-```
-
-##### Downloading by image key
-
-You can download any images which belong to an organization (whether private or public) by using this command. This
-command downloads **private images by default**, to download publicly uploaded images from the given organization you
-need to pass `--private=false` flag. There are some access restrictions when it comes to downloading the data: you have
-to be authenticated as an organization admin/member to download the imagery. Contributors don't have access to this
-command. Attempting to download an organization image (whether public/private) as a contributor will yield
-an `You don't have sufficient organization access to download this image` error.
-
-```bash
-mapillary_tools download --advanced --by_property key \
-    --import_path /dev/null \
-    --output_folder "path/to/output_folder" \
-    --organization_keys "org_key1" "org_key2" \
-    --user_name "mapillary_username"
-```
-
-The command above specifies will attempt to download all privately blurred images for the organization `org_key`. These
-are all flags the command supports:
-
-- `organization_keys` - what are the keys of organizations which own the images
-- `private` - download private/non-private images (default is `--private=true`)
-- `user_name` - the username of the authenticate user (see `authenticate` section)
-- `import_path` - it's ignored in this script
-- `output_folder` - where to download the images to
-- `start_time` - the beginning of the time range, `YYYY-MM-DD`
-- `end_time` - the end of the time range, `YYYY-MM-DD`
-
 ## Camera specific
 
 ### BlackVue
-
-#### Direct Upload (Recommended)
-
-- Upload videos located in `path/to/videos` directly to Mapillary. Videos are moved to `path/to/videos/uploaded` folder
-  after upload. Videos that do not contain valid imagery are not uploaded to minimize bandwitdh usage. Sampling is
-  performed in the cloud so no extra disk space is required. This command supports the front camera video from the
-  Blackvue DR900s (1-channel and 2-channel) models.
-
- ```bash
- mapillary_tools send_videos_for_processing --advanced --video_import_path "path/to/videos" \
-     --user_name "mapillary_username"
-```
-
-#### Local sampling (Deprecated)
+#### Local sampling
 
 - Sample one or more Blackvue videos in directory `path/to/videos` into import path `path/to/images` at a sampling rate
   0.2 seconds, ie 5 frames every second and process resulting video frames for user `mapillary_user`, reading geotag

mapillary_tools/__init__.py

@@ -1 +1,9 @@
-VERSION = "0.6.0"
+import os
+
+VERSION = "0.7.0"
+
+MAPILLARY_API_VERSION = os.getenv("MAPILLARY_API_VERSION", "v4")
+assert MAPILLARY_API_VERSION in [
+    "v3",
+    "v4",
+], "MAPILLARY_API_VERSION must be either v3 or v4"

mapillary_tools/api_v3.py

@@ -4,15 +4,17 @@
 from .error import print_error
 
 API_ENDPOINT = os.getenv("API_PROXY_HOST", "https://a.mapillary.com")
-CLIENT_ID = os.getenv(
+MAPILLARY_WEB_CLIENT_ID = os.getenv(
     "MAPILLARY_WEB_CLIENT_ID", "MkJKbDA0bnZuZlcxeTJHTmFqN3g1dzo1YTM0NjRkM2EyZGU5MzBh"
 )
 
 
 def get_user(jwt):
     headers = {"Authorization": f"Bearer {jwt}"}
     resp = requests.get(
-        f"{API_ENDPOINT}/v3/me", params={"client_id": CLIENT_ID}, headers=headers
+        f"{API_ENDPOINT}/v3/me",
+        params={"client_id": MAPILLARY_WEB_CLIENT_ID},
+        headers=headers,
     )
     resp.raise_for_status()
     return resp.json()
@@ -22,7 +24,7 @@ def fetch_user_organizations(user_key, auth_token):
     headers = {"Authorization": f"Bearer {auth_token}"}
     resp = requests.get(
         f"{API_ENDPOINT}/v3/users/{user_key}/organizations",
-        params={"client_id": CLIENT_ID},
+        params={"client_id": MAPILLARY_WEB_CLIENT_ID},
         headers=headers,
     )
     resp.raise_for_status()
@@ -32,7 +34,9 @@ def fetch_user_organizations(user_key, auth_token):
 def get_upload_token(mail, pwd):
     payload = {"email": mail, "password": pwd}
     resp = requests.post(
-        f"{API_ENDPOINT}/v2/ua/login", params={"client_id": CLIENT_ID}, json=payload
+        f"{API_ENDPOINT}/v2/ua/login",
+        params={"client_id": MAPILLARY_WEB_CLIENT_ID},
+        json=payload,
     )
     if resp.status_code == 401:
         return None
@@ -43,7 +47,7 @@ def get_upload_token(mail, pwd):
 def get_user_key(user_name):
     resp = requests.get(
         f"{API_ENDPOINT}/v3/users",
-        params={"client_id": CLIENT_ID, "usernames": user_name},
+        params={"client_id": MAPILLARY_WEB_CLIENT_ID, "usernames": user_name},
     )
     resp.raise_for_status()
     resp = resp.json()

mapillary_tools/api_v4.py

@@ -0,0 +1,19 @@
+import os
+import requests
+
+MAPILLARY_WEB_CLIENT_ID = os.getenv(
+    "MAPILLARY_WEB_CLIENT_ID", "MLY|5675152195860640|6b02c72e6e3c801e5603ab0495623282"
+)
+MAPILLARY_GRAPH_API_ENDPOINT = os.getenv(
+    "MAPILLARY_GRAPH_API_ENDPOINT", "https://graph.mapillary.com"
+)
+
+
+def get_upload_token(email: str, password: str) -> dict:
+    resp = requests.post(
+        f"{MAPILLARY_GRAPH_API_ENDPOINT}/login",
+        params={"access_token": MAPILLARY_WEB_CLIENT_ID},
+        json={"email": email, "password": password},
+    )
+    resp.raise_for_status()
+    return resp.json()

mapillary_tools/commands/__init__.py

@@ -1,5 +1,4 @@
 from . import authenticate
-from . import download
 from . import exif_insert
 from . import extract_geotag_data
 from . import extract_import_meta_data
@@ -12,7 +11,6 @@
 from . import process_and_upload
 from . import process_csv
 from . import sample_video
-from . import send_videos_for_processing
 from . import upload
 from . import video_process
 from . import video_process_and_upload
@@ -31,8 +29,6 @@
     authenticate,
     interpolate,
     post_process,
-    download,
-    send_videos_for_processing,
 ]
 
 mapillary_tools_commands = [process, upload, process_and_upload]
@@ -73,7 +69,6 @@ def add_general_arguments(parser, command):
         "video_process",
         "video_process_and_upload",
         "sample_video",
-        "send_videos_for_processing",
     ]:
         required = False
 

mapillary_tools/commands/authenticate.py

@@ -8,7 +8,7 @@ class Command:
     def add_basic_arguments(self, parser):
         parser.add_argument(
             "--config_file",
-            help="Full path to the config file to be edited. Default is ~/.config/mapillary/configs/MkJKbDA0bnZuZlcxeTJHTmFqN3g1dzo1YTM0NjRkM2EyZGU5MzBh",
+            help="Full path to the config file to be edited. Default is ~/.config/mapillary/configs/MLY_CLIENT_ID",
             default=None,
             required=False,
         )