diff --git a/mapillary_tools/api_v4.py b/mapillary_tools/api_v4.py index fb6606243..6e5088a7c 100644 --- a/mapillary_tools/api_v4.py +++ b/mapillary_tools/api_v4.py @@ -70,7 +70,7 @@ def _truncate(s, limit=512): return s -def _sanitize(headers: T.Dict): +def _sanitize(headers: T.Mapping[T.Any, T.Any]) -> T.Mapping[T.Any, T.Any]: new_headers = {} for k, v in headers.items(): @@ -81,6 +81,7 @@ def _sanitize(headers: T.Dict): "access-token", "access_token", "password", + "user_upload_token", ]: new_headers[k] = "[REDACTED]" else: @@ -224,6 +225,44 @@ def request_get( return resp +def is_auth_error(resp: requests.Response) -> bool: + if resp.status_code in [401, 403]: + return True + + if resp.status_code in [400]: + try: + error_body = resp.json() + except Exception: + error_body = {} + + type = error_body.get("debug_info", {}).get("type") + if type in ["NotAuthorizedError"]: + return True + + return False + + +def extract_auth_error_message(resp: requests.Response) -> str: + assert is_auth_error(resp), "has to be an auth error" + + try: + error_body = resp.json() + except Exception: + error_body = {} + + # from Graph APIs + message = error_body.get("error", {}).get("message") + if message is not None: + return str(message) + + # from upload service + message = error_body.get("debug_info", {}).get("message") + if message is not None: + return str(message) + + return resp.text + + def get_upload_token(email: str, password: str) -> requests.Response: resp = request_post( f"{MAPILLARY_GRAPH_API_ENDPOINT}/login", @@ -252,6 +291,30 @@ def fetch_organization( return resp +def fetch_user_or_me( + user_access_token: str, + user_id: T.Optional[T.Union[int, str]] = None, +) -> requests.Response: + if user_id is None: + url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/me" + else: + url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/{user_id}" + + resp = request_get( + url, + params={ + "fields": ",".join(["id", "username"]), + }, + headers={ + "Authorization": f"OAuth {user_access_token}", + }, + timeout=REQUESTS_TIMEOUT, + ) + + resp.raise_for_status() + return resp + + ActionType = T.Literal[ "upload_started_upload", "upload_finished_upload", "upload_failed_upload" ] diff --git a/mapillary_tools/authenticate.py b/mapillary_tools/authenticate.py index 4b80b46a4..af956f5e2 100644 --- a/mapillary_tools/authenticate.py +++ b/mapillary_tools/authenticate.py @@ -1,102 +1,362 @@ +from __future__ import annotations + import getpass +import json import logging +import re +import sys import typing as T -import jsonschema import requests +import jsonschema -from . import api_v4, config, types +from . import api_v4, config, constants, exceptions, types LOG = logging.getLogger(__name__) def authenticate( - user_name: T.Optional[str] = None, - user_email: T.Optional[str] = None, - user_password: T.Optional[str] = None, - jwt: T.Optional[str] = None, + user_name: str | None = None, + user_email: str | None = None, + user_password: str | None = None, + jwt: str | None = None, + delete: bool = False, ): - if user_name: - user_name = user_name.strip() + """ + Prompt for authentication information and save it to the config file + """ + + # We still have to accept --user_name for the back compatibility + profile_name = user_name + + all_user_items = config.list_all_users() + if all_user_items: + _list_all_profiles(all_user_items) + else: + _welcome() - while not user_name: - user_name = input( - "Enter the Mapillary username you would like to (re)authenticate: " + # Make sure profile name either validated or existed + if profile_name is not None: + profile_name = profile_name.strip() + else: + if not _prompt_enabled(): + raise exceptions.MapillaryBadParameterError( + "Profile name is required, please specify one with --user_name" + ) + profile_name = _prompt_choose_profile_name( + list(all_user_items.keys()), must_exist=delete ) - user_name = user_name.strip() - if jwt: - user_items: types.UserItem = { - "user_upload_token": jwt, - } - elif user_email and user_password: - resp = api_v4.get_upload_token(user_email, user_password) - data = resp.json() - user_items = { - "MAPSettingsUserKey": data["user_id"], - "user_upload_token": data["access_token"], - } + assert profile_name is not None, "profile_name should be set" + + if delete: + config.remove_config(profile_name) + LOG.info('Profile "%s" deleted', profile_name) + else: + if profile_name in all_user_items: + LOG.warning( + 'The profile "%s" already exists and will be overridden', + profile_name, + ) + else: + LOG.info('Creating new profile: "%s"', profile_name) + + if jwt: + user_items: types.UserItem = {"user_upload_token": jwt} + user_items = _verify_user_auth(_validate_profile(user_items)) + else: + user_items = _prompt_login( + user_email=user_email, user_password=user_password + ) + _validate_profile(user_items) + + # Update the config with the new user items + config.update_config(profile_name, user_items) + + # TODO: print more user information + if profile_name in all_user_items: + LOG.info( + 'Profile "%s" updated: %s', profile_name, api_v4._sanitize(user_items) + ) + else: + LOG.info( + 'Profile "%s" created: %s', profile_name, api_v4._sanitize(user_items) + ) + + +def fetch_user_items( + user_name: str | None = None, + organization_key: str | None = None, +) -> types.UserItem: + """ + Read user information from the config file, + or prompt the user to authenticate if the specified profile does not exist + """ + + # we still have to accept --user_name for the back compatibility + profile_name = user_name + + all_user_items = config.list_all_users() + if not all_user_items: + authenticate(user_name=profile_name) + + # Fetch user information only here + all_user_items = config.list_all_users() + assert len(all_user_items) >= 1, "should have at least 1 profile" + if profile_name is None: + if len(all_user_items) > 1: + if not _prompt_enabled(): + raise exceptions.MapillaryBadParameterError( + "Multiple user profiles found, please choose one with --user_name" + ) + _list_all_profiles(all_user_items) + profile_name = _prompt_choose_profile_name( + list(all_user_items.keys()), must_exist=True + ) + user_items = all_user_items[profile_name] + else: + profile_name, user_items = list(all_user_items.items())[0] else: - user_items = prompt_user_for_user_items(user_name) + if profile_name in all_user_items: + user_items = all_user_items[profile_name] + else: + _list_all_profiles(all_user_items) + raise exceptions.MapillaryBadParameterError( + f'Profile "{profile_name}" not found' + ) + + assert profile_name is not None, "profile_name should be set" - config.update_config(user_name, user_items) + user_items = _verify_user_auth(_validate_profile(user_items)) + LOG.info( + 'Uploading to profile "%s": %s', profile_name, api_v4._sanitize(user_items) + ) + + if organization_key is not None: + resp = api_v4.fetch_organization( + user_items["user_upload_token"], organization_key + ) + LOG.info("Uploading to Mapillary organization: %s", json.dumps(resp.json())) + user_items["MAPOrganizationKey"] = organization_key + + return user_items + + +def _echo(*args, **kwargs): + print(*args, **kwargs, file=sys.stderr) -def prompt_user_for_user_items(user_name: str) -> types.UserItem: - print(f"Sign in for user {user_name}") - user_email = input("Enter your Mapillary user email: ") - user_password = getpass.getpass("Enter Mapillary user password: ") +def _prompt(message: str) -> str: + """Display prompt on stderr and get input from stdin""" + print(message, end="", file=sys.stderr, flush=True) + return input() + + +def _validate_profile(user_items: types.UserItem) -> types.UserItem: try: - resp = api_v4.get_upload_token(user_email, user_password) + jsonschema.validate(user_items, types.UserItemSchema) + except jsonschema.ValidationError as ex: + raise exceptions.MapillaryBadParameterError( + f"Invalid profile format: {ex.message}" + ) + return user_items + + +def _verify_user_auth(user_items: types.UserItem) -> types.UserItem: + """ + Verify that the user access token is valid + """ + if constants._AUTH_VERIFICATION_DISABLED: + return user_items + + try: + resp = api_v4.fetch_user_or_me( + user_access_token=user_items["user_upload_token"] + ) except requests.HTTPError as ex: - if ( - isinstance(ex, requests.HTTPError) - and isinstance(ex.response, requests.Response) - and 400 <= ex.response.status_code < 500 - ): - r = ex.response.json() - subcode = r.get("error", {}).get("error_subcode") - if subcode in [1348028, 1348092, 3404005, 1348131]: - title = r.get("error", {}).get("error_user_title") - message = r.get("error", {}).get("error_user_msg") - LOG.error("%s: %s", title, message) - return prompt_user_for_user_items(user_name) - else: - raise ex + if api_v4.is_auth_error(ex.response): + message = api_v4.extract_auth_error_message(ex.response) + raise exceptions.MapillaryUploadUnauthorizedError(message) else: raise ex - data = resp.json() - upload_token = T.cast(str, data.get("access_token")) - user_key = T.cast(str, data.get("user_id")) - if not isinstance(upload_token, str) or not isinstance(user_key, (str, int)): - raise RuntimeError( - f"Error extracting user_key or token from the login response: {data}" + user_json = resp.json() + + return { + **user_items, + "MAPSettingsUsername": user_json.get("username"), + "MAPSettingsUserKey": user_json.get("id"), + } + + +def _validate_profile_name(profile_name: str): + if not (2 <= len(profile_name) <= 32): + raise exceptions.MapillaryBadParameterError( + "Profile name must be between 2 and 32 characters long" ) - if isinstance(user_key, int): - user_key = str(user_key) + pattern = re.compile(r"^[a-zA-Z]+[a-zA-Z0-9_-]*$") + if not bool(pattern.match(profile_name)): + raise exceptions.MapillaryBadParameterError( + "Invalid profile name. Use only letters, numbers, hyphens and underscores" + ) - return { - "MAPSettingsUserKey": user_key, - "user_upload_token": upload_token, + +def _list_all_profiles(profiles: dict[str, types.UserItem]) -> None: + _echo("Existing Mapillary profiles:") + + # Header + _echo(f"{'':>5} {'Profile name':<32} {'User ID':>16} {'Username':>32}") + + # List all profiles + for idx, name in enumerate(profiles, 1): + items = profiles[name] + user_id = items.get("MAPSettingsUserKey", "N/A") + username = items.get("MAPSettingsUsername", "N/A") + _echo(f"{idx:>5}. {name:<32} {user_id:>16} {username:>32}") + + +def _is_interactive(): + # Check if stdout is connected to a terminal + stdout_interactive = sys.stdout.isatty() if hasattr(sys.stdout, "isatty") else False + + # Optionally, also check stdin and stderr + stdin_interactive = sys.stdin.isatty() if hasattr(sys.stdin, "isatty") else False + stderr_interactive = sys.stderr.isatty() if hasattr(sys.stderr, "isatty") else False + + # Return True if any stream is interactive + return stdout_interactive or stdin_interactive or stderr_interactive + + +def _prompt_enabled() -> bool: + if constants.PROMPT_DISABLED: + return False + + if not _is_interactive(): + return False + + return True + + +def _is_login_retryable(ex: requests.HTTPError) -> bool: + if 400 <= ex.response.status_code < 500: + r = ex.response.json() + subcode = r.get("error", {}).get("error_subcode") + if subcode in [1348028, 1348092, 3404005, 1348131]: + title = r.get("error", {}).get("error_user_title") + message = r.get("error", {}).get("error_user_msg") + LOG.error("%s: %s", title, message) + return True + return False + + +def _prompt_login( + user_email: str | None = None, + user_password: str | None = None, +) -> types.UserItem: + _enabled = _prompt_enabled() + + if user_email is None: + if not _enabled: + raise exceptions.MapillaryBadParameterError("user_email is required") + while not user_email: + user_email = _prompt("Enter Mapillary user email: ").strip() + else: + user_email = user_email.strip() + + if user_password is None: + if not _enabled: + raise exceptions.MapillaryBadParameterError("user_password is required") + while True: + user_password = getpass.getpass("Enter Mapillary user password: ") + if user_password: + break + + try: + resp = api_v4.get_upload_token(user_email, user_password) + except requests.HTTPError as ex: + if not _enabled: + raise ex + + if _is_login_retryable(ex): + return _prompt_login() + + raise ex + + data = resp.json() + + user_items: types.UserItem = { + "user_upload_token": str(data["access_token"]), + "MAPSettingsUserKey": str(data["user_id"]), } + return user_items + + +def _prompt_choose_profile_name( + existing_profile_names: T.Sequence[str], must_exist: bool = False +) -> str: + assert _prompt_enabled(), "should not get here if prompting is disabled" + + existed = set(existing_profile_names) + + while True: + if must_exist: + prompt = "Enter an existing profile: " + else: + prompt = "Enter an existing profile or create a new one: " + + profile_name = _prompt(prompt).strip() -def authenticate_user(user_name: str) -> types.UserItem: - user_items = config.load_user(user_name) - if user_items is not None: + if not profile_name: + continue + + # Exit if it's found + if profile_name in existed: + break + + # Try to find by index try: - jsonschema.validate(user_items, types.UserItemSchema) - except jsonschema.ValidationError: + profile_name = existing_profile_names[int(profile_name) - 1] + except (ValueError, IndexError): pass else: - return user_items + # Exit if it's found + break - user_items = prompt_user_for_user_items(user_name) - jsonschema.validate(user_items, types.UserItemSchema) - config.update_config(user_name, user_items) + assert profile_name not in existed, ( + f"Profile {profile_name} must not exist here" + ) - return user_items + if must_exist: + LOG.error('Profile "%s" not found', profile_name) + else: + try: + _validate_profile_name(profile_name) + except exceptions.MapillaryBadParameterError as ex: + LOG.error("Error validating profile name: %s", ex) + profile_name = "" + else: + break + + if must_exist: + assert profile_name in existed, f"Profile {profile_name} must exist" + + return profile_name + + +def _welcome(): + _echo( + """ +================================================================================ + Welcome to Mapillary! +================================================================================ + If you haven't registered yet, please visit https://www.mapillary.com/signup + to create your account first. + + Once registered, proceed here to sign in. +================================================================================ + """ + ) diff --git a/mapillary_tools/commands/authenticate.py b/mapillary_tools/commands/authenticate.py index b6c6a9242..e30c92014 100644 --- a/mapillary_tools/commands/authenticate.py +++ b/mapillary_tools/commands/authenticate.py @@ -10,7 +10,7 @@ class Command: def add_basic_arguments(self, parser: argparse.ArgumentParser): parser.add_argument( - "--user_name", help="Mapillary user name", default=None, required=False + "--user_name", help="Mapillary user profile", default=None, required=False ) parser.add_argument( "--user_email", @@ -27,6 +27,13 @@ def add_basic_arguments(self, parser: argparse.ArgumentParser): parser.add_argument( "--jwt", help="Mapillary user access token", default=None, required=False ) + parser.add_argument( + "--delete", + help="Delete the specified user profile", + default=False, + required=False, + action="store_true", + ) def run(self, vars_args: dict): authenticate( diff --git a/mapillary_tools/commands/process_and_upload.py b/mapillary_tools/commands/process_and_upload.py index c7ffadd3f..268986b8e 100644 --- a/mapillary_tools/commands/process_and_upload.py +++ b/mapillary_tools/commands/process_and_upload.py @@ -1,3 +1,7 @@ +import inspect + +from ..authenticate import fetch_user_items + from .process import Command as ProcessCommand from .upload import Command as UploadCommand @@ -10,11 +14,20 @@ def add_basic_arguments(self, parser): ProcessCommand().add_basic_arguments(parser) UploadCommand().add_basic_arguments(parser) - def run(self, args: dict): - if args.get("desc_path") is None: + def run(self, vars_args: dict): + if vars_args.get("desc_path") is None: # \x00 is a special path similiar to /dev/null # it tells process command do not write anything - args["desc_path"] = "\x00" + vars_args["desc_path"] = "\x00" + + if "user_items" not in vars_args: + vars_args["user_items"] = fetch_user_items( + **{ + k: v + for k, v in vars_args.items() + if k in inspect.getfullargspec(fetch_user_items).args + } + ) - ProcessCommand().run(args) - UploadCommand().run(args) + ProcessCommand().run(vars_args) + UploadCommand().run(vars_args) diff --git a/mapillary_tools/commands/upload.py b/mapillary_tools/commands/upload.py index f8c6748cc..418ce0451 100644 --- a/mapillary_tools/commands/upload.py +++ b/mapillary_tools/commands/upload.py @@ -1,6 +1,7 @@ import inspect from .. import constants +from ..authenticate import fetch_user_items from ..upload import upload from .process import bold_text @@ -41,9 +42,18 @@ def add_basic_arguments(self, parser): Command.add_common_upload_options(group) def run(self, vars_args: dict): - args = { - k: v - for k, v in vars_args.items() - if k in inspect.getfullargspec(upload).args - } - upload(**args) + if "user_items" not in vars_args: + user_items_args = { + k: v + for k, v in vars_args.items() + if k in inspect.getfullargspec(fetch_user_items).args + } + vars_args["user_items"] = fetch_user_items(**user_items_args) + + upload( + **{ + k: v + for k, v in vars_args.items() + if k in inspect.getfullargspec(upload).args + } + ) diff --git a/mapillary_tools/commands/video_process_and_upload.py b/mapillary_tools/commands/video_process_and_upload.py index 12d77480b..d1407a7d3 100644 --- a/mapillary_tools/commands/video_process_and_upload.py +++ b/mapillary_tools/commands/video_process_and_upload.py @@ -1,3 +1,7 @@ +import inspect + +from ..authenticate import fetch_user_items + from .upload import Command as UploadCommand from .video_process import Command as VideoProcessCommand @@ -10,10 +14,20 @@ def add_basic_arguments(self, parser): VideoProcessCommand().add_basic_arguments(parser) UploadCommand().add_basic_arguments(parser) - def run(self, args: dict): - if args.get("desc_path") is None: + def run(self, vars_args: dict): + if vars_args.get("desc_path") is None: # \x00 is a special path similiar to /dev/null # it tells process command do not write anything - args["desc_path"] = "\x00" - VideoProcessCommand().run(args) - UploadCommand().run(args) + vars_args["desc_path"] = "\x00" + + if "user_items" not in vars_args: + vars_args["user_items"] = fetch_user_items( + **{ + k: v + for k, v in vars_args.items() + if k in inspect.getfullargspec(fetch_user_items).args + } + ) + + VideoProcessCommand().run(vars_args) + UploadCommand().run(vars_args) diff --git a/mapillary_tools/config.py b/mapillary_tools/config.py index a70b8b17a..8f100262f 100644 --- a/mapillary_tools/config.py +++ b/mapillary_tools/config.py @@ -35,37 +35,53 @@ def _load_config(config_path: str) -> configparser.ConfigParser: def load_user( - user_name: str, config_path: T.Optional[str] = None + profile_name: str, config_path: T.Optional[str] = None ) -> T.Optional[types.UserItem]: if config_path is None: config_path = MAPILLARY_CONFIG_PATH config = _load_config(config_path) - if not config.has_section(user_name): + if not config.has_section(profile_name): return None - user_items = dict(config.items(user_name)) + user_items = dict(config.items(profile_name)) return T.cast(types.UserItem, user_items) -def list_all_users(config_path: T.Optional[str] = None) -> T.List[types.UserItem]: +def list_all_users(config_path: T.Optional[str] = None) -> T.Dict[str, types.UserItem]: if config_path is None: config_path = MAPILLARY_CONFIG_PATH cp = _load_config(config_path) - users = [ - load_user(user_name, config_path=config_path) for user_name in cp.sections() - ] - return [item for item in users if item is not None] + users = { + profile_name: load_user(profile_name, config_path=config_path) + for profile_name in cp.sections() + } + return {profile: item for profile, item in users.items() if item is not None} def update_config( - user_name: str, user_items: types.UserItem, config_path: T.Optional[str] = None + profile_name: str, user_items: types.UserItem, config_path: T.Optional[str] = None ) -> None: if config_path is None: config_path = MAPILLARY_CONFIG_PATH config = _load_config(config_path) - if not config.has_section(user_name): - config.add_section(user_name) + if not config.has_section(profile_name): + config.add_section(profile_name) for key, val in user_items.items(): - config.set(user_name, key, T.cast(str, val)) + config.set(profile_name, key, T.cast(str, val)) + os.makedirs(os.path.dirname(os.path.abspath(config_path)), exist_ok=True) + with open(config_path, "w") as fp: + config.write(fp) + + +def remove_config(profile_name: str, config_path: T.Optional[str] = None) -> None: + if config_path is None: + config_path = MAPILLARY_CONFIG_PATH + + config = _load_config(config_path) + if not config.has_section(profile_name): + return + + config.remove_section(profile_name) + os.makedirs(os.path.dirname(os.path.abspath(config_path)), exist_ok=True) with open(config_path, "w") as fp: config.write(fp) diff --git a/mapillary_tools/constants.py b/mapillary_tools/constants.py index 60c13023b..a58cce8ee 100644 --- a/mapillary_tools/constants.py +++ b/mapillary_tools/constants.py @@ -5,6 +5,15 @@ _ENV_PREFIX = "MAPILLARY_TOOLS_" + +def _yes_or_no(val: str) -> bool: + return val.strip().upper() in [ + "1", + "TRUE", + "YES", + ] + + # In meters CUTOFF_DISTANCE = float(os.getenv(_ENV_PREFIX + "CUTOFF_DISTANCE", 600)) # In seconds @@ -55,3 +64,9 @@ MAX_SEQUENCE_FILESIZE: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_FILESIZE", "110G") # Max number of pixels per sequence (sum of image pixels in the sequence) MAX_SEQUENCE_PIXELS: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_PIXELS", "6G") + +PROMPT_DISABLED: bool = _yes_or_no(os.getenv(_ENV_PREFIX + "PROMPT_DISABLED", "NO")) + +_AUTH_VERIFICATION_DISABLED: bool = _yes_or_no( + os.getenv(_ENV_PREFIX + "_AUTH_VERIFICATION_DISABLED", "NO") +) diff --git a/mapillary_tools/upload.py b/mapillary_tools/upload.py index a3969918e..055b64a9b 100644 --- a/mapillary_tools/upload.py +++ b/mapillary_tools/upload.py @@ -12,8 +12,6 @@ from . import ( api_v4, - authenticate, - config, constants, exceptions, history, @@ -139,36 +137,6 @@ def zip_images( uploader.ZipImageSequence.zip_images(image_metadatas, zip_dir) -def fetch_user_items( - user_name: T.Optional[str] = None, organization_key: T.Optional[str] = None -) -> types.UserItem: - if user_name is None: - all_user_items = config.list_all_users() - if not all_user_items: - raise exceptions.MapillaryBadParameterError( - "No Mapillary account found. Add one with --user_name" - ) - if len(all_user_items) == 1: - user_items = all_user_items[0] - else: - raise exceptions.MapillaryBadParameterError( - "Found multiple Mapillary accounts. Please specify one with --user_name" - ) - else: - user_items = authenticate.authenticate_user(user_name) - - if organization_key is not None: - resp = api_v4.fetch_organization( - user_items["user_upload_token"], organization_key - ) - org = resp.json() - LOG.info("Uploading to organization: %s", json.dumps(org)) - user_items = T.cast( - types.UserItem, {**user_items, "MAPOrganizationKey": organization_key} - ) - return user_items - - def _setup_cancel_due_to_duplication(emitter: uploader.EventEmitter) -> None: @emitter.on("upload_start") def upload_start(payload: uploader.Progress): @@ -476,12 +444,94 @@ def _find_metadata_with_filename_existed_in( return [d for d in metadatas if d.filename.resolve() in resolved_image_paths] +def _upload_everything( + mly_uploader: uploader.Uploader, + import_paths: T.Sequence[Path], + metadatas: T.Sequence[types.Metadata], + skip_subfolders: bool, +): + # upload images + image_paths = utils.find_images(import_paths, skip_subfolders=skip_subfolders) + # find descs that match the image paths from the import paths + image_metadatas = [ + metadata + for metadata in (metadatas or []) + if isinstance(metadata, types.ImageMetadata) + ] + specified_image_metadatas = _find_metadata_with_filename_existed_in( + image_metadatas, image_paths + ) + if specified_image_metadatas: + try: + clusters = mly_uploader.upload_images( + specified_image_metadatas, + event_payload={"file_type": FileType.IMAGE.value}, + ) + except Exception as ex: + raise UploadError(ex) from ex + + if clusters: + LOG.debug("Uploaded to cluster: %s", clusters) + + # upload videos + video_paths = utils.find_videos(import_paths, skip_subfolders=skip_subfolders) + video_metadatas = [ + metadata + for metadata in (metadatas or []) + if isinstance(metadata, types.VideoMetadata) + ] + specified_video_metadatas = _find_metadata_with_filename_existed_in( + video_metadatas, video_paths + ) + for idx, video_metadata in enumerate(specified_video_metadatas): + video_metadata.update_md5sum() + assert isinstance(video_metadata.md5sum, str), "md5sum should be updated" + + # extract telemetry measurements from GoPro videos + telemetry_measurements: T.List[camm_parser.TelemetryMeasurement] = [] + if MAPILLARY__EXPERIMENTAL_ENABLE_IMU == "YES": + if video_metadata.filetype is FileType.GOPRO: + with video_metadata.filename.open("rb") as fp: + gopro_info = gpmf_parser.extract_gopro_info(fp, telemetry_only=True) + if gopro_info is not None: + telemetry_measurements.extend(gopro_info.accl or []) + telemetry_measurements.extend(gopro_info.gyro or []) + telemetry_measurements.extend(gopro_info.magn or []) + telemetry_measurements.sort(key=lambda m: m.time) + + generator = camm_builder.camm_sample_generator2( + video_metadata, telemetry_measurements=telemetry_measurements + ) + + with video_metadata.filename.open("rb") as src_fp: + camm_fp = simple_mp4_builder.transform_mp4(src_fp, generator) + event_payload: uploader.Progress = { + "total_sequence_count": len(specified_video_metadatas), + "sequence_idx": idx, + "file_type": video_metadata.filetype.value, + "import_path": str(video_metadata.filename), + } + try: + cluster_id = mly_uploader.upload_stream( + T.cast(T.BinaryIO, camm_fp), + upload_api_v4.ClusterFileType.CAMM, + video_metadata.md5sum, + event_payload=event_payload, + ) + except Exception as ex: + raise UploadError(ex) from ex + LOG.debug("Uploaded to cluster: %s", cluster_id) + + # upload zip files + zip_paths = utils.find_zipfiles(import_paths, skip_subfolders=skip_subfolders) + _upload_zipfiles(mly_uploader, zip_paths) + + def upload( import_path: T.Union[Path, T.Sequence[Path]], + user_items: types.UserItem, desc_path: T.Optional[str] = None, _metadatas_from_process: T.Optional[T.Sequence[types.MetadataOrError]] = None, - user_name: T.Optional[str] = None, - organization_key: T.Optional[str] = None, dry_run=False, skip_subfolders=False, ) -> None: @@ -505,8 +555,6 @@ def upload( metadatas = _load_descs(_metadatas_from_process, desc_path, import_paths) - user_items = fetch_user_items(user_name, organization_key) - # Setup the emitter -- the order matters here emitter = uploader.EventEmitter() @@ -547,81 +595,7 @@ def upload( ) try: - image_paths = utils.find_images(import_paths, skip_subfolders=skip_subfolders) - # find descs that match the image paths from the import paths - image_metadatas = [ - metadata - for metadata in (metadatas or []) - if isinstance(metadata, types.ImageMetadata) - ] - specified_image_metadatas = _find_metadata_with_filename_existed_in( - image_metadatas, image_paths - ) - if specified_image_metadatas: - try: - clusters = mly_uploader.upload_images( - specified_image_metadatas, - event_payload={"file_type": FileType.IMAGE.value}, - ) - except Exception as ex: - raise UploadError(ex) from ex - - if clusters: - LOG.debug("Uploaded to cluster: %s", clusters) - - video_paths = utils.find_videos(import_paths, skip_subfolders=skip_subfolders) - video_metadatas = [ - metadata - for metadata in (metadatas or []) - if isinstance(metadata, types.VideoMetadata) - ] - specified_video_metadatas = _find_metadata_with_filename_existed_in( - video_metadatas, video_paths - ) - for idx, video_metadata in enumerate(specified_video_metadatas): - video_metadata.update_md5sum() - assert isinstance(video_metadata.md5sum, str), "md5sum should be updated" - - # extract telemetry measurements from GoPro videos - telemetry_measurements: T.List[camm_parser.TelemetryMeasurement] = [] - if MAPILLARY__EXPERIMENTAL_ENABLE_IMU == "YES": - if video_metadata.filetype is FileType.GOPRO: - with video_metadata.filename.open("rb") as fp: - gopro_info = gpmf_parser.extract_gopro_info( - fp, telemetry_only=True - ) - if gopro_info is not None: - telemetry_measurements.extend(gopro_info.accl or []) - telemetry_measurements.extend(gopro_info.gyro or []) - telemetry_measurements.extend(gopro_info.magn or []) - telemetry_measurements.sort(key=lambda m: m.time) - - generator = camm_builder.camm_sample_generator2( - video_metadata, telemetry_measurements=telemetry_measurements - ) - - with video_metadata.filename.open("rb") as src_fp: - camm_fp = simple_mp4_builder.transform_mp4(src_fp, generator) - event_payload: uploader.Progress = { - "total_sequence_count": len(specified_video_metadatas), - "sequence_idx": idx, - "file_type": video_metadata.filetype.value, - "import_path": str(video_metadata.filename), - } - try: - cluster_id = mly_uploader.upload_stream( - T.cast(T.BinaryIO, camm_fp), - upload_api_v4.ClusterFileType.CAMM, - video_metadata.md5sum, - event_payload=event_payload, - ) - except Exception as ex: - raise UploadError(ex) from ex - LOG.debug("Uploaded to cluster: %s", cluster_id) - - zip_paths = utils.find_zipfiles(import_paths, skip_subfolders=skip_subfolders) - _upload_zipfiles(mly_uploader, zip_paths) - + _upload_everything(mly_uploader, import_paths, metadatas, skip_subfolders) except UploadError as ex: inner_ex = ex.inner_ex @@ -637,16 +611,10 @@ def upload( if isinstance(inner_ex, requests.HTTPError) and isinstance( inner_ex.response, requests.Response ): - if inner_ex.response.status_code in [400, 401]: - try: - error_body = inner_ex.response.json() - except Exception: - error_body = {} - debug_info = error_body.get("debug_info", {}) - if debug_info.get("type") in ["NotAuthorizedError"]: - raise exceptions.MapillaryUploadUnauthorizedError( - debug_info.get("message") - ) from inner_ex + if api_v4.is_auth_error(inner_ex.response): + raise exceptions.MapillaryUploadUnauthorizedError( + api_v4.extract_auth_error_message(inner_ex.response) + ) from inner_ex raise inner_ex raise inner_ex diff --git a/tests/cli/upload_api_v4.py b/tests/cli/upload_api_v4.py index 54718aa5a..1d1ceaf51 100644 --- a/tests/cli/upload_api_v4.py +++ b/tests/cli/upload_api_v4.py @@ -6,7 +6,7 @@ import requests import tqdm -from mapillary_tools import upload +from mapillary_tools import api_v4, authenticate from mapillary_tools.upload_api_v4 import DEFAULT_CHUNK_SIZE, UploadService @@ -14,16 +14,6 @@ LOG = logging.getLogger("mapillary_tools") -def wrap_http_exception(ex: requests.HTTPError): - resp = ex.response - lines = [ - f"{ex.request.method} {resp.url}", - f"> HTTP Status: {ex.response.status_code}", - f"{resp.content!r}", - ] - return Exception("\n".join(lines)) - - def configure_logger(logger: logging.Logger, stream=None) -> None: formatter = logging.Formatter("%(asctime)s - %(levelname)-7s - %(message)s") handler = logging.StreamHandler(stream) @@ -67,7 +57,7 @@ def main(): with open(parsed.filename, "rb") as fp: entity_size = _file_stats(fp) - user_items = upload.fetch_user_items(parsed.user_name) + user_items = authenticate.fetch_user_items(parsed.user_name) session_key = parsed.session_key user_access_token = user_items.get("user_upload_token", "") @@ -81,11 +71,15 @@ def main(): else DEFAULT_CHUNK_SIZE ), ) - initial_offset = service.fetch_offset() + + try: + initial_offset = service.fetch_offset() + except requests.HTTPError as ex: + raise RuntimeError(api_v4.readable_http_error(ex)) LOG.info("Session key: %s", session_key) - LOG.info("Entity size: %d", entity_size) LOG.info("Initial offset: %s", initial_offset) + LOG.info("Entity size: %d", entity_size) LOG.info("Chunk size: %s MB", service.chunk_size / (1024 * 1024)) with open(parsed.filename, "rb") as fp: @@ -101,9 +95,20 @@ def main(): try: file_handle = service.upload(fp, initial_offset) except requests.HTTPError as ex: - raise wrap_http_exception(ex) + raise RuntimeError(api_v4.readable_http_error(ex)) + except KeyboardInterrupt: + file_handle = None + LOG.warning("Upload interrupted") + + try: + final_offset = service.fetch_offset() + except requests.HTTPError as ex: + raise RuntimeError(api_v4.readable_http_error(ex)) + + LOG.info("Final offset: %s", final_offset) + LOG.info("Entity size: %d", entity_size) - LOG.info(file_handle) + LOG.info("File handle: %s", file_handle) if __name__ == "__main__": diff --git a/tests/integration/fixtures.py b/tests/integration/fixtures.py index cc30f54d3..02335309d 100644 --- a/tests/integration/fixtures.py +++ b/tests/integration/fixtures.py @@ -28,6 +28,8 @@ def setup_config(tmpdir: py.path.local): config_path = tmpdir.mkdir("configs").join("CLIENT_ID") os.environ["MAPILLARY_CONFIG_PATH"] = str(config_path) + os.environ["MAPILLARY_TOOLS_PROMPT_DISABLED"] = "YES" + os.environ["MAPILLARY_TOOLS__AUTH_VERIFICATION_DISABLED"] = "YES" x = subprocess.run( f"{EXECUTABLE} authenticate --user_name {USERNAME} --jwt test_user_token", shell=True, @@ -36,7 +38,9 @@ def setup_config(tmpdir: py.path.local): yield config_path if tmpdir.check(): tmpdir.remove(ignore_errors=True) - del os.environ["MAPILLARY_CONFIG_PATH"] + os.environ.pop("MAPILLARY_CONFIG_PATH", None) + os.environ.pop("MAPILLARY_TOOLS_PROMPT_DISABLED", None) + os.environ.pop("MAPILLARY_TOOLS__AUTH_VERIFICATION_DISABLED", None) @pytest.fixture @@ -53,15 +57,19 @@ def setup_data(tmpdir: py.path.local): def setup_upload(tmpdir: py.path.local): upload_dir = tmpdir.mkdir("mapillary_public_uploads") os.environ["MAPILLARY_UPLOAD_PATH"] = str(upload_dir) + os.environ["MAPILLARY_TOOLS__AUTH_VERIFICATION_DISABLED"] = "YES" + os.environ["MAPILLARY_TOOLS_PROMPT_DISABLED"] = "YES" os.environ["MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN"] = "YES" history_path = tmpdir.join("history") os.environ["MAPILLARY_UPLOAD_HISTORY_PATH"] = str(history_path) yield upload_dir if tmpdir.check(): tmpdir.remove(ignore_errors=True) - del os.environ["MAPILLARY_UPLOAD_PATH"] - del os.environ["MAPILLARY_UPLOAD_HISTORY_PATH"] - del os.environ["MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN"] + os.environ.pop("MAPILLARY_UPLOAD_PATH", None) + os.environ.pop("MAPILLARY_UPLOAD_HISTORY_PATH", None) + os.environ.pop("MAPILLARY_TOOLS__AUTH_VERIFICATION_DISABLED", None) + os.environ.pop("MAPILLARY_TOOLS_PROMPT_DISABLED", None) + os.environ.pop("MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN", None) def _ffmpeg_installed(): diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 9185b30e8..67af45cab 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -23,7 +23,7 @@ def test_config_list_all_users(tmpdir: py.path.local): x = config.list_all_users(config_path=str(c)) assert len(x) == 1 - assert x[0] == {"ThisIsOption": "1"} + assert x["hello"] == {"ThisIsOption": "1"} def test_update_config(tmpdir: py.path.local): @@ -50,3 +50,15 @@ def test_load_user(tmpdir: py.path.local): assert x is None x = config.load_user("world", config_path=str(c)) assert x == {"ThisIsOption": "hello"} + + +def test_remove(tmpdir: py.path.local): + c = tmpdir.join("empty_config.ini") + config.update_config( + "world", T.cast(T.Any, {"ThisIsOption": "hello"}), config_path=str(c) + ) + config.remove_config("world", config_path=str(c)) + u = config.load_user("world", config_path=str(c)) + assert u is None + x = config.list_all_users(config_path=str(c)) + assert not x