From 351b7e483f8d783384cc87089bac98ca02352d2d Mon Sep 17 00:00:00 2001 From: Tao Peng Date: Wed, 26 Mar 2025 14:46:42 -0700 Subject: [PATCH 01/16] improve: authenticate early for upload commands --- mapillary_tools/authenticate.py | 33 ++- .../commands/process_and_upload.py | 23 +- mapillary_tools/commands/upload.py | 22 +- .../commands/video_process_and_upload.py | 24 ++- mapillary_tools/upload.py | 196 ++++++++---------- 5 files changed, 170 insertions(+), 128 deletions(-) diff --git a/mapillary_tools/authenticate.py b/mapillary_tools/authenticate.py index 4b80b46a4..cf16ea8fa 100644 --- a/mapillary_tools/authenticate.py +++ b/mapillary_tools/authenticate.py @@ -1,11 +1,12 @@ import getpass +import json import logging import typing as T import jsonschema import requests -from . import api_v4, config, types +from . import api_v4, config, exceptions, types LOG = logging.getLogger(__name__) @@ -100,3 +101,33 @@ def authenticate_user(user_name: str) -> types.UserItem: config.update_config(user_name, user_items) return user_items + + +def fetch_user_items( + user_name: T.Optional[str] = None, organization_key: T.Optional[str] = None +) -> types.UserItem: + if user_name is None: + all_user_items = config.list_all_users() + if not all_user_items: + raise exceptions.MapillaryBadParameterError( + "No Mapillary account found. Add one with --user_name" + ) + if len(all_user_items) == 1: + user_items = all_user_items[0] + else: + raise exceptions.MapillaryBadParameterError( + "Found multiple Mapillary accounts. Please specify one with --user_name" + ) + else: + user_items = authenticate_user(user_name) + + if organization_key is not None: + resp = api_v4.fetch_organization( + user_items["user_upload_token"], organization_key + ) + org = resp.json() + LOG.info("Uploading to organization: %s", json.dumps(org)) + user_items = T.cast( + types.UserItem, {**user_items, "MAPOrganizationKey": organization_key} + ) + return user_items diff --git a/mapillary_tools/commands/process_and_upload.py b/mapillary_tools/commands/process_and_upload.py index c7ffadd3f..268986b8e 100644 --- a/mapillary_tools/commands/process_and_upload.py +++ b/mapillary_tools/commands/process_and_upload.py @@ -1,3 +1,7 @@ +import inspect + +from ..authenticate import fetch_user_items + from .process import Command as ProcessCommand from .upload import Command as UploadCommand @@ -10,11 +14,20 @@ def add_basic_arguments(self, parser): ProcessCommand().add_basic_arguments(parser) UploadCommand().add_basic_arguments(parser) - def run(self, args: dict): - if args.get("desc_path") is None: + def run(self, vars_args: dict): + if vars_args.get("desc_path") is None: # \x00 is a special path similiar to /dev/null # it tells process command do not write anything - args["desc_path"] = "\x00" + vars_args["desc_path"] = "\x00" + + if "user_items" not in vars_args: + vars_args["user_items"] = fetch_user_items( + **{ + k: v + for k, v in vars_args.items() + if k in inspect.getfullargspec(fetch_user_items).args + } + ) - ProcessCommand().run(args) - UploadCommand().run(args) + ProcessCommand().run(vars_args) + UploadCommand().run(vars_args) diff --git a/mapillary_tools/commands/upload.py b/mapillary_tools/commands/upload.py index f8c6748cc..418ce0451 100644 --- a/mapillary_tools/commands/upload.py +++ b/mapillary_tools/commands/upload.py @@ -1,6 +1,7 @@ import inspect from .. import constants +from ..authenticate import fetch_user_items from ..upload import upload from .process import bold_text @@ -41,9 +42,18 @@ def add_basic_arguments(self, parser): Command.add_common_upload_options(group) def run(self, vars_args: dict): - args = { - k: v - for k, v in vars_args.items() - if k in inspect.getfullargspec(upload).args - } - upload(**args) + if "user_items" not in vars_args: + user_items_args = { + k: v + for k, v in vars_args.items() + if k in inspect.getfullargspec(fetch_user_items).args + } + vars_args["user_items"] = fetch_user_items(**user_items_args) + + upload( + **{ + k: v + for k, v in vars_args.items() + if k in inspect.getfullargspec(upload).args + } + ) diff --git a/mapillary_tools/commands/video_process_and_upload.py b/mapillary_tools/commands/video_process_and_upload.py index 12d77480b..d1407a7d3 100644 --- a/mapillary_tools/commands/video_process_and_upload.py +++ b/mapillary_tools/commands/video_process_and_upload.py @@ -1,3 +1,7 @@ +import inspect + +from ..authenticate import fetch_user_items + from .upload import Command as UploadCommand from .video_process import Command as VideoProcessCommand @@ -10,10 +14,20 @@ def add_basic_arguments(self, parser): VideoProcessCommand().add_basic_arguments(parser) UploadCommand().add_basic_arguments(parser) - def run(self, args: dict): - if args.get("desc_path") is None: + def run(self, vars_args: dict): + if vars_args.get("desc_path") is None: # \x00 is a special path similiar to /dev/null # it tells process command do not write anything - args["desc_path"] = "\x00" - VideoProcessCommand().run(args) - UploadCommand().run(args) + vars_args["desc_path"] = "\x00" + + if "user_items" not in vars_args: + vars_args["user_items"] = fetch_user_items( + **{ + k: v + for k, v in vars_args.items() + if k in inspect.getfullargspec(fetch_user_items).args + } + ) + + VideoProcessCommand().run(vars_args) + UploadCommand().run(vars_args) diff --git a/mapillary_tools/upload.py b/mapillary_tools/upload.py index a3969918e..8f026e397 100644 --- a/mapillary_tools/upload.py +++ b/mapillary_tools/upload.py @@ -12,8 +12,6 @@ from . import ( api_v4, - authenticate, - config, constants, exceptions, history, @@ -139,36 +137,6 @@ def zip_images( uploader.ZipImageSequence.zip_images(image_metadatas, zip_dir) -def fetch_user_items( - user_name: T.Optional[str] = None, organization_key: T.Optional[str] = None -) -> types.UserItem: - if user_name is None: - all_user_items = config.list_all_users() - if not all_user_items: - raise exceptions.MapillaryBadParameterError( - "No Mapillary account found. Add one with --user_name" - ) - if len(all_user_items) == 1: - user_items = all_user_items[0] - else: - raise exceptions.MapillaryBadParameterError( - "Found multiple Mapillary accounts. Please specify one with --user_name" - ) - else: - user_items = authenticate.authenticate_user(user_name) - - if organization_key is not None: - resp = api_v4.fetch_organization( - user_items["user_upload_token"], organization_key - ) - org = resp.json() - LOG.info("Uploading to organization: %s", json.dumps(org)) - user_items = T.cast( - types.UserItem, {**user_items, "MAPOrganizationKey": organization_key} - ) - return user_items - - def _setup_cancel_due_to_duplication(emitter: uploader.EventEmitter) -> None: @emitter.on("upload_start") def upload_start(payload: uploader.Progress): @@ -476,12 +444,94 @@ def _find_metadata_with_filename_existed_in( return [d for d in metadatas if d.filename.resolve() in resolved_image_paths] +def _upload_everything( + mly_uploader: uploader.Uploader, + import_paths: T.Sequence[Path], + metadatas: T.Sequence[types.Metadata], + skip_subfolders: bool, +): + # upload images + image_paths = utils.find_images(import_paths, skip_subfolders=skip_subfolders) + # find descs that match the image paths from the import paths + image_metadatas = [ + metadata + for metadata in (metadatas or []) + if isinstance(metadata, types.ImageMetadata) + ] + specified_image_metadatas = _find_metadata_with_filename_existed_in( + image_metadatas, image_paths + ) + if specified_image_metadatas: + try: + clusters = mly_uploader.upload_images( + specified_image_metadatas, + event_payload={"file_type": FileType.IMAGE.value}, + ) + except Exception as ex: + raise UploadError(ex) from ex + + if clusters: + LOG.debug("Uploaded to cluster: %s", clusters) + + # upload videos + video_paths = utils.find_videos(import_paths, skip_subfolders=skip_subfolders) + video_metadatas = [ + metadata + for metadata in (metadatas or []) + if isinstance(metadata, types.VideoMetadata) + ] + specified_video_metadatas = _find_metadata_with_filename_existed_in( + video_metadatas, video_paths + ) + for idx, video_metadata in enumerate(specified_video_metadatas): + video_metadata.update_md5sum() + assert isinstance(video_metadata.md5sum, str), "md5sum should be updated" + + # extract telemetry measurements from GoPro videos + telemetry_measurements: T.List[camm_parser.TelemetryMeasurement] = [] + if MAPILLARY__EXPERIMENTAL_ENABLE_IMU == "YES": + if video_metadata.filetype is FileType.GOPRO: + with video_metadata.filename.open("rb") as fp: + gopro_info = gpmf_parser.extract_gopro_info(fp, telemetry_only=True) + if gopro_info is not None: + telemetry_measurements.extend(gopro_info.accl or []) + telemetry_measurements.extend(gopro_info.gyro or []) + telemetry_measurements.extend(gopro_info.magn or []) + telemetry_measurements.sort(key=lambda m: m.time) + + generator = camm_builder.camm_sample_generator2( + video_metadata, telemetry_measurements=telemetry_measurements + ) + + with video_metadata.filename.open("rb") as src_fp: + camm_fp = simple_mp4_builder.transform_mp4(src_fp, generator) + event_payload: uploader.Progress = { + "total_sequence_count": len(specified_video_metadatas), + "sequence_idx": idx, + "file_type": video_metadata.filetype.value, + "import_path": str(video_metadata.filename), + } + try: + cluster_id = mly_uploader.upload_stream( + T.cast(T.BinaryIO, camm_fp), + upload_api_v4.ClusterFileType.CAMM, + video_metadata.md5sum, + event_payload=event_payload, + ) + except Exception as ex: + raise UploadError(ex) from ex + LOG.debug("Uploaded to cluster: %s", cluster_id) + + # upload zip files + zip_paths = utils.find_zipfiles(import_paths, skip_subfolders=skip_subfolders) + _upload_zipfiles(mly_uploader, zip_paths) + + def upload( import_path: T.Union[Path, T.Sequence[Path]], + user_items: types.UserItem, desc_path: T.Optional[str] = None, _metadatas_from_process: T.Optional[T.Sequence[types.MetadataOrError]] = None, - user_name: T.Optional[str] = None, - organization_key: T.Optional[str] = None, dry_run=False, skip_subfolders=False, ) -> None: @@ -505,8 +555,6 @@ def upload( metadatas = _load_descs(_metadatas_from_process, desc_path, import_paths) - user_items = fetch_user_items(user_name, organization_key) - # Setup the emitter -- the order matters here emitter = uploader.EventEmitter() @@ -547,81 +595,7 @@ def upload( ) try: - image_paths = utils.find_images(import_paths, skip_subfolders=skip_subfolders) - # find descs that match the image paths from the import paths - image_metadatas = [ - metadata - for metadata in (metadatas or []) - if isinstance(metadata, types.ImageMetadata) - ] - specified_image_metadatas = _find_metadata_with_filename_existed_in( - image_metadatas, image_paths - ) - if specified_image_metadatas: - try: - clusters = mly_uploader.upload_images( - specified_image_metadatas, - event_payload={"file_type": FileType.IMAGE.value}, - ) - except Exception as ex: - raise UploadError(ex) from ex - - if clusters: - LOG.debug("Uploaded to cluster: %s", clusters) - - video_paths = utils.find_videos(import_paths, skip_subfolders=skip_subfolders) - video_metadatas = [ - metadata - for metadata in (metadatas or []) - if isinstance(metadata, types.VideoMetadata) - ] - specified_video_metadatas = _find_metadata_with_filename_existed_in( - video_metadatas, video_paths - ) - for idx, video_metadata in enumerate(specified_video_metadatas): - video_metadata.update_md5sum() - assert isinstance(video_metadata.md5sum, str), "md5sum should be updated" - - # extract telemetry measurements from GoPro videos - telemetry_measurements: T.List[camm_parser.TelemetryMeasurement] = [] - if MAPILLARY__EXPERIMENTAL_ENABLE_IMU == "YES": - if video_metadata.filetype is FileType.GOPRO: - with video_metadata.filename.open("rb") as fp: - gopro_info = gpmf_parser.extract_gopro_info( - fp, telemetry_only=True - ) - if gopro_info is not None: - telemetry_measurements.extend(gopro_info.accl or []) - telemetry_measurements.extend(gopro_info.gyro or []) - telemetry_measurements.extend(gopro_info.magn or []) - telemetry_measurements.sort(key=lambda m: m.time) - - generator = camm_builder.camm_sample_generator2( - video_metadata, telemetry_measurements=telemetry_measurements - ) - - with video_metadata.filename.open("rb") as src_fp: - camm_fp = simple_mp4_builder.transform_mp4(src_fp, generator) - event_payload: uploader.Progress = { - "total_sequence_count": len(specified_video_metadatas), - "sequence_idx": idx, - "file_type": video_metadata.filetype.value, - "import_path": str(video_metadata.filename), - } - try: - cluster_id = mly_uploader.upload_stream( - T.cast(T.BinaryIO, camm_fp), - upload_api_v4.ClusterFileType.CAMM, - video_metadata.md5sum, - event_payload=event_payload, - ) - except Exception as ex: - raise UploadError(ex) from ex - LOG.debug("Uploaded to cluster: %s", cluster_id) - - zip_paths = utils.find_zipfiles(import_paths, skip_subfolders=skip_subfolders) - _upload_zipfiles(mly_uploader, zip_paths) - + _upload_everything(mly_uploader, import_paths, metadatas, skip_subfolders) except UploadError as ex: inner_ex = ex.inner_ex From 27d30f517cd0c6942ce4c5745b5db1a8b5ce3493 Mon Sep 17 00:00:00 2001 From: Tao Peng Date: Thu, 6 Mar 2025 23:52:03 -0800 Subject: [PATCH 02/16] provide an interactive way to choose user profile --- mapillary_tools/authenticate.py | 36 +++++++++++++++++++++++++++------ mapillary_tools/config.py | 11 +++++----- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/mapillary_tools/authenticate.py b/mapillary_tools/authenticate.py index cf16ea8fa..315ab857f 100644 --- a/mapillary_tools/authenticate.py +++ b/mapillary_tools/authenticate.py @@ -103,6 +103,32 @@ def authenticate_user(user_name: str) -> types.UserItem: return user_items +def prompt_choose_user_profile( + all_user_items: T.Dict[str, types.UserItem], +) -> types.UserItem: + print("Found multiple Mapillary profiles:") + profiles = list(all_user_items.keys()) + + for i, name in enumerate(profiles, 1): + print(f"{i:5}. {name}") + + while True: + try: + choice = int( + input("Which user profile would you like to use? Enter the number: ") + ) + except ValueError: + print("Invalid input. Please enter a number.") + else: + if 1 <= choice <= len(all_user_items): + user_items = all_user_items[profiles[choice - 1]] + break + + print(f"Please enter a number between 1 and {len(profiles)}.") + + return user_items + + def fetch_user_items( user_name: T.Optional[str] = None, organization_key: T.Optional[str] = None ) -> types.UserItem: @@ -110,14 +136,12 @@ def fetch_user_items( all_user_items = config.list_all_users() if not all_user_items: raise exceptions.MapillaryBadParameterError( - "No Mapillary account found. Add one with --user_name" + "No Mapillary profile found. Add one with --user_name" ) - if len(all_user_items) == 1: - user_items = all_user_items[0] + elif len(all_user_items) == 1: + user_items = list(all_user_items.values())[0] else: - raise exceptions.MapillaryBadParameterError( - "Found multiple Mapillary accounts. Please specify one with --user_name" - ) + user_items = prompt_choose_user_profile(all_user_items) else: user_items = authenticate_user(user_name) diff --git a/mapillary_tools/config.py b/mapillary_tools/config.py index a70b8b17a..521ddff94 100644 --- a/mapillary_tools/config.py +++ b/mapillary_tools/config.py @@ -46,14 +46,15 @@ def load_user( return T.cast(types.UserItem, user_items) -def list_all_users(config_path: T.Optional[str] = None) -> T.List[types.UserItem]: +def list_all_users(config_path: T.Optional[str] = None) -> T.Dict[str, types.UserItem]: if config_path is None: config_path = MAPILLARY_CONFIG_PATH cp = _load_config(config_path) - users = [ - load_user(user_name, config_path=config_path) for user_name in cp.sections() - ] - return [item for item in users if item is not None] + users = { + user_name: load_user(user_name, config_path=config_path) + for user_name in cp.sections() + } + return {profile: item for profile, item in users.items() if item is not None} def update_config( From 4b830aa621492b513c3367a393e338dce261d770 Mon Sep 17 00:00:00 2001 From: Tao Peng Date: Fri, 7 Mar 2025 00:32:46 -0800 Subject: [PATCH 03/16] add a nice banner --- mapillary_tools/authenticate.py | 86 +++++++++++++++++++++------------ mapillary_tools/config.py | 18 +++---- 2 files changed, 63 insertions(+), 41 deletions(-) diff --git a/mapillary_tools/authenticate.py b/mapillary_tools/authenticate.py index 315ab857f..b5f84ebc0 100644 --- a/mapillary_tools/authenticate.py +++ b/mapillary_tools/authenticate.py @@ -6,7 +6,7 @@ import jsonschema import requests -from . import api_v4, config, exceptions, types +from . import api_v4, config, types LOG = logging.getLogger(__name__) @@ -18,14 +18,17 @@ def authenticate( user_password: T.Optional[str] = None, jwt: T.Optional[str] = None, ): - if user_name: - user_name = user_name.strip() + # we still accept --user_name for the back compatibility + profile_name = user_name - while not user_name: - user_name = input( + if profile_name: + profile_name = profile_name.strip() + + while not profile_name: + profile_name = input( "Enter the Mapillary username you would like to (re)authenticate: " ) - user_name = user_name.strip() + profile_name = profile_name.strip() if jwt: user_items: types.UserItem = { @@ -39,13 +42,30 @@ def authenticate( "user_upload_token": data["access_token"], } else: - user_items = prompt_user_for_user_items(user_name) - - config.update_config(user_name, user_items) - - -def prompt_user_for_user_items(user_name: str) -> types.UserItem: - print(f"Sign in for user {user_name}") + profile_name, user_items = prompt_user_for_user_items(profile_name) + + config.update_config(profile_name, user_items) + + +def prompt_user_for_user_items( + profile_name: str | None, +) -> T.Tuple[str, types.UserItem]: + print( + """ +================================================================================ + Welcome to Mapillary! +================================================================================ +If you haven't registered yet, please visit the following link to sign up first: +https://www.mapillary.com/signup +After the registration, proceed here to sign in. +================================================================================ +""".strip(), + ) + + if profile_name is None: + profile_name = input("Enter the profile name you would like to create: ") + + print(f"Sign in for user {profile_name}") user_email = input("Enter your Mapillary user email: ") user_password = getpass.getpass("Enter Mapillary user password: ") @@ -63,7 +83,7 @@ def prompt_user_for_user_items(user_name: str) -> types.UserItem: title = r.get("error", {}).get("error_user_title") message = r.get("error", {}).get("error_user_msg") LOG.error("%s: %s", title, message) - return prompt_user_for_user_items(user_name) + return prompt_user_for_user_items(profile_name) else: raise ex else: @@ -80,25 +100,26 @@ def prompt_user_for_user_items(user_name: str) -> types.UserItem: if isinstance(user_key, int): user_key = str(user_key) - return { + return profile_name, { "MAPSettingsUserKey": user_key, "user_upload_token": upload_token, } -def authenticate_user(user_name: str) -> types.UserItem: - user_items = config.load_user(user_name) - if user_items is not None: - try: - jsonschema.validate(user_items, types.UserItemSchema) - except jsonschema.ValidationError: - pass - else: - return user_items +def authenticate_user(profile_name: str | None) -> types.UserItem: + if profile_name is not None: + user_items = config.load_user(profile_name) + if user_items is not None: + try: + jsonschema.validate(user_items, types.UserItemSchema) + except jsonschema.ValidationError: + pass + else: + return user_items - user_items = prompt_user_for_user_items(user_name) + profile_name, user_items = prompt_user_for_user_items(profile_name) jsonschema.validate(user_items, types.UserItemSchema) - config.update_config(user_name, user_items) + config.update_config(profile_name, user_items) return user_items @@ -130,20 +151,21 @@ def prompt_choose_user_profile( def fetch_user_items( - user_name: T.Optional[str] = None, organization_key: T.Optional[str] = None + profile_name: T.Optional[str] = None, organization_key: T.Optional[str] = None ) -> types.UserItem: - if user_name is None: + if profile_name is None: all_user_items = config.list_all_users() if not all_user_items: - raise exceptions.MapillaryBadParameterError( - "No Mapillary profile found. Add one with --user_name" - ) + user_items = authenticate_user(None) + # raise exceptions.MapillaryBadParameterError( + # "No Mapillary profile found. Add one with --user_name" + # ) elif len(all_user_items) == 1: user_items = list(all_user_items.values())[0] else: user_items = prompt_choose_user_profile(all_user_items) else: - user_items = authenticate_user(user_name) + user_items = authenticate_user(profile_name) if organization_key is not None: resp = api_v4.fetch_organization( diff --git a/mapillary_tools/config.py b/mapillary_tools/config.py index 521ddff94..9daef33ec 100644 --- a/mapillary_tools/config.py +++ b/mapillary_tools/config.py @@ -35,14 +35,14 @@ def _load_config(config_path: str) -> configparser.ConfigParser: def load_user( - user_name: str, config_path: T.Optional[str] = None + profile_name: str, config_path: T.Optional[str] = None ) -> T.Optional[types.UserItem]: if config_path is None: config_path = MAPILLARY_CONFIG_PATH config = _load_config(config_path) - if not config.has_section(user_name): + if not config.has_section(profile_name): return None - user_items = dict(config.items(user_name)) + user_items = dict(config.items(profile_name)) return T.cast(types.UserItem, user_items) @@ -51,22 +51,22 @@ def list_all_users(config_path: T.Optional[str] = None) -> T.Dict[str, types.Use config_path = MAPILLARY_CONFIG_PATH cp = _load_config(config_path) users = { - user_name: load_user(user_name, config_path=config_path) - for user_name in cp.sections() + profile_name: load_user(profile_name, config_path=config_path) + for profile_name in cp.sections() } return {profile: item for profile, item in users.items() if item is not None} def update_config( - user_name: str, user_items: types.UserItem, config_path: T.Optional[str] = None + profile_name: str, user_items: types.UserItem, config_path: T.Optional[str] = None ) -> None: if config_path is None: config_path = MAPILLARY_CONFIG_PATH config = _load_config(config_path) - if not config.has_section(user_name): - config.add_section(user_name) + if not config.has_section(profile_name): + config.add_section(profile_name) for key, val in user_items.items(): - config.set(user_name, key, T.cast(str, val)) + config.set(profile_name, key, T.cast(str, val)) os.makedirs(os.path.dirname(os.path.abspath(config_path)), exist_ok=True) with open(config_path, "w") as fp: config.write(fp) From 48eaaae28a447ebf3ef4f37c7f410c6638b8126d Mon Sep 17 00:00:00 2001 From: Tao Peng Date: Fri, 7 Mar 2025 00:37:32 -0800 Subject: [PATCH 04/16] make sure all prints going to stderr --- mapillary_tools/authenticate.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/mapillary_tools/authenticate.py b/mapillary_tools/authenticate.py index b5f84ebc0..c24406e18 100644 --- a/mapillary_tools/authenticate.py +++ b/mapillary_tools/authenticate.py @@ -1,6 +1,7 @@ import getpass import json import logging +import sys import typing as T import jsonschema @@ -60,12 +61,13 @@ def prompt_user_for_user_items( After the registration, proceed here to sign in. ================================================================================ """.strip(), + file=sys.stderr, ) if profile_name is None: profile_name = input("Enter the profile name you would like to create: ") - print(f"Sign in for user {profile_name}") + print(f"Sign in for user {profile_name}", file=sys.stderr) user_email = input("Enter your Mapillary user email: ") user_password = getpass.getpass("Enter Mapillary user password: ") @@ -127,11 +129,11 @@ def authenticate_user(profile_name: str | None) -> types.UserItem: def prompt_choose_user_profile( all_user_items: T.Dict[str, types.UserItem], ) -> types.UserItem: - print("Found multiple Mapillary profiles:") + print("Found multiple Mapillary profiles:", file=sys.stderr) profiles = list(all_user_items.keys()) for i, name in enumerate(profiles, 1): - print(f"{i:5}. {name}") + print(f"{i:5}. {name}", file=sys.stderr) while True: try: @@ -139,13 +141,15 @@ def prompt_choose_user_profile( input("Which user profile would you like to use? Enter the number: ") ) except ValueError: - print("Invalid input. Please enter a number.") + print("Invalid input. Please enter a number.", file=sys.stderr) else: if 1 <= choice <= len(all_user_items): user_items = all_user_items[profiles[choice - 1]] break - print(f"Please enter a number between 1 and {len(profiles)}.") + print( + f"Please enter a number between 1 and {len(profiles)}.", file=sys.stderr + ) return user_items From 6dc5de8143d80574b822de943ba54660b0fd6476 Mon Sep 17 00:00:00 2001 From: Tao Peng Date: Fri, 7 Mar 2025 02:01:22 -0800 Subject: [PATCH 05/16] print existing profiles --- mapillary_tools/authenticate.py | 117 ++++++++++++++++++++++---------- 1 file changed, 81 insertions(+), 36 deletions(-) diff --git a/mapillary_tools/authenticate.py b/mapillary_tools/authenticate.py index c24406e18..b9a896bc2 100644 --- a/mapillary_tools/authenticate.py +++ b/mapillary_tools/authenticate.py @@ -19,17 +19,29 @@ def authenticate( user_password: T.Optional[str] = None, jwt: T.Optional[str] = None, ): - # we still accept --user_name for the back compatibility + # we still have to accept --user_name for the back compatibility profile_name = user_name + all_user_items = config.list_all_users() + if all_user_items: + echo("Existing Mapillary profiles:") + _list_all_profiles(all_user_items) + else: + welcome() + if profile_name: profile_name = profile_name.strip() while not profile_name: profile_name = input( - "Enter the Mapillary username you would like to (re)authenticate: " + "Enter the Mapillary profile you would like to (re)authenticate: " + ).strip() + + if profile_name in all_user_items: + LOG.warning( + 'The profile "%s" already exists and will be overridden.', + profile_name, ) - profile_name = profile_name.strip() if jwt: user_items: types.UserItem = { @@ -43,33 +55,43 @@ def authenticate( "user_upload_token": data["access_token"], } else: + if user_email or user_password: + LOG.warning( + "Both user_email and user_password must be provided to authenticate" + ) profile_name, user_items = prompt_user_for_user_items(profile_name) + LOG.info('Authenticated as "%s"', profile_name) config.update_config(profile_name, user_items) +def echo(*args, **kwargs): + print(*args, **kwargs, file=sys.stderr) + + +def _list_all_profiles(profiles: T.Dict[str, types.UserItem]) -> None: + for idx, name in enumerate(profiles, 1): + echo(f"{idx:>5}. {name:<32} {profiles[name].get('MAPSettingsUserKey')}") + + def prompt_user_for_user_items( - profile_name: str | None, + profile_name: T.Optional[str], ) -> T.Tuple[str, types.UserItem]: - print( - """ -================================================================================ - Welcome to Mapillary! -================================================================================ -If you haven't registered yet, please visit the following link to sign up first: -https://www.mapillary.com/signup -After the registration, proceed here to sign in. -================================================================================ -""".strip(), - file=sys.stderr, - ) - if profile_name is None: - profile_name = input("Enter the profile name you would like to create: ") - - print(f"Sign in for user {profile_name}", file=sys.stderr) - user_email = input("Enter your Mapillary user email: ") - user_password = getpass.getpass("Enter Mapillary user password: ") + while not profile_name: + profile_name = input( + "Enter the profile name you would like to authenticate: " + ).strip() + + user_email = "" + while not user_email: + user_email = input( + f'Enter your Mapillary user email for "{profile_name}": ' + ).strip() + + user_password = getpass.getpass( + f'Enter Mapillary user password for "{profile_name}": ' + ) try: resp = api_v4.get_upload_token(user_email, user_password) @@ -108,19 +130,25 @@ def prompt_user_for_user_items( } -def authenticate_user(profile_name: str | None) -> types.UserItem: +def authenticate_user(profile_name: T.Optional[str]) -> types.UserItem: if profile_name is not None: user_items = config.load_user(profile_name) - if user_items is not None: + if user_items is None: + LOG.info('Profile "%s" not found in config', profile_name) + else: try: jsonschema.validate(user_items, types.UserItemSchema) except jsonschema.ValidationError: - pass + # If the user_items in config are invalid, proceed with the user input + LOG.warning("Invalid user items for profile: %s", profile_name) else: return user_items profile_name, user_items = prompt_user_for_user_items(profile_name) jsonschema.validate(user_items, types.UserItemSchema) + + # Update the config with the new user items + LOG.info('Authenticated as "%s"', profile_name) config.update_config(profile_name, user_items) return user_items @@ -129,37 +157,54 @@ def authenticate_user(profile_name: str | None) -> types.UserItem: def prompt_choose_user_profile( all_user_items: T.Dict[str, types.UserItem], ) -> types.UserItem: - print("Found multiple Mapillary profiles:", file=sys.stderr) + echo("Found multiple Mapillary profiles:") profiles = list(all_user_items.keys()) - for i, name in enumerate(profiles, 1): - print(f"{i:5}. {name}", file=sys.stderr) + _list_all_profiles(all_user_items) while True: + c = input( + "Which user profile would you like to use? Enter the number: " + ).strip() + try: - choice = int( - input("Which user profile would you like to use? Enter the number: ") - ) + choice = int(c) except ValueError: - print("Invalid input. Please enter a number.", file=sys.stderr) + echo("Invalid input. Please enter a number.") else: if 1 <= choice <= len(all_user_items): user_items = all_user_items[profiles[choice - 1]] break - print( - f"Please enter a number between 1 and {len(profiles)}.", file=sys.stderr - ) + echo(f"Please enter a number between 1 and {len(profiles)}.") return user_items +def welcome(): + echo( + """ +================================================================================ + Welcome to Mapillary! +================================================================================ +If you haven't registered yet, please visit the following link to sign up first: +https://www.mapillary.com/signup +After the registration, proceed here to sign in. +================================================================================ + """.strip() + ) + + def fetch_user_items( - profile_name: T.Optional[str] = None, organization_key: T.Optional[str] = None + user_name: T.Optional[str] = None, organization_key: T.Optional[str] = None ) -> types.UserItem: + # we still have to accept --user_name for the back compatibility + profile_name = user_name + if profile_name is None: all_user_items = config.list_all_users() if not all_user_items: + welcome() user_items = authenticate_user(None) # raise exceptions.MapillaryBadParameterError( # "No Mapillary profile found. Add one with --user_name" From 433c5550de3aae438bf0741f605cefbb3c04a69b Mon Sep 17 00:00:00 2001 From: Tao Peng Date: Fri, 7 Mar 2025 02:24:11 -0800 Subject: [PATCH 06/16] prompt to stderr --- mapillary_tools/authenticate.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/mapillary_tools/authenticate.py b/mapillary_tools/authenticate.py index b9a896bc2..7a695889c 100644 --- a/mapillary_tools/authenticate.py +++ b/mapillary_tools/authenticate.py @@ -13,6 +13,16 @@ LOG = logging.getLogger(__name__) +def echo(*args, **kwargs): + print(*args, **kwargs, file=sys.stderr) + + +def prompt(message: str) -> str: + """Display prompt on stderr and get input from stdin""" + print(message, end="", file=sys.stderr, flush=True) + return input() + + def authenticate( user_name: T.Optional[str] = None, user_email: T.Optional[str] = None, @@ -33,7 +43,7 @@ def authenticate( profile_name = profile_name.strip() while not profile_name: - profile_name = input( + profile_name = prompt( "Enter the Mapillary profile you would like to (re)authenticate: " ).strip() @@ -65,10 +75,6 @@ def authenticate( config.update_config(profile_name, user_items) -def echo(*args, **kwargs): - print(*args, **kwargs, file=sys.stderr) - - def _list_all_profiles(profiles: T.Dict[str, types.UserItem]) -> None: for idx, name in enumerate(profiles, 1): echo(f"{idx:>5}. {name:<32} {profiles[name].get('MAPSettingsUserKey')}") @@ -79,14 +85,14 @@ def prompt_user_for_user_items( ) -> T.Tuple[str, types.UserItem]: if profile_name is None: while not profile_name: - profile_name = input( + profile_name = prompt( "Enter the profile name you would like to authenticate: " ).strip() user_email = "" while not user_email: - user_email = input( - f'Enter your Mapillary user email for "{profile_name}": ' + user_email = prompt( + f'Enter Mapillary user email for "{profile_name}": ' ).strip() user_password = getpass.getpass( @@ -163,7 +169,7 @@ def prompt_choose_user_profile( _list_all_profiles(all_user_items) while True: - c = input( + c = prompt( "Which user profile would you like to use? Enter the number: " ).strip() From b0d2d85817af6391575083a3ac0e8aab865cb166 Mon Sep 17 00:00:00 2001 From: Tao Peng Date: Fri, 7 Mar 2025 02:40:01 -0800 Subject: [PATCH 07/16] validate profile name --- mapillary_tools/authenticate.py | 96 ++++++++++++++++++--------------- 1 file changed, 54 insertions(+), 42 deletions(-) diff --git a/mapillary_tools/authenticate.py b/mapillary_tools/authenticate.py index 7a695889c..c94224323 100644 --- a/mapillary_tools/authenticate.py +++ b/mapillary_tools/authenticate.py @@ -1,6 +1,7 @@ import getpass import json import logging +import re import sys import typing as T @@ -23,6 +24,37 @@ def prompt(message: str) -> str: return input() +def prompt_profile_name() -> str: + profile_name = "" + + while not profile_name: + profile_name = prompt( + "Enter the Mapillary profile you would like to (re)authenticate: " + ).strip() + + if profile_name: + try: + validate_profile_name(profile_name) + except ValueError as ex: + echo(ex) + profile_name = "" + else: + break + + return profile_name + + +def validate_profile_name(profile_name: str): + if not (2 <= len(profile_name) <= 32): + raise ValueError("Profile name must be between 2 and 32 characters long") + + pattern = re.compile(r"^[a-zA-Z0-9_-]+$") + if not bool(pattern.match(profile_name)): + raise ValueError( + "Invalid profile name. Use only letters, numbers, hyphens and underscores" + ) + + def authenticate( user_name: T.Optional[str] = None, user_email: T.Optional[str] = None, @@ -32,6 +64,10 @@ def authenticate( # we still have to accept --user_name for the back compatibility profile_name = user_name + if profile_name: + profile_name = profile_name.strip() + validate_profile_name(profile_name) + all_user_items = config.list_all_users() if all_user_items: echo("Existing Mapillary profiles:") @@ -39,13 +75,8 @@ def authenticate( else: welcome() - if profile_name: - profile_name = profile_name.strip() - - while not profile_name: - profile_name = prompt( - "Enter the Mapillary profile you would like to (re)authenticate: " - ).strip() + if not profile_name: + profile_name = prompt_profile_name() if profile_name in all_user_items: LOG.warning( @@ -84,20 +115,18 @@ def prompt_user_for_user_items( profile_name: T.Optional[str], ) -> T.Tuple[str, types.UserItem]: if profile_name is None: - while not profile_name: - profile_name = prompt( - "Enter the profile name you would like to authenticate: " - ).strip() + profile_name = prompt_profile_name() + + echo(f'Authenticating as "{profile_name}"') user_email = "" while not user_email: - user_email = prompt( - f'Enter Mapillary user email for "{profile_name}": ' - ).strip() + user_email = prompt("Enter Mapillary user email: ").strip() - user_password = getpass.getpass( - f'Enter Mapillary user password for "{profile_name}": ' - ) + while True: + user_password = getpass.getpass("Enter Mapillary user password: ") + if user_password: + break try: resp = api_v4.get_upload_token(user_email, user_password) @@ -164,26 +193,10 @@ def prompt_choose_user_profile( all_user_items: T.Dict[str, types.UserItem], ) -> types.UserItem: echo("Found multiple Mapillary profiles:") - profiles = list(all_user_items.keys()) - _list_all_profiles(all_user_items) - - while True: - c = prompt( - "Which user profile would you like to use? Enter the number: " - ).strip() - - try: - choice = int(c) - except ValueError: - echo("Invalid input. Please enter a number.") - else: - if 1 <= choice <= len(all_user_items): - user_items = all_user_items[profiles[choice - 1]] - break - - echo(f"Please enter a number between 1 and {len(profiles)}.") - + profile_name = prompt_profile_name() + # TODO: fix KeyError here + user_items = all_user_items[profile_name] return user_items @@ -207,14 +220,13 @@ def fetch_user_items( # we still have to accept --user_name for the back compatibility profile_name = user_name + all_user_items = config.list_all_users() + if not all_user_items: + welcome() + if profile_name is None: - all_user_items = config.list_all_users() - if not all_user_items: - welcome() + if len(all_user_items) == 0: user_items = authenticate_user(None) - # raise exceptions.MapillaryBadParameterError( - # "No Mapillary profile found. Add one with --user_name" - # ) elif len(all_user_items) == 1: user_items = list(all_user_items.values())[0] else: From fa4ff714bab915c66b458bc2b443eae4714ebf3f Mon Sep 17 00:00:00 2001 From: Tao Peng Date: Fri, 7 Mar 2025 12:59:43 -0800 Subject: [PATCH 08/16] disable prompt when the ENV is set --- mapillary_tools/authenticate.py | 257 ++++++++++++++++++++------------ mapillary_tools/constants.py | 6 + 2 files changed, 164 insertions(+), 99 deletions(-) diff --git a/mapillary_tools/authenticate.py b/mapillary_tools/authenticate.py index c94224323..60adce2e5 100644 --- a/mapillary_tools/authenticate.py +++ b/mapillary_tools/authenticate.py @@ -8,53 +8,12 @@ import jsonschema import requests -from . import api_v4, config, types +from . import api_v4, config, constants, exceptions, types LOG = logging.getLogger(__name__) -def echo(*args, **kwargs): - print(*args, **kwargs, file=sys.stderr) - - -def prompt(message: str) -> str: - """Display prompt on stderr and get input from stdin""" - print(message, end="", file=sys.stderr, flush=True) - return input() - - -def prompt_profile_name() -> str: - profile_name = "" - - while not profile_name: - profile_name = prompt( - "Enter the Mapillary profile you would like to (re)authenticate: " - ).strip() - - if profile_name: - try: - validate_profile_name(profile_name) - except ValueError as ex: - echo(ex) - profile_name = "" - else: - break - - return profile_name - - -def validate_profile_name(profile_name: str): - if not (2 <= len(profile_name) <= 32): - raise ValueError("Profile name must be between 2 and 32 characters long") - - pattern = re.compile(r"^[a-zA-Z0-9_-]+$") - if not bool(pattern.match(profile_name)): - raise ValueError( - "Invalid profile name. Use only letters, numbers, hyphens and underscores" - ) - - def authenticate( user_name: T.Optional[str] = None, user_email: T.Optional[str] = None, @@ -66,28 +25,27 @@ def authenticate( if profile_name: profile_name = profile_name.strip() - validate_profile_name(profile_name) all_user_items = config.list_all_users() if all_user_items: - echo("Existing Mapillary profiles:") _list_all_profiles(all_user_items) else: - welcome() + _welcome() if not profile_name: - profile_name = prompt_profile_name() + profile_name = _prompt_profile_name(skip_validation=True) if profile_name in all_user_items: LOG.warning( - 'The profile "%s" already exists and will be overridden.', + 'The profile "%s" already exists and will be overridden', profile_name, ) + else: + # validate only new profile names + _validate_profile_name(profile_name) if jwt: - user_items: types.UserItem = { - "user_upload_token": jwt, - } + user_items: types.UserItem = {"user_upload_token": jwt} elif user_email and user_password: resp = api_v4.get_upload_token(user_email, user_password) data = resp.json() @@ -100,28 +58,150 @@ def authenticate( LOG.warning( "Both user_email and user_password must be provided to authenticate" ) - profile_name, user_items = prompt_user_for_user_items(profile_name) + + if not _prompt_enabled(): + raise exceptions.MapillaryBadParameterError( + "Authentication required, but prompting is disabled" + ) + + profile_name, user_items = _prompt_user_for_user_items(profile_name) LOG.info('Authenticated as "%s"', profile_name) config.update_config(profile_name, user_items) +def fetch_user_items( + user_name: T.Optional[str] = None, organization_key: T.Optional[str] = None +) -> types.UserItem: + # we still have to accept --user_name for the back compatibility + profile_name = user_name + + all_user_items = config.list_all_users() + if not all_user_items: + _welcome() + + if profile_name is None: + if len(all_user_items) == 0: + user_items = _load_or_authenticate_user() + elif len(all_user_items) == 1: + user_items = list(all_user_items.values())[0] + else: + if not _prompt_enabled(): + raise exceptions.MapillaryBadParameterError( + "Multiple user profiles found, please choose one with --user_name" + ) + user_items = _prompt_choose_user_profile(all_user_items) + else: + user_items = _load_or_authenticate_user(profile_name) + + if organization_key is not None: + resp = api_v4.fetch_organization( + user_items["user_upload_token"], organization_key + ) + org = resp.json() + LOG.info("Uploading to organization: %s", json.dumps(org)) + user_items = T.cast( + types.UserItem, {**user_items, "MAPOrganizationKey": organization_key} + ) + return user_items + + +def _echo(*args, **kwargs): + print(*args, **kwargs, file=sys.stderr) + + +def _prompt(message: str) -> str: + """Display prompt on stderr and get input from stdin""" + print(message, end="", file=sys.stderr, flush=True) + return input() + + +def _prompt_profile_name(skip_validation: bool = False) -> str: + assert _prompt_enabled(), "should not get here if prompting is disabled" + + profile_name = "" + + while not profile_name: + profile_name = _prompt( + "Enter the Mapillary profile you would like to (re)authenticate: " + ).strip() + + if profile_name: + if skip_validation: + break + + try: + _validate_profile_name(profile_name) + except ValueError as ex: + LOG.error("Error validating profile name: %s", ex) + profile_name = "" + else: + break + + return profile_name + + +def _validate_profile_name(profile_name: str): + if not (2 <= len(profile_name) <= 32): + raise exceptions.MapillaryBadParameterError( + "Profile name must be between 2 and 32 characters long" + ) + + pattern = re.compile(r"^[a-zA-Z0-9_-]+$") + if not bool(pattern.match(profile_name)): + raise exceptions.MapillaryBadParameterError( + "Invalid profile name. Use only letters, numbers, hyphens and underscores" + ) + + def _list_all_profiles(profiles: T.Dict[str, types.UserItem]) -> None: + _echo("Existing Mapillary profiles:") for idx, name in enumerate(profiles, 1): - echo(f"{idx:>5}. {name:<32} {profiles[name].get('MAPSettingsUserKey')}") + _echo(f"{idx:>5}. {name:<32} {profiles[name].get('MAPSettingsUserKey')}") + + +def _is_interactive(): + """ + Determine if the current environment is interactive by checking + if standard streams are connected to a TTY device. + + Returns: + bool: True if running in an interactive terminal, False otherwise + """ + # Check if stdout is connected to a terminal + stdout_interactive = sys.stdout.isatty() if hasattr(sys.stdout, "isatty") else False + + # Optionally, also check stdin and stderr + stdin_interactive = sys.stdin.isatty() if hasattr(sys.stdin, "isatty") else False + stderr_interactive = sys.stderr.isatty() if hasattr(sys.stderr, "isatty") else False + + # Return True if any stream is interactive + return stdout_interactive or stdin_interactive or stderr_interactive + + +def _prompt_enabled() -> bool: + if constants.PROMPT_DISABLED: + return False + + if not _is_interactive(): + return False + + return True -def prompt_user_for_user_items( +def _prompt_user_for_user_items( profile_name: T.Optional[str], ) -> T.Tuple[str, types.UserItem]: + assert _prompt_enabled(), "should not get here if prompting is disabled" + if profile_name is None: - profile_name = prompt_profile_name() + profile_name = _prompt_profile_name() - echo(f'Authenticating as "{profile_name}"') + LOG.info('Authenticating as "%s"', profile_name) user_email = "" while not user_email: - user_email = prompt("Enter Mapillary user email: ").strip() + user_email = _prompt("Enter Mapillary user email: ").strip() while True: user_password = getpass.getpass("Enter Mapillary user password: ") @@ -142,7 +222,7 @@ def prompt_user_for_user_items( title = r.get("error", {}).get("error_user_title") message = r.get("error", {}).get("error_user_msg") LOG.error("%s: %s", title, message) - return prompt_user_for_user_items(profile_name) + return _prompt_user_for_user_items(profile_name) else: raise ex else: @@ -165,11 +245,13 @@ def prompt_user_for_user_items( } -def authenticate_user(profile_name: T.Optional[str]) -> types.UserItem: +def _load_or_authenticate_user(profile_name: T.Optional[str] = None) -> types.UserItem: if profile_name is not None: user_items = config.load_user(profile_name) if user_items is None: LOG.info('Profile "%s" not found in config', profile_name) + # validate here since we are going to create this profile + _validate_profile_name(profile_name) else: try: jsonschema.validate(user_items, types.UserItemSchema) @@ -179,7 +261,12 @@ def authenticate_user(profile_name: T.Optional[str]) -> types.UserItem: else: return user_items - profile_name, user_items = prompt_user_for_user_items(profile_name) + if not _prompt_enabled(): + raise exceptions.MapillaryBadParameterError( + f'Profile "{profile_name}" not found (and prompting disabled)' + ) + + profile_name, user_items = _prompt_user_for_user_items(profile_name) jsonschema.validate(user_items, types.UserItemSchema) # Update the config with the new user items @@ -189,19 +276,23 @@ def authenticate_user(profile_name: T.Optional[str]) -> types.UserItem: return user_items -def prompt_choose_user_profile( +def _prompt_choose_user_profile( all_user_items: T.Dict[str, types.UserItem], ) -> types.UserItem: - echo("Found multiple Mapillary profiles:") + assert _prompt_enabled(), "should not get here if prompting is disabled" + _list_all_profiles(all_user_items) - profile_name = prompt_profile_name() - # TODO: fix KeyError here - user_items = all_user_items[profile_name] - return user_items + while True: + profile_name = _prompt_profile_name() + if profile_name in all_user_items: + break + _echo(f'Profile "{profile_name}" not found') + return all_user_items[profile_name] -def welcome(): - echo( + +def _welcome(): + _echo( """ ================================================================================ Welcome to Mapillary! @@ -212,35 +303,3 @@ def welcome(): ================================================================================ """.strip() ) - - -def fetch_user_items( - user_name: T.Optional[str] = None, organization_key: T.Optional[str] = None -) -> types.UserItem: - # we still have to accept --user_name for the back compatibility - profile_name = user_name - - all_user_items = config.list_all_users() - if not all_user_items: - welcome() - - if profile_name is None: - if len(all_user_items) == 0: - user_items = authenticate_user(None) - elif len(all_user_items) == 1: - user_items = list(all_user_items.values())[0] - else: - user_items = prompt_choose_user_profile(all_user_items) - else: - user_items = authenticate_user(profile_name) - - if organization_key is not None: - resp = api_v4.fetch_organization( - user_items["user_upload_token"], organization_key - ) - org = resp.json() - LOG.info("Uploading to organization: %s", json.dumps(org)) - user_items = T.cast( - types.UserItem, {**user_items, "MAPOrganizationKey": organization_key} - ) - return user_items diff --git a/mapillary_tools/constants.py b/mapillary_tools/constants.py index 60c13023b..d8676e867 100644 --- a/mapillary_tools/constants.py +++ b/mapillary_tools/constants.py @@ -55,3 +55,9 @@ MAX_SEQUENCE_FILESIZE: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_FILESIZE", "110G") # Max number of pixels per sequence (sum of image pixels in the sequence) MAX_SEQUENCE_PIXELS: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_PIXELS", "6G") + +PROMPT_DISABLED: bool = os.getenv(_ENV_PREFIX + "PROMPT_DISABLED", "NO").upper() in [ + "1", + "TRUE", + "YES", +] From 08c9bf279530505bddc2060bb392e65129e450d4 Mon Sep 17 00:00:00 2001 From: Tao Peng Date: Wed, 26 Mar 2025 14:49:48 -0700 Subject: [PATCH 09/16] fix the test cli --- mapillary_tools/api_v4.py | 62 +++++++++++++++++++++++++++++++++ mapillary_tools/authenticate.py | 54 ++++++++++++++++++++++++---- mapillary_tools/upload.py | 14 +++----- tests/cli/upload_api_v4.py | 37 +++++++++++--------- 4 files changed, 134 insertions(+), 33 deletions(-) diff --git a/mapillary_tools/api_v4.py b/mapillary_tools/api_v4.py index fb6606243..a5f06649d 100644 --- a/mapillary_tools/api_v4.py +++ b/mapillary_tools/api_v4.py @@ -224,6 +224,44 @@ def request_get( return resp +def is_auth_error(resp: requests.Response) -> bool: + if resp.status_code in [401, 403]: + return True + + if resp.status_code in [400]: + try: + error_body = resp.json() + except Exception: + error_body = {} + + type = error_body.get("debug_info", {}).get("type") + if type in ["NotAuthorizedError"]: + return True + + return False + + +def extract_auth_error_message(resp: requests.Response) -> str: + assert is_auth_error(resp), "has to be an auth error" + + try: + error_body = resp.json() + except Exception: + error_body = {} + + # from Graph APIs + message = error_body.get("error", {}).get("message") + if message is not None: + return str(message) + + # from upload service + message = error_body.get("debug_info", {}).get("message") + if message is not None: + return str(message) + + return resp.text + + def get_upload_token(email: str, password: str) -> requests.Response: resp = request_post( f"{MAPILLARY_GRAPH_API_ENDPOINT}/login", @@ -252,6 +290,30 @@ def fetch_organization( return resp +def fetch_user_or_me( + user_access_token: str, + user_id: T.Optional[T.Union[int, str]] = None, +) -> requests.Response: + if user_id is None: + url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/me" + else: + url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/{user_id}" + + resp = request_get( + url, + params={ + "fields": ",".join(["id", "username"]), + }, + headers={ + "Authorization": f"OAuth {user_access_token}", + }, + timeout=REQUESTS_TIMEOUT, + ) + + resp.raise_for_status() + return resp + + ActionType = T.Literal[ "upload_started_upload", "upload_finished_upload", "upload_failed_upload" ] diff --git a/mapillary_tools/authenticate.py b/mapillary_tools/authenticate.py index 60adce2e5..fa2879c00 100644 --- a/mapillary_tools/authenticate.py +++ b/mapillary_tools/authenticate.py @@ -23,18 +23,21 @@ def authenticate( # we still have to accept --user_name for the back compatibility profile_name = user_name - if profile_name: - profile_name = profile_name.strip() - all_user_items = config.list_all_users() if all_user_items: _list_all_profiles(all_user_items) else: _welcome() - if not profile_name: + # Make sure profile name either validated or existed + if profile_name: + profile_name = profile_name.strip() + else: + if not _prompt_enabled(): + raise exceptions.MapillaryBadParameterError( + "Profile name is required, please specify one with --user_name" + ) profile_name = _prompt_profile_name(skip_validation=True) - if profile_name in all_user_items: LOG.warning( 'The profile "%s" already exists and will be overridden', @@ -66,10 +69,41 @@ def authenticate( profile_name, user_items = _prompt_user_for_user_items(profile_name) + _test_auth_and_update_user(user_items) + LOG.info('Authenticated as "%s"', profile_name) config.update_config(profile_name, user_items) +def _test_auth_and_update_user( + user_items: types.UserItem, +) -> T.Optional[T.Dict[str, str]]: + try: + resp = api_v4.fetch_user_or_me( + user_access_token=user_items["user_upload_token"] + ) + except requests.HTTPError as ex: + if api_v4.is_auth_error(ex.response): + message = api_v4.extract_auth_error_message(ex.response) + raise exceptions.MapillaryUploadUnauthorizedError(message) + else: + # The point of this function is to test if the auth works, so we don't throw any non-auth errors + LOG.warning("Error testing the auth: %s", api_v4.readable_http_error(ex)) + return None + + user_json = resp.json() + if user_json is not None: + username = user_json.get("username") + if username is not None: + user_items["MAPSettingsUsername"] = username + + user_id = user_json.get("id") + if user_id is not None: + user_items["MAPSettingsUserKey"] = user_id + + return user_json + + def fetch_user_items( user_name: T.Optional[str] = None, organization_key: T.Optional[str] = None ) -> types.UserItem: @@ -94,15 +128,19 @@ def fetch_user_items( else: user_items = _load_or_authenticate_user(profile_name) + user_json = _test_auth_and_update_user(user_items) + if user_json is not None: + LOG.info("Uploading to Mapillary user: %s", json.dumps(user_json)) + if organization_key is not None: resp = api_v4.fetch_organization( user_items["user_upload_token"], organization_key ) - org = resp.json() - LOG.info("Uploading to organization: %s", json.dumps(org)) + LOG.info("Uploading to Mapillary organization: %s", json.dumps(resp.json())) user_items = T.cast( types.UserItem, {**user_items, "MAPOrganizationKey": organization_key} ) + return user_items @@ -269,6 +307,8 @@ def _load_or_authenticate_user(profile_name: T.Optional[str] = None) -> types.Us profile_name, user_items = _prompt_user_for_user_items(profile_name) jsonschema.validate(user_items, types.UserItemSchema) + _test_auth_and_update_user(user_items) + # Update the config with the new user items LOG.info('Authenticated as "%s"', profile_name) config.update_config(profile_name, user_items) diff --git a/mapillary_tools/upload.py b/mapillary_tools/upload.py index 8f026e397..055b64a9b 100644 --- a/mapillary_tools/upload.py +++ b/mapillary_tools/upload.py @@ -611,16 +611,10 @@ def upload( if isinstance(inner_ex, requests.HTTPError) and isinstance( inner_ex.response, requests.Response ): - if inner_ex.response.status_code in [400, 401]: - try: - error_body = inner_ex.response.json() - except Exception: - error_body = {} - debug_info = error_body.get("debug_info", {}) - if debug_info.get("type") in ["NotAuthorizedError"]: - raise exceptions.MapillaryUploadUnauthorizedError( - debug_info.get("message") - ) from inner_ex + if api_v4.is_auth_error(inner_ex.response): + raise exceptions.MapillaryUploadUnauthorizedError( + api_v4.extract_auth_error_message(inner_ex.response) + ) from inner_ex raise inner_ex raise inner_ex diff --git a/tests/cli/upload_api_v4.py b/tests/cli/upload_api_v4.py index 54718aa5a..1d1ceaf51 100644 --- a/tests/cli/upload_api_v4.py +++ b/tests/cli/upload_api_v4.py @@ -6,7 +6,7 @@ import requests import tqdm -from mapillary_tools import upload +from mapillary_tools import api_v4, authenticate from mapillary_tools.upload_api_v4 import DEFAULT_CHUNK_SIZE, UploadService @@ -14,16 +14,6 @@ LOG = logging.getLogger("mapillary_tools") -def wrap_http_exception(ex: requests.HTTPError): - resp = ex.response - lines = [ - f"{ex.request.method} {resp.url}", - f"> HTTP Status: {ex.response.status_code}", - f"{resp.content!r}", - ] - return Exception("\n".join(lines)) - - def configure_logger(logger: logging.Logger, stream=None) -> None: formatter = logging.Formatter("%(asctime)s - %(levelname)-7s - %(message)s") handler = logging.StreamHandler(stream) @@ -67,7 +57,7 @@ def main(): with open(parsed.filename, "rb") as fp: entity_size = _file_stats(fp) - user_items = upload.fetch_user_items(parsed.user_name) + user_items = authenticate.fetch_user_items(parsed.user_name) session_key = parsed.session_key user_access_token = user_items.get("user_upload_token", "") @@ -81,11 +71,15 @@ def main(): else DEFAULT_CHUNK_SIZE ), ) - initial_offset = service.fetch_offset() + + try: + initial_offset = service.fetch_offset() + except requests.HTTPError as ex: + raise RuntimeError(api_v4.readable_http_error(ex)) LOG.info("Session key: %s", session_key) - LOG.info("Entity size: %d", entity_size) LOG.info("Initial offset: %s", initial_offset) + LOG.info("Entity size: %d", entity_size) LOG.info("Chunk size: %s MB", service.chunk_size / (1024 * 1024)) with open(parsed.filename, "rb") as fp: @@ -101,9 +95,20 @@ def main(): try: file_handle = service.upload(fp, initial_offset) except requests.HTTPError as ex: - raise wrap_http_exception(ex) + raise RuntimeError(api_v4.readable_http_error(ex)) + except KeyboardInterrupt: + file_handle = None + LOG.warning("Upload interrupted") + + try: + final_offset = service.fetch_offset() + except requests.HTTPError as ex: + raise RuntimeError(api_v4.readable_http_error(ex)) + + LOG.info("Final offset: %s", final_offset) + LOG.info("Entity size: %d", entity_size) - LOG.info(file_handle) + LOG.info("File handle: %s", file_handle) if __name__ == "__main__": From 8295bbea863fc429d458178d1fef5f0177b89d8f Mon Sep 17 00:00:00 2001 From: Tao Peng Date: Fri, 7 Mar 2025 17:06:44 -0800 Subject: [PATCH 10/16] handle invalid tokens --- mapillary_tools/authenticate.py | 143 ++++++++++++++++++++------------ mapillary_tools/config.py | 15 ++++ 2 files changed, 105 insertions(+), 53 deletions(-) diff --git a/mapillary_tools/authenticate.py b/mapillary_tools/authenticate.py index fa2879c00..cc5980cbe 100644 --- a/mapillary_tools/authenticate.py +++ b/mapillary_tools/authenticate.py @@ -20,6 +20,10 @@ def authenticate( user_password: T.Optional[str] = None, jwt: T.Optional[str] = None, ): + """ + Prompt for authentication information and save it to the config file + """ + # we still have to accept --user_name for the back compatibility profile_name = user_name @@ -75,38 +79,16 @@ def authenticate( config.update_config(profile_name, user_items) -def _test_auth_and_update_user( - user_items: types.UserItem, -) -> T.Optional[T.Dict[str, str]]: - try: - resp = api_v4.fetch_user_or_me( - user_access_token=user_items["user_upload_token"] - ) - except requests.HTTPError as ex: - if api_v4.is_auth_error(ex.response): - message = api_v4.extract_auth_error_message(ex.response) - raise exceptions.MapillaryUploadUnauthorizedError(message) - else: - # The point of this function is to test if the auth works, so we don't throw any non-auth errors - LOG.warning("Error testing the auth: %s", api_v4.readable_http_error(ex)) - return None - - user_json = resp.json() - if user_json is not None: - username = user_json.get("username") - if username is not None: - user_items["MAPSettingsUsername"] = username - - user_id = user_json.get("id") - if user_id is not None: - user_items["MAPSettingsUserKey"] = user_id - - return user_json - - def fetch_user_items( - user_name: T.Optional[str] = None, organization_key: T.Optional[str] = None + user_name: T.Optional[str] = None, + organization_key: T.Optional[str] = None, + let_user_choose: bool = False, ) -> types.UserItem: + """ + Read user information from the config file, + or prompt the user to authenticate if the specified profile does not exist + """ + # we still have to accept --user_name for the back compatibility profile_name = user_name @@ -116,19 +98,39 @@ def fetch_user_items( if profile_name is None: if len(all_user_items) == 0: - user_items = _load_or_authenticate_user() - elif len(all_user_items) == 1: - user_items = list(all_user_items.values())[0] + profile_name, user_items = _load_or_authenticate_user() else: - if not _prompt_enabled(): - raise exceptions.MapillaryBadParameterError( - "Multiple user profiles found, please choose one with --user_name" - ) - user_items = _prompt_choose_user_profile(all_user_items) + if let_user_choose or len(all_user_items) > 1: + profile_name, user_items = _prompt_choose_user_profile(all_user_items) + else: + profile_name, user_items = list(all_user_items.items())[0] else: - user_items = _load_or_authenticate_user(profile_name) + profile_name, user_items = _load_or_authenticate_user(profile_name) + + try: + user_json = _test_auth_and_update_user(user_items) + except exceptions.MapillaryUploadUnauthorizedError as ex: + if not _prompt_enabled(): + raise ex + + LOG.error( + 'The access token for profile "%s" is invalid or expired: %s', + profile_name, + ex, + ) + + answer = _prompt( + f'Delete the profile "{profile_name}", and try again with another profile? [y/N] ' + ).strip() + + if answer.lower() == "y": + config.remove_config(profile_name) + return fetch_user_items( + user_name=None, organization_key=organization_key, let_user_choose=True + ) + else: + raise ex - user_json = _test_auth_and_update_user(user_items) if user_json is not None: LOG.info("Uploading to Mapillary user: %s", json.dumps(user_json)) @@ -137,9 +139,7 @@ def fetch_user_items( user_items["user_upload_token"], organization_key ) LOG.info("Uploading to Mapillary organization: %s", json.dumps(resp.json())) - user_items = T.cast( - types.UserItem, {**user_items, "MAPOrganizationKey": organization_key} - ) + user_items["MAPOrganizationKey"] = organization_key return user_items @@ -154,6 +154,35 @@ def _prompt(message: str) -> str: return input() +def _test_auth_and_update_user( + user_items: types.UserItem, +) -> T.Optional[T.Dict[str, str]]: + try: + resp = api_v4.fetch_user_or_me( + user_access_token=user_items["user_upload_token"] + ) + except requests.HTTPError as ex: + if api_v4.is_auth_error(ex.response): + message = api_v4.extract_auth_error_message(ex.response) + raise exceptions.MapillaryUploadUnauthorizedError(message) + else: + # The point of this function is to test if the auth works, so we don't throw any non-auth errors + LOG.warning("Error testing the auth: %s", api_v4.readable_http_error(ex)) + return None + + user_json = resp.json() + if user_json is not None: + username = user_json.get("username") + if username is not None: + user_items["MAPSettingsUsername"] = username + + user_id = user_json.get("id") + if user_id is not None: + user_items["MAPSettingsUserKey"] = user_id + + return user_json + + def _prompt_profile_name(skip_validation: bool = False) -> str: assert _prompt_enabled(), "should not get here if prompting is disabled" @@ -195,7 +224,10 @@ def _validate_profile_name(profile_name: str): def _list_all_profiles(profiles: T.Dict[str, types.UserItem]) -> None: _echo("Existing Mapillary profiles:") for idx, name in enumerate(profiles, 1): - _echo(f"{idx:>5}. {name:<32} {profiles[name].get('MAPSettingsUserKey')}") + items = profiles[name] + user_id = items.get("MAPSettingsUserKey", "N/A") + username = items.get("MAPSettingsUsername", "N/A") + _echo(f"{idx:>5}. {name:<32} {user_id:>16} {username:>32}") def _is_interactive(): @@ -283,11 +315,13 @@ def _prompt_user_for_user_items( } -def _load_or_authenticate_user(profile_name: T.Optional[str] = None) -> types.UserItem: +def _load_or_authenticate_user( + profile_name: T.Optional[str] = None, +) -> T.Tuple[str, types.UserItem]: if profile_name is not None: user_items = config.load_user(profile_name) if user_items is None: - LOG.info('Profile "%s" not found in config', profile_name) + LOG.info('Profile "%s" not found', profile_name) # validate here since we are going to create this profile _validate_profile_name(profile_name) else: @@ -297,11 +331,11 @@ def _load_or_authenticate_user(profile_name: T.Optional[str] = None) -> types.Us # If the user_items in config are invalid, proceed with the user input LOG.warning("Invalid user items for profile: %s", profile_name) else: - return user_items + return profile_name, user_items if not _prompt_enabled(): raise exceptions.MapillaryBadParameterError( - f'Profile "{profile_name}" not found (and prompting disabled)' + f'Profile "{profile_name}" not found' ) profile_name, user_items = _prompt_user_for_user_items(profile_name) @@ -313,22 +347,25 @@ def _load_or_authenticate_user(profile_name: T.Optional[str] = None) -> types.Us LOG.info('Authenticated as "%s"', profile_name) config.update_config(profile_name, user_items) - return user_items + return profile_name, user_items def _prompt_choose_user_profile( all_user_items: T.Dict[str, types.UserItem], -) -> types.UserItem: - assert _prompt_enabled(), "should not get here if prompting is disabled" +) -> T.Tuple[str, types.UserItem]: + if not _prompt_enabled(): + raise exceptions.MapillaryBadParameterError( + "Multiple user profiles found, please choose one with --user_name" + ) _list_all_profiles(all_user_items) while True: - profile_name = _prompt_profile_name() + profile_name = _prompt_profile_name(skip_validation=True) if profile_name in all_user_items: break _echo(f'Profile "{profile_name}" not found') - return all_user_items[profile_name] + return profile_name, all_user_items[profile_name] def _welcome(): diff --git a/mapillary_tools/config.py b/mapillary_tools/config.py index 9daef33ec..8f100262f 100644 --- a/mapillary_tools/config.py +++ b/mapillary_tools/config.py @@ -70,3 +70,18 @@ def update_config( os.makedirs(os.path.dirname(os.path.abspath(config_path)), exist_ok=True) with open(config_path, "w") as fp: config.write(fp) + + +def remove_config(profile_name: str, config_path: T.Optional[str] = None) -> None: + if config_path is None: + config_path = MAPILLARY_CONFIG_PATH + + config = _load_config(config_path) + if not config.has_section(profile_name): + return + + config.remove_section(profile_name) + + os.makedirs(os.path.dirname(os.path.abspath(config_path)), exist_ok=True) + with open(config_path, "w") as fp: + config.write(fp) From 29dcb1d6155c3028b42250f877e7f96548dd3df8 Mon Sep 17 00:00:00 2001 From: Tao Peng Date: Sun, 9 Mar 2025 12:22:28 -0700 Subject: [PATCH 11/16] call authenticate directly --- mapillary_tools/authenticate.py | 210 +++++++++++--------------------- 1 file changed, 74 insertions(+), 136 deletions(-) diff --git a/mapillary_tools/authenticate.py b/mapillary_tools/authenticate.py index cc5980cbe..728e2a20c 100644 --- a/mapillary_tools/authenticate.py +++ b/mapillary_tools/authenticate.py @@ -5,7 +5,6 @@ import sys import typing as T -import jsonschema import requests from . import api_v4, config, constants, exceptions, types @@ -41,7 +40,10 @@ def authenticate( raise exceptions.MapillaryBadParameterError( "Profile name is required, please specify one with --user_name" ) - profile_name = _prompt_profile_name(skip_validation=True) + profile_name = _prompt_profile_name() + + assert profile_name is not None, "profile_name should be set" + if profile_name in all_user_items: LOG.warning( 'The profile "%s" already exists and will be overridden', @@ -50,39 +52,28 @@ def authenticate( else: # validate only new profile names _validate_profile_name(profile_name) + LOG.info('Creating new profile: "%s"', profile_name) if jwt: user_items: types.UserItem = {"user_upload_token": jwt} - elif user_email and user_password: - resp = api_v4.get_upload_token(user_email, user_password) - data = resp.json() - user_items = { - "MAPSettingsUserKey": data["user_id"], - "user_upload_token": data["access_token"], - } else: - if user_email or user_password: - LOG.warning( - "Both user_email and user_password must be provided to authenticate" - ) - - if not _prompt_enabled(): - raise exceptions.MapillaryBadParameterError( - "Authentication required, but prompting is disabled" - ) - - profile_name, user_items = _prompt_user_for_user_items(profile_name) + user_items = _prompt_login(user_email=user_email, user_password=user_password) _test_auth_and_update_user(user_items) - LOG.info('Authenticated as "%s"', profile_name) + # Update the config with the new user items config.update_config(profile_name, user_items) + # TODO: print more user information + if profile_name in all_user_items: + LOG.info('Profile "%s" updated', profile_name) + else: + LOG.info('Profile "%s" created', profile_name) + def fetch_user_items( user_name: T.Optional[str] = None, organization_key: T.Optional[str] = None, - let_user_choose: bool = False, ) -> types.UserItem: """ Read user information from the config file, @@ -94,43 +85,28 @@ def fetch_user_items( all_user_items = config.list_all_users() if not all_user_items: - _welcome() + authenticate(user_name=profile_name) + # Fetch user information only here + all_user_items = config.list_all_users() + assert len(all_user_items) >= 1, "should have at least 1 profile" if profile_name is None: - if len(all_user_items) == 0: - profile_name, user_items = _load_or_authenticate_user() + if len(all_user_items) > 1: + profile_name, user_items = _prompt_choose_user_profile(all_user_items) else: - if let_user_choose or len(all_user_items) > 1: - profile_name, user_items = _prompt_choose_user_profile(all_user_items) - else: - profile_name, user_items = list(all_user_items.items())[0] + profile_name, user_items = list(all_user_items.items())[0] else: - profile_name, user_items = _load_or_authenticate_user(profile_name) - - try: - user_json = _test_auth_and_update_user(user_items) - except exceptions.MapillaryUploadUnauthorizedError as ex: - if not _prompt_enabled(): - raise ex - - LOG.error( - 'The access token for profile "%s" is invalid or expired: %s', - profile_name, - ex, - ) - - answer = _prompt( - f'Delete the profile "{profile_name}", and try again with another profile? [y/N] ' - ).strip() - - if answer.lower() == "y": - config.remove_config(profile_name) - return fetch_user_items( - user_name=None, organization_key=organization_key, let_user_choose=True - ) + if profile_name in all_user_items: + user_items = all_user_items[profile_name] else: - raise ex + _list_all_profiles(all_user_items) + raise exceptions.MapillaryBadParameterError( + f'Profile "{profile_name}" not found' + ) + assert profile_name is not None, "profile_name should be set" + + user_json = _test_auth_and_update_user(user_items) if user_json is not None: LOG.info("Uploading to Mapillary user: %s", json.dumps(user_json)) @@ -223,6 +199,11 @@ def _validate_profile_name(profile_name: str): def _list_all_profiles(profiles: T.Dict[str, types.UserItem]) -> None: _echo("Existing Mapillary profiles:") + + # Header + _echo(f"{'':>5} {'Profile name':<32} {'User ID':>16} {'Username':>32}") + + # List all profiles for idx, name in enumerate(profiles, 1): items = profiles[name] user_id = items.get("MAPSettingsUserKey", "N/A") @@ -231,13 +212,6 @@ def _list_all_profiles(profiles: T.Dict[str, types.UserItem]) -> None: def _is_interactive(): - """ - Determine if the current environment is interactive by checking - if standard streams are connected to a TTY device. - - Returns: - bool: True if running in an interactive terminal, False otherwise - """ # Check if stdout is connected to a terminal stdout_interactive = sys.stdout.isatty() if hasattr(sys.stdout, "isatty") else False @@ -259,95 +233,59 @@ def _prompt_enabled() -> bool: return True -def _prompt_user_for_user_items( - profile_name: T.Optional[str], -) -> T.Tuple[str, types.UserItem]: - assert _prompt_enabled(), "should not get here if prompting is disabled" - - if profile_name is None: - profile_name = _prompt_profile_name() +def _retryable_login(ex: requests.HTTPError) -> bool: + if 400 <= ex.response.status_code < 500: + r = ex.response.json() + subcode = r.get("error", {}).get("error_subcode") + if subcode in [1348028, 1348092, 3404005, 1348131]: + title = r.get("error", {}).get("error_user_title") + message = r.get("error", {}).get("error_user_msg") + LOG.error("%s: %s", title, message) + return True + return False - LOG.info('Authenticating as "%s"', profile_name) - user_email = "" - while not user_email: - user_email = _prompt("Enter Mapillary user email: ").strip() +def _prompt_login( + user_email: T.Optional[str] = None, + user_password: T.Optional[str] = None, +) -> types.UserItem: + _enabled = _prompt_enabled() - while True: - user_password = getpass.getpass("Enter Mapillary user password: ") - if user_password: - break + if user_email is None: + if not _enabled: + raise exceptions.MapillaryBadParameterError("user_email is required") + while not user_email: + user_email = _prompt("Enter Mapillary user email: ").strip() + else: + user_email = user_email.strip() + + if user_password is None: + if not _enabled: + raise exceptions.MapillaryBadParameterError("user_password is required") + while True: + user_password = getpass.getpass("Enter Mapillary user password: ") + if user_password: + break try: resp = api_v4.get_upload_token(user_email, user_password) except requests.HTTPError as ex: - if ( - isinstance(ex, requests.HTTPError) - and isinstance(ex.response, requests.Response) - and 400 <= ex.response.status_code < 500 - ): - r = ex.response.json() - subcode = r.get("error", {}).get("error_subcode") - if subcode in [1348028, 1348092, 3404005, 1348131]: - title = r.get("error", {}).get("error_user_title") - message = r.get("error", {}).get("error_user_msg") - LOG.error("%s: %s", title, message) - return _prompt_user_for_user_items(profile_name) - else: - raise ex - else: + if not _enabled: raise ex - data = resp.json() - upload_token = T.cast(str, data.get("access_token")) - user_key = T.cast(str, data.get("user_id")) - if not isinstance(upload_token, str) or not isinstance(user_key, (str, int)): - raise RuntimeError( - f"Error extracting user_key or token from the login response: {data}" - ) + if _retryable_login(ex): + return _prompt_login() - if isinstance(user_key, int): - user_key = str(user_key) + raise ex - return profile_name, { - "MAPSettingsUserKey": user_key, - "user_upload_token": upload_token, - } - - -def _load_or_authenticate_user( - profile_name: T.Optional[str] = None, -) -> T.Tuple[str, types.UserItem]: - if profile_name is not None: - user_items = config.load_user(profile_name) - if user_items is None: - LOG.info('Profile "%s" not found', profile_name) - # validate here since we are going to create this profile - _validate_profile_name(profile_name) - else: - try: - jsonschema.validate(user_items, types.UserItemSchema) - except jsonschema.ValidationError: - # If the user_items in config are invalid, proceed with the user input - LOG.warning("Invalid user items for profile: %s", profile_name) - else: - return profile_name, user_items - - if not _prompt_enabled(): - raise exceptions.MapillaryBadParameterError( - f'Profile "{profile_name}" not found' - ) - - profile_name, user_items = _prompt_user_for_user_items(profile_name) - jsonschema.validate(user_items, types.UserItemSchema) - - _test_auth_and_update_user(user_items) + data = resp.json() - # Update the config with the new user items - LOG.info('Authenticated as "%s"', profile_name) - config.update_config(profile_name, user_items) + user_items: types.UserItem = { + "user_upload_token": str(data["access_token"]), + "MAPSettingsUserKey": str(data["user_id"]), + } - return profile_name, user_items + return user_items def _prompt_choose_user_profile( From 0ae8a3da8c8b0179cc2a461c88ea7756ff4f3b72 Mon Sep 17 00:00:00 2001 From: Tao Peng Date: Mon, 17 Mar 2025 14:59:49 -0700 Subject: [PATCH 12/16] wip --- mapillary_tools/api_v4.py | 3 +- mapillary_tools/authenticate.py | 197 +++++++++++++---------- mapillary_tools/commands/authenticate.py | 9 +- 3 files changed, 126 insertions(+), 83 deletions(-) diff --git a/mapillary_tools/api_v4.py b/mapillary_tools/api_v4.py index a5f06649d..6e5088a7c 100644 --- a/mapillary_tools/api_v4.py +++ b/mapillary_tools/api_v4.py @@ -70,7 +70,7 @@ def _truncate(s, limit=512): return s -def _sanitize(headers: T.Dict): +def _sanitize(headers: T.Mapping[T.Any, T.Any]) -> T.Mapping[T.Any, T.Any]: new_headers = {} for k, v in headers.items(): @@ -81,6 +81,7 @@ def _sanitize(headers: T.Dict): "access-token", "access_token", "password", + "user_upload_token", ]: new_headers[k] = "[REDACTED]" else: diff --git a/mapillary_tools/authenticate.py b/mapillary_tools/authenticate.py index 728e2a20c..cb7073a64 100644 --- a/mapillary_tools/authenticate.py +++ b/mapillary_tools/authenticate.py @@ -6,6 +6,7 @@ import typing as T import requests +import jsonschema from . import api_v4, config, constants, exceptions, types @@ -18,6 +19,7 @@ def authenticate( user_email: T.Optional[str] = None, user_password: T.Optional[str] = None, jwt: T.Optional[str] = None, + delete: bool = False, ): """ Prompt for authentication information and save it to the config file @@ -33,42 +35,56 @@ def authenticate( _welcome() # Make sure profile name either validated or existed - if profile_name: + if profile_name is not None: profile_name = profile_name.strip() else: if not _prompt_enabled(): raise exceptions.MapillaryBadParameterError( "Profile name is required, please specify one with --user_name" ) - profile_name = _prompt_profile_name() + profile_name = _prompt_choose_profile_name( + list(all_user_items.keys()), must_exist=delete + ) assert profile_name is not None, "profile_name should be set" - if profile_name in all_user_items: - LOG.warning( - 'The profile "%s" already exists and will be overridden', - profile_name, - ) + if delete: + if profile_name not in all_user_items: + raise exceptions.MapillaryBadParameterError( + f'Profile "{profile_name}" not found' + ) + config.remove_config(profile_name) + LOG.info('Profile "%s" deleted', profile_name) else: - # validate only new profile names - _validate_profile_name(profile_name) - LOG.info('Creating new profile: "%s"', profile_name) + if profile_name in all_user_items: + LOG.warning( + 'The profile "%s" already exists and will be overridden', + profile_name, + ) + else: + LOG.info('Creating new profile: "%s"', profile_name) - if jwt: - user_items: types.UserItem = {"user_upload_token": jwt} - else: - user_items = _prompt_login(user_email=user_email, user_password=user_password) + if jwt: + user_items: types.UserItem = {"user_upload_token": jwt} + else: + user_items = _prompt_login( + user_email=user_email, user_password=user_password + ) - _test_auth_and_update_user(user_items) + user_items = _validate_and_update_profile(profile_name, user_items) - # Update the config with the new user items - config.update_config(profile_name, user_items) + # Update the config with the new user items + config.update_config(profile_name, user_items) - # TODO: print more user information - if profile_name in all_user_items: - LOG.info('Profile "%s" updated', profile_name) - else: - LOG.info('Profile "%s" created', profile_name) + # TODO: print more user information + if profile_name in all_user_items: + LOG.info( + 'Profile "%s" updated: %s', profile_name, api_v4._sanitize(user_items) + ) + else: + LOG.info( + 'Profile "%s" created: %s', profile_name, api_v4._sanitize(user_items) + ) def fetch_user_items( @@ -92,7 +108,15 @@ def fetch_user_items( assert len(all_user_items) >= 1, "should have at least 1 profile" if profile_name is None: if len(all_user_items) > 1: - profile_name, user_items = _prompt_choose_user_profile(all_user_items) + if not _prompt_enabled(): + raise exceptions.MapillaryBadParameterError( + "Multiple user profiles found, please choose one with --user_name" + ) + _list_all_profiles(all_user_items) + profile_name = _prompt_choose_profile_name( + list(all_user_items.keys()), must_exist=True + ) + user_items = all_user_items[profile_name] else: profile_name, user_items = list(all_user_items.items())[0] else: @@ -106,9 +130,10 @@ def fetch_user_items( assert profile_name is not None, "profile_name should be set" - user_json = _test_auth_and_update_user(user_items) - if user_json is not None: - LOG.info("Uploading to Mapillary user: %s", json.dumps(user_json)) + user_items = _validate_and_update_profile(profile_name, user_items) + LOG.info( + 'Uploading to profile "%s": %s', profile_name, api_v4._sanitize(user_items) + ) if organization_key is not None: resp = api_v4.fetch_organization( @@ -130,9 +155,16 @@ def _prompt(message: str) -> str: return input() -def _test_auth_and_update_user( - user_items: types.UserItem, -) -> T.Optional[T.Dict[str, str]]: +def _validate_and_update_profile( + profile_name: str, user_items: types.UserItem +) -> types.UserItem: + try: + jsonschema.validate(user_items, types.UserItemSchema) + except jsonschema.ValidationError as ex: + raise exceptions.MapillaryBadParameterError( + f'Invalid profile "{profile_name}": {ex.message}' + ) + try: resp = api_v4.fetch_user_or_me( user_access_token=user_items["user_upload_token"] @@ -142,46 +174,15 @@ def _test_auth_and_update_user( message = api_v4.extract_auth_error_message(ex.response) raise exceptions.MapillaryUploadUnauthorizedError(message) else: - # The point of this function is to test if the auth works, so we don't throw any non-auth errors - LOG.warning("Error testing the auth: %s", api_v4.readable_http_error(ex)) - return None + raise ex user_json = resp.json() - if user_json is not None: - username = user_json.get("username") - if username is not None: - user_items["MAPSettingsUsername"] = username - - user_id = user_json.get("id") - if user_id is not None: - user_items["MAPSettingsUserKey"] = user_id - - return user_json - - -def _prompt_profile_name(skip_validation: bool = False) -> str: - assert _prompt_enabled(), "should not get here if prompting is disabled" - - profile_name = "" - - while not profile_name: - profile_name = _prompt( - "Enter the Mapillary profile you would like to (re)authenticate: " - ).strip() - - if profile_name: - if skip_validation: - break - try: - _validate_profile_name(profile_name) - except ValueError as ex: - LOG.error("Error validating profile name: %s", ex) - profile_name = "" - else: - break - - return profile_name + return { + **user_items, + "MAPSettingsUsername": user_json.get("username"), + "MAPSettingsUserKey": user_json.get("id"), + } def _validate_profile_name(profile_name: str): @@ -190,7 +191,7 @@ def _validate_profile_name(profile_name: str): "Profile name must be between 2 and 32 characters long" ) - pattern = re.compile(r"^[a-zA-Z0-9_-]+$") + pattern = re.compile(r"^[a-zA-Z]+[a-zA-Z0-9_-]*$") if not bool(pattern.match(profile_name)): raise exceptions.MapillaryBadParameterError( "Invalid profile name. Use only letters, numbers, hyphens and underscores" @@ -233,7 +234,7 @@ def _prompt_enabled() -> bool: return True -def _retryable_login(ex: requests.HTTPError) -> bool: +def _is_login_retryable(ex: requests.HTTPError) -> bool: if 400 <= ex.response.status_code < 500: r = ex.response.json() subcode = r.get("error", {}).get("error_subcode") @@ -273,7 +274,7 @@ def _prompt_login( if not _enabled: raise ex - if _retryable_login(ex): + if _is_login_retryable(ex): return _prompt_login() raise ex @@ -288,22 +289,56 @@ def _prompt_login( return user_items -def _prompt_choose_user_profile( - all_user_items: T.Dict[str, types.UserItem], -) -> T.Tuple[str, types.UserItem]: - if not _prompt_enabled(): - raise exceptions.MapillaryBadParameterError( - "Multiple user profiles found, please choose one with --user_name" - ) +def _prompt_choose_profile_name( + existing_profile_names: T.Sequence[str], must_exist: bool = False +) -> str: + assert _prompt_enabled(), "should not get here if prompting is disabled" + + existed = set(existing_profile_names) - _list_all_profiles(all_user_items) while True: - profile_name = _prompt_profile_name(skip_validation=True) - if profile_name in all_user_items: + if must_exist: + prompt = "Enter an existing profile: " + else: + prompt = "Enter an existing profile or create a new one: " + + profile_name = _prompt(prompt).strip() + + if not profile_name: + continue + + # Exit if it's found + if profile_name in existed: + break + + # Try to find by index + try: + profile_name = existing_profile_names[int(profile_name) - 1] + except (ValueError, IndexError): + pass + else: + # Exit if it's found break - _echo(f'Profile "{profile_name}" not found') - return profile_name, all_user_items[profile_name] + assert profile_name not in existed, ( + f"Profile {profile_name} must not exist here" + ) + + if must_exist: + LOG.error('Profile "%s" not found', profile_name) + else: + try: + _validate_profile_name(profile_name) + except exceptions.MapillaryBadParameterError as ex: + LOG.error("Error validating profile name: %s", ex) + profile_name = "" + else: + break + + if must_exist: + assert profile_name in existed, f"Profile {profile_name} must exist" + + return profile_name def _welcome(): diff --git a/mapillary_tools/commands/authenticate.py b/mapillary_tools/commands/authenticate.py index b6c6a9242..e30c92014 100644 --- a/mapillary_tools/commands/authenticate.py +++ b/mapillary_tools/commands/authenticate.py @@ -10,7 +10,7 @@ class Command: def add_basic_arguments(self, parser: argparse.ArgumentParser): parser.add_argument( - "--user_name", help="Mapillary user name", default=None, required=False + "--user_name", help="Mapillary user profile", default=None, required=False ) parser.add_argument( "--user_email", @@ -27,6 +27,13 @@ def add_basic_arguments(self, parser: argparse.ArgumentParser): parser.add_argument( "--jwt", help="Mapillary user access token", default=None, required=False ) + parser.add_argument( + "--delete", + help="Delete the specified user profile", + default=False, + required=False, + action="store_true", + ) def run(self, vars_args: dict): authenticate( From 15d3901bfa12d5e71d3460dd5f72b88e8e115b75 Mon Sep 17 00:00:00 2001 From: Tao Peng Date: Wed, 26 Mar 2025 15:10:50 -0700 Subject: [PATCH 13/16] add tests --- tests/unit/test_config.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 9185b30e8..67af45cab 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -23,7 +23,7 @@ def test_config_list_all_users(tmpdir: py.path.local): x = config.list_all_users(config_path=str(c)) assert len(x) == 1 - assert x[0] == {"ThisIsOption": "1"} + assert x["hello"] == {"ThisIsOption": "1"} def test_update_config(tmpdir: py.path.local): @@ -50,3 +50,15 @@ def test_load_user(tmpdir: py.path.local): assert x is None x = config.load_user("world", config_path=str(c)) assert x == {"ThisIsOption": "hello"} + + +def test_remove(tmpdir: py.path.local): + c = tmpdir.join("empty_config.ini") + config.update_config( + "world", T.cast(T.Any, {"ThisIsOption": "hello"}), config_path=str(c) + ) + config.remove_config("world", config_path=str(c)) + u = config.load_user("world", config_path=str(c)) + assert u is None + x = config.list_all_users(config_path=str(c)) + assert not x From 6b2c2f5f4be96518e4e4bee1f2ac11f2d1c0c940 Mon Sep 17 00:00:00 2001 From: Tao Peng Date: Wed, 26 Mar 2025 15:38:42 -0700 Subject: [PATCH 14/16] update --- mapillary_tools/authenticate.py | 42 ++++++++++++++++----------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/mapillary_tools/authenticate.py b/mapillary_tools/authenticate.py index cb7073a64..18bd62afd 100644 --- a/mapillary_tools/authenticate.py +++ b/mapillary_tools/authenticate.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import getpass import json import logging @@ -15,17 +17,17 @@ def authenticate( - user_name: T.Optional[str] = None, - user_email: T.Optional[str] = None, - user_password: T.Optional[str] = None, - jwt: T.Optional[str] = None, + user_name: str | None = None, + user_email: str | None = None, + user_password: str | None = None, + jwt: str | None = None, delete: bool = False, ): """ Prompt for authentication information and save it to the config file """ - # we still have to accept --user_name for the back compatibility + # We still have to accept --user_name for the back compatibility profile_name = user_name all_user_items = config.list_all_users() @@ -49,10 +51,6 @@ def authenticate( assert profile_name is not None, "profile_name should be set" if delete: - if profile_name not in all_user_items: - raise exceptions.MapillaryBadParameterError( - f'Profile "{profile_name}" not found' - ) config.remove_config(profile_name) LOG.info('Profile "%s" deleted', profile_name) else: @@ -66,12 +64,12 @@ def authenticate( if jwt: user_items: types.UserItem = {"user_upload_token": jwt} + user_items = _authenticate_profile(_validate_profile(user_items)) else: user_items = _prompt_login( user_email=user_email, user_password=user_password ) - - user_items = _validate_and_update_profile(profile_name, user_items) + _validate_profile(user_items) # Update the config with the new user items config.update_config(profile_name, user_items) @@ -88,8 +86,8 @@ def authenticate( def fetch_user_items( - user_name: T.Optional[str] = None, - organization_key: T.Optional[str] = None, + user_name: str | None = None, + organization_key: str | None = None, ) -> types.UserItem: """ Read user information from the config file, @@ -130,7 +128,8 @@ def fetch_user_items( assert profile_name is not None, "profile_name should be set" - user_items = _validate_and_update_profile(profile_name, user_items) + user_items = _authenticate_profile(_validate_profile(user_items)) + LOG.info( 'Uploading to profile "%s": %s', profile_name, api_v4._sanitize(user_items) ) @@ -155,16 +154,17 @@ def _prompt(message: str) -> str: return input() -def _validate_and_update_profile( - profile_name: str, user_items: types.UserItem -) -> types.UserItem: +def _validate_profile(user_items: types.UserItem) -> types.UserItem: try: jsonschema.validate(user_items, types.UserItemSchema) except jsonschema.ValidationError as ex: raise exceptions.MapillaryBadParameterError( - f'Invalid profile "{profile_name}": {ex.message}' + f"Invalid profile format: {ex.message}" ) + return user_items + +def _authenticate_profile(user_items: types.UserItem) -> types.UserItem: try: resp = api_v4.fetch_user_or_me( user_access_token=user_items["user_upload_token"] @@ -198,7 +198,7 @@ def _validate_profile_name(profile_name: str): ) -def _list_all_profiles(profiles: T.Dict[str, types.UserItem]) -> None: +def _list_all_profiles(profiles: dict[str, types.UserItem]) -> None: _echo("Existing Mapillary profiles:") # Header @@ -247,8 +247,8 @@ def _is_login_retryable(ex: requests.HTTPError) -> bool: def _prompt_login( - user_email: T.Optional[str] = None, - user_password: T.Optional[str] = None, + user_email: str | None = None, + user_password: str | None = None, ) -> types.UserItem: _enabled = _prompt_enabled() From 0269d040c0856edea7f925e1a033d6e7494cd9bf Mon Sep 17 00:00:00 2001 From: Tao Peng Date: Wed, 26 Mar 2025 15:58:58 -0700 Subject: [PATCH 15/16] introduce env MAPILLARY_TOOLS__DISABLE_AUTH_VERIFICATION --- mapillary_tools/authenticate.py | 12 +++++++++--- mapillary_tools/constants.py | 8 ++++++++ tests/integration/fixtures.py | 12 ++++++++---- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/mapillary_tools/authenticate.py b/mapillary_tools/authenticate.py index 18bd62afd..b66ea2713 100644 --- a/mapillary_tools/authenticate.py +++ b/mapillary_tools/authenticate.py @@ -64,7 +64,7 @@ def authenticate( if jwt: user_items: types.UserItem = {"user_upload_token": jwt} - user_items = _authenticate_profile(_validate_profile(user_items)) + user_items = _verify_user_auth(_validate_profile(user_items)) else: user_items = _prompt_login( user_email=user_email, user_password=user_password @@ -128,7 +128,7 @@ def fetch_user_items( assert profile_name is not None, "profile_name should be set" - user_items = _authenticate_profile(_validate_profile(user_items)) + user_items = _verify_user_auth(_validate_profile(user_items)) LOG.info( 'Uploading to profile "%s": %s', profile_name, api_v4._sanitize(user_items) @@ -164,7 +164,13 @@ def _validate_profile(user_items: types.UserItem) -> types.UserItem: return user_items -def _authenticate_profile(user_items: types.UserItem) -> types.UserItem: +def _verify_user_auth(user_items: types.UserItem) -> types.UserItem: + """ + Verify that the user access token is valid + """ + if constants._DISABLE_AUTH_VERIFICATION: + return user_items + try: resp = api_v4.fetch_user_or_me( user_access_token=user_items["user_upload_token"] diff --git a/mapillary_tools/constants.py b/mapillary_tools/constants.py index d8676e867..5327a5044 100644 --- a/mapillary_tools/constants.py +++ b/mapillary_tools/constants.py @@ -61,3 +61,11 @@ "TRUE", "YES", ] + +_DISABLE_AUTH_VERIFICATION = os.getenv( + _ENV_PREFIX + "_DISABLE_AUTH_VERIFICATION", "NO" +).upper() in [ + "1", + "TRUE", + "YES", +] diff --git a/tests/integration/fixtures.py b/tests/integration/fixtures.py index cc30f54d3..e6182003e 100644 --- a/tests/integration/fixtures.py +++ b/tests/integration/fixtures.py @@ -28,6 +28,7 @@ def setup_config(tmpdir: py.path.local): config_path = tmpdir.mkdir("configs").join("CLIENT_ID") os.environ["MAPILLARY_CONFIG_PATH"] = str(config_path) + os.environ["MAPILLARY_TOOLS__DISABLE_AUTH_VERIFICATION"] = "YES" x = subprocess.run( f"{EXECUTABLE} authenticate --user_name {USERNAME} --jwt test_user_token", shell=True, @@ -36,7 +37,8 @@ def setup_config(tmpdir: py.path.local): yield config_path if tmpdir.check(): tmpdir.remove(ignore_errors=True) - del os.environ["MAPILLARY_CONFIG_PATH"] + os.environ.pop("MAPILLARY_CONFIG_PATH", None) + os.environ.pop("MAPILLARY_TOOLS__DISABLE_AUTH_VERIFICATION", None) @pytest.fixture @@ -53,15 +55,17 @@ def setup_data(tmpdir: py.path.local): def setup_upload(tmpdir: py.path.local): upload_dir = tmpdir.mkdir("mapillary_public_uploads") os.environ["MAPILLARY_UPLOAD_PATH"] = str(upload_dir) + os.environ["MAPILLARY_TOOLS__DISABLE_AUTH_VERIFICATION"] = "YES" os.environ["MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN"] = "YES" history_path = tmpdir.join("history") os.environ["MAPILLARY_UPLOAD_HISTORY_PATH"] = str(history_path) yield upload_dir if tmpdir.check(): tmpdir.remove(ignore_errors=True) - del os.environ["MAPILLARY_UPLOAD_PATH"] - del os.environ["MAPILLARY_UPLOAD_HISTORY_PATH"] - del os.environ["MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN"] + os.environ.pop("MAPILLARY_UPLOAD_PATH", None) + os.environ.pop("MAPILLARY_UPLOAD_HISTORY_PATH", None) + os.environ.pop("MAPILLARY_TOOLS__DISABLE_AUTH_VERIFICATION", None) + os.environ.pop("MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN", None) def _ffmpeg_installed(): From a92648d25ff0cb44502474988ac41402f1724e2f Mon Sep 17 00:00:00 2001 From: Tao Peng Date: Wed, 26 Mar 2025 16:42:09 -0700 Subject: [PATCH 16/16] update banner --- mapillary_tools/authenticate.py | 13 +++++++------ mapillary_tools/constants.py | 25 +++++++++++++------------ tests/integration/fixtures.py | 12 ++++++++---- 3 files changed, 28 insertions(+), 22 deletions(-) diff --git a/mapillary_tools/authenticate.py b/mapillary_tools/authenticate.py index b66ea2713..af956f5e2 100644 --- a/mapillary_tools/authenticate.py +++ b/mapillary_tools/authenticate.py @@ -168,7 +168,7 @@ def _verify_user_auth(user_items: types.UserItem) -> types.UserItem: """ Verify that the user access token is valid """ - if constants._DISABLE_AUTH_VERIFICATION: + if constants._AUTH_VERIFICATION_DISABLED: return user_items try: @@ -351,11 +351,12 @@ def _welcome(): _echo( """ ================================================================================ - Welcome to Mapillary! + Welcome to Mapillary! ================================================================================ -If you haven't registered yet, please visit the following link to sign up first: -https://www.mapillary.com/signup -After the registration, proceed here to sign in. + If you haven't registered yet, please visit https://www.mapillary.com/signup + to create your account first. + + Once registered, proceed here to sign in. ================================================================================ - """.strip() + """ ) diff --git a/mapillary_tools/constants.py b/mapillary_tools/constants.py index 5327a5044..a58cce8ee 100644 --- a/mapillary_tools/constants.py +++ b/mapillary_tools/constants.py @@ -5,6 +5,15 @@ _ENV_PREFIX = "MAPILLARY_TOOLS_" + +def _yes_or_no(val: str) -> bool: + return val.strip().upper() in [ + "1", + "TRUE", + "YES", + ] + + # In meters CUTOFF_DISTANCE = float(os.getenv(_ENV_PREFIX + "CUTOFF_DISTANCE", 600)) # In seconds @@ -56,16 +65,8 @@ # Max number of pixels per sequence (sum of image pixels in the sequence) MAX_SEQUENCE_PIXELS: str = os.getenv(_ENV_PREFIX + "MAX_SEQUENCE_PIXELS", "6G") -PROMPT_DISABLED: bool = os.getenv(_ENV_PREFIX + "PROMPT_DISABLED", "NO").upper() in [ - "1", - "TRUE", - "YES", -] +PROMPT_DISABLED: bool = _yes_or_no(os.getenv(_ENV_PREFIX + "PROMPT_DISABLED", "NO")) -_DISABLE_AUTH_VERIFICATION = os.getenv( - _ENV_PREFIX + "_DISABLE_AUTH_VERIFICATION", "NO" -).upper() in [ - "1", - "TRUE", - "YES", -] +_AUTH_VERIFICATION_DISABLED: bool = _yes_or_no( + os.getenv(_ENV_PREFIX + "_AUTH_VERIFICATION_DISABLED", "NO") +) diff --git a/tests/integration/fixtures.py b/tests/integration/fixtures.py index e6182003e..02335309d 100644 --- a/tests/integration/fixtures.py +++ b/tests/integration/fixtures.py @@ -28,7 +28,8 @@ def setup_config(tmpdir: py.path.local): config_path = tmpdir.mkdir("configs").join("CLIENT_ID") os.environ["MAPILLARY_CONFIG_PATH"] = str(config_path) - os.environ["MAPILLARY_TOOLS__DISABLE_AUTH_VERIFICATION"] = "YES" + os.environ["MAPILLARY_TOOLS_PROMPT_DISABLED"] = "YES" + os.environ["MAPILLARY_TOOLS__AUTH_VERIFICATION_DISABLED"] = "YES" x = subprocess.run( f"{EXECUTABLE} authenticate --user_name {USERNAME} --jwt test_user_token", shell=True, @@ -38,7 +39,8 @@ def setup_config(tmpdir: py.path.local): if tmpdir.check(): tmpdir.remove(ignore_errors=True) os.environ.pop("MAPILLARY_CONFIG_PATH", None) - os.environ.pop("MAPILLARY_TOOLS__DISABLE_AUTH_VERIFICATION", None) + os.environ.pop("MAPILLARY_TOOLS_PROMPT_DISABLED", None) + os.environ.pop("MAPILLARY_TOOLS__AUTH_VERIFICATION_DISABLED", None) @pytest.fixture @@ -55,7 +57,8 @@ def setup_data(tmpdir: py.path.local): def setup_upload(tmpdir: py.path.local): upload_dir = tmpdir.mkdir("mapillary_public_uploads") os.environ["MAPILLARY_UPLOAD_PATH"] = str(upload_dir) - os.environ["MAPILLARY_TOOLS__DISABLE_AUTH_VERIFICATION"] = "YES" + os.environ["MAPILLARY_TOOLS__AUTH_VERIFICATION_DISABLED"] = "YES" + os.environ["MAPILLARY_TOOLS_PROMPT_DISABLED"] = "YES" os.environ["MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN"] = "YES" history_path = tmpdir.join("history") os.environ["MAPILLARY_UPLOAD_HISTORY_PATH"] = str(history_path) @@ -64,7 +67,8 @@ def setup_upload(tmpdir: py.path.local): tmpdir.remove(ignore_errors=True) os.environ.pop("MAPILLARY_UPLOAD_PATH", None) os.environ.pop("MAPILLARY_UPLOAD_HISTORY_PATH", None) - os.environ.pop("MAPILLARY_TOOLS__DISABLE_AUTH_VERIFICATION", None) + os.environ.pop("MAPILLARY_TOOLS__AUTH_VERIFICATION_DISABLED", None) + os.environ.pop("MAPILLARY_TOOLS_PROMPT_DISABLED", None) os.environ.pop("MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN", None)