Merge pull request #9 from ImageMarkup/support-metadata-types

Author: danlamanna

isic_cli/cli/utils.py

@@ -44,15 +44,17 @@ def _extract_metadata(
     metadata_fields = set()
 
     for image in images:
-        metadata_fields |= set(image.get('metadata', {}).keys())
+        metadata_fields |= set(image['metadata']['acquisition'].keys())
+        metadata_fields |= set(image['metadata']['clinical'].keys())
         metadata.append(
             {
                 **{
                     'isic_id': image['isic_id'],
                     'attribution': image['attribution'],
                     'copyright_license': image['copyright_license'],
                 },
-                **image['metadata'],
+                **image['metadata']['acquisition'],
+                **image['metadata']['clinical'],
             }
         )
 

isic_cli/io/http.py

@@ -123,10 +123,18 @@ def get_num_images(
     before_sleep=before_sleep_log(logger, logging.DEBUG),
 )
 def download_image(image: dict, to: Path, progress, task) -> None:
+    dest_path = to / f'{image["isic_id"]}.JPG'
+
+    # Avoid re downloading the image if one of the same name/size exists. This is a decent
+    # enough proxy for detecting file differences without going throw a hashing mechanism.
+    if dest_path.exists() and dest_path.stat().st_size == image['files']['full']['size']:
+        progress.update(task, advance=1)
+        return
+
     # intentionally don't pass auth headers, since these are s3 signed urls that
     # already contain credentials.
     with IsicCliSession() as session:
-        r = session.get(image['urls']['full'], stream=True)
+        r = session.get(image['files']['full']['url'], stream=True)
         r.raise_for_status()
 
         temp_file_name = None
@@ -135,6 +143,6 @@ def download_image(image: dict, to: Path, progress, task) -> None:
             for chunk in r.iter_content(1024 * 1024 * 5):
                 outfile.write(chunk)
 
-        shutil.move(temp_file_name, to / f'{image["isic_id"]}.JPG')
+        shutil.move(temp_file_name, dest_path)
 
     progress.update(task, advance=1)

tests/test_cli_image.py

@@ -36,7 +36,10 @@ def _download_image_side_effect(*args, **kwargs):
                     'isic_id': 'ISIC_0000000',
                     'copyright_license': 'CC-0',
                     'attribution': 'some-institution',
-                    'metadata': {'sex': 'male', 'diagnosis': 'melanoma'},
+                    'metadata': {
+                        'acquisition': {},
+                        'clinical': {'sex': 'male', 'diagnosis': 'melanoma'},
+                    },
                 }
             ]
         ),

tests/test_cli_metadata.py

@@ -23,7 +23,10 @@ def test_metadata_download(cli_run, mocker):
                     'isic_id': 'ISIC_0000000',
                     'attribution': 'Foo',
                     'copyright_license': 'CC-0',
-                    'metadata': {'sex': 'male', 'diagnosis': 'melanoma'},
+                    'metadata': {
+                        'acquisition': {},
+                        'clinical': {'sex': 'male', 'diagnosis': 'melanoma'},
+                    },
                 }
             ]
         ),