Remove unneeded API calls in folder_file_ops

Added construct_path and construct_empty_path to base-
provider for individual providers to make paths for folder-file-ops
without needing to make API calls

Author: AddisonSchiller

tests/core/test_provider.py

@@ -104,6 +104,15 @@ async def test_revalidate_path_is_child(self, provider1):
         assert str(path) == str(new_path.parent)
         assert new_path.name == 'text_file.txt'
 
+    @pytest.mark.asyncio
+    async def test_construct_empty_path(self, provider1):
+        path = await provider1.validate_path('/folder/')
+        data = utils.MockFileMetadata()
+        empty_path = await provider1.construct_empty_path(path, data)
+
+        # Make sure its actually not a real path
+        assert empty_path.identifier is None
+
 
 class TestHandleNameConflict:
 

tests/providers/bitbucket/test_provider.py

@@ -61,7 +61,7 @@ async def test_validate_v1_path_root(self, provider):
 
         assert wb_path_v1 == wb_path_v0
         assert wb_path_v1.branch_name == default_branch_body['name']
-        assert wb_path_v1.commit_sha == None
+        assert wb_path_v1.commit_sha is None
 
     @pytest.mark.asyncio
     @pytest.mark.aiohttpretty
@@ -77,8 +77,8 @@ async def test_validate_v1_path(self, provider, path, kind):
         default_branch_url = provider._build_v1_repo_url('main-branch')
         aiohttpretty.register_json_uri('GET', default_branch_url, body=default_branch_body)
 
-        dir_listing_body =  test_fixtures['root_dir_listing']
-        dir_listing_url = provider._build_v1_repo_url('src', default_branch)  + '/'
+        dir_listing_body = test_fixtures['root_dir_listing']
+        dir_listing_url = provider._build_v1_repo_url('src', default_branch) + '/'
         aiohttpretty.register_json_uri('GET', dir_listing_url, body=dir_listing_body)
 
         try:
@@ -107,9 +107,9 @@ async def test_validate_v1_path(self, provider, path, kind):
     async def test_validate_v1_path_commit_sha(self, provider, arg_name, arg_val, attr_name):
         test_fixtures = fixtures.validate_path
 
-        dir_listing_body =  test_fixtures['root_dir_listing']
+        dir_listing_body = test_fixtures['root_dir_listing']
         base_commit = dir_listing_body['node']
-        dir_listing_url = provider._build_v1_repo_url('src', arg_val)  + '/'
+        dir_listing_url = provider._build_v1_repo_url('src', arg_val) + '/'
         aiohttpretty.register_json_uri('GET', dir_listing_url, body=dir_listing_body)
 
         path = '/foo-file.txt'
@@ -145,9 +145,9 @@ async def test_validate_v1_path_commit_sha(self, provider, arg_name, arg_val, at
     async def test_validate_v1_path_subfolder(self, provider):
         test_fixtures = fixtures.validate_path
 
-        dir_listing_body =  test_fixtures['subfolder_dir_listing']
+        dir_listing_body = test_fixtures['subfolder_dir_listing']
         base_commit = dir_listing_body['node']
-        dir_listing_url = provider._build_v1_repo_url('src', 'main-branch', 'subfolder')  + '/'
+        dir_listing_url = provider._build_v1_repo_url('src', 'main-branch', 'subfolder') + '/'
         aiohttpretty.register_json_uri('GET', dir_listing_url, body=dir_listing_body)
 
         path = '/subfolder/.gitkeep'
@@ -186,8 +186,8 @@ async def test_get_metadata_for_file(self, provider):
         path = BitbucketPath('/foo-file.txt', _ids=[(base_ref, 'develop'), (base_ref, 'develop')])
 
         test_fixtures = fixtures.validate_path
-        dir_listing_body =  test_fixtures['root_dir_listing']
-        dir_listing_url = provider._build_v1_repo_url('src', base_ref)  + '/'
+        dir_listing_body = test_fixtures['root_dir_listing']
+        dir_listing_url = provider._build_v1_repo_url('src', base_ref) + '/'
         aiohttpretty.register_json_uri('GET', dir_listing_url, body=dir_listing_body)
 
         result = await provider.metadata(path)
@@ -211,8 +211,8 @@ async def test_get_metadata_for_folder(self, provider):
         path = BitbucketPath('/', _ids=[(None, 'develop')], folder=True)
 
         test_fixtures = fixtures.validate_path
-        dir_listing_body =  test_fixtures['root_dir_listing']
-        dir_listing_url = provider._build_v1_repo_url('src', 'develop')  + '/'
+        dir_listing_body = test_fixtures['root_dir_listing']
+        dir_listing_url = provider._build_v1_repo_url('src', 'develop') + '/'
         aiohttpretty.register_json_uri('GET', dir_listing_url, body=dir_listing_body)
 
         result = await provider.metadata(path)
@@ -229,8 +229,8 @@ async def test_get_metadata_for_file(self, provider):
         path = BitbucketPath('/foo-file.txt', _ids=[(base_ref, 'develop'), (base_ref, 'develop')])
 
         test_fixtures = fixtures.validate_path
-        dir_listing_body =  test_fixtures['root_dir_listing']
-        dir_listing_url = provider._build_v1_repo_url('src', base_ref)  + '/'
+        dir_listing_body = test_fixtures['root_dir_listing']
+        dir_listing_url = provider._build_v1_repo_url('src', base_ref) + '/'
         aiohttpretty.register_json_uri('GET', dir_listing_url, body=dir_listing_body)
 
         download_url = provider._build_v1_repo_url('raw', path.commit_sha, *path.path_tuple())
@@ -268,17 +268,38 @@ async def test_copy_to(self, provider):
         assert e.value.code == 501
 
     def test_can_intra_move(self, provider):
-        assert provider.can_intra_move(provider) == False
+        assert provider.can_intra_move(provider) is False
 
     def test_can_intra_copy(self, provider):
-        assert provider.can_intra_copy(provider) == False
+        assert provider.can_intra_copy(provider) is False
 
 
 # leftover bits
 class TestMisc:
 
+    @pytest.mark.asyncio
+    async def test_construct_path(self, provider):
+        name = 'aaa-01-2.txt'
+        subdir = 'plaster'
+        full_path = '/{}/{}'.format(subdir, name)
+        branch = 'master'
+        commit_sha = '123abc456def'
+
+        path = BitbucketPath(full_path, _ids=[
+            (commit_sha, branch), (commit_sha, branch), (commit_sha, branch)
+        ])
+
+        metadata = BitbucketFileMetadata(fixtures.file_metadata, path, owner=fixtures.owner, repo=fixtures.repo)
+        child_path = await provider.construct_path(path.parent, metadata)
+        rev_metadata = await provider.revalidate_path(path.parent,
+                        metadata.name, folder=metadata.is_folder)
+
+        assert child_path.full_path == path.full_path
+        assert child_path == path
+        assert child_path == rev_metadata
+
     def test_can_duplicate_name(self, provider):
-        assert provider.can_duplicate_names() == False
+        assert provider.can_duplicate_names() is False
 
     def test_path_from_metadata(self, provider):
         name = 'aaa-01-2.txt'
@@ -292,7 +313,7 @@ def test_path_from_metadata(self, provider):
         ])
 
         metadata = BitbucketFileMetadata(fixtures.file_metadata, path, owner=fixtures.owner, repo=fixtures.repo)
-        child_path =  provider.path_from_metadata(path.parent, metadata)
+        child_path = provider.path_from_metadata(path.parent, metadata)
 
         assert child_path.full_path == path.full_path
         assert child_path == path

tests/providers/box/test_provider.py

@@ -834,23 +834,31 @@ class TestOperations:
 
     @pytest.mark.asyncio
     @pytest.mark.aiohttpretty
-    async def test_can_duplicate_names(self, provider):
+    async def test_construct_path(self, provider, root_provider_fixtures):
+        path = WaterButlerPath('/50 shades of nope/', _ids=(provider.folder, '0'))
+        item = root_provider_fixtures['revalidate_metadata']['entries'][0]
+        data = BoxFileMetadata(item, path)
+
+        url = provider.build_url('folders', path.identifier, 'items',
+                           fields='id,name,type', limit=1000)
+        aiohttpretty.register_json_uri('GET', url, body=root_provider_fixtures['revalidate_metadata'])
+
+        con_metadata = await provider.construct_path(path, data)
+
+        path_metadata = await provider.revalidate_path(path, data.name)
+        assert con_metadata == path_metadata
+
+    def test_can_duplicate_names(self, provider):
         assert provider.can_duplicate_names() is False
 
-    @pytest.mark.asyncio
-    @pytest.mark.aiohttpretty
-    async def test_shares_storage_root(self, provider, other_provider):
+    def test_shares_storage_root(self, provider, other_provider):
         assert provider.shares_storage_root(other_provider) is False
         assert provider.shares_storage_root(provider) is True
 
-    @pytest.mark.asyncio
-    @pytest.mark.aiohttpretty
-    async def test_can_intra_move(self, provider, other_provider):
+    def test_can_intra_move(self, provider, other_provider):
         assert provider.can_intra_move(other_provider) is False
         assert provider.can_intra_move(provider) is True
 
-    @pytest.mark.asyncio
-    @pytest.mark.aiohttpretty
-    async def test_can_intra_copy(self, provider, other_provider):
+    def test_can_intra_copy(self, provider, other_provider):
         assert provider.can_intra_copy(other_provider) is False
         assert provider.can_intra_copy(provider) is True

tests/providers/cloudfiles/test_provider.py

@@ -3,7 +3,6 @@
 import json
 import time
 import hashlib
-import functools
 from unittest import mock
 
 import furl
@@ -17,6 +16,7 @@
 from waterbutler.core.path import WaterButlerPath
 from waterbutler.providers.cloudfiles import CloudFilesProvider
 from waterbutler.providers.cloudfiles import settings as cloud_settings
+from waterbutler.providers.cloudfiles.metadata import CloudFilesFileMetadata
 
 
 @pytest.fixture
@@ -180,7 +180,8 @@ def file_metadata():
         ('ETAG', 'edfa12d00b779b4b37b81fe5b61b2b3f'),
         ('CONTENT-TYPE', 'text/html; charset=UTF-8'),
         ('X-TRANS-ID', 'txf876a4b088e3451d94442-00549b7c6aiad3'),
-        ('DATE', 'Thu, 25 Dec 2014 02:54:34 GMT')
+        ('DATE', 'Thu, 25 Dec 2014 02:54:34 GMT'),
+        ('NAME', 'file.txt')
     ])
 
 
@@ -664,6 +665,17 @@ async def test_v1_validate_path(self, connected_provider):
 
 class TestOperations:
 
+    @pytest.mark.asyncio
+    async def test_construct_path(self, provider, file_metadata):
+        # Construct replaces revalidate_path in some instances, so it should always
+        # be tested against it, even if calls revalidate_path
+        path = WaterButlerPath('/file.txt')
+
+        data = CloudFilesFileMetadata(file_metadata)
+        rev_path = await provider.revalidate_path(path.parent, data.name)
+        con_path = await provider.construct_path(path.parent, data)
+        assert rev_path == con_path
+
     @pytest.mark.asyncio
     @pytest.mark.aiohttpretty
     async def test_ensure_connection(self, provider, auth_json, mock_temp_key):

tests/providers/dataverse/test_provider.py

@@ -110,7 +110,6 @@ async def test_revalidate_path(self, provider, native_dataset_metadata):
                                        status=200,
                                        body=native_dataset_metadata)
 
-
         base = await provider.validate_v1_path('/')
 
         wb_path = await provider.revalidate_path(base, '/thefile.txt')
@@ -301,7 +300,7 @@ async def test_upload_checksum_mismatch(self, provider, file_stream,
         ])
 
         path = await provider.validate_path(path)
-        with pytest.raises(exceptions.UploadChecksumMismatchError) as exc:
+        with pytest.raises(exceptions.UploadChecksumMismatchError):
             await provider.upload(file_stream, path)
 
         assert aiohttpretty.has_call(method='POST', uri=url)
@@ -511,12 +510,39 @@ async def test_metadata_never_published_raises_errors(self, provider):
 
         path = await provider.validate_path('/')
         with pytest.raises(exceptions.MetadataError) as e:
-            result = await provider.metadata(path)
+            await provider.metadata(path)
 
         assert e.value.code == 400
 
 
 class TestUtils:
 
+    @pytest.mark.asyncio
+    @pytest.mark.aiohttpretty
+    async def test_construct_path(self, provider, native_dataset_metadata):
+        entry = native_dataset_metadata['data']['files'][0]['datafile']
+        metadata = DataverseFileMetadata(entry, 'latest')
+
+        draft_url = provider.build_url(dvs.JSON_BASE_URL.format(provider._id, 'latest'),
+                                       key=provider.token)
+        published_url = provider.build_url(dvs.JSON_BASE_URL.format(provider._id,
+                                                                    'latest-published'),
+                                           key=provider.token)
+
+        aiohttpretty.register_json_uri('GET',
+                                       draft_url,
+                                       status=200,
+                                       body=native_dataset_metadata)
+        aiohttpretty.register_json_uri('GET',
+                                       published_url,
+                                       status=200,
+                                       body=native_dataset_metadata)
+
+        base = await provider.validate_v1_path('/')
+
+        rev_path = await provider.revalidate_path(base, metadata.name)
+        con_path = await provider.construct_path(base, metadata)
+        assert rev_path == con_path
+
     def test_utils(self, provider):
         assert not provider.can_duplicate_names()

tests/providers/dropbox/test_provider.py

@@ -722,6 +722,18 @@ async def test_intra_move_casing_change(self, provider):
 
 class TestOperations:
 
+    @pytest.mark.asyncio
+    async def test_construct_path(self, provider, root_provider_fixtures):
+        # Construct replaces revalidate_path in some instances, so it should always
+        # be tested against it, even if calls revalidate_path
+        path = WaterButlerPath('/file.txt')
+
+        data = DropboxFileMetadata(root_provider_fixtures['file_metadata'], folder=False)
+
+        rev_path = await provider.revalidate_path(path.parent, data.name)
+        con_path = await provider.construct_path(path.parent, data)
+        assert rev_path == con_path
+
     def test_can_intra_copy(self, provider):
         assert provider.can_intra_copy(provider)
 
@@ -732,7 +744,7 @@ def test_can_intra_move(self, provider):
         assert provider.can_intra_move(provider)
 
     def test_cannot_intra_move_other(self, provider, other_provider):
-        assert provider.can_intra_move(other_provider) == False
+        assert provider.can_intra_move(other_provider) is False
 
     def test_conflict_error_handler_not_found(self, provider, error_fixtures):
         error_path = '/Photos/folder/file'

tests/providers/figshare/test_provider.py

@@ -1387,6 +1387,36 @@ async def test_article_revalidate_path_file(self, article_provider, crud_fixture
 
 class TestMisc:
 
+    @pytest.mark.asyncio
+    @pytest.mark.aiohttpretty
+    async def test_construct_path(self, project_provider, root_provider_fixtures):
+        file_article_id = str(root_provider_fixtures['list_project_articles'][0]['id'])
+        folder_article_id = str(root_provider_fixtures['list_project_articles'][1]['id'])
+
+        root_parts = project_provider.root_path_parts
+        list_articles_url = project_provider.build_url(False, *root_parts, 'articles')
+        file_article_url = project_provider.build_url(False, *root_parts, 'articles',
+                                                      file_article_id)
+        folder_article_url = project_provider.build_url(False, *root_parts, 'articles',
+                                                        folder_article_id)
+
+        aiohttpretty.register_json_uri('GET', list_articles_url,
+                                       body=root_provider_fixtures['list_project_articles'],
+                                       params={'page': '1', 'page_size': str(MAX_PAGE_SIZE)})
+        aiohttpretty.register_json_uri('GET', list_articles_url, body=[],
+                                       params={'page': '2', 'page_size': str(MAX_PAGE_SIZE)})
+        aiohttpretty.register_json_uri('GET', file_article_url,
+                                       body=root_provider_fixtures['file_article_metadata'])
+        aiohttpretty.register_json_uri('GET', folder_article_url,
+                                       body=root_provider_fixtures['folder_article_metadata'])
+
+        path = FigsharePath('/test/', _ids=('0', folder_article_id), folder=True, parent_is_folder=False)
+        base_meta = root_provider_fixtures['file_article_metadata']
+        data = metadata.FigshareFileMetadata(base_meta, base_meta['files'][0])
+        rev_path = await project_provider.revalidate_path(path, data.name, folder=False)
+        con_path = await project_provider.construct_path(path, data)
+        assert con_path == rev_path
+
     def test_path_from_metadata(self, project_provider, root_provider_fixtures):
         file_article_metadata = root_provider_fixtures['file_article_metadata']
         fig_metadata = metadata.FigshareFileMetadata(file_article_metadata)

tests/providers/filesystem/test_provider.py

@@ -24,6 +24,17 @@ def credentials():
     return {}
 
 
+@pytest.fixture
+def file_metadata():
+    return {
+        'path': '/code/website/osfstoragecache/77094244-aa24-48da-9437-d8ce6f7a94e9',
+        'modified_utc': '2017-09-20T15:16:02.601916+00:00',
+        'mime_type': None,
+        'size': 35981,
+        'modified': 'Wed, 20 Sep 2017 15:16:02 +0000'
+    }
+
+
 @pytest.fixture
 def settings(tmpdir):
     return {'folder': str(tmpdir)}
@@ -271,6 +282,14 @@ async def test_intra_move_file(self, provider):
 
 class TestOperations:
 
+    @pytest.mark.asyncio
+    async def test_construct_path(self, provider, file_metadata):
+        data = FileSystemFileMetadata(file_metadata, '/')
+        path = await provider.validate_path('/folder/')
+        con_path = await provider.construct_path(path, data)
+        rev_path = await provider.revalidate_path(path, data.name)
+        assert con_path == rev_path
+
     def test_can_duplicate_names(self, provider):
         assert provider.can_duplicate_names() is False