update onedrive to use new revisions endpoint

 * Update OneDrive provider's revisions method to use a new endpoint
   that works with all account types and change how OneNote versioning
   behaves to accommodate that. OD revisions were using the viewdelta
   endpoint which identifies revisions by etag.  Since initial
   development, Microsoft has added actual file versioning to OD.
   Revisions are not supported for OneNote files, so create a dummy
   'current' version and return that.

 * Move fixtures and adjust tests.

Author: Johnetordoff

tests/providers/onedrive/fixtures/download.json

@@ -1,11 +1,10 @@
 {
     "root_id": "F4D50E400DFE7D4E!103",
     "file_id": "F4D50E400DFE7D4E!291",
-    "file_revision": "aRjRENTBFNDAwREZFN0Q0RSEyOTEuMg",
     "onenote_id": "F4D50E400DFE7D4E!154",
     "onenote_revision": "aRjRENTBFNDAwREZFN0Q0RSExNTQuMg",
+    "onenote_revision_non_exportable": "F4D50E400DFE7D4E!154",
     "file_download_url" : "https://public.bn1303.livefilestore.com/y4mB8JhDUWbofzVglNap3rO5i6R7jOQyJAz995dPlkrOiQeOV2jgK-EOf916z8YHi9A42WCTMVfNmHjJliYLccUFzJgsEK3j3cviT2YLlZBMRVN-sC0mfvZz_ZeDgiLzfSChMmNXkRoq6Ymh_F8r8jRAvZTzJOgyX3F7jdw4qcY27tz95Rutrl68W0Z8ntuh3bVoPIDHC5kckF8sSWoyv5j4BfRQCckjyrmaV8F1BM5Cb1x10WNdE7CP_X1bBFqY7ZTJzYcsQcDR07BdalvRTDp-A",
-    "file_revision_download_url": "https://kdqyaa.bn1303.livefilestore.com/y4pIg3zgkmcBaQ_b2CpkxiYLihuF-GaqI-zBWermrthafHBogxMCjK6Q2qA_DoELVL0-oogK2WBYfx_CKyjynFBkRe61e6OsMDAfn0NEo4fSLSXamfrMRZ0-Pyf8ZgUujCNHpaihbkj2hwIlvnNJez0ZDerAEdfA7jos7JQnVfEAU2GNXGnsyx9Yrn9VC72xLmeMdDh676UTL9gpG-2xj4BX1AI2Ro7phbbB1n2kwnwNRZOaK1tusL8cyUfjs7joD4X",
     "file_content": "ten of them",
     "file_revisions": {
         "value" : [

tests/providers/onedrive/fixtures/revisions.json

@@ -1,56 +1,71 @@
 {
-    "root_id": "F4D50E400DFE7D4E!103",
-    "file_id": "F4D50E400DFE7D4E!291",
-    "file_revisions": {
-        "value" : [
-            {
-                "@content.downloadUrl" : "https://kdqyaa.bn1303.livefilestore.com/y4pIg3zgkmcBaQ_b2CpkxiYLihuF-GaqI-zBWermrthafHBogxMCjK6Q2qA_DoELVL0-oogK2WBYfx_CKyjynFBkRe61e6OsMDAfn0NEo4fSLSXamfrMRZ0-Pyf8ZgUujCNHpaihbkj2hwIlvnNJez0ZDerAEdfA7jos7JQnVfEAU2GNXGnsyx9Yrn9VC72xLmeMdDh676UTL9gpG-2xj4BX1AI2Ro7phbbB1n2kwnwNRZOaK1tusL8cyUfjs7joD4X",
-                "createdDateTime" : "2017-08-17T17:49:39.613Z",
-                "fileSystemInfo" : {
-                    "lastModifiedDateTime" : "2017-08-17T17:49:50.363Z",
-                    "createdDateTime" : "2017-08-17T17:49:39.613Z"
-                },
-                "name" : "toes.txt",
-                "size" : 11,
-                "eTag" : "aRjRENTBFNDAwREZFN0Q0RSEyOTEuMg",
-                "cTag" : "aYzpGNEQ1MEU0MDBERkU3RDRFITI5MS4yNTg",
-                "createdBy" : {
-                    "user" : {
-                        "displayName" : "Fitz Elliott",
-                        "id" : "f4d50e400dfe7d4e"
-                    },
-                    "application" : {
-                        "id" : "481710a4"
-                    }
-                },
-                "lastModifiedDateTime" : "2017-08-17T17:49:50.38Z",
-                "file" : {
-                    "mimeType" : "text/plain",
-                    "hashes" : {
-                        "sha1Hash" : "D6FAC576DCF80198874C9C9476F021AF3F12688C"
-                    }
-                },
-                "webUrl" : "https://1drv.ms/t/s!AE59_g1ADtX0giM",
-                "id" : "F4D50E400DFE7D4E!291",
-                "parentReference" : {
-                    "id" : "F4D50E400DFE7D4E!103",
-                    "name" : "root:",
-                    "path" : "/drive/root:",
-                    "driveId" : "f4d50e400dfe7d4e"
-                },
-                "lastModifiedBy" : {
-                    "user" : {
-                        "id" : "f4d50e400dfe7d4e",
-                        "displayName" : "Fitz Elliott"
-                    },
-                    "application" : {
-                        "id" : "481710a4"
-                    }
-                }
-            }
-        ],
-        "@odata.context" : "https://api.onedrive.com/v1.0/$metadata#drives('me')/items",
-        "@odata.deltaLink" : "https://api.onedrive.com/v1.0/drives('me')/items('F4D50E400DFE7D4E!291')/view.delta?$top=250&token=aTE09NjM2Mzg1OTM3MTAzMzc7SUQ9RjRENTBFNDAwREZFN0Q0RSEyOTE7TFI9NjM2Mzg5MTk0MDUyMTA7RVA9MTY7U0k9ODA7U0c9MTtTTz0yO1BJPTM",
-        "@delta.token" : "aTE09NjM2Mzg1OTM3MTAzMzc7SUQ9RjRENTBFNDAwREZFN0Q0RSEyOTE7TFI9NjM2Mzg5MTk0MDUyMTA7RVA9MTY7U0k9ODA7U0c9MTtTTz0yO1BJPTM"
-    }
-}
+   "root_id":"F4D50E400DFE7D4E!103",
+   "file_id":"F4D50E400DFE7D4E!291",
+   "revision_id":"107!70",
+   "file_revision_download_url": "https://public.ch.files.1drv.com/y4mMH_4HkyaoHu8_0kFg_Q4bTRlfZiB9OyWlZU6ZFe3mLCFm-r2F5YWV9nOmvvF9I04mT2cnK8S5YZP5cgPtM3jAIcoPugtolMMCwFWf3_ZC_cXcsDWN4gZAqhf07TFoIqvebghUGwEYBwPAnrJ0kM0sa8mrf4eQqK8lEjwXOHnk-FvvPqyLJca1u6_x_cRNtV_wLsM-P97TRWJGqUFKY4uwRofjycO-HfdUm5cOtSHWaHBrymv8dc485fEVq1RVSf2-0Rs1_tl0oqDEaH2An0DgQ",
+   "file_revisions":{
+      "@odata.context":"https://api.onedrive.com/v1.0/$metadata#drives('me'/items('81ACFE813AEB6C20%213910')/versions",
+      "value":[
+         {
+            "@content.downloadUrl":"https://public.ch.files.1drv.com/y4mdCMJd_GU7BsM49MAk6eaWJLcAahSXbD3ACWHlLw6xBLRVhZxH8d-4HKIsq3Qr5eVqIadN0fUaltRxSUsREoQ9NY9aw-FWGQAJDpytApMZIkHQYE02m_pRHfSLVTJAlL-2t8N3MVz_8GL1Zfhs1rcRWEmQUX_159Xaj1FYJLv13BSJSQJOv677x_jZQiZSSgvr-WnjyvMtoH2ZzMUAJxevHFHvFDduP266QhSWl8kX_feLRyxnhG96-gtbNeJbwJ_",
+            "id":"current",
+            "lastModifiedBy":{
+               "user":{
+                  "displayName":"John Tordoff",
+                  "id":"81acfe813aeb6c20"
+               }
+            },
+            "lastModifiedDateTime":"2017-11-30T15:42:33.447Z",
+            "size":12
+         },
+         {
+            "@content.downloadUrl":"https://public.ch.files.1drv.com/y4mMH_4HkyaoHu8_0kFg_Q4bTRlfZiB9OyWlZU6ZFe3mLCFm-r2F5YWV9nOmvvF9I04mT2cnK8S5YZP5cgPtM3jAIcoPugtolMMCwFWf3_ZC_cXcsDWN4gZAqhf07TFoIqvebghUGwEYBwPAnrJ0kM0sa8mrf4eQqK8lEjwXOHnk-FvvPqyLJca1u6_x_cRNtV_wLsM-P97TRWJGqUFKY4uwRofjycO-HfdUm5cOtSHWaHBrymv8dc485fEVq1RVSf2-0Rs1_tl0oqDEaH2An0DgQ",
+            "id":"107!70",
+            "lastModifiedBy":{
+               "user":{
+                  "displayName":"John Tordoff",
+                  "id":"81acfe813aeb6c20"
+               }
+            },
+            "lastModifiedDateTime":"2017-11-30T15:15:04.453Z",
+            "size":6
+         },
+         {
+            "@content.downloadUrl":"https://public.ch.files.1drv.com/y4mrCC1HUStA2PX-L6-pKFuekc9sezEnTqwymnJUtP7kkVXrckReyMUo7I-fz-lR-dTvnZP0gFsE5h6KOV1jwrXqlklo0fw7PITKJ3h_xnox6M97jysra8eIml1uTJrnAKWCDkuC2PL0cwoVZ0Tzwbg6E7Wc3MeIPhdwVLHFEy36QJiOn9ISq12KfATPG-kc6utS31RQlWgGd5BThVyFRsJ2RHW5vT78BfGUYijxNPZEjE-MEQRkjasqcHMnE4s2p_KrcYT6fecXpdPsRrm6ysOMA",
+            "id":"104!70",
+            "lastModifiedBy":{
+               "user":{
+                  "displayName":"John Tordoff",
+                  "id":"81acfe813aeb6c20"
+               }
+            },
+            "lastModifiedDateTime":"2017-11-30T15:13:18.643Z",
+            "size":5
+         },
+         {
+            "@content.downloadUrl":"https://public.ch.files.1drv.com/y4m26zI4Ycikb_c683UBuLdD7WU85eH3lk_ikfWZXrG5q-z16tFB8Y83ERSc_LcsboC2fMIbVZ19xdESP151xmi7kYZpfy0yzCPzvMwS9wF__m6Uu0Lycq3Ocq3Yb1MSphHD7cZn1foExV9cdYyyVoizLXSx-XHu3nTRUuDjZZ13jx7OlTsvEoCkZhJPO_G1yWBTuPpewHQCh5ayG-4g_srhx1dYV1dL5r4e2x-1Oxz4rRdQNiFAHLX-tcvX07_bBD95jNuiEcn6pNAFAQCvR_iXw",
+            "id":"102!70",
+            "lastModifiedBy":{
+               "user":{
+                  "displayName":"John Tordoff",
+                  "id":"81acfe813aeb6c20"
+               }
+            },
+            "lastModifiedDateTime":"2017-11-30T15:08:32.893Z",
+            "size":4
+         },
+         {
+            "@content.downloadUrl":"https://public.ch.files.1drv.com/y4mE4L0ez-GhBbcx-GwiM25Th2xGinltbyKDbZu1f5t6eEDHNOwlLS0RnUJW2u5QDL5IOga1sEt57y7V53hheTdxcPGJTZIaVoSgaRWW2sBprhdqc-o36QAgY104aEMurXqbPGtxYAA-gWu3Ol7qjHKc7ykC2v5kF_JzIuIRqoprrRZvwy1of_d-czQfejZZ3k50Q734Alc_jO-X5ZpmwwUndM3twvzfCwgzynCkujSqtzffdSOrDYuarhFtqOVas_i95-MgJegeIbAiAalPDHfog",
+            "id":"101!70",
+            "lastModifiedBy":{
+               "user":{
+                  "displayName":"John Tordoff",
+                  "id":"81acfe813aeb6c20"
+               }
+            },
+            "lastModifiedDateTime":"2017-11-30T15:08:22.987Z",
+            "size":3
+         }
+      ]
+   }
+}

tests/providers/onedrive/test_metadata.py

@@ -70,8 +70,8 @@ def test_build_revision_metadata(self, revision_fixtures):
 
         assert metadata.serialized() == {
             'extra': {},
-            'version': 'aRjRENTBFNDAwREZFN0Q0RSEyOTEuMg',
-            'modified': '2017-08-17T17:49:50.38Z',
-            'modified_utc': '2017-08-17T17:49:50+00:00',
+            'version': 'current',
+            'modified': '2017-11-30T15:42:33.447Z',
+            'modified_utc': '2017-11-30T15:42:33+00:00',
             'versionIdentifier': 'revision',
         }

tests/providers/onedrive/test_provider.py

@@ -409,13 +409,12 @@ async def test_get_revisions(self, provider, revision_fixtures):
         path = OneDrivePath('/bicuspids.txt', _ids=[revision_fixtures['root_id'], file_id])
 
         revision_response = revision_fixtures['file_revisions']
-        revisions_url = provider._build_drive_url('items', file_id, 'view.delta',
-                                                  top=provider.MAX_REVISIONS)
+        revisions_url = provider._build_drive_url(*path.api_identifier, 'versions')
         aiohttpretty.register_json_uri('GET', revisions_url, body=revision_response)
 
         result = await provider.revisions(path)
 
-        assert len(result) == 1
+        assert len(result) == 5
 
 
 class TestDownload:
@@ -440,20 +439,19 @@ async def test_download_standard_file(self, provider, download_fixtures):
 
     @pytest.mark.asyncio
     @pytest.mark.aiohttpretty
-    async def test_download_by_revision(self, provider, download_fixtures):
+    async def test_download_by_revision(self, provider, download_fixtures, revision_fixtures):
         file_id = download_fixtures['file_id']
         path = OneDrivePath('/toes.txt', _ids=[download_fixtures['root_id'], file_id])
 
-        revision_response = download_fixtures['file_revisions']
-        revisions_url = provider._build_drive_url('items', file_id, 'view.delta',
-                                                  top=provider.MAX_REVISIONS)
+        revision_response = revision_fixtures['file_revisions']
+        revisions_url = provider._build_drive_url(*path.api_identifier, 'versions')
         aiohttpretty.register_json_uri('GET', revisions_url, body=revision_response)
 
-        aiohttpretty.register_uri('GET', download_fixtures['file_revision_download_url'],
+        aiohttpretty.register_uri('GET', revision_fixtures['file_revision_download_url'],
                                   body=download_fixtures['file_content'],
                                   headers={'Content-Length': '11'})
 
-        response = await provider.download(path, revision=download_fixtures['file_revision'])
+        response = await provider.download(path, revision=revision_fixtures['revision_id'])
         content = await response.read()
         assert content == b'ten of them'
 
@@ -470,8 +468,7 @@ async def test_download_by_bad_revision(self, provider, download_fixtures):
         path = OneDrivePath('/toes.txt', _ids=[download_fixtures['root_id'], file_id])
 
         revision_response = download_fixtures['file_revisions']
-        revisions_url = provider._build_drive_url('items', file_id, 'view.delta',
-                                                  top=provider.MAX_REVISIONS)
+        revisions_url = provider._build_drive_url(*path.api_identifier, 'versions')
         aiohttpretty.register_json_uri('GET', revisions_url, body=revision_response)
 
         with pytest.raises(exceptions.NotFoundError) as exc:
@@ -497,12 +494,12 @@ async def test_download_unexportable_by_revision(self, provider, download_fixtur
         path = OneDrivePath('/onenote', _ids=[download_fixtures['root_id'], onenote_id])
 
         revision_response = download_fixtures['onenote_revisions']
-        revisions_url = provider._build_drive_url('items', onenote_id, 'view.delta',
-                                                  top=provider.MAX_REVISIONS)
+        revisions_url = provider._build_drive_url('items', onenote_id, 'versions')
         aiohttpretty.register_json_uri('GET', revisions_url, body=revision_response)
 
         with pytest.raises(exceptions.UnexportableFileTypeError) as exc:
-            await provider.download(path, revision=download_fixtures['onenote_revision'])
+            await provider.download(path,
+                                    revision=download_fixtures['onenote_revision_non_exportable'])
 
 
 class TestReadOnlyProvider:

waterbutler/providers/onedrive/metadata.py

@@ -114,7 +114,7 @@ def version_identifier(self):
 
     @property
     def version(self):
-        return self.raw['eTag']
+        return self.raw['id']
 
     @property
     def modified(self):

waterbutler/providers/onedrive/provider.py

@@ -288,7 +288,6 @@ async def revisions(self,  # type: ignore
         return [
             OneDriveRevisionMetadata(item)
             for item in data['value']
-            if not item.get('deleted')
         ]
 
     async def download(self,  # type: ignore
@@ -318,7 +317,7 @@ async def download(self,  # type: ignore
         if revision:
             items = await self._revisions_json(path)
             for item in items['value']:
-                if item['eTag'] == revision:
+                if item['id'] == revision:
                     try:
                         download_url = item['@content.downloadUrl']
                     except KeyError:
@@ -413,19 +412,40 @@ def _construct_metadata(self, data: dict):
     async def _revisions_json(self, path: OneDrivePath, **kwargs) -> dict:
         """Fetch a list of revisions for the file at ``path``.
 
-        API docs: https://dev.onedrive.com/items/view_delta.htm
+        API docs: https://docs.microsoft.com/en-us/onedrive/developer/rest-api/resources/driveitemversion
 
         :param OneDrivePath path: the path of the file to get revisions for
         :rtype: dict
         :return: list of revision metadata under a ``value`` key
         """
+        try:
+            resp = await self.make_request(
+                'GET',
+                self._build_drive_url(*path.api_identifier, 'versions'),
+                expects=(200, ),
+                throws=exceptions.RevisionsError
+            )
+        except exceptions.RevisionsError as exc:
+            if (
+                    exc.code == 405 and
+                    exc.data['error']['message'] == 'Item does not match expected type'
+            ):
+                # OneNote versioning not supported, instead return a lone revision called 'current'
+                url = self._build_drive_url(*path.api_identifier)
+                resp = await self.make_request(
+                    'GET',
+                    url,
+                    expects=(200,),
+                    throws=exceptions.MetadataError
+                )
+                logger.debug("metadata resp::{}".format(repr(resp)))
+                data = await resp.json()
+                return {'value': [
+                    {'id': 'current', 'lastModifiedDateTime': data['lastModifiedDateTime']},
+                ]}  # fudge a fake revision response
+            else:
+                raise exc
 
-        resp = await self.make_request(
-            'GET',
-            self._build_drive_url(*path.api_identifier, 'view.delta', top=self.MAX_REVISIONS),
-            expects=(200, ),
-            throws=exceptions.RevisionsError
-        )
         logger.debug('_revisions_json: resp::{}'.format(repr(resp)))
         data = await resp.json()
         logger.debug('_revisions_json: data::{}'.format(json.dumps(data)))