[a] Add mirror URI to service /index/file response (#7624)

Author: nadove-ucsc

lambdas/service/app.py

@@ -121,7 +121,7 @@
         # changes and reset the minor version to zero. Otherwise, increment only
         # the minor version for backwards compatible changes. A backwards
         # compatible change is one that does not require updates to clients.
-        'version': '15.0',
+        'version': '15.1',
         'description': fd(f'''
             # Overview
 

lambdas/service/openapi.json

@@ -2,7 +2,7 @@
     "openapi": "3.0.1",
     "info": {
         "title": "azul-service-dev",
-        "version": "15.0",
+        "version": "15.1",
         "description": "\n# Overview\n\nAzul is a REST web service for querying metadata associated with\nboth experimental and analysis data from a data repository. In order\nto deliver response times that make it suitable for interactive use\ncases, the set of metadata properties that it exposes for sorting,\nfiltering, and aggregation is limited. Azul provides a uniform view\nof the metadata over a range of diverse schemas, effectively\nshielding clients from changes in the schemas as they occur over\ntime. It does so, however, at the expense of detail in the set of\nmetadata properties it exposes and in the accuracy with which it\naggregates them.\n\nAzul denormalizes and aggregates metadata into several different\nindices for selected entity types. Metadata entities can be queried\nusing the [Index](#operations-tag-Index) endpoints.\n\nA set of indices forms a catalog. There is a default catalog called\n`dcp2` which will be used unless a\ndifferent catalog name is specified using the `catalog` query\nparameter. Metadata from different catalogs is completely\nindependent: a response obtained by querying one catalog does not\nnecessarily correlate to a response obtained by querying another\none. Two catalogs can contain metadata from the same sources or\ndifferent sources. It is only guaranteed that the body of a\nresponse by any given endpoint adheres to one schema,\nindependently of which catalog was specified in the request.\n\nAzul provides the ability to download data and metadata via the\n[Manifests](#operations-tag-Manifests) endpoints. The\n`curl` format manifests can be used to\ndownload data files. Other formats provide various views of the\nmetadata. Manifests can be generated for a selection of files using\nfilters. These filters are interchangeable with the filters used by\nthe [Index](#operations-tag-Index) endpoints.\n\nAzul also provides a [summary](#operations-Index-get_index_summary)\nview of indexed data.\n\n## Data model\n\nAny index, when queried, returns a JSON array of hits. Each hit\nrepresents a metadata entity. Nested in each hit is a summary of the\nproperties of entities associated with the hit. An entity is\nassociated either by a direct edge in the original metadata graph,\nor indirectly as a series of edges. The nested properties are\ngrouped by the type of the associated entity. The properties of all\ndata files associated with a particular sample, for example, are\nlisted under `hits[*].files` in a `/index/samples` response. It is\nimportant to note that while each _hit_ represents a discrete\nentity, the properties nested within that hit are the result of an\naggregation over potentially many associated entities.\n\nTo illustrate this, consider a data file that is part of two\nprojects (a project is a group of related experiments, typically by\none laboratory, institution or consortium). Querying the `files`\nindex for this file yields a hit looking something like:\n\n```\n{\n    \"projects\": [\n        {\n            \"projectTitle\": \"Project One\"\n            \"laboratory\": ...,\n            ...\n        },\n        {\n            \"projectTitle\": \"Project Two\"\n            \"laboratory\": ...,\n            ...\n        }\n    ],\n    \"files\": [\n        {\n            \"format\": \"pdf\",\n            \"name\": \"Team description.pdf\",\n            ...\n        }\n    ]\n}\n```\n\nThis example hit contains two kinds of nested entities (a hit in an\nactual Azul response will contain more): There are the two projects\nentities, and the file itself. These nested entities contain\nselected metadata properties extracted in a consistent way. This\nmakes filtering and sorting simple.\n\nAlso notice that there is only one file. When querying a particular\nindex, the corresponding entity will always be a singleton like\nthis.\n\n\n## Contact us\n\nFor technical support please file an issue at\n[GitHub](https://github.com/DataBiosphere/azul/issues) or email\n`azul-group@ucsc.edu`. To report a security concern or misconduct please email\n`azul-group@ucsc.edu`.\n"
     },
     "tags": [

src/azul/plugins/metadata/anvil/service/response.py

@@ -218,6 +218,11 @@ def _pivotal_entity(self,
             inner_entity['azul_url'] = self._file_url(uuid=json_str(inner_entity['document_id']),
                                                       version=json_str(inner_entity['version']),
                                                       drs_uri=optional(json_str, inner_entity['drs_uri']))
+<<<<<<< HEAD
+=======
+            inner_entity['azul_mirror_uri'] = self._file_mirror_uri(inner_entity)
+            inner_entity.pop('uuid', None)
+>>>>>>> 5e0b949b8 ([a] Add mirror URI to service /index/file response (#7624))
             inner_entity.pop('version', None)
         return inner_entity
 

src/azul/plugins/metadata/hca/service/response.py

@@ -416,7 +416,8 @@ def make_file(self, file: JSON) -> JSON:
             'drs_uri': file.get('drs_uri'),
             'azul_url': self._file_url(uuid=json_str(file['uuid']),
                                        version=json_str(file['version']),
-                                       drs_uri=optional(json_str, file['drs_uri']))
+                                       drs_uri=optional(json_str, file['drs_uri'])),
+            'azul_mirror_uri': self._file_mirror_uri(file),
         }
         return translated_file
 

src/azul/service/__init__.py

@@ -36,6 +36,7 @@
 )
 from azul.plugins import (
     FieldName,
+    File,
     MetadataPlugin,
 )
 from azul.types import (
@@ -433,3 +434,8 @@ def __call__(self,
                  fetch: bool = True,
                  **params: str
                  ) -> mutable_furl: ...
+
+
+class FileMirrorUriFunc(Protocol):
+
+    def __call__(self, file: File) -> str: ...

src/azul/service/repository_controller.py

@@ -107,6 +107,7 @@ def search(self,
             response = self.service.search(catalog=catalog,
                                            entity_type=entity_type,
                                            file_url_func=self.file_url_func,
+                                           file_mirror_uri_func=self.mirror_service(catalog).mirror_uri,
                                            item_id=item_id,
                                            filters=filters,
                                            pagination=pagination)

src/azul/service/repository_service.py

@@ -32,13 +32,17 @@
     CatalogName,
     config,
 )
+from azul.indexer.mirror_service import (
+    BaseMirrorService,
+)
 from azul.plugins import (
     File,
     RepositoryPlugin,
     dotted,
 )
 from azul.service import (
     BadArgumentException,
+    FileMirrorUriFunc,
     FileUrlFunc,
     Filters,
 )
@@ -73,6 +77,7 @@ def __init__(self, entity_type: str, entity_id: str):
 class SearchResponseStage(_ElasticsearchStage[ResponseTriple, MutableJSON],
                           metaclass=ABCMeta):
     file_url_func: FileUrlFunc
+    file_mirror_uri_func: FileMirrorUriFunc
 
     def prepare_request(self, request: Search) -> Search:
         return request
@@ -93,6 +98,13 @@ def _file_url(self, *, uuid: str, version: str, drs_uri: str | None) -> str | No
                                           file_uuid=uuid,
                                           version=version))
 
+    def _file_mirror_uri(self, file: JSON) -> str | None:
+        file = self.plugin.file_class.from_index(file)
+        if BaseMirrorService.may_mirror(self.catalog, file.size):
+            return self.file_mirror_uri_func(file)
+        else:
+            return None
+
 
 class SummaryResponseStage(ElasticsearchStage[JSON, MutableJSON],
                            metaclass=ABCMeta):
@@ -113,6 +125,7 @@ def search(self,
                catalog: CatalogName,
                entity_type: str,
                file_url_func: FileUrlFunc,
+               file_mirror_uri_func: FileMirrorUriFunc,
                item_id: str | None,
                filters: Filters,
                pagination: Pagination
@@ -127,6 +140,8 @@ def search(self,
         :param file_url_func: A function that is used only when getting a *list* of files data.
         It creates the files URL based on info from the request. It should have the type
         signature `(uuid: str, **params) -> str`
+        :param file_mirror_uri_func: Like `file_url_func`, but creates URIs to
+        where the files are mirrored. It should have the type signature `(File) -> str`.
         :return: The Elasticsearch JSON response
         """
         if item_id is not None:
@@ -138,7 +153,8 @@ def search(self,
                                 pagination=pagination,
                                 aggregate=item_id is None,
                                 entity_type=entity_type,
-                                file_url_func=file_url_func)
+                                file_url_func=file_url_func,
+                                file_mirror_uri_func=file_mirror_uri_func)
 
         special_fields = self.metadata_plugin(catalog).special_fields
         for hit in response['hits']:
@@ -158,7 +174,8 @@ def _search(self,
                 aggregate: bool,
                 filters: Filters,
                 pagination: Pagination,
-                file_url_func: FileUrlFunc
+                file_url_func: FileUrlFunc,
+                file_mirror_uri_func: FileMirrorUriFunc,
                 ) -> MutableJSON:
         """
         This function does the whole transformation process. It takes the path
@@ -217,7 +234,8 @@ def _search(self,
         chain = response_stage_cls(service=self,
                                    catalog=catalog,
                                    entity_type=entity_type,
-                                   file_url_func=file_url_func).wrap(chain)
+                                   file_url_func=file_url_func,
+                                   file_mirror_uri_func=file_mirror_uri_func).wrap(chain)
 
         request = self.create_request(catalog, entity_type)
         request = chain.prepare_request(request)

test/service/test_response.py

@@ -42,6 +42,9 @@
 from azul.collections import (
     none_safe_key,
 )
+from azul.deployment import (
+    aws,
+)
 from azul.indexer import (
     BundleFQID,
     Prefix,
@@ -139,6 +142,11 @@ def tearDownClass(cls):
     def file_url_func(self):
         return self._app.file_url
 
+    @property
+    def file_mirror_uri_func(self):
+        mirror_service = self._app.repository_controller.mirror_service(self.catalog)
+        return mirror_service.mirror_uri
+
     def _get_hits(self, entity_type: str, entity_id: str):
         """
         Fetches hits from ES instance searching for a particular entity ID
@@ -167,7 +175,8 @@ def _response_stage(self, entity_type: str) -> HCASearchResponseStage:
         return HCASearchResponseStage(service=self.index_service,
                                       entity_type=entity_type,
                                       catalog=self.catalog,
-                                      file_url_func=self.file_url_func
+                                      file_url_func=self.file_url_func,
+                                      file_mirror_uri_func=self.file_mirror_uri_func)
 
     @property
     def paginations(self):
@@ -247,6 +256,8 @@ def test_response_stage_files(self):
                         'azul_url': f'{self.base_url}/repository/files/'
                                     f'7b07f99e-4a8a-4ad0-bd4f-db0d7a00c7bb'
                                     f'?catalog=test&version=2018-11-02T11%3A33%3A44.698028Z',
+                        'azul_mirror_uri': f's3://{aws.mirror_bucket}/file/'
+                                           f'77337cb51b2e584b5ae1b99db6c163b988cbc5b894dda2f5d22424978c3bfc7a.sha256',
                         'drs_uri': f'drs://{self._drs_domain_name}/'
                                    f'7b07f99e-4a8a-4ad0-bd4f-db0d7a00c7bb?version=2018-11-02T11%3A33%3A44.698028Z',
                         'uuid': '7b07f99e-4a8a-4ad0-bd4f-db0d7a00c7bb',
@@ -976,6 +987,8 @@ def test_response_stage_files_file(self):
             'azul_url': f'{self.base_url}/repository/files/'
                         f'a8b8479d-cfa9-4f74-909f-49552439e698'
                         f'?catalog=test&version=2019-10-09T17%3A22%3A51.560099Z',
+            'azul_mirror_uri': f's3://{aws.mirror_bucket}/file/'
+                               f'709fede4736213f0f71ae4d76719fd51fa402a9112582a4c52983973cb7d7e47.sha256',
             'drs_uri': f'drs://{self._drs_domain_name}/'
                        f'a8b8479d-cfa9-4f74-909f-49552439e698?version=2019-10-09T17%3A22%3A51.560099Z',
             'uuid': 'a8b8479d-cfa9-4f74-909f-49552439e698',
@@ -2958,7 +2971,10 @@ def test_matrices_tree(self):
                                                 'azul_url': str(self.base_url.set(
                                                     path='/repository/files/bd98f428-881e-501a-ac16-24f27a68ce2f',
                                                     args=dict(catalog='test', version='2021-02-11T23:11:45.000000Z')
-                                                ))
+                                                )),
+                                                'azul_mirror_uri': f's3://{aws.mirror_bucket}/file/'
+                                                                   f'6a6483c2e78da77017e912a4d350f141'
+                                                                   f'bda1ec7b269f20ca718b55145ee5c83c.sha256'
                                             }
                                         ]
                                     }
@@ -2990,7 +3006,10 @@ def test_matrices_tree(self):
                                                 'azul_url': str(self.base_url.set(
                                                     path='/repository/files/538faa28-3235-5e4b-a998-5672e2d964e8',
                                                     args=dict(catalog='test', version='2020-12-03T10:39:17.144517Z')
-                                                ))
+                                                )),
+                                                'azul_mirror_uri': f's3://{aws.mirror_bucket}/file/'
+                                                                   f'edb8e0139fece9702d89ae5fe7f761c4'
+                                                                   f'1c291ef6a71129c6420857e025228a24.sha256',
                                             },
                                             {
                                                 # Supplementary file, source from submitter_id
@@ -3012,7 +3031,10 @@ def test_matrices_tree(self):
                                                 'azul_url': str(self.base_url.set(
                                                     path='/repository/files/6c142250-567c-5b63-bd4f-0d78499863f8',
                                                     args=dict(catalog='test', version='2020-12-03T10:39:17.144517Z')
-                                                ))
+                                                )),
+                                                'azul_mirror_uri': f's3://{aws.mirror_bucket}/file/'
+                                                                   f'cb1467f4d23a2429b4928943b51652b3'
+                                                                   f'2edb949099250d28cf400d13074f5440.sha256',
                                             },
                                             {
                                                 # Supplementary file, source from submitter_id
@@ -3034,7 +3056,10 @@ def test_matrices_tree(self):
                                                 'azul_url': str(self.base_url.set(
                                                     path='/repository/files/8d2ba1c1-bc9f-5c2a-a74d-fe5e09bdfb18',
                                                     args=dict(catalog='test', version='2020-12-03T10:39:17.144517Z')
-                                                ))
+                                                )),
+                                                'azul_mirror_uri': f's3://{aws.mirror_bucket}/file/'
+                                                                   f'724b2c0ddf33c662b362179bc6ca90cd'
+                                                                   f'866b99b340d061463c35d27cfd5a23c5.sha256',
                                             }
                                         ]
                                     }
@@ -3757,6 +3782,8 @@ def test_contributed_analyses_matrix(self):
                 version='2022-07-26T00:16:47.748000Z'
             )
         ))
+        mirror_uri = (f's3://{aws.mirror_bucket}/file/'
+                      f'649c45bd2f01b028c974c7e2a9604b9cf564d8afcf528eb299eaf3d7fe92bae3.sha256')
         expected_file = {
             'contentDescription': ['Count matrix', 'Feature table'],
             'format': 'csv.gz',
@@ -3772,6 +3799,7 @@ def test_contributed_analyses_matrix(self):
                        '_541cc0bb-c54f-4a7e-8cdd-1a70cbd2f20c'
                        '_596c26ba-2c35-4396-8c7c-50c825eb4e75',
             'azul_url': file_url,
+            'azul_mirror_uri': mirror_uri,
         }
         expected_tree = {
             'genusSpecies': {'Homo sapiens': {

test/service/test_response_anvil.py

@@ -1,5 +1,8 @@
 import requests
 
+from azul.deployment import (
+    aws,
+)
 from azul.logging import (
     configure_test_logging,
 )
@@ -1271,7 +1274,8 @@ def test_entity_indices(self):
                                 'azul_url': str(self.base_url.set(
                                     path='/repository/files/15b76f9c-6b46-433f-851d-34e89f1b9ba6',
                                     args=dict(catalog='test', version=self.version)
-                                ))
+                                )),
+                                'azul_mirror_uri': f's3://{aws.mirror_bucket}/file/beec606ee0aa299fdf913f4259316622.md5'
                             }
                         ]
                     },
@@ -1349,7 +1353,8 @@ def test_entity_indices(self):
                                 'azul_url': str(self.base_url.set(
                                     path='/repository/files/3b17377b-16b1-431c-9967-e5d01fc5923f',
                                     args=dict(catalog='test', version=self.version)
-                                ))
+                                )),
+                                'azul_mirror_uri': f's3://{aws.mirror_bucket}/file/7cd9fd7b54a8bf380e44e93706f1fa2d.md5'
                             }
                         ]
                     }