Merge pull request #793 from hubmapconsortium/karlburke/NewProvMetadataEndpoint

Initial #781 changes, exclude nested fields from public responses

Author: yuanzhou

src/app.py

@@ -797,13 +797,6 @@ def get_entity_by_id(id):
         # Response with the dict
         if public_entity and not user_in_hubmap_read_group(request):
             final_result = schema_manager.exclude_properties_from_response(fields_to_exclude, final_result)
-        if normalized_entity_type == 'Collection':
-            for i, dataset in enumerate(final_result.get('datasets', [])):
-                if _get_entity_visibility(normalized_entity_type='Dataset', entity_dict=dataset) != DataVisibilityEnum.PUBLIC or user_in_hubmap_read_group(request):
-                    # If the dataset is non-public, or if the user has read-group access, there is no need to remove fields, continue to the next dataset
-                    continue
-                dataset_excluded_fields = schema_manager.get_fields_to_exclude('Dataset')
-                final_result.get('datasets')[i] = schema_manager.exclude_properties_from_response(dataset_excluded_fields, dataset)
         return jsonify(final_result)
 
 """

src/dev_entity_worker.py

@@ -329,15 +329,6 @@ def _get_entity_by_id_for_auth_level(self, entity_id:Annotated[str, 32], valid_u
         #if public_entity and not user_in_hubmap_read_group(request):
         if public_entity and not user_authorized:
             final_result = self.schemaMgr.exclude_properties_from_response(fields_to_exclude, final_result)
-        if normalized_entity_type == 'Collection':
-            for i, dataset in enumerate(final_result.get('datasets', [])):
-                if self._get_entity_visibility( entity_dict=dataset) != DataVisibilityEnum.PUBLIC \
-                        or user_authorized: # or user_in_hubmap_read_group(request):
-                    # If the dataset is public, or if the user has read-group access, there is
-                    # no need to remove fields, continue to the next dataset
-                    continue
-                dataset_excluded_fields = self.schemaMgr.get_fields_to_exclude('Dataset')
-                final_result.get('datasets')[i] = self.schemaMgr.exclude_properties_from_response(dataset_excluded_fields, dataset)
         return final_result
 
     '''

src/schema/provenance_schema.yaml

@@ -192,6 +192,11 @@ shared_entity_properties: &shared_entity_properties
 ENTITIES:
   ############################################# Collection #############################################
   Collection:
+    excluded_properties_from_public_response:
+      - datasets:
+        - lab_dataset_id
+        - metadata:
+          - lab_id
     # Collection can not be derivation source but not target
     derivation:
       source: false
@@ -304,6 +309,15 @@ ENTITIES:
       - lab_dataset_id
       - metadata:
         - lab_id
+      - direct_ancestors:
+        # Sample ancestors of a Dataset must have these fields removed
+        - lab_tissue_sample_id
+        - submission_id
+        # Dataset ancestors of a Dataset must have these fields removed
+        - lab_dataset_id
+        # Both Sample and Dataset ancestors of a Dataset must have these fields removed
+        - metadata:
+          - lab_id
     derivation:
       source: true
       target: true
@@ -870,6 +884,18 @@ ENTITIES:
     excluded_properties_from_public_response:
       - lab_tissue_sample_id
       - submission_id
+      - metadata:
+        - lab_id
+      - direct_ancestor:
+        # Donor ancestors of a Sample must have these fields removed
+        - lab_donor_id
+        - label
+        # Sample ancestors of a Sample must have these fields removed
+        - lab_tissue_sample_id
+        - metadata:
+          - lab_id
+        # Both Sample and Donor ancestors of a Sample must have these fields removed
+        - submission_id
     properties:
       <<: *shared_properties
       <<: *shared_entity_properties

src/schema/schema_manager.py

@@ -306,13 +306,26 @@ def delete_nested_field(data, nested_path):
                     if isinstance(value, list):
                         for nested_field in value:
                             if isinstance(nested_field, dict):
-                                delete_nested_field(data[key], nested_field)
+                                if isinstance(data[key], list):
+                                    for item in data[key]:
+                                        delete_nested_field(item, nested_field)
+                                else:
+                                    delete_nested_field(data[key], nested_field)
+                            elif isinstance(data[key], list):
+                                for item in data[key]:
+                                    if nested_field in item:
+                                        del item[nested_field]
                             elif nested_field in data[key]:
                                 del data[key][nested_field]
                     elif isinstance(value, dict):
                         delete_nested_field(data[key], value)
         elif nested_path in data:
-            del data[nested_path]
+            if isinstance(data[nested_path], list):
+                for item in data[nested_path]:
+                    if nested_path in item:
+                        del item[nested_path]
+            else:
+                del data[nested_path]
 
     for field in excluded_fields:
         delete_nested_field(output_dict, field)