[r] Remove AnVIL fields from aggregation (#6793)

Author: dsotirho-ucsc

src/azul/plugins/metadata/anvil/__init__.py

@@ -314,12 +314,26 @@ def manifest_config(self) -> ManifestConfig:
         # Note that there is a brittle coupling that must be maintained between
         # the fields listed here and those used in `self._field_mapping`.
         fields_to_omit_from_manifest: list[FieldPath] = [
+            ('contents', 'activities', 'activity_id'),
             ('contents', 'activities', 'activity_table'),
+            ('contents', 'activities', 'document_id'),
+            ('contents', 'activities', 'source_datarepo_row_ids'),
+            ('contents', 'biosamples', 'biosample_id'),
+            ('contents', 'biosamples', 'document_id'),
+            ('contents', 'biosamples', 'source_datarepo_row_ids'),
+            ('contents', 'datasets', 'document_id'),
             # We omit the `duos_id` field from manifests since there is only one
             # DUOS bundle per dataset, and that bundle only contributes to outer
             # entities of the `datasets` type, not to entities of the other
             # types, such as files, which the manifest is generated from.
             ('contents', 'datasets', 'duos_id'),
+            ('contents', 'datasets', 'source_datarepo_row_ids'),
+            ('contents', 'diagnoses', 'diagnosis_id'),
+            ('contents', 'diagnoses', 'document_id'),
+            ('contents', 'diagnoses', 'source_datarepo_row_ids'),
+            ('contents', 'donors', 'document_id'),
+            ('contents', 'donors', 'donor_id'),
+            ('contents', 'donors', 'source_datarepo_row_ids'),
             ('contents', 'files', 'version'),
         ]
 

src/azul/plugins/metadata/anvil/indexer/aggregate.py

@@ -23,13 +23,20 @@
 
 
 class ActivityAggregator(SimpleAggregator):
-    pass
+
+    def _accumulator(self, field: str) -> Accumulator | None:
+        if field in ('activity_id', 'document_id', 'source_datarepo_row_ids'):
+            return None
+        else:
+            return super()._accumulator(field)
 
 
 class BiosampleAggregator(SimpleAggregator):
 
     def _accumulator(self, field: str) -> Accumulator | None:
-        if field == 'donor_age_at_collection':
+        if field in ('biosample_id', 'document_id', 'source_datarepo_row_ids'):
+            return None
+        elif field == 'donor_age_at_collection':
             return SetOfDictAccumulator(max_size=100,
                                         key=compose_keys(none_safe_tuple_key(none_last=True),
                                                          itemgetter('lte', 'gte')))
@@ -38,13 +45,23 @@ def _accumulator(self, field: str) -> Accumulator | None:
 
 
 class DatasetAggregator(SimpleAggregator):
-    pass
+
+    def _accumulator(self, field: str) -> Accumulator | None:
+        if field == 'source_datarepo_row_ids':
+            return None
+        # Aggregation of `document_id` is required for the creation of manifests
+        elif field == 'document_id':
+            return super()._accumulator(field)
+        else:
+            return super()._accumulator(field)
 
 
 class DiagnosisAggregator(SimpleAggregator):
 
     def _accumulator(self, field: str) -> Accumulator | None:
-        if field in ('diagnosis_age', 'onset_age'):
+        if field in ('diagnosis_id', 'document_id', 'source_datarepo_row_ids'):
+            return None
+        elif field in ('diagnosis_age', 'onset_age'):
             return SetOfDictAccumulator(max_size=100,
                                         key=compose_keys(none_safe_tuple_key(none_last=True),
                                                          itemgetter('lte', 'gte')))
@@ -53,7 +70,12 @@ def _accumulator(self, field: str) -> Accumulator | None:
 
 
 class DonorAggregator(SimpleAggregator):
-    pass
+
+    def _accumulator(self, field: str) -> Accumulator | None:
+        if field in ('document_id', 'donor_id', 'source_datarepo_row_ids'):
+            return None
+        else:
+            return super()._accumulator(field)
 
 
 class FileAggregator(GroupingAggregator):
@@ -72,7 +94,17 @@ def _group_keys(self, entity) -> tuple[Any, ...]:
         return entity['file_format'],
 
     def _accumulator(self, field: str) -> Accumulator | None:
-        if field in ('count', 'file_size'):
+        if field in (
+                'document_id',
+                'drs_uri',
+                'file_id',
+                'file_md5sum',
+                'file_name',
+                'source_datarepo_row_ids',
+                'version',
+        ):
+            return None
+        elif field in ('count', 'file_size'):
             return DistinctAccumulator(SumAccumulator())
         else:
             return super()._accumulator(field)