Top level data object

Author: sandorkertesz

docs/experimental/top_level_object/bufr_single.ipynb

@@ -53,8 +53,8 @@ def to_geopandas(self, **kwargs):
         pass
 
     @abstractmethod
-    def to_bufr_list(self, *args, **kwargs):
-        """Convert into a list of bufr messages"""
+    def to_featurelist(self, *args, **kwargs):
+        """Convert into a list of feature messages"""
         pass
 
     @abstractmethod
@@ -115,7 +115,7 @@ def to_pandas(self, *args, **kwargs):
     def to_geopandas(self, *args, **kwargs):
         raise NotImplementedError("Conversion of MultiData to geopandas is not implemented")
 
-    def to_bufr_list(self, *args, **kwargs):
+    def to_featurelist(self, *args, **kwargs):
         pass
 
     def to_numpy(self, *args, **kwargs):
@@ -153,7 +153,7 @@ def to_geojson(self, **kwargs):
     def to_geopandas(self, **kwargs):
         return self._conversion_not_implemented()
 
-    def to_bufr_list(self, *args, **kwargs):
+    def to_featurelist(self, *args, **kwargs):
         self._conversion_not_implemented()
 
     def to_numpy(self, *args, **kwargs):

src/earthkit/data/core/data.py

@@ -23,7 +23,7 @@ def _match_magic(magic, deeper_check):
 
 def reader(source, path, *, magic=None, deeper_check=False, **kwargs):
     if _match_magic(magic, deeper_check):
-        from .message_list import BUFRReader
+        from .message_list import BUFRReader1
 
         parts = source.parts if hasattr(source, "parts") else None
-        return BUFRReader(source, path, parts=parts)
+        return BUFRReader1(source, path, parts=parts)

src/earthkit/data/readers/bufr/__init__.py

@@ -0,0 +1,48 @@
+# (C) Copyright 2020 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+#
+
+from earthkit.data.core.data import Data
+
+
+class BUFRData(Data):
+    _TYPE_NAME = "BUFR"
+
+    def __init__(self, reader):
+        self._reader = reader
+
+    @property
+    def available_types(self):
+        return ["pandas", "featurelist"]
+
+    def describe(self):
+        return f"BUFR data from {self._reader.path}"
+
+    def to_fieldlist(self, *args, **kwargs):
+        return self._conversion_not_implemented()
+
+    def to_pandas(self, **kwargs):
+        return self._reader.to_pandas(**kwargs)
+
+    def to_xarray(self, **kwargs):
+        return self._conversion_not_implemented()
+
+    def to_geojson(self, **kwargs):
+        return self._conversion_not_implemented()
+
+    def to_geopandas(self, **kwargs):
+        return self._conversion_not_implemented()
+
+    def to_featurelist(self, *args, **kwargs):
+        return self._reader.to_featurelist(*args, **kwargs)
+
+    def to_numpy(self, *args, **kwargs):
+        self._conversion_not_implemented()
+
+    def to_array(self, *args, **kwargs):
+        self._conversion_not_implemented()

src/earthkit/data/readers/bufr/data.py

@@ -7,7 +7,6 @@
 # nor does it submit to any jurisdiction.
 #
 
-
 import eccodes
 
 from earthkit.data.utils.message import CodesHandle

src/earthkit/data/readers/bufr/handle.py

@@ -13,6 +13,7 @@
 from earthkit.data.core.index import Index
 from earthkit.data.core.index import MaskIndex
 from earthkit.data.core.index import MultiIndex
+from earthkit.data.sources import Source
 from earthkit.data.utils.parts import Part
 
 from .. import Reader
@@ -505,3 +506,60 @@ def mutate_source(self):
 
     def _default_encoder(self):
         return Reader._default_encoder(self)
+
+
+class BUFRReader1(Source, Reader):
+    def __init__(self, source, path, parts=None, positions=None):
+        self._ori_source = source
+        self._kwargs = {"parts": parts}
+        Reader.__init__(self, source, path)
+
+    def to_featurelist(self, *args, **kwargs):
+        return BUFRInOneFile(self.path, **self._kwargs, **kwargs)
+
+    def to_pandas(self, *args, **kwargs):
+        return self.to_featurelist().to_pandas(*args, **kwargs)
+
+    def mutate_source(self):
+        # A BUFRReader is a source itself
+        return self
+
+    def mutate(self):
+        return self
+
+    def _to_data_object(self):
+        from .data import BUFRData
+
+        return BUFRData(self)
+
+    @classmethod
+    def merge(cls, sources):
+
+        assert all(isinstance(s, BUFRReader1) for s in sources)
+        return MultiBUFRReader1(sources)
+
+
+class MultiBUFRReader1(BUFRReader1):
+    def __init__(self, sources):
+        self.sources = sources
+
+    def to_featurelist(self):
+        fs = [s.to_featurelist() for s in self.sources]
+        from earthkit.data.mergers import merge_by_class
+
+        merged = merge_by_class(fs)
+        if merged is not None:
+            return merged.mutate()
+
+        raise NotImplementedError("Conversion of MultiBUFRReader1 to featurelist is not implemented")
+
+    def to_pandas(self, *args, **kwargs):
+        return self.to_featurelist().to_pandas(*args, **kwargs)
+
+    def __repr__(self):
+        return "MultiBUFRReader1(%s)" % (self.sources,)
+
+    def _to_data_object(self):
+        from earthkit.data.core.data import MultiData
+
+        return MultiData(self)

src/earthkit/data/readers/bufr/message_list.py

@@ -158,7 +158,7 @@ def to_geojson(self, **kwargs):
     def to_geopandas(self, **kwargs):
         return self._conversion_not_implemented()
 
-    def to_bufr_list(self, *args, **kwargs):
+    def to_featurelist(self, *args, **kwargs):
         self._conversion_not_implemented()
 
     def to_numpy(self, *args, **kwargs):

src/earthkit/data/readers/covjson.py

@@ -223,7 +223,7 @@ def to_xarray(self, **kwargs):
     def to_geopandas(self, **kwargs):
         self._conversion_not_implemented()
 
-    def to_bufr_list(self, *args, **kwargs):
+    def to_featurelist(self, *args, **kwargs):
         self._conversion_not_implemented()
 
     def to_numpy(self, *args, **kwargs):

src/earthkit/data/readers/csv.py

@@ -206,7 +206,7 @@ def to_xarray(self, **kwargs):
     def to_geopandas(self, **kwargs):
         self._conversion_not_implemented()
 
-    def to_bufr_list(self, *args, **kwargs):
+    def to_featurelist(self, *args, **kwargs):
         self._conversion_not_implemented()
 
     def to_numpy(self, *args, **kwargs):

src/earthkit/data/readers/geotiff.py

@@ -29,7 +29,7 @@ def to_geojson(self, **kwargs):
     def to_geopandas(self, **kwargs):
         return self._conversion_not_implemented()
 
-    def to_bufr_list(self, *args, **kwargs):
+    def to_featurelist(self, *args, **kwargs):
         self._conversion_not_implemented()
 
     def to_numpy(self, *args, **kwargs):