Merge branch 'develop' into levelist

Author: danovaro

CMakeLists.txt

@@ -60,6 +60,22 @@ ecbuild_add_option( FEATURE ODB
                     DESCRIPTION "Add support for ODB data"
                     REQUIRED_PACKAGES "NAME odc VERSION 1.0" )
 
+# MARS2GRIB encoder
+
+ecbuild_add_option( FEATURE MARS2GRIB
+                    DEFAULT ON
+                    DESCRIPTION "Build the MARS2GRIB encoder" )
+
+# Python bindings for mars2grib encoder
+
+ecbuild_add_option( FEATURE MARS2GRIB_PYTHON
+                    DEFAULT OFF
+                    CONDITION HAVE_MARS2GRIB
+                    DESCRIPTION "Build mars2grib python library"
+                    REQUIRED_PACKAGES
+                        "NAME Python VERSION 3.11 COMPONENTS Interpreter Development REQUIRED"
+                        "NAME pybind11 VERSION 3.0.1 REQUIRED" )
+
 # METKIT config files support
 
 ecbuild_add_option( FEATURE METKIT_CONFIG

src/metkit/CMakeLists.txt

@@ -129,12 +129,12 @@ if ( HAVE_GRIB )
         codes/BUFRDecoder.h
         codes/GRIBDecoder.cc
         codes/GRIBDecoder.h
+        codes/CodesDataContent.cc
+        codes/CodesDataContent.h
         codes/CodesContent.cc
         codes/CodesContent.h
         codes/BufrContent.cc
         codes/BufrContent.h
-        codes/MallocCodesContent.cc
-        codes/MallocCodesContent.h
         codes/CodesSplitter.cc
         codes/CodesSplitter.h
         codes/GribAccessor.cc
@@ -184,6 +184,15 @@ if ( HAVE_ODB )
     set( odc_libs odccore )
 endif()
 
+if ( HAVE_MARS2GRIB )
+    list( APPEND metkit_srcs
+        mars2grib/api/Mars2Grib.cc
+        mars2grib/api/Mars2Grib.h
+        mars2grib/api/Options.h
+    )
+    set( mars2grib_libs eckit_geo )
+endif()
+
 ecbuild_add_library(
 
     TARGET metkit
@@ -211,4 +220,9 @@ ecbuild_add_library(
         eckit
         eckit_option
         ${grib_libs}
+        ${mars2grib_libs}
 )
+
+if( HAVE_MARS2GRIB_PYTHON )
+    add_subdirectory(mars2grib/api/pymars2grib)
+endif()

src/metkit/codes/BUFRDecoder.cc

@@ -89,14 +89,15 @@ void BUFRDecoder::getMetadata(const eckit::message::Message& msg, eckit::message
     // https://confluence.ecmwf.int/display/ECC/bufr_keys_iterator
     h->set("unpack", 1);
 
-    for (auto& k : h->keys()) {
+    for (const auto& k : h->keys()) {
         auto name = k.name();
 
         if (name == "subsetNumber") {
             continue;
         }
 
-        /* get key size to see if it is an array */
+        // Get key size to see if it is an array
+        // Only continue for scalar values
         if (h->size(name) != 1) {
             continue;
         }

src/metkit/codes/BufrContent.cc

@@ -8,14 +8,8 @@
  * does it submit to any jurisdiction.
  */
 
-/// @author Baudouin Raoult
-/// @author Emanuele Danovaro
-/// @date   Mar 2022
-
 #include "metkit/codes/BufrContent.h"
 
-#include "metkit/codes/GribHandle.h"
-
 #include "eccodes.h"
 
 namespace metkit {
@@ -24,29 +18,14 @@ namespace codes {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-BufrContent::BufrContent(codes_handle* handle, bool delete_handle) : CodesContent(handle, delete_handle) {}
-
-BufrContent::BufrContent(const codes_handle* handle) : BufrContent(const_cast<codes_handle*>(handle), false) {}
-
-BufrContent::~BufrContent() {}
-
+BufrContent::BufrContent(std::unique_ptr<CodesHandle> handle) : CodesDataContent(std::move(handle)) {}
 
 //----------------------------------------------------------------------------------------------------------------------
 
 void BufrContent::transform(const eckit::OrderedStringDict& dict) {
-
-    std::vector<codes_values> values;
-
-    for (auto& kv : dict) {
-        codes_values v;
-        v.name       = kv.first.c_str();
-        v.long_value = std::stol(kv.second);
-        v.type       = GRIB_TYPE_LONG;
-
-        values.push_back(v);
+    for (const auto& [key, value] : dict) {
+        handle_->set(key, std::stol(value));
     }
-
-    CODES_CALL(codes_set_values(handle_, values.data(), values.size()));
 }
 
 

src/metkit/codes/BufrContent.h

@@ -14,25 +14,24 @@
 
 #pragma once
 
-#include "metkit/codes/CodesContent.h"
+#include "metkit/codes/CodesDataContent.h"
 
 namespace metkit {
 namespace codes {
 
 
 //----------------------------------------------------------------------------------------------------------------------
 
-class BufrContent : public CodesContent {
+class BufrContent : public CodesDataContent {
 public:
 
-    BufrContent(codes_handle* handle, bool delete_handle);
-    explicit BufrContent(const codes_handle* handle);
+    BufrContent(std::unique_ptr<CodesHandle> handle);
 
-    ~BufrContent();
+    virtual ~BufrContent() = default;
 
 protected:
 
-    using CodesContent::transform;
+    using CodesDataContent::transform;
     void transform(const eckit::OrderedStringDict& dict) override;
 };
 

src/metkit/codes/CodesDataContent.cc

@@ -0,0 +1,157 @@
+/*
+ * (C) Copyright 2017- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+#include "metkit/codes/CodesDataContent.h"
+
+#include "eccodes.h"
+
+#include "eckit/exception/Exceptions.h"
+#include "eckit/io/DataHandle.h"
+#include "eckit/io/MemoryHandle.h"
+
+#include "metkit/codes/GribHandle.h"
+
+namespace metkit {
+namespace codes {
+
+//----------------------------------------------------------------------------------------------------------------------
+
+CodesDataContent::CodesDataContent(std::unique_ptr<CodesHandle> handle) :
+    handle_(std::move(handle)), offset_(handle_->getLong("offset")) {}
+
+
+CodesDataContent::CodesDataContent(std::unique_ptr<CodesHandle> handle, eckit::Offset offset) :
+    handle_(std::move(handle)), offset_(std::move(offset)) {}
+
+
+//----------------------------------------------------------------------------------------------------------------------
+
+size_t CodesDataContent::length() const {
+    return handle_->messageSize();
+}
+
+
+//----------------------------------------------------------------------------------------------------------------------
+
+void CodesDataContent::write(eckit::DataHandle& handle) const {
+    auto data = handle_->messageData();
+    if (handle.write(data.data(), data.size()) != data.size()) {
+        std::ostringstream oss;
+        oss << "Write error to data handle " << handle;
+        throw eckit::WriteError(oss.str(), Here());
+    }
+}
+
+
+//----------------------------------------------------------------------------------------------------------------------
+
+eckit::DataHandle* CodesDataContent::readHandle() const {
+    auto data = handle_->messageData();
+    return new eckit::MemoryHandle(data.data(), data.size());
+}
+
+
+//----------------------------------------------------------------------------------------------------------------------
+
+void CodesDataContent::print(std::ostream& s) const {
+    s << "CodesDataContent[]";
+}
+
+
+//----------------------------------------------------------------------------------------------------------------------
+
+std::string CodesDataContent::getString(const std::string& key) const {
+    return handle_->getString(key);
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+
+long CodesDataContent::getLong(const std::string& key) const {
+    return handle_->getLong(key);
+}
+
+
+//----------------------------------------------------------------------------------------------------------------------
+
+double CodesDataContent::getDouble(const std::string& key) const {
+    return handle_->getDouble(key);
+}
+
+
+//----------------------------------------------------------------------------------------------------------------------
+
+void CodesDataContent::getDoubleArray(const std::string& key, std::vector<double>& values) const {
+    values = handle_->getDoubleArray(key);
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+
+void CodesDataContent::getFloatArray(const std::string& key, std::vector<float>& values) const {
+    values = handle_->getFloatArray(key);
+}
+
+
+//----------------------------------------------------------------------------------------------------------------------
+
+size_t CodesDataContent::getSize(const std::string& key) const {
+    return handle_->size(key);
+}
+
+
+//----------------------------------------------------------------------------------------------------------------------
+
+void CodesDataContent::getDoubleArray(const std::string& key, double* data, size_t len) const {
+    auto arr = handle_->getDoubleArray(key);
+    ASSERT(len == arr.size());
+    std::copy(arr.begin(), arr.end(), data);
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+
+void CodesDataContent::getFloatArray(const std::string& key, float* data, size_t len) const {
+    auto arr = handle_->getFloatArray(key);
+    ASSERT(len == arr.size());
+    std::copy(arr.begin(), arr.end(), data);
+}
+
+
+//----------------------------------------------------------------------------------------------------------------------
+
+void CodesDataContent::transform(const eckit::OrderedStringDict& dict) {
+    for (auto& [key, value] : dict) {
+        handle_->set(key, value);
+    }
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+
+eckit::Offset CodesDataContent::offset() const {
+    return offset_;
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+
+const void* CodesDataContent::data() const {
+    return handle_->messageData().data();
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+
+const CodesHandle& CodesDataContent::codesHandle() const {
+    return *handle_.get();
+}
+
+CodesHandle& CodesDataContent::codesHandle() {
+    return *handle_.get();
+}
+
+
+}  // namespace codes
+}  // namespace metkit

src/metkit/codes/CodesDataContent.h

@@ -0,0 +1,65 @@
+/*
+ * (C) Copyright 2017- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+#pragma once
+
+#include "eckit/io/Offset.h"
+#include "metkit/codes/api/CodesAPI.h"
+
+#include "eckit/message/MessageContent.h"
+
+namespace metkit {
+namespace codes {
+
+//----------------------------------------------------------------------------------------------------------------------
+
+class CodesDataContent : public eckit::message::MessageContent {
+public:
+
+    CodesDataContent(std::unique_ptr<CodesHandle> handle, eckit::Offset offset);
+    CodesDataContent(std::unique_ptr<CodesHandle> handle);
+
+    virtual ~CodesDataContent() = default;
+
+    const CodesHandle& codesHandle() const;
+    CodesHandle& codesHandle();
+
+protected:
+
+    std::unique_ptr<CodesHandle> handle_;
+    eckit::Offset offset_;
+
+    using eckit::message::MessageContent::transform;
+    void transform(const eckit::OrderedStringDict&) override;
+
+private:
+
+    size_t length() const override;
+    void write(eckit::DataHandle& handle) const override;
+    eckit::DataHandle* readHandle() const override;
+    void print(std::ostream& s) const override;
+    std::string getString(const std::string& key) const override;
+    long getLong(const std::string& key) const override;
+    double getDouble(const std::string& key) const override;
+    void getDoubleArray(const std::string& key, std::vector<double>& values) const override;
+    void getFloatArray(const std::string& key, std::vector<float>& values) const override;
+    size_t getSize(const std::string& key) const override;
+    void getDoubleArray(const std::string& key, double* data, size_t lenExpected) const override;
+    void getFloatArray(const std::string& key, float* data, size_t lenExpected) const override;
+
+    eckit::Offset offset() const override;
+    const void* data() const override;
+};
+
+
+//----------------------------------------------------------------------------------------------------------------------
+
+}  // namespace codes
+}  // namespace metkit