Remove json from esri reader headers (PDAL#4722)

* Remove private functions from stac reader header.

* Move interface functions out of EsriReader.

* Add missed file.

Author: abellgithub

io/EsriReader.cpp

@@ -37,16 +37,13 @@
 
 #include <Eigen/Geometry>
 
-
 #include <pdal/util/Algorithm.hpp>
 #include <pdal/util/ThreadPool.hpp>
 #include <pdal/private/MathUtils.hpp>
 #include <pdal/private/SrsTransform.hpp>
 
-
-
-
 #include "private/esri/Obb.hpp"
+#include "private/esri/Interface.hpp"
 #include "lepcc/src/include/lepcc_types.h"
 
 namespace pdal
@@ -124,7 +121,8 @@ class EsriReader::TileContents
     std::string m_error;
 };
 
-EsriReader::EsriReader() : m_args(new Args)
+EsriReader::EsriReader(std::unique_ptr<Interface> interface) :
+    m_interface(std::move(interface)), m_args(new Args)
 {}
 
 
@@ -154,8 +152,6 @@ void EsriReader::initialize(PointTableRef table)
     for (std::string& s : m_args->dimensions)
         s = Utils::toupper(s);
 
-    m_arbiter.reset(new arbiter::Arbiter());
-
     //adjust filename string
     const std::string pre("i3s://");
     if (Utils::startsWith(m_filename, pre))
@@ -170,15 +166,15 @@ void EsriReader::initialize(PointTableRef table)
     //personalize for slpk or i3s
     try
     {
-        m_info = initInfo();
+        m_interface->initInfo();
     }
     catch (std::exception& e)
     {
         throwError(std::string("Failed to fetch info: ") + e.what());
     }
 
     //create const for looking into
-    const NL::json jsonBody = m_info;
+    const NL::json jsonBody = m_interface->getInfo();
 
     //find version
     if (jsonBody["store"].contains("version"))
@@ -215,7 +211,7 @@ void EsriReader::initialize(PointTableRef table)
     }
 
     //create spatial reference objects
-    NL::json wkid = m_info["spatialReference"]["wkid"];
+    NL::json wkid = jsonBody["spatialReference"]["wkid"];
     int system(0);
     if (wkid.is_string())
     {
@@ -254,9 +250,10 @@ void EsriReader::addDimensions(PointLayoutPtr layout)
 {
     using namespace Dimension;
 
-    if (!m_info.contains("attributeStorageInfo"))
+    const NL::json& jsonBody = m_interface->getInfo();
+    if (!jsonBody.contains("attributeStorageInfo"))
         throwError("Attributes do not exist for this object");
-    const NL::json& attributes = m_info["attributeStorageInfo"];
+    const NL::json& attributes = jsonBody["attributeStorageInfo"];
 
     layout->registerDims({Id::X, Id::Y, Id::Z});
 
@@ -353,8 +350,8 @@ void EsriReader::ready(PointTableRef table)
     // actual page number, so we use the page size (m_nodeCap) as a factor from the page
     // index to get the proper filename for a page.
     int indexFactor = m_version >= Version("2.0") ? 1 : m_nodeCap;
-    i3s::FetchFunction fetch = std::bind(&EsriReader::fetchJson, this, std::placeholders::_1);
-    m_pageManager.reset(new PageManager(100, 4, indexFactor, fetch));
+    m_pageManager.reset(new PageManager(100, 4, indexFactor,
+        [this](std::string src) { return m_interface->fetchJson(src); }));
     PagePtr p = m_pageManager->getPage(0);
     traverseTree(p, 0);
     m_pool->await();
@@ -661,7 +658,7 @@ EsriReader::TileContents EsriReader::loadPath(const std::string& filepath)
     TileContents tile(filepath, m_extraDimCount);
 
     const std::string geomUrl = filepath + "/geometries/";
-    auto xyzFetch = fetchBinary(geomUrl, "0", ".bin.pccxyz");
+    auto xyzFetch = m_interface->fetchBinary(geomUrl, "0", ".bin.pccxyz");
     tile.m_xyz = i3s::decompressXYZ(&xyzFetch);
 
     size_t size = tile.m_xyz.size();
@@ -670,22 +667,22 @@ EsriReader::TileContents EsriReader::loadPath(const std::string& filepath)
     {
         if (dim.name == "RGB")
         {
-            auto data = fetchBinary(attrUrl, std::to_string(dim.key),
+            auto data = m_interface->fetchBinary(attrUrl, std::to_string(dim.key),
                 ".bin.pccrgb");
             tile.m_rgb = i3s::decompressRGB(&data);
             tile.m_error = checkSize(dim, size, tile.m_rgb.size());
         }
         else if (dim.name == "INTENSITY")
         {
-            auto data = fetchBinary(attrUrl, std::to_string(dim.key),
+            auto data = m_interface->fetchBinary(attrUrl, std::to_string(dim.key),
                 ".bin.pccint");
             tile.m_intensity = i3s::decompressIntensity(&data);
             tile.m_error = checkSize(dim, size, tile.m_intensity.size());
         }
         else
         {
             std::vector<char>& data = tile.m_data[dim.pos];
-            data = fetchBinary(attrUrl, std::to_string(dim.key), ".bin.gz");
+            data = m_interface->fetchBinary(attrUrl, std::to_string(dim.key), ".bin.gz");
             tile.m_error = checkSize(dim, size * Dimension::size(dim.type),
                 data.size());
         }

io/EsriReader.hpp

@@ -48,40 +48,30 @@
 #include "private/esri/EsriUtil.hpp"
 #include "private/esri/PageManager.hpp"
 
-#include <nlohmann/json.hpp>
-
 namespace pdal
 {
 
-namespace arbiter
-{
-    class Arbiter;
-}
-
 class SrsTransform;
 class ThreadPool;
 
+namespace i3s
+{
+    struct Interface;
+};
+
 class PDAL_EXPORT EsriReader : public Reader, public Streamable
 {
 public:
-    EsriReader();
+    EsriReader(std::unique_ptr<i3s::Interface> interface);
     ~EsriReader();
 
-protected:
-    std::unique_ptr<arbiter::Arbiter> m_arbiter;
-
-    virtual NL::json initInfo() = 0;
-    virtual std::vector<char> fetchBinary(std::string url, std::string attNum,
-            std::string ext) const = 0;
-    virtual std::string fetchJson(std::string) = 0;
-
 private:
     struct Args;
     struct DimData;
     class TileContents;
 
+    std::unique_ptr<i3s::Interface> m_interface;
     std::unique_ptr<Args> m_args;
-    NL::json m_info;
     int m_nodeCap;
     i3s::Version m_version;
     SpatialReference m_nativeSrs;

io/I3SReader.cpp

@@ -35,6 +35,7 @@
 #include <arbiter/arbiter.hpp>
 #include "I3SReader.hpp"
 #include "private/esri/EsriUtil.hpp"
+#include "private/esri/Interface.hpp"
 
 #include <thread>
 
@@ -52,52 +53,75 @@ static PluginInfo const i3sInfo
 
 CREATE_STATIC_STAGE(I3SReader, i3sInfo)
 
+namespace
+{
+
+struct I3SInterface : public i3s::Interface
+{
+    I3SInterface(const std::string& filename) : m_filename(filename)
+    {}
+    ~I3SInterface() override
+    {}
+
+    void initInfo() override;
+    std::vector<char> fetchBinary(std::string url, std::string attNum,
+        std::string ext) const override;
+    std::string fetchJson(std::string) override;
+    NL::json getInfo() override
+        { return m_info; }
+
+    NL::json m_info;
+    const std::string& m_filename;
+    arbiter::Arbiter m_arbiter;
+};
+
+}
+
 std::string I3SReader::getName() const { return i3sInfo.name; }
 
-NL::json I3SReader::initInfo()
+I3SReader::I3SReader() : EsriReader(std::make_unique<I3SInterface>(m_filename))
+{}
+
+void I3SInterface::initInfo()
 {
-    NL::json info;
     try
     {
-        std::string s = fetchJson("");
-        info = i3s::parse(s, "Invalid JSON in file '" + m_filename + "'.");
+        std::string s = m_arbiter.get(m_filename);
+        NL::json info = i3s::parse(s, "Invalid JSON in file '" + m_filename + "'.");
 
         if (info.empty())
-            throwError(std::string("Incorrect Json object"));
+            throw pdal_error(std::string("Incorrect Json object"));
         if (!info.contains("layers"))
-            throwError(std::string("Json object contains no layers"));
+            throw pdal_error(std::string("Json object contains no layers"));
 
-        info = info["layers"][0];
+        m_info = info["layers"][0];
     }
     catch(i3s::EsriError& e)
     {
-        throwError(std::string("Error parsing Json object: ") + e.what());
+        throw pdal_error(std::string("Error parsing Json object: ") + e.what());
     }
-
-    m_filename += "/layers/0";
-    return info;
 }
 
 
-std::string I3SReader::fetchJson(std::string filepath)
+std::string I3SInterface::fetchJson(std::string filepath)
 {
-    filepath = m_filename + "/" + filepath;
-    return m_arbiter->get(filepath);
+    filepath = m_filename + "/layers/0/" + filepath;
+    return m_arbiter.get(filepath);
 }
 
 
-std::vector<char> I3SReader::fetchBinary(std::string url,
+std::vector<char> I3SInterface::fetchBinary(std::string url,
     std::string attNum, std::string ext) const
 {
     const int NumRetries(5);
     int retry = 0;
 
-    std::string filepath = m_filename + "/" + url + attNum;
+    std::string filepath = m_filename + "/layers/0/" + url + attNum;
     // For the REST I3S endpoint there are no file extensions.
     std::vector<char> result;
     while (true)
     {
-        auto data = m_arbiter->tryGetBinary(filepath);
+        auto data = m_arbiter.tryGetBinary(filepath);
         if (data)
         {
             result = std::move(*data);

io/I3SReader.hpp

@@ -41,13 +41,9 @@ namespace pdal
 class PDAL_EXPORT I3SReader : public EsriReader
 {
 public:
-    std::string getName() const override;
+    I3SReader();
 
-protected:
-    virtual NL::json initInfo() override;
-    virtual std::vector<char> fetchBinary(std::string url, std::string attNum,
-        std::string ext) const override;
-    virtual std::string fetchJson(std::string) override;
+    std::string getName() const override;
 };
 
 } // namespace pdal