implement endian-aware reader support

hobu · hobu · commit 1b5a51b244cf · 2020-05-19T19:59:55.000-05:00
diff --git a/pdal/io/NumpyReader.cpp b/pdal/io/NumpyReader.cpp
@@ -41,6 +41,7 @@
 #include <pdal/util/ProgramArgs.hpp>
 #include <pdal/util/FileUtils.hpp>
 #include <pdal/util/Algorithm.hpp>
+#include <pdal/util/Extractor.hpp>
 
 #include "../plang/Environment.hpp"
 
@@ -407,10 +408,13 @@ void NumpyReader::createFields(PointLayoutPtr layout)
             offset = PyLong_AsLong(offset_o);
 
             // Get type.
-            type = getType((PyArray_Descr *)PySequence_Fast_GET_ITEM(tup, 0),
-                name);
+            PyArray_Descr* dt = (PyArray_Descr *)PySequence_Fast_GET_ITEM(tup, 0);
+            type = getType(dt, name);
+
+            char byteorder = dt->byteorder;
+            int elsize = dt->elsize;
             id = registerDim(layout, name, type);
-            m_fields.push_back({id, type, offset});
+            m_fields.push_back({id, type, offset, byteorder, elsize});
         }
     }
 }
@@ -510,6 +514,12 @@ void NumpyReader::ready(PointTableRef table)
     for (npy_intp i = 0; i < m_ndims; ++i)
         log()->get(LogLevel::Debug) << "numpy shape dimension number '" <<
             i << "' is '" << m_shape[i] <<"'" << std::endl;
+
+    PointLayoutPtr layout = table.layout();
+    MetadataNode m = layout->toMetadata();
+
+    pdal::Utils::toJSON(m, log()->get(LogLevel::Debug3));
+
 }
 
 bool NumpyReader::nextPoint()
@@ -530,12 +540,101 @@ bool NumpyReader::nextPoint()
 }
 
 
+
+
 bool NumpyReader::loadPoint(PointRef& point, point_count_t position)
 {
     using namespace Dimension;
 
+    pdal::SwitchableExtractor extractor(p_data, *m_strideptr);
+
+    std::vector<char> buf(*m_strideptr,0);
+
+    float flt(0.0);
+    double dbl(0.0);
+    uint8_t uint8(0);
+    uint16_t uint16(0);
+    uint32_t uint32(0);
+    uint64_t uint64(0);
+    int8_t int8(0);
+    int16_t int16(0);
+    int32_t int32(0);
+    int64_t int64(0);
+
     for (const Field& f : m_fields)
-        point.setField(f.m_id, f.m_type, (void*)(p_data + f.m_offset));
+    {
+        if (f.m_byteorder == '>')
+            extractor.switchToBigEndian();
+        else
+            extractor.switchToLittleEndian();
+
+        if (Dimension::base(f.m_type) == Dimension::BaseType::Floating)
+        {
+            if (f.m_elsize == 4)
+            {
+                extractor >> flt;
+                point.setField<float>(f.m_id, flt);
+            }
+            else if (f.m_elsize == 8)
+            {
+                extractor >> dbl;
+                point.setField<double>(f.m_id, dbl);
+
+            }
+        }
+        else if (Dimension::base(f.m_type) == Dimension::BaseType::Signed)
+        {
+            if (f.m_elsize == 1)
+            {
+                extractor >> int8;
+                point.setField<int8_t>(f.m_id, int8);
+            }
+            if (f.m_elsize == 2)
+            {
+                extractor >> int16;
+                point.setField<int16_t>(f.m_id, int16);
+            }
+            if (f.m_elsize == 4)
+            {
+                extractor >> int32;
+                point.setField<int32_t>(f.m_id, int32);
+            }
+            if (f.m_elsize == 8)
+            {
+                extractor >> int64;
+                point.setField<int64_t>(f.m_id, int64);
+            }
+        }
+        else if (Dimension::base(f.m_type) == Dimension::BaseType::Unsigned)
+        {
+            if (f.m_elsize == 1)
+            {
+                extractor >> uint8;
+                point.setField<uint8_t>(f.m_id, uint8);
+            }
+            if (f.m_elsize == 2)
+            {
+                extractor >> uint16;
+                point.setField<uint16_t>(f.m_id, uint16);
+            }
+            if (f.m_elsize == 4)
+            {
+                extractor >> uint32;
+                point.setField<uint32_t>(f.m_id, uint32);
+            }
+            if (f.m_elsize == 8)
+            {
+                extractor >> int64;
+                point.setField<uint64_t>(f.m_id, uint64);
+            }
+        }
+        else
+        {
+            // skip it
+            extractor.skip(f.m_elsize);
+        }
+
+    }
 
     if (m_storeXYZ)
     {
diff --git a/pdal/io/NumpyReader.hpp b/pdal/io/NumpyReader.hpp
@@ -120,6 +120,8 @@ class PDAL_DLL NumpyReader : public Reader, public Streamable
         Dimension::Id m_id;
         Dimension::Type m_type;
         int m_offset;
+        char m_byteorder;
+        int m_elsize;
     };
     std::vector<Field> m_fields;
     point_count_t m_index;
diff --git a/pdal/test/NumpyReaderTest.cpp b/pdal/test/NumpyReaderTest.cpp
@@ -91,7 +91,7 @@ TEST(NumpyReaderTest, NumpyReaderTest_read_array)
     EXPECT_EQ(view->layout()->pointSize(), 24u);
 
     EXPECT_EQ(view->getFieldAs<double>(pdal::Dimension::Id::Intensity,5000),
-        0.5);
+        0.0);
     EXPECT_EQ(view->getFieldAs<uint32_t>(pdal::Dimension::Id::X,5000), 50u);
     EXPECT_EQ(view->getFieldAs<uint32_t>(pdal::Dimension::Id::X,5023), 50u);
     EXPECT_EQ(view->getFieldAs<uint32_t>(pdal::Dimension::Id::Y,5023), 23u);
@@ -236,3 +236,100 @@ TEST(NumpyReaderTest, threedim_col)
     }
 }
 
+
+TEST(NumpyReaderTest, threedim_row_script)
+{
+    StageFactory f;
+
+    Options opts;
+    opts.add("filename", Support::datapath("threedim.py"));
+    opts.add("order", "row");
+    opts.add("function", "load");
+    opts.add("module", "threedim");
+    opts.add("fargs",  Support::datapath("threedim.npy"));
+
+    NumpyReader reader;
+    reader.setOptions(opts);
+
+    PointTable table;
+
+    reader.prepare(table);
+
+    PointViewSet viewSet = reader.execute(table);
+    PointViewPtr view = *viewSet.begin();
+    PointLayoutPtr layout = view->layout();
+    Dimension::Id health = layout->findDim("Health");
+    Dimension::Id foo = layout->findDim("Foo");
+
+    EXPECT_EQ(view->size(), 24u);
+    int ex = 0;
+    int ey = 0;
+    int ez = 0;
+    for (PointId id = 0; id < view->size(); ++id)
+    {
+        int f = view->getFieldAs<int>(foo, id);
+        int h = view->getFieldAs<int>(health, id);
+        int i = view->getFieldAs<int>(Dimension::Id::Intensity, id);
+        int x = view->getFieldAs<int>(Dimension::Id::X, id);
+        int y = view->getFieldAs<int>(Dimension::Id::Y, id);
+        int z = view->getFieldAs<int>(Dimension::Id::Z, id);
+        EXPECT_EQ(f, ((int)id / 12) % 2);
+        EXPECT_EQ(h, ((int)id / 4) % 3);
+        EXPECT_EQ(i, (int)id % 4);
+        EXPECT_EQ(x, ex);
+        EXPECT_EQ(y, ey);
+        EXPECT_EQ(z, ez);
+        // Array is 2 x 3 x 4
+        ez++;
+        if (ez == 4)
+        {
+            ez = 0;
+            ey++;
+            if (ey == 3)
+            {
+                ey = 0;
+                ex++;
+            }
+        }
+    }
+}
+
+TEST(NumpyReaderTest, four_floats_script)
+{
+    StageFactory f;
+
+    Options opts;
+    opts.add("filename", Support::datapath("four-floats.py"));
+    opts.add("order", "row");
+    opts.add("function", "load");
+    opts.add("module", "four_floats");
+    opts.add("fargs",  Support::datapath("four-floats-be.bin"));
+
+    NumpyReader reader;
+    reader.setOptions(opts);
+
+    PointTable table;
+
+    reader.prepare(table);
+
+    PointViewSet viewSet = reader.execute(table);
+    PointViewPtr view = *viewSet.begin();
+    PointLayoutPtr layout = view->layout();
+
+    EXPECT_EQ(view->size(), 10u);
+    int ex = 0;
+    int ey = 0;
+    int ez = 0;
+    int factor = 10;
+    for (PointId id = 0; id < view->size(); ++id)
+    {
+        int t = view->getFieldAs<int>(Dimension::Id::GpsTime, id);
+        int x = view->getFieldAs<int>(Dimension::Id::X, id);
+        int y = view->getFieldAs<int>(Dimension::Id::Y, id);
+        int z = view->getFieldAs<int>(Dimension::Id::Z, id);
+        EXPECT_EQ(x, (int)id * 1 * factor);
+        EXPECT_EQ(y, (int)id * 2 * factor);
+        EXPECT_EQ(z, (int)id * 3 * factor);
+        EXPECT_EQ(t, (int)id * 4 * factor);
+    }
+}
diff --git a/pdal/test/data/four-floats-be.bin b/pdal/test/data/four-floats-be.bin
diff --git a/pdal/test/data/four-floats.py b/pdal/test/data/four-floats.py
@@ -0,0 +1,10 @@
+import numpy as np
+
+
+dtype = np.dtype([('X', '>f4'), ('Y', '>f4'), ('Z', '>f4'), ('GPSTime', '>f4')])
+
+def load(filename):
+    data = open(filename, 'rb').read()
+    array = np.frombuffer(data, dtype=dtype)
+
+    return array
diff --git a/pdal/test/data/threedim.py b/pdal/test/data/threedim.py
@@ -0,0 +1,5 @@
+import numpy as np
+
+def load(filename):
+    array = np.load(filename)
+    return array
