Skip to content

Commit feda8e1

Browse files
hobuhobu
authored
Merge pull request #61 from PDAL/endian-aware-reading
implement endian-aware reader support
2 parents 63e3a1a + 5a77941 commit feda8e1

File tree

6 files changed

+209
-4
lines changed

6 files changed

+209
-4
lines changed

‎pdal/io/NumpyReader.cpp‎

Lines changed: 95 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
#include <pdal/util/ProgramArgs.hpp>
4242
#include <pdal/util/FileUtils.hpp>
4343
#include <pdal/util/Algorithm.hpp>
44+
#include <pdal/util/Extractor.hpp>
4445

4546
#include "../plang/Environment.hpp"
4647

@@ -407,10 +408,13 @@ void NumpyReader::createFields(PointLayoutPtr layout)
407408
offset = PyLong_AsLong(offset_o);
408409

409410
// Get type.
410-
type = getType((PyArray_Descr *)PySequence_Fast_GET_ITEM(tup, 0),
411-
name);
411+
PyArray_Descr* dt = (PyArray_Descr *)PySequence_Fast_GET_ITEM(tup, 0);
412+
type = getType(dt, name);
413+
414+
char byteorder = dt->byteorder;
415+
int elsize = dt->elsize;
412416
id = registerDim(layout, name, type);
413-
m_fields.push_back({id, type, offset});
417+
m_fields.push_back({id, type, offset, byteorder, elsize});
414418
}
415419
}
416420
}
@@ -510,6 +514,12 @@ void NumpyReader::ready(PointTableRef table)
510514
for (npy_intp i = 0; i < m_ndims; ++i)
511515
log()->get(LogLevel::Debug) << "numpy shape dimension number '" <<
512516
i << "' is '" << m_shape[i] <<"'" << std::endl;
517+
518+
PointLayoutPtr layout = table.layout();
519+
MetadataNode m = layout->toMetadata();
520+
521+
pdal::Utils::toJSON(m, log()->get(LogLevel::Debug3));
522+
513523
}
514524

515525
bool NumpyReader::nextPoint()
@@ -530,12 +540,93 @@ bool NumpyReader::nextPoint()
530540
}
531541

532542

543+
544+
533545
bool NumpyReader::loadPoint(PointRef& point, point_count_t position)
534546
{
535547
using namespace Dimension;
536548

549+
pdal::SwitchableExtractor extractor(p_data, *m_strideptr);
550+
551+
std::vector<char> buf(*m_strideptr,0);
552+
553+
float flt(0.0);
554+
double dbl(0.0);
555+
uint8_t u8(0);
556+
uint16_t u16(0);
557+
uint32_t u32(0);
558+
uint64_t u64(0);
559+
int8_t i8(0);
560+
int16_t i16(0);
561+
int32_t i32(0);
562+
int64_t i64(0);
563+
537564
for (const Field& f : m_fields)
538-
point.setField(f.m_id, f.m_type, (void*)(p_data + f.m_offset));
565+
{
566+
if (f.m_byteorder == '>')
567+
extractor.switchToBigEndian();
568+
else
569+
extractor.switchToLittleEndian();
570+
571+
switch (f.m_type)
572+
{
573+
case Dimension::Type::Signed8:
574+
extractor >> i8;
575+
point.setField(f.m_id, i8);
576+
break;
577+
578+
case Dimension::Type::Signed16:
579+
extractor >> i16;
580+
point.setField(f.m_id, i16);
581+
break;
582+
583+
case Dimension::Type::Signed32:
584+
extractor >> i32;
585+
point.setField(f.m_id, i32);
586+
break;
587+
588+
case Dimension::Type::Signed64:
589+
extractor >> i64;
590+
point.setField(f.m_id, i64);
591+
break;
592+
593+
case Dimension::Type::Unsigned8:
594+
extractor >> u8;
595+
point.setField(f.m_id, u8);
596+
break;
597+
598+
case Dimension::Type::Unsigned16:
599+
extractor >> u16;
600+
point.setField(f.m_id, u16);
601+
break;
602+
603+
case Dimension::Type::Unsigned32:
604+
extractor >> u32;
605+
point.setField(f.m_id, u32);
606+
break;
607+
608+
case Dimension::Type::Unsigned64:
609+
extractor >> u64;
610+
point.setField(f.m_id, u64);
611+
break;
612+
613+
case Dimension::Type::Float:
614+
extractor >> flt;
615+
point.setField(f.m_id, flt);
616+
break;
617+
618+
case Dimension::Type::Double:
619+
extractor >> dbl;
620+
point.setField(f.m_id, dbl);
621+
break;
622+
623+
default:
624+
// skip it
625+
extractor.skip(f.m_elsize);
626+
627+
}
628+
629+
}
539630

540631
if (m_storeXYZ)
541632
{

‎pdal/io/NumpyReader.hpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,8 @@ class PDAL_DLL NumpyReader : public Reader, public Streamable
120120
Dimension::Id m_id;
121121
Dimension::Type m_type;
122122
int m_offset;
123+
char m_byteorder;
124+
int m_elsize;
123125
};
124126
std::vector<Field> m_fields;
125127
point_count_t m_index;

‎pdal/test/NumpyReaderTest.cpp‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -236,3 +236,100 @@ TEST(NumpyReaderTest, threedim_col)
236236
}
237237
}
238238

239+
240+
TEST(NumpyReaderTest, threedim_row_script)
241+
{
242+
StageFactory f;
243+
244+
Options opts;
245+
opts.add("filename", Support::datapath("threedim.py"));
246+
opts.add("order", "row");
247+
opts.add("function", "load");
248+
opts.add("module", "threedim");
249+
opts.add("fargs", Support::datapath("threedim.npy"));
250+
251+
NumpyReader reader;
252+
reader.setOptions(opts);
253+
254+
PointTable table;
255+
256+
reader.prepare(table);
257+
258+
PointViewSet viewSet = reader.execute(table);
259+
PointViewPtr view = *viewSet.begin();
260+
PointLayoutPtr layout = view->layout();
261+
Dimension::Id health = layout->findDim("Health");
262+
Dimension::Id foo = layout->findDim("Foo");
263+
264+
EXPECT_EQ(view->size(), 24u);
265+
int ex = 0;
266+
int ey = 0;
267+
int ez = 0;
268+
for (PointId id = 0; id < view->size(); ++id)
269+
{
270+
int f = view->getFieldAs<int>(foo, id);
271+
int h = view->getFieldAs<int>(health, id);
272+
int i = view->getFieldAs<int>(Dimension::Id::Intensity, id);
273+
int x = view->getFieldAs<int>(Dimension::Id::X, id);
274+
int y = view->getFieldAs<int>(Dimension::Id::Y, id);
275+
int z = view->getFieldAs<int>(Dimension::Id::Z, id);
276+
EXPECT_EQ(f, ((int)id / 12) % 2);
277+
EXPECT_EQ(h, ((int)id / 4) % 3);
278+
EXPECT_EQ(i, (int)id % 4);
279+
EXPECT_EQ(x, ex);
280+
EXPECT_EQ(y, ey);
281+
EXPECT_EQ(z, ez);
282+
// Array is 2 x 3 x 4
283+
ez++;
284+
if (ez == 4)
285+
{
286+
ez = 0;
287+
ey++;
288+
if (ey == 3)
289+
{
290+
ey = 0;
291+
ex++;
292+
}
293+
}
294+
}
295+
}
296+
297+
TEST(NumpyReaderTest, four_floats_script)
298+
{
299+
StageFactory f;
300+
301+
Options opts;
302+
opts.add("filename", Support::datapath("four-floats.py"));
303+
opts.add("order", "row");
304+
opts.add("function", "load");
305+
opts.add("module", "four_floats");
306+
opts.add("fargs", Support::datapath("four-floats-be.bin"));
307+
308+
NumpyReader reader;
309+
reader.setOptions(opts);
310+
311+
PointTable table;
312+
313+
reader.prepare(table);
314+
315+
PointViewSet viewSet = reader.execute(table);
316+
PointViewPtr view = *viewSet.begin();
317+
PointLayoutPtr layout = view->layout();
318+
319+
EXPECT_EQ(view->size(), 10u);
320+
int ex = 0;
321+
int ey = 0;
322+
int ez = 0;
323+
int factor = 10;
324+
for (PointId id = 0; id < view->size(); ++id)
325+
{
326+
int t = view->getFieldAs<int>(Dimension::Id::GpsTime, id);
327+
int x = view->getFieldAs<int>(Dimension::Id::X, id);
328+
int y = view->getFieldAs<int>(Dimension::Id::Y, id);
329+
int z = view->getFieldAs<int>(Dimension::Id::Z, id);
330+
EXPECT_EQ(x, (int)id * 1 * factor);
331+
EXPECT_EQ(y, (int)id * 2 * factor);
332+
EXPECT_EQ(z, (int)id * 3 * factor);
333+
EXPECT_EQ(t, (int)id * 4 * factor);
334+
}
335+
}

‎pdal/test/data/four-floats-be.bin‎

160 Bytes
Binary file not shown.

‎pdal/test/data/four-floats.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
import numpy as np
2+
3+
4+
dtype = np.dtype([('X', '>f4'), ('Y', '>f4'), ('Z', '<f4'), ('GPSTime', '>f4')])
5+
6+
def load(filename):
7+
data = open(filename, 'rb').read()
8+
array = np.frombuffer(data, dtype=dtype)
9+
10+
return array

‎pdal/test/data/threedim.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
import numpy as np
2+
3+
def load(filename):
4+
array = np.load(filename)
5+
return array

0 commit comments

Comments
 (0)