Skip to content

Commit be3f3c3

Browse files
wolfvwolfv
committed
add tests, cmake etc.
1 parent e185add commit be3f3c3

File tree

17 files changed

+404
-59
lines changed

17 files changed

+404
-59
lines changed

‎CMakeLists.txt‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,6 @@ message(STATUS "Found zlib: ${zlib_INCLUDE_DIRS}")
4646

4747
set(XTENSOR_IO_HEADERS
4848
${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/ximageio.hpp
49-
${XTENSOR_IO_INCLUDE_DIR}/thirdparty/zstr/strict_fstream.hpp
50-
${XTENSOR_IO_INCLUDE_DIR}/thirdparty/zstr/zstr.hpp
5149
${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xnpz.hpp
5250
${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xtensor_io_config.hpp
5351
)
@@ -80,6 +78,9 @@ include(CMakePackageConfigHelpers)
8078
install(FILES ${XTENSOR_IO_HEADERS}
8179
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/xtensor-io)
8280

81+
install(DIRECTORY ${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/thirdparty/zstr
82+
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/xtensor-io/thirdparty)
83+
8384
set(XTENSOR_IO_CMAKECONFIG_INSTALL_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}" CACHE
8485
STRING "install path for xtensor-ioConfig.cmake")
8586

include/thirdparty/zstr/strict_fstream.hpp renamed to include/xtensor-io/thirdparty/zstr/strict_fstream.hpp

File renamed without changes.

include/thirdparty/zstr/zstr.hpp renamed to include/xtensor-io/thirdparty/zstr/zstr.hpp

Lines changed: 18 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -146,16 +146,16 @@ class istreambuf
146146
{
147147
// empty input buffer: refill from the start
148148
in_buff_start = in_buff;
149-
std::streamsize sz = sbuf_p->sgetn(in_buff, buff_size);
149+
std::streamsize sz = sbuf_p->sgetn(in_buff, (std::streamsize) buff_size);
150150
in_buff_end = in_buff + sz;
151151
if (in_buff_end == in_buff_start) break; // end of input
152152
}
153153
// auto detect if the stream contains text or deflate data
154154
if (auto_detect && !auto_detect_run)
155155
{
156156
auto_detect_run = true;
157-
unsigned char b0 = *reinterpret_cast< unsigned char * >(in_buff_start);
158-
unsigned char b1 = *reinterpret_cast< unsigned char * >(in_buff_start + 1);
157+
unsigned char b0 = *reinterpret_cast<unsigned char *>(in_buff_start);
158+
unsigned char b1 = *reinterpret_cast<unsigned char *>(in_buff_start + 1);
159159
// Ref:
160160
// http://en.wikipedia.org/wiki/Gzip
161161
// http://stackoverflow.com/questions/9050260/what-does-a-zlib-header-look-like
@@ -177,11 +177,14 @@ class istreambuf
177177
else
178178
{
179179
// run inflate() on input
180-
if (! zstrm_p) zstrm_p = new detail::z_stream_wrapper(true);
181-
zstrm_p->next_in = reinterpret_cast< decltype(zstrm_p->next_in) >(in_buff_start);
182-
zstrm_p->avail_in = in_buff_end - in_buff_start;
183-
zstrm_p->next_out = reinterpret_cast< decltype(zstrm_p->next_out) >(out_buff_free_start);
184-
zstrm_p->avail_out = (out_buff + buff_size) - out_buff_free_start;
180+
if (!zstrm_p)
181+
{
182+
zstrm_p = new detail::z_stream_wrapper(true);
183+
}
184+
zstrm_p->next_in = reinterpret_cast<decltype(zstrm_p->next_in)>(in_buff_start);
185+
zstrm_p->avail_in = static_cast<decltype(zstrm_p->avail_in)>(in_buff_end - in_buff_start);
186+
zstrm_p->next_out = reinterpret_cast<decltype(zstrm_p->next_out)>(out_buff_free_start);
187+
zstrm_p->avail_out = static_cast<decltype(zstrm_p->avail_out)>((out_buff + buff_size) - out_buff_free_start);
185188
int ret = inflate(zstrm_p, Z_NO_FLUSH);
186189
// process return code
187190
if (ret != Z_OK && ret != Z_STREAM_END) throw Exception(zstrm_p, ret);
@@ -247,12 +250,12 @@ class ostreambuf
247250
{
248251
while (true)
249252
{
250-
zstrm_p->next_out = reinterpret_cast< decltype(zstrm_p->next_out) >(out_buff);
251-
zstrm_p->avail_out = buff_size;
253+
zstrm_p->next_out = reinterpret_cast<decltype(zstrm_p->next_out)>(out_buff);
254+
zstrm_p->avail_out = static_cast<decltype(zstrm_p->avail_out)>(buff_size);
252255
int ret = deflate(zstrm_p, flush);
253256
if (ret != Z_OK && ret != Z_STREAM_END && ret != Z_BUF_ERROR) throw Exception(zstrm_p, ret);
254-
std::streamsize sz = sbuf_p->sputn(out_buff, reinterpret_cast< decltype(out_buff) >(zstrm_p->next_out) - out_buff);
255-
if (sz != reinterpret_cast< decltype(out_buff) >(zstrm_p->next_out) - out_buff)
257+
std::streamsize sz = sbuf_p->sputn(out_buff, reinterpret_cast<decltype(out_buff)>(zstrm_p->next_out) - out_buff);
258+
if (sz != reinterpret_cast<decltype(out_buff)>(zstrm_p->next_out) - out_buff)
256259
{
257260
// there was an error in the sink stream
258261
return -1;
@@ -283,7 +286,7 @@ class ostreambuf
283286
virtual std::streambuf::int_type overflow(std::streambuf::int_type c = traits_type::eof())
284287
{
285288
zstrm_p->next_in = reinterpret_cast< decltype(zstrm_p->next_in) >(pbase());
286-
zstrm_p->avail_in = pptr() - pbase();
289+
zstrm_p->avail_in = static_cast<decltype(zstrm_p->avail_in)>(pptr() - pbase());
287290
while (zstrm_p->avail_in > 0)
288291
{
289292
int r = deflate_loop(Z_NO_FLUSH);
@@ -294,7 +297,7 @@ class ostreambuf
294297
}
295298
}
296299
setp(in_buff, in_buff + buff_size);
297-
return traits_type::eq_int_type(c, traits_type::eof()) ? traits_type::eof() : sputc(c);
300+
return traits_type::eq_int_type(c, traits_type::eof()) ? traits_type::eof() : sputc((char) c);
298301
}
299302
virtual int sync()
300303
{
@@ -328,7 +331,7 @@ class istream
328331
exceptions(std::ios_base::badbit);
329332
}
330333

331-
istream(std::istream & is, std::streamsize buff_size)
334+
istream(std::istream & is, std::size_t buff_size)
332335
: std::istream(new istreambuf(is.rdbuf(), buff_size))
333336
{
334337
exceptions(std::ios_base::badbit);

‎include/xtensor-io/ximageio.hpp‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
#include <stdexcept>
22
#include <string>
3+
34
#include <OpenImageIO/imageio.h>
45

56
#include "xtensor/xarray.hpp"
7+
#include "xtensor/xeval.hpp"
68

79
using namespace OIIO;
810

@@ -36,20 +38,22 @@ namespace xt
3638
}
3739

3840
template <class E>
39-
void dump_image(std::string filename, const xexpression<E>& e)
41+
void dump_image(std::string filename, const xexpression<E>& e, int quality = 90)
4042
{
41-
const E& ex = e.derived_cast();
43+
auto&& ex = eval(e.derived_cast());
4244

4345
ImageOutput *out = ImageOutput::create(filename);
4446
if (!out)
4547
{
4648
// something went wrong
4749
throw std::runtime_error("Couldn't open file to write image.");
4850
}
49-
ImageSpec spec(ex.shape()[0], ex.shape()[1], ex.shape()[2], TypeDesc::UINT8);
51+
ImageSpec spec((int) ex.shape()[0], (int) ex.shape()[1], (int) ex.shape()[2], TypeDesc::UINT8);
52+
53+
spec.attribute("CompressionQuality", quality);
5054

5155
out->open(filename, spec);
52-
out->write_image (TypeDesc::UINT8, ex.raw_data());
56+
out->write_image(TypeDesc::UINT8, ex.raw_data());
5357
out->close();
5458
ImageOutput::destroy(out);
5559
}

‎include/xtensor-io/xnpz.hpp‎

Lines changed: 33 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
#include <vector>
1515
#include <typeinfo>
1616
#include <fstream>
17-
#include <iostream>
1817
#include <cassert>
1918
#include <map>
2019
#include <memory>
@@ -61,7 +60,6 @@ namespace xt
6160
//if we've reached the global header, stop reading
6261
if(local_header[2] != 0x03 || local_header[3] != 0x04)
6362
{
64-
std::cout << "Reached global header" << std::endl;
6563
break;
6664
}
6765

@@ -92,9 +90,7 @@ namespace xt
9290
uint16_t compr_method = *reinterpret_cast<uint16_t*>(&local_header[0] + 8);
9391
uint32_t compr_bytes = *reinterpret_cast<uint32_t*>(&local_header[0] + 18);
9492
uint32_t uncompr_bytes = *reinterpret_cast<uint32_t*>(&local_header[0] + 22);
95-
std::cout << compr_bytes << ", " << uncompr_bytes << std::endl;
9693

97-
std::cout << "Compression Method: " << compr_method << std::endl;
9894
if (compr_method == 0)
9995
{
10096
arrays.insert(result_type::value_type(varname,
@@ -202,7 +198,7 @@ namespace xt
202198
}
203199
else
204200
{
205-
stream.seekg(compr_bytes, std::ios_base::cur);
201+
stream.seekg((std::streamsize) compr_bytes, std::ios_base::cur);
206202
}
207203
}
208204
throw std::runtime_error("Array "s + search_varname + " not found in file: "s + filename);
@@ -240,19 +236,19 @@ namespace xt
240236

241237
self_type& operator<<(const char* rhs) {
242238
//write in little endian
243-
size_t len = strlen(rhs);
239+
std::size_t len = strlen(rhs);
244240
this->reserve(len);
245-
for(size_t byte = 0; byte < len; byte++)
241+
for(std::size_t byte = 0; byte < len; byte++)
246242
{
247243
this->push_back(rhs[byte]);
248244
}
249245
return *this;
250246
}
251247

252-
bool write(const char* char_arr, std::streamsize size)
248+
bool write(const char* char_arr, std::size_t size)
253249
{
254250
this->reserve(this->size() + size);
255-
for(size_t byte = 0; byte < size; byte++)
251+
for(std::size_t byte = 0; byte < size; byte++)
256252
{
257253
this->push_back(char_arr[byte]);
258254
}
@@ -268,8 +264,8 @@ namespace xt
268264

269265
void parse_zip_footer(std::istream& stream,
270266
uint16_t& nrecs,
271-
size_t& global_header_size,
272-
size_t& global_header_offset)
267+
std::streamsize& global_header_size,
268+
std::streamoff& global_header_offset)
273269
{
274270
std::vector<char> footer(22);
275271
stream.seekg(-22, std::ios_base::end);
@@ -300,28 +296,27 @@ namespace xt
300296
namespace detail
301297
{
302298
inline uint16_t msdos_time(uint16_t hour, uint16_t min, uint16_t sec) {
303-
return hour << 11 | min << 5 | (sec / 2);
299+
return (uint16_t) (hour << 11 | min << 5 | (sec / 2));
304300
}
305301

306302
inline uint16_t msdos_date(uint16_t year, uint16_t month, uint16_t day) {
307-
return (year - 1980) << 9 | month << 5 | day;
303+
return (uint16_t) ((year - 1980) << 9 | month << 5 | day);
308304
}
309305

310306
inline std::pair<uint16_t, uint16_t> time_pair()
311307
{
312308
std::time_t t = std::time(nullptr);
313309
auto tm = std::localtime(&t);
314-
return {msdos_time(tm->tm_hour, tm->tm_min, tm->tm_sec),
315-
msdos_date((tm->tm_year + 1900), tm->tm_mon + 1, tm->tm_mday)};
310+
return {msdos_time((uint16_t) tm->tm_hour, (uint16_t) tm->tm_min, (uint16_t) tm->tm_sec),
311+
msdos_date((uint16_t) (tm->tm_year + 1900), (uint16_t) (tm->tm_mon + 1), (uint16_t) tm->tm_mday)};
316312
}
317313
}
318314

319315
template <class E>
320316
void dump_npz(std::string filename,
321317
std::string varname, const xt::xexpression<E>& e,
322318
bool compression = false,
323-
bool append_to_existing_file = true,
324-
std::string mode = "w")
319+
bool append_to_existing_file = true)
325320
{
326321
//first, append a .npy to the fname
327322
varname += ".npy";
@@ -330,20 +325,20 @@ namespace xt
330325
std::ifstream in_stream(filename, std::ios_base::in | std::ios_base::binary);
331326

332327
uint16_t nrecs = 0;
333-
size_t global_header_offset = 0;
328+
std::streamoff global_header_offset = 0;
334329
detail::binary_vector global_header;
335330

336-
if (in_stream)
331+
if (in_stream && append_to_existing_file)
337332
{
338333
// zip file exists. we need to add a new npy file to it.
339334
// first read the footer. this gives us the offset and size
340335
// of the global header then read and store the global header.
341336
// below, we will write the the new data at the start of the
342337
// global header then append the global header and footer below it
343-
size_t global_header_size;
338+
std::streamsize global_header_size;
344339
detail::parse_zip_footer(in_stream, nrecs, global_header_size, global_header_offset);
345340
in_stream.seekg(global_header_offset);
346-
global_header.resize(global_header_size);
341+
global_header.resize((std::size_t) global_header_size);
347342
in_stream.read(&global_header[0], global_header_size);
348343
if(!in_stream)
349344
{
@@ -366,9 +361,9 @@ namespace xt
366361
detail::dump_npy_stream(array_contents, e);
367362

368363
//get the CRC of the data to be added
369-
uint32_t crc = crc32(0L, (uint8_t*) &array_contents[0], array_contents.size());
370-
size_t uncompressed_size = array_contents.size();
371-
size_t compressed_size = uncompressed_size;
364+
uint32_t crc = (uint32_t) crc32(0L, (uint8_t*) &array_contents[0], (unsigned int) array_contents.size());
365+
std::size_t uncompressed_size = array_contents.size();
366+
std::size_t compressed_size = uncompressed_size;
372367

373368
uint16_t compression_method = 0;
374369

@@ -377,13 +372,12 @@ namespace xt
377372
std::ostringstream ss;
378373
compression_method = 8;
379374
zstr::ostream compressed_array(ss, compression_method);
380-
compressed_array.write(&array_contents[0], array_contents.size());
375+
compressed_array.write(&array_contents[0], (std::streamsize) array_contents.size());
381376
compressed_array.flush();
382377
array_contents.clear();
383378
array_contents << ss.str();
384379
compressed_size = array_contents.size();
385380
}
386-
387381
auto date_time_pair = detail::time_pair();
388382

389383
//build the local header
@@ -392,9 +386,9 @@ namespace xt
392386
<< (uint16_t) 0x0403 // second part of signature
393387
<< (uint16_t) 20 // minimum version to extract
394388
<< (uint16_t) 0 // general purpose bit flag
395-
<< (uint16_t) compression_method // compression method
396-
<< (uint16_t) date_time_pair.first // file last mod time
397-
<< (uint16_t) date_time_pair.second // file last mod date
389+
<< (uint16_t) compression_method // compression method
390+
<< (uint16_t) date_time_pair.first // file last mod time
391+
<< (uint16_t) date_time_pair.second // file last mod date
398392
<< (uint32_t) crc // checksum
399393
<< (uint32_t) compressed_size // compressed size
400394
<< (uint32_t) uncompressed_size // uncompressed size
@@ -403,9 +397,10 @@ namespace xt
403397
<< varname; // file name
404398

405399
//build global header
406-
global_header << "PK" // first part of sig
407-
<< (uint16_t) 0x0201 // second part of sig
408-
<< (uint16_t) 20; // version made by
400+
global_header << "PK" // first part of sig
401+
<< (uint16_t) 0x0201 // second part of sig
402+
<< (uint16_t) 0x0314; // version made by .. hardcoded to what NumPy uses here
403+
// (not sure if this depends on zlib version)
409404

410405
// Copy from local header
411406
global_header.insert(global_header.end(),
@@ -415,7 +410,7 @@ namespace xt
415410
global_header << (uint16_t) 0 // file comment length
416411
<< (uint16_t) 0 // disk number where file starts
417412
<< (uint16_t) 0 // internal file attributes
418-
<< (uint32_t) 0 // external file attributes
413+
<< (uint32_t) 0x81800000 // external file attributes (taken from numpy)
419414
<< (uint32_t) global_header_offset // relative offset of local file header,
420415
// since it begins where the global header used to begin
421416
<< varname;
@@ -430,14 +425,14 @@ namespace xt
430425
<< (uint16_t) (nrecs + 1) // total number of records
431426
<< (uint32_t) global_header.size() // nbytes of global headers
432427
// offset of start of global headers, since global header now starts after newly written array
433-
<< (uint32_t) (global_header_offset + compressed_size + local_header.size())
428+
<< (uint32_t) ((std::size_t) global_header_offset + compressed_size + local_header.size())
434429
<< (uint16_t) 0; // zip file comment length
435430

436431
// write everything
437-
stream.write(&local_header[0], local_header.size());
438-
stream.write(&array_contents[0], array_contents.size());
439-
stream.write(&global_header[0], global_header.size());
440-
stream.write(&footer[0], footer.size());
432+
stream.write(&local_header[0], (std::streamsize) local_header.size());
433+
stream.write(&array_contents[0], (std::streamsize) array_contents.size());
434+
stream.write(&global_header[0], (std::streamsize) global_header.size());
435+
stream.write(&footer[0], (std::streamsize) footer.size());
441436
}
442437

443438
} // namespace xt

0 commit comments

Comments
 (0)