Skip to content

Commit ad4e472

Browse files
arahlinarahlin
committed
Add support for full-depth compression and decompression of float and int32 timestreams
1 parent 74809df commit ad4e472

File tree

3 files changed

+127
-16
lines changed

3 files changed

+127
-16
lines changed

‎core/include/core/G3Timestream.h‎

Lines changed: 21 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -27,19 +27,21 @@ class G3Timestream : public G3FrameObject {
2727

2828
G3Timestream(const G3Timestream &r);
2929
G3Timestream(std::vector<double>::size_type s = 0, double val = 0) :
30-
units(None), use_flac_(0),
30+
units(None), use_flac_(0), flac_depth_(24),
3131
buffer_((s == 0) ? NULL : new std::vector<double>(s, val)),
3232
data_((s == 0) ? NULL : &(*buffer_)[0]), len_(s),
3333
data_type_(TS_DOUBLE) {}
3434
template <typename Iterator> G3Timestream(Iterator l, Iterator r) :
35-
units(None), use_flac_(0), buffer_(new std::vector<double>(l, r)),
35+
units(None), use_flac_(0), flac_depth_(24),
36+
buffer_(new std::vector<double>(l, r)),
3637
data_(&(*buffer_)[0]), len_(buffer_->size()), data_type_(TS_DOUBLE) {}
3738
virtual ~G3Timestream() {
3839
if (buffer_) delete buffer_;
3940
}
4041

4142
// FLAC compression levels range from 0-9. 0 means do not use FLAC.
4243
void SetFLACCompression(int compression_level);
44+
void SetFLACDepth(int bit_depth);
4345

4446
TimestreamUnits units;
4547
G3Time start, stop;
@@ -105,6 +107,7 @@ class G3Timestream : public G3FrameObject {
105107

106108
double GetSampleRate() const;
107109
uint8_t GetCompressionLevel() const{ return use_flac_; }
110+
uint8_t GetBitDepth() const { return flac_depth_; }
108111

109112
template <class A> void load(A &ar, unsigned v);
110113
template <class A> void save(A &ar, unsigned v) const;
@@ -135,17 +138,19 @@ class G3Timestream : public G3FrameObject {
135138
friend class G3TimestreamPythonHelpers;
136139

137140
uint8_t use_flac_;
141+
uint8_t flac_depth_;
138142

139143
std::vector<double> *buffer_;
140144
std::shared_ptr<void> root_data_ref_;
141145
void *data_;
142146
size_t len_;
143-
enum {
147+
enum DataType {
144148
TS_DOUBLE,
145149
TS_FLOAT,
146150
TS_INT32,
147151
TS_INT64
148-
} data_type_;
152+
};
153+
DataType data_type_;
149154

150155
template<typename T>
151156
struct TimeStreamTypeResolver{
@@ -208,7 +213,9 @@ class G3TimestreamMap : public G3FrameObject,
208213
void SetUnits(G3Timestream::TimestreamUnits units);
209214
/// FLAC compression levels range from 0-9. 0 means do not use FLAC.
210215
uint8_t GetCompressionLevel() const;
216+
uint8_t GetBitDepth() const;
211217
void SetFLACCompression(int compression_level);
218+
void SetFLACDepth(int bit_depth);
212219

213220
// Compact underlying data storage into a contiguous 2D block.
214221
// This invalidates any references to data inside any member
@@ -227,9 +234,10 @@ class G3TimestreamMap : public G3FrameObject,
227234
static G3TimestreamMap MakeCompact(const std::vector<std::string>& keys, std::size_t n_samples,
228235
G3Time start, G3Time stop,
229236
G3Timestream::TimestreamUnits units=G3Timestream::None,
230-
int compression_level=0){
237+
int compression_level=0, int bit_depth=24){
231238
std::shared_ptr<SampleType[]> data(new SampleType[n_samples*keys.size()]);
232-
return MakeCompact(keys, n_samples, data, start, stop, units, compression_level);
239+
return MakeCompact(keys, n_samples, data, start, stop, units,
240+
compression_level, bit_depth);
233241
}
234242

235243
/// Construct a map using an existing contiguous 2D block of data as the underlying storage.
@@ -246,16 +254,18 @@ class G3TimestreamMap : public G3FrameObject,
246254
std::shared_ptr<SampleType[]> data,
247255
G3Time start, G3Time stop,
248256
G3Timestream::TimestreamUnits units=G3Timestream::None,
249-
int compression_level=0){
257+
int compression_level=0, int bit_depth=24){
250258
G3TimestreamMap map;
251-
map.FromBuffer(keys, n_samples, data, start, stop, units, compression_level);
259+
map.FromBuffer(keys, n_samples, data, start, stop, units,
260+
compression_level, bit_depth);
252261
return map;
253262
}
254263

255264
template<typename SampleType>
256265
void FromBuffer(const std::vector<std::string>& keys, std::size_t n_samples,
257266
std::shared_ptr<SampleType[]> data, G3Time start, G3Time stop,
258-
G3Timestream::TimestreamUnits units=G3Timestream::None, int compression_level=0) {
267+
G3Timestream::TimestreamUnits units=G3Timestream::None, int compression_level=0,
268+
int bit_depth=24) {
259269
const auto data_type=G3Timestream::TimeStreamTypeResolver<SampleType>::type_tag;
260270
std::size_t offset = 0;
261271
for (const auto& key : keys) {
@@ -264,6 +274,7 @@ class G3TimestreamMap : public G3FrameObject,
264274
ts->stop = stop;
265275
ts->units = units;
266276
ts->use_flac_ = compression_level;
277+
ts->flac_depth_ = bit_depth;
267278
ts->root_data_ref_ = data;
268279
ts->data_ = data.get() + offset;
269280
ts->data_type_ = data_type;
@@ -284,7 +295,7 @@ namespace cereal {
284295
template <class A> struct specialize<A, G3TimestreamMap, cereal::specialization::member_serialize> {};
285296
}
286297

287-
G3_SERIALIZABLE(G3Timestream, 3);
298+
G3_SERIALIZABLE(G3Timestream, 4);
288299
G3_SERIALIZABLE(G3TimestreamMap, 4);
289300

290301
#endif

‎core/src/G3SuperTimestream.cxx‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -248,13 +248,19 @@ template <class A> void G3SuperTimestream::load(A &ar, unsigned v)
248248
FromBuffer(names, nsamp_filled, data, start, stop);
249249
buf = (char *)data.get();
250250
elsize = sizeof(int32_t);
251+
252+
SetFLACCompression(flac_level);
253+
SetFLACDepth(32);
251254
break;
252255
}
253256
case TYPE_NUM_FLOAT32: {
254257
std::shared_ptr<float[]> data(new float[size]);
255258
FromBuffer(names, nsamp_filled, data, start, stop);
256259
buf = (char *)data.get();
257260
elsize = sizeof(int32_t);
261+
262+
SetFLACCompression(flac_level);
263+
SetFLACDepth(32);
258264
break;
259265
}
260266
case TYPE_NUM_INT64: {

‎core/src/G3Timestream.cxx‎

Lines changed: 100 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -122,9 +122,20 @@ template <class A> void G3Timestream::save(A &ar, unsigned v) const
122122
if (units != Counts && units != None)
123123
log_fatal("Cannot use FLAC on non-counts timestreams");
124124

125+
DataType data_type_out = TS_FLOAT;
126+
if (flac_depth_ == 32) {
127+
if (data_type_ == TS_INT64 || data_type_ == TS_INT32)
128+
data_type_out = TS_INT32;
129+
}
130+
131+
ar & cereal::make_nvp("flac_depth", flac_depth_);
132+
ar & cereal::make_nvp("data_type", data_type_out);
133+
125134
// Copy to 24-bit integers
126135
inbuf.resize(size());
127-
switch (data_type_) {
136+
switch (flac_depth_) {
137+
case 24:
138+
switch (data_type_) {
128139
case TS_DOUBLE:
129140
for (size_t i = 0; i < size(); i++)
130141
inbuf[i] = ((int32_t(((double *)data_)[i]) & 0x00ffffff) << 8) >> 8;
@@ -149,6 +160,31 @@ template <class A> void G3Timestream::save(A &ar, unsigned v) const
149160
break;
150161
default:
151162
log_fatal("Unknown timestream datatype %d", data_type_);
163+
}
164+
break;
165+
case 32:
166+
switch (data_type_) {
167+
case TS_DOUBLE:
168+
for (size_t i = 0; i < size(); i++)
169+
inbuf[i] = int32_t(float(((double *)data_)[i]));
170+
break;
171+
case TS_FLOAT:
172+
for (size_t i = 0; i < size(); i++)
173+
inbuf[i] = int32_t(((float *)data_)[i]);
174+
break;
175+
case TS_INT32:
176+
memcpy(inbuf.data(), data_, size() * sizeof(int32_t));
177+
break;
178+
case TS_INT64:
179+
for (size_t i = 0; i < size(); i++)
180+
inbuf[i] = int32_t(((int64_t *)data_)[i]);
181+
break;
182+
default:
183+
log_fatal("Unknown timestream datatype %d", data_type_);
184+
}
185+
break;
186+
default:
187+
log_fatal("Invalid FLAC bit depth %d", flac_depth_);
152188
}
153189
chanmap[0] = &inbuf[0];
154190

@@ -182,7 +218,7 @@ template <class A> void G3Timestream::save(A &ar, unsigned v) const
182218
FLAC__StreamEncoder *encoder = FLAC__stream_encoder_new();
183219
FLAC__stream_encoder_set_channels(encoder, 1);
184220
// XXX: should assert if high-order 8 bits are not clear
185-
FLAC__stream_encoder_set_bits_per_sample(encoder, 24);
221+
FLAC__stream_encoder_set_bits_per_sample(encoder, flac_depth_);
186222
FLAC__stream_encoder_set_compression_level(encoder, use_flac_);
187223
FLAC__stream_encoder_set_do_md5(encoder, false);
188224
FLAC__stream_encoder_init_stream(encoder,
@@ -262,6 +298,13 @@ template <class A> void G3Timestream::load(A &ar, unsigned v)
262298
if (units != Counts && units != None)
263299
log_fatal("Cannot use FLAC on non-counts timestreams");
264300

301+
if (v >= 4) {
302+
ar & cereal::make_nvp("flac_depth", flac_depth_);
303+
ar & cereal::make_nvp("data_type", data_type_);
304+
} else {
305+
data_type_ = TS_FLOAT;
306+
}
307+
265308
ar & cereal::make_nvp("nanflag", nanflag);
266309
if (nanflag == SomeNan)
267310
ar & cereal::make_nvp("nanmask", nanbuf);
@@ -281,14 +324,22 @@ template <class A> void G3Timestream::load(A &ar, unsigned v)
281324
FLAC__stream_decoder_finish(decoder);
282325
FLAC__stream_decoder_delete(decoder);
283326

327+
// Short-circuit for full-depth INT32.
328+
if (data_type_ == TS_INT32 && flac_depth_ == 32) {
329+
root_data_ref_ = std::shared_ptr<std::vector<int32_t> >(
330+
callback.outbuf);
331+
len_ = callback.outbuf->size();
332+
data_ = &(*callback.outbuf)[0];
333+
return;
334+
}
335+
284336
// Represent data as floats internally. These have the same
285337
// significand depth (24 bits) as the max. bit depth of the
286338
// reference FLAC encoder we use, so no data are lost, and
287339
// allow NaNs, unlike int32_ts, which we try to pull through
288340
// the process to signal missing data.
289341
float *data = new float[callback.outbuf->size()];
290342
root_data_ref_ = std::shared_ptr<float[]>(data);
291-
data_type_ = TS_FLOAT;
292343
len_ = callback.outbuf->size();
293344
data_ = data;
294345

@@ -361,7 +412,7 @@ template <class A> void G3Timestream::load(A &ar, unsigned v)
361412

362413
G3Timestream::G3Timestream(const G3Timestream &r) :
363414
units(r.units), start(r.start), stop(r.stop), use_flac_(r.use_flac_),
364-
len_(r.len_), data_type_(r.data_type_)
415+
flac_depth_(r.flac_depth_), len_(r.len_), data_type_(r.data_type_)
365416
{
366417
// Copy constructor needs to copy data, which always involves
367418
// allocating the internal buffer.
@@ -431,6 +482,14 @@ void G3Timestream::SetFLACCompression(int use_flac)
431482
#endif
432483
}
433484

485+
void G3Timestream::SetFLACDepth(int bit_depth)
486+
{
487+
if (bit_depth != 24 && bit_depth != 32)
488+
log_fatal("Invalid flac bit depth %d", bit_depth);
489+
490+
flac_depth_ = bit_depth;
491+
}
492+
434493
G3Timestream G3Timestream::operator +(const G3Timestream &r) const
435494
{
436495
G3Timestream ret(*this);
@@ -764,12 +823,32 @@ uint8_t G3TimestreamMap::GetCompressionLevel() const
764823
return begin()->second->use_flac_;
765824
}
766825

826+
uint8_t G3TimestreamMap::GetBitDepth() const
827+
{
828+
if (begin() == end())
829+
return 0;
830+
831+
return begin()->second->flac_depth_;
832+
}
833+
767834
void G3TimestreamMap::SetFLACCompression(int compression_level)
768835
{
836+
// Check for errors
837+
begin()->second->SetFLACCompression(compression_level);
838+
769839
for (auto& ts : *this)
770840
ts.second->use_flac_ = compression_level;
771841
}
772842

843+
void G3TimestreamMap::SetFLACDepth(int bit_depth)
844+
{
845+
// Check for errors
846+
begin()->second->SetFLACDepth(bit_depth);
847+
848+
for (auto& ts : *this)
849+
ts.second->flac_depth_ = bit_depth;
850+
}
851+
773852
void G3TimestreamMap::Compactify()
774853
{
775854
// Check if already compacted
@@ -1069,7 +1148,8 @@ struct PyBufferOwner {
10691148
static G3TimestreamMapPtr
10701149
G3TimestreamMap_from_numpy(std::vector<std::string> keys,
10711150
boost::python::object data, G3Time start, G3Time stop,
1072-
G3Timestream::TimestreamUnits units, int compression_level, bool copy_data)
1151+
G3Timestream::TimestreamUnits units, int compression_level,
1152+
bool copy_data, int bit_depth)
10731153
{
10741154
G3TimestreamMapPtr x(new G3TimestreamMap);
10751155

@@ -1116,6 +1196,7 @@ G3TimestreamMap_from_numpy(std::vector<std::string> keys,
11161196
templ.start = start;
11171197
templ.stop = stop;
11181198
templ.SetFLACCompression(compression_level);
1199+
templ.SetFLACDepth(bit_depth);
11191200
if (strcmp(v->v.format, "d") == 0) {
11201201
templ.data_type_ = G3Timestream::TS_DOUBLE;
11211202
} else if (strcmp(v->v.format, "f") == 0) {
@@ -1347,6 +1428,10 @@ PYBINDINGS("core") {
13471428
"Pass True to turn on FLAC compression when serialized. "
13481429
"FLAC compression only works if the timestream is in units of "
13491430
"counts.")
1431+
.def("SetFLACDepth", &G3Timestream::SetFLACDepth,
1432+
"Change the bit depth for FLAC compression. "
1433+
"FLAC compression only works if the timestream is in units of "
1434+
"counts.")
13501435
.def_readwrite("units", &G3Timestream::units,
13511436
"Units of the data in the timestream, stored as one of the "
13521437
"members of core.G3TimestreamUnits.")
@@ -1361,6 +1446,8 @@ PYBINDINGS("core") {
13611446
.add_property("compression_level", &G3Timestream::GetCompressionLevel,
13621447
&G3Timestream::SetFLACCompression, "Level of FLAC compression used for this timestream. "
13631448
"This can only be non-zero if the timestream is in units of counts.")
1449+
.add_property("bit_depth", &G3Timestream::GetBitDepth,
1450+
&G3Timestream::SetFLACDepth, "Bit depth of FLAC compression used for this timestream.")
13641451
.def("_assert_congruence", &G3Timestream::G3TimestreamPythonHelpers::G3Timestream_assert_congruence,
13651452
"log_fatal() if units, length, start, or stop times do not match")
13661453
.def("_cxxslice", &G3Timestream::G3TimestreamPythonHelpers::G3Timestream_getslice, "Slice-only __getitem__")
@@ -1382,7 +1469,7 @@ PYBINDINGS("core") {
13821469
bp::default_call_policies(),
13831470
(bp::arg("keys"), bp::arg("data"), bp::arg("start")=G3Time(0),
13841471
bp::arg("stop")=G3Time(0), bp::arg("units") = G3Timestream::TimestreamUnits::None,
1385-
bp::arg("compression_level") = 0, bp::arg("copy_data") = true)),
1472+
bp::arg("compression_level") = 0, bp::arg("copy_data") = true, bp::arg("bit_depth") = 24)),
13861473
"Create a timestream map from a numpy array or other numeric python iterable. "
13871474
"Each row of the 2D input array will correspond to a single timestream, with "
13881475
"the key set to the correspondingly-indexed entry of <keys>. If <copy_data> "
@@ -1398,6 +1485,10 @@ PYBINDINGS("core") {
13981485
"Pass True to turn on FLAC compression when serialized. "
13991486
"FLAC compression only works if the timestreams are in units of "
14001487
"counts.")
1488+
.def("SetFLACDepth", &G3TimestreamMap::SetFLACDepth,
1489+
"Change the bit depth for FLAC compression. "
1490+
"FLAC compression only works if the timestreams are in units of "
1491+
"counts.")
14011492
.add_property("start", &G3TimestreamMap::GetStartTime,
14021493
&G3TimestreamMap::SetStartTime,
14031494
"Time of the first sample in the time stream")
@@ -1417,6 +1508,9 @@ PYBINDINGS("core") {
14171508
&G3TimestreamMap::SetFLACCompression,
14181509
"Level of FLAC compression used for this timestream map. "
14191510
"This can only be non-zero if the timestream is in units of counts.")
1511+
.add_property("bit_depth", &G3TimestreamMap::GetBitDepth,
1512+
&G3TimestreamMap::SetFLACDepth,
1513+
"Bit depth of FLAC compression used for this timestream map.")
14201514
;
14211515
register_pointer_conversions<G3TimestreamMap>();
14221516

0 commit comments

Comments
 (0)