Consistent G3Frame versioning and serialization (#181)

This PR uses the same versioning and archive serialization methods for G3Frame instances as those for all other serializable objects in the library.

The G3Frame class version is defined in the header, along with specialized load/save methods for IO with cereal binary archives.

The load/save methods for streams are renamed to loads/saves to ensure that the cereal library binds the correct serialization methods.

This PR does not change the serialized blob structure or increment the serialization version, so full compatibility with existing data is maintained.

Author: arahlin

core/include/core/G3Frame.h

@@ -96,8 +96,12 @@ class G3Frame {
 	G3Frame &operator = (const G3Frame &);
 
 	// Serialize (or deserialize) from an IO stream.
-	template <typename T> void save(T &os) const;
-	template <typename T> void load(T &is);
+	template <typename T> void saves(T &os) const;
+	template <typename T> void loads(T &is);
+
+	// Serialize (or deserialize) from a cereal archive.
+	template <class A> void save(A &ar, unsigned) const;
+	template <class A> void load(A &ar, unsigned);
 
 	// Routines for handling the stored serialized copies of data.
 	// These are all const because they only manipulate caches and
@@ -134,6 +138,8 @@ class G3Frame {
 };
 
 G3_POINTERS(G3Frame);
+CEREAL_CLASS_VERSION(G3Frame, 1);
+CEREAL_SPECIALIZE_FOR_ALL_ARCHIVES(G3Frame, cereal::specialization::member_load_save);
 
 template <typename T>
 std::shared_ptr<const T> G3Frame::Get(const std::string &name,

core/src/G3Frame.cxx

@@ -234,14 +234,20 @@ std::ostream& operator<<(std::ostream& os, const G3Frame::FrameType &frame_type)
 }
 
 template <typename T>
-void G3Frame::save(T &os) const
+void G3Frame::saves(T &os) const
 {
 	using cereal::make_nvp;
-	uint32_t crc = 0;
-	uint32_t version(1), size(map_.size());
 
 	cereal::PortableBinaryOutputArchive ar(os);
-	ar << make_nvp("version", version);
+	ar << make_nvp("frame", *this);
+}
+
+template <class A>
+void G3Frame::save(A &ar, unsigned v) const
+{
+	using cereal::make_nvp;
+	uint32_t crc(0), size(map_.size());
+
 	ar << make_nvp("size", size);
 	ar << make_nvp("type", (uint32_t)type);
 	for (auto i = map_.begin(); i != map_.end(); i++) {
@@ -259,16 +265,24 @@ void G3Frame::save(T &os) const
 }
 
 template <typename T>
-void G3Frame::load(T &is)
+void G3Frame::loads(T &is)
 {
 	using cereal::make_nvp;
 
 	cereal::PortableBinaryInputArchive ar(is);
-	int version, size;
-	uint32_t xtype;
-	uint32_t crc(0), testcrc;
+	ar >> make_nvp("frame", *this);
+}
+
+template <class A>
+void G3Frame::load(A &ar, unsigned v)
+{
+	using cereal::make_nvp;
+	G3_CHECK_VERSION(v);
+
+	int size;
+	uint32_t xtype, testcrc;
+	uint32_t crc(0);
 
-	ar >> make_nvp("version", version);
 	ar >> make_nvp("size", size);
 	ar >> make_nvp("type", xtype);
 	type = FrameType(xtype);
@@ -368,13 +382,17 @@ void G3Frame::blob_encode(struct blob_container &blob)
 	item_os.flush();
 }
 
-template void G3Frame::load(g3_istream &);
-template void G3Frame::load(std::istream &);
-template void G3Frame::load(std::istringstream &);
+template void G3Frame::loads(g3_istream &);
+template void G3Frame::loads(G3BufferInputStream &);
+template void G3Frame::loads(std::istream &);
+template void G3Frame::loads(std::istringstream &);
 
-template void G3Frame::save(g3_ostream &) const;
-template void G3Frame::save(std::ostream &) const;
-template void G3Frame::save(std::ostringstream &) const;
+template void G3Frame::saves(g3_ostream &) const;
+template void G3Frame::saves(G3BufferOutputStream &) const;
+template void G3Frame::saves(std::ostream &) const;
+template void G3Frame::saves(std::ostringstream &) const;
+
+G3_SPLIT_SERIALIZABLE_CODE(G3Frame);
 
 G3FramePtr
 g3frame_char_constructor(std::string max_4_chars)
@@ -395,37 +413,6 @@ g3frame_char_constructor(std::string max_4_chars)
 	return G3FramePtr(new G3Frame(G3Frame::FrameType(code)));
 }
 
-// Use G3Frame::save()/G3Frame::load() to provide a pickle interface for frames
-struct g3frame_picklesuite : boost::python::pickle_suite
-{
-	static boost::python::tuple getstate(boost::python::object obj)
-	{
-		namespace bp = boost::python;
-		std::vector<char> buffer;
-		G3BufferOutputStream os(buffer);
-		(bp::extract<const G3Frame &>(obj))().save(os);
-		os.flush();
-
-		return boost::python::make_tuple(obj.attr("__dict__"),
-		    bp::object(bp::handle<>(
-		    PyBytes_FromStringAndSize(&buffer[0], buffer.size()))));
-	}
-
-	static void setstate(boost::python::object obj,
-	    boost::python::tuple state)
-	{
-		namespace bp = boost::python;
-		Py_buffer view;
-		PyObject_GetBuffer(bp::object(state[1]).ptr(), &view,
-		    PyBUF_SIMPLE);
-
-		bp::extract<bp::dict>(obj.attr("__dict__"))().update(state[0]);
-		G3BufferInputStream is((char *)view.buf, view.len);
-		(bp::extract<G3Frame &>(obj))().load(is);
-		PyBuffer_Release(&view);
-	}
-};
-
 static boost::python::list g3frame_keys(const G3Frame &map)
 {
         boost::python::list keys;
@@ -599,7 +586,7 @@ PYBINDINGS("core") {
 	      "where those serialized copies already exist. Saves memory for "
 	      "frames about to be written at the expense of CPU time to "
 	      "re-decode them if they are accessed again later.")
-	    .def_pickle(g3frame_picklesuite())
+	    .def_pickle(g3frameobject_picklesuite<G3Frame>())
 	;
 	register_vector_of<G3FramePtr>("Frame");
 	register_vector_of<G3FrameObjectPtr>("FrameObject");

core/src/G3MultiFileWriter.cxx

@@ -120,7 +120,7 @@ G3MultiFileWriter::CheckNewFile(G3FramePtr frame)
 	g3_ostream_to_path(stream_, filename, false, true);
 
 	for (auto i = metadata_cache_.begin(); i != metadata_cache_.end(); i++)
-		(*i)->save(stream_);
+		(*i)->saves(stream_);
 
 	return true;
 }
@@ -161,7 +161,7 @@ void G3MultiFileWriter::Process(G3FramePtr frame, std::deque<G3FramePtr> &out)
 	// And out to disk, making sure not to write again if it just went into
 	// the metadata cache and onto disk in CheckNewFile()
 	if (!new_file || !meta_cached)
-		frame->save(stream_);
+		frame->saves(stream_);
 
 done:
 	out.push_back(frame);

core/src/G3NetworkSender.cxx

@@ -174,7 +174,7 @@ void G3NetworkSender::SerializeFrame(serialization_task& task)
 	try{
 		netbuf_type buf(new std::vector<char>);
 		G3BufferOutputStream os(*buf);
-		task.input->save(os);
+		task.input->saves(os);
 		os.flush();
 		task.output.set_value(buf);
 	}

core/src/G3Reader.cxx

@@ -84,7 +84,7 @@ void G3Reader::Process(G3FramePtr frame, std::deque<G3FramePtr> &out)
 	}
 	frame = G3FramePtr(new G3Frame);
 	try {
-		frame->load(stream_);
+		frame->loads(stream_);
 	} catch (...) {
 		log_error("Exception raised while reading file %s",
 		    cur_file_.c_str());

core/src/G3Writer.cxx

@@ -31,7 +31,7 @@ void G3Writer::Process(G3FramePtr frame, std::deque<G3FramePtr> &out)
 	else if (streams_.size() == 0 ||
 	    std::find(streams_.begin(), streams_.end(), frame->type) !=
 	    streams_.end())
-		frame->save(stream_);
+		frame->saves(stream_);
 
 	out.push_back(frame);
 }