Use STL streams for buffer IO (#178)

arahlin · web-flow · commit 453f870ae258 · 2025-03-11T15:57:50.000-05:00
This reduces the boost::iostreams footprint to only file IO.
diff --git a/core/include/core/dataio.h b/core/include/core/dataio.h
@@ -50,17 +50,6 @@ off_t g3_istream_seek(g3_istream &stream, off_t offset);
  */
 off_t g3_istream_tell(g3_istream &stream);
 
-/**
- * Configure a filtering stream for G3Frame decompression from a memory buffer.
- *
- * @param  stream   A reference to the filtering istream that will be configured
- *                  by this function.  Must be instantiated prior to this
- *                  function.
- * @param  buffer   A pointer to a char buffer in memory.
- * @param  len      Size of the buffer in bytes.
- */
-void g3_istream_from_buffer(g3_istream &stream, const char *buf, size_t len);
-
 /**
  * Configure a filtering stream for G3Frame compression to a local file.
  *
@@ -87,16 +76,6 @@ void g3_ostream_to_path(g3_ostream &stream, const std::string &path,
  */
 size_t g3_ostream_count(g3_ostream &stream);
 
-/**
- * Configure a filtering stream for G3Frame compression to a memory buffer.
- *
- * @param  stream   A reference to the filtering ostream that will be configured
- *                  by this function.  Must be instantiated prior to this
- *                  function.
- * @param  buffer   A reference a char buffer in memory.
- */
-void g3_ostream_to_buffer(g3_ostream &stream, std::vector<char> &buf);
-
 /**
  * Check that the input filename is a valid filename on disk.
  *
diff --git a/core/include/core/serialization.h b/core/include/core/serialization.h
@@ -1,7 +1,7 @@
 #ifndef _G3_SERIALIZATION_H
 #define _G3_SERIALIZATION_H
 
-#include <dataio.h>
+#include <sstream>
 
 #include <cereal/archives/portable_binary.hpp>
 
@@ -22,15 +22,76 @@ template void x::serialize(cereal::PortableBinaryInputArchive &, unsigned); \
 template void x::save(cereal::PortableBinaryOutputArchive &, unsigned) const; \
 template void x::load(cereal::PortableBinaryInputArchive &, unsigned); \
 
+// Work around crummy slow implementation of xsgetn() in libc++
+class G3InputStreamBuffer : public std::basic_streambuf<char>
+{
+public:
+	G3InputStreamBuffer(std::vector<char> &vec) : basic_streambuf() {
+		setg(&vec[0], &vec[0], &vec[0] + vec.size());
+	}
+	G3InputStreamBuffer(char *buf, size_t len) : basic_streambuf() {
+		setg(buf, buf, buf + len);
+	}
+protected:
+	std::streamsize xsgetn(char_type *buf, std::streamsize n) {
+		std::streamsize to_read = egptr() - gptr();
+		if (to_read > n)
+			to_read = n;
+		memcpy(buf, gptr(), to_read);
+		gbump(to_read);
+		return to_read;
+	}
+};
+
+class G3BufferInputStream : public std::istream
+{
+public:
+	G3BufferInputStream(std::vector<char> &vec) : std::istream(&sbuf_), sbuf_(vec) {}
+	G3BufferInputStream(char *buf, size_t len) : std::istream(&sbuf_), sbuf_(buf, len) {}
+private:
+	G3InputStreamBuffer sbuf_;
+};
+
+class G3OutputStreamBuffer : public std::basic_streambuf<char>
+{
+public:
+	G3OutputStreamBuffer(std::vector<char> &buffer) : buffer_(buffer) {
+		setp(&buffer_[0], &buffer_[0] + buffer_.size());
+	}
+protected:
+	std::streamsize xsputn(const char* s, std::streamsize n) {
+		buffer_.insert(buffer_.end(), s, s + n);
+		pbump(n);
+		return n;
+	}
+
+	int overflow(int c = std::char_traits<char>::eof()) {
+		if (c != std::char_traits<char>::eof()) {
+			buffer_.push_back(static_cast<char>(c));
+			pbump(1);
+		}
+		return c;
+	}
+private:
+	std::vector<char>& buffer_;
+};
+
+class G3BufferOutputStream : public std::ostream
+{
+public:
+	G3BufferOutputStream(std::vector<char> &vec) : std::ostream(&sbuf_), sbuf_(vec) {}
+private:
+	G3OutputStreamBuffer sbuf_;
+};
+
 template <class T>
 struct g3frameobject_picklesuite : boost::python::pickle_suite
 {
 	static boost::python::tuple getstate(boost::python::object obj)
 	{
 		namespace bp = boost::python;
 		std::vector<char> buffer;
-		g3_ostream os;
-		g3_ostream_to_buffer(os, buffer);
+		G3BufferOutputStream os(buffer);
 		{
 			cereal::PortableBinaryOutputArchive ar(os);
 			ar << bp::extract<const T &>(obj)();
@@ -50,8 +111,7 @@ struct g3frameobject_picklesuite : boost::python::pickle_suite
 		PyObject_GetBuffer(bp::object(state[1]).ptr(), &view,
 		    PyBUF_SIMPLE);
 
-		g3_istream fis;
-		g3_istream_from_buffer(fis, (char *)view.buf, view.len);
+		G3BufferInputStream fis((char *)view.buf, view.len);
 		cereal::PortableBinaryInputArchive ar(fis);
 
 		bp::extract<bp::dict>(obj.attr("__dict__"))().update(state[0]);
diff --git a/core/src/G3Frame.cxx b/core/src/G3Frame.cxx
@@ -5,7 +5,6 @@
 #include <pybindings.h>
 #include <dataio.h>
 
-#include <sstream>
 #include <stdlib.h>
 #include <cxxabi.h>
 #include <algorithm>
@@ -14,24 +13,6 @@ namespace bp = boost::python;
 
 extern "C" unsigned long crc32c(unsigned long crc, const uint8_t *buf, unsigned int len);
 
-// Work around crummy slow implementation of xsgetn() in libc++
-class fast_streambuf : public std::basic_streambuf<char>
-{
-public:
-	fast_streambuf(std::vector<char> &vec) : basic_streambuf() {
-		setg(&vec[0], &vec[0], &vec[0] + vec.size());
-	}
-protected:
-	std::streamsize xsgetn(char_type *buf, std::streamsize n) {
-		std::streamsize to_read = egptr() - gptr();
-		if (to_read > n)
-			to_read = n;
-		memcpy(buf, gptr(), to_read);
-		gbump(to_read);
-		return to_read;
-	}
-};
-
 template <typename T>
 static std::string cxx_demangle(const T& v)
 {
@@ -313,15 +294,6 @@ void G3Frame::load(T &is)
 		    testcrc, crc);
 }
 
-template <>
-void G3Frame::load(std::vector<char> &data)
-{
-	fast_streambuf sb(data);
-	std::istream is(&sb);
-
-	load(is);
-}
-
 void G3Frame::DropBlobs(bool decode_all) const
 {
 	for (auto i = map_.begin(); i != map_.end(); i++) {
@@ -370,8 +342,7 @@ void G3Frame::blob_decode(struct blob_container &blob)
 	if (blob.frameobject)
 		return;
 
-	fast_streambuf sb(*blob.blob);
-	std::istream is(&sb);
+	G3BufferInputStream is(*blob.blob);
 	cereal::PortableBinaryInputArchive item_ar(is);
 	item_ar >> make_nvp("val", ptr);
 	blob.frameobject = ptr;
@@ -391,8 +362,7 @@ void G3Frame::blob_encode(struct blob_container &blob)
 
 	// If no encoded frameobject, serialize it
 	blob.blob = std::make_shared<std::vector<char> >();
-	g3_ostream item_os;
-	g3_ostream_to_buffer(item_os, *blob.blob);
+	G3BufferOutputStream item_os(*blob.blob);
 	cereal::PortableBinaryOutputArchive item_ar(item_os);
 	item_ar << make_nvp("val", blob.frameobject);
 	item_os.flush();
@@ -432,8 +402,7 @@ struct g3frame_picklesuite : boost::python::pickle_suite
 	{
 		namespace bp = boost::python;
 		std::vector<char> buffer;
-		g3_ostream os;
-		g3_ostream_to_buffer(os, buffer);
+		G3BufferOutputStream os(buffer);
 		(bp::extract<const G3Frame &>(obj))().save(os);
 		os.flush();
 
@@ -450,9 +419,9 @@ struct g3frame_picklesuite : boost::python::pickle_suite
 		PyObject_GetBuffer(bp::object(state[1]).ptr(), &view,
 		    PyBUF_SIMPLE);
 
-		std::vector<char> buf((char *)view.buf, (char *)view.buf + view.len);
 		bp::extract<bp::dict>(obj.attr("__dict__"))().update(state[0]);
-		(bp::extract<G3Frame &>(obj))().load(buf);
+		G3BufferInputStream is((char *)view.buf, view.len);
+		(bp::extract<G3Frame &>(obj))().load(is);
 		PyBuffer_Release(&view);
 	}
 };
diff --git a/core/src/G3Map.cxx b/core/src/G3Map.cxx
@@ -3,7 +3,6 @@
 #include <pybindings.h>
 #include <container_pybindings.h>
 #include <serialization.h>
-#include <dataio.h>
 #include "int_storage.h"
 
 /* Special load/save for int64_t, using the same encoding at G3VectorInt */
@@ -159,12 +158,11 @@ template <class A> void G3MapFrameObject::save(A &ar, const unsigned v) const
 	uint32_t len = size();
 	ar << cereal::make_nvp("len", len);
 
-	g3_ostream os;
 	for (auto i = begin(); i != end(); i++) {
 		ar << cereal::make_nvp("key", i->first);
 
 		std::vector<char> buffer;
-		g3_ostream_to_buffer(os, buffer);
+		G3BufferOutputStream os(buffer);
 		{
 			A subar(os);
 			subar << cereal::make_nvp("item",
@@ -184,16 +182,14 @@ template <class A> void G3MapFrameObject::load(A &ar, const unsigned v)
 
 	uint32_t len;
 	ar >> cereal::make_nvp("len", len);
-	g3_istream fis;
 	for (uint32_t i = 0; i < len; i++) {
 		std::pair<std::string, G3FrameObjectPtr> item;
 		std::vector<char> buffer;
 
 		ar >> cereal::make_nvp("key", item.first);
 		ar >> cereal::make_nvp("value", buffer);
 
-		g3_istream_from_buffer(fis, (char *)&buffer[0], buffer.size());
-
+		G3BufferInputStream fis(buffer);
 		A subar(fis);
 		subar >> cereal::make_nvp("item", item.second);
 		this->insert(item);
diff --git a/core/src/G3NetworkSender.cxx b/core/src/G3NetworkSender.cxx
@@ -1,7 +1,7 @@
 #include <pybindings.h>
 #include <G3NetworkSender.h>
 
-#include <dataio.h>
+#include <serialization.h>
 
 #include <chrono>
 
@@ -173,8 +173,7 @@ void G3NetworkSender::SerializeFrame(serialization_task& task)
 {
 	try{
 		netbuf_type buf(new std::vector<char>);
-		g3_ostream os;
-		g3_ostream_to_buffer(os, *buf);
+		G3BufferOutputStream os(*buf);
 		task.input->save(os);
 		os.flush();
 		task.output.set_value(buf);
diff --git a/core/src/dataio.cxx b/core/src/dataio.cxx
@@ -10,8 +10,6 @@
 #endif
 #include <boost/iostreams/device/file.hpp>
 #include <boost/iostreams/device/file_descriptor.hpp>
-#include <boost/iostreams/device/array.hpp>
-#include <boost/iostreams/device/back_inserter.hpp>
 #include <filesystem>
 
 #include <sys/types.h>
@@ -178,13 +176,6 @@ g3_istream_tell(g3_istream &stream)
 	return boost::iostreams::seek(stream, 0, std::ios_base::cur);
 }
 
-void
-g3_istream_from_buffer(g3_istream &stream, const char *buf, size_t len)
-{
-	stream.reset();
-	stream.push(boost::iostreams::array_source(buf, len));
-}
-
 void
 g3_ostream_to_path(g3_ostream &stream, const std::string &path,
     bool append, bool counter)
@@ -227,14 +218,6 @@ g3_ostream_count(g3_ostream &stream)
 	return counter->characters();
 }
 
-void
-g3_ostream_to_buffer(g3_ostream &stream,
-    std::vector<char> &buf)
-{
-	stream.reset();
-	stream.push(boost::iostreams::back_inserter(buf));
-}
-
 void
 g3_check_input_path(const std::string &path)
 {
