Merge branch 'master' into arc_reader_subclass

Author: arahlin

.github/workflows/cmake.yml

@@ -35,7 +35,7 @@ jobs:
         sudo apt update
         sudo apt install gcc-14 g++-14 -y
         sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 14 --slave /usr/bin/g++ g++ /usr/bin/g++-14
-        sudo apt-get install libbz2-dev libboost-all-dev libflac-dev libnetcdf-dev libpython3-dev
+        sudo apt-get install libz-dev libbz2-dev liblzma-dev libboost-python-dev libflac-dev libnetcdf-dev libpython3-dev
         sudo pip install numpy scipy matplotlib astropy healpy sphinx --break-system-packages
 
     - name: Setup python on macOS
@@ -47,7 +47,7 @@ jobs:
     - name: Install macOS Dependencies
       if: runner.environment == 'github-hosted' && runner.os == 'macOS'
       run: |
-        brew install --overwrite bzip2 boost boost-python3 flac netcdf
+        brew install --overwrite bzip2 xz boost-python3 flac netcdf
         python3.13 -m pip install numpy scipy matplotlib astropy healpy sphinx --break-system-packages
 
     - name: Create Build Environment

.github/workflows/wheels.yml

@@ -86,7 +86,7 @@ jobs:
         uses: pypa/cibuildwheel@v2.23.0
         env:
           CIBW_BUILD: ${{ matrix.build }}
-          CIBW_BEFORE_ALL_LINUX: yum install -y bzip2-devel netcdf-devel
+          CIBW_BEFORE_ALL_LINUX: yum install -y zlib-devel bzip2-devel xz-devel netcdf-devel
           CIBW_BEFORE_ALL_MACOS: brew install netcdf
           CIBW_BEFORE_BUILD_MACOS: >
             ln -s $(dirname $(readlink -f $(which python3)))/python3-config $(dirname $(which python3))/python3-config &&

CMakeLists.txt

@@ -52,7 +52,7 @@ set(Boost_USE_STATIC_LIBS OFF)
 set(Boost_USE_MULTITHREADED ON)
 set(Boost_USE_STATIC_RUNTIME OFF)
 set(Boost_PYTHON_VERSION ${Python_VERSION})
-find_package(Boost COMPONENTS iostreams python REQUIRED)
+find_package(Boost COMPONENTS python REQUIRED)
 
 # Interface library for flags and library dependencies
 add_library(spt3g INTERFACE)

core/CMakeLists.txt

@@ -12,7 +12,7 @@ add_spt3g_library(core SHARED
 	src/G3Data.cxx src/G3Vector.cxx src/G3Map.cxx src/G3Timestream.cxx
         src/G3Timesample.cxx
 	src/G3TriggeredBuilder.cxx src/G3MultiFileWriter.cxx src/dataio.cxx
-	src/crc32.c ${CORE_EXTRA_SRCS}
+	src/compression.cxx src/crc32.c ${CORE_EXTRA_SRCS}
 	src/G3NetworkSender.cxx src/G3SyslogLogger.cxx
 	src/G3PipelineInfo.cxx src/G3Quat.cxx src/G3Units.cxx
 	src/int_storage.cxx src/pybindings.cxx
@@ -40,8 +40,17 @@ if(FLAC_FOUND)
 	target_link_libraries(core PRIVATE FLAC::FLAC)
 endif()
 
-# Link Boost IOStreams
-target_link_libraries(core PRIVATE Boost::iostreams)
+# Link against Z library
+if(NOT DEFINED WITH_ZLIB)
+	set(WITH_ZLIB TRUE CACHE BOOL "Enable gzip fie compression")
+endif()
+if(WITH_ZLIB)
+	find_package(ZLIB)
+endif()
+if(ZLIB_FOUND)
+	target_compile_definitions(core PRIVATE -DZLIB_FOUND)
+	target_link_libraries(core PRIVATE ZLIB::ZLIB)
+endif()
 
 # Link against BZIP2 library
 if(NOT DEFINED WITH_BZIP2)
@@ -52,6 +61,7 @@ if(WITH_BZIP2)
 endif()
 if(BZIP2_FOUND)
 	target_compile_definitions(core PRIVATE -DBZIP2_FOUND)
+	target_link_libraries(core PRIVATE BZip2::BZip2)
 endif()
 
 if(NOT DEFINED WITH_LZMA)
@@ -62,6 +72,7 @@ if(WITH_LZMA)
 endif()
 if(LIBLZMA_FOUND)
 	target_compile_definitions(core PRIVATE -DLZMA_FOUND)
+	target_link_libraries(core PRIVATE LibLZMA::LibLZMA)
 endif()
 
 link_python_dir()
@@ -88,7 +99,9 @@ add_spt3g_test(cuts)
 add_spt3g_test(fileio)
 add_spt3g_test(multifileio)
 add_spt3g_test(splitfileio)
-add_spt3g_test(compressedfileio)
+if(ZLIB_FOUND)
+	add_spt3g_test(compressedfileio)
+endif()
 if(BZIP2_FOUND)
 	add_spt3g_test(bz2fileio)
 endif()

core/include/core/G3Reader.h

@@ -9,10 +9,10 @@
 
 class G3Reader : public G3Module {
 public:
-	G3Reader(std::string filename, int n_frames_to_read = -1,
+	G3Reader(const std::string &filename, int n_frames_to_read = -1,
                  float timeout = -1., bool track_filename = false,
 	         size_t buffersize = 1024*1024);
-	G3Reader(std::vector<std::string> filenames, int n_frames_to_read = -1,
+	G3Reader(const std::vector<std::string> &filenames, int n_frames_to_read = -1,
                  float timeout = -1., bool track_filename = false,
 	         size_t buffersize = 1024*1024);
 
@@ -21,12 +21,12 @@ class G3Reader : public G3Module {
 	off_t Tell();
 
 protected:
-	virtual void StartFile(std::string path);
+	virtual void StartFile(const std::string &path);
 	virtual G3FramePtr FillFrame();
 	bool prefix_file_;
 	std::string cur_file_;
 	std::deque<std::string> filename_;
-	std::shared_ptr<std::istream> stream_;
+	std::istream stream_;
 	int fd_;
 	int n_frames_to_read_;
 	int n_frames_read_;

core/include/core/G3Writer.h

@@ -9,15 +9,16 @@
 class G3Writer : public G3Module {
 public:
 	G3Writer(std::string filename,
-	    std::vector<G3Frame::FrameType> streams={}, bool append=false);
+	    std::vector<G3Frame::FrameType> streams={}, bool append=false,
+	    size_t buffersize=1024*1024);
 	// Writes to file <filename> all frames with types in <streams>.
 	// If <streams> is empty (default), writes all frames.
 
 	void Process(G3FramePtr frame, std::deque<G3FramePtr> &out);
 	void Flush();
 private:
 	std::string filename_;
-	std::shared_ptr<std::ostream> stream_;
+	std::ostream stream_;
 	std::vector<G3Frame::FrameType> streams_;
 
 	SET_LOGGER("G3Writer");

core/src/G3Frame.cxx

@@ -381,12 +381,10 @@ void G3Frame::blob_encode(struct blob_container &blob)
 	item_os.flush();
 }
 
-template <> void G3Frame::loads(std::shared_ptr<std::istream> &is) { loads(*is); }
 template void G3Frame::loads(G3BufferInputStream &);
 template void G3Frame::loads(std::istream &);
 template void G3Frame::loads(std::istringstream &);
 
-template <> void G3Frame::saves(std::shared_ptr<std::ostream> &os) const { saves(*os); }
 template void G3Frame::saves(G3BufferOutputStream &) const;
 template void G3Frame::saves(std::ostream &) const;
 template void G3Frame::saves(std::ostringstream &) const;

core/src/G3MultiFileWriter.cxx

@@ -8,7 +8,9 @@ class G3MultiFileWriter : public G3Module {
 public:
 	G3MultiFileWriter(boost::python::object filename,
 	    size_t size_limit,
-	    boost::python::object divide_on = boost::python::object());
+	    boost::python::object divide_on = boost::python::object(),
+	    size_t buffersize=1024*1024);
+
 	void Process(G3FramePtr frame, std::deque<G3FramePtr> &out);
 	std::string CurrentFile() { return current_filename_; }
 private:
@@ -18,26 +20,26 @@ class G3MultiFileWriter : public G3Module {
 	boost::python::object filename_callback_;
 	std::string current_filename_;
 	size_t size_limit_;
+	size_t buffersize_;
 
 	std::vector<G3Frame::FrameType> always_break_on_;
 	boost::python::object newfile_callback_;
 
-	std::shared_ptr<std::ostream> stream_;
+	std::ostream stream_;
 	std::vector<G3FramePtr> metadata_cache_;
 	int seqno;
 
 	SET_LOGGER("G3MultiFileWriter");
 };
 
 G3MultiFileWriter::G3MultiFileWriter(boost::python::object filename,
-    size_t size_limit, boost::python::object divide_on)
-    : size_limit_(size_limit), seqno(0)
+    size_t size_limit, boost::python::object divide_on, size_t buffersize)
+    : size_limit_(size_limit), buffersize_(buffersize), stream_(nullptr), seqno(0)
 {
 	boost::python::extract<std::string> fstr(filename);
 
 	if (fstr.check()) {
 		filename_ = fstr();
-		g3_check_output_path(filename_);
 
 		if (snprintf(NULL, 0, filename_.c_str(), 0) < 0)
 			log_fatal("Cannot format filename. Should be "
@@ -77,10 +79,10 @@ G3MultiFileWriter::CheckNewFile(G3FramePtr frame)
 {
 	// If we are already saving data, check file size. Otherwise, open
 	// a new file unconditionally.
-	if (stream_ != nullptr) {
+	if (stream_) {
 		bool start_new_ = false;
 
-		if (g3_ostream_count(stream_) > size_limit_)
+		if ((size_t)stream_.tellp() > size_limit_)
 			start_new_ = true;
 
 		if (newfile_callback_.ptr() != Py_None &&
@@ -95,7 +97,7 @@ G3MultiFileWriter::CheckNewFile(G3FramePtr frame)
 			return false;
 	}
 
-	stream_.reset();
+	stream_.flush();
 
 	std::string filename;
 	if (filename_ != "") {
@@ -111,12 +113,10 @@ G3MultiFileWriter::CheckNewFile(G3FramePtr frame)
 	} else {
 		filename = boost::python::extract<std::string>(
 		    filename_callback_(frame, seqno++))();
-
-		g3_check_output_path(filename);
 	}
 
 	current_filename_ = filename;
-	stream_ = g3_ostream_to_path(filename, false, true);
+	g3_ostream_to_path(stream_, filename, false, buffersize_);
 
 	for (auto i = metadata_cache_.begin(); i != metadata_cache_.end(); i++)
 		(*i)->saves(stream_);
@@ -129,7 +129,7 @@ void G3MultiFileWriter::Process(G3FramePtr frame, std::deque<G3FramePtr> &out)
 	bool new_file(false), meta_cached(false);
 
 	if (frame->type == G3Frame::EndProcessing) {
-		stream_.reset();
+		stream_.flush();
 		goto done;
 	}
 
@@ -186,8 +186,8 @@ PYBINDINGS("core") {
 	      "python callable as divide_on. This callable will be passed each "
 	      "frame in turn. If it returns True (or something with positive "
 	      "truth-value), a new file will be started at that frame.",
-	init<object, size_t, optional<object> >((arg("filename"),
-	    arg("size_limit"), arg("divide_on")=object())))
+	init<object, size_t, optional<object, size_t> >((arg("filename"),
+	    arg("size_limit"), arg("divide_on")=object(), arg("buffersize")=1024*1024)))
 	.def_readonly("current_file", &G3MultiFileWriter::CurrentFile)
 	.def_readonly("__g3module__", true)
 	;

core/src/G3Reader.cxx

@@ -2,39 +2,39 @@
 #include "dataio.h"
 #include <G3Reader.h>
 
-G3Reader::G3Reader(std::string filename, int n_frames_to_read,
+G3Reader::G3Reader(const std::string &filename, int n_frames_to_read,
     float timeout, bool track_filename, size_t buffersize) :
-    prefix_file_(false), fd_(-1), n_frames_to_read_(n_frames_to_read),
+    prefix_file_(false), stream_(nullptr), fd_(-1),
+    n_frames_to_read_(n_frames_to_read),
     n_frames_read_(0), n_frames_cur_(0), timeout_(timeout),
     track_filename_(track_filename), buffersize_(buffersize)
 {
-	g3_check_input_path(filename);
 	StartFile(filename);
 }
 
-G3Reader::G3Reader(std::vector<std::string> filename, int n_frames_to_read,
+G3Reader::G3Reader(const std::vector<std::string> &filename, int n_frames_to_read,
     float timeout, bool track_filename, size_t buffersize) :
-    prefix_file_(false), fd_(-1), n_frames_to_read_(n_frames_to_read),
+    prefix_file_(false), stream_(nullptr), fd_(-1),
+    n_frames_to_read_(n_frames_to_read),
     n_frames_read_(0), n_frames_cur_(0), timeout_(timeout),
     track_filename_(track_filename), buffersize_(buffersize)
 {
 	if (filename.size() == 0)
 		log_fatal("Empty file list provided to G3Reader");
 
-	for (auto i = filename.begin(); i != filename.end(); i++){
-		g3_check_input_path(*i);
+	for (auto i = filename.begin(); i != filename.end(); i++)
 		filename_.push_back(*i);
-	}
+
 	StartFile(filename_.front());
 	filename_.pop_front();
 }
 
-void G3Reader::StartFile(std::string path)
+void G3Reader::StartFile(const std::string &path)
 {
 	log_info("Starting file %s\n", path.c_str());
 	cur_file_ = path;
 	n_frames_cur_ = 0;
-	stream_ = g3_istream_from_path(path, timeout_, buffersize_);
+	g3_istream_from_path(stream_, path, timeout_, buffersize_);
 	fd_ = g3_istream_handle(stream_);
 }
 
@@ -79,7 +79,7 @@ void G3Reader::Process(G3FramePtr frame, std::deque<G3FramePtr> &out)
 	// function reacquire the lock, if it was released.
 	G3PythonContext ctx("G3Reader", false);
 
-	while (stream_->peek() == EOF) {
+	while (stream_.peek() == EOF) {
 		if (n_frames_cur_ == 0)
 			log_error("Empty file %s", cur_file_.c_str());
 		if (filename_.size() > 0) {
@@ -107,14 +107,14 @@ void G3Reader::Process(G3FramePtr frame, std::deque<G3FramePtr> &out)
 }
 
 off_t G3Reader::Seek(off_t offset) {
-	if (stream_->peek() == EOF && offset != Tell())
+	if (stream_.peek() == EOF && offset != Tell())
 		log_fatal("Cannot seek %s; stream closed at EOF.", cur_file_.c_str());
-	stream_->seekg(offset, std::ios_base::beg);
+	stream_.seekg(offset, std::ios_base::beg);
 	return offset;
 }
 
 off_t G3Reader::Tell() {
-	return stream_->tellg();
+	return stream_.tellg();
 }
 
 PYBINDINGS("core") {

core/src/G3Writer.cxx

@@ -4,11 +4,10 @@
 
 G3Writer::G3Writer(std::string filename,
     std::vector<G3Frame::FrameType> streams,
-    bool append) :
-    filename_(filename), streams_(streams)
+    bool append, size_t buffersize) :
+    filename_(filename), stream_(nullptr), streams_(streams)
 {
-	g3_check_output_path(filename);
-	stream_ = g3_ostream_to_path(filename, append);
+	g3_ostream_to_path(stream_, filename, append, buffersize);
 }
 
 void G3Writer::Process(G3FramePtr frame, std::deque<G3FramePtr> &out)
@@ -22,7 +21,7 @@ void G3Writer::Process(G3FramePtr frame, std::deque<G3FramePtr> &out)
 	G3PythonContext ctx("G3Writer", false);
 
 	if (frame->type == G3Frame::EndProcessing)
-		stream_.reset();
+		stream_.flush();
 	else if (streams_.size() == 0 ||
 	    std::find(streams_.begin(), streams_.end(), frame->type) !=
 	    streams_.end())
@@ -33,10 +32,7 @@ void G3Writer::Process(G3FramePtr frame, std::deque<G3FramePtr> &out)
 
 void G3Writer::Flush()
 {
-	try {
-		g3_ostream_flush(stream_);
-	} catch (...) {
-	}
+	stream_.flush();
 }
 
 PYBINDINGS("core") {
@@ -51,9 +47,10 @@ PYBINDINGS("core") {
 	      "types to the second optional argument (streams). If no streams argument "
 	      "is given, writes all types of frames. If append is set to True, will "
 	      "append frames to its output file rather than overwriting it.",
-        init<std::string, std::vector<G3Frame::FrameType>, bool>((arg("filename"),
-	    arg("streams")=std::vector<G3Frame::FrameType>(), arg("append")=false)))
-        .def("Flush", &G3Writer::Flush)
-        .def_readonly("__g3module__", true)
+	init<std::string, std::vector<G3Frame::FrameType>, bool, size_t>((arg("filename"),
+	    arg("streams")=std::vector<G3Frame::FrameType>(), arg("append")=false,
+	    arg("buffersize")=1024*1024)))
+	.def("Flush", &G3Writer::Flush)
+	.def_readonly("__g3module__", true)
 	;
 }