Skip to content

Commit ca5a7d2

Browse files
committed
Remove dependency on Boost.IOStreams
1 parent 5254495 commit ca5a7d2

File tree

13 files changed

+111
-136
lines changed

13 files changed

+111
-136
lines changed

.github/workflows/cmake.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ jobs:
3535
sudo apt update
3636
sudo apt install gcc-14 g++-14 -y
3737
sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 14 --slave /usr/bin/g++ g++ /usr/bin/g++-14
38-
sudo apt-get install libbz2-dev libboost-all-dev libflac-dev libnetcdf-dev libpython3-dev
38+
sudo apt-get install libbz2-dev libboost-python-dev libflac-dev libnetcdf-dev libpython3-dev
3939
sudo pip install numpy scipy matplotlib astropy healpy sphinx --break-system-packages
4040
4141
- name: Setup python on macOS
@@ -47,7 +47,7 @@ jobs:
4747
- name: Install macOS Dependencies
4848
if: runner.environment == 'github-hosted' && runner.os == 'macOS'
4949
run: |
50-
brew install --overwrite bzip2 boost boost-python3 flac netcdf
50+
brew install --overwrite bzip2 boost-python3 flac netcdf
5151
python3.13 -m pip install numpy scipy matplotlib astropy healpy sphinx --break-system-packages
5252
5353
- name: Create Build Environment

CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ set(Boost_USE_STATIC_LIBS OFF)
5252
set(Boost_USE_MULTITHREADED ON)
5353
set(Boost_USE_STATIC_RUNTIME OFF)
5454
set(Boost_PYTHON_VERSION ${Python_VERSION})
55-
find_package(Boost COMPONENTS iostreams python REQUIRED)
55+
find_package(Boost COMPONENTS python REQUIRED)
5656

5757
# Interface library for flags and library dependencies
5858
add_library(spt3g INTERFACE)

core/CMakeLists.txt

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,17 @@ if(FLAC_FOUND)
4040
target_link_libraries(core PRIVATE FLAC::FLAC)
4141
endif()
4242

43-
# Link Boost IOStreams
44-
target_link_libraries(core PRIVATE Boost::iostreams)
43+
# Link against Z library
44+
if(NOT DEFINED WITH_ZLIB)
45+
set(WITH_ZLIB TRUE CACHE BOOL "Enable gzip fie compression")
46+
endif()
47+
if(WITH_ZLIB)
48+
find_package(ZLIB)
49+
endif()
50+
if(ZLIB_FOUND)
51+
target_compile_definitions(core PRIVATE -DZLIB_FOUND)
52+
target_link_libraries(core PRIVATE ZLIB::ZLIB)
53+
endif()
4554

4655
# Link against BZIP2 library
4756
if(NOT DEFINED WITH_BZIP2)
@@ -52,6 +61,7 @@ if(WITH_BZIP2)
5261
endif()
5362
if(BZIP2_FOUND)
5463
target_compile_definitions(core PRIVATE -DBZIP2_FOUND)
64+
target_link_libraries(core PRIVATE BZip2::BZip2)
5565
endif()
5666

5767
if(NOT DEFINED WITH_LZMA)
@@ -62,6 +72,7 @@ if(WITH_LZMA)
6272
endif()
6373
if(LIBLZMA_FOUND)
6474
target_compile_definitions(core PRIVATE -DLZMA_FOUND)
75+
target_link_libraries(core PRIVATE LibLZMA::LibLZMA)
6576
endif()
6677

6778
link_python_dir()
@@ -88,7 +99,9 @@ add_spt3g_test(cuts)
8899
add_spt3g_test(fileio)
89100
add_spt3g_test(multifileio)
90101
add_spt3g_test(splitfileio)
91-
add_spt3g_test(compressedfileio)
102+
if(ZLIB_FOUND)
103+
add_spt3g_test(compressedfileio)
104+
endif()
92105
if(BZIP2_FOUND)
93106
add_spt3g_test(bz2fileio)
94107
endif()

core/include/core/G3Reader.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ class G3Reader : public G3Module {
1515
G3Reader(std::vector<std::string> filenames, int n_frames_to_read = -1,
1616
float timeout = -1., bool track_filename = false,
1717
size_t buffersize = 1024*1024);
18+
virtual ~G3Reader();
1819

1920
void Process(G3FramePtr frame, std::deque<G3FramePtr> &out);
2021
off_t Seek(off_t offset);
@@ -25,7 +26,7 @@ class G3Reader : public G3Module {
2526
bool prefix_file_;
2627
std::string cur_file_;
2728
std::deque<std::string> filename_;
28-
std::shared_ptr<std::istream> stream_;
29+
std::istream stream_;
2930
int n_frames_to_read_;
3031
int n_frames_read_;
3132
int n_frames_cur_;

core/include/core/G3Writer.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,13 @@ class G3Writer : public G3Module {
1212
std::vector<G3Frame::FrameType> streams={}, bool append=false);
1313
// Writes to file <filename> all frames with types in <streams>.
1414
// If <streams> is empty (default), writes all frames.
15+
virtual ~G3Writer();
1516

1617
void Process(G3FramePtr frame, std::deque<G3FramePtr> &out);
1718
void Flush();
1819
private:
1920
std::string filename_;
20-
std::shared_ptr<std::ostream> stream_;
21+
std::ostream stream_;
2122
std::vector<G3Frame::FrameType> streams_;
2223

2324
SET_LOGGER("G3Writer");

core/include/core/dataio.h

Lines changed: 29 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,14 @@
22
#define _G3_DATAIO_H
33

44
#include <string>
5-
#include <memory>
65
#include <iostream>
76

87
/**
98
* Configure a filtering stream for G3Frame decompression from a local or remote
109
* file source.
1110
*
11+
* @param stream A reference to the input stream to be configured by this
12+
* function.
1213
* @param path A valid filename on disk, or a TCP socket address. If a
1314
* filename, the compression scheme is determined from the file
1415
* extension. Supported compression schemes are gzip or bzip2.
@@ -19,11 +20,10 @@
1920
* read until EOF.
2021
* @param timeout Timeout in seconds for socket connections.
2122
* @param buffersize Advisory buffer size in bytes for aggregating reads
22-
* @return stream The input stream configured by this function.
2323
*/
24-
std::shared_ptr<std::istream>
25-
g3_istream_from_path(const std::string &path, float timeout=-1.0,
26-
size_t buffersize=1024*1024);
24+
void
25+
g3_istream_from_path(std::istream &stream, const std::string &path,
26+
float timeout=-1.0, size_t buffersize=1024*1024);
2727

2828
/**
2929
* Return the file descriptor handle for socket connections.
@@ -32,22 +32,32 @@ g3_istream_from_path(const std::string &path, float timeout=-1.0,
3232
* g3_istream_from_path.
3333
* @return fd The socket file descriptor.
3434
*/
35-
int g3_istream_handle(std::shared_ptr<std::istream> &stream);
35+
int g3_istream_handle(std::istream &stream);
36+
37+
/**
38+
* Clean up the input stream.
39+
*
40+
* @param stream A reference to the input stream, as configured by
41+
* g3_istream_from_path.
42+
*/
43+
void g3_istream_close(std::istream &stream);
3644

3745
/**
3846
* Configure a filtering stream for G3Frame compression to a local file.
3947
*
48+
* @param stream A reference to the output stream to be configured by this
49+
* function.
4050
* @param path A valid filename on disk. If a filename, the compression
4151
* scheme is determined from the file extension. Supported
4252
* compression schemes are gzip or bzip2.
4353
* @param append If true, append to an existing file on disk. Otherwise,
4454
* Create a new file or overwrite an existing file.
4555
* @param counter If true, add a counter filter to the stream configuration,
4656
* for use by the g3_ostream_count function.
47-
* @return stream The output stream configured by this function.
4857
*/
49-
std::shared_ptr<std::ostream>
50-
g3_ostream_to_path(const std::string &path, bool append=false, bool counter=false);
58+
void
59+
g3_ostream_to_path(std::ostream &stream, const std::string &path, bool append=false,
60+
bool counter=false);
5161

5262
/**
5363
* Count the number of bytes written to the output file stream.
@@ -56,15 +66,23 @@ g3_ostream_to_path(const std::string &path, bool append=false, bool counter=fals
5666
* g3_ostream_to_path with the counter argument set to true.
5767
* @return Number of bytes written to disk.
5868
*/
59-
size_t g3_ostream_count(std::shared_ptr<std::ostream> &stream);
69+
size_t g3_ostream_count(std::ostream &stream);
6070

6171
/**
6272
* Flush the output file stream.
6373
*
6474
* @param stream A reference to the output stream, as configured by
6575
* g3_ostream_to_path.
6676
*/
67-
void g3_ostream_flush(std::shared_ptr<std::ostream> &stream);
77+
void g3_ostream_flush(std::ostream &stream);
78+
79+
/**
80+
* Clean up the output stream.
81+
*
82+
* @param stream A reference to the output stream, as configured by
83+
* g3_istream_from_path.
84+
*/
85+
void g3_ostream_close(std::ostream &stream);
6886

6987
/**
7088
* Check that the input filename is a valid filename on disk.

core/src/G3Frame.cxx

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -381,12 +381,10 @@ void G3Frame::blob_encode(struct blob_container &blob)
381381
item_os.flush();
382382
}
383383

384-
template <> void G3Frame::loads(std::shared_ptr<std::istream> &is) { loads(*is); }
385384
template void G3Frame::loads(G3BufferInputStream &);
386385
template void G3Frame::loads(std::istream &);
387386
template void G3Frame::loads(std::istringstream &);
388387

389-
template <> void G3Frame::saves(std::shared_ptr<std::ostream> &os) const { saves(*os); }
390388
template void G3Frame::saves(G3BufferOutputStream &) const;
391389
template void G3Frame::saves(std::ostream &) const;
392390
template void G3Frame::saves(std::ostringstream &) const;

core/src/G3MultiFileWriter.cxx

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ class G3MultiFileWriter : public G3Module {
99
G3MultiFileWriter(boost::python::object filename,
1010
size_t size_limit,
1111
boost::python::object divide_on = boost::python::object());
12+
~G3MultiFileWriter();
1213
void Process(G3FramePtr frame, std::deque<G3FramePtr> &out);
1314
std::string CurrentFile() { return current_filename_; }
1415
private:
@@ -22,7 +23,7 @@ class G3MultiFileWriter : public G3Module {
2223
std::vector<G3Frame::FrameType> always_break_on_;
2324
boost::python::object newfile_callback_;
2425

25-
std::shared_ptr<std::ostream> stream_;
26+
std::ostream stream_;
2627
std::vector<G3FramePtr> metadata_cache_;
2728
int seqno;
2829

@@ -31,7 +32,7 @@ class G3MultiFileWriter : public G3Module {
3132

3233
G3MultiFileWriter::G3MultiFileWriter(boost::python::object filename,
3334
size_t size_limit, boost::python::object divide_on)
34-
: size_limit_(size_limit), seqno(0)
35+
: size_limit_(size_limit), stream_(nullptr), seqno(0)
3536
{
3637
boost::python::extract<std::string> fstr(filename);
3738

@@ -72,12 +73,17 @@ G3MultiFileWriter::G3MultiFileWriter(boost::python::object filename,
7273
}
7374
}
7475

76+
G3MultiFileWriter::~G3MultiFileWriter()
77+
{
78+
g3_ostream_close(stream_);
79+
}
80+
7581
bool
7682
G3MultiFileWriter::CheckNewFile(G3FramePtr frame)
7783
{
7884
// If we are already saving data, check file size. Otherwise, open
7985
// a new file unconditionally.
80-
if (stream_ != nullptr) {
86+
if (stream_) {
8187
bool start_new_ = false;
8288

8389
if (g3_ostream_count(stream_) > size_limit_)
@@ -95,7 +101,7 @@ G3MultiFileWriter::CheckNewFile(G3FramePtr frame)
95101
return false;
96102
}
97103

98-
stream_.reset();
104+
g3_ostream_close(stream_);
99105

100106
std::string filename;
101107
if (filename_ != "") {
@@ -116,7 +122,7 @@ G3MultiFileWriter::CheckNewFile(G3FramePtr frame)
116122
}
117123

118124
current_filename_ = filename;
119-
stream_ = g3_ostream_to_path(filename, false, true);
125+
g3_ostream_to_path(stream_, filename, false, true);
120126

121127
for (auto i = metadata_cache_.begin(); i != metadata_cache_.end(); i++)
122128
(*i)->saves(stream_);
@@ -129,7 +135,7 @@ void G3MultiFileWriter::Process(G3FramePtr frame, std::deque<G3FramePtr> &out)
129135
bool new_file(false), meta_cached(false);
130136

131137
if (frame->type == G3Frame::EndProcessing) {
132-
stream_.reset();
138+
g3_ostream_close(stream_);
133139
goto done;
134140
}
135141

core/src/G3Reader.cxx

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
G3Reader::G3Reader(std::string filename, int n_frames_to_read,
66
float timeout, bool track_filename, size_t buffersize) :
7-
prefix_file_(false), n_frames_to_read_(n_frames_to_read),
7+
prefix_file_(false), stream_(nullptr), n_frames_to_read_(n_frames_to_read),
88
n_frames_read_(0), n_frames_cur_(0), timeout_(timeout),
99
track_filename_(track_filename), buffersize_(buffersize)
1010
{
@@ -14,7 +14,7 @@ G3Reader::G3Reader(std::string filename, int n_frames_to_read,
1414

1515
G3Reader::G3Reader(std::vector<std::string> filename, int n_frames_to_read,
1616
float timeout, bool track_filename, size_t buffersize) :
17-
prefix_file_(false), n_frames_to_read_(n_frames_to_read),
17+
prefix_file_(false), stream_(nullptr), n_frames_to_read_(n_frames_to_read),
1818
n_frames_read_(0), n_frames_cur_(0), timeout_(timeout),
1919
track_filename_(track_filename), buffersize_(buffersize)
2020
{
@@ -29,12 +29,17 @@ G3Reader::G3Reader(std::vector<std::string> filename, int n_frames_to_read,
2929
filename_.pop_front();
3030
}
3131

32+
G3Reader::~G3Reader()
33+
{
34+
g3_istream_close(stream_);
35+
}
36+
3237
void G3Reader::StartFile(std::string path)
3338
{
3439
log_info("Starting file %s\n", path.c_str());
3540
cur_file_ = path;
3641
n_frames_cur_ = 0;
37-
stream_ = g3_istream_from_path(path, timeout_, buffersize_);
42+
g3_istream_from_path(stream_, path, timeout_, buffersize_);
3843
}
3944

4045
void G3Reader::Process(G3FramePtr frame, std::deque<G3FramePtr> &out)
@@ -71,7 +76,7 @@ void G3Reader::Process(G3FramePtr frame, std::deque<G3FramePtr> &out)
7176
// function reacquire the lock, if it was released.
7277
G3PythonContext ctx("G3Reader", false);
7378

74-
while (stream_->peek() == EOF) {
79+
while (stream_.peek() == EOF) {
7580
if (n_frames_cur_ == 0)
7681
log_error("Empty file %s", cur_file_.c_str());
7782
if (filename_.size() > 0) {
@@ -100,14 +105,14 @@ void G3Reader::Process(G3FramePtr frame, std::deque<G3FramePtr> &out)
100105
}
101106

102107
off_t G3Reader::Seek(off_t offset) {
103-
if (stream_->peek() == EOF && offset != Tell())
108+
if (stream_.peek() == EOF && offset != Tell())
104109
log_fatal("Cannot seek %s; stream closed at EOF.", cur_file_.c_str());
105-
stream_->seekg(offset, std::ios_base::beg);
110+
stream_.seekg(offset, std::ios_base::beg);
106111
return offset;
107112
}
108113

109114
off_t G3Reader::Tell() {
110-
return stream_->tellg();
115+
return stream_.tellg();
111116
}
112117

113118
PYBINDINGS("core") {

core/src/G3Writer.cxx

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,15 @@
55
G3Writer::G3Writer(std::string filename,
66
std::vector<G3Frame::FrameType> streams,
77
bool append) :
8-
filename_(filename), streams_(streams)
8+
filename_(filename), stream_(nullptr), streams_(streams)
99
{
1010
g3_check_output_path(filename);
11-
stream_ = g3_ostream_to_path(filename, append);
11+
g3_ostream_to_path(stream_, filename, append);
12+
}
13+
14+
G3Writer::~G3Writer()
15+
{
16+
g3_ostream_close(stream_);
1217
}
1318

1419
void G3Writer::Process(G3FramePtr frame, std::deque<G3FramePtr> &out)
@@ -22,7 +27,7 @@ void G3Writer::Process(G3FramePtr frame, std::deque<G3FramePtr> &out)
2227
G3PythonContext ctx("G3Writer", false);
2328

2429
if (frame->type == G3Frame::EndProcessing)
25-
stream_.reset();
30+
g3_ostream_close(stream_);
2631
else if (streams_.size() == 0 ||
2732
std::find(streams_.begin(), streams_.end(), frame->type) !=
2833
streams_.end())

0 commit comments

Comments
 (0)