Merge branch 'master' into json_output

Author: arahlin

.github/workflows/cmake.yml

@@ -25,7 +25,7 @@ jobs:
     runs-on: ${{matrix.os}}
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
 
     - name: Install Linux Dependencies
       if: runner.environment == 'github-hosted' && runner.os == 'Linux'
@@ -36,19 +36,19 @@ jobs:
         sudo apt install gcc-14 g++-14 -y
         sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 14 --slave /usr/bin/g++ g++ /usr/bin/g++-14
         sudo apt-get install libz-dev libbz2-dev liblzma-dev libflac-dev libnetcdf-dev libpython3-dev
-        sudo pip install numpy scipy matplotlib astropy healpy sphinx sphinx-rtd-theme --break-system-packages
+        sudo pip install numpy scipy matplotlib astropy healpy --break-system-packages
 
     - name: Setup python on macOS
       if: runner.environment == 'github-hosted' && runner.os == 'macOS'
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: '3.13'
 
     - name: Install macOS Dependencies
       if: runner.environment == 'github-hosted' && runner.os == 'macOS'
       run: |
         brew install --overwrite bzip2 xz flac netcdf
-        python3.13 -m pip install numpy scipy matplotlib astropy healpy sphinx sphinx-rtd-theme --break-system-packages
+        python3.13 -m pip install numpy scipy matplotlib astropy healpy --break-system-packages
 
     - name: Create Build Environment
       # Some projects don't allow in-source building, so create a separate build directory
@@ -85,19 +85,25 @@ jobs:
       # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail
       run: ctest -C $BUILD_TYPE --output-on-failure
 
+    - name: Docs Requirements
+      if: runner.environment == 'github-hosted' && github.ref_name == 'master' && matrix.docs
+      shell: bash
+      run: |
+        python3 -m pip install -r $GITHUB_WORKSPACE/doc/requirements.txt
+
     - name: Docs
       if: github.ref_name == 'master' && matrix.docs
       working-directory: ${{runner.workspace}}/${{matrix.toolset}}
       shell: bash
       # Build docs
       run: |
-        rm -f ${{runner.workspace}}/doc/moddoc_*.rst
+        rm -f $GITHUB_WORKSPACE/doc/moddoc_*.rst
         rm -rf docs/
         cmake --build . --config $BUILD_TYPE --target docs
 
     - name: Checkout gh_pages
       if: github.ref_name == 'master' && matrix.docs
-      uses: actions/checkout@v4
+      uses: actions/checkout@v5
       with:
         ref: gh-pages
         path: gh-pages

.github/workflows/wheels.yml

@@ -11,60 +11,21 @@ concurrency:
 
 jobs:
   build_wheels:
-    name: Build wheels for ${{ matrix.build }}
+    name: Build wheels for ${{ matrix.pyver }}-${{ matrix.build }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
+        pyver: [cp39, cp310, cp311, cp312, cp313]
+        os: [macos-14, ubuntu-24.04, ubuntu-24.04-arm]
         include:
-          # macos-13 is an intel runner, macos-14 is apple silicon
-          - os: macos-13
-            build: cp38-macosx_x86_64
-            target: 13.0
-          - os: macos-13
-            build: cp39-macosx_x86_64
-            target: 13.0
-          - os: macos-13
-            build: cp310-macosx_x86_64
-            target: 13.0
-          - os: macos-13
-            build: cp311-macosx_x86_64
-            target: 13.0
-          - os: macos-13
-            build: cp312-macosx_x86_64
-            target: 13.0
-          - os: macos-13
-            build: cp313-macosx_x86_64
-            target: 13.0
-
-          - os: macos-14
-            build: cp39-macosx_arm64
-            target: 14.0
-          - os: macos-14
-            build: cp310-macosx_arm64
-            target: 14.0
           - os: macos-14
-            build: cp311-macosx_arm64
+            build: macosx_arm64
             target: 14.0
-          - os: macos-14
-            build: cp312-macosx_arm64
-            target: 14.0
-          - os: macos-14
-            build: cp313-macosx_arm64
-            target: 14.0
-
-          - os: ubuntu-latest
-            build: cp38-manylinux_x86_64
-          - os: ubuntu-latest
-            build: cp39-manylinux_x86_64
-          - os: ubuntu-latest
-            build: cp310-manylinux_x86_64
-          - os: ubuntu-latest
-            build: cp311-manylinux_x86_64
-          - os: ubuntu-latest
-            build: cp312-manylinux_x86_64
-          - os: ubuntu-latest
-            build: cp313-manylinux_x86_64
+          - os: ubuntu-24.04
+            build: manylinux_x86_64
+          - os: ubuntu-24.04-arm
+            build: manylinux_aarch64
 
     steps:
       - name: Set macOS deployment target
@@ -77,17 +38,17 @@ jobs:
         run: brew install automake libtool
 
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
           fetch-tags: true
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.23.3
+        uses: pypa/cibuildwheel@v3.2.0
         env:
-          CIBW_BUILD: ${{ matrix.build }}
-          CIBW_BEFORE_ALL_LINUX: yum install -y zlib-devel bzip2-devel xz-devel netcdf-devel
-          CIBW_BEFORE_ALL_MACOS: brew install netcdf
+          CIBW_BUILD: ${{ matrix.pyver }}-${{ matrix.build }}
+          CIBW_BEFORE_ALL_LINUX: yum install -y zlib-devel bzip2-devel xz-devel netcdf-devel cfitsio-devel
+          CIBW_BEFORE_ALL_MACOS: brew install netcdf cfitsio
           CIBW_BEFORE_BUILD_MACOS: >
             ln -s $(dirname $(readlink -f $(which python3)))/python3-config $(dirname $(which python3))/python3-config &&
             pip install cmake && ./deps/install_flac.sh
@@ -115,14 +76,14 @@ jobs:
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         with:
-          name: cibw-wheel-${{ matrix.build }}
+          name: cibw-wheel-${{ matrix.pyver }}-${{ matrix.build }}
           path: ./wheelhouse/spt3g*.whl
 
   build_sdist:
     name: Build source distribution
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           fetch-depth: 0
           fetch-tags: true
@@ -144,7 +105,7 @@ jobs:
     if: github.event_name == 'release' && github.event.action == 'published'
     steps:
       - name: Download artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v5
         with:
           # unpacks all CIBW artifacts into dist/
           pattern: cibw-*

CMakeLists.txt

@@ -32,6 +32,7 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall")
 
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+set(CMAKE_INTERPROCEDURAL_OPTIMIZATION OFF)
 
 # Find all the Python libraries
 set(THREADS_PREFER_PTHREAD_FLAG ON)
@@ -163,6 +164,7 @@ endif()
 add_custom_target(tarball
 	COMMAND rm -rf ${CMAKE_BINARY_DIR}/tarball ${CMAKE_BINARY_DIR}/spt3g.tgz
 	COMMAND mkdir ${CMAKE_BINARY_DIR}/tarball
+	COMMAND ${CMAKE_MAKE_PROGRAM} -C ${CMAKE_BINARY_DIR}
 	COMMAND ${CMAKE_SOURCE_DIR}/cmake/copy_build_dir.sh ${CMAKE_BINARY_DIR}
 	COMMAND ${Python_EXECUTABLE} -m compileall -fq ${CMAKE_BINARY_DIR}/tarball
 	COMMAND tar -C ${CMAKE_BINARY_DIR}/tarball/ -czf ${CMAKE_BINARY_DIR}/spt3g.tgz .

README.rst

@@ -52,14 +52,16 @@ If your system defaults to Python 2, but you wish to use Python 3, please do the
 
 On any system, this software requires numpy and scipy (hard requirements), plus astropy and healpy (optional).
 
-Setup on RHEL6
---------------
+Setup on RHEL Systems with OASIS
+--------------------------------
 
-Note that on any RHEL6 system, you will need a newer compiler than ships with the OS. Please follow whatever directions apply at your site to achieve this. Alternately, if you have OASIS set up on your local system, run this before anything else (also works on several other operating systems, including RHEL7):
+If using RHEL8 or RHEL9 and you have OASIS set up on your local system, run this before anything else:
 
 .. code-block:: shell
 
-	eval `/cvmfs/spt.opensciencegrid.org/py3-v4/setup.sh`
+	eval `/cvmfs/spt.opensciencegrid.org/py3-v5/setup.sh`
+
+This will add a relatively new compiler and all necessary development packages (FLAC, GSL, etc) for compiling this software.
 
 
 How to Build
@@ -91,6 +93,16 @@ Alternatively, for users that only use a single build environment, set the follo
 	export LD_LIBRARY_PATH=$SPT3G_SOFTWARE_BUILD_PATH/lib:$LD_LIBRARY_PATH
 	export PATH=$SPT3G_SOFTWARE_BUILD_PATH/bin:$PATH
 
+Building the Documentation
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You may find that you are missing some of the required packages for building the documentation.  To fix this, run the following commands:
+
+.. code-block:: shell
+
+	cd spt3g_software
+	pip install -r doc/requirements.txt
+
 To build the documentation in the build directory type:
 
 .. code-block:: shell

calibration/python/bolopropertiesutils.py

@@ -94,7 +94,7 @@ def to_value(self, string):
     def extract_string(self, string, include_units=True):
         """Return the band substring from the input string, or None if not
         found."""
-        m = self._regex.match(string)
+        m = self._regex.search(string)
         if not m:
             return None
         if not include_units:
@@ -104,10 +104,10 @@ def extract_string(self, string, include_units=True):
     def extract_value(self, value):
         """Return the band in G3Units extracted from the input string, or None
         if not found."""
-        s = extract_string(value, include_units=False)
+        s = self.extract_string(value, include_units=False)
         if not s:
             return None
-        return float(v) * self._uvalue
+        return float(s) * self._uvalue
 
 
 # global instance used by functions below

core/CMakeLists.txt

@@ -116,6 +116,7 @@ add_spt3g_test(pipeline_module)
 add_spt3g_test(multiproc)
 add_spt3g_test(frames)
 add_spt3g_test(frame_types)
+add_spt3g_test(frameobjects)
 add_spt3g_test(datatypes)
 if(FLAC_FOUND)
 	add_spt3g_test(ts_nanencoding)
@@ -145,6 +146,7 @@ add_spt3g_test(timestrings)
 add_spt3g_test(testcomplexconv)
 add_spt3g_test(droporphanmetadata)
 add_spt3g_test(networktest SLOWTEST)
+add_spt3g_test(non_contiguous_vectors)
 add_spt3g_test(non_double_timestreams)
 add_spt3g_test(endprocessing)
 add_spt3g_test(lazyreader)

core/include/core/G3Map.h

@@ -73,6 +73,7 @@ G3MAP_OF(std::string, std::vector<bool>, G3MapVectorBool);
 G3MAP_OF(std::string, std::vector<std::string>, G3MapVectorString);
 G3MAP_OF(std::string, G3VectorVectorString, G3MapVectorVectorString);
 G3MAP_OF(std::string, std::vector<std::complex<double> >, G3MapVectorComplexDouble);
+G3MAP_OF(std::string, G3Time, G3MapTime);
 G3MAP_OF(std::string, G3VectorTime, G3MapVectorTime);
 G3MAP_OF(std::string, std::string, G3MapString);
 

core/include/core/G3Timestream.h

@@ -271,8 +271,8 @@ class G3TimestreamMap : public G3FrameObject,
 			ts->start = start;
 			ts->stop = stop;
 			ts->units = units;
-			ts->use_flac_ = compression_level;
-			ts->flac_depth_ = bit_depth;
+			ts->SetFLACCompression(compression_level);
+			ts->SetFLACBitDepth(bit_depth);
 			ts->root_data_ref_ = data;
 			ts->data_ = data.get() + offset;
 			ts->data_type_ = data_type;

core/include/core/container_pybindings.h

@@ -163,6 +163,11 @@ register_g3vector(py::module_ &scope, std::string name, Args &&...args)
 
 	cls.def(g3frameobject_picklesuite<V>());
 
+	cls.def("__str__", &V::Summary)
+	    .def("Summary", &V::Summary, "Short (one-line) description of the object")
+	    .def("Description", &V::Description,
+	        "Long-form human-readable description of the object");
+
 	return cls;
 }
 
@@ -390,6 +395,11 @@ register_g3map(py::module_ &scope, std::string name, Args &&...args)
 
 	cls.def(g3frameobject_picklesuite<M>());
 
+	cls.def("__str__", &M::Summary)
+	    .def("Summary", &M::Summary, "Short (one-line) description of the object")
+	    .def("Description", &M::Description,
+	        "Long-form human-readable description of the object");
+
 	return cls;
 }
 

core/include/core/serialization.h

@@ -118,16 +118,21 @@ class G3Pickler
 {
 public:
 	template <class T> static
-	auto getstate(const py::object &self) {
+	auto dumpstate(const T &obj) {
 		std::vector<char> buffer;
 		G3BufferOutputStream os(buffer);
 		{
 			cereal::PortableBinaryOutputArchive ar(os);
-			ar << self.cast<const T &>();
+			ar << obj;
 		}
 		os.flush();
 
-		py::bytes data(buffer.data(), buffer.size());
+		return py::bytes(buffer.data(), buffer.size());
+	}
+
+	template <class T> static
+	auto getstate(const py::object &self) {
+		py::bytes data = dumpstate<T>(self.cast<const T &>());
 
 		py::dict py_state;
 		if (py::hasattr(self, "__dict__"))
@@ -136,27 +141,40 @@ class G3Pickler
 		return py::make_tuple(py_state, data);
 	}
 
+	template <class T> static
+	auto loadstate(T &obj, const std::string_view &buffer) {
+		G3BufferInputStream fis((char *)&buffer[0], buffer.size());
+		cereal::PortableBinaryInputArchive ar(fis);
+
+		ar >> obj;
+	}
+
 	template <class T> static
 	auto setstate(const py::tuple &state) {
 		auto py_state = state[0].cast<py::dict>();
-
 		auto buffer = state[1].cast<std::string_view>();
-		G3BufferInputStream fis((char *)&buffer[0], buffer.size());
-		cereal::PortableBinaryInputArchive ar(fis);
 
 		T obj;
-		ar >> obj;
+		loadstate<T>(obj, buffer);
 
 		return std::make_pair(std::move(obj), py_state);
 	}
 };
 
 // Call this function in a class .def method to enable pickling
 template <class T>
-auto
-g3frameobject_picklesuite()
+struct g3frameobject_picklesuite
 {
-	return py::pickle(&G3Pickler::getstate<T>, &G3Pickler::setstate<T>);
+	static constexpr bool op_enable_if_hook = true;
+
+	template <typename Class>
+	void execute(Class &cl) const {
+		cl.def(py::pickle(&G3Pickler::getstate<T>, &G3Pickler::setstate<T>));
+		cl.def("_cereal_loads", &G3Pickler::loadstate<T>,
+		    "Populate this instance from a serialized data buffer");
+		cl.def("_cereal_dumps", &G3Pickler::dumpstate<T>,
+		    "Save the state of this instance to a serialized data buffer");
+	}
 };
 
 // Register a G3FrameObject-derived class.  Includes a copy constructor,