Merge branch 'master' into json_output

Author: arahlin

core/include/core/dataio.h

@@ -30,26 +30,6 @@ typedef boost::iostreams::filtering_ostream g3_ostream;
 int g3_istream_from_path(g3_istream &stream, const std::string &path,
     float timeout=-1.0, size_t buffersize=1024*1024);
 
-/**
- * Seek to a byte offset in an open input file stream.
- *
- * @param  stream   A reference to the filtering istream, e.g. as configured by
- *                  g3_istream_from_path.
- * @param  offset   File offset to seek to, in bytes, relative to the beginning
- *                  of the file.
- * @return New read head position, or -1 on error.
- */
-off_t g3_istream_seek(g3_istream &stream, off_t offset);
-
-/**
- * Return the current read head position in an open input file stream.
- *
- * @param  stream   A reference to the filtering istream, e.g. as configured by
- *                  g3_istream_from_path.
- * @return Current read head position, or -1 on error.
- */
-off_t g3_istream_tell(g3_istream &stream);
-
 /**
  * Configure a filtering stream for G3Frame compression to a local file.
  *

core/src/G3Reader.cxx

@@ -100,15 +100,14 @@ void G3Reader::Process(G3FramePtr frame, std::deque<G3FramePtr> &out)
 }
 
 off_t G3Reader::Seek(off_t offset) {
-	try {
-		return g3_istream_seek(stream_, offset);
-	} catch (...) {
+	if (stream_.peek() == EOF && offset != Tell())
 		log_fatal("Cannot seek %s; stream closed at EOF.", cur_file_.c_str());
-	}
+	stream_.seekg(offset, std::ios_base::beg);
+	return offset;
 }
 
 off_t G3Reader::Tell() {
-	return g3_istream_tell(stream_);
+	return stream_.tellg();
 }
 
 PYBINDINGS("core") {

core/src/dataio.cxx

@@ -162,20 +162,6 @@ g3_istream_from_path(g3_istream &stream, const std::string &path,
 	return fd;
 }
 
-off_t
-g3_istream_seek(g3_istream &stream, off_t offset)
-{
-	if (stream.peek() == EOF && offset != g3_istream_tell(stream))
-		log_fatal("Cannot seek stream, closed at EOF.");
-	return boost::iostreams::seek(stream, offset, std::ios_base::beg);
-}
-
-off_t
-g3_istream_tell(g3_istream &stream)
-{
-	return boost::iostreams::seek(stream, 0, std::ios_base::cur);
-}
-
 void
 g3_ostream_to_path(g3_ostream &stream, const std::string &path,
     bool append, bool counter)

core/tests/fileio.py

@@ -3,9 +3,13 @@
 from spt3g import core
 import time
 
+nframes = 1000
+verbose = False
+
 # File to disk
+core.log_notice("Writing %d frames to disk" % nframes)
 pipe = core.G3Pipeline()
-pipe.Add(core.G3InfiniteSource, type=core.G3FrameType.Timepoint, n=10)
+pipe.Add(core.G3InfiniteSource, type=core.G3FrameType.Timepoint, n=nframes)
 # Drop a wiring frame in the middle
 count = 0
 def addwiring(fr):
@@ -25,16 +29,18 @@ def addinfo(fr):
     fr['count'] = n
     n += 1
 pipe.Add(addinfo)
-pipe.Add(core.Dump)
+if verbose:
+    pipe.Add(core.Dump)
 pipe.Add(core.G3Writer, filename='test.g3')
-pipe.Run()
-assert n == 10, 'Wrong number of frames written (%d should be %d)' % (n, 10)
+pipe.Run(profile=True)
+assert n == nframes, 'Wrong number of frames written (%d should be %d)' % (n, nframes)
 
 # And back from disk
-print('Reading')
+core.log_notice("Reading %d frames from disk" % nframes)
 pipe = core.G3Pipeline()
 pipe.Add(core.G3Reader, filename='test.g3', track_filename=True)
-pipe.Add(core.Dump)
+if verbose:
+    pipe.Add(core.Dump)
 n = 0
 def checkinfo(fr):
     global n
@@ -45,20 +51,21 @@ def checkinfo(fr):
     assert fr._filename == 'test.g3', 'Wrong filename'
     n += 1
 pipe.Add(checkinfo)
-pipe.Run()
+pipe.Run(profile=True)
 
-assert n == 10, 'Wrong number of frames read (%d should be %d)' % (n, 10)
+assert n == nframes, 'Wrong number of frames read (%d should be %d)' % (n, nframes)
 
 # Skip empty files
+core.log_notice("Reading %d frames from disk with empty files" % nframes)
 with core.G3Writer("empty.g3") as wr:
     pass
 n = 0
 pipe = core.G3Pipeline()
 pipe.Add(core.G3Reader, filename=["empty.g3", "test.g3", "empty.g3"], track_filename=True)
 pipe.Add(checkinfo)
-pipe.Run()
+pipe.Run(profile=True)
 
-assert n == 10, 'Wrong number of frames read (%d should be %d)' % (n, 10)
+assert n == nframes, 'Wrong number of frames read (%d should be %d)' % (n, nframes)
 
 # Indexing
 class CachingReader:
@@ -78,10 +85,12 @@ def __call__(self, frame):
 
 cacher = CachingReader()
 
+core.log_notice("Reading %d frames from disk with frame indexing" % nframes)
 pipe = core.G3Pipeline()
 pipe.Add(cacher)
-pipe.Add(core.Dump)
-pipe.Run()
+if verbose:
+    pipe.Add(core.Dump)
+pipe.Run(profile=True)
 
 assert cacher.w_pos is not None, 'Missing wiring frame'
 
@@ -107,10 +116,12 @@ def __call__(self, frame):
 
 cached = CachedReader(start=cacher.w_pos)
 
+core.log_notice("Reading %d frames from disk with seek" % nframes)
 pipe = core.G3Pipeline()
 pipe.Add(cached)
-pipe.Add(core.Dump)
-pipe.Run()
+if verbose:
+    pipe.Add(core.Dump)
+pipe.Run(profile=True)
 
 assert cached.pos is None, "Missing wiring frame"
 

maps/python/map_modules.py

@@ -15,6 +15,7 @@
     "InjectMaps",
     "ReplicateMaps",
     "CoaddMaps",
+    "RebinMaps",
     "ReprojectMaps",
     "coadd_map_files",
 ]
@@ -835,6 +836,49 @@ def coadd_map_files(
     return coadder.coadd_frame
 
 
+@core.indexmod
+def RebinMaps(frame, scale=1, weighted=True, norm=False):
+    """
+    Rebin any maps in the input frame into larger pixels by summing scale-x-scale
+    blocks of pixels together.  Map dimensions must be a multiple of the rebinning
+    scale.
+
+    Arguments
+    ---------
+    scale : int
+        Rebinning scale factor, such that scale-x-scale blocks of pixels are summed
+        into larger pixels.  Must divide evenly into all map pixel dimensions.
+    weighted : bool
+        If True (default), ensure that maps have had weights applied before
+        rebinning.  Otherwise, rebin maps without checking the weights.
+    norm : bool
+        If False (default), sum all of the sub-pixels into each larger output pixel.
+        If True, normalize the output pixel by the number of sub-pixels each
+        encloses.  Only applies to Stokes parameters T/Q/U, not weights or hits.
+    """
+    if scale <= 1:
+        return
+
+    if weighted:
+        ApplyWeights(frame)
+
+    for k in ["T", "Q", "U", "Wunpol", "Wpol", "H"]:
+        if k in frame:
+            m = frame.pop(k)
+            if k in "TQU":
+                frame[k] = m.rebin(scale, norm=norm)
+            elif k == "H":
+                frame[k] = m.rebin(scale, norm=False)
+            else:
+                if weighted and norm:
+                    core.log_warn(
+                        "Rebinning weighted frames with norm=True may result "
+                        "in inconsistent maps and weights",
+                        unit="RebinMaps",
+                    )
+                frame[k] = m.rebin(scale)
+
+
 @core.indexmod
 class ReprojectMaps(object):
     """