Add (optional at compile-time) G3Frame JSON output

core/CMakeLists.txt

@@ -25,6 +25,15 @@ add_spt3g_library(core SHARED
 	src/G3Units.cxx ${CORE_EXTRA_SRCS} src/pybindings.cxx
 )
 
+
+# json output option
+set(ENABLE_JSON_OUTPUT "FALSE" CACHE BOOL "Enable JSON output of frames (adds to compile time and binary size)")
+
+if (${ENABLE_JSON_OUTPUT})
+	target_compile_definitions(core PUBLIC -DSPT3G_ENABLE_JSON_OUTPUT)
+endif()
+
+
 # Link dependencies
 target_link_libraries(core PUBLIC spt3g)
 
@@ -92,6 +101,7 @@ endif()
 link_python_dir()
 
 add_spt3g_program(bin/spt3g-dump)
+add_spt3g_program(bin/spt3g-jsonify)
 add_spt3g_program(bin/spt3g-verify)
 add_spt3g_program(bin/spt3g-inspect)
 add_spt3g_program(bin/gen-analysis-doc)

core/bin/spt3g-jsonify

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+
+import sys
+from spt3g import core
+
+if len(sys.argv) < 2:
+    print('Usage: %s file.g3 ' % sys.argv[0])
+    print('Dumps the given file to standard output as JSON (not quite indented properly, sorry)')
+    sys.exit(1)
+
+#Crappy hand-crafted json wrapper to make this stream valid json
+print('{')
+print('    "filename": "%s",' % (sys.argv[1]))
+print('    "frames":')
+print('     [')
+
+first = True
+indent = 8*' '
+print_indent = 7*' '
+
+for f in core.G3File(sys.argv[1]):
+	if not first:
+		print (print_indent,',')
+	print(print_indent,indent.join(f.as_json().splitlines(True)))
+	first = False
+print('    ]')
+print('}')
+

core/include/core/G3Frame.h

@@ -100,6 +100,10 @@ class G3Frame {
 	template <class A> void save(A &ar, unsigned) const;
 	template <class A> void load(A &ar, unsigned);
 
+	// JSON representation of a frame (if JSON output is enabled, otherwise
+	// outputs valid JSON telling you that JSON output isn't enabled)
+	std::string asJSON() const;
+
 	// Routines for handling the stored serialized copies of data.
 	// These are all const because they only manipulate caches and
 	// so change only performance rather than data contents of the frame.

core/include/core/serialization.h

@@ -4,6 +4,9 @@
 #include <sstream>
 
 #include <cereal/archives/portable_binary.hpp>
+#ifdef SPT3G_ENABLE_JSON_OUTPUT
+#include <cereal/archives/json.hpp>
+#endif
 
 #include <cereal/types/polymorphic.hpp>
 #include <cereal/types/string.hpp>
@@ -14,14 +17,39 @@
 #include <pybindings.h>
 #include <G3Logging.h>
 
-#define G3_SERIALIZABLE_CODE(x) \
+#define G3_SERIALIZABLE_CODE_BINARY(x) \
 template void x::serialize(cereal::PortableBinaryOutputArchive &, unsigned); \
 template void x::serialize(cereal::PortableBinaryInputArchive &, unsigned); \
 
-#define G3_SPLIT_SERIALIZABLE_CODE(x) \
+#define G3_SPLIT_SERIALIZABLE_CODE_BINARY(x) \
 template void x::save(cereal::PortableBinaryOutputArchive &, unsigned) const; \
 template void x::load(cereal::PortableBinaryInputArchive &, unsigned); \
 
+#ifdef SPT3G_ENABLE_JSON_OUTPUT
+
+#define G3_SERIALIZABLE_CODE_JSON(x) \
+template void x::serialize(cereal::JSONOutputArchive &, unsigned); \
+template void x::serialize(cereal::JSONInputArchive &, unsigned); \
+
+#define G3_SPLIT_SERIALIZABLE_CODE_JSON(x) \
+template void x::save(cereal::JSONOutputArchive &, unsigned) const; \
+template void x::load(cereal::JSONInputArchive &, unsigned); \
+
+#define G3_SERIALIZABLE_CODE(x) \
+G3_SERIALIZABLE_CODE_BINARY(x) \
+G3_SERIALIZABLE_CODE_JSON(x) \
+
+#define G3_SPLIT_SERIALIZABLE_CODE(x) \
+G3_SPLIT_SERIALIZABLE_CODE_BINARY(x) \
+G3_SPLIT_SERIALIZABLE_CODE_JSON(x) \
+
+#else
+
+#define G3_SERIALIZABLE_CODE(x) G3_SERIALIZABLE_CODE_BINARY(x)
+#define G3_SPLIT_SERIALIZABLE_CODE(x) G3_SPLIT_SERIALIZABLE_CODE_BINARY(x)
+
+#endif
+
 // Work around crummy slow implementation of xsgetn() in libc++
 class G3InputStreamBuffer : public std::basic_streambuf<char>
 {

core/src/G3Frame.cxx

@@ -242,6 +242,61 @@ void G3Frame::save(A &ar, unsigned v) const
 	ar << make_nvp("crc", crc);
 }
 
+#ifdef SPT3G_ENABLE_JSON_OUTPUT
+template<>
+void G3Frame::save(cereal::JSONOutputArchive &ar, unsigned v) const
+{
+	using cereal::make_nvp;
+	uint32_t size(map_.size());
+
+	ar << make_nvp("size", size);
+	std::string typestr(1,(char) type);
+	ar << make_nvp("type", typestr);
+	for (auto i = map_.begin(); i != map_.end(); i++) {
+		//make sure it's deserialized so we don't just write a blob
+		blob_decode(i->second);
+		ar << make_nvp("name", i->first);
+		ar << make_nvp("val", i->second.frameobject);
+	}
+}
+
+template<>
+void G3Frame::load(cereal::JSONInputArchive &ar, unsigned v)
+{
+	using cereal::make_nvp;
+	G3_CHECK_VERSION(v);
+
+	int size;
+	std::string typestr;
+
+	ar >> make_nvp("size", size);
+	ar >> make_nvp("type", typestr);
+	type = FrameType((uint32_t)typestr[0]);
+
+	map_.clear();
+	for (int i = 0; i < size; i++) {
+		std::string name;
+		struct blob_container blob;
+		ar >> make_nvp("name", name);
+		ar >> make_nvp("val", blob.frameobject);
+		map_.insert(G3MapType::value_type(name, blob));
+	}
+}
+#endif
+
+std::string
+G3Frame::asJSON() const
+{
+	std::stringstream str;
+#ifdef SPT3G_ENABLE_JSON_OUTPUT
+	cereal::JSONOutputArchive ar(str);
+	ar << cereal::make_nvp("frame", *this);
+#else
+	str << "{error: \"spt3g_software compiled without JSON support\"}" << std::endl;
+#endif
+	return str.str();
+}
+
 template <typename T>
 void G3Frame::loads(T &is)
 {
@@ -373,7 +428,7 @@ template void G3Frame::saves(G3BufferOutputStream &) const;
 template void G3Frame::saves(std::ostream &) const;
 template void G3Frame::saves(std::ostringstream &) const;
 
-G3_SPLIT_SERIALIZABLE_CODE(G3Frame);
+G3_SPLIT_SERIALIZABLE_CODE_BINARY(G3Frame);
 
 G3FramePtr
 g3frame_char_constructor(std::string max_4_chars)
@@ -620,6 +675,7 @@ PYBINDINGS("core", scope) {
 	      "where those serialized copies already exist. Saves memory for "
 	      "frames about to be written at the expense of CPU time to "
 	      "re-decode them if they are accessed again later.")
+	    .def("as_json", &G3Frame::asJSON, "JSON representation of frame")
 	    .def(g3frameobject_picklesuite<G3Frame>())
 	    .def_property_readonly("hash", &g3frame_hash,
 	      "Return the serialized representation of the frame")

core/src/G3Timestream.cxx

@@ -437,6 +437,111 @@ template <class A> void G3Timestream::load(A &ar, unsigned v)
 	}
 }
 
+#ifdef SPT3G_ENABLE_JSON_OUTPUT
+template <> void G3Timestream::save(cereal::JSONOutputArchive &ar, unsigned v) const
+{
+	ar & cereal::make_nvp("G3FrameObject",
+	    cereal::base_class<G3FrameObject>(this));
+	ar & cereal::make_nvp("units", units);
+	ar & cereal::make_nvp("start", start);
+	ar & cereal::make_nvp("stop", stop);
+
+	ar & cereal::make_nvp("data_type", data_type_);
+	if (buffer_) {
+		ar & cereal::make_nvp("data", *buffer_);
+	} else {
+		switch (data_type_) {
+		case TS_DOUBLE: {
+			std::vector<double> data((double *)data_,
+			    (double *)data_ + len_);
+			ar & cereal::make_nvp("data", data);
+			break;
+		}
+		case TS_FLOAT: {
+			std::vector<float> data((float *)data_,
+			    (float *)data_ + len_);
+			ar & cereal::make_nvp("data", data);
+			break;
+		}
+		case TS_INT32: {
+			std::vector<int32_t> data((int32_t *)data_,
+			    (int32_t *)data_ + len_);
+			ar & cereal::make_nvp("data", data);
+			break;
+		}
+		case TS_INT64: {
+			std::vector<int64_t> data((int64_t *)data_,
+			    (int64_t *)data_ + len_);
+			ar & cereal::make_nvp("data", data);
+			break;
+		}
+		default:
+			log_fatal("Unknown timestream datatype %d", data_type_);
+		}
+	}
+}
+
+template <> void G3Timestream::load(cereal::JSONInputArchive &ar, unsigned v)
+{
+	G3_CHECK_VERSION(v);
+
+	ar & cereal::make_nvp("G3FrameObject",
+	    cereal::base_class<G3FrameObject>(this));
+	ar & cereal::make_nvp("units", units);
+	if (v >= 2) {
+		ar & cereal::make_nvp("start", start);
+		ar & cereal::make_nvp("stop", stop);
+	}
+
+	if (buffer_)
+		delete buffer_;
+	buffer_ = NULL;
+	root_data_ref_.reset();
+
+	if (v >= 3)
+		ar & cereal::make_nvp("data_type", data_type_);
+	else
+		data_type_ = TS_DOUBLE;
+	switch (data_type_) {
+	case TS_DOUBLE:
+		buffer_ = new std::vector<double>();
+		ar & cereal::make_nvp("data", *buffer_);
+		len_ = buffer_->size();
+		data_ = &(*buffer_)[0];
+		break;
+	case TS_FLOAT: {
+		std::vector<float> *data = new std::vector<float>();
+		ar & cereal::make_nvp("data", *data);
+		root_data_ref_ = std::shared_ptr<std::vector<float> >(
+		    data);
+		len_ = data->size();
+		data_ = &(*data)[0];
+		break;
+		}
+	case TS_INT32: {
+		std::vector<int32_t> *data = new std::vector<int32_t>();
+		ar & cereal::make_nvp("data", *data);
+		root_data_ref_ = std::shared_ptr<
+		    std::vector<int32_t> >(data);
+		len_ = data->size();
+		data_ = &(*data)[0];
+		break;
+	}
+	case TS_INT64: {
+		std::vector<int64_t> *data = new std::vector<int64_t>();
+		ar & cereal::make_nvp("data", *data);
+		root_data_ref_ = std::shared_ptr<
+		    std::vector<int64_t> >(data);
+		len_ = data->size();
+		data_ = &(*data)[0];
+		break;
+	}
+	default:
+		log_fatal("Unknown timestream datatype %d", data_type_);
+	}
+}
+#endif
+
 G3Timestream::G3Timestream(const G3Timestream &r) :
     units(r.units), start(r.start), stop(r.stop), use_flac_(r.use_flac_),
     flac_depth_(r.flac_depth_), len_(r.len_), data_type_(r.data_type_)
@@ -990,7 +1095,7 @@ void G3TimestreamMap::Compactify()
 	}
 }
 
-G3_SPLIT_SERIALIZABLE_CODE(G3Timestream);
+G3_SPLIT_SERIALIZABLE_CODE_BINARY(G3Timestream);
 G3_SERIALIZABLE_CODE(G3TimestreamMap);
 
 static void
@@ -1396,7 +1501,7 @@ PYBINDINGS("core", scope) {
 	    .def("SetFLACCompression", &G3Timestream::SetFLACCompression,
 	      "Pass True to turn on FLAC compression when serialized. "
 	      "FLAC compression only works if the timestream is in units of "
-	      "counts.")
+	      "counts. FLAC compression is ignored when outputing to JSON.")
 	    .def("SetFLACBitDepth", &G3Timestream::SetFLACBitDepth,
 	      "Change the bit depth for FLAC compression, may be 24 or 32 "
 	      "(default, requires version 1.4+).")

serve/CMakeLists.txt

@@ -0,0 +1,22 @@
+
+function (make_embedded_resource var_name file_name out_file)
+  if (${file_name} IS_NEWER_THAN ${out_file})
+    message(Generating ${out_file} " from " ${file_name})
+    file (WRITE ${out_file} "// automatically generated from ${file_name}\n")
+    file (READ  ${file_name} filedata HEX)
+    string (REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\1," cfiledata ${filedata})
+    file (APPEND ${out_file} "unsigned char ${var_name}[] = { ${cfiledata} };\n")
+  endif()
+endfunction()
+
+make_embedded_resource(favicon favicon.ico ${CMAKE_BINARY_DIR}/serve/favicon.h)
+
+if(NOT DEFINED ENV{CIBUILDWHEEL})
+  add_spt3g_executable(spt3g-json-serve-files src/spt3g-json-serve-files.cxx)
+
+  target_link_libraries(spt3g-json-serve-files core z)
+  target_include_directories(spt3g-json-serve-files PRIVATE
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+    $<BUILD_INTERFACE:${CMAKE_BINARY_DIR}>
+  )
+endif()