* rawmerge.hpp: generalize/fix condition for consistent timeseries

* introduce propagation of C++ exceptions to Cython/Python
* convert all cout/cerr output to exceptions
* Cython: introduce separate 'do_conversion' => avoid constructor to
trigger conversion

Author: Mario Fink

.gitignore

@@ -1,18 +1,19 @@
 
-/build
-
 eatraw
 eatdev
 
 nohup.out
 
 .DS_Store
 
-*.so
-
 raw_eater.cpp
 raw_meat.cpp
 
 cyt/*.cpp
+/build
 
+*.log
+*.so
 *.o
+*.csv
+*.parquet

cyt/raw_eater.pxd

@@ -14,14 +14,18 @@ from libcpp cimport bool
 #
 # for how to overload the constructor see
 # https://cython.readthedocs.io/en/latest/src/userguide/wrapping_CPlusPlus.html
+# and propagating exceptions from C++ to Python
+# http://docs.cython.org/en/latest/src/userguide/wrapping_CPlusPlus.html#exceptions
 
 cdef extern from "../lib/raweat.hpp":
     cdef cppclass raw_eater:
         # constructor(s)
-        raw_eater() except+
+        raw_eater() except +
         raw_eater(string) except +
         # set new file for decoding
         void set_file(string)
+        # perform conversion (pass any C++ exceptions to Python)
+        void setup_and_conversion() except +
         # get validity of data format
         bool get_valid()
         # get channel name and unit

cyt/raw_eater.pyx

@@ -30,6 +30,9 @@ cdef class raweater:
       raise ValueError("'" + str(rawfile) + "' does not exist")
     self.rawit.set_file(rawfile)
 
+  def do_conversion(self):
+      self.rawit.setup_and_conversion()
+
   def validity(self):
     return self.rawit.get_valid()
 

cyt/raw_meat.pxd

@@ -23,8 +23,8 @@ cdef extern from "../lib/rawmerge.hpp":
         vector[double] get_data()
         # dump all data to .csv
         void write_table(const char*,char)
-        # add channel and try to merge it
-        bool add_channel(string)
+        # add channel and try to merge it (pass C++ exceptions to Python)
+        bool add_channel(string) except +
         # get total number of (added) channels
         int get_num_channels()
         # get list of channel names

lib/raweat.hpp

@@ -80,14 +80,20 @@ class raw_eater
   raw_eater(std::string rawfile, bool showlog = false) : rawfile_(rawfile)
   {
     // trigger setup and conversion
-    if ( ! rawfile.empty() ) this->setup_and_conversion(showlog);
+    //if ( ! rawfile.empty() ) this->setup_and_conversion(showlog);
   }
 
   raw_eater()
   {
 
   }
 
+  // destructor
+  ~raw_eater()
+  {
+
+  }
+
   // provide/set new raw file
   void set_file(std::string rawfile, bool showlog = false)
   {
@@ -100,11 +106,11 @@ class raw_eater
     datmes_.clear();
 
     // do setup and conversion
-    setup_and_conversion(showlog);
+    //setup_and_conversion(showlog);
   }
 
   // set up and conversion
-  void setup_and_conversion(bool showlog)
+  void setup_and_conversion(bool showlog = false)
   {
     // make sure 'raw_file' is already set
     assert ( !rawfile_.empty() );
@@ -123,10 +129,8 @@ class raw_eater
 
     if ( showlog )
     {
-      // show raw data
+      // show raw data and display predefined markers
       this->show_buffer();
-
-      // display predefined markers
       this->show_markers();
     }
 
@@ -150,12 +154,6 @@ class raw_eater
     }
   }
 
-  // destructor
-  ~raw_eater()
-  {
-
-  }
-
   // display buffer/data properties
   void show_buffer(int numel = 32)
   {
@@ -239,18 +237,23 @@ class raw_eater
     unsigned long int totalmarksize = 0;
     for (std::pair<std::string,std::vector<unsigned char>> mrk : markers_ )
     {
-      //assert ( datasec_[mrk.first].size() > 0 && "marker segment of length zero" );
       if ( datasec_[mrk.first].size() == 0 )
       {
-        std::cout<<"warning: "<<mrk.first<<" not found in buffer\n";
+        std::string errmess = mrk.first + std::string(" not found in buffer");
+        // std::cerr<<errmess;
+        try {
+          throw std::runtime_error(errmess);
+        } catch( const std::exception& e ) {
+          throw;
+	        //std::cout<<e.what()<<"\n";
+        }
       }
       totalmarksize += datasec_[mrk.first].size();
     }
     // std::cout<<"totalmarksize "<<totalmarksize<<"\n";
     // std::cout<<"\n";
 
     // check validity of format
-    // assert ( totalmarksize > 0 && "didn't find any predefined marker => probably not a valid .raw-file" );
     valid_ = ( totalmarksize < 100 ) ? false : true;
   }
 
@@ -389,7 +392,6 @@ class raw_eater
     }
     else
     {
-
       // switch for datatypes
       switch ( dattype )
       {
@@ -414,9 +416,9 @@ class raw_eater
           convert_data_as_type<unsigned long int>(datbuf,factor,offset);
           break;
         case 6 :
-          std::cout<<"warning: 'signed long int' datatype with experimental support\n";
           // assert ( sizeof(signed long int)*8 == typesize );
           // convert_data_as_type<signed long int>(datbuf,factor,offset);
+          // throw std::runtime_error("warning: 'signed long int' datatype with experimental support");
           assert ( sizeof(int)*8 == typesize );
           convert_data_as_type<int>(datbuf,factor,offset);
           break;
@@ -429,13 +431,13 @@ class raw_eater
           convert_data_as_type<double>(datbuf,factor,offset);
           break;
         case 9 :
-          std::cerr<<"'imc Devices Transitional Recording' datatype not supported\n";
+          throw std::runtime_error("'imc Devices Transitional Recording' datatype not supported");
           break;
         case 10 :
-          std::cerr<<"'Timestamp Ascii' datatype not supported\n";
+          throw std::runtime_error("'Timestamp Ascii' datatype not supported");
           break;
         case 11 :
-          std::cout<<"warning: '2-Byte-Word digital' datatype with experimental support\n";
+          // throw std::runtime_error("warning: '2-Byte-Word digital' datatype with experimental support");
           assert ( sizeof(short int)*8 == typesize );
           convert_data_as_type<short int>(datbuf,factor,offset);
           break;

lib/rawmerge.hpp

@@ -83,13 +83,19 @@ class raw_merger : public raw_eater
   {
     // set raw file and perform conversion
     this->set_file(rawfile,showlog);
+    // try {
+    this->setup_and_conversion();
+    // } catch (const std::exception& e) {
+    //   // throw;
+    //   std::cout<<e.what();
+    // }
 
     // show channel name, unit, timestep, time unit, etc.
     if ( showlog && this->get_valid() )
     {
-      std::cout<<this->get_name()<<" ["<<this->get_unit()<<"]"<<"\n";
       std::cout<<"Time ["<<this->get_temp_unit()<<"]"<<"\n";
       for ( unsigned long int i = 0; i < 5; i++ ) std::cout<<this->get_time()[i]<<"\n";
+      std::cout<<this->get_name()<<" ["<<this->get_unit()<<"]"<<"\n";
       for ( unsigned long int i = 0; i < 5; i++ ) std::cout<<this->get_data()[i]<<"\n";
       std::cout<<"lenght of channel "<<this->get_time().size()<<"\n";
       std::cout<<"\n";
@@ -126,12 +132,9 @@ class raw_merger : public raw_eater
         std::vector<double> td = this->get_data();
 
         // compare start/end of timeseries (define tolerance)
-        double deltat = 7*fmax(this->get_dt(),this->dt_);
-        if ( ( this->timeseries_[0]     - ts[0]     < deltat  )
-          && ( this->timeseries_.back() - ts.back() < deltat ) )
-        // double tol = 0.001;
-        // if ( ( (this->timeseries_[0]-ts[0])/ts[0] < tol )
-        //   && ( (this->timeseries_.back()-ts.back())/ts.back() < tol ) )
+        double deltat = 9*fmax(this->get_dt(),this->dt_);
+        if ( ( fabs( this->timeseries_[0]     - ts[0] )     < deltat  )
+          && ( fabs( this->timeseries_.back() - ts.back() ) < deltat ) )
         {
           // resulting new time series
           std::vector<double> newts;
@@ -155,27 +158,38 @@ class raw_merger : public raw_eater
         else
         {
           // refuse to merge due to inconsistent start of timeseries
-          std::cerr<<"rawmerge: add_channel '"<<rawfile
-                   <<"' : inconsistent start of time series, i.e. "
-                   <<timeseries_[0]<<" vs. "<<ts[0]<<"\n";
+          std::string errmess = std::string("rawmerge: add_channel '") + rawfile
+                              + std::string("' : inconsistent start of time series, i.e. ")
+                              + std::to_string(timeseries_[0]) + std::string(" vs. ")
+                              + std::to_string(ts[0])
+                              + std::string("( timesteps: ") + std::to_string(this->get_dt())
+                              + std::string(" and ") + std::to_string(this->dt_) + std::string(")");
+          // std::cerr<<errmess<<"\n";
+          throw std::runtime_error(errmess);
 
           return false;
         }
       }
       else
       {
         // refuse to merge due to different temporal units
-        std::cerr<<"rawmerge: add_channel '"<<rawfile
-                 <<"' : inconsistent time units: '"
-                 <<this->get_temp_unit()<<"' versus '"<<this->temp_unit_<<"'\n";
+        std::string errmess = std::string("rawmerge: add_channel '") + rawfile
+                            + std::string("' : inconsistent time units: ', i.e. '")
+                            + this->get_temp_unit() + std::string("' vs. '")
+                            + this->temp_unit_ + std::string("'");
+        // std::cerr<<errmess<<"\n";
+        throw std::runtime_error(errmess);
 
         return false;
       }
     }
     else
     {
       // provided file does not feature a valid .raw format
-      std::cerr<<"rawmerge: add_channel '"<<rawfile<<"' : invalid .raw file\n";
+      std::string errmess = std::string("rawmerge: add_channel '") + rawfile
+                          + std::string("' : invalid .raw file");
+      // std::cerr<<errmess<<"\n";
+      throw std::runtime_error(errmess);
 
       return false;
     }

makefile

@@ -15,13 +15,14 @@ EXE = eatraw
 
 # compiler and its options
 CCC = g++ -std=c++11
-OPT = -O3 -Wall -mavx -mno-tbm -mf16c -mno-f16c
+#OPT = -O3 -Wall -mavx -mno-tbm -mf16c -mno-f16c
+OPT = -O3 -Wall -Werror
 
 #-----------------------------------------------------------------------------#
 # C++
 
 # build executable
-$(EXE) : $(SRC)main.cpp $(LIB)raweat.hpp $(LIB)hexshow.hpp $(LIB)rawmerge.hpp
+$(EXE) : $(SRC)main.cpp $(LIB)raweat.hpp $(LIB)hexshow.hpp $(LIB)rawmerge.hpp output
 	$(CCC) $(OPT) $< -o $@
 
 # development version
@@ -37,6 +38,7 @@ clean :
 	rm -f $(EXE)
 	rm -f eatall
 	rm -f eatdev
+	rm -rf output/
 
 # check existence of name of executable globally
 chexe:=$(shell command -v $(EXE))
@@ -59,7 +61,8 @@ uninstall :
 
 # build python module
 py : $(CYT)setup_raw_eater.py $(CYT)raw_eater.pyx $(CYT)raw_eater.pxd $(LIB)raweat.hpp \
-     $(CYT)setup_raw_meat.py $(CYT)raw_meat.pyx $(CYT)raw_meat.pxd $(LIB)rawmerge.hpp
+     $(CYT)setup_raw_meat.py $(CYT)raw_meat.pyx $(CYT)raw_meat.pxd $(LIB)rawmerge.hpp \
+		 output
 	python3 $(CYT)setup_raw_eater.py build_ext --inplace
 	python3 $(CYT)setup_raw_meat.py build_ext --inplace
 	cp raw_eater.cpython-*.so pyt/
@@ -80,5 +83,10 @@ py-clean :
 	rm -f $(CYT)raw_meat.cpp
 	rm -rf build/
 	rm -f *.txt
+	rm -rf output/
+
+# prepare directory for test output
+output :
+	mkdir -pv output/
 
 #-----------------------------------------------------------------------------#