saving data to npy format in streamer is now bug free. Added three tests: 2 python and 1 at cpp level.

Author: Dilawar Singh

basecode/Id.cpp

@@ -76,18 +76,26 @@ string Id::id2str( Id id )
 string Id::path( const string& separator) const
 {
     string ret = Neutral::path( eref() );
-    // Trim off trailing []
-    assert( ret.length() > 0 );
-    // the 'back' operation is not supported by pre 2011 compilers
 
-    while ( ret[ ret.length() - 1 ] == ']' )
+#if 0
+    // NOTE/FIXME: Monday 09 March 2020 12:30:27 PM IST, Dilawar Singh
+    // This beaks the path comparison. Getting x.path from x returned in a 
+    // list by moose.wildcardFind() and getting path from here doesn't math
+    // when this is enabled.
+    // If we want to remove [0] then it should be done globally and not just
+    // here.
+
+    // Trim off trailing [] 
+    assert( ret.length() > 0 );
+    while ( ret.back() == ']' )
     {
         size_t pos = ret.find_last_of( '[' );
         if ( pos != string::npos && pos > 0 )
         {
             ret = ret.substr( 0, pos );
         }
     }
+#endif 
 
     return ret;
 }

builtins/Streamer.cpp

@@ -185,10 +185,10 @@ void Streamer::reinit(const Eref& e, ProcPtr p)
             if( tick != tableDt_[0] )
             {
                 moose::showWarn( "Table " + tableIds_[i].path() + " has "
-                        " different clock dt. "
-                        " Make sure all tables added to Streamer have the same "
-                        " dt value."
-                        );
+                                 " different clock dt. "
+                                 " Make sure all tables added to Streamer have the same "
+                                 " dt value."
+                               );
             }
         }
     }
@@ -209,12 +209,12 @@ void Streamer::reinit(const Eref& e, ProcPtr p)
         if( tableTick_[i] != tableTick_[0] )
         {
             LOG( moose::warning
-                    , "Table " << tableIds_[i].path()
-                    << " has tick (dt) which is different than the first table."
-                    << endl
-                    << " Got " << tableTick_[i] << " expected " << tableTick_[0]
-                    << endl << " Disabling this table."
-                    );
+                 , "Table " << tableIds_[i].path()
+                 << " has tick (dt) which is different than the first table."
+                 << endl
+                 << " Got " << tableTick_[i] << " expected " << tableTick_[0]
+                 << endl << " Disabling this table."
+               );
             invalidTables.push_back( i );
         }
     }
@@ -236,7 +236,7 @@ void Streamer::reinit(const Eref& e, ProcPtr p)
     // write now.
     currTime_ = 0.0;
     zipWithTime( );
-    StreamerBase::writeToOutFile(outfilePath_, format_, "w", data_, columns_);
+    StreamerBase::writeToOutFile(outfilePath_, format_, WRITE, data_, columns_);
     data_.clear( );
 }
 
@@ -247,7 +247,7 @@ void Streamer::reinit(const Eref& e, ProcPtr p)
 void Streamer::cleanUp( )
 {
     zipWithTime( );
-    StreamerBase::writeToOutFile( outfilePath_, format_, "a", data_, columns_ );
+    StreamerBase::writeToOutFile( outfilePath_, format_, APPEND, data_, columns_ );
     data_.clear( );
 }
 
@@ -261,7 +261,7 @@ void Streamer::process(const Eref& e, ProcPtr p)
 {
     // LOG( moose::debug, "Writing Streamer data to file." );
     zipWithTime( );
-    StreamerBase::writeToOutFile( outfilePath_, format_, "a", data_, columns_ );
+    StreamerBase::writeToOutFile( outfilePath_, format_, APPEND, data_, columns_ );
     data_.clear();
     numWriteEvents_ += 1;
 }
@@ -284,12 +284,12 @@ void Streamer::addTable( Id table )
     tables_.push_back( t );
     tableTick_.push_back( table.element()->getTick() );
 
-    // NOTE: If user can make sure that names are unique in table, using name is
-    // better than using the full path.
-    if( t->getColumnName().size() > 0 )
-        columns_.push_back( t->getColumnName( ) );
+    // NOTE: Table can also have name. If name is set by User, use the name to
+    // for column name, else use full path of the table.
+    if(t->getColumnName().size() > 0)
+        columns_.push_back(t->getColumnName());
     else
-        columns_.push_back( moose::moosePathToUserPath( table.path() ) );
+        columns_.push_back(table.path());
 }
 
 /**
@@ -355,7 +355,7 @@ size_t Streamer::getNumTables( void ) const
  * @Synopsis  Get number of write events in streamer. Useful for debugging and
  * performance measuerments.
  *
- * @Returns   
+ * @Returns
  */
 /* ----------------------------------------------------------------------------*/
 size_t Streamer::getNumWriteEvents( void ) const
@@ -414,8 +414,8 @@ void Streamer::zipWithTime( )
         {
 #if 0
             LOG( moose::debug
-                    , "Table " << tables_[i]->getName( ) << " is not functional. Filling with zero "
-                    );
+                 , "Table " << tables_[i]->getName( ) << " is not functional. Filling with zero "
+               );
 #endif
             tVec.resize( numEntriesInEachTable, 0 );
         }

builtins/StreamerBase.cpp

@@ -57,41 +57,49 @@ void StreamerBase::setOutFilepath( string filepath )
 
 void StreamerBase::writeToOutFile( const string& filepath
         , const string& outputFormat
-        , const string& openmode
+        , const OpenMode openmode
         , const vector<double>& data
         , const vector<string>& columns
         )
 {
     if( data.size() == 0 )
         return;
 
-    if( "npy" == outputFormat )
-        writeToNPYFile( filepath, openmode, data, columns );
+    if("npy" == outputFormat  || "npz" == outputFormat)
+    {
+        OpenMode m = (openmode == WRITE)?WRITE_BIN:APPEND_BIN;
+        writeToNPYFile( filepath, m, data, columns );
+    }
     else if( "csv" == outputFormat or "dat" == outputFormat )
-        writeToCSVFile( filepath, openmode, data, columns );
+    {
+        OpenMode m = (openmode == WRITE)?WRITE_STR:APPEND_STR;
+        writeToCSVFile( filepath, m, data, columns );
+    }
     else
     {
         LOG( moose::warning, "Unsupported format " << outputFormat
                 << ". Use npy or csv. Falling back to default csv"
            );
-        writeToCSVFile( filepath, openmode, data, columns );
+        OpenMode m = (openmode == WRITE)?WRITE_STR:APPEND_STR;
+        writeToCSVFile( filepath, m, data, columns );
     }
 }
 
 /*  Write to a csv file.  */
-void StreamerBase::writeToCSVFile( const string& filepath, const string& openmode
+void StreamerBase::writeToCSVFile( const string& filepath, const OpenMode openmode
         , const vector<double>& data, const vector<string>& columns )
 {
+    string m = (openmode == WRITE_STR)?"w":"a";
+    FILE* fp = fopen( filepath.c_str(), m.c_str());
 
-    FILE* fp = fopen( filepath.c_str(), openmode.c_str() );
     if( NULL == fp )
     {
         LOG( moose::warning, "Failed to open " << filepath );
         return;
     }
 
     // If writing in "w" mode, write the header first.
-    if(openmode == "w")
+    if(openmode == WRITE_STR)
     {
         string headerText = "";
         for( vector<string>::const_iterator it = columns.begin();
@@ -116,16 +124,23 @@ void StreamerBase::writeToCSVFile( const string& filepath, const string& openmod
 }
 
 /*  write data to a numpy file */
-void StreamerBase::writeToNPYFile( const string& filepath, const string& openmode
+void StreamerBase::writeToNPYFile( const string& filepath, const OpenMode openmode
         , const vector<double>& data, const vector<string>& columns )
 {
-    cnpy2::save_numpy<double>( filepath, data, columns, openmode );
+    //for(auto v: data) cout << v << ' ';
+    //cout << endl;
+
+    if(openmode == APPEND_BIN)
+        return cnpy2::appendNumpy( filepath, data, columns);
+
+    if(openmode == WRITE_BIN)
+        return cnpy2::writeNumpy( filepath, data, columns);
 }
 
 string StreamerBase::vectorToCSV( const vector<double>& ys, const string& fmt )
 {
-    stringstream ss;
+    string res{""};
     for( auto v : ys )
-        ss << v << ",";
-    return ss.str();
+        res += std::to_string(v) + ",";
+    return res;
 }

builtins/StreamerBase.h

@@ -28,6 +28,8 @@ using namespace std;
 
 class TableBase;
 
+enum OpenMode {WRITE, APPEND, WRITE_STR, APPEND_STR, WRITE_BIN, APPEND_BIN};
+
 class StreamerBase : public TableBase
 {
 
@@ -63,7 +65,7 @@ class StreamerBase : public TableBase
      */
     static void writeToOutFile(
             const string& filepath, const string& format
-            , const string& openmode
+            , const OpenMode openmode
             , const vector<double>& data
             , const vector<string>& columns
             );
@@ -72,15 +74,15 @@ class StreamerBase : public TableBase
      * @brief Write data to csv file. See the documentation of writeToOutfile
      * for details.
      */
-    static void writeToCSVFile( const string& filepath, const string& openmode
+    static void writeToCSVFile( const string& filepath, const OpenMode openMode
             , const vector<double>& data, const vector<string>& columns
             );
 
     /**
      * @brief  Write to NPY format. See the documentation of
      * writeToOutfile for more details.
      */
-    static void writeToNPYFile( const string& filepath, const string& openmode
+    static void writeToNPYFile( const string& filepath, const OpenMode openmode
             , const vector<double>& data
             , const vector<string>& columns
             );

builtins/Table.cpp

@@ -228,7 +228,7 @@ Table::~Table( )
     if( useFileStreamer_ )
     {
         mergeWithTime( data_ );
-        StreamerBase::writeToOutFile( outfile_, format_, "a", data_, columns_ );
+        StreamerBase::writeToOutFile( outfile_, format_, APPEND, data_, columns_ );
         clearAllVecs();
     }
 }
@@ -269,7 +269,7 @@ void Table::process( const Eref& e, ProcPtr p )
         if( fmod(lastTime_, 5.0) == 0.0 || getVecSize() >= 10000 )
         {
             mergeWithTime( data_ );
-            StreamerBase::writeToOutFile( outfile_, format_, "a", data_, columns_ );
+            StreamerBase::writeToOutFile( outfile_, format_, APPEND, data_, columns_ );
             clearAllVecs();
         }
         }
@@ -332,7 +332,7 @@ void Table::reinit( const Eref& e, ProcPtr p )
     if( useFileStreamer_ )
     {
         mergeWithTime( data_ );
-        StreamerBase::writeToOutFile( outfile_, format_, "w", data_, columns_);
+        StreamerBase::writeToOutFile( outfile_, format_, WRITE, data_, columns_);
         clearAllVecs();
     }
 }

pymoose/CMakeLists.txt

@@ -59,8 +59,7 @@ execute_process( COMMAND ${PYTHON_EXECUTABLE}-config --libs
 
 set_target_properties(_moose PROPERTIES
     COMPILE_DEFINITIONS "PYMOOSE"
-  COMPILE_FLAGS "${COMPILE_FLAGS} ${PYTHON_INCLUDE_FLAGS}"
-    )
+    COMPILE_FLAGS "${COMPILE_FLAGS} ${PYTHON_INCLUDE_FLAGS}")
 
 # Remove prefix lib from python module.
 if(NOT(PYTHON_SO_EXTENSION STREQUAL ""))

shell/Neutral.cpp

@@ -764,6 +764,7 @@ string Neutral::path( const Eref& e )
     stringstream ss;
 
     pathVec.push_back( curr );
+
     while ( curr.id != Id() )
     {
         ObjId mid = curr.eref().element()->findCaller( pafid );
@@ -776,8 +777,10 @@ string Neutral::path( const Eref& e )
         curr = Msg::getMsg( mid )->findOtherEnd( curr );
         pathVec.push_back( curr );
     }
+
     if ( pathVec.size() <= 1 )
         return "/";
+
     for ( unsigned int i = 1; i < pathVec.size(); ++i )
     {
         ss << "/";
@@ -792,6 +795,7 @@ string Neutral::path( const Eref& e )
         	ss << "[" << oid.dataIndex << "]";
         	*/
     }
+
     // Append braces if Eref was for a fieldElement. This should
     // work even if it is off-node.
     if ( e.element()->hasFields() )

shell/Wildcard.cpp

@@ -104,11 +104,6 @@ static int innerFind( const string& path, vector< ObjId >& ret)
     return wildcardRelativeFind( start, names, 0, ret );
 }
 
-/*
-static int wildcardRelativeFind( Id start, const vector< string >& path,
-		unsigned int depth, vector< Id >& ret )
-		*/
-
 /**
  * This is the basic wildcardFind function, working on a single
  * tree. It adds entries into the vector 'ret' with Ids found according