Merge pull request #241 from ComputationalRadiationPhysics/revert-234-master

Revert "Make filename creation consistent"

Author: ax3l

src/HandleMgr.cpp

@@ -1,5 +1,5 @@
 /**
- * Copyright 2013-2016 Felix Schmitt, Alexander Grund
+ * Copyright 2013 Felix Schmitt
  *
  * This file is part of libSplash.
  *
@@ -27,17 +27,17 @@
 #include <hdf5.h>
 
 #include "splash/core/HandleMgr.hpp"
-#include "splash/core/logging.hpp"
-#include "splash/core/DCHelper.hpp"
 
 namespace splash
 {
 
     HandleMgr::HandleMgr(uint32_t maxHandles, FileNameScheme fileNameScheme) :
     maxHandles(maxHandles),
+    numHandles(0),
     mpiSize(1, 1, 1),
     fileNameScheme(fileNameScheme),
     fileFlags(0),
+    singleFile(true),
     fileCreateCallback(NULL),
     fileCreateUserData(NULL),
     fileOpenCallback(NULL),
@@ -69,53 +69,26 @@ namespace splash
         return full_msg.str();
     }
 
-    void HandleMgr::setFileNameScheme(FileNameScheme fileNameScheme) throw (DCException)
-    {
-        if (this->fileNameScheme == fileNameScheme)
-            return;
-        if (!filename.empty())
-            throw DCException(getExceptionString("setFileNameScheme",
-                                    "Tried to change scheme while file(s) were still open", ""));
-        this->fileNameScheme = fileNameScheme;
-    }
-
     void HandleMgr::open(Dimensions mpiSize, const std::string baseFilename,
             hid_t fileAccProperties, unsigned flags)
-    throw (DCException)
     {
+        this->numHandles = mpiSize.getScalarSize();
         this->mpiSize.set(mpiSize);
         this->filename = baseFilename;
         this->fileAccProperties = fileAccProperties;
         this->fileFlags = flags;
-        // Validation: For parallel files we normally append MPI rank or iteration number.
-        //             This is disabled by using FNS_FULLNAME
-        //             or when the filename already contains an h5-extension.
-        if (fileNameScheme != FNS_FULLNAME && baseFilename.find(".h5") == baseFilename.length() - 3)
-        {
-            if (mpiSize.getScalarSize() > 1)
-            {
-                throw DCException(getExceptionString("open",
-                        "Passed full filename for parallel file operations",
-                        baseFilename.c_str()));
-            } else
-            {
-                log_msg(1, "\n"
-                        "\tWarning: Passed full filename for parallel file operations: %s\n"
-                        "It is recommended to pass only the base name (no extension)"
-                        "and let the implementation choose a filename.\n", filename.c_str());
-            }
-        }
+        this->singleFile = false;
     }
 
     void HandleMgr::open(const std::string fullFilename,
             hid_t fileAccProperties, unsigned flags)
-    throw (DCException)
     {
-        setFileNameScheme(FNS_FULLNAME);
+        this->numHandles = 1;
         this->mpiSize.set(1, 1, 1);
         this->filename = fullFilename;
         this->fileAccProperties = fileAccProperties;
         this->fileFlags = flags;
+        this->singleFile = true;
     }
 
     uint32_t HandleMgr::indexFromPos(Dimensions& mpiPos)
@@ -152,7 +125,7 @@ namespace splash
     throw (DCException)
     {
         uint32_t index = 0;
-        if (fileNameScheme != FNS_FULLNAME)
+        if (!singleFile)
             index = indexFromPos(mpiPos);
 
         HandleMap::iterator iter = handles.find(index);
@@ -177,34 +150,28 @@ namespace splash
                 leastAccIndex.ctr = 0;
             }
 
-            // Append prefix and extension if we don't have a full filename (extension)
-            std::string fullFilename;
-            if (fileNameScheme != FNS_FULLNAME && filename.find(".h5") != filename.length() - 3)
+            std::stringstream filenameStream;
+            filenameStream << filename;
+            if (!singleFile)
             {
-                std::stringstream filenameStream;
-                filenameStream << filename;
                 if (fileNameScheme == FNS_MPI)
                 {
                     filenameStream << "_" << mpiPos[0] << "_" << mpiPos[1] <<
                             "_" << mpiPos[2] << ".h5";
-                } else if(fileNameScheme == FNS_ITERATIONS)
+                } else
                     filenameStream << "_" << mpiPos[0] << ".h5";
-                fullFilename = filenameStream.str();
-            }else
-                fullFilename = filename;
+            }
 
             H5Handle newHandle = 0;
 
             // serve requests to create files once as create and as read/write afterwards
             if ((fileFlags & H5F_ACC_TRUNC) && (createdFiles.find(index) == createdFiles.end()))
             {
-                DCHelper::testFilename(fullFilename);
-
-                newHandle = H5Fcreate(fullFilename.c_str(), fileFlags,
+                newHandle = H5Fcreate(filenameStream.str().c_str(), fileFlags,
                         H5P_FILE_CREATE_DEFAULT, fileAccProperties);
                 if (newHandle < 0)
                     throw DCException(getExceptionString("get", "Failed to create file",
-                            fullFilename.c_str()));
+                        filenameStream.str().c_str()));
 
                 createdFiles.insert(index);
 
@@ -217,10 +184,10 @@ namespace splash
                 if (fileFlags & H5F_ACC_TRUNC)
                     tmp_flags = H5F_ACC_RDWR;
 
-                newHandle = H5Fopen(fullFilename.c_str(), tmp_flags, fileAccProperties);
+                newHandle = H5Fopen(filenameStream.str().c_str(), tmp_flags, fileAccProperties);
                 if (newHandle < 0)
                     throw DCException(getExceptionString("get", "Failed to open file",
-                            fullFilename.c_str()));
+                        filenameStream.str().c_str()));
 
                 if (fileOpenCallback)
                     fileOpenCallback(newHandle, index, fileOpenUserData);
@@ -257,6 +224,8 @@ namespace splash
         leastAccIndex.ctr = 0;
         leastAccIndex.index = 0;
         mpiSize.set(1, 1, 1);
+        numHandles = 0;
+        singleFile = true;
 
         // close all remaining handles
         HandleMap::const_iterator iter = handles.begin();

src/ParallelDataCollector.cpp

@@ -1,5 +1,5 @@
 /**
- * Copyright 2013-2016 Felix Schmitt, Axel Huebl, Alexander Grund
+ * Copyright 2013-2015 Felix Schmitt, Axel Huebl
  *
  * This file is part of libSplash.
  *
@@ -67,6 +67,14 @@ namespace splash
         log_msg(3, "Raw Data Cache (File) = %llu KiB", (long long unsigned) (rawCacheSize / 1024));
     }
 
+    std::string ParallelDataCollector::getFullFilename(uint32_t id, std::string baseFilename)
+    {
+        std::stringstream serial_filename;
+        serial_filename << baseFilename << "_" << id << ".h5";
+
+        return serial_filename.str();
+    }
+
     std::string ParallelDataCollector::getExceptionString(std::string func, std::string msg,
             const char *info)
     {
@@ -193,16 +201,15 @@ namespace splash
 
     ParallelDataCollector::~ParallelDataCollector()
     {
-        close();
         H5Pclose(fileAccProperties);
-        finalize();
     }
 
     void ParallelDataCollector::finalize()
     {
+        log_msg(1, "finalizing data collector");
+
         if (options.mpiComm != MPI_COMM_NULL)
         {
-            log_msg(1, "finalizing data collector");
             MPI_Comm_free(&options.mpiComm);
             options.mpiComm = MPI_COMM_NULL;
         }
@@ -238,9 +245,6 @@ namespace splash
 
     void ParallelDataCollector::close()
     {
-        if (fileStatus == FST_CLOSED)
-            return;
-
         log_msg(1, "closing parallel data collector");
 
         // close opened hdf5 file handles

src/SerialDataCollector.cpp

@@ -1,5 +1,5 @@
 /**
- * Copyright 2013-2016 Felix Schmitt, Axel Huebl, Alexander Grund
+ * Copyright 2013-2015 Felix Schmitt, Axel Huebl
  *
  * This file is part of libSplash.
  *
@@ -35,6 +35,7 @@
 #include "splash/core/DCAttribute.hpp"
 #include "splash/core/DCDataSet.hpp"
 #include "splash/core/DCGroup.hpp"
+#include "splash/core/DCHelper.hpp"
 #include "splash/core/SDCHelper.hpp"
 #include "splash/core/logging.hpp"
 #include "splash/basetypes/basetypes.hpp"
@@ -69,18 +70,8 @@ namespace splash
         return (stat(filename.c_str(), &fileInfo) == 0);
     }
 
-    std::string SerialDataCollector::getFullFilename(const Dimensions mpiPos, std::string baseFilename,
-            bool isFullNameAllowed) const throw (DCException)
+    std::string SerialDataCollector::getFullFilename(const Dimensions mpiPos, std::string baseFilename) const
     {
-        // Check for existing extension
-        if (baseFilename.find(".h5") == baseFilename.length() - 3)
-        {
-            if (isFullNameAllowed)
-                return baseFilename;
-            else
-                throw DCException("Full filename is not allowed!");
-        }
-
         std::stringstream serial_filename;
         serial_filename << baseFilename << "_" << mpiPos[0] << "_" << mpiPos[1] <<
                 "_" << mpiPos[2] << ".h5";
@@ -106,7 +97,7 @@ namespace splash
      *******************************************************************************/
 
     SerialDataCollector::SerialDataCollector(uint32_t maxFileHandles) :
-    handles(maxFileHandles, HandleMgr::FNS_FULLNAME),
+    handles(maxFileHandles, HandleMgr::FNS_MPI),
     fileStatus(FST_CLOSED),
     maxID(-1),
     mpiTopology(1, 1, 1)
@@ -122,7 +113,7 @@ namespace splash
                 "failed to initialize/open HDF5 library"));
 
 #ifndef SPLASH_VERBOSE_HDF5
-        // Suppress automatic output of HDF5 exception messages
+        // surpress automatic output of HDF5 exception messages
         if (H5Eset_auto2(H5E_DEFAULT, NULL, NULL) < 0)
             throw DCException(getExceptionString("SerialDataCollector",
                 "failed to disable error printing"));
@@ -134,7 +125,6 @@ namespace splash
 
     SerialDataCollector::~SerialDataCollector()
     {
-        close();
     }
 
     void SerialDataCollector::open(const char* filename, FileCreationAttr &attr)
@@ -167,19 +157,16 @@ namespace splash
 
     void SerialDataCollector::close()
     {
-        if (fileStatus == FST_CLOSED)
-            return;
-
         log_msg(1, "closing serial data collector");
 
-        if ((fileStatus == FST_CREATING || fileStatus == FST_WRITING) &&
-            maxID >= 0)
+        if (fileStatus == FST_CREATING || fileStatus == FST_WRITING)
         {
+            DCGroup group;
+            group.open(handles.get(0), SDC_GROUP_HEADER);
+
             // write number of iterations
             try
             {
-                DCGroup group;
-                group.open(handles.get(0), SDC_GROUP_HEADER);
                 ColTypeInt32 ctInt32;
                 DCAttribute::writeAttribute(SDC_ATTR_MAX_ID, ctInt32.getDataType(),
                         group.getHandle(), &maxID);
@@ -759,8 +746,9 @@ namespace splash
     {
         this->fileStatus = FST_CREATING;
 
-        std::string full_filename = getFullFilename(attr.mpiPosition, filename,
-                attr.mpiSize.getScalarSize() == 1);
+        // appends the mpiPosition to the filename (e.g. myfile_0_1_0.h5)
+        std::string full_filename = getFullFilename(attr.mpiPosition, filename);
+        DCHelper::testFilename(full_filename);
 
         this->enableCompression = attr.enableCompression;
 
@@ -791,8 +779,11 @@ namespace splash
     {
         fileStatus = FST_WRITING;
 
-        std::string full_filename = getFullFilename(attr.mpiPosition, filename,
-                attr.mpiSize.getScalarSize() == 1);
+        std::string full_filename = filename;
+        if (full_filename.find(".h5") == std::string::npos)
+            full_filename = getFullFilename(attr.mpiPosition, filename);
+
+        DCHelper::testFilename(full_filename);
 
         this->enableCompression = attr.enableCompression;
 
@@ -814,7 +805,9 @@ namespace splash
         this->fileStatus = FST_MERGING;
 
         // open reference file to get mpi information
-        std::string full_filename = getFullFilename(Dimensions(0, 0, 0), filename, true);
+        std::string full_filename = getFullFilename(Dimensions(0, 0, 0), filename);
+
+        DCHelper::testFilename(full_filename);
 
         if (!fileExists(full_filename))
         {
@@ -828,7 +821,6 @@ namespace splash
         // no compression for in-memory datasets
         this->enableCompression = false;
 
-        handles.setFileNameScheme(HandleMgr::FNS_MPI);
         handles.open(mpiTopology, filename, fileAccProperties, H5F_ACC_RDONLY);
     }
 
@@ -837,7 +829,11 @@ namespace splash
     {
         this->fileStatus = FST_READING;
 
-        std::string full_filename = getFullFilename(attr.mpiPosition, filename, true);
+        std::string full_filename = filename;
+        if (full_filename.find(".h5") == std::string::npos)
+            full_filename = getFullFilename(attr.mpiPosition, filename);
+
+        DCHelper::testFilename(full_filename);
 
         if (!fileExists(full_filename))
         {

src/include/splash/ParallelDataCollector.hpp

@@ -54,6 +54,16 @@ namespace splash
          */
         void setFileAccessParams(hid_t& fileAccProperties);
 
+        /**
+         * Constructs a filename from a base filename and the current id
+         * such as baseFilename+id+.h5
+         *
+         * @param id Iteration ID.
+         * @param baseFilename Base filename for the new file.
+         * @return newly Constructed filename including file extension.
+         */
+        static std::string getFullFilename(uint32_t id, std::string baseFilename);
+
         /**
          * Internal function for formatting exception messages.
          *

src/include/splash/SerialDataCollector.hpp

@@ -60,17 +60,13 @@ namespace splash
 
         /**
          * Constructs a filename from a base filename and the process' mpi position
-         * such as myfile_0_1_0.h5
-         * Does nothing if the filename already ends with ".h5" (only allowed
-         * for Prod(mpiSize)==1 or reading).
+         * such as baseFilename+mpiPos+.h5
          *
          * @param mpiPos MPI position of the process
          * @param baseFilename base filename for the new file
-         * @param isFullNameAllowed If false, an exception is raised when a full name is passed
-         * @return newly constructed filename including file extension
+         * @return newly constructed filename iucluding file exitension
          */
-        std::string getFullFilename(const Dimensions mpiPos, std::string baseFilename,
-                bool isFullNameAllowed) const throw (DCException);
+        std::string getFullFilename(const Dimensions mpiPos, std::string baseFilename) const;
 
         /**
          * Internal function for formatting exception messages.