Merge pull request ComputationalRadiationPhysics#5403 from psychocoderHPC/fix-folderCreation

fix folder creation

Author: ikbuibui

include/picongpu/plugins/PhaseSpace/PhaseSpace.x.cpp

@@ -318,9 +318,6 @@ namespace picongpu
 
             if(activatePlugin)
             {
-                /** create dir */
-                pmacc::Filesystem::get().createDirectoryWithPermissions("phaseSpace");
-
                 uint32_t const r_element = axis_element.space;
 
                 /* CORE + BORDER elements for spatial bins */
@@ -382,6 +379,12 @@ namespace picongpu
                         std::lower_bound(planeReduceRootRanks.begin(), planeReduceRootRanks.end(), 0),
                         planeReduceRootRanks.end());
 
+                    if(static_cast<int>(gc.getGlobalRank()) == ranks.front())
+                    {
+                        /** only one rank is creating the output directory */
+                        pmacc::Filesystem::get().createDirectoryWithPermissions("phaseSpace");
+                    }
+
                     MPI_CHECK(MPI_Comm_group(MPI_COMM_WORLD, &world_group));
                     MPI_CHECK(MPI_Group_incl(world_group, ranks.size(), ranks.data(), &new_group));
                     MPI_CHECK(MPI_Comm_create(MPI_COMM_WORLD, new_group, &commFileWriter));

include/picongpu/plugins/makroParticleCounter/PerSuperCell.x.cpp

@@ -185,8 +185,14 @@ namespace picongpu
                 DataSpace<simDim> localSuperCells(subGrid.getLocalDomain().size / SuperCellSize::toRT());
                 localResult = std::make_unique<GridBufferType>(localSuperCells);
 
-                /* create folder for hdf5 files*/
-                pmacc::Filesystem::get().createDirectoryWithPermissions(foldername);
+                GridController<simDim>& gc = Environment<simDim>::get().GridController();
+                if(gc.getGlobalRank() == 0)
+                {
+                    /* Only one rank is allowed to create the directory.
+                     * All ranks participate the IO therefore selecting rank zero is fine.
+                     */
+                    pmacc::Filesystem::get().createDirectoryWithPermissions(foldername);
+                }
             }
         }
 

include/picongpu/plugins/openPMD/openPMDWriter.x.cpp

@@ -1126,16 +1126,26 @@ make sure that environment variable OPENPMD_BP_BACKEND is not set to ADIOS1.
 
                         Environment<>::get().PluginConnector().setNotificationPeriod(this, emplaced->second.periods());
 
-                        /** create output directory */
-                        pmacc::Filesystem::get().createDirectoryWithPermissions(outputDirectory);
+                        if(gc.getGlobalRank() == 0)
+                        {
+                            /* Only one rank is allowed to create the directory.
+                             * All ranks participate the IO therefore selecting rank zero is fine.
+                             */
+                            pmacc::Filesystem::get().createDirectoryWithPermissions(outputDirectory);
+                        }
                     }
                     else if(not tomlSourcesSpecified && notifyPeriodSpecified)
                     {
                         std::string const& notifyPeriod = m_help->notifyPeriod.get(id);
                         Environment<>::get().PluginConnector().setNotificationPeriod(this, notifyPeriod);
 
-                        /** create output directory */
-                        pmacc::Filesystem::get().createDirectoryWithPermissions(outputDirectory);
+                        if(gc.getGlobalRank() == 0)
+                        {
+                            /* Only one rank is allowed to create the directory.
+                             * All ranks participate the IO therefore selecting rank zero is fine.
+                             */
+                            pmacc::Filesystem::get().createDirectoryWithPermissions(outputDirectory);
+                        }
                     }
                     else
                     {

include/picongpu/plugins/output/images/PngCreator.hpp

@@ -101,6 +101,10 @@ namespace picongpu
         }
 
     private:
+        /** Write image to disk
+         *
+         * @attention Only one MPI rank is allowed to call this method.
+         */
         template<typename T_DataType>
         void createImage(std::shared_ptr<std::vector<T_DataType>> imageBuffer, MessageHeader const header);
 

include/picongpu/plugins/particleCalorimeter/ParticleCalorimeter.x.cpp

@@ -209,11 +209,15 @@ namespace picongpu
 
         void checkpoint(uint32_t currentStep, std::string const& checkpointDirectory) override
         {
-            /*
-             * Create folder for openPMD checkpoint files.
-             * openPMD would also do it automatically, but let's keep things explicit.
-             */
-            pmacc::Filesystem::get().createDirectoryWithPermissions(checkpointDirectory + "/" + this->foldername);
+            GridController<simDim>& gc = Environment<simDim>::get().GridController();
+            if(gc.getGlobalRank() == 0)
+            {
+                /*
+                 * Create folder for openPMD checkpoint files.
+                 * openPMD would also do it automatically, but let's keep things explicit.
+                 */
+                pmacc::Filesystem::get().createDirectoryWithPermissions(checkpointDirectory + "/" + this->foldername);
+            }
             auto dataSize = this->dBufLeftParsCalorimeter->capacityND();
             HBufCalorimeter hBufLeftParsCalorimeter(dataSize);
             HBufCalorimeter hBufTotal(dataSize);
@@ -356,8 +360,12 @@ namespace picongpu
                 this->calorimeterFrameVecY,
                 this->calorimeterFrameVecZ);
 
-            /* create folder for openPMD files*/
-            pmacc::Filesystem::get().createDirectoryWithPermissions(this->foldername);
+            bool const isIORank = this->allGPU_reduce->hasResult(mpi::reduceMethods::Reduce());
+            if(isIORank)
+            {
+                /* create folder for openPMD files*/
+                pmacc::Filesystem::get().createDirectoryWithPermissions(this->foldername);
+            }
 
             // set how often the plugin should be executed while PIConGPU is running
             Environment<>::get().PluginConnector().setNotificationPeriod(this, m_help->notifyPeriod.get(m_id));

include/pmacc/mappings/simulation/Filesystem.cpp

@@ -46,15 +46,9 @@ namespace pmacc
 
     void Filesystem::createDirectoryWithPermissions(std::string const dir) const
     {
-        auto const mpiRank = Environment<>::get().EnvironmentController().getCommunicator().getRank();
-        bool const isRootRank = mpiRank == 0;
-
-        if(isRootRank)
-        {
-            createDirectory(dir);
-            /* must be set by only one process to avoid races */
-            setDirectoryPermissions(dir);
-        }
+        createDirectory(dir);
+        /* must be set by only one process to avoid races */
+        setDirectoryPermissions(dir);
     }
 
     std::string Filesystem::basename(std::string const pathFilename) const

include/pmacc/mappings/simulation/Filesystem.hpp

@@ -34,35 +34,36 @@ namespace pmacc
     class Filesystem
     {
     public:
-        /**
-         * Create directory with default permissions
+        /** Create directory with default permissions
+         *
+         * @attention Only one MPI rank is allowed to call this method.
          *
          * @param dir name of directory
          */
         void createDirectory(std::string const dir) const;
-        /**
-         * Set 755 permissions for a directory
+        /** Set 755 permissions for a directory
+         *
+         * @attention Only one MPI rank is allowed to call this method.
          *
          * @param dir name of directory
          */
         void setDirectoryPermissions(std::string const dir) const;
 
-        /**
-         * Create directory and set 755 permissions by root rank.
+        /** Create directory and set 755 permissions by root rank.
+         *
+         * @attention Only one MPI rank is allowed to call this method.
          *
          * @param dir name of directory
          */
         void createDirectoryWithPermissions(std::string const dir) const;
 
-        /**
-         * Strip path from absolute or relative paths to filenames
+        /** Strip path from absolute or relative paths to filenames
          *
          * @param path and filename
          */
         std::string basename(std::string const pathFilename) const;
 
-        /**
-         * Returns the instance of the filesystem class.
+        /** Returns the instance of the filesystem class.
          *
          * This class is a singleton class.
          *

include/pmacc/simulationControl/SimulationHelper.cpp

@@ -111,7 +111,7 @@ namespace pmacc
             MPI_CHECK(MPI_Barrier(gc.getCommunicator().getMPIComm()));
 
             /* create directory containing checkpoints  */
-            if(numCheckpoints == 0)
+            if(numCheckpoints == 0 && gc.getGlobalRank() == 0)
             {
                 pmacc::Filesystem::get().createDirectoryWithPermissions(checkpointDirectory);
             }