fix folder creation

fix ComputationalRadiationPhysics#5401

The method `createDirectory()` and `createDirectoryWithPermissions()` was not clearly mention that only one MPI rank is allowed to call this method.
The original implementation guarded the permission setting method by using the global rank zero only but in cases where the IO rank was not rank zero the permission was not set and since the refactoring ComputationalRadiationPhysics#5094 also the output folder was not created.

Author: psychocoderHPC

include/picongpu/plugins/PhaseSpace/PhaseSpace.x.cpp

@@ -318,9 +318,6 @@ namespace picongpu
 
             if(activatePlugin)
             {
-                /** create dir */
-                pmacc::Filesystem::get().createDirectoryWithPermissions("phaseSpace");
-
                 uint32_t const r_element = axis_element.space;
 
                 /* CORE + BORDER elements for spatial bins */
@@ -382,6 +379,12 @@ namespace picongpu
                         std::lower_bound(planeReduceRootRanks.begin(), planeReduceRootRanks.end(), 0),
                         planeReduceRootRanks.end());
 
+                    if(static_cast<int>(gc.getGlobalRank()) == ranks.front())
+                    {
+                        /** only one rank is creating the output directory */
+                        pmacc::Filesystem::get().createDirectoryWithPermissions("phaseSpace");
+                    }
+
                     MPI_CHECK(MPI_Comm_group(MPI_COMM_WORLD, &world_group));
                     MPI_CHECK(MPI_Group_incl(world_group, ranks.size(), ranks.data(), &new_group));
                     MPI_CHECK(MPI_Comm_create(MPI_COMM_WORLD, new_group, &commFileWriter));

include/picongpu/plugins/makroParticleCounter/PerSuperCell.x.cpp

@@ -185,8 +185,14 @@ namespace picongpu
                 DataSpace<simDim> localSuperCells(subGrid.getLocalDomain().size / SuperCellSize::toRT());
                 localResult = std::make_unique<GridBufferType>(localSuperCells);
 
-                /* create folder for hdf5 files*/
-                pmacc::Filesystem::get().createDirectoryWithPermissions(foldername);
+                GridController<simDim>& gc = Environment<simDim>::get().GridController();
+                if(gc.getGlobalRank() == 0)
+                {
+                    /* Only one rank is allowed to create the directory.
+                     * All ranks participate the IO therefore selecting rank zero is fine.
+                     */
+                    pmacc::Filesystem::get().createDirectoryWithPermissions(foldername);
+                }
             }
         }
 

include/picongpu/plugins/openPMD/openPMDWriter.x.cpp

@@ -1126,16 +1126,26 @@ make sure that environment variable OPENPMD_BP_BACKEND is not set to ADIOS1.
 
                         Environment<>::get().PluginConnector().setNotificationPeriod(this, emplaced->second.periods());
 
-                        /** create output directory */
-                        pmacc::Filesystem::get().createDirectoryWithPermissions(outputDirectory);
+                        if(gc.getGlobalRank() == 0)
+                        {
+                            /* Only one rank is allowed to create the directory.
+                             * All ranks participate the IO therefore selecting rank zero is fine.
+                             */
+                            pmacc::Filesystem::get().createDirectoryWithPermissions(outputDirectory);
+                        }
                     }
                     else if(not tomlSourcesSpecified && notifyPeriodSpecified)
                     {
                         std::string const& notifyPeriod = m_help->notifyPeriod.get(id);
                         Environment<>::get().PluginConnector().setNotificationPeriod(this, notifyPeriod);
 
-                        /** create output directory */
-                        pmacc::Filesystem::get().createDirectoryWithPermissions(outputDirectory);
+                        if(gc.getGlobalRank() == 0)
+                        {
+                            /* Only one rank is allowed to create the directory.
+                             * All ranks participate the IO therefore selecting rank zero is fine.
+                             */
+                            pmacc::Filesystem::get().createDirectoryWithPermissions(outputDirectory);
+                        }
                     }
                     else
                     {

include/picongpu/plugins/output/images/PngCreator.hpp

@@ -101,6 +101,10 @@ namespace picongpu
         }
 
     private:
+        /** Write image to disk
+         *
+         * @attention Only one MPI rank is allowed to call this method.
+         */
         template<typename T_DataType>
         void createImage(std::shared_ptr<std::vector<T_DataType>> imageBuffer, MessageHeader const header);
 

include/picongpu/plugins/particleCalorimeter/ParticleCalorimeter.x.cpp

@@ -209,11 +209,15 @@ namespace picongpu
 
         void checkpoint(uint32_t currentStep, std::string const& checkpointDirectory) override
         {
-            /*
-             * Create folder for openPMD checkpoint files.
-             * openPMD would also do it automatically, but let's keep things explicit.
-             */
-            pmacc::Filesystem::get().createDirectoryWithPermissions(checkpointDirectory + "/" + this->foldername);
+            GridController<simDim>& gc = Environment<simDim>::get().GridController();
+            if(gc.getGlobalRank() == 0)
+            {
+                /*
+                 * Create folder for openPMD checkpoint files.
+                 * openPMD would also do it automatically, but let's keep things explicit.
+                 */
+                pmacc::Filesystem::get().createDirectoryWithPermissions(checkpointDirectory + "/" + this->foldername);
+            }
             auto dataSize = this->dBufLeftParsCalorimeter->capacityND();
             HBufCalorimeter hBufLeftParsCalorimeter(dataSize);
             HBufCalorimeter hBufTotal(dataSize);
@@ -356,8 +360,12 @@ namespace picongpu
                 this->calorimeterFrameVecY,
                 this->calorimeterFrameVecZ);
 
-            /* create folder for openPMD files*/
-            pmacc::Filesystem::get().createDirectoryWithPermissions(this->foldername);
+            bool const isIORank = this->allGPU_reduce->hasResult(mpi::reduceMethods::Reduce());
+            if(isIORank)
+            {
+                /* create folder for openPMD files*/
+                pmacc::Filesystem::get().createDirectoryWithPermissions(this->foldername);
+            }
 
             // set how often the plugin should be executed while PIConGPU is running
             Environment<>::get().PluginConnector().setNotificationPeriod(this, m_help->notifyPeriod.get(m_id));

include/pmacc/mappings/simulation/Filesystem.cpp

@@ -46,15 +46,9 @@ namespace pmacc
 
     void Filesystem::createDirectoryWithPermissions(std::string const dir) const
     {
-        auto const mpiRank = Environment<>::get().EnvironmentController().getCommunicator().getRank();
-        bool const isRootRank = mpiRank == 0;
-
-        if(isRootRank)
-        {
-            createDirectory(dir);
-            /* must be set by only one process to avoid races */
-            setDirectoryPermissions(dir);
-        }
+        createDirectory(dir);
+        /* must be set by only one process to avoid races */
+        setDirectoryPermissions(dir);
     }
 
     std::string Filesystem::basename(std::string const pathFilename) const

include/pmacc/mappings/simulation/Filesystem.hpp

@@ -34,35 +34,36 @@ namespace pmacc
     class Filesystem
     {
     public:
-        /**
-         * Create directory with default permissions
+        /** Create directory with default permissions
+         *
+         * @attention Only one MPI rank is allowed to call this method.
          *
          * @param dir name of directory
          */
         void createDirectory(std::string const dir) const;
-        /**
-         * Set 755 permissions for a directory
+        /** Set 755 permissions for a directory
+         *
+         * @attention Only one MPI rank is allowed to call this method.
          *
          * @param dir name of directory
          */
         void setDirectoryPermissions(std::string const dir) const;
 
-        /**
-         * Create directory and set 755 permissions by root rank.
+        /** Create directory and set 755 permissions by root rank.
+         *
+         * @attention Only one MPI rank is allowed to call this method.
          *
          * @param dir name of directory
          */
         void createDirectoryWithPermissions(std::string const dir) const;
 
-        /**
-         * Strip path from absolute or relative paths to filenames
+        /** Strip path from absolute or relative paths to filenames
          *
          * @param path and filename
          */
         std::string basename(std::string const pathFilename) const;
 
-        /**
-         * Returns the instance of the filesystem class.
+        /** Returns the instance of the filesystem class.
          *
          * This class is a singleton class.
          *

include/pmacc/simulationControl/SimulationHelper.cpp

@@ -111,7 +111,7 @@ namespace pmacc
             MPI_CHECK(MPI_Barrier(gc.getCommunicator().getMPIComm()));
 
             /* create directory containing checkpoints  */
-            if(numCheckpoints == 0)
+            if(numCheckpoints == 0 && gc.getGlobalRank() == 0)
             {
                 pmacc::Filesystem::get().createDirectoryWithPermissions(checkpointDirectory);
             }