Retry a few times if directory creation fails (#4537)

This tries to fix the following error reported by @BenWibking

    Writing checkpoint chk02500
    amrex::UtilCreateDirectory:: path errno:  chk02500 :: File exists
amrex::UtilCreateDirectory:: path errno: chk02500/Level_2 ::
Input/output error
    amrex::Error::0::Couldn't create directory: chk02500/Level_2 !!!

Author: WeiqunZhang

Src/Base/AMReX_Utility.cpp

@@ -116,7 +116,13 @@ bool
 amrex::UtilCreateDirectory (const std::string& path,
                             mode_t mode, bool verbose)
 {
-    return FileSystem::CreateDirectories(path, mode, verbose);
+    double sleep = 0.1;
+    while (sleep < 2.0) {
+        if (FileSystem::CreateDirectories(path, mode, verbose)) { return true; }
+        amrex::Sleep(sleep);
+        sleep *= 2;
+    }
+    return false;
 }
 
 void

Src/Base/AMReX_VisMF.cpp

@@ -1114,6 +1114,7 @@ VisMF::Write (const FabArray<FArrayBox>&    mf,
             nfi.Stream().write(allFabData, bytesWritten);
             nfi.Stream().flush();
             delete [] allFabData;
+            if (! nfi.Stream().good()) { amrex::Error("VisMF::Write failed"); }
 
         } else {    // ---- write fabs individually
             for(MFIter mfi(mf); mfi.isValid(); ++mfi) {
@@ -1154,6 +1155,8 @@ VisMF::Write (const FabArray<FArrayBox>&    mf,
             if (!noFlushAfterWrite) {
                 nfi.Stream().flush();
             }
+
+            if (! nfi.Stream().good()) { amrex::Error("VisMF::Write failed"); }
         }
     }
 
@@ -1485,6 +1488,8 @@ VisMF::readFAB (int                  idx,
 #endif
     }
 
+    if (!(infs->good())) { amrex::Error("VisMF::readFAB failed"); }
+
     VisMF::CloseStream(FullName);
 
     return fab;
@@ -1532,6 +1537,8 @@ VisMF::readFAB (FabArray<FArrayBox> &mf,
       fab.readFrom(*infs);
     }
 
+    if (!(infs->good())) { amrex::Error("VisMF::readFAB failed"); }
+
     VisMF::CloseStream(FullName);
 }
 
@@ -1831,6 +1838,8 @@ VisMF::Read (FabArray<FArrayBox> &mf,
                 }
               }
 
+              if (! nfi.Stream().good()) { amrex::Error("VisMF::Read failed"); }
+
           }    // ---- end NFilesIter
         }
 
@@ -2524,6 +2533,7 @@ VisMF::AsyncWriteDoit (const FabArray<FArrayBox>& mf, const std::string& mf_name
                 fabio->write(ofs, fab, 0, fab.nComp());
             }
             ofs.flush();
+            if (!ofs.good()) { amrex::Error("VisMF::AsyncWriteDoit failed"); }
             ofs.close();
         }