better retry (#3098)

Author: Guihao Liang

src/core/storage/fileio/s3_api.cpp

@@ -512,9 +512,9 @@ list_objects_response list_objects_impl(s3url parsed_url, std::string proxy,
         break;
 
       } else {
-        auto error = outcome.GetError().GetResponseCode();
+        auto error = outcome.GetError();
 
-        if (error == Aws::Http::HttpResponseCode::TOO_MANY_REQUESTS) {
+        if (error.ShouldRetry()) {
           n_retry++;
 
           if (n_retry == 3) {

src/core/storage/fileio/s3_api.hpp

@@ -286,11 +286,14 @@ std::ostream& reportS3Error(std::ostream& ss, const s3url& parsed_url,
                             S3Operation::ops_enum operation,
                             const Aws::Client::ClientConfiguration& config,
                             const Response& outcome) {
+  auto error = outcome.GetError();
   ss << "('" << parsed_url << ", proxy: '" << config.proxyHost << "', region: '"
      << config.region << "')"
      << " Error while performing " << S3Operation::toString(operation)
-     << ". Error Name: " << outcome.GetError().GetExceptionName()
-     << ". Error Message: " << outcome.GetError().GetMessage();
+     << ". Error Name: " << error.GetExceptionName()
+     << ". Error Message: " << error.GetMessage()
+     << ". HTTP Error Code: " << static_cast<int>(error.GetResponseCode());
+
   return ss;
 }
 

test/fileio/s3_filesys_test.cpp

@@ -91,15 +91,15 @@ int main(int argc, char** argv) {
     // force write all entries
     sa_ptr->materialize();
 
-    /* teardown manually */
-    turi::file_download_cache::get_instance().clear();
-    turi::block_cache::release_instance();
-
   } catch (std::string& e) {
     std::cerr << "Exception: " << e << std::endl;
   } catch (std::exception& e) {
     std::cerr << "Exception: " << e.what() << std::endl;
   }
 
+  /* teardown manually */
+  turi::file_download_cache::get_instance().clear();
+  turi::block_cache::release_instance();
+
   return 0;
 }