Merge pull request OSGeo#12802 from rouault/s3_listdir_path_specific_option

/vsis3/: retrieve path specific options in ReadDir()

Author: rouault

autotest/gcore/vsis3.py

@@ -1244,9 +1244,16 @@ def method(request):
                 </Contents>
             </ListBucketResult>
         """,
+        expected_headers={"foo": "bar"},
+    )
+    gdal.SetPathSpecificOption(
+        "/vsis3/s3_fake_bucket2/a_dir",
+        "GDAL_HTTP_HEADERS",
+        "foo:bar",
     )
     with webserver.install_http_handler(handler):
         dir_contents = gdal.ReadDir("/vsis3/s3_fake_bucket2/a_dir")
+    gdal.ClearPathSpecificOptions()
     assert dir_contents == ["test.txt"]
 
     # Test CPL_VSIL_CURL_IGNORE_GLACIER_STORAGE=NO

port/cpl_vsil_az.cpp

@@ -61,7 +61,8 @@ const char GDAL_MARKER_FOR_DIR[] = ".gdal_marker_for_dir";
 
 struct VSIDIRAz : public VSIDIRS3Like
 {
-    explicit VSIDIRAz(IVSIS3LikeFSHandler *poFSIn) : VSIDIRS3Like(poFSIn)
+    VSIDIRAz(const std::string &osDirName, IVSIS3LikeFSHandler *poFSIn)
+        : VSIDIRS3Like(osDirName, poFSIn)
     {
     }
 
@@ -2479,7 +2480,7 @@ VSIDIR *VSIAzureFSHandler::OpenDir(const char *pszPath, int nRecurseDepth,
         return nullptr;
     }
 
-    VSIDIRAz *dir = new VSIDIRAz(this);
+    VSIDIRAz *dir = new VSIDIRAz(pszPath, this);
     dir->nRecurseDepth = nRecurseDepth;
     dir->poHandleHelper = std::move(poHandleHelper);
     dir->osBucket = std::move(osBucket);

port/cpl_vsil_curl_class.h

@@ -1026,6 +1026,8 @@ struct VSIDIRWithMissingDirSynthesis : public VSIDIR
 
 struct VSIDIRS3Like : public VSIDIRWithMissingDirSynthesis
 {
+    const std::string m_osDirName;
+
     int nRecurseDepth = 0;
 
     std::string osNextMarker{};
@@ -1045,12 +1047,14 @@ struct VSIDIRS3Like : public VSIDIRWithMissingDirSynthesis
     std::unique_ptr<VSIDIR, decltype(&VSICloseDir)> m_subdir{nullptr,
                                                              VSICloseDir};
 
-    explicit VSIDIRS3Like(IVSIS3LikeFSHandler *poFSIn)
-        : poFS(poFSIn), poS3FS(poFSIn)
+    VSIDIRS3Like(const std::string &osDirName, IVSIS3LikeFSHandler *poFSIn)
+        : m_osDirName(osDirName), poFS(poFSIn), poS3FS(poFSIn)
     {
     }
 
-    explicit VSIDIRS3Like(VSICurlFilesystemHandlerBase *poFSIn) : poFS(poFSIn)
+    VSIDIRS3Like(const std::string &osDirName,
+                 VSICurlFilesystemHandlerBase *poFSIn)
+        : m_osDirName(osDirName), poFS(poFSIn)
     {
     }
 

port/cpl_vsil_s3.cpp

@@ -73,12 +73,13 @@ struct VSIDIRS3 : public VSIDIRS3Like
     bool m_bDirectoryBucketListingDone = false;
     bool m_bListBucket = false;
 
-    explicit VSIDIRS3(IVSIS3LikeFSHandler *poFSIn) : VSIDIRS3Like(poFSIn)
+    VSIDIRS3(const std::string &osDirName, IVSIS3LikeFSHandler *poFSIn)
+        : VSIDIRS3Like(osDirName, poFSIn)
     {
     }
 
-    explicit VSIDIRS3(VSICurlFilesystemHandlerBase *poFSIn)
-        : VSIDIRS3Like(poFSIn)
+    VSIDIRS3(const std::string &osDirName, VSICurlFilesystemHandlerBase *poFSIn)
+        : VSIDIRS3Like(osDirName, poFSIn)
     {
     }
 
@@ -515,6 +516,9 @@ bool VSIDIRS3::IssueListDir()
         l_poHandlerHelper = poTmpHandleHelper.get();
     }
 
+    const CPLStringList aosHTTPOptions(
+        CPLHTTPGetOptionsFromEnv(m_osDirName.c_str()));
+
     while (true)
     {
         l_poHandlerHelper->ResetQueryParameters();
@@ -541,8 +545,9 @@ bool VSIDIRS3::IssueListDir()
                                                      m_osFilterPrefix);
         }
 
-        struct curl_slist *headers = VSICurlSetOptions(
-            hCurlHandle, l_poHandlerHelper->GetURL().c_str(), nullptr);
+        struct curl_slist *headers =
+            VSICurlSetOptions(hCurlHandle, poHandleHelper->GetURL().c_str(),
+                              aosHTTPOptions.List());
 
         headers = l_poHandlerHelper->GetCurlHeaders("GET", headers);
         // Disable automatic redirection
@@ -703,7 +708,7 @@ bool VSICurlFilesystemHandlerBase::AnalyseS3FileList(
     int nMaxFiles, const std::set<std::string> &oSetIgnoredStorageClasses,
     bool &bIsTruncated)
 {
-    VSIDIRS3 oDir(this);
+    VSIDIRS3 oDir(std::string(), this);
     oDir.nMaxFiles = nMaxFiles;
     bool ret = oDir.AnalyseS3FileList(osBaseURL, pszXML,
                                       oSetIgnoredStorageClasses, bIsTruncated);
@@ -3403,7 +3408,7 @@ VSIDIR *IVSIS3LikeFSHandler::OpenDir(const char *pszPath, int nRecurseDepth,
         return nullptr;
     }
 
-    VSIDIRS3 *dir = new VSIDIRS3(this);
+    VSIDIRS3 *dir = new VSIDIRS3(pszPath, this);
     dir->nRecurseDepth = nRecurseDepth;
     dir->poHandleHelper = std::move(poS3HandleHelper);
     dir->osBucket = std::move(osBucket);