Add back handling for use_delimiter, making globs like s3://bucket/*-something/somethingelse/*.csv faster

Author: carlopi

src/include/s3fs.hpp

@@ -290,7 +290,7 @@ class S3FileSystem : public HTTPFileSystem {
 // Helper class to do s3 ListObjectV2 api call https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
 struct AWSListObjectV2 {
 	static string Request(const string &path, HTTPParams &http_params, const S3AuthParams &s3_auth_params,
-	                      string &continuation_token);
+	                      string &continuation_token, bool use_delimiter = false);
 	static void ParseFileList(string &aws_response, vector<OpenFileInfo> &result);
 	static vector<string> ParseCommonPrefix(string &aws_response);
 	static string ParseContinuationToken(string &aws_response);

src/s3fs.cpp

@@ -1114,12 +1114,19 @@ bool S3GlobResult::ExpandNextPath() const {
 		// we have common prefixes left to scan - perform the request
 		auto prefix_path = parsed_s3_url.prefix + parsed_s3_url.bucket + '/' + current_common_prefix;
 
-		auto prefix_res =
-		    AWSListObjectV2::Request(prefix_path, *http_params, s3_auth_params, common_prefix_continuation_token);
-		AWSListObjectV2::ParseFileList(prefix_res, s3_keys);
-		auto more_prefixes = AWSListObjectV2::ParseCommonPrefix(prefix_res);
-		common_prefixes.insert(common_prefixes.end(), more_prefixes.begin(), more_prefixes.end());
-		common_prefix_continuation_token = AWSListObjectV2::ParseContinuationToken(prefix_res);
+		vector<string> pattern_splits = StringUtil::Split(parsed_s3_url.key, "/");
+		vector<string> key_splits = StringUtil::Split(current_common_prefix, "/");
+		pattern_splits.resize(key_splits.size());
+		bool is_match = Match(key_splits.begin(), key_splits.end(), pattern_splits.begin(), pattern_splits.end());
+		if (is_match) {
+			auto prefix_res = AWSListObjectV2::Request(prefix_path, *http_params, s3_auth_params,
+			                                           common_prefix_continuation_token, true);
+
+			AWSListObjectV2::ParseFileList(prefix_res, s3_keys);
+			auto more_prefixes = AWSListObjectV2::ParseCommonPrefix(prefix_res);
+			common_prefixes.insert(common_prefixes.end(), more_prefixes.begin(), more_prefixes.end());
+			common_prefix_continuation_token = AWSListObjectV2::ParseContinuationToken(prefix_res);
+		}
 		if (common_prefix_continuation_token.empty()) {
 			// we are done with the current common prefix
 			// either move on to the next one, or finish up
@@ -1138,7 +1145,7 @@ bool S3GlobResult::ExpandNextPath() const {
 		}
 		// issue the main request
 		string response_str =
-		    AWSListObjectV2::Request(shared_path, *http_params, s3_auth_params, main_continuation_token);
+		    AWSListObjectV2::Request(shared_path, *http_params, s3_auth_params, main_continuation_token, true);
 		main_continuation_token = AWSListObjectV2::ParseContinuationToken(response_str);
 		AWSListObjectV2::ParseFileList(response_str, s3_keys);
 
@@ -1270,7 +1277,7 @@ HTTPException S3FileSystem::GetHTTPError(FileHandle &handle, const HTTPResponse
 	return GetS3Error(s3_handle.auth_params, response, url);
 }
 string AWSListObjectV2::Request(const string &path, HTTPParams &http_params, const S3AuthParams &s3_auth_params,
-                                string &continuation_token) {
+                                string &continuation_token, bool use_delimiter) {
 	auto parsed_url = S3FileSystem::S3UrlParse(path, s3_auth_params);
 
 	// Construct the ListObjectsV2 call
@@ -1284,6 +1291,10 @@ string AWSListObjectV2::Request(const string &path, HTTPParams &http_params, con
 	req_params += "encoding-type=url&list-type=2";
 	req_params += "&prefix=" + S3FileSystem::UrlEncode(parsed_url.key, true);
 
+	if (use_delimiter) {
+		req_params += "&delimiter=%2F";
+	}
+
 	auto header_map =
 	    CreateS3Header(req_path, req_params, parsed_url.host, "s3", "GET", s3_auth_params, "", "", "", "");