Skip to content

Commit 21a2d6c

Browse files
MytherinMytherin
committed
Format fix
1 parent c8cc7c3 commit 21a2d6c

17 files changed

+125
-101
lines changed

‎src/CMakeLists.txt‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
11
set(HTTPFS_SOURCES
2-
hffs.cpp
3-
s3fs.cpp
4-
httpfs.cpp
5-
http_state.cpp
6-
crypto.cpp
7-
hash_functions.cpp
8-
create_secret_functions.cpp
9-
httpfs_extension.cpp
10-
)
11-
if (NOT EMSCRIPTEN)
12-
set(HTTPFS_SOURCES ${HTTPFS_SOURCES} crypto.cpp httpfs_httplib_client.cpp httpfs_curl_client.cpp)
2+
hffs.cpp
3+
s3fs.cpp
4+
httpfs.cpp
5+
http_state.cpp
6+
crypto.cpp
7+
hash_functions.cpp
8+
create_secret_functions.cpp
9+
httpfs_extension.cpp)
10+
if(NOT EMSCRIPTEN)
11+
set(HTTPFS_SOURCES ${HTTPFS_SOURCES} crypto.cpp httpfs_httplib_client.cpp
12+
httpfs_curl_client.cpp)
1313
else()
1414
set(HTTPFS_SOURCES ${HTTPFS_SOURCES} httpfs_client_wasm.cpp)
1515
endif()

‎src/crypto.cpp‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@
2323

2424
namespace duckdb {
2525

26-
AESStateSSL::AESStateSSL(EncryptionTypes::CipherType cipher_p, idx_t key_len) : EncryptionState(cipher_p, key_len), context(EVP_CIPHER_CTX_new()) {
26+
AESStateSSL::AESStateSSL(EncryptionTypes::CipherType cipher_p, idx_t key_len)
27+
: EncryptionState(cipher_p, key_len), context(EVP_CIPHER_CTX_new()) {
2728
if (!(context)) {
2829
throw InternalException("OpenSSL AES failed with initializing context");
2930
}
@@ -83,7 +84,8 @@ void AESStateSSL::GenerateRandomData(data_ptr_t data, idx_t len) {
8384
RAND_bytes(data, len);
8485
}
8586

86-
void AESStateSSL::InitializeEncryption(const_data_ptr_t iv, idx_t iv_len, const_data_ptr_t key, idx_t key_len_p, const_data_ptr_t aad, idx_t aad_len) {
87+
void AESStateSSL::InitializeEncryption(const_data_ptr_t iv, idx_t iv_len, const_data_ptr_t key, idx_t key_len_p,
88+
const_data_ptr_t aad, idx_t aad_len) {
8789
mode = EncryptionTypes::ENCRYPT;
8890

8991
if (key_len_p != key_len) {
@@ -101,14 +103,15 @@ void AESStateSSL::InitializeEncryption(const_data_ptr_t iv, idx_t iv_len, const_
101103
}
102104

103105
int len;
104-
if (aad_len > 0){
106+
if (aad_len > 0) {
105107
if (!EVP_DecryptUpdate(context, NULL, &len, aad, aad_len)) {
106108
throw InternalException("Setting Additional Authenticated Data failed");
107109
}
108110
}
109111
}
110112

111-
void AESStateSSL::InitializeDecryption(const_data_ptr_t iv, idx_t iv_len, const_data_ptr_t key, idx_t key_len_p, const_data_ptr_t aad, idx_t aad_len) {
113+
void AESStateSSL::InitializeDecryption(const_data_ptr_t iv, idx_t iv_len, const_data_ptr_t key, idx_t key_len_p,
114+
const_data_ptr_t aad, idx_t aad_len) {
112115
mode = EncryptionTypes::DECRYPT;
113116
if (key_len_p != key_len) {
114117
throw InternalException("Invalid encryption key length, expected %llu, got %llu", key_len, key_len_p);
@@ -126,7 +129,7 @@ void AESStateSSL::InitializeDecryption(const_data_ptr_t iv, idx_t iv_len, const_
126129
throw InternalException("EVP_DecryptInit_ex failed to set iv/key");
127130
}
128131
int len;
129-
if (aad_len > 0){
132+
if (aad_len > 0) {
130133
if (!EVP_DecryptUpdate(context, NULL, &len, aad, aad_len)) {
131134
throw InternalException("Setting Additional Authenticated Data failed");
132135
}

‎src/httpfs.cpp‎

Lines changed: 34 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -44,13 +44,14 @@ unique_ptr<HTTPParams> HTTPFSUtil::InitializeParameters(optional_ptr<FileOpener>
4444
// Setting lookups
4545
FileOpener::TryGetCurrentSetting(opener, "http_timeout", result->timeout, info);
4646
FileOpener::TryGetCurrentSetting(opener, "force_download", result->force_download, info);
47-
FileOpener::TryGetCurrentSetting(opener, "auto_fallback_to_full_download", result->auto_fallback_to_full_download, info);
47+
FileOpener::TryGetCurrentSetting(opener, "auto_fallback_to_full_download", result->auto_fallback_to_full_download,
48+
info);
4849
FileOpener::TryGetCurrentSetting(opener, "http_retries", result->retries, info);
4950
FileOpener::TryGetCurrentSetting(opener, "http_retry_wait_ms", result->retry_wait_ms, info);
5051
FileOpener::TryGetCurrentSetting(opener, "http_retry_backoff", result->retry_backoff, info);
5152
FileOpener::TryGetCurrentSetting(opener, "http_keep_alive", result->keep_alive, info);
52-
FileOpener::TryGetCurrentSetting(opener, "enable_curl_server_cert_verification", result->enable_curl_server_cert_verification,
53-
info);
53+
FileOpener::TryGetCurrentSetting(opener, "enable_curl_server_cert_verification",
54+
result->enable_curl_server_cert_verification, info);
5455
FileOpener::TryGetCurrentSetting(opener, "enable_server_cert_verification", result->enable_server_cert_verification,
5556
info);
5657
FileOpener::TryGetCurrentSetting(opener, "ca_cert_file", result->ca_cert_file, info);
@@ -262,19 +263,23 @@ unique_ptr<HTTPResponse> HTTPFileSystem::GetRangeRequest(FileHandle &handle, str
262263
out_offset = 0;
263264

264265
if (!hfh.http_params.unsafe_disable_etag_checks && !hfh.etag.empty() && response.HasHeader("ETag")) {
265-
string responseEtag = response.GetHeaderValue("ETag");
266-
267-
if (!responseEtag.empty() && responseEtag != hfh.etag) {
268-
throw HTTPException(response, "ETag was initially %s and now it returned %s, this likely means the remote file has changed.\nTry to restart the read or close the file-handle and read the file again (e.g. `DETACH` in the file is a database file).\nYou can disable checking etags via `SET unsafe_disable_etag_checks = true;`", hfh.etag, response.GetHeaderValue("ETag"));
269-
}
270-
}
271-
272-
266+
string responseEtag = response.GetHeaderValue("ETag");
267+
268+
if (!responseEtag.empty() && responseEtag != hfh.etag) {
269+
throw HTTPException(
270+
response,
271+
"ETag was initially %s and now it returned %s, this likely means the remote file has "
272+
"changed.\nTry to restart the read or close the file-handle and read the file again (e.g. "
273+
"`DETACH` in the file is a database file).\nYou can disable checking etags via `SET "
274+
"unsafe_disable_etag_checks = true;`",
275+
hfh.etag, response.GetHeaderValue("ETag"));
276+
}
277+
}
273278

274279
if (response.HasHeader("Content-Length")) {
275280
auto content_length = stoll(response.GetHeaderValue("Content-Length"));
276281
if ((idx_t)content_length != buffer_out_len) {
277-
RangeRequestNotSupportedException::Throw();
282+
RangeRequestNotSupportedException::Throw();
278283
}
279284
}
280285
}
@@ -388,31 +393,33 @@ unique_ptr<FileHandle> HTTPFileSystem::OpenFileExtended(const OpenFileInfo &file
388393
return std::move(handle);
389394
}
390395

391-
bool HTTPFileSystem::TryRangeRequest(FileHandle &handle, string url, HTTPHeaders header_map, idx_t file_offset, char *buffer_out, idx_t buffer_out_len) {
396+
bool HTTPFileSystem::TryRangeRequest(FileHandle &handle, string url, HTTPHeaders header_map, idx_t file_offset,
397+
char *buffer_out, idx_t buffer_out_len) {
392398
auto res = GetRangeRequest(handle, url, header_map, file_offset, buffer_out, buffer_out_len);
393399

394400
if (res) {
395401
// Request succeeded TODO: fix upstream that 206 is not considered success
396-
if (res->Success() || res->status == HTTPStatusCode::PartialContent_206 || res->status == HTTPStatusCode::Accepted_202) {
402+
if (res->Success() || res->status == HTTPStatusCode::PartialContent_206 ||
403+
res->status == HTTPStatusCode::Accepted_202) {
397404
return true;
398405
}
399406

400407
// Request failed and we have a request error
401408
if (res->HasRequestError()) {
402-
ErrorData error (res->GetRequestError());
409+
ErrorData error(res->GetRequestError());
403410

404411
// Special case: we can do a retry with a full file download
405-
if (error.Type() == RangeRequestNotSupportedException::TYPE && error.RawMessage() == RangeRequestNotSupportedException::MESSAGE) {
412+
if (error.Type() == RangeRequestNotSupportedException::TYPE &&
413+
error.RawMessage() == RangeRequestNotSupportedException::MESSAGE) {
406414
auto &hfh = handle.Cast<HTTPFileHandle>();
407415
if (hfh.http_params.auto_fallback_to_full_download) {
408416
return false;
409417
}
410-
411418
}
412419
error.Throw();
413420
}
414-
throw HTTPException(*res, "Request returned HTTP %d for HTTP %s to '%s'",
415-
static_cast<int>(res->status), EnumUtil::ToString(RequestType::GET_REQUEST), res->url);
421+
throw HTTPException(*res, "Request returned HTTP %d for HTTP %s to '%s'", static_cast<int>(res->status),
422+
EnumUtil::ToString(RequestType::GET_REQUEST), res->url);
416423
}
417424
throw IOException("Unknown error for HTTP %s to '%s'", EnumUtil::ToString(RequestType::GET_REQUEST), url);
418425
}
@@ -485,15 +492,17 @@ bool HTTPFileSystem::ReadInternal(FileHandle &handle, void *buffer, int64_t nr_b
485492

486493
// Bypass buffer if we read more than buffer size
487494
if (to_read > new_buffer_available) {
488-
if (!TryRangeRequest(hfh, hfh.path, {}, location + buffer_offset, (char *)buffer + buffer_offset, to_read)) {
495+
if (!TryRangeRequest(hfh, hfh.path, {}, location + buffer_offset, (char *)buffer + buffer_offset,
496+
to_read)) {
489497
return false;
490498
}
491499
hfh.buffer_available = 0;
492500
hfh.buffer_idx = 0;
493501
start_offset += to_read;
494502
break;
495503
} else {
496-
if (!TryRangeRequest(hfh, hfh.path, {}, start_offset, (char *)hfh.read_buffer.get(), new_buffer_available)) {
504+
if (!TryRangeRequest(hfh, hfh.path, {}, start_offset, (char *)hfh.read_buffer.get(),
505+
new_buffer_available)) {
497506
return false;
498507
}
499508
hfh.buffer_available = new_buffer_available;
@@ -520,7 +529,10 @@ void HTTPFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes, id
520529
// attempt to download the full file and retry.
521530

522531
if (handle.logger) {
523-
DUCKDB_LOG_WARN(handle.logger, "Falling back to full file download for file '%s': the server does not support HTTP range requests. Performance and memory usage are potentially degraded.", handle.path);
532+
DUCKDB_LOG_WARN(handle.logger,
533+
"Falling back to full file download for file '%s': the server does not support HTTP range "
534+
"requests. Performance and memory usage are potentially degraded.",
535+
handle.path);
524536
}
525537

526538
auto &hfh = handle.Cast<HTTPFileHandle>();
@@ -744,7 +756,6 @@ void HTTPFileHandle::LoadFileInfo() {
744756
initialized = true;
745757
}
746758

747-
748759
void HTTPFileHandle::TryAddLogger(FileOpener &opener) {
749760
auto context = opener.TryGetClientContext();
750761
if (context) {

‎src/httpfs_config.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,7 @@
22

33
# list all include directories
44
include_directories = [
5-
os.path.sep.join(x.split('/'))
6-
for x in ['src/include', 'third_party/httplib', 'extension/parquet/include']
5+
os.path.sep.join(x.split('/')) for x in ['src/include', 'third_party/httplib', 'extension/parquet/include']
76
]
87
# source files
98
source_files = [

‎src/httpfs_curl_client.cpp‎

Lines changed: 34 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -17,17 +17,16 @@ namespace duckdb {
1717
// place curl will look. But not every distro has this file in the same location, so we search a
1818
// number of common locations and use the first one we find.
1919
static std::string certFileLocations[] = {
20-
// Arch, Debian-based, Gentoo
21-
"/etc/ssl/certs/ca-certificates.crt",
22-
// RedHat 7 based
23-
"/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem",
24-
// Redhat 6 based
25-
"/etc/pki/tls/certs/ca-bundle.crt",
26-
// OpenSUSE
27-
"/etc/ssl/ca-bundle.pem",
28-
// Alpine
29-
"/etc/ssl/cert.pem"};
30-
20+
// Arch, Debian-based, Gentoo
21+
"/etc/ssl/certs/ca-certificates.crt",
22+
// RedHat 7 based
23+
"/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem",
24+
// Redhat 6 based
25+
"/etc/pki/tls/certs/ca-bundle.crt",
26+
// OpenSUSE
27+
"/etc/ssl/ca-bundle.pem",
28+
// Alpine
29+
"/etc/ssl/cert.pem"};
3130

3231
//! Grab the first path that exists, from a list of well-known locations
3332
static std::string SelectCURLCertPath() {
@@ -44,15 +43,15 @@ static std::string cert_path = SelectCURLCertPath();
4443

4544
static size_t RequestWriteCallback(void *contents, size_t size, size_t nmemb, void *userp) {
4645
size_t totalSize = size * nmemb;
47-
std::string* str = static_cast<std::string*>(userp);
48-
str->append(static_cast<char*>(contents), totalSize);
46+
std::string *str = static_cast<std::string *>(userp);
47+
str->append(static_cast<char *>(contents), totalSize);
4948
return totalSize;
5049
}
5150

5251
static size_t RequestHeaderCallback(void *contents, size_t size, size_t nmemb, void *userp) {
5352
size_t totalSize = size * nmemb;
54-
std::string header(static_cast<char*>(contents), totalSize);
55-
HeaderCollector* header_collection = static_cast<HeaderCollector*>(userp);
53+
std::string header(static_cast<char *>(contents), totalSize);
54+
HeaderCollector *header_collection = static_cast<HeaderCollector *>(userp);
5655

5756
// Trim trailing \r\n
5857
if (!header.empty() && header.back() == '\n') {
@@ -86,7 +85,7 @@ static size_t RequestHeaderCallback(void *contents, size_t size, size_t nmemb, v
8685
return totalSize;
8786
}
8887

89-
CURLHandle::CURLHandle(const string &token, const string &cert_path) {
88+
CURLHandle::CURLHandle(const string &token, const string &cert_path) {
9089
curl = curl_easy_init();
9190
if (!curl) {
9291
throw InternalException("Failed to initialize curl");
@@ -104,15 +103,13 @@ CURLHandle::~CURLHandle() {
104103
curl_easy_cleanup(curl);
105104
}
106105

107-
108106
struct RequestInfo {
109107
string url = "";
110108
string body = "";
111109
uint16_t response_code = 0;
112110
std::vector<HTTPHeaders> header_collection;
113111
};
114112

115-
116113
static idx_t httpfs_client_count = 0;
117114

118115
class HTTPFSCurlClient : public HTTPClient {
@@ -140,11 +137,12 @@ class HTTPFSCurlClient : public HTTPClient {
140137
}
141138

142139
if (http_params.enable_curl_server_cert_verification) {
143-
curl_easy_setopt(*curl, CURLOPT_SSL_VERIFYPEER, 1L); // Verify the cert
144-
curl_easy_setopt(*curl, CURLOPT_SSL_VERIFYHOST, 2L); // Verify that the cert matches the hostname
140+
curl_easy_setopt(*curl, CURLOPT_SSL_VERIFYPEER, 1L); // Verify the cert
141+
curl_easy_setopt(*curl, CURLOPT_SSL_VERIFYHOST, 2L); // Verify that the cert matches the hostname
145142
} else {
146-
curl_easy_setopt(*curl, CURLOPT_SSL_VERIFYPEER, 0L); // Override default, don't verify the cert
147-
curl_easy_setopt(*curl, CURLOPT_SSL_VERIFYHOST, 0L); // Override default, don't verify that the cert matches the hostname
143+
curl_easy_setopt(*curl, CURLOPT_SSL_VERIFYPEER, 0L); // Override default, don't verify the cert
144+
curl_easy_setopt(*curl, CURLOPT_SSL_VERIFYHOST,
145+
0L); // Override default, don't verify that the cert matches the hostname
148146
}
149147

150148
// set read timeout
@@ -164,7 +162,8 @@ class HTTPFSCurlClient : public HTTPClient {
164162
curl_easy_setopt(*curl, CURLOPT_WRITEDATA, &request_info->body);
165163

166164
if (!http_params.http_proxy.empty()) {
167-
curl_easy_setopt(*curl, CURLOPT_PROXY, StringUtil::Format("%s:%s", http_params.http_proxy, http_params.http_proxy_port).c_str());
165+
curl_easy_setopt(*curl, CURLOPT_PROXY,
166+
StringUtil::Format("%s:%s", http_params.http_proxy, http_params.http_proxy_port).c_str());
168167

169168
if (!http_params.http_proxy_username.empty()) {
170169
curl_easy_setopt(*curl, CURLOPT_PROXYUSERNAME, http_params.http_proxy_username.c_str());
@@ -201,7 +200,8 @@ class HTTPFSCurlClient : public HTTPClient {
201200
curl_easy_getinfo(*curl, CURLINFO_RESPONSE_CODE, &request_info->response_code);
202201

203202
idx_t bytes_received = 0;
204-
if (!request_info->header_collection.empty() && request_info->header_collection.back().HasHeader("content-length")) {
203+
if (!request_info->header_collection.empty() &&
204+
request_info->header_collection.back().HasHeader("content-length")) {
205205
bytes_received = std::stoi(request_info->header_collection.back().GetHeaderValue("content-length"));
206206
D_ASSERT(bytes_received == request_info->body.size());
207207
} else {
@@ -211,9 +211,9 @@ class HTTPFSCurlClient : public HTTPClient {
211211
state->total_bytes_received += bytes_received;
212212
}
213213

214-
const char* data = request_info->body.c_str();
214+
const char *data = request_info->body.c_str();
215215
if (info.content_handler) {
216-
info.content_handler(const_data_ptr_cast(data), bytes_received);
216+
info.content_handler(const_data_ptr_cast(data), bytes_received);
217217
}
218218

219219
return TransformResponseCurl(res);
@@ -321,7 +321,7 @@ class HTTPFSCurlClient : public HTTPClient {
321321

322322
// Get HTTP response status code
323323
curl_easy_getinfo(*curl, CURLINFO_RESPONSE_CODE, &request_info->response_code);
324-
return TransformResponseCurl( res);
324+
return TransformResponseCurl(res);
325325
}
326326

327327
unique_ptr<HTTPResponse> Post(PostRequestInfo &info) override {
@@ -359,7 +359,7 @@ class HTTPFSCurlClient : public HTTPClient {
359359
curl_easy_getinfo(*curl, CURLINFO_RESPONSE_CODE, &request_info->response_code);
360360
info.buffer_out = request_info->body;
361361
// Construct HTTPResponse
362-
return TransformResponseCurl( res);
362+
return TransformResponseCurl(res);
363363
}
364364

365365
private:
@@ -406,15 +406,16 @@ class HTTPFSCurlClient : public HTTPClient {
406406
auto response = make_uniq<HTTPResponse>(status_code);
407407
if (res != CURLcode::CURLE_OK) {
408408
// TODO: request error can come from HTTPS Status code toString() value.
409-
if (!request_info->header_collection.empty() && request_info->header_collection.back().HasHeader("__RESPONSE_STATUS__")) {
409+
if (!request_info->header_collection.empty() &&
410+
request_info->header_collection.back().HasHeader("__RESPONSE_STATUS__")) {
410411
response->request_error = request_info->header_collection.back().GetHeaderValue("__RESPONSE_STATUS__");
411412
} else {
412413
response->request_error = curl_easy_strerror(res);
413414
}
414415
return response;
415416
}
416417
response->body = request_info->body;
417-
response->url= request_info->url;
418+
response->url = request_info->url;
418419
if (!request_info->header_collection.empty()) {
419420
for (auto &header : request_info->header_collection.back()) {
420421
response->headers.Insert(header.first, header.second);
@@ -465,7 +466,8 @@ unordered_map<string, string> HTTPFSCurlUtil::ParseGetParameters(const string &t
465466
unordered_map<std::string, std::string> params;
466467

467468
auto pos = text.find('?');
468-
if (pos == std::string::npos) return params;
469+
if (pos == std::string::npos)
470+
return params;
469471

470472
std::string query = text.substr(pos + 1);
471473
std::stringstream ss(query);
@@ -478,7 +480,7 @@ unordered_map<string, string> HTTPFSCurlUtil::ParseGetParameters(const string &t
478480
std::string value = StringUtil::URLDecode(item.substr(eq_pos + 1));
479481
params[key] = value;
480482
} else {
481-
params[item] = ""; // key with no value
483+
params[item] = ""; // key with no value
482484
}
483485
}
484486

0 commit comments

Comments
 (0)