Skip to content

Commit abcde70

Browse files
lmanganilmangani
committed
refactor, deduplicate
1 parent dd65a86 commit abcde70

File tree

1 file changed

+98
-146
lines changed

1 file changed

+98
-146
lines changed

‎src/http_client_extension.cpp‎

Lines changed: 98 additions & 146 deletions
Original file line numberDiff line numberDiff line change
@@ -15,141 +15,133 @@
1515

1616
namespace duckdb {
1717

18+
// Helper function to parse URL and setup client
19+
static std::pair<duckdb_httplib_openssl::Client, std::string> SetupHttpClient(const std::string &url) {
20+
std::string scheme, domain, path;
21+
size_t pos = url.find("://");
22+
std::string mod_url = url;
23+
if (pos != std::string::npos) {
24+
scheme = mod_url.substr(0, pos);
25+
mod_url.erase(0, pos + 3);
26+
}
27+
28+
pos = mod_url.find("/");
29+
if (pos != std::string::npos) {
30+
domain = mod_url.substr(0, pos);
31+
path = mod_url.substr(pos);
32+
} else {
33+
domain = mod_url;
34+
path = "/";
35+
}
36+
37+
// Create client and set a reasonable timeout (e.g., 10 seconds)
38+
duckdb_httplib_openssl::Client client(domain.c_str());
39+
client.set_read_timeout(10, 0); // 10 seconds
40+
client.set_follow_location(true); // Follow redirects
41+
42+
return std::make_pair(std::move(client), path);
43+
}
44+
45+
static void HandleHttpError(const duckdb_httplib_openssl::Result &res, const std::string &request_type) {
46+
std::string err_message = "HTTP " + request_type + " request failed. ";
47+
48+
switch (res.error()) {
49+
case duckdb_httplib_openssl::Error::Connection:
50+
err_message += "Connection error.";
51+
break;
52+
case duckdb_httplib_openssl::Error::BindIPAddress:
53+
err_message += "Failed to bind IP address.";
54+
break;
55+
case duckdb_httplib_openssl::Error::Read:
56+
err_message += "Error reading response.";
57+
break;
58+
case duckdb_httplib_openssl::Error::Write:
59+
err_message += "Error writing request.";
60+
break;
61+
case duckdb_httplib_openssl::Error::ExceedRedirectCount:
62+
err_message += "Too many redirects.";
63+
break;
64+
case duckdb_httplib_openssl::Error::Canceled:
65+
err_message += "Request was canceled.";
66+
break;
67+
case duckdb_httplib_openssl::Error::SSLConnection:
68+
err_message += "SSL connection failed.";
69+
break;
70+
case duckdb_httplib_openssl::Error::SSLLoadingCerts:
71+
err_message += "Failed to load SSL certificates.";
72+
break;
73+
case duckdb_httplib_openssl::Error::SSLServerVerification:
74+
err_message += "SSL server verification failed.";
75+
break;
76+
case duckdb_httplib_openssl::Error::UnsupportedMultipartBoundaryChars:
77+
err_message += "Unsupported characters in multipart boundary.";
78+
break;
79+
case duckdb_httplib_openssl::Error::Compression:
80+
err_message += "Error during compression.";
81+
break;
82+
default:
83+
err_message += "Unknown error.";
84+
break;
85+
}
86+
throw std::runtime_error(err_message);
87+
}
88+
89+
1890
static void HTTPGetRequestFunction(DataChunk &args, ExpressionState &state, Vector &result) {
1991
D_ASSERT(args.data.size() == 1);
2092

2193
UnaryExecutor::Execute<string_t, string_t>(args.data[0], result, args.size(), [&](string_t input) {
2294
std::string url = input.GetString();
2395

24-
// Parse the URL to extract the domain and path
25-
std::string scheme, domain, path;
26-
size_t pos = url.find("://");
27-
if (pos != std::string::npos) {
28-
scheme = url.substr(0, pos);
29-
url.erase(0, pos + 3);
30-
}
31-
32-
pos = url.find("/");
33-
if (pos != std::string::npos) {
34-
domain = url.substr(0, pos);
35-
path = url.substr(pos);
36-
} else {
37-
domain = url;
38-
path = "/";
39-
}
40-
41-
// Create client and set a reasonable timeout (e.g., 10 seconds)
42-
duckdb_httplib_openssl::Client client(domain.c_str());
43-
client.set_read_timeout(10, 0); // 10 seconds
44-
45-
// Follow redirects
46-
client.set_follow_location(true);
96+
// Use helper to setup client and parse URL
97+
auto client_and_path = SetupHttpClient(url);
98+
auto &client = client_and_path.first;
99+
auto &path = client_and_path.second;
47100

48101
// Make the GET request
49102
auto res = client.Get(path.c_str());
50103
if (res) {
51104
if (res->status == 200) {
52105
return StringVector::AddString(result, res->body);
53106
} else {
54-
throw std::runtime_error("HTTP error: " + std::to_string(res->status) + " - " + res->reason);
107+
throw std::runtime_error("HTTP GET error: " + std::to_string(res->status) + " - " + res->reason);
55108
}
56109
} else {
57-
// Handle the error case
58-
std::string err_message = "HTTP request failed. ";
59-
60-
// Convert httplib error codes to a descriptive message
61-
switch (res.error()) {
62-
case duckdb_httplib_openssl::Error::Connection:
63-
err_message += "Connection error.";
64-
break;
65-
case duckdb_httplib_openssl::Error::BindIPAddress:
66-
err_message += "Failed to bind IP address.";
67-
break;
68-
case duckdb_httplib_openssl::Error::Read:
69-
err_message += "Error reading response.";
70-
break;
71-
case duckdb_httplib_openssl::Error::Write:
72-
err_message += "Error writing request.";
73-
break;
74-
case duckdb_httplib_openssl::Error::ExceedRedirectCount:
75-
err_message += "Too many redirects.";
76-
break;
77-
case duckdb_httplib_openssl::Error::Canceled:
78-
err_message += "Request was canceled.";
79-
break;
80-
case duckdb_httplib_openssl::Error::SSLConnection:
81-
err_message += "SSL connection failed.";
82-
break;
83-
case duckdb_httplib_openssl::Error::SSLLoadingCerts:
84-
err_message += "Failed to load SSL certificates.";
85-
break;
86-
case duckdb_httplib_openssl::Error::SSLServerVerification:
87-
err_message += "SSL server verification failed.";
88-
break;
89-
case duckdb_httplib_openssl::Error::UnsupportedMultipartBoundaryChars:
90-
err_message += "Unsupported characters in multipart boundary.";
91-
break;
92-
case duckdb_httplib_openssl::Error::Compression:
93-
err_message += "Error during compression.";
94-
break;
95-
default:
96-
err_message += "Unknown error.";
97-
break;
98-
}
99-
throw std::runtime_error(err_message);
110+
// Handle errors
111+
HandleHttpError(res, "GET");
100112
}
113+
// Ensure a return value in case of an error
114+
return string_t();
101115
});
102116
}
103117

104118
static void HTTPPostRequestFunction(DataChunk &args, ExpressionState &state, Vector &result) {
105119
D_ASSERT(args.data.size() == 3);
106120

107121
auto &url_vector = args.data[0];
108-
auto &headers_vector = args.data[1]; // Already passed as a serialized string
109-
auto &body_vector = args.data[2]; // Already passed as a JSON string
122+
auto &headers_vector = args.data[1];
123+
auto &body_vector = args.data[2];
110124

111-
// Use TernaryExecutor instead of UnaryExecutor
112125
TernaryExecutor::Execute<string_t, string_t, string_t, string_t>(
113126
url_vector, headers_vector, body_vector, result, args.size(),
114-
[&](string_t url, string_t headers_varchar, string_t body_varchar) {
127+
[&](string_t url, string_t headers, string_t body) {
115128
std::string url_str = url.GetString();
116129

117-
// Parse the URL to extract the domain and path
118-
std::string scheme, domain, path;
119-
size_t pos = url_str.find("://");
120-
if (pos != std::string::npos) {
121-
scheme = url_str.substr(0, pos);
122-
url_str.erase(0, pos + 3);
123-
}
124-
125-
pos = url_str.find("/");
126-
if (pos != std::string::npos) {
127-
domain = url_str.substr(0, pos);
128-
path = url_str.substr(pos);
129-
} else {
130-
domain = url_str;
131-
path = "/";
132-
}
133-
134-
// Create the client and set a timeout
135-
duckdb_httplib_openssl::Client client(domain.c_str());
136-
client.set_read_timeout(10, 0); // 10 seconds
137-
// Follow redirects
138-
client.set_follow_location(true);
139-
140-
// Follow redirects for POST as well
141-
client.set_follow_location(true);
130+
// Use helper to setup client and parse URL
131+
auto client_and_path = SetupHttpClient(url_str);
132+
auto &client = client_and_path.first;
133+
auto &path = client_and_path.second;
142134

143-
// Deserialize the header string into a header map
135+
// Prepare headers
144136
duckdb_httplib_openssl::Headers header_map;
145-
std::istringstream header_stream(headers_varchar.GetString());
137+
std::istringstream header_stream(headers.GetString());
146138
std::string header;
147139
while (std::getline(header_stream, header)) {
148140
size_t colon_pos = header.find(':');
149141
if (colon_pos != std::string::npos) {
150142
std::string key = header.substr(0, colon_pos);
151143
std::string value = header.substr(colon_pos + 1);
152-
// Trim leading/trailing whitespace
144+
// Trim leading and trailing whitespace
153145
key.erase(0, key.find_first_not_of(" \t"));
154146
key.erase(key.find_last_not_of(" \t") + 1);
155147
value.erase(0, value.find_first_not_of(" \t"));
@@ -158,60 +150,20 @@ static void HTTPPostRequestFunction(DataChunk &args, ExpressionState &state, Vec
158150
}
159151
}
160152

161-
// Prepare the POST body (it is passed as a string)
162-
std::string body_str = body_varchar.GetString();
163-
164-
// Make the POST request
165-
auto res = client.Post(path.c_str(), header_map, body_str, "application/json");
153+
// Make the POST request with headers and body
154+
auto res = client.Post(path.c_str(), header_map, body.GetString(), "application/json");
166155
if (res) {
167156
if (res->status == 200) {
168157
return StringVector::AddString(result, res->body);
169158
} else {
170-
throw std::runtime_error("HTTP error: " + std::to_string(res->status) + " - " + res->reason);
159+
throw std::runtime_error("HTTP POST error: " + std::to_string(res->status) + " - " + res->reason);
171160
}
172161
} else {
173-
// Handle the error case
174-
std::string err_message = "HTTP POST request failed. ";
175-
switch (res.error()) {
176-
case duckdb_httplib_openssl::Error::Connection:
177-
err_message += "Connection error.";
178-
break;
179-
case duckdb_httplib_openssl::Error::BindIPAddress:
180-
err_message += "Failed to bind IP address.";
181-
break;
182-
case duckdb_httplib_openssl::Error::Read:
183-
err_message += "Error reading response.";
184-
break;
185-
case duckdb_httplib_openssl::Error::Write:
186-
err_message += "Error writing request.";
187-
break;
188-
case duckdb_httplib_openssl::Error::ExceedRedirectCount:
189-
err_message += "Too many redirects.";
190-
break;
191-
case duckdb_httplib_openssl::Error::Canceled:
192-
err_message += "Request was canceled.";
193-
break;
194-
case duckdb_httplib_openssl::Error::SSLConnection:
195-
err_message += "SSL connection failed.";
196-
break;
197-
case duckdb_httplib_openssl::Error::SSLLoadingCerts:
198-
err_message += "Failed to load SSL certificates.";
199-
break;
200-
case duckdb_httplib_openssl::Error::SSLServerVerification:
201-
err_message += "SSL server verification failed.";
202-
break;
203-
case duckdb_httplib_openssl::Error::UnsupportedMultipartBoundaryChars:
204-
err_message += "Unsupported characters in multipart boundary.";
205-
break;
206-
case duckdb_httplib_openssl::Error::Compression:
207-
err_message += "Error during compression.";
208-
break;
209-
default:
210-
err_message += "Unknown error.";
211-
break;
212-
}
213-
throw std::runtime_error(err_message);
162+
// Handle errors
163+
HandleHttpError(res, "POST");
214164
}
165+
// Ensure a return value in case of an error
166+
return string_t();
215167
});
216168
}
217169

0 commit comments

Comments
 (0)