Skip to content

Commit b52e502

Browse files
puzza007puzza007
committed
feat: add dns_cache_timeout, max_concurrent_streams, and modernize libevent
- Add dns_cache_timeout option (0=disable, -1=forever, positive=seconds) - Add max_concurrent_streams pool option for HTTP/2 stream limits - Modernize libevent to 2.x API (event_base_new, bufferevent_socket_new) - Add proper cleanup on exit (bufferevent_free, curl_*_cleanup, event_base_free) - Fix curl type warnings (1L for CURLOPT_POST/NOBODY, long response_code) - Fix multi_timer_cb signature for curl type checking - Add COVERAGE=1 and SANITIZE=1 build flags to Makefile
1 parent 79a04fd commit b52e502

File tree

6 files changed

+103
-16
lines changed

6 files changed

+103
-16
lines changed

‎.gitignore‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,3 +24,9 @@ local/
2424
/rebar3.crashdump
2525
doc
2626
/.claude/
27+
28+
# C coverage artifacts
29+
*.gcda
30+
*.gcno
31+
*.gcov
32+
coverage.info

‎README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ katipo:Method(Pool :: atom(), URL :: binary(), ReqOptions :: map()).
8686
| `capath` | `binary()` | `undefined` | [docs](https://curl.haxx.se/libcurl/c/CURLOPT_CAPATH.html) |
8787
| `cacert` | `binary()` | `undefined` | [docs](https://curl.haxx.se/libcurl/c/CURLOPT_CAINFO.html) |
8888
| `timeout_ms` | `pos_integer()` | 30000 | [docs](https://curl.haxx.se/libcurl/c/CURLOPT_TIMEOUT_MS.html) |
89+
| `dns_cache_timeout` | `integer()` | 60 | [docs](https://curl.haxx.se/libcurl/c/CURLOPT_DNS_CACHE_TIMEOUT.html) (0=disable, -1=forever) |
8990
| `maxredirs` | `non_neg_integer()` | 9 | [docs](https://curl.haxx.se/libcurl/c/CURLOPT_MAXREDIRS.html) |
9091
| `proxy` | `binary()` | `undefined` | [docs](https://curl.haxx.se/libcurl/c/CURLOPT_PROXY.html) |
9192
| `tcp_fastopen` | `boolean()` | `false` | [docs](https://curl.haxx.se/libcurl/c/CURLOPT_TCP_FASTOPEN.html) curl >= 7.49.0 |
@@ -123,6 +124,7 @@ katipo:Method(Pool :: atom(), URL :: binary(), ReqOptions :: map()).
123124
| `pipelining` | `nothing` <br> `http1` <br> `multiplex` | `nothing` | HTTP pipelining [CURLMOPT_PIPELINING](https://curl.haxx.se/libcurl/c/CURLMOPT_PIPELINING.html) |
124125
| `max_pipeline_length` | `non_neg_integer()` | 100 | |
125126
| `max_total_connections` | `non_neg_integer()` | 0 (no limit) | [docs](https://curl.haxx.se/libcurl/c/CURLMOPT_MAX_TOTAL_CONNECTIONS.html) |
127+
| `max_concurrent_streams`| `non_neg_integer()` | 100 | [docs](https://curl.haxx.se/libcurl/c/CURLMOPT_MAX_CONCURRENT_STREAMS.html) curl >= 7.67.0 |
126128

127129
#### Observability
128130

‎c_src/Makefile‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,12 @@ endif
4949
CFLAGS += -fPIC -I $(ERTS_INCLUDE_DIR) -I $(ERL_INTERFACE_INCLUDE_DIR)
5050
CXXFLAGS += -fPIC -I $(ERTS_INCLUDE_DIR) -I $(ERL_INTERFACE_INCLUDE_DIR)
5151

52+
# Coverage support: build with COVERAGE=1 to enable
53+
ifdef COVERAGE
54+
CFLAGS += --coverage -O0 -g
55+
LDFLAGS += --coverage
56+
endif
57+
5258
# Add curl include path from curl-config
5359
CFLAGS += $(shell $(CURL_CONFIG) --cflags)
5460

@@ -144,3 +150,19 @@ $(C_SRC_OUTPUT): $(OBJECTS)
144150

145151
clean:
146152
@rm -f $(C_SRC_OUTPUT) $(OBJECTS)
153+
154+
# Coverage targets
155+
.PHONY: coverage-html coverage-clean
156+
157+
coverage-html:
158+
@if ls *.gcda 1>/dev/null 2>&1; then \
159+
lcov --capture --directory . --output-file coverage.info; \
160+
genhtml coverage.info --output-directory $(BASEDIR)/_build/cover/c_src; \
161+
rm -f coverage.info; \
162+
echo "C coverage report: $(BASEDIR)/_build/cover/c_src/index.html"; \
163+
else \
164+
echo "No coverage data found. Run tests first with COVERAGE=1 build."; \
165+
fi
166+
167+
coverage-clean:
168+
@rm -f *.gcda *.gcno *.gcov coverage.info

‎c_src/katipo.c‎

Lines changed: 37 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
#include <stdlib.h>
2-
#include <event.h>
2+
#include <event2/event.h>
3+
#include <event2/bufferevent.h>
4+
#include <event2/buffer.h>
35
#include <sys/types.h>
46
#include <sys/uio.h>
57
#include <unistd.h>
@@ -47,6 +49,7 @@
4749
#define K_CURLOPT_KEYPASSWD 27
4850
#define K_CURLOPT_USERPWD 28
4951
#define K_CURLOPT_SSLVERSION 29
52+
#define K_CURLOPT_DNS_CACHE_TIMEOUT 31
5053

5154
#define K_CURLAUTH_BASIC 100
5255
#define K_CURLAUTH_DIGEST 101
@@ -80,7 +83,7 @@ typedef struct _ConnInfo {
8083
struct curl_slist *resp_headers;
8184
struct curl_slist *req_headers;
8285
struct curl_slist *req_cookies;
83-
int response_code;
86+
long response_code;
8487
char *post_data;
8588
long post_data_size;
8689
// metrics
@@ -130,6 +133,7 @@ typedef struct _EasyOpts {
130133
char *curlopt_keypasswd;
131134
char *curlopt_userpwd;
132135
long curlopt_sslversion;
136+
long curlopt_dns_cache_timeout;
133137
} EasyOpts;
134138

135139
static const char *curl_error_code(CURLcode error) {
@@ -659,7 +663,8 @@ static int sock_cb(CURL *e, curl_socket_t s, int what, void *cbp, void *sockp) {
659663
return 0;
660664
}
661665

662-
static int multi_timer_cb(CURLM *multi, long timeout_ms, GlobalInfo *global) {
666+
static int multi_timer_cb(CURLM *multi, long timeout_ms, void *userp) {
667+
GlobalInfo *global = (GlobalInfo *)userp;
663668
struct timeval timeout;
664669

665670
timeout.tv_sec = timeout_ms / 1000;
@@ -715,7 +720,7 @@ static void set_method(long method, ConnInfo *conn) {
715720
case KATIPO_GET:
716721
break;
717722
case KATIPO_POST:
718-
curl_easy_setopt(conn->easy, CURLOPT_POST, 1);
723+
curl_easy_setopt(conn->easy, CURLOPT_POST, 1L);
719724
curl_easy_setopt(conn->easy, CURLOPT_POSTFIELDS, conn->post_data);
720725
curl_easy_setopt(conn->easy, CURLOPT_POSTFIELDSIZE, conn->post_data_size);
721726
break;
@@ -731,7 +736,7 @@ static void set_method(long method, ConnInfo *conn) {
731736
break;
732737
case KATIPO_HEAD:
733738
curl_easy_setopt(conn->easy, CURLOPT_CUSTOMREQUEST, "HEAD");
734-
curl_easy_setopt(conn->easy, CURLOPT_NOBODY, 1);
739+
curl_easy_setopt(conn->easy, CURLOPT_NOBODY, 1L);
735740
break;
736741
case KATIPO_DELETE:
737742
curl_easy_setopt(conn->easy, CURLOPT_CUSTOMREQUEST, "DELETE");
@@ -819,6 +824,7 @@ static void new_conn(long method, char *url, struct curl_slist *req_headers,
819824
eopts.curlopt_cacert);
820825
}
821826
curl_easy_setopt(conn->easy, CURLOPT_TIMEOUT_MS, eopts.curlopt_timeout_ms);
827+
curl_easy_setopt(conn->easy, CURLOPT_DNS_CACHE_TIMEOUT, eopts.curlopt_dns_cache_timeout);
822828
curl_easy_setopt(conn->easy, CURLOPT_MAXREDIRS, eopts.curlopt_maxredirs);
823829
if (eopts.curlopt_http_auth != -1) {
824830
curl_easy_setopt(conn->easy, CURLOPT_HTTPAUTH,
@@ -1147,6 +1153,9 @@ static void erl_input(struct bufferevent *ev, void *arg) {
11471153
case K_CURLOPT_SSLVERSION:
11481154
eopts.curlopt_sslversion = eopt_long;
11491155
break;
1156+
case K_CURLOPT_DNS_CACHE_TIMEOUT:
1157+
eopts.curlopt_dns_cache_timeout = eopt_long;
1158+
break;
11501159
default:
11511160
errx(2, "Unknown eopt long value %ld", eopt);
11521161
}
@@ -1231,15 +1240,17 @@ static void erl_error(struct bufferevent *ev, short event, void *ud) {
12311240

12321241
static void erlang_init(GlobalInfo *global) {
12331242
from_erlang =
1234-
bufferevent_new(STDIN_FILENO, erl_input, NULL, erl_error, global);
1243+
bufferevent_socket_new(global->evbase, STDIN_FILENO, 0);
12351244
if (from_erlang == NULL) {
1236-
errx(2, "bufferevent_new");
1245+
errx(2, "bufferevent_socket_new");
12371246
}
1247+
bufferevent_setcb(from_erlang, erl_input, NULL, erl_error, global);
12381248

1239-
to_erlang = bufferevent_new(STDOUT_FILENO, NULL, NULL, erl_error, global);
1249+
to_erlang = bufferevent_socket_new(global->evbase, STDOUT_FILENO, 0);
12401250
if (to_erlang == NULL) {
1241-
errx(2, "bufferevent_new");
1251+
errx(2, "bufferevent_socket_new");
12421252
}
1253+
bufferevent_setcb(to_erlang, NULL, NULL, erl_error, global);
12431254

12441255
bufferevent_setwatermark(from_erlang, EV_READ, 4, 0);
12451256
bufferevent_enable(from_erlang, EV_READ);
@@ -1257,11 +1268,15 @@ int main(int argc, char **argv) {
12571268
{ "pipelining", required_argument, 0, 'p' },
12581269
{ "max-pipeline-length", required_argument, 0, 'a' },
12591270
{ "max-total-connections", required_argument, 0, 'c' },
1271+
{ "max-concurrent-streams", required_argument, 0, 's' },
12601272
{ 0, 0, 0, 0 }
12611273
};
12621274

12631275
memset(&global, 0, sizeof(GlobalInfo));
1264-
global.evbase = event_init();
1276+
global.evbase = event_base_new();
1277+
if (!global.evbase) {
1278+
errx(2, "event_base_new failed");
1279+
}
12651280

12661281
if (curl_global_init(CURL_GLOBAL_ALL)) {
12671282
errx(2, "curl_global_init failed");
@@ -1308,6 +1323,10 @@ int main(int argc, char **argv) {
13081323
curl_multi_setopt(global.multi, CURLMOPT_MAX_TOTAL_CONNECTIONS,
13091324
atoi(optarg));
13101325
break;
1326+
case 's':
1327+
curl_multi_setopt(global.multi, CURLMOPT_MAX_CONCURRENT_STREAMS,
1328+
atoi(optarg));
1329+
break;
13111330
default:
13121331
errx(2, "Unknown option '%c'\n", c);
13131332
}
@@ -1317,5 +1336,13 @@ int main(int argc, char **argv) {
13171336

13181337
event_base_dispatch(global.evbase);
13191338

1339+
/* Cleanup */
1340+
bufferevent_free(from_erlang);
1341+
bufferevent_free(to_erlang);
1342+
curl_multi_cleanup(global.multi);
1343+
curl_share_cleanup(global.shobject);
1344+
curl_global_cleanup();
1345+
event_base_free(global.evbase);
1346+
13201347
return (0);
13211348
}

‎src/katipo.erl‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,7 @@
110110
-define(KEYPASSWD, 27).
111111
-define(USERPWD, 28).
112112
-define(SSLVERSION, 29).
113+
-define(DNS_CACHE_TIMEOUT, 31).
113114

114115
-define(DEFAULT_REQ_TIMEOUT, 30000).
115116
-define(FOLLOWLOCATION_TRUE, 1).
@@ -374,7 +375,8 @@
374375
%% see [https://curl.se/libcurl/c/CURLOPT_SSLVERSION.html]
375376
-type curlmopts() :: [{max_pipeline_length, non_neg_integer()} |
376377
{pipelining, pipelining()} |
377-
{max_total_connections, non_neg_integer()}].
378+
{max_total_connections, non_neg_integer()} |
379+
{max_concurrent_streams, non_neg_integer()}].
378380

379381
-export_type([method/0]).
380382
-export_type([url/0]).
@@ -436,7 +438,8 @@
436438
sslkey = undefined :: undefined | binary() | file:name_all(),
437439
sslkey_blob = undefined :: undefined | binary(),
438440
keypasswd = undefined :: undefined | binary(),
439-
userpwd = undefined :: undefined | binary()
441+
userpwd = undefined :: undefined | binary(),
442+
dns_cache_timeout = 60 :: integer()
440443
}).
441444

442445
-type req() :: #req{}.
@@ -667,7 +670,8 @@ handle_call(#req{method = Method,
667670
sslkey = SSLKey,
668671
sslkey_blob = SSLKeyBlob,
669672
keypasswd = KeyPasswd,
670-
userpwd = UserPwd},
673+
userpwd = UserPwd,
674+
dns_cache_timeout = DNSCacheTimeout},
671675
From,
672676
State = #state{port = Port, reqs = Reqs}) ->
673677
{Self, Ref} = From,
@@ -695,7 +699,8 @@ handle_call(#req{method = Method,
695699
{?SSLKEY, SSLKey},
696700
{?SSLKEY_BLOB, SSLKeyBlob},
697701
{?KEYPASSWD, KeyPasswd},
698-
{?USERPWD, UserPwd}],
702+
{?USERPWD, UserPwd},
703+
{?DNS_CACHE_TIMEOUT, DNSCacheTimeout}],
699704
Command = {Self, Ref, Method, Url, Headers, CookieJar, Body, Opts},
700705
true = port_command(Port, term_to_binary(Command)),
701706
Tref = erlang:start_timer(Timeout, self(), {req_timeout, From}),
@@ -816,6 +821,9 @@ mopt_supported({pipelining, multiplex}) ->
816821
mopt_supported({max_total_connections, Val})
817822
when is_integer(Val) andalso Val >= 0 ->
818823
{true, "--max-total-connections " ++ integer_to_list(Val)};
824+
mopt_supported({max_concurrent_streams, Val})
825+
when is_integer(Val) andalso Val >= 0 ->
826+
{true, "--max-concurrent-streams " ++ integer_to_list(Val)};
819827
mopt_supported({_, _}) ->
820828
false.
821829

@@ -944,6 +952,8 @@ opt(keypasswd, Pass, {Req, Errors}) when is_binary(Pass) ->
944952
{Req#req{keypasswd = Pass}, Errors};
945953
opt(userpwd, UserPwd, {Req, Errors}) when is_binary(UserPwd) ->
946954
{Req#req{userpwd = UserPwd}, Errors};
955+
opt(dns_cache_timeout, Secs, {Req, Errors}) when is_integer(Secs) andalso Secs >= -1 ->
956+
{Req#req{dns_cache_timeout = Secs}, Errors};
947957
opt(K, V, {Req, Errors}) ->
948958
{Req, [{K, V} | Errors]}.
949959

‎test/katipo_SUITE.erl‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,8 @@ groups() ->
164164
lock_data_ssl_session_false,
165165
doh_url,
166166
badopts,
167-
protocol_restriction]},
167+
protocol_restriction,
168+
dns_cache_timeout]},
168169
{digest, [],
169170
[basic_authorised,
170171
basic_authorised_userpwd,
@@ -178,7 +179,8 @@ groups() ->
178179
port_death,
179180
port_late_response,
180181
pool_opts,
181-
max_pipeline_length]},
182+
max_pipeline_length,
183+
max_concurrent_streams]},
182184
{https, [parallel],
183185
[verify_host_verify_peer_ok,
184186
%% TODO :Fix this test. See https://github.com/puzza007/katipo/runs/5281801454?check_suite_focus=true
@@ -712,6 +714,16 @@ proxy_couldnt_connect(Config) ->
712714
protocol_restriction(_) ->
713715
{error, #{code := unsupported_protocol}} = katipo:get(?POOL, <<"dict.org">>).
714716

717+
dns_cache_timeout(Config) ->
718+
Url = httpbin_url(Config, <<"/get">>),
719+
BaseOpts = ?config(httpbin_opts, Config),
720+
%% cache disabled
721+
{ok, #{status := 200}} = katipo:get(?POOL, Url, BaseOpts#{dns_cache_timeout => 0}),
722+
%% 120 second cache
723+
{ok, #{status := 200}} = katipo:get(?POOL, Url, BaseOpts#{dns_cache_timeout => 120}),
724+
%% forever cache
725+
{ok, #{status := 200}} = katipo:get(?POOL, Url, BaseOpts#{dns_cache_timeout => -1}).
726+
715727
timeout_ms(Config) ->
716728
{req_opts, Opts} = lists:keyfind(req_opts, 1, Config),
717729
ok = case katipo:get(?POOL, httpbin_url(Config, <<"/delay/1">>), Opts#{timeout_ms => 500}) of
@@ -826,6 +838,14 @@ max_pipeline_length(_) ->
826838
{ok, _} = katipo_pool:start(PoolName, PoolSize, PoolOpts),
827839
ok = katipo_pool:stop(PoolName).
828840

841+
max_concurrent_streams(_) ->
842+
PoolName = pool_max_streams,
843+
PoolSize = 1,
844+
PoolOpts = [{pipelining, multiplex},
845+
{max_concurrent_streams, 50}],
846+
{ok, _} = katipo_pool:start(PoolName, PoolSize, PoolOpts),
847+
ok = katipo_pool:stop(PoolName).
848+
829849
verify_host_verify_peer_ok(_) ->
830850
Opts = [#{ssl_verifyhost => true, ssl_verifypeer => true},
831851
#{ssl_verifyhost => false, ssl_verifypeer => true},

0 commit comments

Comments
 (0)