Skip to content

Commit f0586c3

Browse files
rpmoorerpmoore
authored
Merge pull request #149 from DenverM80/3_0_curl_connections
3 0 curl connection pool
2 parents 48dcf14 + 3d03072 commit f0586c3

File tree

11 files changed

+343
-10
lines changed

11 files changed

+343
-10
lines changed

‎src/Makefile.am‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
lib_LTLIBRARIES = libds3.la
2-
include_HEADERS = ds3.h ds3_string_multimap.h ds3_net.h ds3_request.h ds3_utils.h ds3_string_multimap_impl.h ds3_string.h
2+
include_HEADERS = ds3.h ds3_string_multimap.h ds3_net.h ds3_request.h ds3_utils.h ds3_string_multimap_impl.h ds3_string.h ds3_connection.h
33

4-
libds3_la_SOURCES = ds3.c ds3_string_multimap.c ds3_net.c ds3_string_multimap_impl.c ds3_string.c ds3_utils.c
4+
libds3_la_SOURCES = ds3.c ds3_string_multimap.c ds3_net.c ds3_string_multimap_impl.c ds3_string.c ds3_utils.c ds3_connection.c
55
libds3_la_CFLAGS = $(LIBXML2_CFLAGS) $(GLIB2_CFLAGS) $(LIBCURL_CFLAGS) -Wall
66
libds3_la_LIBADD = $(LIBXML2_LIBS) $(GLIB2_LIBS) $(LIBCURL_LIBS)
77
libds3_la_LDFLAGS = -version-info 0:0:0

‎src/ds3.c‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
#include "ds3_string_multimap_impl.h"
3333
#include "ds3_net.h"
3434
#include "ds3_utils.h"
35+
#include "ds3_connection.h"
3536

3637
#ifdef _WIN32
3738
#include <io.h>
@@ -893,6 +894,8 @@ ds3_client* ds3_create_client(const char* endpoint, ds3_creds* creds) {
893894

894895
ds3_client_register_net( client, net_process_request );
895896

897+
client->connection_pool = ds3_connection_pool_init();
898+
896899
return client;
897900
}
898901

@@ -949,9 +952,9 @@ void ds3_client_free(ds3_client* client) {
949952

950953
ds3_str_free(client->endpoint);
951954
ds3_str_free(client->proxy);
952-
if (client->log != NULL) {
953-
g_free(client->log);
954-
}
955+
g_free(client->log);
956+
ds3_connection_pool_clear(client->connection_pool);
957+
g_free(client->connection_pool);
955958
g_free(client);
956959
}
957960

‎src/ds3.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@ extern "C" {
4848

4949
typedef struct _ds3_request ds3_request;
5050

51+
typedef struct _ds3_connection_pool ds3_connection_pool;
52+
5153
typedef struct {
5254
ds3_str* name;
5355
ds3_str** values;
@@ -1586,8 +1588,10 @@ typedef struct _ds3_client {
15861588
void* write_user_struct,
15871589
size_t (*write_handler_func)(void*, size_t, size_t, void*),
15881590
ds3_string_multimap** return_headers);
1591+
ds3_connection_pool* connection_pool;
15891592
}ds3_client;
15901593

1594+
15911595
LIBRARY_API void ds3_blob_response_free(ds3_blob_response* response_data);
15921596
LIBRARY_API void ds3_bucket_response_free(ds3_bucket_response* response_data);
15931597
LIBRARY_API void ds3_bucket_acl_response_free(ds3_bucket_acl_response* response_data);

‎src/ds3_connection.c‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
2+
/*
3+
* ******************************************************************************
4+
* Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved.
5+
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use
6+
* this file except in compliance with the License. A copy of the License is located at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* or in the "license" file accompanying this file.
11+
* This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
12+
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
13+
* specific language governing permissions and limitations under the License.
14+
* ****************************************************************************
15+
*/
16+
17+
#include <string.h>
18+
#include <curl/curl.h>
19+
#include <glib.h>
20+
#include "ds3_net.h"
21+
22+
ds3_connection_pool* ds3_connection_pool_init(void) {
23+
ds3_connection_pool* pool = g_new0(ds3_connection_pool, 1);
24+
g_mutex_init(&pool->mutex);
25+
g_cond_init(&pool->available_connections);
26+
return pool;
27+
}
28+
29+
void ds3_connection_pool_clear(ds3_connection_pool* pool) {
30+
int index;
31+
32+
if (pool == NULL) {
33+
return;
34+
}
35+
36+
g_mutex_lock(&pool->mutex);
37+
38+
for (index = 0; index < CONNECTION_POOL_SIZE; index++) {
39+
if (pool->connections[index] != NULL) {
40+
curl_easy_cleanup(pool->connections[index]);
41+
}
42+
}
43+
44+
g_mutex_unlock(&pool->mutex);
45+
g_mutex_clear(&pool->mutex);
46+
g_cond_clear(&pool->available_connections);
47+
}
48+
49+
static int _pool_inc(ds3_connection_pool* pool, int index) {
50+
return (index+1) % CONNECTION_POOL_SIZE;
51+
}
52+
53+
static int _pool_full(ds3_connection_pool* pool) {
54+
return (_pool_inc(pool, pool->tail) == pool->head);
55+
}
56+
57+
58+
ds3_connection* ds3_connection_acquire(ds3_connection_pool* pool) {
59+
ds3_connection* connection = NULL;
60+
61+
g_mutex_lock(&pool->mutex);
62+
while (_pool_full(pool)) {
63+
g_cond_wait(&pool->available_connections, &pool->mutex);
64+
}
65+
66+
if (pool->connections[pool->head] == NULL) {
67+
connection = curl_easy_init();
68+
69+
pool->connections[pool->head] = connection;
70+
} else {
71+
connection = pool->connections[pool->head];
72+
}
73+
pool->head = _pool_inc(pool, pool->head);
74+
75+
g_mutex_unlock(&pool->mutex);
76+
77+
return connection;
78+
}
79+
80+
void ds3_connection_release(ds3_connection_pool* pool, ds3_connection* connection) {
81+
g_mutex_lock(&pool->mutex);
82+
83+
curl_easy_reset(connection);
84+
pool->tail = _pool_inc(pool, pool->tail);
85+
86+
g_mutex_unlock(&pool->mutex);
87+
g_cond_signal(&pool->available_connections);
88+
}
89+

‎src/ds3_connection.h‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
2+
/*
3+
* ******************************************************************************
4+
* Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved.
5+
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use
6+
* this file except in compliance with the License. A copy of the License is located at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* or in the "license" file accompanying this file.
11+
* This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
12+
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
13+
* specific language governing permissions and limitations under the License.
14+
* ****************************************************************************
15+
*/
16+
17+
#ifndef __DS3_CONNECTION_H__
18+
#define __DS3_CONNECTION_H__
19+
20+
#ifdef __cplusplus
21+
extern "C" {
22+
#endif
23+
24+
#include <curl/curl.h>
25+
#include <glib.h>
26+
27+
#define CONNECTION_POOL_SIZE 100
28+
29+
typedef GMutex ds3_mutex;
30+
typedef GCond ds3_condition;
31+
32+
typedef CURL ds3_connection;
33+
34+
//-- Opaque struct
35+
struct _ds3_connection_pool{
36+
ds3_connection* connections[CONNECTION_POOL_SIZE];
37+
int head;
38+
int tail;
39+
ds3_mutex mutex;
40+
ds3_condition available_connections;
41+
};
42+
43+
ds3_connection_pool* ds3_connection_pool_init(void);
44+
void ds3_connection_pool_clear(ds3_connection_pool* pool);
45+
46+
ds3_connection* ds3_connection_acquire(ds3_connection_pool* pool);
47+
void ds3_connection_release(ds3_connection_pool* pool, ds3_connection* handle);
48+
49+
#ifdef __cplusplus
50+
}
51+
#endif
52+
#endif

‎src/ds3_net.c‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
#include "ds3_utils.h"
2525
#include "ds3_string_multimap.h"
2626
#include "ds3_string_multimap_impl.h"
27+
#include "ds3_connection.h"
2728

2829
static void _init_curl(void) {
2930
static ds3_bool initialized = False;
@@ -398,7 +399,7 @@ ds3_error* net_process_request(const ds3_client* client,
398399
}
399400

400401
while (retry_count < client->num_redirects) {
401-
handle = curl_easy_init();
402+
handle = (CURL*)ds3_connection_acquire(client->connection_pool);
402403

403404
if (handle) {
404405
char* amz_headers;
@@ -514,7 +515,7 @@ ds3_error* net_process_request(const ds3_client* client,
514515
g_free(signature);
515516
g_free(auth_header);
516517
curl_slist_free_all(headers);
517-
curl_easy_cleanup(handle);
518+
ds3_connection_release(client->connection_pool, handle);
518519

519520
//process the response
520521
if (res != CURLE_OK) {
@@ -584,4 +585,4 @@ ds3_error* net_process_request(const ds3_client* client,
584585

585586
void net_cleanup(void) {
586587
curl_global_cleanup();
587-
}
588+
}

‎src/ds3_net.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ extern "C" {
2222

2323
#include "ds3.h"
2424
#include "ds3_string_multimap.h"
25+
#include "ds3_connection.h"
2526

2627
char* escape_url(const char* url);
2728
char* escape_url_extended(const char* url, const char** delimiters, uint32_t num_delimiters);

‎test/Makefile‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ run_single: test
4040
deps:
4141
./build_local.sh
4242

43-
test: test.o get_physical_placement.o search_tests.o metadata_tests.o negative_tests.o checksum.o service_tests.o bucket_tests.o multimap_tests.o deletes_test.o job_tests.o get_object.o bulk_get.o
43+
test: test.o get_physical_placement.o search_tests.o metadata_tests.o negative_tests.o checksum.o service_tests.o bucket_tests.o multimap_tests.o deletes_test.o job_tests.o get_object.o bulk_get.o bulk_put.o
4444
$(CPP) *.o $(CFLAGS) $(LIBS) -o test
4545

4646
test.o: ../install/lib/pkgconfig/libds3.pc
@@ -67,6 +67,9 @@ job_tests.o:
6767
bulk_get.o:
6868
$(CPP) -c bulk_get.cpp $(CFLAGS)
6969

70+
bulk_put.o:
71+
$(CPP) -c bulk_put.cpp $(CFLAGS)
72+
7073
get_object.o:
7174
$(CPP) -c get_object.cpp $(CFLAGS)
7275

0 commit comments

Comments
 (0)