Skip to content

Commit fa998c5

Browse files
rpmoorerpmoore
authored
Merge pull request #142 from DenverM80/fix_get_objects_request
fix get_objects_request passing of user_data and callback
2 parents ee24114 + e085cbd commit fa998c5

File tree

4 files changed

+154
-68
lines changed

4 files changed

+154
-68
lines changed

‎src/ds3.c‎

Lines changed: 61 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -13375,55 +13375,6 @@ static ds3_error* _parse_top_level_ds3_list_multi_part_uploads_result_response(c
1337513375
return error;
1337613376
}
1337713377

13378-
ds3_error* ds3_head_bucket_request(const ds3_client* client, const ds3_request* request) {
13379-
if (client == NULL || request == NULL) {
13380-
return ds3_create_error(DS3_ERROR_MISSING_ARGS, "All arguments must be filled in for request processing");
13381-
}
13382-
if (g_ascii_strncasecmp(request->path->value, "//", 2) == 0) {
13383-
return ds3_create_error(DS3_ERROR_MISSING_ARGS, "The bucket name parameter is required.");
13384-
}
13385-
13386-
return _internal_request_dispatcher(client, request, NULL, NULL, NULL, NULL, NULL);
13387-
}
13388-
13389-
ds3_error* ds3_head_object_request(const ds3_client* client, const ds3_request* request, ds3_metadata** _metadata) {
13390-
ds3_error* error;
13391-
ds3_string_multimap* return_headers;
13392-
ds3_metadata* metadata;
13393-
13394-
int num_slashes = num_chars_in_ds3_str(request->path, '/');
13395-
if (num_slashes < 2 || ((num_slashes == 2) && ('/' == request->path->value[request->path->size-1]))) {
13396-
return ds3_create_error(DS3_ERROR_MISSING_ARGS, "The object name parameter is required.");
13397-
} else if (g_ascii_strncasecmp(request->path->value, "//", 2) == 0) {
13398-
return ds3_create_error(DS3_ERROR_MISSING_ARGS, "The bucket name parameter is required.");
13399-
}
13400-
13401-
error = _internal_request_dispatcher(client, request, NULL, NULL, NULL, NULL, &return_headers);
13402-
13403-
if (error == NULL) {
13404-
metadata = _init_metadata(return_headers);
13405-
*_metadata = metadata;
13406-
ds3_string_multimap_free(return_headers);
13407-
}
13408-
13409-
return error;
13410-
}
13411-
13412-
13413-
ds3_error* ds3_get_object_with_metadata(const ds3_client* client, const ds3_request* request, void* user_data, size_t (* callback)(void*, size_t, size_t, void*), ds3_metadata** _metadata) {
13414-
ds3_error* error;
13415-
ds3_string_multimap* return_headers;
13416-
ds3_metadata* metadata;
13417-
13418-
error = _internal_request_dispatcher(client, request, user_data, callback, NULL, NULL, &return_headers);
13419-
if (error == NULL) {
13420-
metadata = _init_metadata(return_headers);
13421-
*_metadata = metadata;
13422-
ds3_string_multimap_free(return_headers);
13423-
}
13424-
13425-
return error;
13426-
}
1342713378

1342813379
ds3_error* ds3_abort_multi_part_upload_request(const ds3_client* client, const ds3_request* request) {
1342913380

@@ -13570,7 +13521,26 @@ ds3_error* ds3_get_service_request(const ds3_client* client, const ds3_request*
1357013521

1357113522
return _parse_top_level_ds3_list_all_my_buckets_result_response(client, request, response, xml_blob);
1357213523
}
13573-
ds3_error* ds3_get_object_request(const ds3_client* client, const ds3_request* request, void* user_data, size_t (*callback)(void*, size_t, size_t, void*)) {
13524+
13525+
ds3_error* ds3_get_object_request(const ds3_client* client, const ds3_request* request, void* user_data, size_t (* callback)(void*, size_t, size_t, void*)) {
13526+
ds3_error* error;
13527+
13528+
int num_slashes = num_chars_in_ds3_str(request->path, '/');
13529+
if (num_slashes < 2 || ((num_slashes == 2) && ('/' == request->path->value[request->path->size-1]))) {
13530+
return ds3_create_error(DS3_ERROR_MISSING_ARGS, "The bucket name parameter is required.");
13531+
} else if (g_ascii_strncasecmp(request->path->value, "//", 2) == 0) {
13532+
return ds3_create_error(DS3_ERROR_MISSING_ARGS, "The object name parameter is required.");
13533+
}
13534+
13535+
error = _internal_request_dispatcher(client, request, user_data, callback, NULL, NULL, NULL);
13536+
13537+
return error;
13538+
}
13539+
13540+
ds3_error* ds3_get_object_with_metadata(const ds3_client* client, const ds3_request* request, void* user_data, size_t (* callback)(void*, size_t, size_t, void*), ds3_metadata** _metadata) {
13541+
ds3_error* error;
13542+
ds3_string_multimap* return_headers;
13543+
ds3_metadata* metadata;
1357413544

1357513545
int num_slashes = num_chars_in_ds3_str(request->path, '/');
1357613546
if (num_slashes < 2 || ((num_slashes == 2) && ('/' == request->path->value[request->path->size-1]))) {
@@ -13579,8 +13549,49 @@ ds3_error* ds3_get_object_request(const ds3_client* client, const ds3_request* r
1357913549
return ds3_create_error(DS3_ERROR_MISSING_ARGS, "The object name parameter is required.");
1358013550
}
1358113551

13552+
error = _internal_request_dispatcher(client, request, user_data, callback, NULL, NULL, &return_headers);
13553+
if (error == NULL) {
13554+
metadata = _init_metadata(return_headers);
13555+
*_metadata = metadata;
13556+
ds3_string_multimap_free(return_headers);
13557+
}
13558+
13559+
return error;
13560+
}
13561+
ds3_error* ds3_head_bucket_request(const ds3_client* client, const ds3_request* request) {
13562+
if (client == NULL || request == NULL) {
13563+
return ds3_create_error(DS3_ERROR_MISSING_ARGS, "All arguments must be filled in for request processing");
13564+
}
13565+
if (g_ascii_strncasecmp(request->path->value, "//", 2) == 0) {
13566+
return ds3_create_error(DS3_ERROR_MISSING_ARGS, "The bucket name parameter is required.");
13567+
}
13568+
1358213569
return _internal_request_dispatcher(client, request, NULL, NULL, NULL, NULL, NULL);
1358313570
}
13571+
13572+
ds3_error* ds3_head_object_request(const ds3_client* client, const ds3_request* request, ds3_metadata** _metadata) {
13573+
ds3_error* error;
13574+
ds3_string_multimap* return_headers;
13575+
ds3_metadata* metadata;
13576+
13577+
int num_slashes = num_chars_in_ds3_str(request->path, '/');
13578+
if (num_slashes < 2 || ((num_slashes == 2) && ('/' == request->path->value[request->path->size-1]))) {
13579+
return ds3_create_error(DS3_ERROR_MISSING_ARGS, "The object name parameter is required.");
13580+
} else if (g_ascii_strncasecmp(request->path->value, "//", 2) == 0) {
13581+
return ds3_create_error(DS3_ERROR_MISSING_ARGS, "The bucket name parameter is required.");
13582+
}
13583+
13584+
error = _internal_request_dispatcher(client, request, NULL, NULL, NULL, NULL, &return_headers);
13585+
13586+
if (error == NULL) {
13587+
metadata = _init_metadata(return_headers);
13588+
*_metadata = metadata;
13589+
ds3_string_multimap_free(return_headers);
13590+
}
13591+
13592+
return error;
13593+
}
13594+
1358413595
ds3_error* ds3_initiate_multi_part_upload_request(const ds3_client* client, const ds3_request* request, ds3_initiate_multipart_upload_result_response** response) {
1358513596
ds3_error* error;
1358613597
GByteArray* xml_blob;

‎test/Makefile‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ run_single: test
4040
deps:
4141
./build_local.sh
4242

43-
test: test.o get_physical_placement.o search_tests.o metadata_tests.o negative_tests.o checksum.o service_tests.o bucket_tests.o multimap_tests.o deletes_test.o job_tests.o # bulk_get.o
43+
test: test.o get_physical_placement.o search_tests.o metadata_tests.o negative_tests.o checksum.o service_tests.o bucket_tests.o multimap_tests.o deletes_test.o job_tests.o get_object.o bulk_get.o
4444
$(CPP) *.o $(CFLAGS) $(LIBS) -o test
4545

4646
test.o: ../install/lib/pkgconfig/libds3.pc
@@ -67,6 +67,9 @@ job_tests.o:
6767
bulk_get.o:
6868
$(CPP) -c bulk_get.cpp $(CFLAGS)
6969

70+
get_object.o:
71+
$(CPP) -c get_object.cpp $(CFLAGS)
72+
7073
get_physical_placement.o:
7174
$(CPP) -c get_physical_placement.cpp $(CFLAGS)
7275

‎test/bulk_get.cpp‎

Lines changed: 20 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -47,9 +47,6 @@ uint32_t get_sum_of_chunks(uint32_t num_files, checksum_result* results) {
4747
}
4848

4949
uint32_t getFileIndexForChunk(uint64_t* max_file_index, ds3_str* current_obj_name, checksum_result* results) {
50-
printf("getFileIndexForChunk\n");
51-
printf(" curr_obj_name[%s]\n", current_obj_name->value);
52-
printf("max_file_index: %lu\n", *max_file_index);
5350
int64_t file_index = -1;
5451
for (uint64_t current_file_index = 0; current_file_index < *max_file_index; current_file_index++) {
5552
if (g_strcmp0(current_obj_name->value, results[current_file_index].original_name) == 0) {
@@ -58,20 +55,16 @@ uint32_t getFileIndexForChunk(uint64_t* max_file_index, ds3_str* current_obj_nam
5855
}
5956
}
6057

61-
printf("file_index: %lu\n", file_index);
6258
if (file_index == -1) {
6359

6460
file_index = *max_file_index;
65-
printf(" file_index: %lu\n", file_index);
6661
(*max_file_index)++;
67-
printf(" max_file_index: %lu\n", *max_file_index);
6862

6963
memcpy(results[file_index].original_name, current_obj_name->value, current_obj_name->size);
7064

7165
memcpy(results[file_index].tmp_name, FOLDER_PREFIX, LENGTH_OF_FOLDER_PREFIX); // "resources/"
7266
memcpy(results[file_index].tmp_name+LENGTH_OF_FOLDER_PREFIX-1, TEMP_PREFIX, LENGTH_OF_TEMP_PREFIX); // "resources/temp-"
7367
memcpy(results[file_index].tmp_name+LENGTH_OF_FOLDER_PREFIX+LENGTH_OF_TEMP_PREFIX-2, current_obj_name->value+LENGTH_OF_FOLDER_PREFIX-1, current_obj_name->size-LENGTH_OF_FOLDER_PREFIX+1);
74-
printf(" tmp_name[%s]\n", results[file_index].tmp_name);
7568
}
7669
results[file_index].num_chunks++;
7770
return file_index;
@@ -88,14 +81,14 @@ void checkChunkResponse(ds3_client* client, uint32_t num_files, ds3_master_objec
8881
for (uint64_t chunk_object_index = 0; chunk_object_index < chunk_object_list->num_objects; chunk_object_index++) {
8982
FILE* w_file;
9083
ds3_bulk_object_response* current_obj = chunk_object_list->objects[chunk_object_index];
91-
printf("checkChunkResponse: obj %s in bucket %s\n", current_obj->name->value, chunk_response->bucket_name->value);
9284
file_index = getFileIndexForChunk(&max_file_index, current_obj->name, results);
9385

9486
const uint64_t length = current_obj->length;
95-
printf("checkChunkResponse length: %lu\n", length);
9687
const uint64_t offset = current_obj->offset;
97-
printf("checkChunkResponse offset: %lu\n", offset);
98-
request = ds3_init_get_object_request(chunk_response->bucket_name->value, current_obj->name->value, &length, chunk_response->job_id->value, &offset);
88+
89+
request = ds3_init_get_object_request(chunk_response->bucket_name->value, current_obj->name->value, length);
90+
ds3_request_set_job(request, chunk_response->job_id->value);
91+
ds3_request_set_offset(request, offset);
9992

10093
w_file = fopen(results[file_index].tmp_name, "a+");
10194
fseek(w_file, current_obj->offset, SEEK_SET);
@@ -129,7 +122,9 @@ void checkChunkResponsePartials(ds3_client* client, uint32_t num_files, ds3_mast
129122

130123
const uint64_t length = current_obj->length;
131124
const uint64_t offset = current_obj->offset;
132-
request = ds3_init_get_object_request(chunk_response->bucket_name->value, current_obj->name->value, &length, chunk_response->job_id->value, &offset);
125+
request = ds3_init_get_object_request(chunk_response->bucket_name->value, current_obj->name->value, length);
126+
ds3_request_set_job(request, chunk_response->job_id->value);
127+
ds3_request_set_offset(request, offset);
133128

134129
ds3_request_set_byte_range(request, segment_size, segment_size*2-1);
135130
ds3_request_set_byte_range(request, segment_size*3, segment_size*4-1);
@@ -168,8 +163,10 @@ BOOST_AUTO_TEST_CASE( bulk_get ) {
168163

169164
populate_with_objects(client, bucket_name);
170165

166+
request = ds3_init_get_bulk_job_spectra_s3_request(bucket_name, object_list);
171167
ds3_job_chunk_client_processing_order_guarantee chunk_order = DS3_JOB_CHUNK_CLIENT_PROCESSING_ORDER_GUARANTEE_NONE;
172-
request = ds3_init_get_bulk_job_spectra_s3_request(bucket_name, NULL, &chunk_order, NULL, NULL, object_list);
168+
ds3_request_set_chunk_client_processing_order_guarantee_ds3_job_chunk_client_processing_order_guarantee(request, chunk_order);
169+
173170
error = ds3_get_bulk_job_spectra_s3_request(client, request, &bulk_response);
174171
ds3_request_free(request);
175172
ds3_bulk_object_list_response_free(object_list);
@@ -226,8 +223,10 @@ BOOST_AUTO_TEST_CASE( max_upload_size ) {
226223
populate_with_objects_from_bulk(client, bucket_name, bulk_response);
227224
ds3_master_object_list_response_free(bulk_response);
228225

226+
request = ds3_init_get_bulk_job_spectra_s3_request(bucket_name, object_list);
229227
ds3_job_chunk_client_processing_order_guarantee chunk_order = DS3_JOB_CHUNK_CLIENT_PROCESSING_ORDER_GUARANTEE_NONE;
230-
request = ds3_init_get_bulk_job_spectra_s3_request(bucket_name, NULL, &chunk_order, NULL, NULL, object_list);
228+
ds3_request_set_chunk_client_processing_order_guarantee_ds3_job_chunk_client_processing_order_guarantee(request, chunk_order);
229+
231230
error = ds3_get_bulk_job_spectra_s3_request(client, request, &bulk_response);
232231
ds3_request_free(request);
233232
ds3_bulk_object_list_response_free(object_list);
@@ -286,16 +285,18 @@ BOOST_AUTO_TEST_CASE( chunk_preference ) {
286285

287286
populate_with_objects(client, bucket_name);
288287

288+
request = ds3_init_get_bulk_job_spectra_s3_request(bucket_name, object_list);
289289
ds3_job_chunk_client_processing_order_guarantee chunk_order = DS3_JOB_CHUNK_CLIENT_PROCESSING_ORDER_GUARANTEE_NONE;
290-
request = ds3_init_get_bulk_job_spectra_s3_request(bucket_name, NULL, &chunk_order, NULL, NULL, object_list);
290+
ds3_request_set_chunk_client_processing_order_guarantee_ds3_job_chunk_client_processing_order_guarantee(request, chunk_order);
291+
291292
error = ds3_get_bulk_job_spectra_s3_request(client, request, &bulk_response);
292293
ds3_request_free(request);
293294
ds3_bulk_object_list_response_free(object_list);
294295
BOOST_REQUIRE(handle_error_and_return_is_null(error));
295296

296297
do {
297298
retry_get = false;
298-
request = ds3_init_get_job_chunks_ready_for_client_processing_spectra_s3_request(bulk_response->job_id->value, NULL);
299+
request = ds3_init_get_job_chunks_ready_for_client_processing_spectra_s3_request(bulk_response->job_id->value);
299300
error = ds3_get_job_chunks_ready_for_client_processing_spectra_s3_request(client, request, &chunk_response);
300301
ds3_request_free(request);
301302

@@ -359,8 +360,10 @@ BOOST_AUTO_TEST_CASE( partial_get ) {
359360

360361
populate_with_objects(client, bucket_name);
361362

363+
request = ds3_init_get_bulk_job_spectra_s3_request(bucket_name, object_list);
362364
ds3_job_chunk_client_processing_order_guarantee chunk_order = DS3_JOB_CHUNK_CLIENT_PROCESSING_ORDER_GUARANTEE_NONE;
363-
request = ds3_init_get_bulk_job_spectra_s3_request(bucket_name, NULL, &chunk_order, NULL, NULL, object_list);
365+
ds3_request_set_chunk_client_processing_order_guarantee_ds3_job_chunk_client_processing_order_guarantee(request, chunk_order);
366+
364367
error = ds3_get_bulk_job_spectra_s3_request(client, request, &bulk_response);
365368
ds3_request_free(request);
366369
ds3_bulk_object_list_response_free(object_list);

‎test/get_object.cpp‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
#include <stdio.h>
2+
#include "ds3.h"
3+
#include "test.h"
4+
#include <boost/test/unit_test.hpp>
5+
#include <glib.h>
6+
#include <sys/stat.h>
7+
8+
ds3_contents_response* get_contents_by_name(const ds3_list_bucket_result_response* bucket_list, const char* object_name) {
9+
if (bucket_list == NULL) {
10+
return NULL;
11+
}
12+
13+
ds3_contents_response* object = NULL;
14+
for (size_t object_index = 0; object_index < bucket_list->num_objects; object_index++) {
15+
object = bucket_list->objects[object_index];
16+
if (g_strcmp0(object_name, object->key->value) == 0) {
17+
return object;
18+
}
19+
}
20+
21+
return NULL;
22+
}
23+
24+
BOOST_AUTO_TEST_CASE( get_object ) {
25+
printf("-----Testing GET object-------\n");
26+
27+
const char* bucket_name = "test_get_object_bucket";
28+
const char* object_name = "resources/sherlock_holmes.txt";
29+
const char* get_object_name = "sherlock_holmes_get.txt";
30+
ds3_request* request = NULL;
31+
ds3_error* error = NULL;
32+
ds3_list_bucket_result_response* get_bucket_response;
33+
ds3_contents_response* sherlock = NULL;
34+
uint64_t sherlock_size = 0;
35+
FILE* fp;
36+
struct stat fstat;
37+
int status = 0;
38+
39+
ds3_client* client = get_client();
40+
populate_with_objects(client, bucket_name);
41+
42+
request = ds3_init_get_bucket_request(bucket_name);
43+
error = ds3_get_bucket_request(client, request, &get_bucket_response);
44+
ds3_request_free(request);
45+
handle_error(error);
46+
47+
sherlock = get_contents_by_name(get_bucket_response, object_name);
48+
BOOST_CHECK(sherlock != NULL);
49+
50+
sherlock_size = sherlock->size;
51+
ds3_list_bucket_result_response_free(get_bucket_response);
52+
53+
request = ds3_init_get_object_request(bucket_name, object_name, sherlock_size);
54+
fp = fopen(get_object_name, "w+");
55+
56+
error = ds3_get_object_request(client, request, fp, ds3_write_to_file);
57+
ds3_request_free(request);
58+
fclose(fp);
59+
handle_error(error);
60+
61+
status = stat(get_object_name, &fstat);
62+
BOOST_CHECK(status != -1);
63+
BOOST_CHECK(fstat.st_size > 0);
64+
65+
remove(get_object_name);
66+
clear_bucket(client, bucket_name);
67+
free_client(client);
68+
}
69+

0 commit comments

Comments
 (0)