Merge pull request #180 from DenverM80/64_bit_offsets

Fix request query params for 64bit values

Author: rpmoore

src/ds3.c

@@ -26,6 +26,7 @@
 #include <libxml/parser.h>
 #include <libxml/xmlmemory.h>
 #include <errno.h>
+#include <inttypes.h>
 
 #include "ds3.h"
 #include "ds3_request.h"
@@ -45,6 +46,13 @@
 #define S_ISDIR(mode)  (((mode) & S_IFMT) == S_IFDIR)
 #endif
 
+//The max size of an uint32_t should be 10 characters + NULL
+static const char UNSIGNED_LONG_BASE_10[] = "4294967296";
+static const unsigned int UNSIGNED_LONG_BASE_10_STR_LEN = sizeof(UNSIGNED_LONG_BASE_10) + 1;
+//The max size of an uint64_t should be 20 characters + NULL
+static const char UNSIGNED_LONG_LONG_BASE_10[] = "18446744073709551615";
+static const unsigned int UNSIGNED_LONG_LONG_BASE_10_STR_LEN = sizeof(UNSIGNED_LONG_LONG_BASE_10) + 1;
+
 static char* _get_ds3_bucket_acl_permission_str(ds3_bucket_acl_permission input) {
     if (input == DS3_BUCKET_ACL_PERMISSION_LIST) {
         return "LIST";
@@ -1010,9 +1018,6 @@ void ds3_client_free(ds3_client* client) {
 }
 
 
-static const char UNSIGNED_LONG_BASE_10[] = "4294967296";
-static const unsigned int UNSIGNED_LONG_BASE_10_STR_LEN = sizeof(UNSIGNED_LONG_BASE_10);
-
 typedef struct {
     char* buff;
     size_t size;
@@ -1150,9 +1155,9 @@ static void _set_query_param_flag(const ds3_request* _request, const char* key,
 }
 
 static void _set_query_param_uint64_t(const ds3_request* _request, const char* key, uint64_t value) {
-    char string_buffer[UNSIGNED_LONG_BASE_10_STR_LEN];
+    char string_buffer[UNSIGNED_LONG_LONG_BASE_10_STR_LEN];
     memset(string_buffer, 0, sizeof(string_buffer));
-    snprintf(string_buffer, sizeof(string_buffer), "%lu", value);
+    snprintf(string_buffer, sizeof(string_buffer), "%"PRIu64, value);
     _set_query_param(_request, key, string_buffer);
 }
 
@@ -3579,10 +3584,9 @@ static ds3_error* _get_request_xml_nodes(
     return NULL;
 }
 
-#define LENGTH_BUFF_SIZE 21
 
 static xmlDocPtr _generate_xml_bulk_objects_list(const ds3_bulk_object_list_response* obj_list, object_list_type list_type, ds3_job_chunk_client_processing_order_guarantee order) {
-    char size_buff[LENGTH_BUFF_SIZE]; //The max size of an uint64_t should be 20 characters
+    char size_buff[UNSIGNED_LONG_LONG_BASE_10_STR_LEN];
     xmlDocPtr doc;
     ds3_bulk_object_response* obj;
     xmlNodePtr objects_node, object_node;
@@ -3598,7 +3602,7 @@ static xmlDocPtr _generate_xml_bulk_objects_list(const ds3_bulk_object_list_resp
 
     for (obj_index = 0; obj_index < obj_list->num_objects; obj_index++) {
         obj = obj_list->objects[obj_index];
-        g_snprintf(size_buff, sizeof(char) * LENGTH_BUFF_SIZE, "%llu", (unsigned long long int) obj->length);
+        g_snprintf(size_buff, sizeof(char) * UNSIGNED_LONG_LONG_BASE_10_STR_LEN, "%"PRIu64, obj->length);
 
         object_node = xmlNewNode(NULL, (xmlChar*) "Object");
         xmlAddChild(objects_node, object_node);
@@ -3615,7 +3619,7 @@ static xmlDocPtr _generate_xml_bulk_objects_list(const ds3_bulk_object_list_resp
 }
 
 static xmlDocPtr _generate_xml_complete_mpu(const ds3_complete_multipart_upload_response* mpu_list) {
-    char size_buff[LENGTH_BUFF_SIZE]; //The max size of an uint64_t should be 20 characters
+    char size_buff[UNSIGNED_LONG_LONG_BASE_10_STR_LEN]; //The max size of an uint64_t should be 20 characters
     xmlDocPtr doc;
     ds3_multipart_upload_part_response* part;
     xmlNodePtr parts_node, part_node;
@@ -3631,7 +3635,7 @@ static xmlDocPtr _generate_xml_complete_mpu(const ds3_complete_multipart_upload_
         part_node = xmlNewNode(NULL, (xmlChar*) "Part");
         xmlAddChild(parts_node, part_node);
 
-        g_snprintf(size_buff, sizeof(char) * LENGTH_BUFF_SIZE, "%d", part->part_number);
+        g_snprintf(size_buff, sizeof(char) * UNSIGNED_LONG_LONG_BASE_10_STR_LEN, "%d", part->part_number);
         xmlNewTextChild(part_node, NULL, (xmlChar*) "PartNumber", (xmlChar*) size_buff);
 
         xmlNewTextChild(part_node, NULL, (xmlChar*) "ETag", (xmlChar*) part->etag->value);

src/ds3.h

@@ -30,6 +30,7 @@
 #include "ds3_string_multimap.h"
 
 #ifdef __cplusplus
+#define __STDC_FORMAT_MACROS
 extern "C" {
 #endif
 

test/bulk_put.cpp

@@ -19,6 +19,7 @@
 #include <glib.h>
 #include <sys/stat.h>
 #include <boost/test/unit_test.hpp>
+#include <inttypes.h>
 #include "ds3.h"
 #include "ds3_net.h"
 #include "test.h"
@@ -228,7 +229,56 @@ BOOST_AUTO_TEST_CASE( put_utf_object_name ) {
     ds3_request_free(request);
     ds3_bulk_object_list_response_free(obj_list);
     handle_error(error);
+    ds3_master_object_list_response_free(bulk_response);
+
+    clear_bucket(client, bucket_name);
+    free_client(client);
+}
+
+BOOST_AUTO_TEST_CASE( put_64bit_object_sizes ) {
+    printf("-----Testing PUT objects with 64bit object sizes in name-------\n");
+
+    uint64_t test_objects_size = 429496729600; // 400gb > 32bit
+    const char* bucket_name = "test_64_bit_object_sizes_bucket";
+    ds3_request* request = NULL;
+    ds3_master_object_list_response* bulk_response = NULL;
+
+    ds3_client* client = get_client();
+    ds3_error* error = create_bucket_with_data_policy(client, bucket_name, ids.data_policy_id->value);
+
+    // Create obj_list with 1 folder with UTF-8 characters
+    ds3_bulk_object_list_response* obj_list = ds3_init_bulk_object_list();
+    GPtrArray* ds3_bulk_object_response_array = g_ptr_array_new();
+
+    ds3_bulk_object_response* obj1 = g_new0(ds3_bulk_object_response, 1);
+    obj1->name = ds3_str_init("max_size_obj_1");
+    obj1->length = test_objects_size;
+    g_ptr_array_add(ds3_bulk_object_response_array, obj1);
+    ds3_bulk_object_response* obj2 = g_new0(ds3_bulk_object_response, 1);
+    obj2->name = ds3_str_init("max_size_obj_2");
+    obj2->length = test_objects_size;
+    g_ptr_array_add(ds3_bulk_object_response_array, obj2);
+
+    obj_list->objects = (ds3_bulk_object_response**)ds3_bulk_object_response_array->pdata;
+    obj_list->num_objects = ds3_bulk_object_response_array->len;
+    g_ptr_array_free(ds3_bulk_object_response_array, FALSE);
+
+    request = ds3_init_put_bulk_job_spectra_s3_request(bucket_name, obj_list);
+    error = ds3_put_bulk_job_spectra_s3_request(client, request, &bulk_response);
+    ds3_request_free(request);
+    ds3_bulk_object_list_response_free(obj_list);
+    handle_error(error);
+
+    BOOST_CHECK_EQUAL(bulk_response->num_objects, 14);
 
+    BOOST_CHECK_EQUAL(bulk_response->objects[0]->objects[0]->length, 68719476736);
+    BOOST_CHECK_EQUAL(bulk_response->objects[0]->objects[0]->offset, 0);
+
+    BOOST_CHECK_EQUAL(bulk_response->objects[13]->objects[0]->length, 17179869184);
+    BOOST_CHECK_EQUAL(bulk_response->objects[13]->objects[0]->offset, 412316860416);
+
+    ds3_master_object_list_response_free(bulk_response);
     clear_bucket(client, bucket_name);
     free_client(client);
 }
+

test/metadata_tests.cpp

@@ -391,7 +391,7 @@ BOOST_AUTO_TEST_CASE( put_metadata_using_get_object_retrieval ) {
     ds3_metadata_entry* metadata_entry;
     const char* file_name[1] = {"resources/beowulf.txt"};
     ds3_client* client = get_client();
-    const char* bucket_name = "get_object_metadata_test";
+    const char* bucket_name = "put_metadata_using_get_object_retrieval";
     FILE* file;
 
     ds3_error* error = create_bucket_with_data_policy(client, bucket_name, ids.data_policy_id->value);
@@ -405,6 +405,7 @@ BOOST_AUTO_TEST_CASE( put_metadata_using_get_object_retrieval ) {
     handle_error(error);
 
     request = ds3_init_put_object_request(bucket_name, "resources/beowulf.txt", obj_list->objects[0]->length);
+    ds3_request_set_job(request, bulk_response->job_id->value);
     file = fopen(obj_list->objects[0]->name->value, "r");
 
     ds3_request_set_metadata(request, "name", "value");

test/search_tests.cpp

@@ -244,7 +244,7 @@ BOOST_AUTO_TEST_CASE( get_folder_and_objects ) {
 BOOST_AUTO_TEST_CASE( get_incorrect_bucket_name ) {
     ds3_client* client = get_client();
     ds3_s3_object_list_response* response;
-    const char* bucket_name = "search_bucket_test";
+    const char* bucket_name = "search_incorrect_bucket_test";
     populate_with_objects(client, bucket_name);
 
     ds3_request* request = ds3_init_get_objects_details_spectra_s3_request();