buffer io operations

Author: TheAssembler1

run_test.sh

@@ -11,6 +11,5 @@ echo "deleting existing data"
 rm -rf /home/nlewi26/src/work_space/source/pdc/build/bin/pdc_data
 
 pushd ./build
-ctest -L serial --stop-on-failure --output-on-failure
-#ctest -L serial -I 1,37 -I 39,68
+ctest -L serial -I 20 --stop-on-failure --output-on-failure
 popd

src/api/pdc_client_connect.c

@@ -6313,7 +6313,6 @@ static perr_t
 PDC_add_storage_meta_to_io_list(pdc_data_server_io_list_t **list, region_storage_meta_t *storage_meta,
                                 void *buf)
 {
-
     pdc_data_server_io_list_t *io_list_elt, *io_list_target = NULL;
     region_list_t *            new_region;
     int                        j;

src/server/include/pdc_client_server_common.h

@@ -339,7 +339,6 @@ typedef struct data_server_region_t {
     // For non-mapped object analysis
     // Used primarily as a local_temp
     void *obj_data_ptr;
-#ifdef PDC_USE_REGION_PER_FILE
     // FIXME: (Noah) we should dynamically allocate this number
     char *storage_location[4096]; // save the file location to enable reopening
     /**
@@ -354,7 +353,6 @@ typedef struct data_server_region_t {
      * to chunk size
      */
     uint8_t use_region_per_file;
-#endif
     struct data_server_region_t *prev;
     struct data_server_region_t *next;
 } data_server_region_t;

src/server/pdc_server_region/pdc_server_data.c

@@ -499,8 +499,6 @@ PDC_Data_Server_region_lock(region_lock_in_t *in, region_lock_out_t *out, hg_han
     region_buf_map_t *    eltt;
     int                   error      = 0;
     int                   found_lock = 0;
-    // time_t                t;
-    // struct tm             tm;
 
     FUNC_ENTER(NULL);
 
@@ -3334,10 +3332,8 @@ PDC_Server_add_region_storage_meta_to_bulk_buf(region_list_t *region, bulk_xfer_
 
     // Sanity check
     if (NULL == region || region->storage_location[0] == 0 || NULL == region->meta) {
-        LOG_ERROR("==PDC_SERVER[%d]: invalid region data!\n", pdc_server_rank_g);
         PDC_print_region_list(region);
-        ret_value = FAIL;
-        goto done;
+        PGOTO_ERROR(FAIL, "==PDC_SERVER[%d]: invalid region data!\n", pdc_server_rank_g);
     }
 
     // Alloc space and init if it's empty
@@ -4532,6 +4528,8 @@ is_fd_valid(int fd)
     return fcntl(fd, F_GETFD) != -1 || errno != EBADF;
 }
 
+#undef NOAH_DEBUG
+
 static perr_t
 PDC_Server_data_io_from_region_per_file(uint64_t obj_id, struct pdc_region_info *region_info, void *buf,
                                         size_t unit, REGION_PER_FILE_IO_TYPE io_type)
@@ -4540,6 +4538,7 @@ PDC_Server_data_io_from_region_per_file(uint64_t obj_id, struct pdc_region_info
 
     FUNC_ENTER(NULL);
 
+#ifdef NOAH_DEBUG
     LOG_JUST_PRINT("===============================================\n");
     LOG_JUST_PRINT("IO (%s) called with parameters:\n",
                    io_type == REGION_PER_FILE_IO_TYPE_READ ? "READ" : "WRITE");
@@ -4552,6 +4551,7 @@ PDC_Server_data_io_from_region_per_file(uint64_t obj_id, struct pdc_region_info
         printf("\tregion_info->offset[%d] = %" PRIu64 "\n", i, region_info->offset[i]);
         printf("\tregion_info->size[%d]   = %" PRIu64 "\n", i, region_info->size[i]);
     }
+#endif
 
     // Compute total number of elements
     uint64_t num_elements = 1;
@@ -4573,10 +4573,12 @@ PDC_Server_data_io_from_region_per_file(uint64_t obj_id, struct pdc_region_info
     uint64_t *            offset      = region_info->offset;
     int                   ndim        = region_info->ndim;
 
+#ifdef NOAH_DEBUG
     LOG_JUST_PRINT("\tobj meta ndim: %d\n", ndim);
     for (int i = 0; i < ndim; i++) {
         LOG_JUST_PRINT("\tobj meta dims[%d] = %" PRIu64 "\n", i, global_size[i]);
     }
+#endif
 
     // Compute flat offset from global start
     uint64_t flat_offset = 0, stride = 1;
@@ -4585,11 +4587,14 @@ PDC_Server_data_io_from_region_per_file(uint64_t obj_id, struct pdc_region_info
         stride *= global_size[i];
     }
 
+#ifdef NOAH_DEBUG
     LOG_JUST_PRINT("\tnum_elements: %d\n", num_elements);
     LOG_JUST_PRINT("\tflat_offset: %d\n", flat_offset);
+#endif
 
     // Determine initial region index
     uint64_t region_index = 0;
+    uint64_t region_num_elements;
     if (region == NULL) {
         region_index = flat_offset / num_elements;
     }
@@ -4600,8 +4605,9 @@ PDC_Server_data_io_from_region_per_file(uint64_t obj_id, struct pdc_region_info
         region_index = flat_offset / region->region_size_elements;
     }
 
+#ifdef NOAH_DEBUG
     LOG_JUST_PRINT("\tregion_index: %d\n", region_index);
-
+#endif
     // Register the initial region
     PDC_Server_register_obj_region_by_pointer(&region, obj_id, 0, region_index);
 
@@ -4613,6 +4619,19 @@ PDC_Server_data_io_from_region_per_file(uint64_t obj_id, struct pdc_region_info
         region->region_size_elements = num_elements;
     }
 
+    // Used to buffer I/O operations
+    uint8_t* buf_io_ops = (uint8_t*)malloc(region->region_size_elements * unit);
+    errno = 0;
+    if (ftruncate(region->fd, region->region_size_elements * unit) != 0) {
+        LOG_ERROR("Error: %s\n", strerror(errno));
+        PGOTO_ERROR(FAIL, "Failed to tfruncate");
+    }
+    errno = 0;
+    if(pread(region->fd, buf_io_ops, region->region_size_elements * unit, 0) != region->region_size_elements * unit) {
+        LOG_ERROR("Error: %s\n", strerror(errno));
+        PGOTO_ERROR(FAIL, "Failed to pread");
+    }
+
     // Allocate indices for iteration
     uint64_t *indices = (uint64_t *)calloc(ndim, sizeof(uint64_t));
     if (indices == NULL) {
@@ -4633,15 +4652,29 @@ PDC_Server_data_io_from_region_per_file(uint64_t obj_id, struct pdc_region_info
         if (new_region_index != region_index) {
             uint64_t old_region_size = region->region_size_elements;
             if (region) {
+                pwrite(region->fd, buf_io_ops, region->region_size_elements * unit, 0);
                 PDC_Server_unregister_obj_region_by_pointer(region, 0);
                 region->fd = -1;
             }
+#ifdef NOAH_DEBUG
             LOG_JUST_PRINT("\told region index: %d\n", region_index);
             LOG_JUST_PRINT("\tnew region index: %d\n", new_region_index);
             LOG_JUST_PRINT("\tflat index: %d\n", flat_index);
+#endif
             region_index = new_region_index;
             PDC_Server_register_obj_region_by_pointer(&region, obj_id, 0, region_index);
 
+            errno = 0;
+            if (ftruncate(region->fd, region->region_size_elements * unit) != 0) {
+                LOG_ERROR("Error: %s\n", strerror(errno));
+                PGOTO_ERROR(FAIL, "Failed to tfruncate");
+            }
+            errno = 0;
+            if(pread(region->fd, buf_io_ops, region->region_size_elements * unit, 0) != region->region_size_elements * unit) {
+                LOG_ERROR("Error: %s\n", strerror(errno));
+                PGOTO_ERROR(FAIL, "Failed to pread");
+            }
+
             if (region == NULL) {
                 PGOTO_ERROR(FAIL, "Failed to register obj_region by pointer");
             }
@@ -4657,27 +4690,11 @@ PDC_Server_data_io_from_region_per_file(uint64_t obj_id, struct pdc_region_info
         void *   target_buf    = (char *)buf + (count * unit);
         off_t    file_offset   = offset_region * unit;
 
-        ssize_t res;
-        errno = 0;
         if (io_type == REGION_PER_FILE_IO_TYPE_READ) {
-            res = pread(region->fd, target_buf, unit, file_offset);
+            memcpy(target_buf, &(buf_io_ops[file_offset]), unit);
         }
         else {
-            res = pwrite(region->fd, target_buf, unit, file_offset);
-        }
-
-        if (res != (ssize_t)unit) {
-            if (is_fd_valid(region->fd)) {
-                LOG_JUST_PRINT("region->fd was valid\n");
-            }
-            else {
-                LOG_JUST_PRINT("region->fd %d was invalid\n", region->fd);
-            }
-            LOG_JUST_PRINT("expected io size %d: actual io size %d\n", unit, res);
-            LOG_JUST_PRINT("region size %d, region index: %d\n", region->region_size_elements, region_index);
-            LOG_JUST_PRINT("io error storage location: %s\n", region->storage_location[region_index]);
-            perror(io_type == REGION_PER_FILE_IO_TYPE_READ ? "pread" : "pwrite");
-            PGOTO_ERROR(FAIL, "I/O operation failed\n");
+            memcpy(&(buf_io_ops[file_offset]), target_buf, unit);
         }
 
         // Advance indices
@@ -4694,14 +4711,18 @@ PDC_Server_data_io_from_region_per_file(uint64_t obj_id, struct pdc_region_info
 
 done:
     if (region) {
+        pwrite(region->fd, buf_io_ops, region->region_size_elements * unit, 0);
         PDC_Server_unregister_obj_region_by_pointer(region, 0);
         region->fd = -1;
     }
-
     if (indices)
         free(indices);
+    if (buf_io_ops)
+        free(buf_io_ops);
 
+#ifdef NOAH_DEBUG
     LOG_JUST_PRINT("===============================================\n");
+#endif
 
     FUNC_LEAVE(ret_value);
 }