Merge pull request open-mpi#770 from edgargabriel/pr/commits-of-week-47

Pr/commits of week 47

Author: jsquyres

config/ompi_check_lustre.m4

@@ -61,6 +61,30 @@ AC_DEFUN([OMPI_CHECK_LUSTRE],[
                        [$ompi_check_lustre_dir], [$ompi_check_lustre_libdir], [ompi_check_lustre_happy="yes"],
                        [ompi_check_lustre_happy="no"])
 
+    AC_MSG_CHECKING([for required lustre data structures])
+   cat > conftest.c <<EOF
+#include "lustre/liblustreapi.h"
+void alloc_lum()
+{
+  int v1, v3;
+  v1 = sizeof(struct lov_user_md_v1) +
+   LOV_MAX_STRIPE_COUNT * sizeof(struct lov_user_ost_data_v1);
+  v3 = sizeof(struct lov_user_md_v3) +
+    LOV_MAX_STRIPE_COUNT * sizeof(struct lov_user_ost_data_v1);
+}
+EOF
+
+# Try the compile
+OPAL_LOG_COMMAND(
+    [$CC $CFLAGS -I$with_lustre/include -c conftest.c],
+    [ompi_check_lustre_struct_happy="yes"],
+    [ompi_check_lustre_struct_happy="no"
+     ompi_check_lustre_happy="no"]
+)
+    rm -f conftest.c conftest.o
+    AC_MSG_RESULT([$ompi_check_lustre_struct_happy])
+
+
     AS_IF([test "$ompi_check_lustre_happy" = "yes"],
           [$2],
           [AS_IF([test ! -z "$with_lustre" && test "$with_lustre" != "no"],

ompi/mca/io/ompio/io_ompio.c

@@ -1031,13 +1031,29 @@ int ompi_io_ompio_set_aggregator_props (struct mca_io_ompio_file_t *fh,
     fh->f_flags |= OMPIO_AGGREGATOR_IS_SET;
 
     if (-1 == num_aggregators) {
-        mca_io_ompio_create_groups(fh,bytes_per_proc);
+        if ( SIMPLE == mca_io_ompio_grouping_option ||
+            NO_REFINEMENT == mca_io_ompio_grouping_option ) {
+            fh->f_aggregator_index = 0;
+            fh->f_final_num_aggrs  = fh->f_init_num_aggrs;
+            fh->f_procs_per_group  = fh->f_init_procs_per_group;
+
+            fh->f_procs_in_group = (int*)malloc (fh->f_procs_per_group * sizeof(int));
+            if (NULL == fh->f_procs_in_group) {
+                opal_output (1, "OUT OF MEMORY\n");
+                return OMPI_ERR_OUT_OF_RESOURCE;
+            }
+
+            for (j=0 ; j<fh->f_procs_per_group ; j++) {
+                fh->f_procs_in_group[j] = fh->f_init_procs_in_group[j];
+            }
+        }
+        else {
+            mca_io_ompio_create_groups(fh,bytes_per_proc);
+        }
         return OMPI_SUCCESS;
     }
 
    //Forced number of aggregators
-   else
-   {
     /* calculate the offset at which each group of processes will start */
     procs_per_group = ceil ((float)fh->f_size/num_aggregators);
 
@@ -1063,7 +1079,6 @@ int ompi_io_ompio_set_aggregator_props (struct mca_io_ompio_file_t *fh,
     fh->f_final_num_aggrs  = num_aggregators;
 
     return OMPI_SUCCESS;
-   }
  }
 
 

ompi/mca/io/ompio/io_ompio.h

@@ -101,15 +101,22 @@ OMPI_DECLSPEC extern int mca_io_ompio_coll_timing_info;
 #define OMPIO_MERGE                     1
 #define OMPIO_SPLIT                     2
 #define OMPIO_RETAIN                    3
+
 #define DATA_VOLUME                     1
 #define UNIFORM_DISTRIBUTION            2
-#define OMPIO_UNIFORM_DIST_THRESHOLD  0.5
 #define CONTIGUITY                      3
-#define OMPIO_CONTG_THRESHOLD     1048576
 #define OPTIMIZE_GROUPING               4
-#define OMPIO_PROCS_PER_GROUP_TAG       0
-#define OMPIO_PROCS_IN_GROUP_TAG        1
-#define OMPIO_MERGE_THRESHOLD         0.5
+#define SIMPLE                          5
+#define NO_REFINEMENT                   6
+
+
+#define OMPIO_UNIFORM_DIST_THRESHOLD     0.5
+#define OMPIO_CONTG_THRESHOLD        1048576
+#define OMPIO_CONTG_FACTOR                 8
+#define OMPIO_DEFAULT_STRIPE_SIZE    1048576
+#define OMPIO_PROCS_PER_GROUP_TAG          0
+#define OMPIO_PROCS_IN_GROUP_TAG           1
+#define OMPIO_MERGE_THRESHOLD            0.5
 
 /*---------------------------*/
 
@@ -543,6 +550,9 @@ int mca_io_ompio_cart_based_grouping(mca_io_ompio_file_t *ompio_fh);
 int mca_io_ompio_fview_based_grouping(mca_io_ompio_file_t *fh,
 		                                    int *num_groups,
 						    contg *contg_groups);
+int mca_io_ompio_simple_grouping(mca_io_ompio_file_t *fh,
+                                 int *num_groups,
+                                 contg *contg_groups);
 
 int mca_io_ompio_finalize_initial_grouping(mca_io_ompio_file_t *fh,
                                            int num_groups,

ompi/mca/io/ompio/io_ompio_component.c

@@ -38,7 +38,7 @@ int mca_io_ompio_record_offset_info = 0;
 int mca_io_ompio_coll_timing_info = 0;
 int mca_io_ompio_sharedfp_lazy_open = 1;
 
-int mca_io_ompio_grouping_option=0;
+int mca_io_ompio_grouping_option=5;
 
 /*
  * Private functions
@@ -202,10 +202,13 @@ static int register_component(void)
                                            MCA_BASE_VAR_SCOPE_READONLY,
                                            &mca_io_ompio_sharedfp_lazy_open);
 
-    mca_io_ompio_grouping_option = 0;
+    mca_io_ompio_grouping_option = 5;
     (void) mca_base_component_var_register(&mca_io_ompio_component.io_version,
                                            "grouping_option",
-                                           "Option for grouping of processes in the aggregator selection",
+                                           "Option for grouping of processes in the aggregator selection "
+                                           "1: Data volume based grouping 2: maximizing group size uniformity 3: maximimze "
+                                           "data contiguity 4: hybrid optimization  5: simple (default) "
+                                           "6: skip refinement step",
                                            MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
                                            OPAL_INFO_LVL_9,
                                            MCA_BASE_VAR_SCOPE_READONLY,