pandasGEXpress/concat_gctoo: add doc strings to methods, slight rename

Author: Dave Lahr

cmapPy/pandasGEXpress/concat_gctoo.py

@@ -276,7 +276,7 @@ def assemble_common_meta(common_meta_dfs, fields_to_remove, sources, remove_all_
         all_meta_df_sorted (pandas df)
 
     """
-    all_meta_df, all_meta_df_with_dups = build_common_all_meta_dfs(common_meta_dfs, fields_to_remove, remove_all_metadata_fields)
+    all_meta_df, all_meta_df_with_dups = build_common_all_meta_df(common_meta_dfs, fields_to_remove, remove_all_metadata_fields)
 
     if not all_meta_df.index.is_unique:
         all_report_df = build_mismatched_common_meta_report([x.shape for x in common_meta_dfs],
@@ -299,8 +299,25 @@ def assemble_common_meta(common_meta_dfs, fields_to_remove, sources, remove_all_
     return all_meta_df_sorted
 
 
-def build_common_all_meta_dfs(common_meta_dfs, fields_to_remove, remove_all_metadata_fields):
-    # Remove any column headers that are not present in all dfs (and sort)
+def build_common_all_meta_df(common_meta_dfs, fields_to_remove, remove_all_metadata_fields):
+    """
+    concatenate the entries in common_meta_dfs, removing columns selectively (fields_to_remove) or entirely (
+        remove_all_metadata_fields=True; in this case, effectively just merges all the indexes in common_meta_dfs).
+
+        Returns 2 dataframes (in a tuple):  the first has duplicates removed, the second does not.
+
+    Args:
+        common_meta_dfs: collection of pandas DataFrames containing the metadata in the "common" direction of the
+            concatenation operation
+        fields_to_remove: columns to be removed (if present) from the common_meta_dfs
+        remove_all_metadata_fields: boolean indicating that all metadata fields should be removed from the
+            common_meta_dfs; overrides fields_to_remove if present
+
+    Returns:
+        tuple containing
+            all_meta_df:  pandas dataframe that is the concatenation of the dataframes in common_meta_dfs,
+            all_meta_df_with_dups:
+    """
 
     if remove_all_metadata_fields:
         trimmed_common_meta_dfs = [pd.DataFrame(index=df.index) for df in common_meta_dfs]
@@ -338,6 +355,19 @@ def build_common_all_meta_dfs(common_meta_dfs, fields_to_remove, remove_all_meta
 
 
 def build_mismatched_common_meta_report(common_meta_df_shapes, sources, all_meta_df, all_meta_df_with_dups):
+    """
+    Generate a report (dataframe) that indicates for the common metadata that does not match across the common metadata
+        which source file had which of the different mismatch values
+
+    Args:
+        common_meta_df_shapes:  list of tuples that are the shapes of the common meta dataframes
+        sources: list of the source files that the dataframes were loaded from
+        all_meta_df: produced from build_common_all_meta_df
+        all_meta_df_with_dups: produced from build_common_all_meta_df
+
+    Returns:
+        all_report_df:  dataframe indicating the mismatched row metadata values and the corresponding source file
+    """
     expanded_sources = []
     for (i, shape) in enumerate(common_meta_df_shapes):
         src = sources[i]

cmapPy/pandasGEXpress/tests/test_concat_gctoo.py

@@ -219,7 +219,7 @@ def test_do_reset_ids(self):
         pd.util.testing.assert_frame_equal(meta_df, e_meta_df)
         pd.util.testing.assert_frame_equal(data_df, e_data_df)
 
-    def test_build_common_all_meta_dfs(self):
+    def test_build_common_all_meta_df(self):
         # rhd3 header needs to be removed
         meta1 = pd.DataFrame(
             [["r1_1", "r1_2", "r1_3"],
@@ -240,15 +240,15 @@ def test_build_common_all_meta_dfs(self):
             index=["r1", "r2", "r3"],
             columns=["rhd1", "rhd2"])
 
-        r_all, r_all_w_dups = cg.build_common_all_meta_dfs([meta1, meta2], ["rhd3"], False)
+        r_all, r_all_w_dups = cg.build_common_all_meta_df([meta1, meta2], ["rhd3"], False)
         logger.debug("rhd3 header needs to be removed - r_all:\n{}".format(r_all))
         logger.debug("r_all_w_dups:\n{}".format(r_all_w_dups))
         self.assertEqual((3,2), r_all.shape)
         self.assertEqual((6,2), r_all_w_dups.shape)
         pd.util.testing.assert_frame_equal(e_meta1, r_all)
 
         #remove all metadata fields
-        r_all, r_all_w_dups = cg.build_common_all_meta_dfs([meta1, meta2], [], True)
+        r_all, r_all_w_dups = cg.build_common_all_meta_df([meta1, meta2], [], True)
         logger.debug("remove all metadata fields - r_all\n{}".format(r_all))
         logger.debug("r_all_w_dups:\n{}".format(r_all_w_dups))
         self.assertEqual((3,0), r_all.shape)
@@ -273,7 +273,7 @@ def test_build_common_all_meta_dfs(self):
 
         # rhd5 not in meta4, so it should be dropped even without being
         # explicitly provided
-        out_meta3, _ = cg.build_common_all_meta_dfs([meta1, meta4], ["rhd2"], False)
+        out_meta3, _ = cg.build_common_all_meta_df([meta1, meta4], ["rhd2"], False)
         logger.debug("""rhd5 not in meta4 so it should be automatically dropped without being
         explictly listed in fields_to_remove - out_meta3:
         {}""".format(out_meta3))
@@ -282,14 +282,14 @@ def test_build_common_all_meta_dfs(self):
         # Empty metadata
         empty_meta = pd.DataFrame([], index=["a", "b", "c"])
         logger.debug("empty metadata provided - empty_meta.empty: {}".format(empty_meta.empty))
-        out_meta4, _ = cg.build_common_all_meta_dfs([empty_meta, empty_meta], [], False)
+        out_meta4, _ = cg.build_common_all_meta_df([empty_meta, empty_meta], [], False)
         logger.debug("empty metadata provided - out_meta4:\n{}".format(out_meta4))
         pd.util.testing.assert_frame_equal(out_meta4, empty_meta)
 
         #metadata has duplicates but index is unique
         meta5 = pd.DataFrame({"rhd1":[0,0,1]}, index=range(3))
         meta6 = pd.DataFrame({"rhd1":[0,0,1]}, index=range(3))
-        out_meta5, _ = cg.build_common_all_meta_dfs([meta5, meta6], [], False)
+        out_meta5, _ = cg.build_common_all_meta_df([meta5, meta6], [], False)
         logger.debug("metadata has duplicates but index is unique - out_meta5:\n{}".format(out_meta5))
         self.assertEqual((3,1), out_meta5.shape, "metadata contains duplicates but index is unique - should have been kept")
 
@@ -319,7 +319,7 @@ def test_build_mismatched_common_meta_report(self):
         logger.debug("meta3:\n{}".format(meta3))
 
         common_meta_dfs = [meta1, meta2, meta3]
-        all_meta_df, all_meta_df_with_dups = cg.build_common_all_meta_dfs(common_meta_dfs, [], False)
+        all_meta_df, all_meta_df_with_dups = cg.build_common_all_meta_df(common_meta_dfs, [], False)
         common_meta_df_shapes = [x.shape for x in common_meta_dfs]
         sources = ["my_src1", "my_src2", "my_src3"]
         self.assertFalse(all_meta_df.index.is_unique, "during setup expected the index to not be unique")