Merge pull request #13 from LIVVkit/mkstratos/tsc_merge_fix

Bugfix for new pandas version

Author: mkstratos

.git-blame-ignore-revs

@@ -0,0 +1 @@
+0e03e4395fae0aa0b6ab475feb054e53135f2352

evv4esm/__init__.py

@@ -28,35 +28,50 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
-__version_info__ = (0, 5, 0)
-__version__ = '.'.join(str(vi) for vi in __version_info__)
+__version_info__ = (0, 5, 1)
+__version__ = ".".join(str(vi) for vi in __version_info__)
 
-PASS_COLOR = '#389933'
-L_PASS_COLOR = '#93DA90'
+PASS_COLOR = "#389933"
+L_PASS_COLOR = "#93DA90"
 
-FAIL_COLOR = '#BF3F46'
-L_FAIL_COLOR = '#E68388'
+FAIL_COLOR = "#BF3F46"
+L_FAIL_COLOR = "#E68388"
 
-human_color_names = {'pass': ('green', 'light green'),
-                     PASS_COLOR: 'green',
-                     L_PASS_COLOR: 'light green',
-                     'fail': ('red', 'light red'),
-                     FAIL_COLOR: 'red',
-                     L_FAIL_COLOR: 'light red'}
+human_color_names = {
+    "pass": ("green", "light green"),
+    PASS_COLOR: "green",
+    L_PASS_COLOR: "light green",
+    "fail": ("red", "light red"),
+    FAIL_COLOR: "red",
+    L_FAIL_COLOR: "light red",
+}
 
-pf_color_picker = {'Pass': PASS_COLOR, 'pass': PASS_COLOR,
-                   'Accept': PASS_COLOR, 'accept': PASS_COLOR,
-                   'Fail': FAIL_COLOR, 'fail': FAIL_COLOR,
-                   'Reject': FAIL_COLOR, 'reject': FAIL_COLOR}
+pf_color_picker = {
+    "Pass": PASS_COLOR,
+    "pass": PASS_COLOR,
+    "Accept": PASS_COLOR,
+    "accept": PASS_COLOR,
+    "Fail": FAIL_COLOR,
+    "fail": FAIL_COLOR,
+    "Reject": FAIL_COLOR,
+    "reject": FAIL_COLOR,
+}
 
-light_pf_color_picker = {'Pass': L_PASS_COLOR, 'pass': L_PASS_COLOR,
-                         'Accept': L_PASS_COLOR, 'accept': L_PASS_COLOR,
-                         PASS_COLOR: L_PASS_COLOR,
-                         'Fail': L_FAIL_COLOR, 'fail': L_FAIL_COLOR,
-                         'Reject': L_FAIL_COLOR, 'reject': L_FAIL_COLOR,
-                         FAIL_COLOR: L_FAIL_COLOR}
+light_pf_color_picker = {
+    "Pass": L_PASS_COLOR,
+    "pass": L_PASS_COLOR,
+    "Accept": L_PASS_COLOR,
+    "accept": L_PASS_COLOR,
+    PASS_COLOR: L_PASS_COLOR,
+    "Fail": L_FAIL_COLOR,
+    "fail": L_FAIL_COLOR,
+    "Reject": L_FAIL_COLOR,
+    "reject": L_FAIL_COLOR,
+    FAIL_COLOR: L_FAIL_COLOR,
+}
 
 
 class EVVException(Exception):
     """Base class for EVV exceptions"""
+
     pass

evv4esm/__main__.py

@@ -2,21 +2,21 @@
 # coding=utf-8
 # Copyright (c) 2018 UT-BATTELLE, LLC
 # All rights reserved.
-# 
+#
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are met:
-# 
+#
 # 1. Redistributions of source code must retain the above copyright notice, this
 # list of conditions and the following disclaimer.
-# 
+#
 # 2. Redistributions in binary form must reproduce the above copyright notice,
 # this list of conditions and the following disclaimer in the documentation
 # and/or other materials provided with the distribution.
-# 
+#
 # 3. Neither the name of the copyright holder nor the names of its contributors
 # may be used to endorse or promote products derived from this software without
 # specific prior written permission.
-# 
+#
 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
@@ -39,53 +39,72 @@
 
 
 def parse_args(args=None):
-    parser = argparse.ArgumentParser(description=__doc__,
-                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-
-    parser.add_argument('-e', '--extensions',
-                        action='store',
-                        nargs='+',
-                        default=None,
-                        help='Specify the location of the JSON configuration files for the extended V&V tests to run.')
-  
-    parser.add_argument('-o', '--out-dir',
-                        default=os.path.join(os.getcwd(), "vv_" + time.strftime("%Y-%m-%d")),
-                        help='Location to output the EVV webpages.')
-
-    parser.add_argument('-s', '--serve',
-                        nargs='?', type=int, const=8000,
-                        help=' '.join(['Start a simple HTTP server for the output website specified',
-                                       'by OUT_DIR on port SERVE.'
-                                       ])
-                        )
-
-    parser.add_argument('-p', '--pool-size',
-                        nargs='?',
-                        type=int,
-                        default=(options.mp.cpu_count() - 1 or 1),
-                        help='The number of multiprocessing processes to run '
-                             'analyses in. If zero, processes will run serially '
-                             'outside of the multiprocessing module.')
-
-    parser.add_argument('--version',
-                        action='version',
-                        version='EVV {}'.format(evv4esm.__version__),
-                        help="Show EVV's version number and exit"
-                        )
+    parser = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+
+    parser.add_argument(
+        "-e",
+        "--extensions",
+        action="store",
+        nargs="+",
+        default=None,
+        help="Specify the location of the JSON configuration files for the extended V&V tests to run.",
+    )
+
+    parser.add_argument(
+        "-o",
+        "--out-dir",
+        default=os.path.join(os.getcwd(), "vv_" + time.strftime("%Y-%m-%d")),
+        help="Location to output the EVV webpages.",
+    )
+
+    parser.add_argument(
+        "-s",
+        "--serve",
+        nargs="?",
+        type=int,
+        const=8000,
+        help=" ".join(
+            [
+                "Start a simple HTTP server for the output website specified",
+                "by OUT_DIR on port SERVE.",
+            ]
+        ),
+    )
+
+    parser.add_argument(
+        "-p",
+        "--pool-size",
+        nargs="?",
+        type=int,
+        default=(options.mp.cpu_count() - 1 or 1),
+        help="The number of multiprocessing processes to run "
+        "analyses in. If zero, processes will run serially "
+        "outside of the multiprocessing module.",
+    )
+
+    parser.add_argument(
+        "--version",
+        action="version",
+        version="EVV {}".format(evv4esm.__version__),
+        help="Show EVV's version number and exit",
+    )
 
     args = parser.parse_args(args)
 
     if args.extensions:
-        options.parse_args(['-V']+args.extensions + ['-o', args.out_dir])
-    
+        options.parse_args(["-V"] + args.extensions + ["-o", args.out_dir])
+
     from evv4esm import resources
+
     args.livv_resource_dir = livvkit.resource_dir
     livvkit.resource_dir = os.sep.join(resources.__path__)
     return args
 
 
 def main(cl_args=None):
-    """ Direct execution. """
+    """Direct execution."""
 
     if cl_args is None and len(sys.argv) > 1:
         cl_args = sys.argv[1:]
@@ -123,10 +142,14 @@ def main(cl_args=None):
         print(" -----------------------------------------------------------------")
         print("")
         for conf in livvkit.validation_model_configs:
-            validation_config = functions.merge_dicts(validation_config,
-                                                      functions.read_json(conf))
-            summary_elements.extend(scheduler.run_quiet("validation", validation, validation_config,
-                                                        group=False))
+            validation_config = functions.merge_dicts(
+                validation_config, functions.read_json(conf)
+            )
+            summary_elements.extend(
+                scheduler.run_quiet(
+                    "validation", validation, validation_config, group=False
+                )
+            )
         print(" -----------------------------------------------------------------")
         print("   Extensions test suite complete ")
         print(" -----------------------------------------------------------------")
@@ -137,28 +160,34 @@ def main(cl_args=None):
             index_data.write(result._repr_json())
         print("-------------------------------------------------------------------")
         print(" Done!  Results can be seen in a web browser at:")
-        print("   " + os.path.join(livvkit.output_dir, 'index.html'))
+        print("   " + os.path.join(livvkit.output_dir, "index.html"))
         print("-------------------------------------------------------------------")
 
     if args.serve:
         import http.server as server
         import socketserver as socket
 
-        httpd = socket.TCPServer(('', args.serve), server.SimpleHTTPRequestHandler)
+        httpd = socket.TCPServer(("", args.serve), server.SimpleHTTPRequestHandler)
 
         sa = httpd.socket.getsockname()
-        print('\nServing HTTP on {host} port {port} (http://{host}:{port}/)'.format(host=sa[0], port=sa[1]))
-        print('\nView the generated website by navigating to:')
-        print('\n    http://{host}:{port}/{path}/index.html'.format(host=sa[0], port=sa[1],
-                                                                    path=os.path.relpath(args.out_dir)
-                                                                    ))
-        print('\nExit by pressing `ctrl+c` to send a keyboard interrupt.\n')
+        print(
+            "\nServing HTTP on {host} port {port} (http://{host}:{port}/)".format(
+                host=sa[0], port=sa[1]
+            )
+        )
+        print("\nView the generated website by navigating to:")
+        print(
+            "\n    http://{host}:{port}/{path}/index.html".format(
+                host=sa[0], port=sa[1], path=os.path.relpath(args.out_dir)
+            )
+        )
+        print("\nExit by pressing `ctrl+c` to send a keyboard interrupt.\n")
         try:
             httpd.serve_forever()
         except KeyboardInterrupt:
-            print('\nKeyboard interrupt received, exiting.\n')
+            print("\nKeyboard interrupt received, exiting.\n")
             sys.exit(0)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

evv4esm/ensembles/e3sm.py

@@ -64,15 +64,19 @@ def file_date_str(case_file, style="short", hist_name="h0"):
     return result.replace("{}.".format(hist_name), "").replace(".nc", "")
 
 
-def component_monthly_files(dir_, component, ninst, hist_name="h0", nmonth_max=12, date_style="short"):
+def component_monthly_files(
+    dir_, component, ninst, hist_name="h0", nmonth_max=12, date_style="short"
+):
     if date_style == "full":
         date_search = "????-??-??-??"
     elif date_style == "med":
         date_search = "????-??-??"
     else:
         date_search = "????-??"
 
-    base = "{d}/*{c}_????.{n}.{ds}.nc".format(d=dir_, c=component, n=hist_name, ds=date_search)
+    base = "{d}/*{c}_????.{n}.{ds}.nc".format(
+        d=dir_, c=component, n=hist_name, ds=date_search
+    )
     search = os.path.normpath(base)
     result = sorted(glob.glob(search))
 
@@ -115,22 +119,36 @@ def gather_monthly_averages(ensemble_files, variable_set=None):
                     if variable_set is None:
                         variable_set = set(data.variables.keys())
                 except OSError as E:
-                    six.raise_from(BaseException('Could not open netCDF dataset: {}'.format(file_)), E)
+                    six.raise_from(
+                        BaseException(
+                            "Could not open netCDF dataset: {}".format(file_)
+                        ),
+                        E,
+                    )
 
                 for var in data.variables.keys():
                     if var not in variable_set:
                         continue
-                    if len(data.variables[var].shape) < 2 or var in ['time_bnds', 'date_written', 'time_written']:
+                    if len(data.variables[var].shape) < 2 or var in [
+                        "time_bnds",
+                        "date_written",
+                        "time_written",
+                    ]:
                         continue
-                    elif 'ncol' not in data.variables[var].dimensions:
+                    elif "ncol" not in data.variables[var].dimensions:
                         continue
                     else:
                         m = np.mean(data.variables[var][0, ...])
 
                     desc = "{long_name}{units}".format(**get_variable_meta(data, var))
-                    monthly_avgs.append((case, var, '{:04}'.format(inst), date_str, m, desc))
-
-    monthly_avgs = pd.DataFrame(monthly_avgs, columns=('case', 'variable', 'instance', 'date', 'monthly_mean', 'desc'))
+                    monthly_avgs.append(
+                        (case, var, "{:04}".format(inst), date_str, m, desc)
+                    )
+
+    monthly_avgs = pd.DataFrame(
+        monthly_avgs,
+        columns=("case", "variable", "instance", "date", "monthly_mean", "desc"),
+    )
     return monthly_avgs
 
 

evv4esm/ensembles/tools.py

@@ -170,10 +170,16 @@ def prob_plot(
 
         else:
             ax3.hist(
-                norm_ref, bins=n_q, color=pf_color_picker.get(pf, "#1F77B4"), edgecolor="k"
+                norm_ref,
+                bins=n_q,
+                color=pf_color_picker.get(pf, "#1F77B4"),
+                edgecolor="k",
             )
             ax4.hist(
-                norm_test, bins=n_q, color=pf_color_picker.get(pf, "#1F77B4"), edgecolor="k"
+                norm_test,
+                bins=n_q,
+                color=pf_color_picker.get(pf, "#1F77B4"),
+                edgecolor="k",
             )
 
             # Check if these distributions are wildly different. If they are, use different