Merge pull request #385 from xylar/add_build_mesh_logging

Add support for logging (e.g. to log files)

Author: xylar

conda_package/mpas_tools/logging.py

@@ -0,0 +1,187 @@
+import sys
+import logging
+import subprocess
+
+
+def check_call(args, logger):
+    """
+    Call the given subprocess with logging to the given logger.
+
+    Parameters
+    ----------
+    args : list
+        A list of argument to the subprocess
+
+    logger : logging.Logger
+        The logger to write output to
+
+    Raises
+    ------
+    subprocess.CalledProcessError
+        If the given subprocess exists with nonzero status
+
+    """
+
+    process = subprocess.Popen(args, stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE)
+    stdout, stderr = process.communicate()
+
+    if stdout:
+        stdout = stdout.decode('utf-8')
+        for line in stdout.split('\n'):
+            logger.info(line)
+    if stderr:
+        stderr = stderr.decode('utf-8')
+        for line in stderr.split('\n'):
+            logger.error(line)
+
+    if process.returncode != 0:
+        raise subprocess.CalledProcessError(process.returncode,
+                                            ' '.join(args))
+
+
+class LoggingContext(object):
+
+    """
+    A context manager for creating a logger or using an existing logger
+
+    Attributes
+    ----------
+    logger : logging.Logger
+        A logger that sends output to a log file or stdout/stderr
+    """
+
+    def __init__(self, name, logger=None, log_filename=None):
+        """
+        If ``logger`` is ``None``, create a new logger either to a log file
+        or stdout/stderr.  If ``logger`` is anything else, just set the logger
+        attribute
+
+        Parameters
+        ----------
+        name : str
+            A unique name for the logger (e.g. ``__name__`` of the calling
+            module)
+
+        logger : logging.Logger, optional
+           An existing logger that sends output to a log file or stdout/stderr
+           to be used in this context
+
+        log_filename : str, optional
+            The name of a file where output should be written.  If none is
+            supplied, output goes to stdout/stderr
+        """
+        self.logger = logger
+        self.name = name
+        self.log_filename = log_filename
+        self.handler = None
+        self.old_stdout = None
+        self.old_stderr = None
+        self.existing_logger = logger is not None
+
+    def __enter__(self):
+        if not self.existing_logger:
+            if self.log_filename is not None:
+                # redirect output to a log file
+                logger = logging.getLogger(self.name)
+                handler = logging.FileHandler(self.log_filename)
+            else:
+                logger = logging.getLogger(self.name)
+                handler = logging.StreamHandler(sys.stdout)
+
+            formatter = MpasFormatter()
+            handler.setFormatter(formatter)
+            logger.addHandler(handler)
+            logger.setLevel(logging.INFO)
+            logger.propagate = False
+            self.logger = logger
+            self.handler = handler
+
+            if self.log_filename is not None:
+                self.old_stdout = sys.stdout
+                self.old_stderr = sys.stderr
+                sys.stdout = StreamToLogger(logger, logging.INFO)
+                sys.stderr = StreamToLogger(logger, logging.ERROR)
+        return self.logger
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if not self.existing_logger:
+            if self.old_stdout is not None:
+                self.handler.close()
+                # restore stdout and stderr
+                sys.stdout = self.old_stdout
+                sys.stderr = self.old_stderr
+
+            # remove the handlers from the logger (probably only necessary if
+            # writeLogFile==False)
+            self.logger.handlers = []
+
+        self.stdout = self.original_stdout = sys.stdout
+        self.stderr = self.original_stderr = sys.stderr
+
+
+class MpasFormatter(logging.Formatter):
+    """
+    A custom formatter for logging
+    Modified from:
+    https://stackoverflow.com/a/8349076/7728169
+    https://stackoverflow.com/a/14859558/7728169
+    """
+    # Authors
+    # -------
+    # Xylar Asay-Davis
+
+    # printing error messages without a prefix because they are sometimes
+    # errors and sometimes only warnings sent to stderr
+    dbg_fmt = "DEBUG: %(module)s: %(lineno)d: %(msg)s"
+    info_fmt = "%(msg)s"
+    err_fmt = info_fmt
+
+    def __init__(self, fmt=info_fmt):
+        logging.Formatter.__init__(self, fmt)
+
+    def format(self, record):
+
+        # Save the original format configured by the user
+        # when the logger formatter was instantiated
+        format_orig = self._fmt
+
+        # Replace the original format with one customized by logging level
+        if record.levelno == logging.DEBUG:
+            self._fmt = MpasFormatter.dbg_fmt
+
+        elif record.levelno == logging.INFO:
+            self._fmt = MpasFormatter.info_fmt
+
+        elif record.levelno == logging.ERROR:
+            self._fmt = MpasFormatter.err_fmt
+
+        # Call the original formatter class to do the grunt work
+        result = logging.Formatter.format(self, record)
+
+        # Restore the original format configured by the user
+        self._fmt = format_orig
+
+        return result
+
+
+class StreamToLogger(object):
+    """
+    Modified based on code by:
+    https://www.electricmonk.nl/log/2011/08/14/redirect-stdout-and-stderr-to-a-logger-in-python/
+    Copyright (C) 2011 Ferry Boender
+    License: GPL, see https://www.electricmonk.nl/log/posting-license/
+    Fake file-like stream object that redirects writes to a logger instance.
+    """
+
+    def __init__(self, logger, log_level=logging.INFO):
+        self.logger = logger
+        self.log_level = log_level
+        self.linebuf = ''
+
+    def write(self, buf):
+        for line in buf.rstrip().splitlines():
+            self.logger.log(self.log_level, str(line.rstrip()))
+
+    def flush(self):
+        pass

conda_package/mpas_tools/mesh/creation/build_mesh.py

@@ -12,11 +12,12 @@
 from mpas_tools.mesh.creation.jigsaw_driver import jigsaw_driver
 from mpas_tools.mesh.creation.jigsaw_to_netcdf import jigsaw_to_netcdf
 from mpas_tools.viz.colormaps import register_sci_viz_colormaps
+from mpas_tools.logging import LoggingContext
 
 
 def build_spherical_mesh(cellWidth, lon, lat, earth_radius,
                          out_filename='base_mesh.nc', plot_cellWidth=True,
-                         dir='./'):
+                         dir='./', logger=None):
     """
     Build an MPAS mesh using JIGSAW with the given cell sizes as a function of
     latitude and longitude.
@@ -43,64 +44,70 @@ def build_spherical_mesh(cellWidth, lon, lat, earth_radius,
         The file name of the resulting MPAS mesh
 
     plot_cellWidth : bool, optional
-        Whether to produce a plot of ``cellWidth``. If so, it will be written to
-        ``cellWidthGlobal.png``.
+        Whether to produce a plot of ``cellWidth``. If so, it will be written
+        to ``cellWidthGlobal.png``.
 
     dir : str, optional
         A directory in which a temporary directory will be added with files
         produced during mesh conversion and then deleted upon completion.
+
+    logger : logging.Logger, optional
+        A logger for the output if not stdout
     """
 
-    da = xarray.DataArray(cellWidth,
-                          dims=['lat', 'lon'],
-                          coords={'lat': lat, 'lon': lon},
-                          name='cellWidth')
-    cw_filename = 'cellWidthVsLatLon.nc'
-    da.to_netcdf(cw_filename)
-    if plot_cellWidth:
-        register_sci_viz_colormaps()
-        fig = plt.figure(figsize=[16.0, 8.0])
-        ax = plt.axes(projection=ccrs.PlateCarree())
-        ax.set_global()
-        im = ax.imshow(cellWidth, origin='lower',
-                       transform=ccrs.PlateCarree(),
-                       extent=[-180, 180, -90, 90], cmap='3Wbgy5',
-                       zorder=0)
-        ax.add_feature(cartopy.feature.LAND, edgecolor='black', zorder=1)
-        gl = ax.gridlines(
-            crs=ccrs.PlateCarree(),
-            draw_labels=True,
-            linewidth=1,
-            color='gray',
-            alpha=0.5,
-            linestyle='-', zorder=2)
-        gl.top_labels = False
-        gl.right_labels = False
-        plt.title(
-            'Grid cell size, km, min: {:.1f} max: {:.1f}'.format(
-            cellWidth.min(),cellWidth.max()))
-        plt.colorbar(im, shrink=.60)
-        fig.canvas.draw()
-        plt.tight_layout()
-        plt.savefig('cellWidthGlobal.png', bbox_inches='tight')
-        plt.close()
-
-    print('Step 1. Generate mesh with JIGSAW')
-    jigsaw_driver(cellWidth, lon, lat, on_sphere=True,
-                  earth_radius=earth_radius)
-
-    print('Step 2. Convert triangles from jigsaw format to netcdf')
-    jigsaw_to_netcdf(msh_filename='mesh-MESH.msh',
-                     output_name='mesh_triangles.nc', on_sphere=True,
-                     sphere_radius=earth_radius)
-
-    print('Step 3. Convert from triangles to MPAS mesh')
-    write_netcdf(convert(xarray.open_dataset('mesh_triangles.nc'), dir=dir),
-                 out_filename)
+    with LoggingContext(__name__, logger=logger) as logger:
+
+        da = xarray.DataArray(cellWidth,
+                              dims=['lat', 'lon'],
+                              coords={'lat': lat, 'lon': lon},
+                              name='cellWidth')
+        cw_filename = 'cellWidthVsLatLon.nc'
+        da.to_netcdf(cw_filename)
+        if plot_cellWidth:
+            register_sci_viz_colormaps()
+            fig = plt.figure(figsize=[16.0, 8.0])
+            ax = plt.axes(projection=ccrs.PlateCarree())
+            ax.set_global()
+            im = ax.imshow(cellWidth, origin='lower',
+                           transform=ccrs.PlateCarree(),
+                           extent=[-180, 180, -90, 90], cmap='3Wbgy5',
+                           zorder=0)
+            ax.add_feature(cartopy.feature.LAND, edgecolor='black', zorder=1)
+            gl = ax.gridlines(
+                crs=ccrs.PlateCarree(),
+                draw_labels=True,
+                linewidth=1,
+                color='gray',
+                alpha=0.5,
+                linestyle='-', zorder=2)
+            gl.top_labels = False
+            gl.right_labels = False
+            plt.title(
+                'Grid cell size, km, min: {:.1f} max: {:.1f}'.format(
+                    cellWidth.min(),cellWidth.max()))
+            plt.colorbar(im, shrink=.60)
+            fig.canvas.draw()
+            plt.tight_layout()
+            plt.savefig('cellWidthGlobal.png', bbox_inches='tight')
+            plt.close()
+
+        logger.info('Step 1. Generate mesh with JIGSAW')
+        jigsaw_driver(cellWidth, lon, lat, on_sphere=True,
+                      earth_radius=earth_radius, logger=logger)
+
+        logger.info('Step 2. Convert triangles from jigsaw format to netcdf')
+        jigsaw_to_netcdf(msh_filename='mesh-MESH.msh',
+                         output_name='mesh_triangles.nc', on_sphere=True,
+                         sphere_radius=earth_radius)
+
+        logger.info('Step 3. Convert from triangles to MPAS mesh')
+        write_netcdf(convert(xarray.open_dataset('mesh_triangles.nc'), dir=dir,
+                             logger=logger),
+                     out_filename)
 
 
 def build_planar_mesh(cellWidth, x, y, geom_points, geom_edges,
-                      out_filename='base_mesh.nc'):
+                      out_filename='base_mesh.nc', logger=None):
     """
     Build a planar MPAS mesh
 
@@ -121,28 +128,31 @@ def build_planar_mesh(cellWidth, x, y, geom_points, geom_edges,
 
     out_filename : str, optional
         The file name of the resulting MPAS mesh
+
+    logger : logging.Logger, optional
+        A logger for the output if not stdout
     """
-    da = xarray.DataArray(cellWidth,
-                          dims=['y', 'x'],
-                          coords={'y': y, 'x': x},
-                          name='cellWidth')
-    cw_filename = 'cellWidthVsXY.nc'
-    da.to_netcdf(cw_filename)
-
-    print('Step 1. Generate mesh with JIGSAW')
-    jigsaw_driver(
-        cellWidth,
-        x,
-        y,
-        on_sphere=False,
-        geom_points=geom_points,
-        geom_edges=geom_edges)
-
-    print('Step 2. Convert triangles from jigsaw format to netcdf')
-    jigsaw_to_netcdf(msh_filename='mesh-MESH.msh',
-                     output_name='mesh_triangles.nc', on_sphere=False)
-
-    print('Step 3. Convert from triangles to MPAS mesh')
-    write_netcdf(convert(xarray.open_dataset('mesh_triangles.nc')),
-                 out_filename)
+
+    with LoggingContext(__name__, logger=logger) as logger:
+
+        da = xarray.DataArray(cellWidth,
+                              dims=['y', 'x'],
+                              coords={'y': y, 'x': x},
+                              name='cellWidth')
+        cw_filename = 'cellWidthVsXY.nc'
+        da.to_netcdf(cw_filename)
+
+        logger.info('Step 1. Generate mesh with JIGSAW')
+        jigsaw_driver(cellWidth, x, y, on_sphere=False,
+                      geom_points=geom_points, geom_edges=geom_edges,
+                      logger=logger)
+
+        logger.info('Step 2. Convert triangles from jigsaw format to netcdf')
+        jigsaw_to_netcdf(msh_filename='mesh-MESH.msh',
+                         output_name='mesh_triangles.nc', on_sphere=False)
+
+        logger.info('Step 3. Convert from triangles to MPAS mesh')
+        write_netcdf(convert(xarray.open_dataset('mesh_triangles.nc'),
+                             logger=logger),
+                     out_filename)