add better console logger (#2467)

fixes #2466

Author: Vladimir Kotal

opengrok-tools/src/main/python/opengrok_tools/deploy.py

@@ -29,6 +29,7 @@
 import tempfile
 from zipfile import ZipFile
 from shutil import copyfile
+from .utils.log import get_console_logger
 
 """
  deploy war file
@@ -109,12 +110,10 @@ def main():
 
     args = parser.parse_args()
 
+    loglevel = logging.INFO
     if args.debug:
-        logging.basicConfig(level=logging.DEBUG)
-    else:
-        logging.basicConfig()
-
-    logger = logging.getLogger(os.path.basename(sys.argv[0]))
+        loglevel = logging.DEBUG
+    logger = get_console_logger(os.path.basename(sys.argv[0]), loglevel)
 
     deploy_war(logger, args.source_war[0], args.target_war[0], args.config)
 

opengrok-tools/src/main/python/opengrok_tools/indexer.py

@@ -30,6 +30,7 @@
 
 from .utils.indexer import FindCtags, Indexer
 from .utils.java import get_javaparser
+from .utils.log import get_console_logger
 
 """
   opengrok.jar wrapper
@@ -46,12 +47,10 @@ def main():
 
     args = parser.parse_args()
 
+    loglevel = logging.INFO
     if args.debug:
-        logging.basicConfig(level=logging.DEBUG)
-    else:
-        logging.basicConfig()
-
-    logger = logging.getLogger(os.path.basename(sys.argv[0]))
+        loglevel = logging.DEBUG
+    logger = get_console_logger(os.path.basename(sys.argv[0]), loglevel)
 
     #
     # Since it is not possible to tell what kind of action is performed,

opengrok-tools/src/main/python/opengrok_tools/java.py

@@ -28,6 +28,7 @@
 import sys
 
 from .utils.java import Java, get_javaparser
+from .utils.log import get_console_logger
 
 
 def main():
@@ -38,12 +39,10 @@ def main():
 
     args = parser.parse_args()
 
+    loglevel = logging.INFO
     if args.debug:
-        logging.basicConfig(level=logging.DEBUG)
-    else:
-        logging.basicConfig()
-
-    logger = logging.getLogger(os.path.basename(sys.argv[0]))
+        loglevel = logging.DEBUG
+    logger = get_console_logger(os.path.basename(sys.argv[0]), loglevel)
 
     java = Java(args.options, logger=logger, java=args.java,
                 jar=args.jar, java_opts=args.java_opts,

opengrok-tools/src/main/python/opengrok_tools/mirror.py

@@ -39,6 +39,7 @@
 
 from .utils.filelock import Timeout, FileLock
 from .utils.hook import run_hook
+from .utils.log import get_console_logger
 from .utils.opengrok import get_repos, get_config_value, get_repo_type
 from .utils.readconfig import read_config
 from .utils.repofactory import get_repository
@@ -52,7 +53,7 @@
     print("Need Python 3, you are running {}".format(major_version))
     sys.exit(1)
 
-__version__ = "0.4"
+__version__ = "0.5"
 
 # "constants"
 HOOK_TIMEOUT_PROPERTY = 'hook_timeout'
@@ -134,12 +135,10 @@ def main():
                              'processing if not found.')
     args = parser.parse_args()
 
+    loglevel = logging.INFO
     if args.debug:
-        logging.basicConfig(level=logging.DEBUG)
-    else:
-        logging.basicConfig()
-
-    logger = logging.getLogger(os.path.basename(sys.argv[0]))
+        loglevel = logging.DEBUG
+    logger = get_console_logger(os.path.basename(sys.argv[0]), loglevel)
 
     if args.config:
         config = read_config(logger, args.config)
@@ -301,31 +300,21 @@ def main():
         if not logdir:
             logger.error("The logdir property is required in batch mode")
             sys.exit(1)
+
         logfile = os.path.join(logdir, args.project + ".log")
         logger.debug("Switching logging to the {} file".
                      format(logfile))
-        logging.shutdown()
-
-        # Remove the existing handler so that logger can be reconfigured.
-        for handler in logging.root.handlers[:]:
-            logging.root.removeHandler(handler)
 
-        logging.basicConfig(filename=logfile, filemode='a',
-                            level=logging.DEBUG if args.debug
-                            else logging.INFO)
-        logger = logging.getLogger(os.path.basename(sys.argv[0]))
-        handler = RotatingFileHandler(logfile, maxBytes=0,
+        logger = logger.getChild("rotating")
+        logger.setLevel(logging.DEBUG if args.debug
+                        else logging.INFO)
+        logger.propagate = False
+        handler = RotatingFileHandler(logfile, maxBytes=0, mode='a',
                                       backupCount=args.backupcount)
         formatter = logging.Formatter("%(asctime)s - %(levelname)s: "
                                       "%(message)s", '%m/%d/%Y %I:%M:%S %p')
         handler.setFormatter(formatter)
         handler.doRollover()
-        #
-        # Technically, adding a handler to the logger is not necessary
-        # since log rotation is done above using doRollover() however
-        # it is done anyway in case the handler changes to use implicit
-        # rotation in the future.
-        #
         logger.addHandler(handler)
 
     # We want this to be logged to the log file (if any).

opengrok-tools/src/main/python/opengrok_tools/projadm.py

@@ -37,6 +37,7 @@
 
 from .utils.command import Command
 from .utils.filelock import Timeout, FileLock
+from .utils.log import get_console_logger
 from .utils.opengrok import get_configuration, set_configuration, \
     add_project, delete_project, get_config_value
 from .utils.utils import get_command, is_web_uri
@@ -247,12 +248,10 @@ def main():
     # Setup logger as a first thing after parsing arguments so that it can be
     # used through the rest of the program.
     #
+    loglevel = logging.INFO
     if args.debug:
-        logging.basicConfig(level=logging.DEBUG)
-    else:
-        logging.basicConfig(format="%(message)s", level=logging.INFO)
-
-    logger = logging.getLogger(os.path.basename(sys.argv[0]))
+        loglevel = logging.DEBUG
+    logger = get_console_logger(os.path.basename(sys.argv[0]), loglevel)
 
     if args.nosourcedelete and not args.delete:
         logger.error("The no source delete option is only valid for delete")

opengrok-tools/src/main/python/opengrok_tools/reindex_project.py

@@ -30,6 +30,7 @@
 
 from .utils.indexer import Indexer
 from .utils.java import get_javaparser
+from .utils.log import get_console_logger
 from .utils.opengrok import get_configuration
 
 """
@@ -86,12 +87,10 @@ def main():
 
     args = parser.parse_args()
 
+    loglevel = logging.INFO
     if args.debug:
-        logging.basicConfig(level=logging.DEBUG)
-    else:
-        logging.basicConfig()
-
-    logger = logging.getLogger(os.path.basename(sys.argv[0]))
+        loglevel = logging.DEBUG
+    logger = get_console_logger(os.path.basename(sys.argv[0]), loglevel)
 
     # Make sure the log directory exists.
     if not os.path.isdir(args.directory):

opengrok-tools/src/main/python/opengrok_tools/sync.py

@@ -44,13 +44,14 @@
 from .utils.opengrok import list_indexed_projects, get_config_value
 from .utils.readconfig import read_config
 from .utils.utils import is_web_uri
+from .utils.log import get_console_logger
 
 major_version = sys.version_info[0]
 if (major_version < 3):
     print("Need Python 3, you are running {}".format(major_version))
     sys.exit(1)
 
-__version__ = "0.5"
+__version__ = "0.6"
 
 
 def worker(base):
@@ -80,13 +81,8 @@ def main():
                         help='List of projects to process')
     parser.add_argument('-I', '--indexed', action='store_true',
                         help='Sync indexed projects only')
-
-    group2 = parser.add_mutually_exclusive_group()
-    group2.add_argument('-D', '--debug', action='store_true',
+    parser.add_argument('-D', '--debug', action='store_true',
                         help='Enable debug prints')
-    group2.add_argument('-p', '--logplain', action='store_true',
-                        help='log plain messages')
-
     parser.add_argument('-i', '--ignore_errors', nargs='*',
                         help='ignore errors from these projects')
     parser.add_argument('-c', '--config', required=True,
@@ -95,15 +91,10 @@ def main():
                         help='URI of the webapp with context path')
     args = parser.parse_args()
 
+    loglevel = logging.INFO
     if args.debug:
-        logging.basicConfig(level=logging.DEBUG)
-    else:
-        if args.logplain:
-            logging.basicConfig(format="%(message)s")
-        else:
-            logging.basicConfig()
-
-    logger = logging.getLogger(os.path.basename(sys.argv[0]))
+        loglevel = logging.DEBUG
+    logger = get_console_logger(os.path.basename(sys.argv[0]), loglevel)
 
     uri = args.uri
     if not is_web_uri(uri):
@@ -112,7 +103,7 @@ def main():
     logger.debug("web application URI = {}".format(uri))
 
     # Changing working directory to root will avoid problems when running
-    # via sudo/su.
+    # programs via sudo/su.
     try:
         os.chdir("/")
     except OSError:

opengrok-tools/src/main/python/opengrok_tools/utils/log.py

@@ -0,0 +1,58 @@
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# See LICENSE.txt included in this distribution for the specific
+# language governing permissions and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at LICENSE.txt.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+#
+
+import logging
+import sys
+
+
+def get_console_logger(name, level=logging.INFO, format='%(message)s'):
+    """
+    Get logger that logs logging.ERROR and higher to stderr, the rest
+    to stdout. For logging.DEBUG level more verbose format is used.
+
+    :param name: name of the logger
+    :param level: base logging level
+    :param format: format string to use
+    :return: logger
+    """
+    if level == logging.DEBUG:
+        format = '%(asctime)s %(levelname)8s %(name)s | %(message)s'
+
+    formatter = logging.Formatter(format)
+
+    stderr_handler = logging.StreamHandler(stream=sys.stderr)
+    stderr_handler.setFormatter(formatter)
+
+    stdout_handler = logging.StreamHandler(stream=sys.stdout)
+    stdout_handler.setFormatter(formatter)
+
+    stderr_handler.addFilter(lambda rec: rec.levelno >= logging.ERROR)
+    stdout_handler.addFilter(lambda rec: rec.levelno <= logging.INFO)
+
+    logger = logging.getLogger(name)
+    logger.setLevel(level)
+    logger.propagate = False
+    logger.addHandler(stdout_handler)
+    logger.addHandler(stderr_handler)
+
+    return logger