fixups to mirroring functions (#2747)

Vladimir Kotal · web-flow · commit 339e8d05f386 · 2019-04-10T15:14:38.000+02:00
diff --git a/opengrok-tools/src/main/python/opengrok_tools/config_merge.py b/opengrok-tools/src/main/python/opengrok_tools/config_merge.py
@@ -26,7 +26,7 @@
 import sys
 
 from .utils.java import Java
-from .utils.log import print_exc_exit
+from .utils.log import fatal
 from .utils.parsers import get_javaparser
 
 """
@@ -42,7 +42,7 @@ def main():
     try:
         args = parser.parse_args()
     except ValueError as e:
-        print_exc_exit(e)
+        fatal(e)
 
     # Avoid using utils.log.get_console_level() since the stdout of the program
     # is interpreted as data.
diff --git a/opengrok-tools/src/main/python/opengrok_tools/deploy.py b/opengrok-tools/src/main/python/opengrok_tools/deploy.py
@@ -29,7 +29,7 @@
 from zipfile import ZipFile
 
 from .utils.log import get_console_logger, get_class_basename, \
-    print_exc_exit
+    fatal
 from .utils.parsers import get_baseparser
 
 """
@@ -111,7 +111,7 @@ def main():
     try:
         args = parser.parse_args()
     except ValueError as e:
-        print_exc_exit(e)
+        fatal(e)
 
     logger = get_console_logger(get_class_basename(), args.loglevel)
 
diff --git a/opengrok-tools/src/main/python/opengrok_tools/groups.py b/opengrok-tools/src/main/python/opengrok_tools/groups.py
@@ -25,7 +25,7 @@
 import sys
 
 from .utils.java import Java
-from .utils.log import get_console_logger, get_class_basename, print_exc_exit
+from .utils.log import get_console_logger, get_class_basename, fatal
 from .utils.parsers import get_javaparser
 
 """
@@ -41,7 +41,7 @@ def main():
     try:
         args = parser.parse_args()
     except ValueError as e:
-        print_exc_exit(e)
+        fatal(e)
 
     logger = get_console_logger(get_class_basename(), args.loglevel)
 
diff --git a/opengrok-tools/src/main/python/opengrok_tools/indexer.py b/opengrok-tools/src/main/python/opengrok_tools/indexer.py
@@ -27,7 +27,7 @@
 import sys
 
 from .utils.indexer import FindCtags, Indexer
-from .utils.log import get_console_logger, get_class_basename, print_exc_exit
+from .utils.log import get_console_logger, get_class_basename, fatal
 from .utils.parsers import get_javaparser
 
 """
@@ -46,7 +46,7 @@ def main():
     try:
         args = parser.parse_args()
     except ValueError as e:
-        print_exc_exit(e)
+        fatal(e)
 
     logger = get_console_logger(get_class_basename(), args.loglevel)
 
diff --git a/opengrok-tools/src/main/python/opengrok_tools/java.py b/opengrok-tools/src/main/python/opengrok_tools/java.py
@@ -26,7 +26,7 @@
 import sys
 
 from .utils.java import Java
-from .utils.log import get_console_logger, get_class_basename, print_exc_exit
+from .utils.log import get_console_logger, get_class_basename, fatal
 from .utils.parsers import get_javaparser
 
 
@@ -39,7 +39,7 @@ def main():
     try:
         args = parser.parse_args()
     except ValueError as e:
-        print_exc_exit(e)
+        fatal(e)
 
     logger = get_console_logger(get_class_basename(), args.loglevel)
 
diff --git a/opengrok-tools/src/main/python/opengrok_tools/mirror.py b/opengrok-tools/src/main/python/opengrok_tools/mirror.py
@@ -39,7 +39,7 @@
 from filelock import Timeout, FileLock
 
 from .utils.log import get_console_logger, get_class_basename, \
-    print_exc_exit, get_batch_logger
+    fatal, get_batch_logger
 from .utils.opengrok import get_config_value, list_indexed_projects
 from .utils.parsers import get_baseparser
 from .utils.readconfig import read_config
@@ -50,8 +50,7 @@
 
 major_version = sys.version_info[0]
 if major_version < 3:
-    print("Need Python 3, you are running {}".format(major_version))
-    sys.exit(1)
+    fatal("Need Python 3, you are running {}".format(major_version))
 
 __version__ = "0.8"
 
@@ -99,30 +98,26 @@ def main():
     try:
         args = parser.parse_args()
     except ValueError as e:
-        print_exc_exit(e)
+        fatal(e)
 
     logger = get_console_logger(get_class_basename(), args.loglevel)
 
     if len(args.project) > 0 and args.all:
-        logger.fatal("Cannot use both project list and -a/--all")
-        sys.exit(1)
+        fatal("Cannot use both project list and -a/--all")
 
     if not args.all and len(args.project) == 0:
-        logger.fatal("Need at least one project or --all")
-        sys.exit(1)
+        fatal("Need at least one project or --all")
 
     if args.config:
         config = read_config(logger, args.config)
         if config is None:
-            logger.fatal("Cannot read config file from {}".format(args.config))
-            sys.exit(1)
+            fatal("Cannot read config file from {}".format(args.config))
     else:
         config = {}
 
     uri = args.uri
     if not is_web_uri(uri):
-        logger.fatal("Not a URI: {}".format(uri))
-        sys.exit(1)
+        fatal("Not a URI: {}".format(uri))
     logger.debug("web application URI = {}".format(uri))
 
     if not check_configuration(config):
@@ -153,9 +148,8 @@ def main():
     if args.batch:
         logdir = config.get(LOGDIR_PROPERTY)
         if not logdir:
-            logger.fatal("The {} property is required in batch mode".
-                         format(LOGDIR_PROPERTY))
-            sys.exit(1)
+            fatal("The {} property is required in batch mode".
+                  format(LOGDIR_PROPERTY))
 
     projects = args.project
     if len(projects) == 1:
diff --git a/opengrok-tools/src/main/python/opengrok_tools/projadm.py b/opengrok-tools/src/main/python/opengrok_tools/projadm.py
@@ -38,7 +38,7 @@
 
 from .utils.command import Command
 from .utils.log import get_console_logger, get_class_basename, \
-    print_exc_exit
+    fatal
 from .utils.opengrok import get_configuration, set_configuration, \
     add_project, delete_project, get_config_value
 from .utils.parsers import get_baseparser
@@ -247,7 +247,7 @@ def main():
     try:
         args = parser.parse_args()
     except ValueError as e:
-        print_exc_exit(e)
+        fatal(e)
 
     doit = not args.noop
     configmerge = None
diff --git a/opengrok-tools/src/main/python/opengrok_tools/reindex_project.py b/opengrok-tools/src/main/python/opengrok_tools/reindex_project.py
@@ -28,7 +28,7 @@
 import tempfile
 
 from .utils.indexer import Indexer
-from .utils.log import get_console_logger, get_class_basename, print_exc_exit
+from .utils.log import get_console_logger, get_class_basename, fatal
 from .utils.opengrok import get_configuration
 from .utils.parsers import get_javaparser
 
@@ -87,7 +87,7 @@ def main():
     try:
         args = parser.parse_args()
     except ValueError as e:
-        print_exc_exit(e)
+        fatal(e)
 
     logger = get_console_logger(get_class_basename(), args.loglevel)
 
diff --git a/opengrok-tools/src/main/python/opengrok_tools/sync.py b/opengrok-tools/src/main/python/opengrok_tools/sync.py
@@ -37,7 +37,7 @@
 from filelock import Timeout, FileLock
 
 from .utils.commandsequence import CommandSequence, CommandSequenceBase
-from .utils.log import get_console_logger, get_class_basename, print_exc_exit
+from .utils.log import get_console_logger, get_class_basename, fatal
 from .utils.opengrok import list_indexed_projects, get_config_value
 from .utils.parsers import get_baseparser
 from .utils.readconfig import read_config
@@ -95,7 +95,7 @@ def main():
     try:
         args = parser.parse_args()
     except ValueError as e:
-        print_exc_exit(e)
+        fatal(e)
 
     logger = get_console_logger(get_class_basename(), args.loglevel)
 
diff --git a/opengrok-tools/src/main/python/opengrok_tools/utils/log.py b/opengrok-tools/src/main/python/opengrok_tools/utils/log.py
@@ -28,13 +28,12 @@
 from logging.handlers import RotatingFileHandler
 
 
-def print_exc_exit(e):
+def fatal(msg):
     """
-    Print exception and exit
-    :param e: exception
-    :return: nothing
+    Print message to standard error output and exit
+    :param msg: message
     """
-    print(e, file=sys.stderr)
+    print(msg, file=sys.stderr)
     sys.exit(1)
 
 
diff --git a/opengrok-tools/src/main/python/opengrok_tools/utils/mirror.py b/opengrok-tools/src/main/python/opengrok_tools/utils/mirror.py
@@ -27,7 +27,7 @@
 import fnmatch
 import logging
 
-from .utils import is_exe, check_create_dir, get_int, diff_list
+from .utils import is_exe, check_create_dir, get_int
 from .opengrok import get_repos, get_repo_type
 from .hook import run_hook
 
@@ -50,28 +50,33 @@
 # This is a special exit code that is recognized by sync.py to terminate
 # the processing of the command sequence.
 CONTINUE_EXITVAL = 2
+SUCCESS_EXITVAL = 0
+FAILURE_EXITVAL = 1
 
 
-def get_repos_for_project(project_name, ignored_repos, **kwargs):
+def get_repos_for_project(project_name, ignored_repos, uri, source_root,
+                          **kwargs):
     """
     :param project_name: project name
     :param ignored_repos: list of ignored repositories
+    :param uri: web application URI
+    :param source_root source root
     :param kwargs: argument dictionary
     :return: list of Repository objects
     """
 
     logger = logging.getLogger(__name__)
 
     repos = []
-    for repo_path in get_repos(logger, project_name, kwargs['uri']):
+    for repo_path in get_repos(logger, project_name, uri):
         logger.debug("Repository path = {}".format(repo_path))
 
         r_path = os.path.relpath(repo_path, '/' + project_name)
         if any(map(lambda repo: fnmatch.fnmatch(r_path, repo), ignored_repos)):
             logger.info("repository {} ignored".format(repo_path))
             continue
 
-        repo_type = get_repo_type(logger, repo_path, kwargs['uri'])
+        repo_type = get_repo_type(logger, repo_path, uri)
         if not repo_type:
             raise RepositoryException("cannot determine type of repository {}".
                                       format(repo_path))
@@ -81,7 +86,7 @@ def get_repos_for_project(project_name, ignored_repos, **kwargs):
         repo = None
         try:
             # Not joining the path since the form of repo_path is absolute.
-            repo = get_repository(kwargs['source_root'] + repo_path,
+            repo = get_repository(source_root + repo_path,
                                   repo_type,
                                   project_name,
                                   kwargs[COMMANDS_PROPERTY],
@@ -226,15 +231,15 @@ def mirror_project(config, project_name, check_incoming, uri,
     # Cache the repositories first. This way it will be known that
     # something is not right, avoiding any needless pre-hook run.
     #
-    ret = 0
+    ret = SUCCESS_EXITVAL
     repos = get_repos_for_project(project_name,
                                   ignored_repos,
+                                  uri,
+                                  source_root,
                                   commands=config.
                                   get(COMMANDS_PROPERTY),
                                   proxy=proxy,
-                                  command_timeout=command_timeout,
-                                  source_root=source_root,
-                                  uri=uri)
+                                  command_timeout=command_timeout)
     if not repos:
         logger.info("No repositories for project {}".
                     format(project_name))
@@ -253,7 +258,7 @@ def mirror_project(config, project_name, check_incoming, uri,
             except RepositoryException:
                 logger.error('Cannot determine incoming changes for '
                              'repository {}'.format(repo))
-                return 1
+                return FAILURE_EXITVAL
 
         if not got_incoming:
             logger.info('No incoming changes for repositories in '
@@ -268,7 +273,7 @@ def mirror_project(config, project_name, check_incoming, uri,
                     hook_timeout) != 0:
             logger.error("pre hook failed for project {}".
                          format(project_name))
-            return 1
+            return FAILURE_EXITVAL
 
     #
     # If one of the repositories fails to sync, the whole project sync
@@ -280,7 +285,7 @@ def mirror_project(config, project_name, check_incoming, uri,
         if repo.sync() != 0:
             logger.error("failed to synchronize repository {}".
                          format(repo.path))
-            ret = 1
+            ret = FAILURE_EXITVAL
 
     if posthook:
         logger.info("Running post hook")
@@ -289,7 +294,7 @@ def mirror_project(config, project_name, check_incoming, uri,
                     hook_timeout) != 0:
             logger.error("post hook failed for project {}".
                          format(project_name))
-            return 1
+            return FAILURE_EXITVAL
 
     return ret
 
@@ -314,10 +319,8 @@ def check_project_configuration(multiple_project_config, hookdir=False,
     if not multiple_project_config:
         return True
 
-    for project_name in multiple_project_config.keys():
-        project_config = multiple_project_config.get(project_name)
-        diff = diff_list(project_config.keys(),
-                         known_project_tunables)
+    for project_name, project_config in multiple_project_config.items():
+        diff = set(project_config.keys()).difference(known_project_tunables)
         if diff:
             logger.error("unknown project configuration option(s) '{}' "
                          "for project {}".format(diff, project_name))
@@ -381,7 +384,7 @@ def check_configuration(config):
     global_tunables = [HOOKDIR_PROPERTY, PROXY_PROPERTY, LOGDIR_PROPERTY,
                        COMMANDS_PROPERTY, PROJECTS_PROPERTY,
                        HOOK_TIMEOUT_PROPERTY, CMD_TIMEOUT_PROPERTY]
-    diff = diff_list(config.keys(), global_tunables)
+    diff = set(config.keys()).difference(global_tunables)
     if diff:
         logger.error("unknown global configuration option(s): '{}'"
                      .format(diff))
diff --git a/opengrok-tools/src/main/python/opengrok_tools/utils/utils.py b/opengrok-tools/src/main/python/opengrok_tools/utils/utils.py
@@ -94,14 +94,6 @@ def get_int(logger, name, value):
         return None
 
 
-def diff_list(first, second):
-    """
-    Get difference of lists.
-    """
-    second = set(second)
-    return [item for item in first if item not in second]
-
-
 def is_web_uri(url):
     """
     Check if given string is web URL.
