make it possible to mirror more projects at once (#2731)

Author: Vladimir Kotal

opengrok-tools/src/main/python/opengrok_tools/mirror.py

@@ -22,7 +22,7 @@
 #
 
 from ..utils.command import Command
-from .repository import Repository
+from .repository import Repository, RepositoryException
 from shutil import which
 
 
@@ -37,8 +37,7 @@ def __init__(self, logger, path, project, command, env, hooks, timeout):
             self.command = which("cvs")
 
         if not self.command:
-            self.logger.error("Cannot get cvs command")
-            raise OSError
+            raise RepositoryException("Cannot get cvs command")
 
     def reposync(self):
         hg_command = [self.command, "update", "-dP"]

opengrok-tools/src/main/python/opengrok_tools/scm/cvs.py

@@ -37,8 +37,7 @@ def __init__(self, logger, path, project, command, env, hooks, timeout):
             self.command = which("git")
 
         if not self.command:
-            self.logger.error("Cannot get git command")
-            raise OSError
+            raise RepositoryException("Cannot get git command")
 
     def reposync(self):
         git_command = [self.command, "pull", "--ff-only"]

opengrok-tools/src/main/python/opengrok_tools/scm/git.py

@@ -37,8 +37,7 @@ def __init__(self, logger, path, project, command, env, hooks, timeout):
             self.command = which("hg")
 
         if not self.command:
-            self.logger.error("Cannot get hg command")
-            raise OSError
+            raise RepositoryException("Cannot get hg command")
 
     def get_branch(self):
         hg_command = [self.command, "branch"]

opengrok-tools/src/main/python/opengrok_tools/scm/mercurial.py

@@ -37,8 +37,7 @@ def __init__(self, logger, path, project, command, env, hooks, timeout):
             self.command = which("repo")
 
         if not self.command:
-            self.logger.error("Cannot get repo command")
-            raise OSError
+            raise RepositoryException("Cannot get repo command")
 
     def reposync(self):
         repo_command = [self.command, "sync", "-cf"]

opengrok-tools/src/main/python/opengrok_tools/scm/repo.py

@@ -18,24 +18,35 @@
 #
 
 #
-# Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
 #
 
+import logging
+
 from .cvs import CVSRepository
 from .git import GitRepository
 from .mercurial import MercurialRepository
 from .repo import RepoRepository
 from .svn import SubversionRepository
 from .teamware import TeamwareRepository
 
+logger = logging.getLogger(__name__)
+
 
-def get_repository(logger, path, repo_type, project, commands, env, hooks,
+def get_repository(path, repo_type, project, commands, env, hooks,
                    timeout):
     """
-    Repository factory. Returns a Repository derived object according
-    to the type specified or None if given repository type cannot
-    be found.
+    :param path: full path
+    :param repo_type: repository type
+    :param project: project name
+    :param commands: commands list
+    :param env: environment varibables dictionary
+    :param hooks: hook dictionary
+    :param timeout: timeout in seconds
+    :return: a Repository derived object according to the type specified
+    or None if given repository type cannot be found.
     """
+
     repo_lower = repo_type.lower()
 
     logger.debug("Constructing repo object for path {}".format(path))
@@ -68,4 +79,6 @@ def get_repository(logger, path, repo_type, project, commands, env, hooks,
                               commands.get("repo"),
                               env, hooks, timeout)
     else:
+        logger.warning("Unsupported repository type {}: {}".
+                       format(repo_type, path))
         return None

opengrok-tools/src/main/python/opengrok_tools/scm/repofactory.py

@@ -22,7 +22,7 @@
 #
 
 from ..utils.command import Command
-from .repository import Repository
+from .repository import Repository, RepositoryException
 from shutil import which
 
 
@@ -37,8 +37,7 @@ def __init__(self, logger, path, project, command, env, hooks, timeout):
             self.command = which("svn")
 
         if not self.command:
-            self.logger.error("Cannot get svn command")
-            raise OSError
+            raise RepositoryException("Cannot get svn command")
 
     def reposync(self):
         svn_command = [self.command]

opengrok-tools/src/main/python/opengrok_tools/scm/svn.py

@@ -22,7 +22,7 @@
 #
 
 from ..utils.command import Command
-from .repository import Repository
+from .repository import Repository, RepositoryException
 import os
 
 
@@ -41,21 +41,19 @@ def __init__(self, logger, path, project, command, env, hooks, timeout):
         #
         if command:
             if not os.path.isdir(command):
-                self.logger.error("Cannot construct Teamware repository:"
-                                  " {} is not a directory".format(command))
-                raise OSError
+                raise RepositoryException("Cannot construct Teamware "
+                                          "repository: {} is not a "
+                                          "directory".format(command))
 
             try:
                 path = os.environ['PATH']
             except KeyError:
-                self.logger.error("Cannot get PATH env var")
-                raise OSError
+                raise OSError("Cannot get PATH env var")
 
             path += ":" + command
             self.env['PATH'] = path
         else:
-            self.logger.error("Cannot get path to Teamware commands")
-            raise OSError
+            raise RepositoryException("Cannot get path to Teamware commands")
 
     def reposync(self):
         #

opengrok-tools/src/main/python/opengrok_tools/scm/teamware.py

@@ -18,12 +18,14 @@
 #
 
 #
-# Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
 #
 
 import logging
 import sys
+import os
 import argparse
+from logging.handlers import RotatingFileHandler
 
 
 def print_exc_exit(e):
@@ -92,7 +94,8 @@ def get_class_basename():
     return __name__.split('.')[0]
 
 
-def get_console_logger(name, level=logging.INFO, format='%(message)s'):
+def get_console_logger(name=__name__, level=logging.INFO,
+                       format='%(message)s'):
     """
     Get logger that logs logging.ERROR and higher to stderr, the rest
     to stdout. For logging.DEBUG level more verbose format is used.
@@ -122,7 +125,41 @@ def get_console_logger(name, level=logging.INFO, format='%(message)s'):
     logger = logging.getLogger(name)
     logger.setLevel(level)
     logger.propagate = False
+    logger.handlers = []
     logger.addHandler(stdout_handler)
     logger.addHandler(stderr_handler)
 
     return logger
+
+
+def get_batch_logger(logdir, project_name, loglevel, backupcount,
+                     name=__name__):
+    """
+    Get rotating file logger for storing logs of mirroring of given project.
+    :param logdir: log directory
+    :param project_name: name of the project
+    :param loglevel: logging level
+    :param backupcount count of log files to keep around
+    :param name name of the logger
+    :return logger
+    """
+
+    logger = logging.getLogger(name)
+
+    logfile = os.path.join(logdir, project_name + ".log")
+    logger.debug("Switching logging to the {} file".
+                 format(logfile))
+
+    handler = RotatingFileHandler(logfile, maxBytes=0, mode='a',
+                                  backupCount=backupcount)
+    formatter = logging.Formatter("%(asctime)s - %(levelname)s: "
+                                  "%(message)s", '%m/%d/%Y %I:%M:%S %p')
+    handler.setFormatter(formatter)
+    handler.doRollover()
+
+    logger.setLevel(loglevel)
+    logger.propagate = False
+    logger.handlers = []
+    logger.addHandler(handler)
+
+    return logger