make get_repos_for_project(), get_repository use kwargs

add tests

Author: Vladimir Kotal

opengrok-tools/src/main/python/opengrok_tools/scm/repofactory.py

@@ -18,7 +18,7 @@
 #
 
 #
-# Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
 #
 
 import logging
@@ -33,14 +33,14 @@
 logger = logging.getLogger(__name__)
 
 
-def get_repository(path, repo_type, project, commands, env, hooks,
-                   timeout):
+def get_repository(path, repo_type, project,
+                   commands=None, env=None, hooks=None, timeout=None):
     """
     :param path: full path
     :param repo_type: repository type
     :param project: project name
-    :param commands: commands list
-    :param env: environment varibables dictionary
+    :param commands: commands dictionary with paths to SCM utilities
+    :param env: environment variables dictionary
     :param hooks: hook dictionary
     :param timeout: timeout in seconds
     :return: a Repository derived object according to the type specified
@@ -62,7 +62,7 @@ def get_repository(path, repo_type, project, commands, env, hooks,
         return TeamwareRepository(logger, path, project,
                                   commands.get("teamware"),
                                   env, hooks, timeout)
-    elif repo_lower.lower() == "cvs":
+    elif repo_lower == "cvs":
         return CVSRepository(logger, path, project,
                              commands.get("cvs"),
                              env, hooks, timeout)

opengrok-tools/src/main/python/opengrok_tools/utils/mirror.py

@@ -61,14 +61,18 @@
 DISABLED_CMD_PROPERTY = 'disabled_command'
 
 
-def get_repos_for_project(project_name, ignored_repos, uri, source_root,
-                          **kwargs):
+def get_repos_for_project(project_name, uri, source_root,
+                          ignored_repos=None,
+                          commands=None, proxy=None, command_timeout=None):
     """
     :param project_name: project name
-    :param ignored_repos: list of ignored repositories
     :param uri: web application URI
     :param source_root source root
-    :param kwargs: argument dictionary
+    :param ignored_repos: list of ignored repositories
+    :param commands: dictionary of commands - paths to SCM programs
+    :param proxy: dictionary of proxy servers - to be used as environment
+                  variables
+    :param command_timeout: command timeout value in seconds
     :return: list of Repository objects
     """
 
@@ -78,10 +82,11 @@ def get_repos_for_project(project_name, ignored_repos, uri, source_root,
     for repo_path in get_repos(logger, project_name, uri):
         logger.debug("Repository path = {}".format(repo_path))
 
-        r_path = os.path.relpath(repo_path, '/' + project_name)
-        if any(map(lambda repo: fnmatch.fnmatch(r_path, repo), ignored_repos)):
-            logger.info("repository {} ignored".format(repo_path))
-            continue
+        if ignored_repos:
+            r_path = os.path.relpath(repo_path, '/' + project_name)
+            if any(map(lambda repo: fnmatch.fnmatch(r_path, repo), ignored_repos)):
+                logger.info("repository {} ignored".format(repo_path))
+                continue
 
         repo_type = get_repo_type(logger, repo_path, uri)
         if not repo_type:
@@ -92,14 +97,20 @@ def get_repos_for_project(project_name, ignored_repos, uri, source_root,
 
         repo = None
         try:
-            # Not joining the path since the form of repo_path is absolute.
-            repo = get_repository(source_root + repo_path,
+            # The OpenGrok convention is that the form of repo_path is absolute
+            # so joining would the paths would actually spoil things. Hence, be
+            # careful.
+            if repo_path.startswith(os.path.sep):
+                path = source_root + repo_path
+            else:
+                path = os.path.join(source_root, repo_path)
+
+            repo = get_repository(path,
                                   repo_type,
                                   project_name,
-                                  kwargs[COMMANDS_PROPERTY],
-                                  kwargs[PROXY_PROPERTY],
-                                  None,
-                                  kwargs[CMD_TIMEOUT_PROPERTY])
+                                  env=proxy,
+                                  timeout=command_timeout,
+                                  commands=commands)
         except (RepositoryException, OSError) as e:
             logger.error("Cannot get repository for {}: {}".
                          format(repo_path, e))
@@ -361,9 +372,9 @@ def mirror_project(config, project_name, check_changes, uri,
     # something is not right, avoiding any needless pre-hook run.
     #
     repos = get_repos_for_project(project_name,
-                                  ignored_repos,
                                   uri,
                                   source_root,
+                                  ignored_repos=ignored_repos,
                                   commands=config.
                                   get(COMMANDS_PROPERTY),
                                   proxy=proxy,

opengrok-tools/src/test/python/test_mirror.py

@@ -35,7 +35,7 @@
 
 from opengrok_tools.scm.repofactory import get_repository
 from opengrok_tools.utils.mirror import check_project_configuration, \
-    check_configuration, mirror_project, run_command, \
+    check_configuration, mirror_project, run_command, get_repos_for_project, \
     HOOKS_PROPERTY, PROXY_PROPERTY, IGNORED_REPOS_PROPERTY, \
     PROJECTS_PROPERTY, DISABLED_CMD_PROPERTY, DISABLED_PROPERTY, \
     CMD_TIMEOUT_PROPERTY, HOOK_TIMEOUT_PROPERTY
@@ -152,8 +152,7 @@ def raise_for_status():
 
         def mock_get_repos(*args, **kwargs):
             return [get_repository(cloned_repo_path,
-                                   "git", project_name,
-                                   None, None, None, None)]
+                                   "git", project_name)]
 
         def mock_get(*args, **kwargs):
             return MockResponse()
@@ -280,3 +279,43 @@ def test_mirror_project(config):
 
         test_mirror_project(global_config_1)
         test_mirror_project(global_config_2)
+
+
+def test_get_repos_for_project(monkeypatch):
+    """
+    Test argument passing between get_repos_for_project() and get_repository()
+    """
+    project_name = 'foo'
+    proxy_dict = {}
+    commands = {}
+    timeout = 314159
+    test_repo = "/" + project_name
+
+    def mock_get_repos(*args):
+        return [test_repo]
+
+    def mock_get_repo_type(*args):
+        return "Git"
+
+    with tempfile.TemporaryDirectory() as source_root:
+        # Note that it is actually not necessary to create real
+        # Git repository for the test to work. This is due to
+        # the way how Repository objects are created.
+        with monkeypatch.context() as m:
+            m.setattr("opengrok_tools.utils.mirror.get_repos",
+                      mock_get_repos)
+            m.setattr("opengrok_tools.utils.mirror.get_repo_type",
+                      mock_get_repo_type)
+
+            repos = get_repos_for_project(project_name, None, source_root,
+                                          commands=commands,
+                                          proxy=proxy_dict,
+                                          command_timeout=timeout)
+            print(repos)
+            assert len(repos) == 1
+
+            # Now ignore the repository
+            repos = get_repos_for_project(project_name, None, source_root,
+                                          ignored_repos=['.'])
+            print(repos)
+            assert len(repos) == 0

opengrok-tools/src/test/python/test_repofactory.py

@@ -41,7 +41,7 @@ def test_repofactory_timeout():
         project_name = "foo"    # does not matter for this test
         repo = get_repository(repo_path,
                               "git", project_name,
-                              None, None, None, timeout)
+                              timeout=timeout)
         assert repo is not None
         assert isinstance(repo, GitRepository)
         assert repo.timeout == timeout