add command handling to mirror.py

- refactor disabled command handling
- add tests

fixes #2973

Author: Vladimir Kotal

opengrok-tools/setup.py

@@ -47,6 +47,8 @@ def readme():
         'pytest',
         'GitPython',
         'pytest-xdist',
+        'mockito',
+        'pytest-mockito',
     ],
     entry_points={
         'console_scripts': [

opengrok-tools/src/main/python/opengrok_tools/utils/commandsequence.py

@@ -29,6 +29,7 @@
     SUCCESS_EXITVAL
 )
 from .restful import call_rest_api
+from .patterns import PROJECT_SUBST, COMMAND_PROPERTY
 import re
 
 
@@ -72,7 +73,6 @@ def fill(self, retcodes, outputs, failed):
 
 
 class CommandSequence(CommandSequenceBase):
-    PROJECT_SUBST = '%PROJECT%'
 
     re_program = re.compile('ERROR[:]*\\s+')
 
@@ -87,11 +87,11 @@ def run_command(self, command):
         """
         Execute a command and return its return code.
         """
-        command_args = command.get("command")
+        command_args = command.get(COMMAND_PROPERTY)
         cmd = Command(command_args,
                       env_vars=command.get("env"),
                       resource_limits=command.get("limits"),
-                      args_subst={self.PROJECT_SUBST: self.name},
+                      args_subst={PROJECT_SUBST: self.name},
                       args_append=[self.name], excl_subst=True)
         cmd.execute()
         self.retcodes[str(cmd)] = cmd.getretcode()
@@ -116,8 +116,8 @@ def run(self):
         """
 
         for command in self.commands:
-            if is_web_uri(command.get("command")[0]):
-                call_rest_api(command, self.PROJECT_SUBST, self.name)
+            if is_web_uri(command.get(COMMAND_PROPERTY)[0]):
+                call_rest_api(command, PROJECT_SUBST, self.name)
             else:
                 retcode = self.run_command(command)
 
@@ -155,14 +155,14 @@ def run_cleanup(self):
             return
 
         for cleanup_cmd in self.cleanup:
-            if is_web_uri(cleanup_cmd.get("command")[0]):
-                call_rest_api(cleanup_cmd, self.PROJECT_SUBST, self.name)
+            if is_web_uri(cleanup_cmd.get(COMMAND_PROPERTY)[0]):
+                call_rest_api(cleanup_cmd, PROJECT_SUBST, self.name)
             else:
-                command_args = cleanup_cmd.get("command")
+                command_args = cleanup_cmd.get(COMMAND_PROPERTY)
                 self.logger.debug("Running cleanup command '{}'".
                                   format(command_args))
                 cmd = Command(command_args,
-                              args_subst={self.PROJECT_SUBST: self.name},
+                              args_subst={PROJECT_SUBST: self.name},
                               args_append=[self.name], excl_subst=True)
                 cmd.execute()
                 if cmd.getretcode() != SUCCESS_EXITVAL:

opengrok-tools/src/main/python/opengrok_tools/utils/mirror.py

@@ -35,10 +35,13 @@
     CONTINUE_EXITVAL,
     SUCCESS_EXITVAL
 )
-from .utils import is_exe, check_create_dir, get_int
+from .patterns import PROJECT_SUBST, COMMAND_PROPERTY
+from .utils import is_exe, check_create_dir, get_int, is_web_uri
 from .webutil import get
 from .opengrok import get_repos, get_repo_type, get_uri
 from .hook import run_hook
+from .command import Command
+from .restful import call_rest_api
 
 from ..scm.repofactory import get_repository
 from ..scm.repository import RepositoryException
@@ -55,6 +58,7 @@
 HOOKS_PROPERTY = 'hooks'
 LOGDIR_PROPERTY = 'logdir'
 PROJECTS_PROPERTY = 'projects'
+DISABLED_CMD_PROPERTY = 'disabled_command'
 
 
 def get_repos_for_project(project_name, ignored_repos, uri, source_root,
@@ -277,6 +281,41 @@ def process_changes(repos, project_name, uri):
     return SUCCESS_EXITVAL
 
 
+def run_command(cmd, project_name):
+    cmd.execute()
+    if cmd.getretcode() != 0:
+        logger = logging.getLogger(__name__)
+
+        logger.error("Command for disabled project '{}' failed "
+                     "with error code {}: {}".
+                     format(project_name, cmd.getretcode(),
+                            cmd.getoutputstr()))
+
+
+def handle_disabled_project(config, project_name):
+    disabled_command = config.get(DISABLED_CMD_PROPERTY)
+    if disabled_command:
+        logger = logging.getLogger(__name__)
+
+        logger.debug("Calling disabled command: {}".format(disabled_command))
+        command_args = disabled_command.get(COMMAND_PROPERTY)
+        if is_web_uri(command_args[0]):
+            r = call_rest_api(disabled_command, PROJECT_SUBST, project_name)
+            try:
+                r.raise_for_status()
+            except HTTPError:
+                logger.error("API call failed for disabled command of "
+                             "project '{}': {}".
+                             format(project_name, r))
+        else:
+            cmd = Command(command_args,
+                          env_vars=disabled_command.get("env"),
+                          resource_limits=disabled_command.get("limits"),
+                          args_subst={PROJECT_SUBST: project_name},
+                          args_append=[project_name], excl_subst=True)
+            run_command(cmd, project_name)
+
+
 def mirror_project(config, project_name, check_changes, uri,
                    source_root):
     """
@@ -296,7 +335,8 @@ def mirror_project(config, project_name, check_changes, uri,
 
     project_config = get_project_config(config, project_name)
     prehook, posthook, hook_timeout, command_timeout, use_proxy, \
-        ignored_repos = get_project_properties(project_config, project_name,
+        ignored_repos = get_project_properties(project_config,
+                                               project_name,
                                                config.get(HOOKDIR_PROPERTY))
 
     proxy = None
@@ -306,7 +346,8 @@ def mirror_project(config, project_name, check_changes, uri,
     # We want this to be logged to the log file (if any).
     if project_config:
         if project_config.get(DISABLED_PROPERTY):
-            logger.info("Project {} disabled, exiting".
+            handle_disabled_project(config, project_name)
+            logger.info("Project '{}' disabled, exiting".
                         format(project_name))
             return CONTINUE_EXITVAL
 
@@ -450,7 +491,8 @@ def check_configuration(config):
 
     global_tunables = [HOOKDIR_PROPERTY, PROXY_PROPERTY, LOGDIR_PROPERTY,
                        COMMANDS_PROPERTY, PROJECTS_PROPERTY,
-                       HOOK_TIMEOUT_PROPERTY, CMD_TIMEOUT_PROPERTY]
+                       HOOK_TIMEOUT_PROPERTY, CMD_TIMEOUT_PROPERTY,
+                       DISABLED_CMD_PROPERTY]
     diff = set(config.keys()).difference(global_tunables)
     if diff:
         logger.error("unknown global configuration option(s): '{}'"
@@ -462,6 +504,10 @@ def check_configuration(config):
     if logdir:
         check_create_dir(logger, logdir)
 
+    disabled_command = config.get(DISABLED_CMD_PROPERTY)
+    if disabled_command:
+        logger.debug("Disabled command: {}".format(disabled_command))
+
     if not check_project_configuration(config.get(PROJECTS_PROPERTY),
                                        config.get(HOOKDIR_PROPERTY),
                                        config.get(PROXY_PROPERTY)):

opengrok-tools/src/main/python/opengrok_tools/utils/patterns.py

@@ -0,0 +1,25 @@
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# See LICENSE.txt included in this distribution for the specific
+# language governing permissions and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at LICENSE.txt.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+#
+
+PROJECT_SUBST = '%PROJECT%'
+COMMAND_PROPERTY = "command"

opengrok-tools/src/test/python/test_command_sequence.py

@@ -31,6 +31,7 @@
 
 from opengrok_tools.utils.commandsequence import CommandSequence, \
     CommandSequenceBase
+from opengrok_tools.utils.patterns import PROJECT_SUBST
 
 
 def test_str():
@@ -46,9 +47,9 @@ def test_str():
 def test_run_retcodes():
     cmd_list = [{"command": ["/bin/echo"]},
                 {"command": ["/bin/sh", "-c",
-                 "echo " + CommandSequence.PROJECT_SUBST + "; exit 0"]},
+                 "echo " + PROJECT_SUBST + "; exit 0"]},
                 {"command": ["/bin/sh", "-c",
-                 "echo " + CommandSequence.PROJECT_SUBST + "; exit 1"]}]
+                 "echo " + PROJECT_SUBST + "; exit 1"]}]
     cmds = CommandSequence(CommandSequenceBase("opengrok-master", cmd_list))
     cmds.run()
     assert cmds.retcodes == {
@@ -63,7 +64,7 @@ def test_run_retcodes():
                     reason="requires Unix")
 def test_terminate_after_non_zero_code():
     cmd_list = [{"command": ["/bin/sh", "-c",
-                 "echo " + CommandSequence.PROJECT_SUBST + "; exit 255"]},
+                 "echo " + PROJECT_SUBST + "; exit 255"]},
                 {"command": ["/bin/echo"]}]
     cmds = CommandSequence(CommandSequenceBase("opengrok-master", cmd_list))
     cmds.run()
@@ -77,7 +78,7 @@ def test_terminate_after_non_zero_code():
                     reason="requires Unix")
 def test_exit_2_handling():
     cmd_list = [{"command": ["/bin/sh", "-c",
-                 "echo " + CommandSequence.PROJECT_SUBST + "; exit 2"]},
+                 "echo " + PROJECT_SUBST + "; exit 2"]},
                 {"command": ["/bin/echo"]}]
     cmds = CommandSequence(CommandSequenceBase("opengrok-master", cmd_list))
     cmds.run()
@@ -92,13 +93,13 @@ def test_exit_2_handling():
                     reason="requires Unix")
 def test_driveon_flag():
     cmd_list = [{"command": ["/bin/sh", "-c",
-                 "echo " + CommandSequence.PROJECT_SUBST + "; exit 2"]},
+                 "echo " + PROJECT_SUBST + "; exit 2"]},
                 {"command": ["/bin/echo"]},
                 {"command": ["/bin/sh", "-c",
-                             "echo " + CommandSequence.PROJECT_SUBST +
+                             "echo " + PROJECT_SUBST +
                              "; exit 1"]},
                 {"command": ["/bin/sh", "-c",
-                             "echo " + CommandSequence.PROJECT_SUBST]}]
+                             "echo " + PROJECT_SUBST]}]
     cmds = CommandSequence(CommandSequenceBase("opengrok-master",
                                                cmd_list, driveon=True))
     cmds.run()
@@ -113,7 +114,7 @@ def test_driveon_flag():
 @pytest.mark.skipif(not os.path.exists('/bin/echo'),
                     reason="requires Unix")
 def test_project_subst():
-    cmd_list = [{"command": ["/bin/echo", CommandSequence.PROJECT_SUBST]}]
+    cmd_list = [{"command": ["/bin/echo", PROJECT_SUBST]}]
     cmds = CommandSequence(CommandSequenceBase("test-subst", cmd_list))
     cmds.run()
 
@@ -125,7 +126,7 @@ def test_cleanup_exception():
     If cleanup is not a list, Exception should be thrown when initializing
     the CommandSequence object.
     """
-    cleanup = {"cleanup": ["foo", CommandSequence.PROJECT_SUBST]}
+    cleanup = {"cleanup": ["foo", PROJECT_SUBST]}
     with pytest.raises(Exception):
         CommandSequence(CommandSequenceBase("test-cleanup-list", None,
                                             cleanup=cleanup))

opengrok-tools/src/test/python/test_mirror.py

@@ -30,16 +30,20 @@
 from git import Repo
 import pytest
 import sys
+from mockito import verify, patch, spy2, mock, ANY
+import requests
 
 from opengrok_tools.scm.repofactory import get_repository
 from opengrok_tools.utils.mirror import check_project_configuration, \
-    check_configuration, \
+    check_configuration, mirror_project, run_command, \
     HOOKS_PROPERTY, PROXY_PROPERTY, IGNORED_REPOS_PROPERTY, \
-    PROJECTS_PROPERTY
+    PROJECTS_PROPERTY, DISABLED_CMD_PROPERTY, DISABLED_PROPERTY
 import opengrok_tools.mirror
 from opengrok_tools.utils.exitvals import (
     CONTINUE_EXITVAL,
 )
+from opengrok_tools.utils.patterns import COMMAND_PROPERTY, PROJECT_SUBST
+from opengrok_tools.utils.command import Command
 
 
 def test_empty_project_configuration():
@@ -173,3 +177,45 @@ def mock_get(*args, **kwargs):
 
 def test_empty_project_config():
     assert check_project_configuration({'foo': None})
+
+
+def test_disabled_command_api():
+    with patch(opengrok_tools.utils.mirror.call_rest_api,
+               lambda a, b, c: mock(spec=requests.Response)):
+        project_name = "foo"
+        config = {DISABLED_CMD_PROPERTY:
+                  {COMMAND_PROPERTY:
+                   ["http://localhost:8080/source/api/v1/foo",
+                    "POST", "data"]},
+                  PROJECTS_PROPERTY: {project_name: {DISABLED_PROPERTY: True}}}
+
+        assert mirror_project(config, project_name, False,
+                              None, None) == CONTINUE_EXITVAL
+        verify(opengrok_tools.utils.mirror). \
+            call_rest_api(config.get(DISABLED_CMD_PROPERTY),
+                          PROJECT_SUBST, project_name)
+
+
+def test_disabled_command_run():
+    """
+    Make sure that mirror_project() results in calling run_command().
+    """
+    spy2(opengrok_tools.utils.mirror.run_command)
+    project_name = "foo"
+    config = {DISABLED_CMD_PROPERTY:
+              {COMMAND_PROPERTY: ["cat"]},
+              PROJECTS_PROPERTY: {project_name: {DISABLED_PROPERTY: True}}}
+
+    assert mirror_project(config, project_name, False,
+                          None, None) == CONTINUE_EXITVAL
+    verify(opengrok_tools.utils.mirror).run_command(ANY, project_name)
+
+
+def test_disabled_command_run_args():
+    """
+    Make sure that run_command() calls Command.execute().
+    """
+    cmd = mock(spec=Command)
+    project_name = "foo"
+    run_command(cmd, project_name)
+    verify(cmd).execute()