add driveon option to sync.py (#2511)

fixes #2510

Author: Vladimir Kotal

opengrok-tools/pom.xml

@@ -289,8 +289,8 @@ Portions Copyright (c) 2017-2018, Chris Fraire <[email protected]>.
                                 <argument>-m</argument>
                                 <argument>flake8</argument>
                                 <argument>-v</argument>
-                                <argument>--exclude=test_command.py,test_commands.py</argument>
                                 <argument>${project.build.sourceDirectory}</argument>
+                                <argument>${project.build.testSourceDirectory}</argument>
                                 <argument>setup.py</argument>
                             </arguments>
                         </configuration>

opengrok-tools/src/main/python/opengrok_tools/sync.py

@@ -90,6 +90,9 @@ def main():
                         help='config file in JSON format')
     parser.add_argument('-U', '--uri', default='http://localhost:8080/source',
                         help='URI of the webapp with context path')
+    parser.add_argument('-f', '--driveon', action='store_true', default=False,
+                        help='continue command sequence processing even '
+                        'if one of the commands requests break')
     try:
         args = parser.parse_args()
     except ValueError as e:
@@ -177,7 +180,8 @@ def main():
             cmds_base = []
             for d in dirs_to_process:
                 cmd_base = CommandSequenceBase(d, commands, args.loglevel,
-                                               config.get("cleanup"))
+                                               config.get("cleanup"),
+                                               args.driveon)
                 cmds_base.append(cmd_base)
 
             # Map the commands into pool of workers so they can be processed.

opengrok-tools/src/main/python/opengrok_tools/utils/commandsequence.py

@@ -36,14 +36,16 @@ class CommandSequenceBase:
     so that it can be passed through Pool.map().
     """
 
-    def __init__(self, name, commands, loglevel=logging.INFO, cleanup=None):
+    def __init__(self, name, commands, loglevel=logging.INFO, cleanup=None,
+                 driveon=False):
         self.name = name
         self.commands = commands
         self.failed = False
         self.retcodes = {}
         self.outputs = {}
         self.cleanup = cleanup
         self.loglevel = loglevel
+        self.driveon = driveon
 
     def __str__(self):
         return str(self.name)
@@ -67,7 +69,7 @@ class CommandSequence(CommandSequenceBase):
 
     def __init__(self, base):
         super().__init__(base.name, base.commands, loglevel=base.loglevel,
-                         cleanup=base.cleanup)
+                         cleanup=base.cleanup, driveon=base.driveon)
 
         self.logger = logging.getLogger(__name__)
         self.logger.setLevel(base.loglevel)
@@ -120,7 +122,8 @@ def run(self):
         Run the sequence of commands and capture their output and return code.
         First command that returns code other than 0 terminates the sequence.
         If the command has return code 2, the sequence will be terminated
-        however it will not be treated as error.
+        however it will not be treated as error (unless the 'driveon' parameter
+        is True).
 
         If a command contains PROJECT_SUBST pattern, it will be replaced
         by project name, otherwise project name will be appended to the
@@ -136,19 +139,29 @@ def run(self):
             else:
                 retcode = self.run_command(command)
 
-                # If a command fails, terminate the sequence of commands.
+                # If a command exits with non-zero return code,
+                # terminate the sequence of commands.
                 if retcode != 0:
                     if retcode == 2:
-                        self.logger.debug("command '{}' for project {} "
-                                          "requested break".
-                                          format(self.name, command))
-                        self.run_cleanup()
+                        if not self.driveon:
+                            self.logger.debug("command '{}' for project {} "
+                                              "requested break".
+                                              format(self.name, command))
+                            self.run_cleanup()
+                        else:
+                            self.logger.debug("command '{}' for project {} "
+                                              "requested break however "
+                                              "the 'driveon' option is set "
+                                              "so driving on.".
+                                              format(self.name, command))
+                            continue
                     else:
                         self.logger.error("command '{}' for project {} failed "
                                           "with code {}, breaking".
                                           format(command, self.name, retcode))
                         self.failed = True
                         self.run_cleanup()
+
                     break
 
     def run_cleanup(self):

opengrok-tools/src/test/python/test_command.py

@@ -66,7 +66,7 @@ def test_subst_append_exclsubst_nosubst():
 def test_execute_nonexistent():
     cmd = Command(['/baaah', '/etc/passwd'])
     cmd.execute()
-    assert cmd.getretcode() == None
+    assert cmd.getretcode() is None
     assert cmd.getstate() == Command.ERRORED
 
 
@@ -88,7 +88,8 @@ def test_work_dir():
     assert os.getcwd() == orig_cwd
 
 
-@pytest.mark.skipif(not os.path.exists('/usr/bin/env'), reason="requires posix")
+@pytest.mark.skipif(not os.path.exists('/usr/bin/env'),
+                    reason="requires posix")
 def test_env():
     cmd = Command(['/usr/bin/env'],
                   env_vars={'FOO': 'BAR', 'A': 'B'})
@@ -141,7 +142,7 @@ def test_command_timeout():
     elapsed_time = time.time() - start_time
     assert elapsed_time < timeout
     assert cmd.getstate() == Command.TIMEDOUT
-    assert cmd.getretcode() == None
+    assert cmd.getretcode() is None
 
 
 @pytest.mark.skipif(not os.path.exists('/bin/sleep'),

opengrok-tools/src/test/python/test_command_sequence.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# See LICENSE.txt included in this distribution for the specific
+# language governing permissions and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at LICENSE.txt.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved.
+#
+
+import os
+
+import pytest
+
+from opengrok_tools.utils.commandsequence import CommandSequence, \
+    CommandSequenceBase
+
+
+def test_str():
+    cmds = CommandSequence(CommandSequenceBase("opengrok-master",
+                                               [{"command": ['foo']},
+                                                {"command": ["bar"]}]))
+    assert str(cmds) == "opengrok-master"
+
+
+@pytest.mark.skipif(not os.path.exists('/bin/sh')
+                    or not os.path.exists('/bin/echo'),
+                    reason="requires Unix")
+def test_run_retcodes():
+    cmd_list = [{"command": ["/bin/echo"]},
+                {"command": ["/bin/sh", "-c",
+                 "echo " + CommandSequence.PROJECT_SUBST + "; exit 0"]},
+                {"command": ["/bin/sh", "-c",
+                 "echo " + CommandSequence.PROJECT_SUBST + "; exit 1"]}]
+    cmds = CommandSequence(CommandSequenceBase("opengrok-master", cmd_list))
+    cmds.run()
+    assert cmds.retcodes == {
+        '/bin/echo opengrok-master': 0,
+        '/bin/sh -c echo opengrok-master; exit 0': 0,
+        '/bin/sh -c echo opengrok-master; exit 1': 1
+    }
+
+
+@pytest.mark.skipif(not os.path.exists('/bin/sh')
+                    or not os.path.exists('/bin/echo'),
+                    reason="requires Unix")
+def test_terminate_after_non_zero_code():
+    cmd_list = [{"command": ["/bin/sh", "-c",
+                 "echo " + CommandSequence.PROJECT_SUBST + "; exit 255"]},
+                {"command": ["/bin/echo"]}]
+    cmds = CommandSequence(CommandSequenceBase("opengrok-master", cmd_list))
+    cmds.run()
+    assert cmds.retcodes == {
+        '/bin/sh -c echo opengrok-master; exit 255': 255
+    }
+
+
+@pytest.mark.skipif(not os.path.exists('/bin/sh')
+                    or not os.path.exists('/bin/echo'),
+                    reason="requires Unix")
+def test_exit_2_handling():
+    cmd_list = [{"command": ["/bin/sh", "-c",
+                 "echo " + CommandSequence.PROJECT_SUBST + "; exit 2"]},
+                {"command": ["/bin/echo"]}]
+    cmds = CommandSequence(CommandSequenceBase("opengrok-master", cmd_list))
+    cmds.run()
+    assert cmds.retcodes == {
+        '/bin/sh -c echo opengrok-master; exit 2': 2
+    }
+    assert not cmds.failed
+
+
+@pytest.mark.skipif(not os.path.exists('/bin/sh')
+                    or not os.path.exists('/bin/echo'),
+                    reason="requires Unix")
+def test_driveon_flag():
+    cmd_list = [{"command": ["/bin/sh", "-c",
+                 "echo " + CommandSequence.PROJECT_SUBST + "; exit 2"]},
+                {"command": ["/bin/echo"]},
+                {"command": ["/bin/sh", "-c",
+                             "echo " + CommandSequence.PROJECT_SUBST +
+                             "; exit 1"]},
+                {"command": ["/bin/sh", "-c",
+                             "echo " + CommandSequence.PROJECT_SUBST]}]
+    cmds = CommandSequence(CommandSequenceBase("opengrok-master",
+                                               cmd_list, driveon=True))
+    cmds.run()
+    assert cmds.retcodes == {
+        '/bin/sh -c echo opengrok-master; exit 2': 2,
+        '/bin/echo opengrok-master': 0,
+        '/bin/sh -c echo opengrok-master; exit 1': 1
+    }
+    assert cmds.failed
+
+
+@pytest.mark.skipif(not os.path.exists('/bin/echo'),
+                    reason="requires Unix")
+def test_project_subst():
+    cmd_list = [{"command": ["/bin/echo", CommandSequence.PROJECT_SUBST]}]
+    cmds = CommandSequence(CommandSequenceBase("test-subst", cmd_list))
+    cmds.run()
+
+    assert cmds.outputs['/bin/echo test-subst'] == ['test-subst\n']