convert API call configuration to dictionary

fixes #3453
fixes #3923

Author: vladak

tools/src/main/python/opengrok_tools/mirror.py

@@ -49,7 +49,8 @@
 from .utils.opengrok import get_config_value, list_indexed_projects
 from .utils.parsers import get_base_parser, add_http_headers, get_headers
 from .utils.readconfig import read_config
-from .utils.utils import get_int, is_web_uri
+from .utils.utils import get_int
+from .utils.webutil import is_web_uri
 from .utils.mirror import check_configuration, LOGDIR_PROPERTY, \
     mirror_project, HOOKDIR_PROPERTY, CMD_TIMEOUT_PROPERTY, \
     HOOK_TIMEOUT_PROPERTY
@@ -58,7 +59,7 @@
 if major_version < 3:
     fatal("Need Python 3, you are running {}".format(major_version))
 
-__version__ = "1.2"
+__version__ = "1.3"
 
 OPENGROK_NO_MIRROR_ENV = "OPENGROK_NO_MIRROR"
 

tools/src/main/python/opengrok_tools/projadm.py

@@ -42,7 +42,8 @@
 from .utils.opengrok import get_configuration, set_configuration, \
     add_project, delete_project, get_config_value, get_repos
 from .utils.parsers import get_base_parser, get_headers, add_http_headers
-from .utils.utils import get_command, is_web_uri
+from .utils.utils import get_command
+from .utils.webutil import is_web_uri
 from .utils.exitvals import (
     FAILURE_EXITVAL,
     SUCCESS_EXITVAL

tools/src/main/python/opengrok_tools/sync.py

@@ -41,7 +41,7 @@
 from .utils.opengrok import list_indexed_projects, get_config_value
 from .utils.parsers import get_base_parser, add_http_headers, get_headers
 from .utils.readconfig import read_config
-from .utils.utils import is_web_uri
+from .utils.webutil import is_web_uri
 from .utils.exitvals import (
     FAILURE_EXITVAL,
     SUCCESS_EXITVAL
@@ -52,7 +52,7 @@
     print("Need Python 3, you are running {}".format(major_version))
     sys.exit(1)
 
-__version__ = "1.3"
+__version__ = "1.4"
 
 
 def worker(base):

tools/src/main/python/opengrok_tools/utils/commandsequence.py

@@ -26,14 +26,13 @@
 from requests.exceptions import RequestException
 
 from .command import Command
-from .utils import is_web_uri
 from .exitvals import (
     CONTINUE_EXITVAL,
     SUCCESS_EXITVAL,
     FAILURE_EXITVAL
 )
 from .restful import call_rest_api
-from .patterns import PROJECT_SUBST, COMMAND_PROPERTY, URL_SUBST
+from .patterns import PROJECT_SUBST, COMMAND_PROPERTY, CALL_PROPERTY, URL_SUBST
 import re
 
 
@@ -51,13 +50,16 @@ def check_command_property(command):
         raise CommandConfigurationException("command '{}' is not a dictionary".format(command))
 
     command_value = command.get(COMMAND_PROPERTY)
-    if command_value is None:
-        raise CommandConfigurationException("command dictionary has no '{}' key: {}".
-                                            format(COMMAND_PROPERTY, command))
+    call_value = command.get(CALL_PROPERTY)
+    if command_value is None and call_value is None:
+        raise CommandConfigurationException(f"command dictionary has unknown key: {command}")
 
-    if not isinstance(command_value, list):
+    if command_value and not isinstance(command_value, list):
         raise CommandConfigurationException("command value not a list: {}".
                                             format(command_value))
+    if call_value and not isinstance(call_value, dict):
+        raise CommandConfigurationException("call value not a dictionary: {}".
+                                            format(call_value))
 
 
 class CommandSequenceBase:
@@ -123,7 +125,6 @@ def fill(self, retcodes, outputs, failed):
 
 
 class CommandSequence(CommandSequenceBase):
-
     re_program = re.compile('ERROR[:]*\\s+')
 
     def __init__(self, base):
@@ -163,20 +164,20 @@ def run(self):
         """
 
         for command in self.commands:
-            cmd_value = command.get(COMMAND_PROPERTY)[0]
-            if cmd_value.startswith(URL_SUBST) or is_web_uri(cmd_value):
+            if command.get(CALL_PROPERTY):
                 try:
-                    call_rest_api(command, {PROJECT_SUBST: self.name,
-                                            URL_SUBST: self.url},
+                    call_rest_api(command.get(CALL_PROPERTY),
+                                  {PROJECT_SUBST: self.name,
+                                   URL_SUBST: self.url},
                                   self.http_headers, self.api_timeout)
                 except RequestException as e:
-                    self.logger.error("RESTful command {} failed: {}".
+                    self.logger.error("REST API call {} failed: {}".
                                       format(command, e))
                     self.failed = True
                     self.retcodes[str(command)] = FAILURE_EXITVAL
 
                     break
-            else:
+            elif command.get(COMMAND_PROPERTY):
                 command_args = command.get(COMMAND_PROPERTY)
                 command = Command(command_args,
                                   env_vars=command.get("env"),
@@ -185,12 +186,12 @@ def run(self):
                                   args_subst={PROJECT_SUBST: self.name,
                                               URL_SUBST: self.url},
                                   args_append=[self.name], excl_subst=True)
-                retcode = self.run_command(command)
+                ret_code = self.run_command(command)
 
                 # If a command exits with non-zero return code,
                 # terminate the sequence of commands.
-                if retcode != SUCCESS_EXITVAL:
-                    if retcode == CONTINUE_EXITVAL:
+                if ret_code != SUCCESS_EXITVAL:
+                    if ret_code == CONTINUE_EXITVAL:
                         if not self.driveon:
                             self.logger.debug("command '{}' for project {} "
                                               "requested break".
@@ -206,11 +207,13 @@ def run(self):
                     else:
                         self.logger.error("command '{}' for project {} failed "
                                           "with code {}, breaking".
-                                          format(command, self.name, retcode))
+                                          format(command, self.name, ret_code))
                         self.failed = True
                         self.run_cleanup()
 
                     break
+            else:
+                raise Exception(f"unknown command: {command}")
 
     def run_cleanup(self):
         """
@@ -221,16 +224,16 @@ def run_cleanup(self):
             return
 
         for cleanup_cmd in self.cleanup:
-            arg0 = cleanup_cmd.get(COMMAND_PROPERTY)[0]
-            if arg0.startswith(URL_SUBST) or is_web_uri(arg0):
+            if cleanup_cmd.get(CALL_PROPERTY):
                 try:
-                    call_rest_api(cleanup_cmd, {PROJECT_SUBST: self.name,
-                                                URL_SUBST: self.url},
+                    call_rest_api(cleanup_cmd.get(CALL_PROPERTY),
+                                  {PROJECT_SUBST: self.name,
+                                   URL_SUBST: self.url},
                                   self.http_headers, self.api_timeout)
                 except RequestException as e:
-                    self.logger.error("RESTful command {} failed: {}".
+                    self.logger.error("API call {} failed: {}".
                                       format(cleanup_cmd, e))
-            else:
+            elif cleanup_cmd.get(COMMAND_PROPERTY):
                 command_args = cleanup_cmd.get(COMMAND_PROPERTY)
                 self.logger.debug("Running cleanup command '{}'".
                                   format(command_args))
@@ -245,6 +248,8 @@ def run_cleanup(self):
                                       "code {}".
                                       format(cmd.cmd, cmd.getretcode()))
                     self.logger.info('output: {}'.format(cmd.getoutputstr()))
+            else:
+                raise Exception(f"unknown type of action: {cleanup_cmd}")
 
     def print_outputs(self, logger, loglevel=logging.INFO, lines=False):
         """

tools/src/main/python/opengrok_tools/utils/mirror.py

@@ -36,12 +36,12 @@
     CONTINUE_EXITVAL,
     SUCCESS_EXITVAL
 )
-from .patterns import PROJECT_SUBST, COMMAND_PROPERTY
-from .utils import is_exe, check_create_dir, get_int, is_web_uri, get_bool
+from .patterns import PROJECT_SUBST, COMMAND_PROPERTY, CALL_PROPERTY
+from .utils import is_exe, check_create_dir, get_int, get_bool
 from .opengrok import get_repos, get_repo_type, get_uri, delete_project_data
 from .hook import run_hook
 from .command import Command
-from .restful import call_rest_api, do_api_call
+from .restful import call_rest_api, do_api_call, get_call_props
 
 from ..scm.repofactory import get_repository
 from ..scm.repository import RepositoryException
@@ -355,30 +355,30 @@ def handle_disabled_project(config, project_name, disabled_msg, headers=None,
     if disabled_command:
         logger = logging.getLogger(__name__)
 
-        logger.debug("Calling disabled command: {}".format(disabled_command))
-        command_args = disabled_command.get(COMMAND_PROPERTY)
-        uri = command_args[0]
-        if is_web_uri(uri):
-            # Is this perhaps OpenGrok API call to supply a Message ?
-            # If so and there was a string supplied, append it
-            # to the message text.
-            data = command_args[2]
+        if disabled_command.get(CALL_PROPERTY):
+            call = disabled_command.get(CALL_PROPERTY)
+            uri, _, data, _ = get_call_props(call)
             text = None
+
             if type(data) is dict:
                 text = data.get("text")
+
+            # Is this perhaps OpenGrok API call to supply a Message for the UI ?
+            # If so and there was a string supplied, append it to the message text.
             if text and uri.find("/api/v1/") > 0 and type(disabled_msg) is str:
                 logger.debug("Appending text to message: {}".
                              format(disabled_msg))
-                command_args[2]["text"] = text + ": " + disabled_msg
+                data["text"] = text + ": " + disabled_msg
 
             try:
-                call_rest_api(disabled_command, {PROJECT_SUBST: project_name},
+                call_rest_api(call, {PROJECT_SUBST: project_name},
                               http_headers=headers, timeout=timeout, api_timeout=api_timeout)
             except RequestException as e:
                 logger.error("API call failed for disabled command of "
                              "project '{}': {}".
                              format(project_name, e))
-        else:
+        elif disabled_command.get(COMMAND_PROPERTY):
+            command_args = disabled_command.get(COMMAND_PROPERTY)
             args = [project_name]
             if disabled_msg and type(disabled_msg) is str:
                 args.append(disabled_command)
@@ -389,6 +389,8 @@ def handle_disabled_project(config, project_name, disabled_msg, headers=None,
                           args_subst={PROJECT_SUBST: project_name},
                           args_append=args, excl_subst=True)
             run_command(cmd, project_name)
+        else:
+            raise Exception(f"unknown disabled action: {disabled_command}")
 
 
 def get_mirror_retcode(ignore_errors, value):

tools/src/main/python/opengrok_tools/utils/patterns.py

@@ -18,9 +18,10 @@
 #
 
 #
-# Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
 #
 
 PROJECT_SUBST = '%PROJECT%'
 URL_SUBST = '%URL%'
 COMMAND_PROPERTY = "command"
+CALL_PROPERTY = "call"