sync.py needs to use RESTful API calls (#2199)

Author: Vladimir Kotal

tools/sync/commands.py

@@ -25,6 +25,9 @@
 import os
 import command
 from command import Command
+from opengrok import put, post, delete
+from utils import is_web_uri
+import json
 
 
 class CommandsBase:
@@ -67,50 +70,88 @@ def __init__(self, base):
         self.logger = logging.getLogger(__name__)
         logging.basicConfig()
 
+    def run_command(self, command):
+        cmd = Command(command,
+                      args_subst={"PROJECT": self.name},
+                      args_append=[self.name], excl_subst=True)
+        cmd.execute()
+        self.retcodes[str(cmd)] = cmd.getretcode()
+        self.outputs[str(cmd)] = cmd.getoutput()
+
+    def call_rest_api(self, command):
+        """
+        Make RESTful API call.
+        """
+        PROJECT_SUBST = '%PROJECT%'
+
+        uri = command[0].replace(PROJECT_SUBST, self.name)
+        verb = command[1]
+        data = command[2]
+
+        if len(data) > 0:
+            headers = {'Content-Type': 'application/json'}
+            json_data = json.dumps(data).replace(PROJECT_SUBST, self.name)
+            self.logger.debug("JSON data: {}".format(json_data))
+
+        if verb == 'PUT':
+            put(self.logger, uri, headers=headers, data=json_data)
+        elif verb == 'POST':
+            post(self.logger, uri, headers=headers, data=json_data)
+        elif verb == 'DELETE':
+            delete(self.logger, uri, data)
+        else:
+            self.logger.error('Unknown verb in command {}'.
+                              format(command))
+
     def run(self):
         """
         Run the sequence of commands and capture their output and return code.
         First command that returns code other than 0 terminates the sequence.
         If the command has return code 2, the sequence will be terminated
         however it will not be treated as error.
+
+        Any command entry that is a URI, will be used to submit RESTful API
+        request. Return codes for these requests are not checked.
         """
 
         for command in self.commands:
-            cmd = Command(command,
-                          args_subst={"ARG": self.name},
-                          args_append=[self.name], excl_subst=True)
-            cmd.execute()
-            self.retcodes[str(cmd)] = cmd.getretcode()
-            self.outputs[str(cmd)] = cmd.getoutput()
-
-            # If a command fails, terminate the sequence of commands.
-            retcode = cmd.getretcode()
-            if retcode != 0:
-                if retcode == 2:
-                    self.logger.info("command '{}' requested break".
-                                     format(cmd))
-                    self.run_cleanup()
-                else:
-                    self.logger.info("command '{}' failed with code {}, "
-                                     "breaking".format(cmd, retcode))
-                    self.failed = True
-                    self.run_cleanup()
-                break
+            if is_web_uri(command[0]):
+                self.call_rest_api(command)
+            else:
+                self.run_command(command)
+
+                # If a command fails, terminate the sequence of commands.
+                retcode = cmd.getretcode()
+                if retcode != 0:
+                    if retcode == 2:
+                        self.logger.info("command '{}' requested break".
+                                         format(cmd))
+                        self.run_cleanup()
+                    else:
+                        self.logger.info("command '{}' failed with code {}, "
+                                         "breaking".format(cmd, retcode))
+                        self.failed = True
+                        self.run_cleanup()
+                    break
 
     def run_cleanup(self):
         """
         Call cleanup in case the sequence failed or termination was requested.
         """
         if self.cleanup:
-            self.logger.debug("Running cleanup command '{}'".
-                              format(self.cleanup))
-            cmd = Command(self.cleanup,
-                          args_subst={"ARG": self.name},
-                          args_append=[self.name], excl_subst=True)
-            cmd.execute()
-            if cmd.getretcode() != 0:
-                self.logger.info("cleanup command '{}' failed with code {}".
-                                 format(self.cleanup, cmd.getretcode()))
+            if is_web_uri(self.cleanup[0]):
+                self.call_rest_api(self.cleanup)
+            else:
+                self.logger.debug("Running cleanup command '{}'".
+                                  format(self.cleanup))
+                cmd = Command(self.cleanup,
+                              args_subst={"ARG": self.name},
+                              args_append=[self.name], excl_subst=True)
+                cmd.execute()
+                if cmd.getretcode() != 0:
+                    self.logger.info("cleanup command '{}' failed with "
+                                     "code {}".
+                                     format(self.cleanup, cmd.getretcode()))
 
     def check(self, ignore_errors):
         """

tools/sync/opengrok.py

@@ -25,39 +25,50 @@
 import requests
 import urllib.parse
 import traceback
+from urllib.parse import urlparse
 
 
-def get(logger, uri, params=None):
+def get(logger, uri, params=None, headers=None):
     try:
-        return requests.get(uri, params=params)
+        proxies = get_proxies(uri)
+        return requests.get(uri, params=params, proxies=proxies)
     except Exception as e:
         logger.debug(traceback.format_exc())
         return None
 
 
-def delete(logger, uri, params=None):
+def delete(logger, uri, params=None, headers=None):
     try:
-        return requests.delete(uri, params=params)
+        proxies = get_proxies(uri)
+        return requests.delete(uri, params=params, proxies=proxies)
     except Exception as e:
         logger.debug(traceback.format_exc())
         return None
 
 
-def post(logger, uri, data=None):
+def post(logger, uri, headers=None, data=None):
+    rv = None
     try:
-        return requests.post(uri, data=data)
+        proxies = get_proxies(uri)
+        rv = requests.post(uri, data=data, headers=headers, proxies=proxies)
     except Exception as e:
         logger.debug(traceback.format_exc())
         return None
 
+    return rv
 
-def put(logger, uri, data=None):
+
+def put(logger, uri, headers=None, data=None):
+    rv = None
     try:
-        return requests.put(uri, data=data)
+        proxies = get_proxies(uri)
+        rv = requests.put(uri, data=data, headers=headers, proxies=proxies)
     except Exception as e:
         logger.debug(traceback.format_exc())
         return None
 
+    return rv
+
 
 def get_repos(logger, project, uri):
     """
@@ -168,3 +179,22 @@ def delete_project(logger, project, uri):
 
 def get_uri(*uri_parts):
     return '/'.join(s.strip('/') for s in uri_parts)
+
+
+def is_localhost_url(url):
+    """
+    Check if given URL is based on localhost.
+    """
+
+    o = urlparse(url)
+    return o.hostname in ['localhost', '127.0.0.1', '::1']
+
+
+def get_proxies(url):
+    """
+    For localhost based requests it is undesirable to use proxies.
+    """
+    if is_localhost_url(url):
+        return {'http': None, 'https': None}
+    else:
+        return None

tools/sync/sync.py

@@ -173,23 +173,25 @@ def worker(base):
 
             logger.debug("to process: {}".format(dirs_to_process))
 
-            projects = []
+            cmds_base = []
             for d in dirs_to_process:
-                proj = CommandsBase(d, config.get("commands"),
-                                    config.get("cleanup"))
-                projects.append(proj)
+                cmd_base = CommandsBase(d, config.get("commands"),
+                                        config.get("cleanup"))
+                cmds_base.append(cmd_base)
 
+            # Map the commands into pool of workers so they can be processed.
             try:
-                projects = pool.map(worker, projects, 1)
+                cmds_base_results = pool.map(worker, cmds_base, 1)
             except KeyboardInterrupt:
                 # XXX lock.release() or return 1 ?
                 sys.exit(1)
             else:
-                for proj in projects:
+                for cmds_base in cmds_base_results:
                     logger.debug("Checking results of project {}".
-                                 format(proj))
-                    cmds = Commands(proj)
-                    cmds.fill(proj.retcodes, proj.outputs, proj.failed)
+                                 format(cmds_base))
+                    cmds = Commands(cmds_base)
+                    cmds.fill(cmds_base.retcodes, cmds_base.outputs,
+                              cmds_base.failed)
                     cmds.check(ignore_errors)
     except Timeout:
         logger.warning("Already running, exiting.")

tools/sync/utils.py

@@ -25,6 +25,7 @@
 from shutil import which
 import logging
 import sys
+from urllib.parse import urlparse
 
 
 def is_exe(fpath):
@@ -95,3 +96,11 @@ def diff_list(first, second):
     """
     second = set(second)
     return [item for item in first if item not in second]
+
+
+def is_web_uri(url):
+    """
+    Check if given string is web URL.
+    """
+    o = urlparse(url)
+    return o.scheme in ['http', 'https']