move timeout defaults to programs

fixes #3867

Author: vladak

tools/src/main/python/opengrok_tools/mirror.py

@@ -110,7 +110,7 @@ def main():
     parser.add_argument('-w', '--workers', default=cpu_count(), type=int,
                         help='Number of worker processes')
     add_http_headers(parser)
-    parser.add_argument('--api_timeout', type=int,
+    parser.add_argument('--api_timeout', type=int, default=3,
                         help='Set response timeout in seconds '
                              'for RESTful API calls')
 

tools/src/main/python/opengrok_tools/projadm.py

@@ -266,9 +266,9 @@ def main():
                         default=False, help='Do not delete source code when '
                                             'deleting a project')
     add_http_headers(parser)
-    parser.add_argument('--api_timeout', type=int,
+    parser.add_argument('--api_timeout', type=int, default=3,
                         help='Set response timeout in seconds for RESTful API calls')
-    parser.add_argument('--async_api_timeout', type=int,
+    parser.add_argument('--async_api_timeout', type=int, default=300,
                         help='Set timeout in seconds for asynchronous RESTful API calls')
 
     group = parser.add_mutually_exclusive_group()

tools/src/main/python/opengrok_tools/reindex_project.py

@@ -18,7 +18,7 @@
 # CDDL HEADER END
 
 #
-# Copyright (c) 2009, 2021, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2009, 2022, Oracle and/or its affiliates. All rights reserved.
 #
 
 
@@ -43,7 +43,7 @@
 """
 
 
-def get_logprop_file(logger, template, pattern, project):
+def get_logprop_file(template, pattern, project):
     """
     Return the filename of file with logging properties specific for given
     project.
@@ -93,7 +93,7 @@ def main():
                         help='URI of the webapp with context path')
     parser.add_argument('--printoutput', action='store_true', default=False)
     add_http_headers(parser)
-    parser.add_argument('--api_timeout', type=int,
+    parser.add_argument('--api_timeout', type=int, default=3,
                         help='Set response timeout in seconds for RESTful API calls')
 
     cmd_args = sys.argv[1:]
@@ -105,18 +105,17 @@ def main():
     logger = get_console_logger(get_class_basename(), args.loglevel)
 
     # Make sure the log directory exists.
-    if args.directory:
-        if not os.path.isdir(args.directory):
-            os.makedirs(args.directory)
+    if args.directory and not os.path.isdir(args.directory):
+        os.makedirs(args.directory)
 
     # Get files needed for per-project reindex.
     headers = get_headers(args.header)
-    conf_file = get_config_file(logger, args.uri, headers=headers)
+    conf_file = get_config_file(logger, args.uri, headers=headers, timeout=args.api_timeout)
     if conf_file is None:
         fatal("could not get config file to run the indexer")
     logprop_file = None
     if args.template and args.pattern:
-        logprop_file = get_logprop_file(logger, args.template, args.pattern,
+        logprop_file = get_logprop_file(args.template, args.pattern,
                                         args.project)
 
     # Reindex with the modified logging.properties file and read-only config.

tools/src/main/python/opengrok_tools/utils/mirror.py

@@ -80,6 +80,7 @@ def get_repos_for_project(project_name, uri, source_root,
                   variables
     :param command_timeout: command timeout value in seconds
     :param headers: optional HTTP headers dictionary
+    :param timeout: connect timeout for API calls
     :return: list of Repository objects
     """
 

tools/src/main/python/opengrok_tools/utils/restful.py

@@ -34,10 +34,30 @@
 APPLICATION_JSON = 'application/json'   # default
 
 
-def wait_for_async_api(response, api_timeout, headers=None, timeout=None):
+def call_finished(location_uri, headers, timeout):
+    """
+    :param location_uri: URI to check the status of API call
+    :param headers: HTTP headers
+    :param timeout: connect timeout
+    """
+    logger = logging.getLogger(__name__)
+
+    logger.debug(f"GET API call: {location_uri}, timeout {timeout} seconds and headers: {headers}")
+    response = requests.get(location_uri, headers=headers, proxies=get_proxies(location_uri), timeout=timeout)
+    if response is None:
+        raise Exception("API call failed")
+
+    response.raise_for_status()
+    if response.status_code == 202:
+        return False
+    else:
+        return True
+
+
+def wait_for_async_api(response, api_timeout=None, headers=None, timeout=None):
     """
     :param response: request
-    :param api_timeout: asynchronous API timeout
+    :param api_timeout: asynchronous API timeout (will wait forever or until error if None)
     :param headers: request headers
     :param timeout: connect timeout
     :return: request
@@ -49,16 +69,16 @@ def wait_for_async_api(response, api_timeout, headers=None, timeout=None):
         raise Exception(f"no Location header in {response}")
 
     start_time = time.time()
-    for _ in range(api_timeout):
-        logger.debug(f"GET API call: {location_uri}, timeout {timeout} seconds and headers: {headers}")
-        response = requests.get(location_uri, headers=headers, proxies=get_proxies(location_uri), timeout=timeout)
-        if response is None:
-            raise Exception("API call failed")
-
-        if response.status_code == 202:
+    if api_timeout is None:
+        while True:
+            if call_finished(location_uri, headers, timeout):
+                break
+            time.sleep(1)
+    else:
+        for _ in range(api_timeout):
+            if call_finished(location_uri, headers, timeout):
+                break
             time.sleep(1)
-        else:
-            break
 
     if response.status_code == 202:
         wait_time = time.time() - start_time
@@ -81,9 +101,7 @@ def do_api_call(verb, uri, params=None, headers=None, data=None, timeout=None, a
     :param data: data or None
     :param timeout: optional connect timeout in seconds.
                     Applies also to asynchronous API status calls.
-                    If None, default (60 seconds) will be used.
     :param api_timeout: optional timeout for asynchronous API requests in seconds.
-                    If None, default (300 seconds) will be used.
     :return: the result of the handler call, can be None
     """
     logger = logging.getLogger(__name__)
@@ -92,12 +110,6 @@ def do_api_call(verb, uri, params=None, headers=None, data=None, timeout=None, a
     if handler is None or not callable(handler):
         raise Exception('Unknown HTTP verb: {}'.format(verb))
 
-    if timeout is None:
-        timeout = 60
-
-    if api_timeout is None:
-        api_timeout = 300
-
     logger.debug("{} API call: {} with data '{}', connect timeout {} seconds, API timeout {} seconds and headers: {}".
                  format(verb, uri, data, timeout, api_timeout, headers))
     r = handler(
@@ -113,7 +125,7 @@ def do_api_call(verb, uri, params=None, headers=None, data=None, timeout=None, a
         raise Exception("API call failed")
 
     if r.status_code == 202:
-        r = wait_for_async_api(r, api_timeout, headers=headers, timeout=timeout)
+        r = wait_for_async_api(r, api_timeout=api_timeout, headers=headers, timeout=timeout)
 
     r.raise_for_status()
 
@@ -142,7 +154,7 @@ def call_rest_api(command, substitutions=None, http_headers=None, timeout=None):
                           data substitution
     :param http_headers: optional dictionary of HTTP headers to be appended
     :param timeout: optional timeout in seconds for API call response
-    :return return value from given requests method
+    :return value from given requests method
     """
 
     logger = logging.getLogger(__name__)