Skip to content

Commit 732be1c

Browse files
author
Vladimir Kotal
authored
requests timeout for tools (#3454)
fixes #3449
1 parent 91193e6 commit 732be1c

File tree

12 files changed

+175
-90
lines changed

12 files changed

+175
-90
lines changed

‎tools/src/main/python/opengrok_tools/mirror.py

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -58,12 +58,12 @@
5858
if major_version < 3:
5959
fatal("Need Python 3, you are running {}".format(major_version))
6060

61-
__version__ = "1.0"
61+
__version__ = "1.1"
6262

6363

6464
def worker(args):
6565
project_name, logdir, loglevel, backupcount, config, check_changes, uri, \
66-
source_root, batch, headers = args
66+
source_root, batch, headers, api_timeout = args
6767

6868
if batch:
6969
get_batch_logger(logdir, project_name,
@@ -73,7 +73,8 @@ def worker(args):
7373

7474
return mirror_project(config, project_name,
7575
check_changes,
76-
uri, source_root, headers=headers)
76+
uri, source_root, headers=headers,
77+
timeout=api_timeout)
7778

7879

7980
def main():
@@ -105,6 +106,8 @@ def main():
105106
parser.add_argument('-w', '--workers', default=cpu_count(),
106107
help='Number of worker processes')
107108
add_http_headers(parser)
109+
parser.add_argument('--api_timeout', type=int,
110+
help='Set response timeout in seconds for RESTful API calls')
108111

109112
try:
110113
args = parser.parse_args()
@@ -145,7 +148,7 @@ def main():
145148

146149
# Save the source root to avoid querying the web application.
147150
source_root = get_config_value(logger, 'sourceRoot', uri,
148-
headers=headers)
151+
headers=headers, timeout=args.api_timeout)
149152
if not source_root:
150153
return 1
151154

@@ -183,7 +186,9 @@ def main():
183186
lockfile = os.path.basename(sys.argv[0])
184187

185188
if args.all:
186-
projects = list_indexed_projects(logger, args.uri, headers=headers)
189+
projects = list_indexed_projects(logger, args.uri,
190+
headers=headers,
191+
timeout=args.api_timeout)
187192

188193
lock = FileLock(os.path.join(tempfile.gettempdir(), lockfile + ".lock"))
189194
try:
@@ -195,7 +200,8 @@ def main():
195200
args.backupcount, config,
196201
args.check_changes,
197202
args.uri, source_root,
198-
args.batch, headers])
203+
args.batch, headers,
204+
args.api_timeout])
199205
try:
200206
project_results = pool.map(worker, worker_args, 1)
201207
except KeyboardInterrupt:

‎tools/src/main/python/opengrok_tools/projadm.py

Lines changed: 24 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ def install_config(doit, logger, src, dst):
111111

112112

113113
def config_refresh(doit, logger, basedir, uri, configmerge, jar_file,
114-
roconfig, java, headers=None):
114+
roconfig, java, headers=None, timeout=None):
115115
"""
116116
Refresh current configuration file with configuration retrieved
117117
from webapp. If roconfig is not None, the current config is merged with
@@ -127,7 +127,8 @@ def config_refresh(doit, logger, basedir, uri, configmerge, jar_file,
127127
sys.exit(FAILURE_EXITVAL)
128128

129129
if doit:
130-
current_config = get_configuration(logger, uri, headers=headers)
130+
current_config = get_configuration(logger, uri,
131+
headers=headers, timeout=timeout)
131132
if not current_config:
132133
sys.exit(FAILURE_EXITVAL)
133134
else:
@@ -162,7 +163,7 @@ def config_refresh(doit, logger, basedir, uri, configmerge, jar_file,
162163
install_config(doit, logger, fmerged.name, main_config)
163164

164165

165-
def project_add(doit, logger, project, uri, headers=None):
166+
def project_add(doit, logger, project, uri, headers=None, timeout=None):
166167
"""
167168
Adds a project to configuration. Works in multiple steps:
168169
@@ -173,10 +174,11 @@ def project_add(doit, logger, project, uri, headers=None):
173174
logger.info("Adding project {}".format(project))
174175

175176
if doit:
176-
add_project(logger, project, uri, headers=headers)
177+
add_project(logger, project, uri, headers=headers, timeout=timeout)
177178

178179

179-
def project_delete(logger, project, uri, doit=True, deletesource=False, headers=None):
180+
def project_delete(logger, project, uri, doit=True, deletesource=False,
181+
headers=None, timeout=None):
180182
"""
181183
Delete the project for configuration and all its data.
182184
Works in multiple steps:
@@ -193,10 +195,11 @@ def project_delete(logger, project, uri, doit=True, deletesource=False, headers=
193195
logger.info("Deleting project {} and its index data".format(project))
194196

195197
if doit:
196-
delete_project(logger, project, uri, headers=headers)
198+
delete_project(logger, project, uri, headers=headers, timeout=timeout)
197199

198200
if deletesource:
199-
src_root = get_config_value(logger, 'sourceRoot', uri, headers=headers)
201+
src_root = get_config_value(logger, 'sourceRoot', uri, headers=headers,
202+
timeout=timeout)
200203
if not src_root or len(src_root) == 0:
201204
raise Exception("source root empty")
202205
logger.debug("Source root = {}".format(src_root))
@@ -236,6 +239,8 @@ def main():
236239
default=False, help='Do not delete source code when '
237240
'deleting a project')
238241
add_http_headers(parser)
242+
parser.add_argument('--api_timeout', type=int,
243+
help='Set response timeout in seconds for RESTful API calls')
239244

240245
group = parser.add_mutually_exclusive_group()
241246
group.add_argument('-a', '--add', metavar='project', nargs='+',
@@ -322,7 +327,8 @@ def main():
322327
for proj in args.add:
323328
project_add(doit=doit, logger=logger,
324329
project=proj,
325-
uri=uri, headers=headers)
330+
uri=uri, headers=headers,
331+
timeout=args.api_timeout)
326332

327333
config_refresh(doit=doit, logger=logger,
328334
basedir=args.base,
@@ -331,14 +337,16 @@ def main():
331337
jar_file=args.jar,
332338
roconfig=args.roconfig,
333339
java=args.java,
334-
headers=headers)
340+
headers=headers,
341+
timeout=args.api_timeout)
335342
elif args.delete:
336343
for proj in args.delete:
337344
project_delete(logger=logger,
338345
project=proj,
339346
uri=uri, doit=doit,
340347
deletesource=not args.nosourcedelete,
341-
headers=headers)
348+
headers=headers,
349+
timeout=args.api_timeout)
342350

343351
config_refresh(doit=doit, logger=logger,
344352
basedir=args.base,
@@ -347,7 +355,8 @@ def main():
347355
jar_file=args.jar,
348356
roconfig=args.roconfig,
349357
java=args.java,
350-
headers=headers)
358+
headers=headers,
359+
timeout=args.api_timeout)
351360
elif args.refresh:
352361
config_refresh(doit=doit, logger=logger,
353362
basedir=args.base,
@@ -356,7 +365,8 @@ def main():
356365
jar_file=args.jar,
357366
roconfig=args.roconfig,
358367
java=args.java,
359-
headers=headers)
368+
headers=headers,
369+
timeout=args.api_timeout)
360370
else:
361371
parser.print_help()
362372
sys.exit(FAILURE_EXITVAL)
@@ -370,7 +380,8 @@ def main():
370380
config_data = config_file.read().encode("utf-8")
371381
if not set_configuration(logger,
372382
config_data, uri,
373-
headers=headers):
383+
headers=headers,
384+
timeout=args.api_timeout):
374385
sys.exit(FAILURE_EXITVAL)
375386
else:
376387
logger.error("file {} does not exist".format(main_config))

‎tools/src/main/python/opengrok_tools/reindex_project.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,11 +60,11 @@ def get_logprop_file(logger, template, pattern, project):
6060
return tmpf.name
6161

6262

63-
def get_config_file(logger, uri, headers=None):
63+
def get_config_file(logger, uri, headers=None, timeout=None):
6464
"""
6565
Get fresh configuration from the webapp and store it in temporary file.
6666
"""
67-
config = get_configuration(logger, uri, headers)
67+
config = get_configuration(logger, uri, headers=headers, timeout=timeout)
6868

6969
with tempfile.NamedTemporaryFile(delete=False) as tmpf:
7070
tmpf.write(config.encode())
@@ -90,6 +90,8 @@ def main():
9090
help='URI of the webapp with context path')
9191
parser.add_argument('--printoutput', action='store_true', default=False)
9292
add_http_headers(parser)
93+
parser.add_argument('--api_timeout', type=int,
94+
help='Set response timeout in seconds for RESTful API calls')
9395

9496
cmd_args = sys.argv[1:]
9597
extra_opts = os.environ.get("OPENGROK_INDEXER_OPTIONAL_ARGS")

‎tools/src/main/python/opengrok_tools/sync.py

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@
5252
print("Need Python 3, you are running {}".format(major_version))
5353
sys.exit(1)
5454

55-
__version__ = "1.2"
55+
__version__ = "1.3"
5656

5757

5858
def worker(base):
@@ -69,7 +69,7 @@ def worker(base):
6969

7070
def do_sync(loglevel, commands, cleanup, dirs_to_process, ignore_errors,
7171
uri, numworkers, driveon=False, print_output=False, logger=None,
72-
http_headers=None):
72+
http_headers=None, timeout=None):
7373
"""
7474
Process the list of directories in parallel.
7575
:param logger: logger to be used in this function
@@ -85,6 +85,7 @@ def do_sync(loglevel, commands, cleanup, dirs_to_process, ignore_errors,
8585
using the supplied logger
8686
:param logger: optional logger
8787
:param http_headers: optional dictionary of HTTP headers
88+
:param timeout: optional timeout in seconds for API call response
8889
:return SUCCESS_EXITVAL on success, FAILURE_EXITVAL on error
8990
"""
9091

@@ -93,7 +94,8 @@ def do_sync(loglevel, commands, cleanup, dirs_to_process, ignore_errors,
9394
cmd_base = CommandSequenceBase(dir, commands, loglevel=loglevel,
9495
cleanup=cleanup,
9596
driveon=driveon, url=uri,
96-
http_headers=http_headers)
97+
http_headers=http_headers,
98+
api_timeout=timeout)
9799
cmds_base.append(cmd_base)
98100

99101
# Map the commands into pool of workers so they can be processed.
@@ -150,6 +152,9 @@ def main():
150152
parser.add_argument('--nolock', action='store_false', default=True,
151153
help='do not acquire lock that prevents multiple '
152154
'instances from running')
155+
parser.add_argument('--api_timeout', type=int,
156+
help='Set response timeout in seconds'
157+
'for RESTful API calls')
153158
add_http_headers(parser)
154159

155160
try:
@@ -197,7 +202,8 @@ def main():
197202
directory = args.directory
198203
if not args.directory and not args.project and not args.indexed:
199204
# Assume directory, get the source root value from the webapp.
200-
directory = get_config_value(logger, 'sourceRoot', uri, headers=headers)
205+
directory = get_config_value(logger, 'sourceRoot', uri,
206+
headers=headers, timeout=args.api_timeout)
201207
if not directory:
202208
logger.error("Neither -d or -P or -I specified and cannot get "
203209
"source root from the webapp")
@@ -221,7 +227,9 @@ def main():
221227
logger.debug("Processing directories: {}".
222228
format(dirs_to_process))
223229
elif args.indexed:
224-
indexed_projects = list_indexed_projects(logger, uri, headers=headers)
230+
indexed_projects = list_indexed_projects(logger, uri,
231+
headers=headers,
232+
timeout=args.api_timeout)
225233
logger.debug("Processing indexed projects: {}".
226234
format(indexed_projects))
227235

@@ -264,7 +272,8 @@ def main():
264272
r = do_sync(args.loglevel, commands, config.get("cleanup"),
265273
dirs_to_process,
266274
ignore_errors, uri, args.workers,
267-
driveon=args.driveon, http_headers=headers)
275+
driveon=args.driveon, http_headers=headers,
276+
timeout=args.api_timeout)
268277
else:
269278
lock = FileLock(os.path.join(tempfile.gettempdir(),
270279
lockfile_name + ".lock"))
@@ -273,7 +282,8 @@ def main():
273282
r = do_sync(args.loglevel, commands, config.get("cleanup"),
274283
dirs_to_process,
275284
ignore_errors, uri, args.workers,
276-
driveon=args.driveon, http_headers=headers)
285+
driveon=args.driveon, http_headers=headers,
286+
timeout=args.api_timeout)
277287
except Timeout:
278288
logger.warning("Already running")
279289
return FAILURE_EXITVAL

‎tools/src/main/python/opengrok_tools/utils/commandsequence.py

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,8 @@ class CommandSequenceBase:
4646
"""
4747

4848
def __init__(self, name, commands, loglevel=logging.INFO, cleanup=None,
49-
driveon=False, url=None, env=None, http_headers=None):
49+
driveon=False, url=None, env=None, http_headers=None,
50+
api_timeout=None):
5051
self.name = name
5152
self.commands = commands
5253
self.failed = False
@@ -60,6 +61,7 @@ def __init__(self, name, commands, loglevel=logging.INFO, cleanup=None,
6061
self.driveon = driveon
6162
self.env = env
6263
self.http_headers = http_headers
64+
self.api_timeout = api_timeout
6365

6466
self.url = url
6567

@@ -93,7 +95,8 @@ def __init__(self, base):
9395
super().__init__(base.name, base.commands, loglevel=base.loglevel,
9496
cleanup=base.cleanup, driveon=base.driveon,
9597
url=base.url, env=base.env,
96-
http_headers=base.http_headers)
98+
http_headers=base.http_headers,
99+
api_timeout=base.api_timeout)
97100

98101
self.logger = logging.getLogger(__name__)
99102
self.logger.setLevel(base.loglevel)
@@ -130,7 +133,7 @@ def run(self):
130133
try:
131134
call_rest_api(command, {PROJECT_SUBST: self.name,
132135
URL_SUBST: self.url},
133-
self.http_headers)
136+
self.http_headers, self.api_timeout)
134137
except HTTPError as e:
135138
self.logger.error("RESTful command {} failed: {}".
136139
format(command, e))
@@ -188,7 +191,7 @@ def run_cleanup(self):
188191
try:
189192
call_rest_api(cleanup_cmd, {PROJECT_SUBST: self.name,
190193
URL_SUBST: self.url},
191-
self.http_headers)
194+
self.http_headers, self.api_timeout)
192195
except HTTPError as e:
193196
self.logger.error("RESTful command {} failed: {}".
194197
format(cleanup_cmd, e))

0 commit comments

Comments
 (0)