Skip to content

Commit f26bd37

Browse files
author
Vladimir Kotal
committed
introduce projadm script for automating project data management
1 parent 3d82206 commit f26bd37

File tree

3 files changed

+334
-0
lines changed

3 files changed

+334
-0
lines changed

‎platform/solaris/ips/create.sh

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -276,6 +276,8 @@ PKG pkgsend add file tools/sync/teamware.py \
276276
mode=0555 owner=root group=bin path=/usr/opengrok/bin/teamware.py
277277
PKG pkgsend add file tools/sync/utils.py \
278278
mode=0555 owner=root group=bin path=/usr/opengrok/bin/utils.py
279+
PKG pkgsend add file tools/sync/projadm.py \
280+
mode=0555 owner=root group=bin path=/usr/opengrok/bin/projadm.py
279281

280282
PKG pkgsend add file dist/opengrok.jar \
281283
mode=0444 owner=root group=bin path=/usr/opengrok/lib/opengrok.jar

‎tools/sync/projadm.py

Lines changed: 299 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,299 @@
1+
#!/usr/bin/env python3
2+
3+
# CDDL HEADER START
4+
#
5+
# The contents of this file are subject to the terms of the
6+
# Common Development and Distribution License (the "License").
7+
# You may not use this file except in compliance with the License.
8+
#
9+
# See LICENSE.txt included in this distribution for the specific
10+
# language governing permissions and limitations under the License.
11+
#
12+
# When distributing Covered Code, include this CDDL HEADER in each
13+
# file and include the License file at LICENSE.txt.
14+
# If applicable, add the following below this CDDL HEADER, with the
15+
# fields enclosed by brackets "[]" replaced with your own identifying
16+
# information: Portions Copyright [yyyy] [name of copyright owner]
17+
#
18+
# CDDL HEADER END
19+
20+
#
21+
# Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
22+
#
23+
24+
"""
25+
This script is wrapper of commands to add/remove project or refresh
26+
configuration using read-only configuration.
27+
"""
28+
29+
30+
import os
31+
from os import path
32+
import sys
33+
import argparse
34+
import filelock
35+
from filelock import Timeout
36+
from command import Command
37+
import logging
38+
import tempfile
39+
import shutil
40+
import stat
41+
from utils import get_command
42+
43+
44+
MAJOR_VERSION = sys.version_info[0]
45+
if (MAJOR_VERSION < 3):
46+
print("Need Python 3, you are running {}".format(MAJOR_VERSION))
47+
sys.exit(1)
48+
49+
__version__ = "0.1"
50+
51+
52+
def exec_command(doit, logger, cmd, msg):
53+
"""
54+
Execute given command and return its output.
55+
Exit the program on failure.
56+
"""
57+
cmd = Command(cmd, logger=logger)
58+
if not doit:
59+
logger.info(cmd)
60+
return
61+
cmd.execute()
62+
if cmd.getstate() is not Command.FINISHED or cmd.getretcode() != 0:
63+
logger.error(msg)
64+
logger.error(cmd.getoutput())
65+
sys.exit(1)
66+
67+
return cmd.getoutput()
68+
69+
70+
def get_config_file(basedir):
71+
"""
72+
Return configuration file in basedir
73+
"""
74+
75+
return path.join(basedir, "etc", "configuration.xml")
76+
77+
78+
def config_refresh(doit, logger, basedir, messages, configmerge, roconfig):
79+
"""
80+
Refresh current configuration file with configuration retrieved
81+
from webapp merged with readonly configuration.
82+
83+
1. retrieves current configuration from the webapp,
84+
stores it into temporary file
85+
2. merge the read-only config with the config from previous step
86+
- this is done as a workaround for
87+
https://github.com/oracle/opengrok/issues/2002
88+
"""
89+
90+
if not roconfig:
91+
logger.debug("No read-only configuration specified, not refreshing")
92+
return
93+
94+
logger.info('Refreshing configuration and merging with read-only '
95+
'configuration')
96+
current_config = exec_command(doit, logger,
97+
[messages, '-n', 'config', '-t', 'getconf'],
98+
"getting configuration failed")
99+
with tempfile.NamedTemporaryFile() as fc:
100+
logger.debug("Temporary file for current config: {}".format(fc.name))
101+
if doit:
102+
fc.write(bytearray(''.join(current_config), "UTF-8"))
103+
merged_config = exec_command(doit, logger,
104+
[configmerge, roconfig, fc.name],
105+
"cannot merge configuration")
106+
with tempfile.NamedTemporaryFile() as fm:
107+
logger.debug("Temporary file for merged config: {}".
108+
format(fm.name))
109+
if doit:
110+
fm.write(bytearray(''.join(merged_config), "UTF-8"))
111+
main_config = get_config_file(basedir)
112+
if path.isfile(main_config):
113+
if doit:
114+
#
115+
# Copy the file so that close() triggered unlink()
116+
# does not fail.
117+
#
118+
logger.debug("Copying {} to {}".
119+
format(fm.name, main_config))
120+
try:
121+
shutil.copyfile(fm.name, main_config)
122+
except PermissionError:
123+
logger.error('Failed to copy {} to {} (permissions)'.
124+
format(fm.name, main_config))
125+
sys.exit(1)
126+
except OSError:
127+
logger.error('Failed to copy {} to {} (I/O)'.
128+
format(fm.name, main_config))
129+
sys.exit(1)
130+
else:
131+
logger.error("file {} does not exist".format(main_config))
132+
sys.exit(1)
133+
134+
135+
def project_add(doit, logger, project, messages):
136+
"""
137+
Adds a project to configuration. Works in multiple steps:
138+
139+
1. add the project to configuration
140+
2. refresh on disk configuration
141+
"""
142+
143+
logger.info("Adding project {}".format(project))
144+
exec_command(doit, logger,
145+
[messages, '-n', 'project', '-t', project, 'add'],
146+
"adding of the project failed")
147+
148+
149+
def project_delete(doit, logger, project, messages):
150+
"""
151+
Delete the project for configuration and all its data.
152+
Works in multiple steps:
153+
154+
1. delete the project from configuration and its indexed data
155+
2. refresh on disk configuration
156+
3. delete the source code for the project
157+
"""
158+
159+
# Be extra careful as we will be recursively removing directory structure.
160+
if not project or len(project) == 0:
161+
raise Exception("invalid call to project_delete(): missing project")
162+
163+
logger.info("Deleting project {} and its index data".format(project))
164+
exec_command(doit, logger,
165+
[messages, '-n', 'project', '-t', project, 'delete'],
166+
"deletion of the project failed")
167+
168+
src_root = exec_command(True, logger,
169+
[messages, '-n', 'config', '-t', 'get',
170+
'sourceRoot'], "cannot get config")
171+
src_root = src_root[0].rstrip()
172+
logger.debug("Source root = {}".format(src_root))
173+
if not src_root or len(src_root) == 0:
174+
raise Exception("source root empty")
175+
sourcedir = path.join(src_root, project)
176+
logger.debug("Removing directory tree {}".format(sourcedir))
177+
if doit:
178+
logger.info("Removing source code under {}".format(sourcedir))
179+
shutil.rmtree(sourcedir)
180+
181+
182+
if __name__ == '__main__':
183+
parser = argparse.ArgumentParser(description='grok configuration '
184+
'management.',
185+
formatter_class=argparse.
186+
ArgumentDefaultsHelpFormatter)
187+
parser.add_argument('-D', '--debug', action='store_true',
188+
help='Enable debug prints')
189+
parser.add_argument('-b', '--base', default="/var/opengrok",
190+
help='OpenGrok instance base directory')
191+
parser.add_argument('-R', '--roconfig',
192+
help='OpenGrok read-only configuration file')
193+
parser.add_argument('-m', '--messages',
194+
help='path to the Messages binary')
195+
parser.add_argument('-c', '--configmerge',
196+
help='path to the ConfigMerge binary')
197+
parser.add_argument('-u', '--upload', action='store_true',
198+
help='Upload configuration at the end')
199+
parser.add_argument('-n', '--noop', action='store_false', default=True,
200+
help='Do not run any commands or modify any config'
201+
', just report. Usually implies the --debug option.')
202+
203+
group = parser.add_mutually_exclusive_group()
204+
group.add_argument('-a', '--add', metavar='project', nargs='+',
205+
help='Add project (assumes its source is available '
206+
'under source root')
207+
group.add_argument('-d', '--delete', metavar='project', nargs='+',
208+
help='Delete project and its data and source code')
209+
group.add_argument('-r', '--refresh', action='store_true',
210+
help='Refresh configuration from read-only '
211+
'configuration')
212+
213+
args = parser.parse_args()
214+
215+
#
216+
# Setup logger as a first thing after parsing arguments so that it can be
217+
# used through the rest of the program.
218+
#
219+
if args.debug:
220+
logging.basicConfig(level=logging.DEBUG)
221+
else:
222+
logging.basicConfig(format="%(message)s", level=logging.INFO)
223+
224+
logger = logging.getLogger(os.path.basename(sys.argv[0]))
225+
226+
# Set the base directory
227+
if args.base:
228+
if path.isdir(args.base):
229+
logger.debug("Using {} as instance base".
230+
format(args.base))
231+
else:
232+
logger.error("Not a directory: {}".format(args.base))
233+
sys.exit(1)
234+
235+
# read-only configuration file.
236+
if args.roconfig:
237+
if path.isfile(args.roconfig):
238+
logger.debug("Using {} as read-only config".format(args.roconfig))
239+
else:
240+
logger.error("File {} does not exist".format(args.roconfig))
241+
sys.exit(1)
242+
243+
if args.refresh and not args.roconfig:
244+
logger.error("-r requires -R")
245+
sys.exit(1)
246+
247+
# XXX replace Messages with REST request after issue #1801
248+
messages_file = get_command(logger, args.messages, "Messages")
249+
configmerge_file = get_command(logger, args.configmerge, "ConfigMerge")
250+
251+
lock = filelock.FileLock(os.path.join(tempfile.gettempdir(),
252+
os.path.basename(sys.argv[0]) + ".lock"))
253+
try:
254+
with lock.acquire(timeout=0):
255+
if args.add:
256+
for proj in args.add:
257+
project_add(doit=args.noop, logger=logger,
258+
project=proj,
259+
messages=messages_file)
260+
261+
config_refresh(doit=args.noop, logger=logger,
262+
basedir=args.base,
263+
messages=messages_file,
264+
configmerge=configmerge_file,
265+
roconfig=args.roconfig)
266+
elif args.delete:
267+
for proj in args.delete:
268+
project_delete(doit=args.noop, logger=logger,
269+
project=proj,
270+
messages=messages_file)
271+
272+
config_refresh(doit=args.noop, logger=logger,
273+
basedir=args.base,
274+
messages=messages_file,
275+
configmerge=configmerge_file,
276+
roconfig=args.roconfig)
277+
elif args.refresh:
278+
config_refresh(doit=args.noop, logger=logger,
279+
basedir=args.base,
280+
messages=messages_file,
281+
configmerge=configmerge_file,
282+
roconfig=args.roconfig)
283+
else:
284+
parser.print_help()
285+
sys.exit(1)
286+
287+
if args.upload:
288+
main_config = get_config_file(basedir=args.base)
289+
if path.isfile(main_config):
290+
exec_command(doit=args.noop, logger=logger,
291+
cmd=[messages_file, '-n', 'config', '-t',
292+
'setconf', main_config],
293+
msg="cannot upload configuration to webapp")
294+
else:
295+
logger.error("file {} does not exist".format(main_config))
296+
sys.exit(1)
297+
except Timeout:
298+
logger.warning("Already running, exiting.")
299+
sys.exit(1)

‎tools/sync/utils.py

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
#
2323

2424
import os
25+
from shutil import which
26+
import logging
2527

2628

2729
def is_exe(fpath):
@@ -38,3 +40,34 @@ def check_create_dir(path):
3840
except OSError:
3941
logger.error("cannot create {} directory".format(path))
4042
sys.exit(1)
43+
44+
45+
def get_command(logger, path, name):
46+
"""
47+
Get the path to the command specified by path and name.
48+
If the path does not contain executable, search for the command
49+
according to name in OS environment and/or dirname.
50+
51+
Return path to the command or None.
52+
"""
53+
54+
cmd_file = None
55+
if path:
56+
cmd_file = which(path)
57+
if not is_exe(cmd_file):
58+
logger.error("file {} is not executable file".
59+
format(path))
60+
sys.exit(1)
61+
else:
62+
cmd_file = which(name)
63+
if not cmd_file:
64+
# try to search within dirname()
65+
cmd_file = which(name,
66+
path=os.path.dirname(sys.argv[0]))
67+
if not cmd_file:
68+
logger.error("cannot determine path to the {} script".
69+
format(name))
70+
sys.exit(1)
71+
logger.debug("{} = {}".format(name, cmd_file))
72+
73+
return cmd_file

0 commit comments

Comments
 (0)