rename REINDEX to SYNC_PERIOD_MINUTES and allow disable (#3458)

fixes #3455

Author: Vladimir Kotal

docker/README.md

@@ -22,7 +22,7 @@ If you happen to have one of the following:
   - Source Code Management systems not supported in the image (e.g. Perforce,
     Clearcase, etc.)
   - need for authentication/authorization
-  
+
 then it is advisable to run OpenGrok standalone or construct your own Docker
 image based on the official one.
 
@@ -74,13 +74,13 @@ The volume mounted to `/opengrok/src` should contain the projects you want to ma
 
 | Docker Environment Var. | Default value | Description |
 | ----------------------- | ------------- | ----------- |
-`REINDEX` | 10 | Period of automatic mirroring/reindexing in minutes. Setting to `0` will disable automatic indexing. You can manually trigger an reindex using docker exec: `docker exec <container> /scripts/index.sh`
+`SYNC_PERIOD_MINUTES` | 10 | Period of automatic synchronization (i.e. mirroring + reindexing) in minutes. Setting to `0` will disable automatic syncing.
 `INDEXER_OPT` | empty | pass extra options to OpenGrok Indexer. For example, `-i d:vendor` will remove all the `*/vendor/*` files from the index. You can check the indexer options on https://github.com/oracle/opengrok/wiki/Python-scripts-transition-guide
 `NOMIRROR` | empty | To avoid the mirroring step, set the variable to non-empty value.
 `URL_ROOT` | `/` | Override the sub-URL that OpenGrok should run on.
 `WORKERS` | number of CPUs in the container | number of workers to use for syncing
 
-To specify environment variable for `docker run`, use the `-e` option, e.g. `-e REINDEX=30`
+To specify environment variable for `docker run`, use the `-e` option, e.g. `-e SYNC_PERIOD_MINUTES=30`
 
 ## OpenGrok Web-Interface
 
@@ -105,7 +105,7 @@ services:
     ports:
       - "8080:8080/tcp"
     environment:
-      REINDEX: '60'
+      SYNC_PERIOD_MINUTES: '60'
     # Volumes store your data between container upgrades
     volumes:
        - '~/opengrok/src/:/opengrok/src/'  # source code
@@ -123,7 +123,7 @@ Equivalent `docker run` command would look like this:
 docker run -d \
     --name opengrok \
     -p 8080:8080/tcp \
-    -e REINDEX="60" \
+    -e SYNC_PERIOD_MINUTES="60" \
     -v ~/opengrok-src/:/opengrok/src/ \
     -v ~/opengrok-etc/:/opengrok/etc/ \
     -v ~/opengrok-data/:/opengrok/data/ \

docker/start.py

@@ -218,7 +218,7 @@ def merge_commands_env(commands, env):
     return commands
 
 
-def syncer(logger, loglevel, uri, config_path, reindex, numworkers, env):
+def syncer(logger, loglevel, uri, config_path, sync_period, numworkers, env):
     """
     Wrapper for running opengrok-sync.
     To be run in a thread/process in the background.
@@ -242,7 +242,7 @@ def syncer(logger, loglevel, uri, config_path, reindex, numworkers, env):
         #
         # The driveon=True is needed for the initial indexing of newly
         # added project, otherwise the incoming check in the opengrok-mirror
-        # would short circuit it.
+        # program would short circuit it.
         #
         if env:
             logger.info('Merging commands with environment')
@@ -262,7 +262,7 @@ def syncer(logger, loglevel, uri, config_path, reindex, numworkers, env):
 
         save_config(logger, uri, config_path)
 
-        sleep_seconds = int(reindex) * 60
+        sleep_seconds = sync_period * 60
         logger.info("Sleeping for {} seconds".format(sleep_seconds))
         time.sleep(sleep_seconds)
 
@@ -305,13 +305,22 @@ def main():
     logger.debug("URL_ROOT = {}".format(url_root))
     logger.debug("URI = {}".format(uri))
 
-    # default period for reindexing (in minutes)
-    reindex_env = os.environ.get('REINDEX')
-    if reindex_env:
-        reindex_min = reindex_env
+    # default period for syncing (in minutes)
+    sync_period = 10
+    sync_env = os.environ.get('SYNC_TIME_MINUTES')
+    if sync_env:
+        try:
+            n = int(sync_env)
+            if n >= 0:
+                sync_period = n
+        except ValueError:
+            logger.error("SYNC_TIME_MINUTES is not a number: {}".
+                         format(sync_env))
+
+    if sync_period == 0:
+        logger.info("synchronization disabled")
     else:
-        reindex_min = 10
-    logger.debug("reindex period = {} minutes".format(reindex_min))
+        logger.info("synchronization period = {} minutes".format(sync_period))
 
     # Note that deploy is done before Tomcat is started.
     deploy(logger, url_root)
@@ -335,18 +344,25 @@ def main():
             os.path.getsize(OPENGROK_CONFIG_FILE) == 0:
         create_bare_config(logger)
 
-    if os.environ.get('WORKERS'):
-        num_workers = os.environ.get('WORKERS')
-    else:
+    if sync_period > 0:
         num_workers = multiprocessing.cpu_count()
-    logger.info('Number of sync workers: {}'.format(num_workers))
-
-    logger.debug("Starting sync thread")
-    thread = threading.Thread(target=syncer, name="Sync thread",
-                              args=(logger, log_level, uri,
-                                    OPENGROK_CONFIG_FILE,
-                                    reindex_min, num_workers, env))
-    thread.start()
+        workers_env = os.environ.get('WORKERS')
+        if workers_env:
+            try:
+                n = int(workers_env)
+                if n > 0:
+                    num_workers = n
+            except ValueError:
+                logger.error("WORKERS is not a number: {}".format(workers_env))
+
+        logger.info('Number of sync workers: {}'.format(num_workers))
+
+        logger.debug("Starting sync thread")
+        thread = threading.Thread(target=syncer, name="Sync thread",
+                                  args=(logger, log_level, uri,
+                                        OPENGROK_CONFIG_FILE,
+                                        sync_period, num_workers, env))
+        thread.start()
 
     # Start Tomcat last. It will be the foreground process.
     logger.info("Starting Tomcat")

tools/src/main/python/opengrok_tools/mirror.py

@@ -96,14 +96,14 @@ def main():
                         help='batch mode - will log into a file')
     parser.add_argument('-L', '--logdir',
                         help='log directory')
-    parser.add_argument('-B', '--backupcount', default=8,
+    parser.add_argument('-B', '--backupcount', default=8, type=int,
                         help='how many log files to keep around in batch mode')
     parser.add_argument('-I', '--check-changes', action='store_true',
                         help='Check for changes in the project or its'
                              ' repositories,'
                              ' terminate the processing'
                              ' if no change is found.')
-    parser.add_argument('-w', '--workers', default=cpu_count(),
+    parser.add_argument('-w', '--workers', default=cpu_count(), type=int,
                         help='Number of worker processes')
     add_http_headers(parser)
     parser.add_argument('--api_timeout', type=int,

tools/src/main/python/opengrok_tools/sync.py

@@ -100,7 +100,7 @@ def do_sync(loglevel, commands, cleanup, dirs_to_process, ignore_errors,
 
     # Map the commands into pool of workers so they can be processed.
     retval = SUCCESS_EXITVAL
-    with Pool(processes=int(numworkers)) as pool:
+    with Pool(processes=numworkers) as pool:
         try:
             cmds_base_results = pool.map(worker, cmds_base, 1)
         except KeyboardInterrupt:
@@ -127,7 +127,7 @@ def main():
                                              tool_version=__version__)
                                      ])
     parser.add_argument('-w', '--workers', default=multiprocessing.cpu_count(),
-                        help='Number of worker processes')
+                        help='Number of worker processes', type=int)
 
     # There can be only one way how to supply list of projects to process.
     group1 = parser.add_mutually_exclusive_group()