introduce check_incoming configuration property

allow to specify mirror configuration in Docker

Author: Vladimir Kotal

docker/README.md

@@ -71,6 +71,17 @@ The container exports ports 8080 for OpenGrok.
 
 The volume mounted to `/opengrok/src` should contain the projects you want to make searchable (in sub directories). You can use common revision control checkouts (git, svn, etc...) and OpenGrok will make history and blame information available.
 
+## Directories
+
+The image contains these directories:
+
+| Directory | Description |
+| --------- | ----------- |
+`/opengrok/etc` | stores the configuration for both web app and indexer
+`/opengrok/data` | data root - index data
+`/opengrok/src` | source root - input data
+`/scripts` | startup script and top level configuration. Do not override unless debugging.
+
 ## Environment Variables
 
 | Docker Environment Var. | Default value | Description |
@@ -80,7 +91,7 @@ The volume mounted to `/opengrok/src` should contain the projects you want to ma
 `NOMIRROR` | empty | To avoid the mirroring step, set the variable to non-empty value.
 `URL_ROOT` | `/` | Override the sub-URL that OpenGrok should run on.
 `WORKERS` | number of CPUs in the container | number of workers to use for syncing (applies only to setup with projects enabled)
-`AVOID_PROJECTS` | empty | run in project less configuration. Set to non empty value disables projects.
+`AVOID_PROJECTS` | empty | run in project less configuration. Set to non empty value disables projects. Also disables repository synchronization.
 `REST_PORT` | 5000 | TCP port where simple REST app listens for GET requests on `/reindex` to trigger manual reindex.
 `REST_TOKEN` | None | if set, the REST app will require this token as Bearer token in order to trigger reindex.
 `READONLY_CONFIG_FILE` | None | if set, the configuration will be merged with configuration from this file. This is run when the container starts.

docker/start.py

@@ -64,7 +64,8 @@
 OPENGROK_DATA_ROOT = os.path.join(OPENGROK_BASE_DIR, "data")
 OPENGROK_SRC_ROOT = os.path.join(OPENGROK_BASE_DIR, "src")
 BODY_INCLUDE_FILE = os.path.join(OPENGROK_DATA_ROOT, "body_include")
-OPENGROK_CONFIG_FILE = os.path.join(OPENGROK_BASE_DIR, "etc",
+OPENGROK_CONFIG_DIR = os.path.join(OPENGROK_BASE_DIR, "etc")
+OPENGROK_CONFIG_FILE = os.path.join(OPENGROK_CONFIG_DIR,
                                     "configuration.xml")
 OPENGROK_WEBAPPS_DIR = os.path.join(tomcat_root, "webapps")
 OPENGROK_JAR = os.path.join(OPENGROK_LIB_DIR, 'opengrok.jar')
@@ -532,6 +533,11 @@ def main():
                                        multiprocessing.cpu_count())
         logger.info('Number of sync workers: {}'.format(num_workers))
 
+        mirror_config = os.path.join(OPENGROK_CONFIG_DIR, "mirror.yml")
+        if not os.path.exists(mirror_config):
+            with open(mirror_config, 'w') as fp:
+                pass
+
         worker_function = project_syncer
         syncer_args = (logger, log_level, uri,
                        OPENGROK_CONFIG_FILE,

docker/sync.yml

@@ -6,7 +6,7 @@ commands:
     duration: PT1H
     tags: ['%PROJECT%']
     text: resync + reindex in progress
-- command: [opengrok-mirror, -I, -U, '%URL%', '%PROJECT%']
+- command: [opengrok-mirror, -c, '/opengrok/etc/mirror.yml', -I, -U, '%URL%', '%PROJECT%']
 - command: [opengrok-reindex-project, --printoutput,
     --jar, /opengrok/lib/opengrok.jar, -U, '%URL%', -P, '%PROJECT%', --,
     -r, dirbased, -G, -m, '256', --leadingWildCards, 'on',

tools/src/main/python/opengrok_tools/utils/mirror.py

@@ -36,7 +36,7 @@
     SUCCESS_EXITVAL
 )
 from .patterns import PROJECT_SUBST, COMMAND_PROPERTY
-from .utils import is_exe, check_create_dir, get_int, is_web_uri
+from .utils import is_exe, check_create_dir, get_int, is_web_uri, get_bool
 from .opengrok import get_repos, get_repo_type, get_uri
 from .hook import run_hook
 from .command import Command
@@ -51,6 +51,7 @@
 CMD_TIMEOUT_PROPERTY = 'command_timeout'
 IGNORED_REPOS_PROPERTY = 'ignored_repos'
 PROXY_PROPERTY = 'proxy'
+INCOMING_PROPERTY = 'incoming_check'
 COMMANDS_PROPERTY = 'commands'
 DISABLED_PROPERTY = 'disabled'
 DISABLED_REASON_PROPERTY = 'disabled-reason'
@@ -158,7 +159,7 @@ def get_project_properties(project_config, project_name, hookdir):
     :param project_name: name of the project
     :param hookdir: directory with hooks
     :return: list of properties: prehook, posthook, hook_timeout,
-    command_timeout, use_proxy, ignored_repos
+    command_timeout, use_proxy, ignored_repos, check_changes
     """
 
     prehook = None
@@ -167,6 +168,7 @@ def get_project_properties(project_config, project_name, hookdir):
     command_timeout = None
     use_proxy = False
     ignored_repos = None
+    check_changes = None
 
     logger = logging.getLogger(__name__)
 
@@ -213,11 +215,17 @@ def get_project_properties(project_config, project_name, hookdir):
             logger.debug("will use proxy")
             use_proxy = True
 
+        if project_config.get(INCOMING_PROPERTY):
+            check_changes = get_bool(logger, ("incoming check for project {}".
+                                              format(project_name)),
+                                     project_config.get(INCOMING_PROPERTY))
+            logger.debug("incoming check = {}".format(check_changes))
+
     if not ignored_repos:
         ignored_repos = []
 
     return prehook, posthook, hook_timeout, command_timeout, \
-        use_proxy, ignored_repos
+        use_proxy, ignored_repos, check_changes
 
 
 def process_hook(hook_ident, hook, source_root, project_name, proxy,
@@ -361,7 +369,7 @@ def mirror_project(config, project_name, check_changes, uri,
     :param uri
     :param source_root
     :param headers: optional dictionary of HTTP headers
-    :param timeout: optional timeount in seconds for API call response
+    :param timeout: optional timeout in seconds for API call response
     :return exit code
     """
 
@@ -371,7 +379,8 @@ def mirror_project(config, project_name, check_changes, uri,
 
     project_config = get_project_config(config, project_name)
     prehook, posthook, hook_timeout, command_timeout, use_proxy, \
-        ignored_repos = get_project_properties(project_config,
+        ignored_repos, \
+        check_changes_proj = get_project_properties(project_config,
                                                project_name,
                                                config.get(HOOKDIR_PROPERTY))
 
@@ -385,16 +394,15 @@ def mirror_project(config, project_name, check_changes, uri,
         proxy = config.get(PROXY_PROPERTY)
 
     # We want this to be logged to the log file (if any).
-    if project_config:
-        if project_config.get(DISABLED_PROPERTY):
-            handle_disabled_project(config, project_name,
-                                    project_config.
-                                    get(DISABLED_REASON_PROPERTY),
-                                    headers=headers,
-                                    timeout=timeout)
-            logger.info("Project '{}' disabled, exiting".
-                        format(project_name))
-            return CONTINUE_EXITVAL
+    if project_config and project_config.get(DISABLED_PROPERTY):
+        handle_disabled_project(config, project_name,
+                                project_config.
+                                get(DISABLED_REASON_PROPERTY),
+                                headers=headers,
+                                timeout=timeout)
+        logger.info("Project '{}' disabled, exiting".
+                    format(project_name))
+        return CONTINUE_EXITVAL
 
     #
     # Cache the repositories first. This way it will be known that
@@ -415,6 +423,9 @@ def mirror_project(config, project_name, check_changes, uri,
                     format(project_name))
         return CONTINUE_EXITVAL
 
+    if check_changes_proj is not None:
+        check_changes = check_changes_proj
+
     # Check if the project or any of its repositories have changed.
     if check_changes:
         r = process_changes(repos, project_name, uri, headers=headers)

tools/src/main/python/opengrok_tools/utils/utils.py

@@ -27,6 +27,7 @@
 import logging
 import sys
 from urllib.parse import urlparse
+from distutils import util
 from .exitvals import (
     FAILURE_EXITVAL,
 )
@@ -97,6 +98,20 @@ def get_int(logger, name, value):
         return None
 
 
+def get_bool(logger, name, value):
+    """
+    If the supplied value is bool, return it. Otherwise return None.
+    """
+    if value is None:
+        return None
+
+    try:
+        return bool(util.strtobool(value))
+    except ValueError:
+        logger.error("'{}' is not a number: {}".format(name, value))
+        return None
+
+
 def is_web_uri(url):
     """
     Check if given string is web URL.