Python tools improved incoming check (#2847)

Improving --incoming check to check also the index presence
for the particular project.

After that, the name "incoming" could be a little misleading
so I refactored the name to "check changes".

fixes #2844

Author: tulinkry

opengrok-tools/src/main/python/opengrok_tools/mirror.py

@@ -20,6 +20,7 @@
 
 #
 # Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
+# Portions Copyright (c) 2019, Krystof Tulinger <[email protected]>
 #
 
 """
@@ -57,11 +58,11 @@
 if major_version < 3:
     fatal("Need Python 3, you are running {}".format(major_version))
 
-__version__ = "0.8"
+__version__ = "0.9"
 
 
 def worker(args):
-    project_name, logdir, loglevel, backupcount, config, incoming, uri, \
+    project_name, logdir, loglevel, backupcount, config, check_changes, uri, \
         source_root, batch = args
 
     if batch:
@@ -71,7 +72,7 @@ def worker(args):
                          get_class_basename())
 
     return mirror_project(config, project_name,
-                          incoming,
+                          check_changes,
                           uri, source_root)
 
 
@@ -94,9 +95,11 @@ def main():
                         help='batch mode - will log into a file')
     parser.add_argument('-B', '--backupcount', default=8,
                         help='how many log files to keep around in batch mode')
-    parser.add_argument('-I', '--incoming', action='store_true',
-                        help='Check for incoming changes, terminate the '
-                             'processing if not found.')
+    parser.add_argument('-I', '--check-changes', action='store_true',
+                        help='Check for changes in the project or its'
+                             ' repositories,'
+                             ' terminate the processing'
+                             ' if no change is found.')
     parser.add_argument('-w', '--workers', default=cpu_count(),
                         help='Number of worker processes')
 
@@ -174,7 +177,7 @@ def main():
                 for x in projects:
                     worker_args.append([x, logdir, args.loglevel,
                                         args.backupcount, config,
-                                        args.incoming,
+                                        args.check_changes,
                                         args.uri, source_root,
                                         args.batch])
                 try:

opengrok-tools/src/main/python/opengrok_tools/utils/mirror.py

@@ -26,14 +26,18 @@
 import os
 import fnmatch
 import logging
+import urllib
+
+from requests.exceptions import HTTPError
 
 from .exitvals import (
     FAILURE_EXITVAL,
     CONTINUE_EXITVAL,
     SUCCESS_EXITVAL
 )
 from .utils import is_exe, check_create_dir, get_int
-from .opengrok import get_repos, get_repo_type
+from .webutil import get
+from .opengrok import get_repos, get_repo_type, get_uri
 from .hook import run_hook
 
 from ..scm.repofactory import get_repository
@@ -196,13 +200,14 @@ def get_project_properties(project_config, project_name, hookdir):
         use_proxy, ignored_repos
 
 
-def mirror_project(config, project_name, check_incoming, uri,
+def mirror_project(config, project_name, check_changes, uri,
                    source_root):
     """
     Mirror the repositories of single project.
     :param config global configuration dictionary
     :param project_name: name of the project
-    :param check_incoming:
+    :param check_changes: check for changes in the project or its repositories
+     and terminate if no change is found
     :param uri
     :param source_root
     :return exit code
@@ -244,22 +249,44 @@ def mirror_project(config, project_name, check_incoming, uri,
                     format(project_name))
         return CONTINUE_EXITVAL
 
-    # Check if any of the repositories contains incoming changes.
-    if check_incoming:
-        got_incoming = False
-        for repo in repos:
-            try:
-                if repo.incoming():
-                    logger.debug('Repository {} has incoming changes'.
-                                 format(repo))
-                    got_incoming = True
-                    break
-            except RepositoryException:
-                logger.error('Cannot determine incoming changes for '
-                             'repository {}'.format(repo))
-                return FAILURE_EXITVAL
+    # Check if the project or any of its repositories have changed.
+    if check_changes:
+        changes_detected = False
+
+        # check if the project is a new project - full index is necessary
+        try:
+            r = get(logger, get_uri(uri, 'api', 'v1', 'projects',
+                                    urllib.parse.quote_plus(project_name),
+                                    'property', 'indexed'))
+            r.raise_for_status()
+            if not bool(r.json()):
+                changes_detected = True
+                logger.debug('Project {} has not been indexed yet'
+                             .format(project_name))
+        except ValueError as e:
+            logger.error('Unable to parse project \'{}\' indexed flag: {}'
+                         .format(project_name, e))
+            return FAILURE_EXITVAL
+        except HTTPError as e:
+            logger.error('Unable to determine project \'{}\' indexed flag: {}'
+                         .format(project_name, e))
+            return FAILURE_EXITVAL
 
-        if not got_incoming:
+        # check if the project has any new changes in the SCM
+        if not changes_detected:
+            for repo in repos:
+                try:
+                    if repo.incoming():
+                        logger.debug('Repository {} has incoming changes'.
+                                     format(repo))
+                        changes_detected = True
+                        break
+                except RepositoryException:
+                    logger.error('Cannot determine incoming changes for '
+                                 'repository {}'.format(repo))
+                    return FAILURE_EXITVAL
+
+        if not changes_detected:
             logger.info('No incoming changes for repositories in '
                         'project {}'.
                         format(project_name))