add repository mirroring scripts

Author: Vladimir Kotal

README.md

@@ -141,6 +141,10 @@ Currently supported message types:
   * **get-repos** – get list of repositories in the form of relative paths to source root for given project(s)
   * **get-repos-type** – get repository type(s) for given project(s)
 
+6. **RepositoryMessage** – used for getting repository info.
+
+  * **get-repo-type** – get repository type
+
 ## 4. OpenGrok install
 
 ### 4.1 Installing on Solaris from *.p5p file

build.xml

@@ -682,9 +682,23 @@ Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
                 <include name="Groups"/>
             </tarfileset>
             <tarfileset dir="tools/sync" prefix="${distname}-${version}/bin/" mode="555">
-                <include name="sync.py"/>
                 <include name="command.py"/>
                 <include name="commands.py"/>
+                <include name="cvs.py"/>
+                <include name="filelock.py"/>
+                <include name="git.py"/>
+                <include name="hook.py"/>
+                <include name="mercurial.py"/>
+                <include name="mirror.py"/>
+                <include name="opengrok.py"/>
+                <include name="readconfig.py"/>
+                <include name="repofactory.py"/>
+                <include name="repository.py"/>
+                <include name="svn.py"/>
+                <include name="sync.py"/>
+                <include name="teamware.py"/>
+                <include name="testmercurial.py"/>
+                <include name="utils.py"/>
                 <include name="reindex-project.ksh"/>
             </tarfileset>
             <tarfileset dir="doc" prefix="${distname}-${version}/doc">

platform/solaris/ips/create.sh

@@ -252,6 +252,30 @@ PKG pkgsend add file tools/sync/filelock.py \
     mode=0555 owner=root group=bin path=/usr/opengrok/bin/filelock.py
 PKG pkgsend add file tools/sync/reindex-project.ksh \
     mode=0555 owner=root group=bin path=/usr/opengrok/bin/reindex-project.ksh
+PKG pkgsend add file tools/sync/cvs.py
+    mode=0555 owner=root group=bin path=/usr/opengrok/bin/cvs.py
+PKG pkgsend add file tools/sync/git.py
+    mode=0555 owner=root group=bin path=/usr/opengrok/bin/git.py
+PKG pkgsend add file tools/sync/hook.py
+    mode=0555 owner=root group=bin path=/usr/opengrok/bin/hook.py
+PKG pkgsend add file tools/sync/mercurial.py
+    mode=0555 owner=root group=bin path=/usr/opengrok/bin/mercurial.py
+PKG pkgsend add file tools/sync/mirror.py
+    mode=0555 owner=root group=bin path=/usr/opengrok/bin/mirror.py
+PKG pkgsend add file tools/sync/opengrok.py
+    mode=0555 owner=root group=bin path=/usr/opengrok/bin/opengrok.py
+PKG pkgsend add file tools/sync/readconfig.py
+    mode=0555 owner=root group=bin path=/usr/opengrok/bin/readconfig.py
+PKG pkgsend add file tools/sync/repofactory.py
+    mode=0555 owner=root group=bin path=/usr/opengrok/bin/repofactory.py
+PKG pkgsend add file tools/sync/repository.py
+    mode=0555 owner=root group=bin path=/usr/opengrok/bin/repository.py
+PKG pkgsend add file tools/sync/svn.py
+    mode=0555 owner=root group=bin path=/usr/opengrok/bin/svn.py
+PKG pkgsend add file tools/sync/teamware.py
+    mode=0555 owner=root group=bin path=/usr/opengrok/bin/teamware.py
+PKG pkgsend add file tools/sync/utils.py
+    mode=0555 owner=root group=bin path=/usr/opengrok/bin/utils.py
 
 PKG pkgsend add file dist/opengrok.jar \
     mode=0444 owner=root group=bin path=/usr/opengrok/lib/opengrok.jar

tools/sync/README

@@ -0,0 +1,70 @@
+
+Set of scripts to facilitate parallel project synchronization and mirroring
+
+Use e.g. like this:
+
+  # sync.py -c /scripts/sync.conf -d /ws-local/ -p
+
+where the sync.conf file contents might look like this:
+
+```
+  {
+     "commands": [["/usr/opengrok/bin/Messages", "-c", "info", "-e", "+1 hour",
+                   "-n", "normal", "-t", "ARG", "resync + reindex in progress"],
+                  ["sudo", "-u", "wsmirror", "/usr/opengrok/bin/mirror.py",
+                    "-c", "/opengrok/etc/mirror-config.yml", "-b",
+                    "--messages", "/usr/opengrok/bin/Messages"],
+                  ["sudo", "-u", "webservd", "/usr/opengrok/bin/reindex-project.ksh",
+                   "/opengrok/etc/opengrok.conf", "/usr/opengrok/bin"],
+                  ["/usr/opengrok/bin/Messages", "-n", "abort", "-t"],
+                  ["/scripts/check-indexer-logs.ksh"]]
+  }
+```
+
+The commands got appended project name unless one of their arguments is equal
+to 'ARG', in which case it is substituted with project name and no append is
+done.
+
+For per-project reindexing to work properly, reindex-project.ksh uses
+the logging.properties.template to make sure each project has its own
+log directory.
+
+The mirror-config.yml can look e.g. like this:
+
+```
+#
+# Commands (or paths - for specific repository types only)
+#
+commands:
+  hg: /usr/bin/hg
+  svn: /usr/bin/svn
+  teamware: /ontools/onnv-tools-i386/teamware/bin
+#
+# The proxy environment variables will be set for a project's repositories
+# if the 'proxy' property is True.
+#
+proxy:
+  http_proxy: proxy.example.com:80
+  https_proxy: proxy.example.com:80
+  ftp_proxy: proxy.example.com:80
+  no_proxy: example.com,foo.example.com
+hookdir: /tmp/hooks
+# per-project hooks relative to 'hookdir' above
+logdir: /tmp/logs
+#
+# Per project configuration.
+#
+projects:
+  http:
+    proxy: true
+  history:
+    disable: true
+  userland:
+    proxy: true
+    hooks:
+      pre: userland-pre.ksh
+      post: userland-post.ksh
+```
+
+See https://github.com/OpenGrok/OpenGrok/wiki/Per-project-management
+for more details on per-project management.

tools/sync/README.md

@@ -18,7 +18,7 @@
 #
 
 #
-# Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved.
 #
 
 import os
@@ -40,10 +40,12 @@ class Command:
     ERRORED = "errored"
 
     def __init__(self, cmd, args_subst=None, args_append=None, logger=None,
-                 excl_subst=False):
+                 excl_subst=False, work_dir=None, env_vars=None):
         self.cmd = cmd
         self.state = "notrun"
         self.excl_subst = excl_subst
+        self.work_dir = work_dir
+        self.env_vars = env_vars
 
         self.logger = logger or logging.getLogger(__name__)
         logging.basicConfig()
@@ -96,11 +98,27 @@ def fileno(self):
             def close(self):
                 os.close(self.write_fd)
 
+        if self.work_dir:
+            try:
+                os.chdir(self.work_dir)
+            except OSError as e:
+                self.state = Command.ERRORED
+                self.logger.error("Cannot change working directory to {}".
+                    format(self.work_dir))
+                return
+
         othr = OutputThread()
         try:
+            self.logger.debug("working directory = {}".format(os.getcwd()))
             self.logger.debug("command = {}".format(self.cmd))
-            p = subprocess.Popen(self.cmd, stderr=subprocess.STDOUT,
-                                 stdout=othr)
+            if self.env_vars:
+                my_env = os.environ.copy()
+                my_env.update(self.env_vars)
+                p = subprocess.Popen(self.cmd, stderr=subprocess.STDOUT,
+                                     stdout=othr, env=my_env)
+            else:
+                p = subprocess.Popen(self.cmd, stderr=subprocess.STDOUT,
+                                     stdout=othr)
             p.wait()
         except KeyboardInterrupt as e:
             self.logger.debug("Got KeyboardException while processing ",

tools/sync/command.py

@@ -0,0 +1,53 @@
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# See LICENSE.txt included in this distribution for the specific
+# language governing permissions and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at LICENSE.txt.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+#
+
+from command import Command
+from repository import Repository
+from utils import which
+
+
+class CVSRepository(Repository):
+    def __init__(self, logger, path, project, command, env, hooks):
+
+        super().__init__(logger, path, project, command, env, hooks)
+
+        if command:
+            self.command = command
+        else:
+            self.command = which("cvs")
+
+        if not self.command:
+            self.logger.error("Cannot get cvs command")
+            raise OSError
+
+    def reposync(self):
+        hg_command = [self.command, "update", "-dP"]
+        cmd = Command(hg_command, work_dir=self.path, env_vars=self.env)
+        cmd.execute()
+        self.logger.info(cmd.getoutput())
+        if cmd.getretcode() != 0 or cmd.getstate() != Command.FINISHED:
+            self.logger.error("failed to perform update for {}".
+                              format(self.path))
+            return 1
+
+        return 0

tools/sync/cvs.py

@@ -0,0 +1,52 @@
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# See LICENSE.txt included in this distribution for the specific
+# language governing permissions and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at LICENSE.txt.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+#
+
+from command import Command
+from repository import Repository
+from utils import which
+
+class GitRepository(Repository):
+    def __init__(self, logger, path, project, command, env, hooks):
+
+        super().__init__(logger, path, project, command, env, hooks)
+
+        if command:
+            self.command = command
+        else:
+            self.command = which("git")
+
+        if not self.command:
+            self.logger.error("Cannot get git command")
+            raise OSError
+
+    def reposync(self):
+        hg_command = [self.command, "pull", "--ff-only"]
+        cmd = Command(hg_command, work_dir=self.path, env_vars=self.env)
+        cmd.execute()
+        self.logger.info(cmd.getoutput())
+        if cmd.getretcode() != 0 or cmd.getstate() != Command.FINISHED:
+            self.logger.error("failed to perform pull for {}".
+                format(self.path))
+            return 1
+
+        return 0

tools/sync/git.py

@@ -0,0 +1,59 @@
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# See LICENSE.txt included in this distribution for the specific
+# language governing permissions and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at LICENSE.txt.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+#
+
+import os
+from command import Command
+import logging
+
+def run_hook(logger, script, path):
+    """
+    Change a working directory to specified path, run a command
+    and change the working directory back to its original value.
+
+    Return 0 on success, 1 on failure.
+    """
+
+    orig_cwd = os.getcwd()
+
+    try:
+        os.chdir(path)
+    except:
+        logger.error("Cannot chdir to {}".format(path))
+        return 1
+
+    cmd = Command([script])
+    cmd.execute()
+    if cmd.state is not "finished" or cmd.getretcode() != 0:
+        logger.error("failed to execute {}".format(cmd))
+        logger.debug(cmd.getoutput())
+        return 1
+
+    logger.info(cmd.getoutput())
+
+    try:
+        os.chdir(orig_cwd)
+    except:
+        logger.error("Cannot chdir to {}".format(orig_cwd))
+        return 1
+
+    return 0