Skip to content

Commit 3ca7a59

Browse files
vladakvladak
committed
add support for parallel indexing
fixes #876
1 parent c58b007 commit 3ca7a59

File tree

3 files changed

+107
-31
lines changed

3 files changed

+107
-31
lines changed

‎OpenGrok

Lines changed: 68 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@
7474
# increase it to 256MB, but do increase JVM to
7575
# 4/8/16GB ! Lucene defaults to 8 threads.
7676
# Increase JVM memory as noted using JAVA_OPTS
77+
# - OPENGROK_LOGGER_CONFIG_PATH Set path to custom logging.properties file.
7778
#
7879
# Notes:
7980
# (*) Any Non-Empty String will enable these options
@@ -93,6 +94,7 @@ Usage()
9394
echo ""
9495
echo "Usage: ${progname} <deploy|derby|update|updateQuietly|usage|help>"
9596
echo " ${progname} index [<directory>]"
97+
echo " ${progname} indexpart [<directory1> ..]"
9698
echo " ${progname} clearHistory [<repository_name>]"
9799
echo ""
98100
echo " Optional environment variables:"
@@ -193,7 +195,15 @@ DefaultInstanceConfiguration()
193195
fi
194196

195197
# REQUIRED: Logger Configuration
196-
LOGGER_CONFIG_PATH="${OPENGROK_INSTANCE_BASE}/${LOGGER_CONFIG_FILE}"
198+
if [ -n "${OPENGROK_LOGGER_CONFIG_PATH}" ]; then
199+
if [ ! -f "${OPENGROK_LOGGER_CONFIG_PATH}" ]; then
200+
echo "File ${OPENGROK_LOGGER_CONFIG_PATH} does not exist"
201+
exit 1
202+
fi
203+
LOGGER_CONFIG_PATH=${OPENGROK_LOGGER_CONFIG_PATH}
204+
else
205+
LOGGER_CONFIG_PATH="${OPENGROK_INSTANCE_BASE}/${LOGGER_CONFIG_FILE}"
206+
fi
197207
LOGGER_PROPERTIES="-Djava.util.logging.config.file=${LOGGER_CONFIG_PATH}"
198208

199209
# REQUIRED: Java Archive of OpenGrok (Installation Location)
@@ -778,35 +788,56 @@ CreateRuntimeRequirements()
778788
fi
779789
}
780790

781-
StdInvocation()
791+
CommonInvocation()
782792
{
783-
${DO} ${JAVA} ${JAVA_OPTS} ${PROPERTIES} \
784-
${LOGGER_PROPERTIES} \
785-
${JAVA_DEBUG} \
786-
-jar ${OPENGROK_JAR} \
787-
${IGNORE_PATTERNS} ${ENABLE_PROJECTS} \
788-
${DERBY_OPTIONS} \
789-
${HISTORY_TAGS} \
790-
${SCAN_FOR_REPOSITORY} ${REMOTE_REPOSITORIES} \
791-
${SCAN_DEPTH} \
792-
${VERBOSE} ${QUIET} \
793-
${PROGRESS} \
794-
${EXUBERANT_CTAGS:+-c} ${EXUBERANT_CTAGS} \
795-
${CTAGS_OPTIONS_FILE:+-o} ${CTAGS_OPTIONS_FILE} \
796-
${OPENGROK_FLUSH_RAM_BUFFER_SIZE} ${SKIN} ${LEADING_WILDCARD} \
797-
${READ_XML_CONF} \
798-
-W ${XML_CONFIGURATION} \
799-
${WEBAPP_CONFIG} \
800-
${WEBAPP_CONTEXT} \
801-
-s ${SRC_ROOT} -d ${DATA_ROOT} \
793+
${DO} ${JAVA} ${JAVA_OPTS} ${PROPERTIES} \
794+
${LOGGER_PROPERTIES} \
795+
${JAVA_DEBUG} \
796+
-jar ${OPENGROK_JAR} \
797+
${IGNORE_PATTERNS} \
798+
${DERBY_OPTIONS} \
799+
${HISTORY_TAGS} \
800+
${REMOTE_REPOSITORIES} \
801+
${SCAN_DEPTH} \
802+
${VERBOSE} ${QUIET} \
803+
${PROGRESS} \
804+
${EXUBERANT_CTAGS:+-c} ${EXUBERANT_CTAGS} \
805+
${CTAGS_OPTIONS_FILE:+-o} ${CTAGS_OPTIONS_FILE} \
806+
${OPENGROK_FLUSH_RAM_BUFFER_SIZE} ${SKIN} ${LEADING_WILDCARD} \
807+
${READ_XML_CONF} \
802808
"${@}"
803809
}
804810

811+
StdInvocation()
812+
{
813+
CommonInvocation \
814+
-W ${XML_CONFIGURATION} \
815+
${SCAN_FOR_REPOSITORY} \
816+
${ENABLE_PROJECTS} \
817+
${WEBAPP_CONFIG} \
818+
${WEBAPP_CONTEXT} \
819+
-s ${SRC_ROOT} \
820+
-d ${DATA_ROOT}
821+
}
822+
805823
UpdateGeneratedData()
806824
{
807825
StdInvocation -H
808826
}
809827

828+
UpdateDataPartial()
829+
{
830+
extra_args=""
831+
for arg in "$@"; do
832+
if [ -n "$extra_args" ]; then
833+
extra_args="$extra_args -h $arg"
834+
else
835+
extra_args="-h $arg"
836+
fi
837+
done
838+
CommonInvocation $extra_args "$@"
839+
}
840+
810841
UpdateDescriptionCache()
811842
{
812843
# OPTIONAL : Update the EftarFile data
@@ -908,7 +939,7 @@ ClearHistory()
908939
# Main Program
909940
#
910941

911-
if [ $# -eq 0 -o $# -gt 2 ]
942+
if [ $# -eq 0 ]
912943
then
913944
Usage
914945
fi
@@ -945,6 +976,10 @@ case "${1}" in
945976
;;
946977

947978
index)
979+
if [ $# -gt 2 ]
980+
then
981+
Usage
982+
fi
948983
if [ -n "${2}" ]
949984
then
950985
SRC_ROOT="${2}"
@@ -955,6 +990,17 @@ case "${1}" in
955990
UpdateDescriptionCache
956991
;;
957992

993+
indexpart)
994+
if [ -z "${2}" ]
995+
then
996+
Usage
997+
fi
998+
shift
999+
ValidateConfiguration
1000+
CreateRuntimeRequirements
1001+
UpdateDataPartial $@
1002+
;;
1003+
9581004
clearHistory)
9591005
if [ -z "${2}" ]; then
9601006
Usage

‎src/org/opensolaris/opengrok/index/CommandLineOptions.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ private void spool(BufferedReader reader, PrintWriter out, String tag) throws IO
149149
public String getUsage() {
150150
StringWriter wrt = new StringWriter();
151151
try (PrintWriter out = new PrintWriter(wrt)) {
152-
out.println("Usage: opengrok.jar [options]");
152+
out.println("Usage: opengrok.jar [options] [subDir1 ..]");
153153
for (Option o : options) {
154154
out.println(o.getUsage());
155155
}

‎src/org/opensolaris/opengrok/index/Indexer.java

Lines changed: 38 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
import org.opensolaris.opengrok.analysis.AnalyzerGuru;
4848
import org.opensolaris.opengrok.configuration.Configuration;
4949
import org.opensolaris.opengrok.configuration.Project;
50+
import static org.opensolaris.opengrok.configuration.Project.getProject;
5051
import org.opensolaris.opengrok.configuration.RuntimeEnvironment;
5152
import org.opensolaris.opengrok.history.HistoryException;
5253
import org.opensolaris.opengrok.history.HistoryGuru;
@@ -479,13 +480,6 @@ public static void main(String argv[]) {
479480
// don't care
480481
}
481482
}
482-
int optind = getopt.getOptind();
483-
if (optind != -1) {
484-
while (optind < argv.length) {
485-
subFiles.add(argv[optind]);
486-
++optind;
487-
}
488-
}
489483

490484
//logging starts here
491485
if (cfg.isVerbose()) {
@@ -540,24 +534,61 @@ public static void main(String argv[]) {
540534
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
541535
env.setConfiguration(cfg);
542536

537+
/*
538+
* Add paths to directories under source root. If projects
539+
* are enabled the path should correspond to a project because
540+
* project path is necessary to correctly set index directory
541+
* (otherwise the index files will end up in index data root
542+
* directory and not per project data root directory).
543+
* For the check we need to have 'env' already set.
544+
*/
545+
int optind = getopt.getOptind();
546+
int orig_optind = optind;
547+
if (optind != -1) {
548+
while (optind < argv.length) {
549+
if (env.hasProjects()) {
550+
// The paths need to correspond to a project.
551+
if (Project.getProject(argv[optind]) != null) {
552+
subFiles.add(argv[optind]);
553+
} else {
554+
System.err.println("The path " + argv[optind] +
555+
" does not correspond to a project");
556+
}
557+
} else {
558+
subFiles.add(argv[optind]);
559+
}
560+
++optind;
561+
}
562+
563+
if ((orig_optind < argv.length) && subFiles.isEmpty()) {
564+
System.err.println("None of the paths were added, exiting");
565+
System.exit(1);
566+
}
567+
}
568+
543569
// Issue a warning when JDBC is used with renamed file handling.
544570
// This causes heavy slowdown when used with JavaDB (issue #774).
545571
if (RuntimeEnvironment.isRenamedFilesEnabled() && cfg.isHistoryCacheInDB()) {
546572
System.out.println("History stored in DB and renamed file handling is on - possible performance degradation");
547573
}
548574

575+
// Get history first.
549576
getInstance().prepareIndexer(env, searchRepositories, addProjects,
550577
defaultProject, configFilename, refreshHistory,
551578
listFiles, createDict, subFiles, repositories,
552579
zapCache, listRepos);
553580
if (listRepos || !zapCache.isEmpty()) {
554581
return;
555582
}
583+
584+
// And now index it all.
556585
if (runIndex || (optimizedChanged && env.isOptimizeDatabase())) {
557586
IndexChangedListener progress = new DefaultIndexChangedListener();
558587
getInstance().doIndexerExecution(update, noThreads, subFiles,
559588
progress);
560589
}
590+
591+
// Finally send new configuration to the web application.
561592
getInstance().sendToConfigHost(env, configHost);
562593
} catch (IndexerException ex) {
563594
log.log(Level.SEVERE, "Exception running indexer", ex);
@@ -572,7 +603,6 @@ public static void main(String argv[]) {
572603
stats.report(log);
573604
}
574605
}
575-
576606
}
577607

578608
/*

0 commit comments

Comments
 (0)