Skip to content

Commit 1091aaa

Browse files
author
Vladimir Kotal
committed
check index version before reindex
fixes #2084
1 parent bef09db commit 1091aaa

File tree

7 files changed

+281
-5
lines changed

7 files changed

+281
-5
lines changed

‎OpenGrok

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -857,6 +857,20 @@ CreateRuntimeRequirements()
857857
fi
858858
}
859859

860+
CheckIndexVersion()
861+
{
862+
if [ ! -f "${XML_CONFIGURATION}" ]
863+
then
864+
return
865+
fi
866+
if ! MinimalInvocation --checkIndexVersion -R "${XML_CONFIGURATION}" -d "${DATA_ROOT}"
867+
then
868+
echo "\nIndex version check failed. You might want to remove all data" \
869+
"under the data root directory ${DATA_ROOT} and reindex.\n"
870+
exit 1
871+
fi
872+
}
873+
860874
MinimalInvocation()
861875
{
862876
${DO} ${JAVA} ${JAVA_OPTS} ${PROPERTIES} \
@@ -892,7 +906,7 @@ CommonInvocation()
892906
StdInvocation()
893907
{
894908
CommonInvocation \
895-
-W ${XML_CONFIGURATION} \
909+
-W "${XML_CONFIGURATION}" \
896910
${SCAN_FOR_REPOSITORY} \
897911
${ENABLE_PROJECTS} \
898912
${DEFAULT_PROJECTS} \
@@ -1057,12 +1071,14 @@ case "${1}" in
10571071

10581072
update)
10591073
ValidateConfiguration
1074+
CheckIndexVersion
10601075
CreateRuntimeRequirements
10611076
UpdateGeneratedData && UpdateDescriptionCache
10621077
;;
10631078

10641079
updateQuietly)
10651080
ValidateConfiguration
1081+
CheckIndexVersion
10661082
CreateRuntimeRequirements
10671083
QUIET="-q"
10681084
VERBOSE=""
@@ -1089,6 +1105,7 @@ case "${1}" in
10891105
SRC_ROOT="${2}"
10901106
fi
10911107
ValidateConfiguration
1108+
CheckIndexVersion
10921109
CreateRuntimeRequirements
10931110
UpdateGeneratedData && UpdateDescriptionCache
10941111
;;
@@ -1103,6 +1120,7 @@ case "${1}" in
11031120
FatalError "need configuration via OPENGROK_READ_XML_CONFIGURATION"
11041121
fi
11051122
ValidateConfiguration
1123+
CheckIndexVersion
11061124
CreateRuntimeRequirements
11071125
UpdateDataPartial $@
11081126
;;

‎nbproject/project.properties

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
file.reference.junit-4.12.jar=lib/junit-4.12.jar
2+
file.reference.lucene-backward-codecs-7.3.0.jar=lib/lucene-backward-codecs-7.3.0.jar
13
javac.external.vm=false
24
jnlp.offline-allowed=false
35
javadoc.splitindex=true
@@ -92,7 +94,9 @@ javac.classpath=\
9294
${file.reference.lucene-queries.jar}:\
9395
${file.reference.lucene-queryparser.jar}:\
9496
${file.reference.lucene-suggest.jar}:\
95-
${file.reference.servlet-api.jar}
97+
${file.reference.servlet-api.jar}:\
98+
${file.reference.junit-4.12.jar}:\
99+
${file.reference.lucene-backward-codecs-7.3.0.jar}
96100
javadoc.noindex=false
97101
manifest.custom.codebase=
98102
annotation.processing.enabled.in.editor=false

‎opengrok-indexer/pom.xml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -284,6 +284,13 @@ Portions Copyright (c) 2017-2018, Chris Fraire <[email protected]>.
284284
<exclude>*.java</exclude>
285285
</excludes>
286286
</testResource>
287+
<testResource>
288+
<targetPath>org/opensolaris/opengrok/index/</targetPath>
289+
<directory>../test/org/opensolaris/opengrok/index/</directory>
290+
<excludes>
291+
<exclude>*.java</exclude>
292+
</excludes>
293+
</testResource>
287294
</testResources>
288295

289296
<plugins>
@@ -497,6 +504,11 @@ Portions Copyright (c) 2017-2018, Chris Fraire <[email protected]>.
497504
<groupId>org.apache.lucene</groupId>
498505
<artifactId>lucene-queryparser</artifactId>
499506
</dependency>
507+
<dependency>
508+
<groupId>org.apache.lucene</groupId>
509+
<artifactId>lucene-backward-codecs</artifactId>
510+
<version>${lucene.version}</version>
511+
</dependency>
500512
<dependency>
501513
<groupId>org.apache.lucene</groupId>
502514
<artifactId>lucene-suggest</artifactId>

‎src/org/opensolaris/opengrok/index/IndexVersion.java

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
22+
*/
23+
package org.opensolaris.opengrok.index;
24+
25+
import java.io.File;
26+
import java.io.IOException;
27+
import org.apache.lucene.index.IndexNotFoundException;
28+
import org.apache.lucene.index.SegmentInfos;
29+
import org.apache.lucene.store.Directory;
30+
import org.apache.lucene.store.FSDirectory;
31+
import org.apache.lucene.store.LockFactory;
32+
import org.apache.lucene.store.NativeFSLockFactory;
33+
import org.apache.lucene.util.Version;
34+
import org.opensolaris.opengrok.configuration.Configuration;
35+
36+
/**
37+
* Index version checker
38+
*
39+
* @author Vladimir Kotal
40+
*/
41+
public class IndexVersion {
42+
/**
43+
* exception thrown when index version does not match Lucene version
44+
*/
45+
public static class IndexVersionException extends Exception {
46+
47+
public IndexVersionException(String s) {
48+
super(s);
49+
}
50+
}
51+
52+
/**
53+
* Check if version of index(es) matches major Lucene version.
54+
* @param cfg configuration
55+
* @throws Exception otherwise
56+
*/
57+
public static void check(Configuration cfg) throws Exception {
58+
File indexRoot = new File(cfg.getDataRoot(), IndexDatabase.INDEX_DIR);
59+
if (cfg.isProjectsEnabled()) {
60+
for (String projectName : cfg.getProjects().keySet()) {
61+
checkDir(getDirectory(new File(indexRoot, projectName)));
62+
}
63+
} else {
64+
checkDir(getDirectory(indexRoot));
65+
}
66+
}
67+
68+
private static Directory getDirectory(File indexDir) throws IOException {
69+
LockFactory lockfact = NativeFSLockFactory.INSTANCE;
70+
FSDirectory indexDirectory = FSDirectory.open(indexDir.toPath(), lockfact);
71+
return indexDirectory;
72+
}
73+
74+
/**
75+
* Check index version in given directory. It assumes that that all commits
76+
* in the Lucene segment file were done with the same version.
77+
*
78+
* @param dir directory with index
79+
* @thows IOException if the directory cannot be opened
80+
*/
81+
private static void checkDir(Directory dir) throws IOException, Exception {
82+
int segVersion;
83+
try {
84+
segVersion = SegmentInfos.readLatestCommit(dir).getIndexCreatedVersionMajor();
85+
} catch (IndexNotFoundException e) {
86+
return;
87+
}
88+
if (segVersion != Version.LATEST.major) {
89+
throw new IndexVersionException(
90+
String.format("Directory %s has index of version %d and Lucene has %d",
91+
dir.toString(), segVersion, Version.LATEST.major));
92+
}
93+
}
94+
}

‎src/org/opensolaris/opengrok/index/Indexer.java

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,6 @@
4141
import java.util.Scanner;
4242
import java.util.Set;
4343
import java.util.TreeSet;
44-
import java.util.concurrent.ExecutorService;
45-
import java.util.concurrent.Executors;
4644
import java.util.concurrent.TimeUnit;
4745
import java.util.logging.Level;
4846
import java.util.logging.Logger;
@@ -61,6 +59,7 @@
6159
import org.opensolaris.opengrok.history.Repository;
6260
import org.opensolaris.opengrok.history.RepositoryFactory;
6361
import org.opensolaris.opengrok.history.RepositoryInfo;
62+
import org.opensolaris.opengrok.index.IndexVersion.IndexVersionException;
6463
import org.opensolaris.opengrok.logger.LoggerFactory;
6564
import org.opensolaris.opengrok.logger.LoggerUtil;
6665
import org.opensolaris.opengrok.util.Executor;
@@ -91,6 +90,7 @@ public final class Indexer {
9190
private static boolean searchRepositories = false;
9291
private static boolean noindex = false;
9392
private static boolean awaitProfiler;
93+
private static boolean checkIndexVersion = false;
9494

9595
private static boolean help;
9696
private static String helpUsage;
@@ -108,7 +108,7 @@ public final class Indexer {
108108
private static int port = 0;
109109

110110
public static OptionParser openGrok = null;
111-
111+
112112
public static Indexer getInstance() {
113113
return index;
114114
}
@@ -143,6 +143,20 @@ public static void main(String argv[]) {
143143
}
144144
System.exit(status);
145145
}
146+
147+
// Check version of index(es) versus current Lucene version and exit
148+
// with return code indicating success or failure.
149+
if (checkIndexVersion) {
150+
int retval = 0;
151+
try {
152+
IndexVersion.check(cfg);
153+
} catch (IndexVersionException e) {
154+
System.err.printf("Index version check failed: %s", e);
155+
retval = 1;
156+
}
157+
System.exit(retval);
158+
}
159+
146160
if (awaitProfiler) pauseToAwaitProfiler();
147161

148162
env = RuntimeEnvironment.getInstance();
@@ -430,6 +444,11 @@ public static String[] parseOptions(String[] argv) throws ParseException {
430444
}
431445
);
432446

447+
parser.on("--checkIndexVersion",
448+
"Check if current Lucene version matches index version").Do( v -> {
449+
checkIndexVersion = true;
450+
});
451+
433452
parser.on("-d", "--dataRoot", "=/path/to/data/root",
434453
"The directory where OpenGrok stores the generated data.").
435454
Do( drPath -> {

‎test/org/opensolaris/opengrok/index/IndexVersionTest.java

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
22+
*/
23+
package org.opensolaris.opengrok.index;
24+
25+
import java.io.File;
26+
import java.io.IOException;
27+
import java.net.URL;
28+
import java.nio.file.Files;
29+
import java.nio.file.Path;
30+
import java.nio.file.Paths;
31+
import java.util.ArrayList;
32+
import org.junit.After;
33+
import static org.junit.Assert.assertNotNull;
34+
import static org.junit.Assert.assertTrue;
35+
import org.junit.Before;
36+
import org.junit.BeforeClass;
37+
import org.junit.Test;
38+
import org.opensolaris.opengrok.configuration.Configuration;
39+
import org.opensolaris.opengrok.configuration.RuntimeEnvironment;
40+
import org.opensolaris.opengrok.history.RepositoryFactory;
41+
import org.opensolaris.opengrok.util.FileUtilities;
42+
import org.opensolaris.opengrok.util.IOUtils;
43+
import org.opensolaris.opengrok.util.TestRepository;
44+
45+
/**
46+
* Verify index version check.
47+
* @author Vladimir Kotal
48+
*/
49+
public class IndexVersionTest {
50+
51+
TestRepository repository;
52+
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
53+
private File oldIndexDataDir;
54+
55+
@BeforeClass
56+
public static void setUpClass() throws Exception {
57+
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
58+
RepositoryFactory.initializeIgnoredNames(env);
59+
}
60+
61+
@Before
62+
public void setUp() throws IOException {
63+
repository = new TestRepository();
64+
repository.create(IndexerTest.class.
65+
getResourceAsStream("/org/opensolaris/opengrok/history/repositories.zip"));
66+
oldIndexDataDir = null;
67+
}
68+
69+
@After
70+
public void tearDown() throws IOException {
71+
repository.destroy();
72+
73+
if (oldIndexDataDir != null) {
74+
IOUtils.removeRecursive(Paths.get(oldIndexDataDir.getPath()));
75+
}
76+
}
77+
78+
/**
79+
* Generate index(es) and check version.
80+
* @throws Exception
81+
*/
82+
private void testIndexVersion(boolean projectsEnabled) throws Exception {
83+
if (env.validateExuberantCtags()) {
84+
env.setVerbose(true);
85+
env.setHistoryEnabled(false);
86+
env.setProjectsEnabled(projectsEnabled);
87+
Indexer.getInstance().prepareIndexer(env, true, true, null,
88+
false, false, null, null, new ArrayList<>(), false);
89+
Indexer.getInstance().doIndexerExecution(true, null, null);
90+
} else {
91+
System.out.println("Skipping test. Could not find a ctags program that could be used.");
92+
}
93+
94+
IndexVersion.check(env.getConfiguration());
95+
}
96+
97+
@Test
98+
public void testIndexVersionNoIndex() throws Exception {
99+
IndexVersion.check(env.getConfiguration());
100+
}
101+
102+
@Test
103+
public void testIndexVersionNoProjects() throws Exception {
104+
testIndexVersion(true);
105+
}
106+
107+
@Test
108+
public void testIndexVersionProjects() throws Exception {
109+
testIndexVersion(false);
110+
}
111+
112+
@Test(expected = IndexVersion.IndexVersionException.class)
113+
public void testIndexVersionOldIndex() throws Exception {
114+
Configuration cfg = new Configuration();
115+
oldIndexDataDir = FileUtilities.createTemporaryDirectory("data");
116+
Path indexPath = Paths.get(oldIndexDataDir.getPath(), "index");
117+
Files.createDirectory(indexPath);
118+
File indexDir = new File(indexPath.toString());
119+
assertTrue("index directory check", indexDir.isDirectory());
120+
URL oldindex = getClass().getResource("oldindex.zip");
121+
assertNotNull("resource needs to be non null", oldindex);
122+
File archive = new File(oldindex.getPath());
123+
assertTrue("archive exists", archive.isFile());
124+
FileUtilities.extractArchive(archive, indexDir);
125+
cfg.setDataRoot(oldIndexDataDir.getPath());
126+
cfg.setProjectsEnabled(false);
127+
IndexVersion.check(cfg);
128+
}
129+
}

‎test/org/opensolaris/opengrok/index/oldindex.zip

3.79 KB
Binary file not shown.

0 commit comments

Comments
 (0)