Skip to content

Commit 1277512

Browse files
author
Vladimir Kotal
committed
check project indexes on webapp startup
1 parent 4095db0 commit 1277512

File tree

4 files changed

+96
-53
lines changed

4 files changed

+96
-53
lines changed

‎opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexVersion.java renamed to ‎opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexCheck.java

Lines changed: 52 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -18,16 +18,16 @@
1818
*/
1919

2020
/*
21-
* Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
21+
* Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
2222
* Portions Copyright (c) 2018, Chris Fraire <[email protected]>.
2323
*/
2424
package org.opengrok.indexer.index;
2525

2626
import java.io.File;
27+
import java.io.IOException;
2728
import java.util.List;
2829
import java.util.logging.Level;
2930
import java.util.logging.Logger;
30-
import org.apache.lucene.index.IndexNotFoundException;
3131
import org.apache.lucene.index.SegmentInfos;
3232
import org.apache.lucene.store.Directory;
3333
import org.apache.lucene.store.FSDirectory;
@@ -38,89 +38,113 @@
3838
import org.opengrok.indexer.logger.LoggerFactory;
3939

4040
/**
41-
* Index version checker.
41+
* Index checker.
4242
*
43-
* @author Vladimir Kotal
43+
* @author Vladimír Kotal
4444
*/
45-
public class IndexVersion {
46-
private static final Logger LOGGER = LoggerFactory.getLogger(IndexVersion.class);
45+
public class IndexCheck {
46+
private static final Logger LOGGER = LoggerFactory.getLogger(IndexCheck.class);
4747

4848
/**
4949
* Exception thrown when index version does not match Lucene version.
5050
*/
5151
public static class IndexVersionException extends Exception {
5252

53-
private static final long serialVersionUID = -756788782277552544L;
53+
private static final long serialVersionUID = 5693446916108385595L;
5454

55-
public IndexVersionException(String s) {
55+
private int luceneIndexVersion = -1;
56+
private int indexVersion = -1;
57+
58+
public IndexVersionException(String s, int luceneIndexVersion, int indexVersion) {
5659
super(s);
60+
61+
this.indexVersion = indexVersion;
62+
this.luceneIndexVersion = luceneIndexVersion;
63+
}
64+
65+
@Override
66+
public String toString() {
67+
return getMessage() + ": " + String.format("Lucene version = %d", luceneIndexVersion) + ", " +
68+
String.format("index version = %d", indexVersion);
5769
}
5870
}
5971

60-
private IndexVersion() {
72+
private IndexCheck() {
73+
// utility class
6174
}
6275

6376
/**
6477
* Check if version of index(es) matches major Lucene version.
6578
* @param subFilesList list of paths. If non-empty, only projects matching these paths will be checked.
66-
* @throws Exception otherwise
79+
* @return true on success, false on failure
6780
*/
68-
public static void check(List<String> subFilesList) throws Exception {
81+
public static boolean check(List<String> subFilesList) {
6982
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
7083
File indexRoot = new File(env.getDataRootPath(), IndexDatabase.INDEX_DIR);
7184
LOGGER.log(Level.FINE, "Checking for Lucene index version mismatch in {0}",
7285
indexRoot);
86+
int ret = 0;
7387

7488
if (!subFilesList.isEmpty()) {
7589
// Assumes projects are enabled.
7690
for (String projectName : subFilesList) {
7791
LOGGER.log(Level.FINER,
7892
"Checking Lucene index version in project {0}",
7993
projectName);
80-
checkDir(new File(indexRoot, projectName));
94+
ret |= checkDirNoExceptions(new File(indexRoot, projectName));
8195
}
8296
} else {
8397
if (env.isProjectsEnabled()) {
8498
for (String projectName : env.getProjects().keySet()) {
8599
LOGGER.log(Level.FINER,
86100
"Checking Lucene index version in project {0}",
87101
projectName);
88-
checkDir(new File(indexRoot, projectName));
102+
ret |= checkDirNoExceptions(new File(indexRoot, projectName));
89103
}
90104
} else {
91105
LOGGER.log(Level.FINER, "Checking Lucene index version in {0}",
92106
indexRoot);
93-
checkDir(indexRoot);
107+
ret |= checkDirNoExceptions(indexRoot);
94108
}
95109
}
110+
111+
return ret == 0;
112+
}
113+
114+
private static int checkDirNoExceptions(File dir) {
115+
try {
116+
checkDir(dir);
117+
} catch (Exception e) {
118+
LOGGER.log(Level.WARNING, "Index check for directory " + dir.toString() + " failed", e);
119+
return 1;
120+
}
121+
122+
return 0;
96123
}
97124

98125
/**
99-
* Check index version in given directory. It assumes that that all commits
126+
* Check index in given directory. It assumes that that all commits (if any)
100127
* in the Lucene segment file were done with the same version.
101128
*
102129
* @param dir directory with index
103-
* @thows IOException if the directory cannot be opened
130+
* @throws IOException if the directory cannot be opened
131+
* @throws IndexVersionException if the version of the index does not match Lucene index version
104132
*/
105-
private static void checkDir(File dir) throws Exception {
106-
LockFactory lockfact = NativeFSLockFactory.INSTANCE;
133+
public static void checkDir(File dir) throws IndexVersionException, IOException {
134+
LockFactory lockFactory = NativeFSLockFactory.INSTANCE;
107135
int segVersion;
108136

109-
try (Directory indexDirectory = FSDirectory.open(dir.toPath(), lockfact)) {
110-
SegmentInfos segInfos = null;
137+
try (Directory indexDirectory = FSDirectory.open(dir.toPath(), lockFactory)) {
138+
SegmentInfos segInfos;
111139

112-
try {
113-
segInfos = SegmentInfos.readLatestCommit(indexDirectory);
114-
segVersion = segInfos.getIndexCreatedVersionMajor();
115-
} catch (IndexNotFoundException e) {
116-
return;
117-
}
140+
segInfos = SegmentInfos.readLatestCommit(indexDirectory);
141+
segVersion = segInfos.getIndexCreatedVersionMajor();
118142
}
119143

120144
if (segVersion != Version.LATEST.major) {
121145
throw new IndexVersionException(
122-
String.format("Directory %s has index of version %d and Lucene has %d",
123-
dir.toString(), segVersion, Version.LATEST.major));
146+
String.format("Directory %s has index version discrepancy", dir.toString()),
147+
Version.LATEST.major, segVersion);
124148
}
125149
}
126150
}

‎opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,6 @@
7272
import org.opengrok.indexer.history.Repository;
7373
import org.opengrok.indexer.history.RepositoryFactory;
7474
import org.opengrok.indexer.history.RepositoryInfo;
75-
import org.opengrok.indexer.index.IndexVersion.IndexVersionException;
7675
import org.opengrok.indexer.logger.LoggerFactory;
7776
import org.opengrok.indexer.logger.LoggerUtil;
7877
import org.opengrok.indexer.util.CtagsUtil;
@@ -112,7 +111,7 @@ public final class Indexer {
112111

113112
private static final Indexer index = new Indexer();
114113
private static Configuration cfg = null;
115-
private static boolean checkIndexVersion = false;
114+
private static boolean checkIndex = false;
116115
private static boolean runIndex = true;
117116
private static boolean optimizedChanged = false;
118117
private static boolean addProjects = false;
@@ -253,22 +252,19 @@ public static void main(String[] argv) {
253252

254253
// Check version of index(es) versus current Lucene version and exit
255254
// with return code upon failure.
256-
if (checkIndexVersion) {
255+
if (checkIndex) {
257256
if (cfg == null) {
258-
System.err.println("Need configuration to check index version (use -R)");
257+
System.err.println("Need configuration to check index (use -R)");
259258
System.exit(1);
260259
}
261260

262-
try {
263-
IndexVersion.check(subFilesList);
264-
} catch (IndexVersionException e) {
265-
System.err.printf("Index version check failed: %s\n", e);
261+
if (!IndexCheck.check(subFilesList)) {
262+
System.err.printf("Index check failed\n");
266263
System.err.print("You might want to remove " +
267264
(subFilesList.size() > 0 ?
268265
"data for projects " + String.join(",", subFilesList) : "all data") +
269-
" under the DATA_ROOT and to reindex\n");
270-
status = 1;
271-
System.exit(status);
266+
" under the data root and reindex\n");
267+
System.exit(1);
272268
}
273269
}
274270

@@ -505,9 +501,7 @@ public static String[] parseOptions(String[] argv) throws ParseException {
505501
canonicalRoots.add(root);
506502
});
507503

508-
parser.on("--checkIndexVersion",
509-
"Check if current Lucene version matches index version.").execute(v ->
510-
checkIndexVersion = true);
504+
parser.on("--checkIndex", "Check index.").execute(v -> checkIndex = true);
511505

512506
parser.on("-d", "--dataRoot", "=/path/to/data/root",
513507
"The directory where OpenGrok stores the generated data.").

‎opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexVersionTest.java renamed to ‎opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexCheckTest.java

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
*/
1919

2020
/*
21-
* Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
21+
* Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
2222
* Portions Copyright (c) 2019, Chris Fraire <[email protected]>.
2323
*/
2424
package org.opengrok.indexer.index;
@@ -36,6 +36,7 @@
3636

3737
import static org.junit.Assert.assertNotNull;
3838
import static org.junit.Assert.assertTrue;
39+
import static org.junit.jupiter.api.Assertions.assertFalse;
3940

4041
import org.junit.Before;
4142
import org.junit.BeforeClass;
@@ -47,10 +48,10 @@
4748
import org.opengrok.indexer.util.TestRepository;
4849

4950
/**
50-
* Verify index version check.
51-
* @author Vladimir Kotal
51+
* Verify index check.
52+
* @author Vladimír Kotal
5253
*/
53-
public class IndexVersionTest {
54+
public class IndexCheckTest {
5455

5556
private TestRepository repository;
5657
private RuntimeEnvironment env = RuntimeEnvironment.getInstance();
@@ -88,12 +89,12 @@ private void testIndexVersion(boolean projectsEnabled, List<String> subFiles) th
8889
false, null, null);
8990
Indexer.getInstance().doIndexerExecution(true, null, null);
9091

91-
IndexVersion.check(subFiles);
92+
IndexCheck.check(subFiles);
9293
}
9394

9495
@Test
9596
public void testIndexVersionNoIndex() throws Exception {
96-
IndexVersion.check(new ArrayList<>());
97+
IndexCheck.check(new ArrayList<>());
9798
}
9899

99100
@Test
@@ -111,20 +112,21 @@ public void testIndexVersionNoProjects() throws Exception {
111112
testIndexVersion(false, new ArrayList<>());
112113
}
113114

114-
@Test(expected = IndexVersion.IndexVersionException.class)
115+
@Test(expected = IndexCheck.IndexVersionException.class)
115116
public void testIndexVersionOldIndex() throws Exception {
116117
oldIndexDataDir = Files.createTempDirectory("data");
117118
Path indexPath = oldIndexDataDir.resolve("index");
118119
Files.createDirectory(indexPath);
119120
File indexDir = new File(indexPath.toString());
120121
assertTrue("index directory check", indexDir.isDirectory());
121-
URL oldindex = getClass().getResource("/index/oldindex.zip");
122-
assertNotNull("resource needs to be non null", oldindex);
123-
File archive = new File(oldindex.getPath());
122+
URL oldIndex = getClass().getResource("/index/oldindex.zip");
123+
assertNotNull("resource needs to be non null", oldIndex);
124+
File archive = new File(oldIndex.getPath());
124125
assertTrue("archive exists", archive.isFile());
125126
FileUtilities.extractArchive(archive, indexDir);
126127
env.setDataRoot(oldIndexDataDir.toString());
127128
env.setProjectsEnabled(false);
128-
IndexVersion.check(new ArrayList<>());
129+
assertFalse(IndexCheck.check(new ArrayList<>()));
130+
IndexCheck.checkDir(indexDir);
129131
}
130132
}

‎opengrok-web/src/main/java/org/opengrok/web/WebappListener.java

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,10 @@
2929
import org.opengrok.indexer.analysis.AnalyzerGuru;
3030
import org.opengrok.indexer.authorization.AuthorizationFramework;
3131
import org.opengrok.indexer.configuration.CommandTimeoutType;
32+
import org.opengrok.indexer.configuration.Project;
3233
import org.opengrok.indexer.configuration.RuntimeEnvironment;
34+
import org.opengrok.indexer.index.IndexCheck;
35+
import org.opengrok.indexer.index.IndexDatabase;
3336
import org.opengrok.indexer.logger.LoggerFactory;
3437
import org.opengrok.indexer.web.SearchHelper;
3538
import org.opengrok.web.api.v1.suggester.provider.service.SuggesterServiceFactory;
@@ -43,6 +46,7 @@
4346
import java.io.IOException;
4447
import java.time.Duration;
4548
import java.time.Instant;
49+
import java.util.Map;
4650
import java.util.logging.Level;
4751
import java.util.logging.Logger;
4852

@@ -100,6 +104,25 @@ public void contextInitialized(final ServletContextEvent servletContextEvent) {
100104
env.watchDog.start(new File(pluginDirectory));
101105
}
102106

107+
// Check project index(es).
108+
if (env.isProjectsEnabled()) {
109+
LOGGER.log(Level.FINE, "Checking indexes for all projects");
110+
Map<String, Project> projects = env.getProjects();
111+
File indexRoot = new File(env.getDataRootPath(), IndexDatabase.INDEX_DIR);
112+
if (indexRoot.exists()) {
113+
for (String projectName : projects.keySet()) {
114+
try {
115+
IndexCheck.checkDir(new File(indexRoot, projectName));
116+
} catch (Exception e) {
117+
LOGGER.log(Level.WARNING,
118+
String.format("Project %s index check failed, marking as not indexed", projectName), e);
119+
projects.get(projectName).setIndexed(false);
120+
}
121+
}
122+
}
123+
LOGGER.log(Level.FINE, "Index check for all projects done");
124+
}
125+
103126
env.startExpirationTimer();
104127
startupTimer.record(Duration.between(start, Instant.now()));
105128
}

0 commit comments

Comments
 (0)