Skip to content

Commit 3d2d635

Browse files
ahornaceVladimir Kotal
ahornace
authored and
Vladimir Kotal
committed
Create suggester data only for enabled fields in configuration
1 parent 4d6d801 commit 3d2d635

File tree

7 files changed

+129
-76
lines changed

7 files changed

+129
-76
lines changed

‎src/org/opensolaris/opengrok/configuration/SuggesterConfig.java

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,11 @@
2525
import com.cronutils.model.CronType;
2626
import com.cronutils.model.definition.CronDefinitionBuilder;
2727
import com.cronutils.parser.CronParser;
28+
import org.opensolaris.opengrok.search.QueryBuilder;
2829

30+
import java.util.Arrays;
31+
import java.util.Collections;
32+
import java.util.HashSet;
2933
import java.util.Objects;
3034
import java.util.Set;
3135

@@ -47,7 +51,14 @@ public class SuggesterConfig {
4751
public static final int SUGGESTER_BUILD_TERMINATION_TIME_DEFAULT = 1800; // half an hour should be enough
4852

4953
public static final Set<String> allowedProjectsDefault = null;
50-
public static final Set<String> allowedFieldsDefault = null;
54+
public static final Set<String> allowedFieldsDefault = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(
55+
QueryBuilder.FULL,
56+
QueryBuilder.DEFS,
57+
QueryBuilder.REFS,
58+
QueryBuilder.PATH,
59+
QueryBuilder.HIST,
60+
QueryBuilder.TYPE
61+
)));
5162

5263
/**
5364
* Specifies if the suggester is enabled.

‎src/org/opensolaris/opengrok/web/api/v1/suggester/provider/service/impl/SuggesterServiceImpl.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -273,7 +273,7 @@ private void initSuggester() {
273273
suggester = new Suggester(suggesterDir,
274274
suggesterConfig.getMaxResults(),
275275
Duration.ofSeconds(suggesterConfig.getSuggesterBuildTerminationTimeSec()),
276-
suggesterConfig.isAllowMostPopular(), env.isProjectsEnabled());
276+
suggesterConfig.isAllowMostPopular(), env.isProjectsEnabled(), suggesterConfig.getAllowedFields());
277277

278278
new Thread(() -> {
279279
suggester.init(getAllProjectIndexDirs());

‎suggester/src/main/java/org/opengrok/suggest/FieldWFSTCollection.java

Lines changed: 64 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,10 @@
4848
import java.nio.charset.StandardCharsets;
4949
import java.nio.file.Files;
5050
import java.nio.file.Path;
51+
import java.util.Collection;
5152
import java.util.Collections;
5253
import java.util.HashMap;
54+
import java.util.HashSet;
5355
import java.util.List;
5456
import java.util.Map;
5557
import java.util.Map.Entry;
@@ -94,10 +96,38 @@ class FieldWFSTCollection implements Closeable {
9496

9597
private final ReadWriteLock lock = new ReentrantReadWriteLock();
9698

97-
FieldWFSTCollection(final Directory indexDir, final Path suggesterDir, final boolean allowMostPopular) {
99+
private Set<String> fields;
100+
101+
FieldWFSTCollection(
102+
final Directory indexDir,
103+
final Path suggesterDir,
104+
final boolean allowMostPopular,
105+
final Set<String> fields
106+
) throws IOException {
98107
this.indexDir = indexDir;
99108
this.suggesterDir = suggesterDir;
100109
this.allowMostPopular = allowMostPopular;
110+
111+
initFields(fields);
112+
}
113+
114+
private void initFields(final Set<String> fields) throws IOException {
115+
try (IndexReader indexReader = DirectoryReader.open(indexDir)) {
116+
Collection<String> indexedFields = MultiFields.getIndexedFields(indexReader);
117+
if (fields == null) {
118+
this.fields = new HashSet<>(indexedFields);
119+
} else if (!indexedFields.containsAll(fields)) {
120+
Set<String> copy = new HashSet<>(fields);
121+
copy.removeAll(indexedFields);
122+
logger.log(Level.WARNING, "Fields {0} will be ignored because they are not indexed", copy);
123+
124+
copy = new HashSet<>(fields);
125+
copy.retainAll(indexedFields);
126+
this.fields = copy;
127+
} else {
128+
this.fields = new HashSet<>(fields);
129+
}
130+
}
101131
}
102132

103133
/**
@@ -147,14 +177,14 @@ private boolean hasStoredData() {
147177

148178
private void loadStoredWFSTs() throws IOException {
149179
try (IndexReader indexReader = DirectoryReader.open(indexDir)) {
150-
for (String field : MultiFields.getIndexedFields(indexReader)) {
180+
for (String field : fields) {
151181

152182
File WFSTfile = getWFSTFile(field);
153183
if (WFSTfile.exists()) {
154184
WFSTCompletionLookup WFST = loadStoredWFST(WFSTfile);
155185
lookups.put(field, WFST);
156186
} else {
157-
logger.log(Level.INFO, "Missing FieldWFSTCollection file for {0} field in {1}, creating a new one",
187+
logger.log(Level.INFO, "Missing WFST file for {0} field in {1}, creating a new one",
158188
new Object[] {field, suggesterDir});
159189

160190
WFSTCompletionLookup lookup = build(indexReader, field);
@@ -207,7 +237,7 @@ public void rebuild() throws IOException {
207237

208238
private void build() throws IOException {
209239
try (IndexReader indexReader = DirectoryReader.open(indexDir)) {
210-
for (String field : MultiFields.getIndexedFields(indexReader)) {
240+
for (String field : fields) {
211241
WFSTCompletionLookup lookup = build(indexReader, field);
212242
store(lookup, field);
213243

@@ -246,45 +276,44 @@ private void createSuggesterDir() throws IOException {
246276

247277
private void initSearchCountMap() throws IOException {
248278
searchCountMaps.clear();
249-
try (IndexReader indexReader = DirectoryReader.open(indexDir)) {
250-
for (String field : MultiFields.getIndexedFields(indexReader)) {
251-
ChronicleMapConfiguration conf = ChronicleMapConfiguration.load(suggesterDir, field);
252-
if (conf == null) { // it was not yet initialized
253-
conf = new ChronicleMapConfiguration((int) lookups.get(field).getCount(), getAverageLength(field));
254-
conf.save(suggesterDir, field);
255-
}
256279

257-
File f = getChronicleMapFile(field);
280+
for (String field : fields) {
281+
ChronicleMapConfiguration conf = ChronicleMapConfiguration.load(suggesterDir, field);
282+
if (conf == null) { // it was not yet initialized
283+
conf = new ChronicleMapConfiguration((int) lookups.get(field).getCount(), getAverageLength(field));
284+
conf.save(suggesterDir, field);
285+
}
258286

259-
ChronicleMapAdapter m;
260-
try {
261-
m = new ChronicleMapAdapter(field, conf.getAverageKeySize(), conf.getEntries(), f);
262-
} catch (Exception e) {
263-
logger.log(Level.SEVERE,
264-
"Could not create ChronicleMap, most popular completion disabled, if you are using "
265-
+ "JDK9+ make sure to specify: "
266-
+ "--add-exports java.base/jdk.internal.ref=ALL-UNNAMED "
267-
+ "--add-exports java.base/jdk.internal.misc=ALL-UNNAMED "
268-
+ "--add-exports java.base/sun.nio.ch=ALL-UNNAMED", e);
269-
return;
270-
}
287+
File f = getChronicleMapFile(field);
271288

272-
if (getCommitVersion() != getDataVersion()) {
273-
removeOldTerms(m, lookups.get(field));
289+
ChronicleMapAdapter m;
290+
try {
291+
m = new ChronicleMapAdapter(field, conf.getAverageKeySize(), conf.getEntries(), f);
292+
} catch (Exception e) {
293+
logger.log(Level.SEVERE,
294+
"Could not create ChronicleMap, most popular completion disabled, if you are using "
295+
+ "JDK9+ make sure to specify: "
296+
+ "--add-exports java.base/jdk.internal.ref=ALL-UNNAMED "
297+
+ "--add-exports java.base/jdk.internal.misc=ALL-UNNAMED "
298+
+ "--add-exports java.base/sun.nio.ch=ALL-UNNAMED", e);
299+
return;
300+
}
274301

275-
if (conf.getEntries() < lookups.get(field).getCount()) {
276-
int newEntriesCount = (int) lookups.get(field).getCount();
277-
double newKeyAvgLength = getAverageLength(field);
302+
if (getCommitVersion() != getDataVersion()) {
303+
removeOldTerms(m, lookups.get(field));
278304

279-
conf.setEntries(newEntriesCount);
280-
conf.setAverageKeySize(newKeyAvgLength);
281-
conf.save(suggesterDir, field);
305+
if (conf.getEntries() < lookups.get(field).getCount()) {
306+
int newEntriesCount = (int) lookups.get(field).getCount();
307+
double newKeyAvgLength = getAverageLength(field);
308+
309+
conf.setEntries(newEntriesCount);
310+
conf.setAverageKeySize(newKeyAvgLength);
311+
conf.save(suggesterDir, field);
282312

283-
m.resize(newEntriesCount, newKeyAvgLength);
284-
}
313+
m.resize(newEntriesCount, newKeyAvgLength);
285314
}
286-
searchCountMaps.put(field, m);
287315
}
316+
searchCountMaps.put(field, m);
288317
}
289318
}
290319

‎suggester/src/main/java/org/opengrok/suggest/Suggester.java

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,11 @@
4040
import java.time.Instant;
4141
import java.util.Collection;
4242
import java.util.Collections;
43+
import java.util.HashSet;
4344
import java.util.List;
4445
import java.util.Map;
4546
import java.util.Map.Entry;
47+
import java.util.Set;
4648
import java.util.concurrent.ConcurrentHashMap;
4749
import java.util.concurrent.ExecutorService;
4850
import java.util.concurrent.Executors;
@@ -75,19 +77,24 @@ public final class Suggester implements Closeable {
7577

7678
private boolean projectsEnabled;
7779

80+
private final Set<String> allowedFields;
81+
7882
/**
7983
* @param suggesterDir directory under which the suggester data should be created
8084
* @param resultSize maximum number of suggestions that should be returned
8185
* @param awaitTerminationTime how much time to wait for suggester to initialize
8286
* @param allowMostPopular specifies if the most popular completion is enabled
8387
* @param projectsEnabled specifies if the OpenGrok projects are enabled
88+
* @param allowedFields fields for which should the suggester be enabled,
89+
* if {@code null} then enabled for all fields
8490
*/
8591
public Suggester(
8692
final File suggesterDir,
8793
final int resultSize,
8894
final Duration awaitTerminationTime,
8995
final boolean allowMostPopular,
90-
final boolean projectsEnabled
96+
final boolean projectsEnabled,
97+
final Set<String> allowedFields
9198
) {
9299
if (suggesterDir == null) {
93100
throw new IllegalArgumentException("Suggester needs to have directory specified");
@@ -103,6 +110,7 @@ public Suggester(
103110

104111
this.allowMostPopular = allowMostPopular;
105112
this.projectsEnabled = projectsEnabled;
113+
this.allowedFields = new HashSet<>(allowedFields);
106114
}
107115

108116
/**
@@ -149,8 +157,8 @@ private Runnable getInitRunnable(final Path indexDir) {
149157
Instant start = Instant.now();
150158
logger.log(Level.FINE, "Initializing {0}", indexDir);
151159

152-
FieldWFSTCollection wfst = new FieldWFSTCollection(FSDirectory.open(indexDir), getSuggesterDir(indexDir),
153-
allowMostPopular);
160+
FieldWFSTCollection wfst = new FieldWFSTCollection(FSDirectory.open(indexDir),
161+
getSuggesterDir(indexDir), allowMostPopular, allowedFields);
154162
wfst.init();
155163
if (projectsEnabled) {
156164
projectData.put(indexDir.getFileName().toString(), wfst);

‎suggester/src/main/java/org/opengrok/suggest/popular/impl/chronicle/ChronicleMapConfiguration.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@ public class ChronicleMapConfiguration implements Serializable {
4343

4444
private static final Logger logger = Logger.getLogger(ChronicleMapConfiguration.class.getName());
4545

46+
private static final String FILE_NAME_SUFFIX = "_map.cfg";
47+
4648
private int entries;
4749

4850
private double averageKeySize;
@@ -101,7 +103,7 @@ public static ChronicleMapConfiguration load(final Path dir, final String field)
101103
}
102104

103105
private static File getFile(final Path dir, final String field) {
104-
return dir.resolve(field + "_map.cfg").toFile();
106+
return dir.resolve(field + FILE_NAME_SUFFIX).toFile();
105107
}
106108

107109
}

0 commit comments

Comments
 (0)