Skip to content

Commit fc98ae9

Browse files
committed
Store file analyzer versions in IndexAnalysisSettings2
... with upgrader from binary serialization of v1 IndexAnalysisSettings.
1 parent 8f6d1b9 commit fc98ae9

12 files changed

+529
-43
lines changed

src/org/opensolaris/opengrok/analysis/AnalyzerGuru.java

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,6 @@
5454
import org.apache.lucene.document.Field.Store;
5555
import org.apache.lucene.document.FieldType;
5656
import org.apache.lucene.document.SortedDocValuesField;
57-
import org.apache.lucene.document.StoredField;
5857
import org.apache.lucene.document.StringField;
5958
import org.apache.lucene.document.TextField;
6059
import org.apache.lucene.util.BytesRef;
@@ -354,6 +353,10 @@ public static long getAnalyzerVersionNo(String fileTypeName) {
354353
return ANALYZER_VERSIONS.getOrDefault(fileTypeName, Long.MIN_VALUE);
355354
}
356355

356+
public static Map<String, Long> getAnalyzersVersionNos() {
357+
return Collections.unmodifiableMap(ANALYZER_VERSIONS);
358+
}
359+
357360
public static Map<String, FileAnalyzerFactory> getExtensionsMap() {
358361
return Collections.unmodifiableMap(ext);
359362
}
@@ -571,8 +574,6 @@ public void populateDocument(Document doc, File file, String path,
571574

572575
String type = fa.getFileTypeName();
573576
doc.add(new StringField(QueryBuilder.TYPE, type, Store.YES));
574-
575-
doc.add(new StoredField(QueryBuilder.ZVER, fa.getVersionNo()));
576577
}
577578
}
578579

src/org/opensolaris/opengrok/analysis/FileAnalyzer.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
/*
2121
* Copyright (c) 2005, 2017, Oracle and/or its affiliates. All rights reserved.
2222
* Use is subject to license terms.
23-
* Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
23+
* Portions Copyright (c) 2017-2018, Chris Fraire <[email protected]>.
2424
*/
2525
package org.opensolaris.opengrok.analysis;
2626

src/org/opensolaris/opengrok/index/IndexAnalysisSettings.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
* re-compared upon each indexing run since changes to them might require
3737
* re-indexing particular files or in certain cases all files.
3838
*/
39+
@Deprecated
3940
public final class IndexAnalysisSettings implements Serializable {
4041

4142
private static final long serialVersionUID = 1005610724146719938L;
Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Copyright (c) 2018, Chris Fraire <[email protected]>.
22+
*/
23+
24+
package org.opensolaris.opengrok.index;
25+
26+
import java.io.ByteArrayInputStream;
27+
import java.io.ByteArrayOutputStream;
28+
import java.io.IOException;
29+
import java.io.ObjectInputStream;
30+
import java.io.ObjectOutputStream;
31+
import java.io.Serializable;
32+
import java.util.Collections;
33+
import java.util.HashMap;
34+
import java.util.Map;
35+
36+
/**
37+
* Represents a serializable gathering of some top-level metadata concerning the
38+
* operation of {@link IndexDatabase} -- and persisted therein too -- which are
39+
* re-compared upon each indexing run since changes to them might require
40+
* re-indexing particular files or in certain cases all files.
41+
*/
42+
public final class IndexAnalysisSettings2 implements Serializable {
43+
44+
private static final long serialVersionUID = 1172403004716059609L;
45+
46+
private String projectName;
47+
48+
/**
49+
* (nullable to allow easing this object into existing OpenGrok indexes
50+
* without forcing a re-indexing)
51+
* @serial
52+
*/
53+
private Integer tabSize;
54+
55+
/**
56+
* (nullable to allow easing this object into existing OpenGrok indexes
57+
* without forcing a re-indexing)
58+
* @serial
59+
*/
60+
private Long analyzerGuruVersion;
61+
62+
/**
63+
* (nullable because otherwise custom de-serialization does not work, as a
64+
* {@code final} initialized value may not actually happen because Java
65+
* de-serialization circumvents normal construction.)
66+
* @serial
67+
*/
68+
private Map<String, Long> analyzersVersions = new HashMap<>();
69+
70+
/**
71+
* Gets the project name to be used to distinguish different instances of
72+
* {@link IndexAnalysisSettings} that might be returned by a Lucene
73+
* {@code MultiReader} search across projects.
74+
* @return projectName
75+
*/
76+
public String getProjectName() {
77+
return projectName;
78+
}
79+
80+
/**
81+
* Sets the project name to be used to distinguish different instances of
82+
* {@link IndexAnalysisSettings} that might be returned by a Lucene
83+
* {@code MultiReader} search across projects.
84+
* @param value
85+
*/
86+
public void setProjectName(String value) {
87+
this.projectName = value;
88+
}
89+
90+
public Integer getTabSize() {
91+
return tabSize;
92+
}
93+
94+
public void setTabSize(Integer value) {
95+
this.tabSize = value;
96+
}
97+
98+
public Long getAnalyzerGuruVersion() {
99+
return analyzerGuruVersion;
100+
}
101+
102+
public void setAnalyzerGuruVersion(Long value) {
103+
this.analyzerGuruVersion = value;
104+
}
105+
106+
/**
107+
* Gets the version number for the specified file type name if it exists
108+
* @return a defined value or {@code null} if unknown
109+
*/
110+
public Long getAnalyzerVersion(String fileTypeName) {
111+
return analyzersVersions.get(fileTypeName);
112+
}
113+
114+
/**
115+
* Gets an unmodifiable view of the map of file type names to version
116+
* numbers.
117+
* @return a defined instance
118+
*/
119+
public Map<String, Long> getAnalyzersVersions() {
120+
return Collections.unmodifiableMap(analyzersVersions);
121+
}
122+
123+
/**
124+
* Replaces the contents of the instance's map with the {@code values}.
125+
* @param values a defined instance
126+
*/
127+
public void setAnalyzersVersions(Map<String, Long> values) {
128+
analyzersVersions.clear();
129+
analyzersVersions.putAll(values);
130+
}
131+
132+
/**
133+
* Creates a binary representation of this object.
134+
* @return a byte array representing this object
135+
* @throws IOException Any exception thrown by the underlying
136+
* OutputStream.
137+
*/
138+
public byte[] serialize() throws IOException {
139+
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
140+
new ObjectOutputStream(bytes).writeObject(this);
141+
return bytes.toByteArray();
142+
}
143+
144+
/**
145+
* De-serializes a binary representation of an {@link IndexAnalysisSettings}
146+
* object.
147+
* @param bytes a byte array containing the serialization
148+
* @return a defined instance
149+
* @throws IOException Any of the usual Input/Output related exceptions.
150+
* @throws ClassNotFoundException Class of a serialized object cannot be
151+
* found.
152+
* @throws ClassCastException if the array contains an object of another
153+
* type than {@code IndexAnalysisSettings}
154+
*/
155+
public static IndexAnalysisSettings2 deserialize(byte[] bytes)
156+
throws IOException, ClassNotFoundException {
157+
ObjectInputStream in = new ObjectInputStream(
158+
new ByteArrayInputStream(bytes));
159+
return (IndexAnalysisSettings2)in.readObject();
160+
}
161+
162+
private void readObject(ObjectInputStream in) throws ClassNotFoundException,
163+
IOException {
164+
165+
boolean hasValue = in.readBoolean();
166+
String vstring = in.readUTF();
167+
projectName = hasValue ? vstring : null;
168+
169+
hasValue = in.readBoolean();
170+
int vint = in.readInt();
171+
tabSize = hasValue ? vint : null;
172+
173+
hasValue = in.readBoolean();
174+
long vlong = in.readLong();
175+
analyzerGuruVersion = hasValue ? vlong : null;
176+
177+
/**
178+
* De-serialization circumvents normal construction, so the following
179+
* field could be null.
180+
*/
181+
if (analyzersVersions == null) {
182+
analyzersVersions = new HashMap<>();
183+
}
184+
int analyzerCount = in.readInt();
185+
for (int i = 0; i < analyzerCount; ++i) {
186+
vstring = in.readUTF();
187+
vlong = in.readLong();
188+
analyzersVersions.put(vstring, vlong);
189+
}
190+
}
191+
192+
private void writeObject(ObjectOutputStream out) throws IOException {
193+
out.writeBoolean(projectName != null); // hasValue
194+
out.writeUTF(projectName == null ? "" : projectName);
195+
196+
out.writeBoolean(tabSize != null); // hasValue
197+
out.writeInt(tabSize == null ? 0 : tabSize);
198+
199+
out.writeBoolean(analyzerGuruVersion != null); // hasValue
200+
out.writeLong(analyzerGuruVersion == null ? 0 : analyzerGuruVersion);
201+
202+
int analyzerCount = analyzersVersions.size();
203+
out.writeInt(analyzerCount);
204+
for (Map.Entry<String, Long> entry : analyzersVersions.entrySet()) {
205+
out.writeUTF(entry.getKey());
206+
out.writeLong(entry.getValue());
207+
--analyzerCount;
208+
}
209+
if (analyzerCount != 0) {
210+
throw new IllegalStateException("analyzersVersions were modified");
211+
}
212+
}
213+
}

src/org/opensolaris/opengrok/index/IndexAnalysisSettingsAccessor.java

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@ public class IndexAnalysisSettingsAccessor {
5353
public static final String INDEX_ANALYSIS_SETTINGS_OBJUID =
5454
"uthuslvotkgltggqqjmurqojpjpjjkutkujktnkk";
5555

56+
private static final int INDEX_ANALYSIS_SETTINGS_OBJVER = 2;
57+
5658
/**
5759
* Searches for a document with a {@link QueryBuilder#OBJUID} value matching
5860
* {@link #INDEX_ANALYSIS_SETTINGS_OBJUID}. The first document found is
@@ -61,8 +63,8 @@ public class IndexAnalysisSettingsAccessor {
6163
* @return a defined instance or {@code null} if none could be found
6264
* @throws IOException if I/O error occurs while searching Lucene
6365
*/
64-
public IndexAnalysisSettings read(IndexReader reader) throws IOException {
65-
IndexAnalysisSettings[] res = read(reader, 1);
66+
public IndexAnalysisSettings2 read(IndexReader reader) throws IOException {
67+
IndexAnalysisSettings2[] res = read(reader, 1);
6668
return res.length > 0 ? res[0] : null;
6769
}
6870

@@ -75,7 +77,7 @@ public IndexAnalysisSettings read(IndexReader reader) throws IOException {
7577
* @return a defined instance, which is empty if none could be found
7678
* @throws IOException if I/O error occurs while searching Lucene
7779
*/
78-
public IndexAnalysisSettings[] read(IndexReader reader, int n)
80+
public IndexAnalysisSettings2[] read(IndexReader reader, int n)
7981
throws IOException {
8082
IndexSearcher searcher = new IndexSearcher(reader);
8183
Query q;
@@ -89,15 +91,17 @@ public IndexAnalysisSettings[] read(IndexReader reader, int n)
8991
TopDocs top = searcher.search(q, n);
9092

9193
int nres = top.totalHits > n ? n : (int)top.totalHits;
92-
IndexAnalysisSettings[] res = new IndexAnalysisSettings[nres];
94+
IndexAnalysisSettings2[] res = new IndexAnalysisSettings2[nres];
9395

96+
IndexAnalysisSettingsUpgrader upgrader =
97+
new IndexAnalysisSettingsUpgrader();
9498
for (int i = 0; i < nres; ++i) {
9599
Document doc = searcher.doc(top.scoreDocs[i].doc);
96100
IndexableField objser = doc.getField(QueryBuilder.OBJSER);
101+
int objver = readObjectVersion(doc);
97102
try {
98-
res[i] = objser == null ? null :
99-
IndexAnalysisSettings.deserialize(
100-
objser.binaryValue().bytes);
103+
res[i] = objser == null ? null : upgrader.upgrade(
104+
objser.binaryValue().bytes, objver);
101105
} catch (ClassNotFoundException ex) {
102106
// This is not expected, so translate to RuntimeException.
103107
throw new RuntimeException(ex);
@@ -115,7 +119,7 @@ public IndexAnalysisSettings[] read(IndexReader reader, int n)
115119
* @param settings a defined instance
116120
* @throws IOException if I/O error occurs while writing Lucene
117121
*/
118-
public void write(IndexWriter writer, IndexAnalysisSettings settings)
122+
public void write(IndexWriter writer, IndexAnalysisSettings2 settings)
119123
throws IOException {
120124
byte[] objser = settings.serialize();
121125

@@ -127,6 +131,13 @@ public void write(IndexWriter writer, IndexAnalysisSettings settings)
127131
INDEX_ANALYSIS_SETTINGS_OBJUID, Field.Store.NO);
128132
doc.add(uidfield);
129133
doc.add(new StoredField(QueryBuilder.OBJSER, objser));
134+
doc.add(new StoredField(QueryBuilder.OBJVER,
135+
INDEX_ANALYSIS_SETTINGS_OBJVER));
130136
writer.addDocument(doc);
131137
}
138+
139+
private int readObjectVersion(Document doc) {
140+
IndexableField objver = doc.getField(QueryBuilder.OBJVER);
141+
return objver == null ? 1 : objver.numericValue().intValue();
142+
}
132143
}

0 commit comments

Comments
 (0)