Skip to content

Commit 7d00439

Browse files
idodeclareidodeclare
committed
Save symlinks data during indexing for use by UI
Also: fix to include File.separator in comparison for IndexDatabase isLocal().
1 parent 8eaf39d commit 7d00439

File tree

10 files changed

+780
-99
lines changed

10 files changed

+780
-99
lines changed

‎opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/AnalyzerGuru.java

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,6 @@
111111
import org.opengrok.indexer.history.HistoryReader;
112112
import org.opengrok.indexer.logger.LoggerFactory;
113113
import org.opengrok.indexer.search.QueryBuilder;
114-
import org.opengrok.indexer.util.ForbiddenSymlinkException;
115114
import org.opengrok.indexer.util.IOUtils;
116115
import org.opengrok.indexer.web.Util;
117116

@@ -525,12 +524,10 @@ public static void returnAnalyzers() {
525524
* @param xrefOut Where to write the xref (possibly {@code null})
526525
* @throws IOException If an exception occurs while collecting the data
527526
* @throws InterruptedException if a timeout occurs
528-
* @throws ForbiddenSymlinkException if symbolic-link checking encounters
529-
* an ineligible link
530527
*/
531528
public void populateDocument(Document doc, File file, String path,
532529
AbstractAnalyzer fa, Writer xrefOut) throws IOException,
533-
InterruptedException, ForbiddenSymlinkException {
530+
InterruptedException {
534531

535532
String date = DateTools.timeToString(file.lastModified(),
536533
DateTools.Resolution.MILLISECOND);

‎opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexAnalysisSettings3.java

Lines changed: 270 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,270 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Copyright (c) 2018-2019, Chris Fraire <[email protected]>.
22+
*/
23+
24+
package org.opengrok.indexer.index;
25+
26+
import java.io.ByteArrayInputStream;
27+
import java.io.ByteArrayOutputStream;
28+
import java.io.IOException;
29+
import java.io.ObjectInputStream;
30+
import java.io.ObjectOutputStream;
31+
import java.io.Serializable;
32+
import java.util.Collections;
33+
import java.util.HashMap;
34+
import java.util.Map;
35+
36+
/**
37+
* Represents a serializable gathering of some top-level metadata concerning the
38+
* operation of {@link IndexDatabase} -- and persisted therein too -- which are
39+
* re-compared upon each indexing run since changes to them might require
40+
* re-indexing particular files or in certain cases all files.
41+
*/
42+
public final class IndexAnalysisSettings3 implements Serializable {
43+
44+
private static final long serialVersionUID = -4700122690707062276L;
45+
46+
private String projectName;
47+
48+
/**
49+
* Nullable to allow easing this object into existing OpenGrok indexes
50+
* without forcing a re-indexing.
51+
* @serial
52+
*/
53+
private Integer tabSize;
54+
55+
/**
56+
* Nullable to allow easing this object into existing OpenGrok indexes
57+
* without forcing a re-indexing.
58+
* @serial
59+
*/
60+
private Long analyzerGuruVersion;
61+
62+
/**
63+
* Nullable because otherwise custom de-serialization does not work, as a
64+
* {@code final} initialized value may not actually happen because Java
65+
* de-serialization circumvents normal construction.
66+
* @serial
67+
*/
68+
private Map<String, Long> analyzersVersions = new HashMap<>();
69+
70+
/**
71+
* Nullable because otherwise custom de-serialization does not work, as a
72+
* {@code final} initialized value may not actually happen because Java
73+
* de-serialization circumvents normal construction. We don't bother with
74+
* anything but a simple {@link HashMap} here.
75+
* @serial
76+
*/
77+
private Map<String, IndexedSymlink> indexedSymlinks = new HashMap<>();
78+
79+
/**
80+
* Gets the project name to be used to distinguish different instances of
81+
* {@link IndexAnalysisSettings3} that might be returned by a Lucene
82+
* {@code MultiReader} search across projects.
83+
* @return projectName
84+
*/
85+
public String getProjectName() {
86+
return projectName;
87+
}
88+
89+
/**
90+
* Sets the project name to be used to distinguish different instances of
91+
* {@link IndexAnalysisSettings3} that might be returned by a Lucene
92+
* {@code MultiReader} search across projects.
93+
* @param value project name
94+
*/
95+
public void setProjectName(String value) {
96+
this.projectName = value;
97+
}
98+
99+
public Integer getTabSize() {
100+
return tabSize;
101+
}
102+
103+
public void setTabSize(Integer value) {
104+
this.tabSize = value;
105+
}
106+
107+
public Long getAnalyzerGuruVersion() {
108+
return analyzerGuruVersion;
109+
}
110+
111+
public void setAnalyzerGuruVersion(Long value) {
112+
this.analyzerGuruVersion = value;
113+
}
114+
115+
/**
116+
* Gets the version number for the specified file type name if it exists.
117+
* @param fileTypeName name of the file type
118+
* @return a defined value or {@code null} if unknown
119+
*/
120+
public Long getAnalyzerVersion(String fileTypeName) {
121+
return analyzersVersions.get(fileTypeName);
122+
}
123+
124+
/**
125+
* Gets an unmodifiable view of the map of file type names to version
126+
* numbers.
127+
* @return a defined instance
128+
*/
129+
public Map<String, Long> getAnalyzersVersions() {
130+
return Collections.unmodifiableMap(analyzersVersions);
131+
}
132+
133+
/**
134+
* Replaces the contents of the instance's map with the {@code values}.
135+
* @param values a defined instance
136+
*/
137+
public void setAnalyzersVersions(Map<String, Long> values) {
138+
analyzersVersions.clear();
139+
analyzersVersions.putAll(values);
140+
}
141+
142+
/**
143+
* Gets an unmodifiable view of the map of canonical file names to symlinks.
144+
* @return a defined instance
145+
*/
146+
public Map<String, IndexedSymlink> getIndexedSymlinks() {
147+
return Collections.unmodifiableMap(indexedSymlinks);
148+
}
149+
150+
/**
151+
* Replaces the contents of the instance's map with the {@code values}.
152+
* @param values a defined instance
153+
*/
154+
public void setIndexedSymlinks(Map<String, IndexedSymlink> values) {
155+
indexedSymlinks.clear();
156+
indexedSymlinks.putAll(values);
157+
}
158+
159+
/**
160+
* Creates a binary representation of this object.
161+
* @return a byte array representing this object
162+
* @throws IOException Any exception thrown by the underlying
163+
* OutputStream.
164+
*/
165+
public byte[] serialize() throws IOException {
166+
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
167+
new ObjectOutputStream(bytes).writeObject(this);
168+
return bytes.toByteArray();
169+
}
170+
171+
/**
172+
* De-serializes a binary representation of an {@link IndexAnalysisSettings3}
173+
* object.
174+
* @param bytes a byte array containing the serialization
175+
* @return a defined instance
176+
* @throws IOException Any of the usual Input/Output related exceptions.
177+
* @throws ClassNotFoundException Class of a serialized object cannot be
178+
* found.
179+
* @throws ClassCastException if the array contains an object of another
180+
* type than {@code IndexAnalysisSettings}
181+
*/
182+
public static IndexAnalysisSettings3 deserialize(byte[] bytes)
183+
throws IOException, ClassNotFoundException {
184+
ObjectInputStream in = new ObjectInputStream(
185+
new ByteArrayInputStream(bytes));
186+
return (IndexAnalysisSettings3) in.readObject();
187+
}
188+
189+
@SuppressWarnings("Duplicates")
190+
private void readObject(ObjectInputStream in) throws ClassNotFoundException,
191+
IOException {
192+
193+
boolean hasValue = in.readBoolean();
194+
String vString = in.readUTF();
195+
projectName = hasValue ? vString : null;
196+
197+
hasValue = in.readBoolean();
198+
int vInteger = in.readInt();
199+
tabSize = hasValue ? vInteger : null;
200+
201+
hasValue = in.readBoolean();
202+
long vLong = in.readLong();
203+
analyzerGuruVersion = hasValue ? vLong : null;
204+
205+
/*
206+
* De-serialization circumvents normal construction, so the following
207+
* field could be null.
208+
*/
209+
if (analyzersVersions == null) {
210+
analyzersVersions = new HashMap<>();
211+
}
212+
int analyzerCount = in.readInt();
213+
for (int i = 0; i < analyzerCount; ++i) {
214+
vString = in.readUTF();
215+
vLong = in.readLong();
216+
analyzersVersions.put(vString, vLong);
217+
}
218+
219+
/*
220+
* De-serialization circumvents normal construction, so the following
221+
* field could be null.
222+
*/
223+
if (indexedSymlinks == null) {
224+
indexedSymlinks = new HashMap<>();
225+
}
226+
int symlinkCount = in.readInt();
227+
for (int i = 0; i < symlinkCount; ++i) {
228+
String absolute = in.readUTF();
229+
String canonical = in.readUTF();
230+
boolean isLocal = in.readBoolean();
231+
IndexedSymlink indexed = new IndexedSymlink(absolute, canonical, isLocal);
232+
indexedSymlinks.put(canonical, indexed);
233+
}
234+
}
235+
236+
@SuppressWarnings("Duplicates")
237+
private void writeObject(ObjectOutputStream out) throws IOException {
238+
out.writeBoolean(projectName != null); // hasValue
239+
out.writeUTF(projectName == null ? "" : projectName);
240+
241+
out.writeBoolean(tabSize != null); // hasValue
242+
out.writeInt(tabSize == null ? 0 : tabSize);
243+
244+
out.writeBoolean(analyzerGuruVersion != null); // hasValue
245+
out.writeLong(analyzerGuruVersion == null ? 0 : analyzerGuruVersion);
246+
247+
int collectionCount = analyzersVersions.size();
248+
out.writeInt(collectionCount);
249+
for (Map.Entry<String, Long> entry : analyzersVersions.entrySet()) {
250+
out.writeUTF(entry.getKey());
251+
out.writeLong(entry.getValue());
252+
--collectionCount;
253+
}
254+
if (collectionCount != 0) {
255+
throw new IllegalStateException("analyzersVersions were modified");
256+
}
257+
258+
collectionCount = indexedSymlinks.size();
259+
out.writeInt(collectionCount);
260+
for (IndexedSymlink entry : indexedSymlinks.values()) {
261+
out.writeUTF(entry.getAbsolute());
262+
out.writeUTF(entry.getCanonical());
263+
out.writeBoolean(entry.isLocal());
264+
--collectionCount;
265+
}
266+
if (collectionCount != 0) {
267+
throw new IllegalStateException("indexedSymlinks were modified");
268+
}
269+
}
270+
}

‎opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexAnalysisSettingsAccessor.java

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
*/
1919

2020
/*
21-
* Copyright (c) 2018, Chris Fraire <[email protected]>.
21+
* Copyright (c) 2018-2019, Chris Fraire <[email protected]>.
2222
*/
2323

2424
package org.opengrok.indexer.index;
@@ -50,7 +50,7 @@ public class IndexAnalysisSettingsAccessor {
5050
*/
5151
static final String INDEX_ANALYSIS_SETTINGS_OBJUID = "uthuslvotkgltggqqjmurqojpjpjjkutkujktnkk";
5252

53-
private static final int INDEX_ANALYSIS_SETTINGS_OBJVER = 2;
53+
private static final int INDEX_ANALYSIS_SETTINGS_OBJVER = 3;
5454

5555
/**
5656
* Searches for a document with a {@link QueryBuilder#OBJUID} value matching
@@ -60,8 +60,8 @@ public class IndexAnalysisSettingsAccessor {
6060
* @return a defined instance or {@code null} if none could be found
6161
* @throws IOException if I/O error occurs while searching Lucene
6262
*/
63-
public IndexAnalysisSettings read(IndexReader reader) throws IOException {
64-
IndexAnalysisSettings[] res = read(reader, 1);
63+
public IndexAnalysisSettings3 read(IndexReader reader) throws IOException {
64+
IndexAnalysisSettings3[] res = read(reader, 1);
6565
return res.length > 0 ? res[0] : null;
6666
}
6767

@@ -74,7 +74,7 @@ public IndexAnalysisSettings read(IndexReader reader) throws IOException {
7474
* @return a defined instance, which is empty if none could be found
7575
* @throws IOException if I/O error occurs while searching Lucene
7676
*/
77-
public IndexAnalysisSettings[] read(IndexReader reader, int n)
77+
public IndexAnalysisSettings3[] read(IndexReader reader, int n)
7878
throws IOException {
7979
IndexSearcher searcher = new IndexSearcher(reader);
8080
Query q;
@@ -88,18 +88,16 @@ public IndexAnalysisSettings[] read(IndexReader reader, int n)
8888
TopDocs top = searcher.search(q, n);
8989

9090
int nres = top.totalHits.value > n ? n : (int) top.totalHits.value;
91-
IndexAnalysisSettings[] res = new IndexAnalysisSettings[nres];
91+
IndexAnalysisSettings3[] res = new IndexAnalysisSettings3[nres];
9292

93+
IndexAnalysisSettingsUpgrader upgrader = new IndexAnalysisSettingsUpgrader();
9394
for (int i = 0; i < nres; ++i) {
9495
Document doc = searcher.doc(top.scoreDocs[i].doc);
9596
IndexableField objser = doc.getField(QueryBuilder.OBJSER);
9697
int objver = readObjectVersion(doc);
97-
if (objver != INDEX_ANALYSIS_SETTINGS_OBJVER) {
98-
throw new IllegalArgumentException("Unknown version " + objver);
99-
}
10098
try {
101-
res[i] = objser == null ? null : IndexAnalysisSettings.deserialize(
102-
objser.binaryValue().bytes);
99+
res[i] = objser == null ? null : upgrader.upgrade(
100+
objser.binaryValue().bytes, objver);
103101
} catch (ClassNotFoundException ex) {
104102
// This is not expected, so translate to RuntimeException.
105103
throw new RuntimeException(ex);
@@ -117,7 +115,7 @@ public IndexAnalysisSettings[] read(IndexReader reader, int n)
117115
* @param settings a defined instance
118116
* @throws IOException if I/O error occurs while writing Lucene
119117
*/
120-
public void write(IndexWriter writer, IndexAnalysisSettings settings)
118+
public void write(IndexWriter writer, IndexAnalysisSettings3 settings)
121119
throws IOException {
122120
byte[] objser = settings.serialize();
123121

0 commit comments

Comments
 (0)