Skip to content

Commit 108cb68

Browse files
committed
Created StopWords class and test for retrieving words from stopwords files
1 parent 8b73ab7 commit 108cb68

File tree

5 files changed

+1355
-678
lines changed

5 files changed

+1355
-678
lines changed

logicaldoc-core/src/main/java/com/logicaldoc/core/i18n/Language.java

Lines changed: 4 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,6 @@
11
package com.logicaldoc.core.i18n;
22

3-
import java.io.BufferedReader;
4-
import java.io.InputStream;
5-
import java.io.InputStreamReader;
63
import java.lang.reflect.Constructor;
7-
import java.nio.charset.StandardCharsets;
84
import java.util.HashSet;
95
import java.util.Locale;
106
import java.util.Set;
@@ -40,7 +36,6 @@ public class Language implements Comparable<Language> {
4036

4137
public Language(Locale locale) {
4238
this.locale = locale;
43-
loadStopwords();
4439
}
4540

4641
public Locale getLocale() {
@@ -59,44 +54,12 @@ public String getDefaultDisplayLanguage() {
5954
return locale.getDisplayLanguage(Locale.ENGLISH);
6055
}
6156

62-
/**
63-
* Populates the field stopWords reading the resource
64-
* /stopwords/stopwords_<b>locale</b>.txt
65-
*/
66-
void loadStopwords() {
67-
try {
68-
Set<String> swSet = new HashSet<>();
69-
String stopwordsResource = "/stopwords/stopwords_" + getLocale().toString() + ".txt";
70-
log.debug("Loading stopwords from: {}", stopwordsResource);
71-
InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream(stopwordsResource);
72-
if (is == null)
73-
is = getClass().getResourceAsStream(stopwordsResource);
74-
75-
if (is == null) {
76-
log.warn("No stopwords found for locale {}", getLocale());
77-
} else {
78-
InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8);
79-
BufferedReader br = new BufferedReader(isr);
80-
String line = null;
81-
while ((line = br.readLine()) != null) {
82-
line = line.trim();
83-
if (line.indexOf("|") != -1) {
84-
line = line.substring(0, line.indexOf("|"));
85-
line = line.trim();
86-
}
87-
if (line != null && line.length() > 0 && !swSet.contains(line)) {
88-
swSet.add(line);
89-
}
90-
}
91-
}
92-
stopWords = swSet;
93-
} catch (Exception e) {
94-
log.warn(e.getMessage(), e);
95-
}
57+
public Set<String> getStopWords() {
58+
return StopWords.getStopWords(locale);
9659
}
9760

98-
public Set<String> getStopWords() {
99-
return stopWords;
61+
public void setStopWords(Set<String> stopWords) {
62+
this.stopWords = stopWords;
10063
}
10164

10265
public String getAnalyzerClass() {

0 commit comments

Comments
 (0)