fixes #10 False matches in Java class files

Author: Lubos Kosco

src/org/opensolaris/opengrok/analysis/TagFilter.java

@@ -24,6 +24,7 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.StringReader;
 import java.io.StringWriter;
 import java.io.Writer;
 import java.util.ArrayList;
@@ -48,14 +49,14 @@
 import org.apache.bcel.classfile.LocalVariable;
 import org.apache.bcel.classfile.LocalVariableTable;
 import org.apache.bcel.classfile.Utility;
+import org.apache.lucene.analysis.charfilter.HTMLStripCharFilter;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field.Store;
 import org.apache.lucene.document.TextField;
 import org.opensolaris.opengrok.analysis.FileAnalyzer;
 import org.opensolaris.opengrok.analysis.FileAnalyzerFactory;
 import org.opensolaris.opengrok.analysis.IteratorReader;
 import org.opensolaris.opengrok.analysis.StreamSource;
-import org.opensolaris.opengrok.analysis.TagFilter;
 import org.opensolaris.opengrok.configuration.RuntimeEnvironment;
 
 /**
@@ -107,8 +108,7 @@ void analyze(Document doc, InputStream in, Writer xrefOut) throws IOException {
 
         doc.add(new TextField("defs", new IteratorReader(defs)));
         doc.add(new TextField("refs", new IteratorReader(refs)));
-        // TODO could be improved, lucene has xhtml parsers/readers
-        doc.add(new TextField("full", new TagFilter(xref)));
+        doc.add(new TextField("full", new HTMLStripCharFilter(new StringReader(xref))));
         doc.add(new TextField("full", constants, Store.NO));
     }
 

src/org/opensolaris/opengrok/analysis/executables/JavaClassAnalyzer.java

@@ -37,6 +37,7 @@
 import java.util.Map;
 import java.util.logging.Level;
 import java.util.zip.GZIPInputStream;
+import org.apache.lucene.analysis.charfilter.HTMLStripCharFilter;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.IndexableField;
@@ -45,7 +46,6 @@
 import org.opensolaris.opengrok.OpenGrokLogger;
 import org.opensolaris.opengrok.analysis.Definitions;
 import org.opensolaris.opengrok.analysis.FileAnalyzer.Genre;
-import org.opensolaris.opengrok.analysis.TagFilter;
 import org.opensolaris.opengrok.history.HistoryException;
 import org.opensolaris.opengrok.web.Prefix;
 import org.opensolaris.opengrok.web.SearchHelper;
@@ -92,7 +92,7 @@ private static Map<String, ArrayList<Document>> createMap(IndexSearcher searcher
 
     private static String getTags(File basedir, String path, boolean compressed) {
         char[] content = new char[1024 * 8];
-        try (TagFilter r = new TagFilter(getXrefReader(basedir, path, compressed))) {
+        try (HTMLStripCharFilter r = new HTMLStripCharFilter(getXrefReader(basedir, path, compressed))) {
             int len = r.read(content);
             return new String(content, 0, len);
         } catch (Exception e) {

src/org/opensolaris/opengrok/search/Results.java

@@ -29,6 +29,7 @@
 import java.util.concurrent.Executors;
 import java.util.logging.Level;
 import java.util.zip.GZIPInputStream;
+import org.apache.lucene.analysis.charfilter.HTMLStripCharFilter;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
@@ -44,7 +45,6 @@
 import org.opensolaris.opengrok.analysis.CompatibleAnalyser;
 import org.opensolaris.opengrok.analysis.Definitions;
 import org.opensolaris.opengrok.analysis.FileAnalyzer.Genre;
-import org.opensolaris.opengrok.analysis.TagFilter;
 import org.opensolaris.opengrok.configuration.Project;
 import org.opensolaris.opengrok.configuration.RuntimeEnvironment;
 import org.opensolaris.opengrok.history.HistoryException;
@@ -358,8 +358,8 @@ public void results(int start, int end, List<Hit> ret) {
                         } else if (Genre.XREFABLE == genre && data != null && summarizer != null) {
                             int l = 0;
                             try (Reader r = RuntimeEnvironment.getInstance().isCompressXref() ?
-                                     new TagFilter(new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(data + "/xref" + filename + ".gz"))))) :
-                                     new TagFilter(new BufferedReader(new FileReader(data + "/xref" + filename)))) {
+                                     new HTMLStripCharFilter(new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(data + "/xref" + filename + ".gz"))))) :
+                                     new HTMLStripCharFilter(new BufferedReader(new FileReader(data + "/xref" + filename)))) {
                                 l = r.read(content);
                             }
                             //TODO FIX below fragmenter according to either summarizer or context (to get line numbers, might be hard, since xref writers will need to be fixed too, they generate just one line of html code now :( )