Merge pull request #1635 from tetsurom/master

Improve UTF-8/16 BOM handling.

Author: tarzanek

src/org/opensolaris/opengrok/analysis/TextAnalyzer.java

@@ -28,6 +28,7 @@
 import java.io.InputStreamReader;
 import java.io.Reader;
 import java.nio.charset.Charset;
+import org.opensolaris.opengrok.util.IOUtils;
 
 public abstract class TextAnalyzer extends FileAnalyzer {
 
@@ -36,33 +37,6 @@ public TextAnalyzer(FileAnalyzerFactory factory) {
     }
 
     protected Reader getReader(InputStream stream) throws IOException {
-        InputStream in = stream.markSupported() ?
-                stream : new BufferedInputStream(stream);
-
-        String charset = null;
-
-        in.mark(3);
-
-        byte[] head = new byte[3];
-        int br = in.read(head, 0, 3);
-
-        if (br >= 2
-                && (head[0] == (byte) 0xFE && head[1] == (byte) 0xFF)
-                || (head[0] == (byte) 0xFF && head[1] == (byte) 0xFE)) {
-            charset = "UTF-16";
-            in.reset();
-        } else if (br >= 3 && head[0] == (byte) 0xEF && head[1] == (byte) 0xBB
-                && head[2] == (byte) 0xBF) {
-            // InputStreamReader does not properly discard BOM on UTF8 streams,
-            // so don't reset the stream.
-            charset = "UTF-8";
-        }
-
-        if (charset == null) {
-            in.reset();
-            charset = Charset.defaultCharset().name();
-        }
-
-        return new InputStreamReader(in, charset);
+        return IOUtils.createBOMStrippedReader(stream);
     }
 }

src/org/opensolaris/opengrok/search/Results.java

@@ -55,6 +55,7 @@
 import org.opensolaris.opengrok.configuration.RuntimeEnvironment;
 import org.opensolaris.opengrok.history.HistoryException;
 import org.opensolaris.opengrok.logger.LoggerFactory;
+import org.opensolaris.opengrok.util.IOUtils;
 import org.opensolaris.opengrok.web.Prefix;
 import org.opensolaris.opengrok.web.SearchHelper;
 import org.opensolaris.opengrok.web.Util;
@@ -118,10 +119,11 @@ private static Reader getXrefReader(
                     File basedir, String path, boolean compressed)
             throws IOException {
         if (compressed) {
-            return new BufferedReader(new InputStreamReader(new GZIPInputStream(
-                    new FileInputStream(new File(basedir, path + ".gz")))));
+            return new BufferedReader(IOUtils.createBOMStrippedReader(
+                    new GZIPInputStream(new FileInputStream(new File(basedir, path + ".gz")))));
         } else {
-            return new BufferedReader(new FileReader(new File(basedir, path)));
+            return new BufferedReader(IOUtils.createBOMStrippedReader(
+                    new FileInputStream(new File(basedir, path))));
         }
     }
 
@@ -233,8 +235,8 @@ public static void prettyPrint(Writer out, SearchHelper sh, int start,
                         String htags = getTags(sh.sourceRoot, rpath, false);
                         out.write(sh.summarizer.getSummary(htags).toString());
                     } else {
-                        FileReader r = genre == Genre.PLAIN
-                                ? new FileReader(new File(sh.sourceRoot, rpath))
+                        Reader r = genre == Genre.PLAIN
+                                ? IOUtils.createBOMStrippedReader(new FileInputStream(new File(sh.sourceRoot, rpath)))
                                 : null;
                         sh.sourceContext.getContext(r, out, xrefPrefix, morePrefix, 
                                 rpath, tags, true, sh.builder.isDefSearch(), null, scopes);

src/org/opensolaris/opengrok/util/IOUtils.java

@@ -23,10 +23,16 @@
  */
 package org.opensolaris.opengrok.util;
 
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
 import java.io.Closeable;
 import java.io.File;
 import java.io.FilenameFilter;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.nio.charset.Charset;
 import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -165,4 +171,52 @@ public boolean accept(File dir, String name) {
         }
         return Arrays.asList(files);
     }
+
+    /**
+     * Create BOM stripped reader from the stream.
+     * Charset of the reader is set to UTF-8, UTF-16 or system's default.
+     * @param stream input stream
+     * @return reader for the stream without BOM
+     */
+    public static Reader createBOMStrippedReader(InputStream stream) throws IOException {
+        return createBOMStrippedReader(stream, Charset.defaultCharset().name());
+    }
+
+    /**
+     * Create BOM stripped reader from the stream.
+     * Charset of the reader is set to UTF-8, UTF-16 or default.
+     * @param stream input stream
+     * @param defaultCharset default charset
+     * @return reader for the stream without BOM
+     */    
+    public static Reader createBOMStrippedReader(InputStream stream, String defaultCharset) throws IOException {
+        InputStream in = stream.markSupported() ?
+                stream : new BufferedInputStream(stream);
+
+        String charset = null;
+
+        in.mark(3);
+
+        byte[] head = new byte[3];
+        int br = in.read(head, 0, 3);
+
+        if (br >= 2
+                && (head[0] == (byte) 0xFE && head[1] == (byte) 0xFF)
+                || (head[0] == (byte) 0xFF && head[1] == (byte) 0xFE)) {
+            charset = "UTF-16";
+            in.reset();
+        } else if (br >= 3 && head[0] == (byte) 0xEF && head[1] == (byte) 0xBB
+                && head[2] == (byte) 0xBF) {
+            // InputStreamReader does not properly discard BOM on UTF8 streams,
+            // so don't reset the stream.
+            charset = "UTF-8";
+        }
+
+        if (charset == null) {
+            in.reset();
+            charset = defaultCharset;
+        }
+
+        return new InputStreamReader(in, charset);
+    }
 }

src/org/opensolaris/opengrok/web/PageConfig.java

@@ -267,7 +267,7 @@ public DiffData getDiffData() {
                 Project p = getProject();
                 for (int i = 0; i < 2; i++) {
                     try (BufferedReader br = new BufferedReader(
-                            ExpandTabsReader.wrap(new InputStreamReader(in[i]), p))) {
+                            ExpandTabsReader.wrap(IOUtils.createBOMStrippedReader(in[i]), p))) {
                         String line;
                         while ((line = br.readLine()) != null) {
                             lines.add(line);

web/list.jsp

@@ -45,6 +45,7 @@ org.opensolaris.opengrok.analysis.FileAnalyzer.Genre,
 org.opensolaris.opengrok.analysis.FileAnalyzerFactory,
 org.opensolaris.opengrok.history.Annotation,
 org.opensolaris.opengrok.index.IndexDatabase,
+org.opensolaris.opengrok.util.IOUtils,
 org.opensolaris.opengrok.web.DirectoryListing"
 %><%
 {
@@ -180,7 +181,7 @@ Binary file [Click <a href="<%= rawPath %>?r=<%= Util.URIEncode(rev) %>">here</a
                             Annotation annotation = cfg.getAnnotation();
                             //not needed yet
                             //annotation.writeTooltipMap(out);
-                            r = new InputStreamReader(in);
+                            r = IOUtils.createBOMStrippedReader(in);
                             AnalyzerGuru.writeXref(a, r, out, defs,
                                 annotation, Project.getProject(resourceFile));
                         } else if (g == Genre.IMAGE) {
@@ -267,7 +268,7 @@ Binary file [Click <a href="<%= rawPath %>?r=<%= Util.URIEncode(rev) %>">here</a
                     // find the definitions in the index.
                     Definitions defs = IndexDatabase.getDefinitions(resourceFile);
                     Annotation annotation = cfg.getAnnotation();
-                    r = new InputStreamReader(bin);
+                    r = IOUtils.createBOMStrippedReader(bin);
                     AnalyzerGuru.writeXref(a, r, out, defs, annotation,
                         Project.getProject(resourceFile));
     %></pre>

web/more.jsp

@@ -23,14 +23,16 @@ Copyright (c) 2010, 2017, Oracle and/or its affiliates. All rights reserved.
 Portions Copyright 2011 Jens Elkner.
 
 --%><%@page errorPage="error.jsp" import="
-java.io.FileReader,
+java.io.FileInputStream,
+java.io.Reader,
 java.util.logging.Level,
 java.util.logging.Logger,
 
 org.apache.lucene.search.Query,
 org.opensolaris.opengrok.search.QueryBuilder,
 org.opensolaris.opengrok.search.context.Context,
-org.opensolaris.opengrok.logger.LoggerFactory"
+org.opensolaris.opengrok.logger.LoggerFactory,
+org.opensolaris.opengrok.util.IOUtils"
 %>
 <%
 {
@@ -54,7 +56,9 @@ file="mast.jsp"
 %><p><span class="pagetitle">Lines Matching <b><%= tquery %></b></span></p>
 <div id="more" style="line-height:1.5em;">
     <pre><%
-            sourceContext.getContext(new FileReader(cfg.getResourceFile()), out,
+            Reader r = IOUtils.createBOMStrippedReader(
+                    new FileInputStream(cfg.getResourceFile()));
+            sourceContext.getContext(r, out,
                 request.getContextPath() + Prefix.XREF_P, null, cfg.getPath(),
                 null, false, false, null, null);
     %></pre>