Merge pull request #2142 from guwirth/fix-2139

support BOM in #include files

Author: guwirth

cxx-squid/pom.xml

@@ -43,6 +43,14 @@
       <artifactId>jackson-databind</artifactId>
       <version>2.12.3</version>
     </dependency>
+    <dependency>
+      <groupId>commons-io</groupId>
+      <artifactId>commons-io</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>commons-lang</groupId>
+      <artifactId>commons-lang</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.sonarsource.sslr</groupId>
       <artifactId>sslr-testing-harness</artifactId>

cxx-squid/src/main/java/org/sonar/cxx/preprocessor/SourceCodeProvider.java

@@ -20,6 +20,7 @@
 package org.sonar.cxx.preprocessor;
 
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.IOException;
 import java.nio.charset.Charset;
 import java.nio.file.Files;
@@ -32,6 +33,8 @@
 import java.util.Objects;
 import javax.annotation.CheckForNull;
 import javax.annotation.Nullable;
+import org.apache.commons.io.ByteOrderMark;
+import org.apache.commons.io.input.BOMInputStream;
 import org.sonar.api.utils.log.Logger;
 import org.sonar.api.utils.log.Loggers;
 
@@ -199,9 +202,18 @@ public File getSourceCodeFile(String filename, boolean quoted) {
     return result;
   }
 
-  public String getSourceCode(File file, Charset charset) throws IOException {
-    byte[] encoded = Files.readAllBytes(Paths.get(file.getAbsolutePath()));
-    return new String(encoded, charset);
+  public String getSourceCode(File file, Charset defaultCharset) throws IOException {
+    try (BOMInputStream bomInputStream = new BOMInputStream(new FileInputStream(file),
+                                                            ByteOrderMark.UTF_8,
+                                                            ByteOrderMark.UTF_16LE,
+                                                            ByteOrderMark.UTF_16BE,
+                                                            ByteOrderMark.UTF_32LE,
+                                                            ByteOrderMark.UTF_32BE)) {
+      ByteOrderMark bom = bomInputStream.getBOM();
+      Charset charset = bom != null ? Charset.forName(bom.getCharsetName()) : defaultCharset;
+      byte[] bytes = bomInputStream.readAllBytes();
+      return new String(bytes, charset);
+    }
   }
 
   private static class State {

cxx-squid/src/test/java/org/sonar/cxx/preprocessor/SourceCodeProviderTest.java

@@ -32,6 +32,7 @@ public class SourceCodeProviderTest {
 
   private final File expected1 = new File(new File("src/test/resources/codeprovider/source.hh").getAbsolutePath());
   private final File expected2 = new File(new File("src/test/resources/codeprovider/source").getAbsolutePath());
+  private final File root = new File(new File("src/test/resources/codeprovider").getAbsolutePath());
 
   // ////////////////////////////////////////////////////////////////////////////
   // Behavior in the absolute path case
@@ -205,4 +206,26 @@ public void getting_source_code2() throws IOException {
     assertEquals("source code", codeProvider.getSourceCode(expected2, Charset.defaultCharset()));
   }
 
+  @Test
+  public void getting_source_code_utf_8() throws IOException {
+    SourceCodeProvider codeProvider = new SourceCodeProvider(new File("dummy"));
+    assertEquals("UTF-8", codeProvider.getSourceCode(new File(root, "./utf-8.hh"),
+                                                     Charset.defaultCharset()));
+  }
+
+  @Test
+  public void getting_source_code_utf_8_bom() throws IOException {
+    SourceCodeProvider codeProvider = new SourceCodeProvider(new File("dummy"));
+    assertEquals("UTF-8-BOM", codeProvider.getSourceCode(new File(root, "./utf-8-bom.hh"),
+                                                         Charset.defaultCharset()));
+  }
+
+  @Test
+  public void getting_source_code_utf_16_le_bom() throws IOException {
+    SourceCodeProvider codeProvider = new SourceCodeProvider(new File("dummy"));
+    assertEquals("UTF-16LE-BOM",
+                 codeProvider.getSourceCode(new File(root, "./utf-16le-bom.hh"),
+                                            Charset.defaultCharset()));
+  }
+
 }

cxx-squid/src/test/resources/codeprovider/utf-16le-bom.hh

@@ -0,0 +1 @@
+UTF-8-BOM

cxx-squid/src/test/resources/codeprovider/utf-8-bom.hh

@@ -0,0 +1 @@
+UTF-8