Bugfix for a bug introducd in 0.6.3

Author: JohannesJander

pom.xml

@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>io.frictionlessdata</groupId>
     <artifactId>tableschema-java</artifactId>
-    <version>0.6.2-SNAPSHOT</version>
+    <version>0.6.4-SNAPSHOT</version>
     <packaging>jar</packaging>
     <issueManagement>
         <url>https://github.com/frictionlessdata/tableschema-java/issues</url>

src/main/java/io/frictionlessdata/tableschema/tabledatasource/TableDataSource.java

@@ -68,30 +68,13 @@ public interface TableDataSource {
      * @return DataSource created from input String
      */
     static TableDataSource fromSource(String input) {
-        String cleanedInput = input;
-        try {
-            byte[] bytes = new byte[input.length()];
-            input.getBytes(0, input.length(), bytes, 0);
-            ByteOrderMarkStrippingInputStream is
-                    = new ByteOrderMarkStrippingInputStream(new ByteArrayInputStream(bytes));
-            is.skipBOM();
-
-            InputStreamReader rdr = new InputStreamReader(is);
-            BufferedReader bfr = new BufferedReader(rdr);
-            cleanedInput = bfr.lines().collect(Collectors.joining("\n"));
-            is.close();
-            rdr.close();
-            bfr.close();
-        } catch (Exception ex) {
-            throw new TableIOException(ex);
-        }
         try {
             // JSON array generation. If an exception is thrown -> probably CSV data
-            ArrayNode json = JsonUtil.getInstance().createArrayNode(cleanedInput);
+            ArrayNode json = JsonUtil.getInstance().createArrayNode(input);
             return new JsonArrayTableDataSource(input);
         } catch (Exception ex) {
             // JSON parsing failed, treat it as a CSV
-            return new CsvTableDataSource(cleanedInput);
+            return new CsvTableDataSource(input);
         }
     }
 

src/test/java/io/frictionlessdata/tableschema/table_tests/TableEncodingTests.java

@@ -1,6 +1,7 @@
 package io.frictionlessdata.tableschema.table_tests;
 
 import io.frictionlessdata.tableschema.Table;
+import io.frictionlessdata.tableschema.schema.Schema;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.DisplayName;
@@ -9,6 +10,7 @@
 import java.io.File;
 import java.util.Iterator;
 
+import static io.frictionlessdata.tableschema.TestHelper.getTestDataDirectory;
 import static io.frictionlessdata.tableschema.TestHelper.getTestsuiteDataDirectory;
 
 public class TableEncodingTests {
@@ -28,4 +30,23 @@ void createTableFromIso8859() throws Exception{
         Object[] row = iter.next();
         Assertions.assertEquals("Réunion", row[0]);
     }
+
+    @Test
+    @DisplayName("Create a Table from a UTF-8 encoded file with non-ASCII chars")
+    @Disabled
+    void createTableFromUTF8() throws Exception{
+        File testDataDir = getTestDataDirectory();
+        Schema schema = Schema.fromJson(new File(testDataDir, "schema/units_schema.json"), true);
+
+        Table table
+                = Table.fromSource(new File("data/units.csv"), testDataDir, schema, null);
+
+        Iterator<Object[]> iter = table.iterator();
+        Object[] row = iter.next();
+        Assertions.assertEquals("°C", row[1]);
+        row = iter.next();
+        Assertions.assertEquals("μS/cm", row[1]);
+        row = iter.next();
+        Assertions.assertEquals("°F", row[1]);
+    }
 }

src/test/resources/fixtures/data/units.csv

@@ -0,0 +1,4 @@
+id,unit
+1,°C
+2,μS/cm
+2,°F

src/test/resources/fixtures/schema/units_schema.json

@@ -0,0 +1,19 @@
+{
+  "fields": [
+    {
+      "name": "id",
+      "format": "default",
+      "description": "primary key",
+      "type": "integer",
+      "title": "id"
+    },
+    {
+      "name": "unit",
+      "format": "default",
+      "description": "The unit.",
+      "type": "string",
+      "title": "The unit."
+    }
+  ],
+  "primaryKey": "id"
+}