Buf fix for data resources that should be serialized to file

Author: JohannesJander

pom.xml

@@ -4,7 +4,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>io.frictionlessdata</groupId>
     <artifactId>datapackage-java</artifactId>
-    <version>0.9.1-SNAPSHOT</version>
+    <version>0.9.2-SNAPSHOT</version>
     <packaging>jar</packaging>
     <issueManagement>
         <url>https://github.com/frictionlessdata/datapackage-java/issues</url>

src/main/java/io/frictionlessdata/datapackage/Package.java

@@ -606,17 +606,17 @@ public void write (File outputDir, Consumer<Path> callback, boolean zipCompresse
         for (Resource r : resourceList) {
             r.writeData(outFs.getPath(parentDirName ));
             r.writeSchema(outFs.getPath(parentDirName));
+            r.writeDialect(outFs.getPath(parentDirName));
 
             // write out dialect file only if not null or URL
-            String dialectP = r.getPathForWritingDialect();
-            if (null != dialectP) {
-                Path dialectPath = outFs.getPath(parentDirName + File.separator + dialectP);
-                r.getDialect().writeDialect(dialectPath);
-            }
-            Dialect dia = r.getDialect();
+
+            /*Dialect dia = r.getDialect();
             if (null != dia) {
+                String dialectP = r.getPathForWritingDialect();
+                Path dialectPath = outFs.getPath(parentDirName + File.separator + dialectP);
+                dia.writeDialect(dialectPath);
                 dia.setReference(new LocalFileReference(new File(dialectP)));
-            }
+            }*/
         }
         writeDescriptor(outFs, parentDirName);
 
@@ -750,6 +750,8 @@ private ObjectNode getJsonNode(){
                 } else {
                     obj.set(JSON_KEY_PATH, JsonUtil.getInstance().createArrayNode(outPaths));
                 }
+                // If a data resource should be saved to file, it should not be inlined as well
+                obj.remove(JSON_KEY_DATA);
                 obj.put(JSON_KEY_FORMAT, resource.getSerializationFormat());
             }
             obj.remove("originalReferences");

src/main/java/io/frictionlessdata/datapackage/resource/AbstractDataResource.java

@@ -34,9 +34,7 @@ public abstract class AbstractDataResource<T> extends AbstractResource<T> {
             throw new DataPackageException("Invalid Resource. The data property cannot be null for a Data-based Resource.");
     }
 
-    /**
-     * @return the data
-     */
+
     @Override
     @JsonProperty(JSON_KEY_DATA)
     public Object getRawData() throws IOException {

src/main/java/io/frictionlessdata/datapackage/resource/AbstractResource.java

@@ -91,6 +91,17 @@ public Object getSchemaForJson() {
         return null;
     }
 
+    @JsonProperty(JSON_KEY_DIALECT)
+    public Object getDialectForJson() {
+        if (originalReferences.containsKey(JSON_KEY_DIALECT)) {
+            return originalReferences.get(JSON_KEY_DIALECT);
+        }
+        if (null != dialect) {
+            return dialect;
+        }
+        return null;
+    }
+
     @Override
     public Iterator<Object[]> objectArrayIterator() throws Exception{
         return this.objectArrayIterator(false, false);
@@ -484,6 +495,33 @@ private static void writeSchema(Path parentFilePath, Schema schema) throws IOExc
         }
     }
 
+    public void writeDialect(Path parentFilePath) throws IOException {
+        if (null == dialect)
+            return;
+        String relPath = getPathForWritingDialect();
+        if (null == originalReferences.get(JSONBase.JSON_KEY_DIALECT) && Objects.nonNull(relPath)) {
+            originalReferences.put(JSONBase.JSON_KEY_DIALECT, relPath);
+        }
+
+        if (null != relPath) {
+            boolean isRemote;
+            try {
+                // don't try to write schema files that came from remote, let's just add the URL to the descriptor
+                URI uri = new URI(relPath);
+                isRemote = (null != uri.getScheme()) &&
+                        (uri.getScheme().equals("http") || uri.getScheme().equals("https"));
+                if (isRemote)
+                    return;
+            } catch (URISyntaxException ignored) {}
+            Path p;
+            if (parentFilePath.toString().isEmpty()) {
+                p = parentFilePath.getFileSystem().getPath(relPath);
+            } else {
+                p = parentFilePath.resolve(relPath);
+            }
+            writeDialect(p, dialect);
+        }
+    }
 
     private static void writeDialect(Path parentFilePath, Dialect dialect) throws IOException {
         if (!Files.exists(parentFilePath)) {
@@ -583,6 +621,7 @@ public String getProfile() {
      * @return the dialect
      */
     @Override
+    @JsonIgnore
     public Dialect getDialect() {
         return dialect;
     }

src/main/java/io/frictionlessdata/datapackage/resource/Resource.java

@@ -209,6 +209,15 @@ public interface Resource<T> extends BaseInterface {
      */
     void writeSchema(Path parentFilePath) throws IOException;
 
+    /**
+     * Write the Resource {@link Dialect} to `outputDir`.
+     *
+     * @param parentFilePath the directory to write to. Code must create
+     *                  files as needed.
+     * @throws IOException if something fails while writing
+     */
+    void writeDialect(Path parentFilePath) throws IOException;
+
     /**
      * Returns an Iterator that returns rows as object-arrays. Values in each column
      * are parsed and converted ("cast") to Java objects based on the Field definitions of the Schema.

src/test/java/io/frictionlessdata/datapackage/resource/NonTabularResourceTest.java

@@ -254,6 +254,10 @@ public void writeData(Writer writer) {
         public void writeSchema(Path path) {
         }
 
+        @Override
+        public void writeDialect(Path parentFilePath) throws IOException {
+        }
+
         @Override
         public Iterator<Object[]> objectArrayIterator() {
             throw new UnsupportedOperationException("Not supported on non-tabular Resources");

src/test/java/io/frictionlessdata/datapackage/resource/RoundtripTest.java

@@ -18,31 +18,15 @@
 import java.util.List;
 import java.util.stream.Collectors;
 
+import static io.frictionlessdata.datapackage.Profile.PROFILE_TABULAR_DATA_RESOURCE;
+
 /**
  * Ensure datapackages are written in a valid format and can be read back. Compare data to see it matches
  */
 public class RoundtripTest {
     private static final CSVFormat csvFormat = TableDataSource
             .getDefaultCsvFormat().builder().setDelimiter('\t').get();
 
-    private static final String resourceContent = "[\n" +
-            "    {\n" +
-            "\t  \"city\": \"london\",\n" +
-            "\t  \"year\": 2017,\n" +
-            "\t  \"population\": 8780000\n" +
-            "\t},\n" +
-            "\t{\n" +
-            "\t  \"city\": \"paris\",\n" +
-            "\t  \"year\": 2017,\n" +
-            "\t  \"population\": 2240000\n" +
-            "\t},\n" +
-            "\t{\n" +
-            "\t  \"city\": \"rome\",\n" +
-            "\t  \"year\": 2017,\n" +
-            "\t  \"population\": 2860000\n" +
-            "\t}\n" +
-            "  ]";
-
     @Test
     @DisplayName("Roundtrip test - write datapackage, read again and compare data")
     // this lead to the discovery of https://github.com/frictionlessdata/specs/issues/666, just in case
@@ -87,16 +71,73 @@ public void dogfoodingTest() throws Exception {
     void validateResourceRoundtrip() throws Exception {
         Path resourcePath = TestUtil.getResourcePath("/fixtures/datapackages/roundtrip/datapackage.json");
         Package dp = new Package(resourcePath, true);
-        StringBuffer buf = new StringBuffer();
-        Resource v4Resource = dp.getResource("test2");
-        List<Object[]> testData = v4Resource.getData(false, false, true, false);
-        String data = createCSV(v4Resource.getHeaders(), testData);
-        Resource v5Resource = new CSVDataResource(v4Resource.getName(),data);
-        v5Resource.setDescription(v4Resource.getDescription());
-        v5Resource.setSchema(v4Resource.getSchema());
-        v5Resource.setSerializationFormat(Resource.FORMAT_CSV);
-        List data1 = v5Resource.getData(false, false, true, false);
-        Assertions.assertArrayEquals(testData.toArray(), data1.toArray());
+        Resource referenceResource = dp.getResource("test2");
+        List<Object[]> referenceData = referenceResource.getData(false, false, true, false);
+        String data = createCSV(referenceResource.getHeaders(), referenceData);
+        Resource newResource = new CSVDataResource(referenceResource.getName(),data);
+        newResource.setDescription(referenceResource.getDescription());
+        newResource.setSchema(referenceResource.getSchema());
+        newResource.setSerializationFormat(Resource.FORMAT_CSV);
+        List testData = newResource.getData(false, false, true, false);
+        Assertions.assertArrayEquals(referenceData.toArray(), testData.toArray());
+    }
+
+    @Test
+    @DisplayName("Create data resource, compare descriptor")
+    void validateCreateResourceDescriptorRoundtrip() throws Exception {
+        Path resourcePath = TestUtil.getResourcePath("/fixtures/datapackages/roundtrip/datapackage.json");
+        Package pkg = new Package(resourcePath, true);
+        JSONDataResource resource = new JSONDataResource("test3", resourceContent);
+        resource.setSchema(Schema.fromJson(
+                new File(TestUtil.getBasePath().toFile(), "/schema/population_schema.json"), true));
+
+        resource.setShouldSerializeToFile(false);
+        resource.setSerializationFormat(Resource.FORMAT_CSV);
+        resource.setDialect(Dialect.fromCsvFormat(csvFormat));
+        resource.setProfile(PROFILE_TABULAR_DATA_RESOURCE);
+        resource.setEncoding("utf-8");
+        pkg.addResource(resource);
+        Path tempDirPath = Files.createTempDirectory("datapackage-");
+        File createdFile = new File(tempDirPath.toFile(), "test_save_datapackage.zip");
+        pkg.write(createdFile, true);
+        System.out.println(tempDirPath);
+
+        // create new Package from the serialized form and check they are equal
+        Package testPkg = new Package(createdFile.toPath(), true);
+        String json = testPkg.asJson();
+        Assertions.assertEquals(
+                descriptorContentInlined.replaceAll("[\n\r]+", "\n"),
+                json.replaceAll("[\n\r]+", "\n")
+        );
+    }
+
+
+    @Test
+    @DisplayName("Create data resource and make it write to file, compare descriptor")
+    void validateCreateResourceDescriptorRoundtrip2() throws Exception {
+        Path resourcePath = TestUtil.getResourcePath("/fixtures/datapackages/roundtrip/datapackage.json");
+        Package pkg = new Package(resourcePath, true);
+        JSONDataResource resource = new JSONDataResource("test3", resourceContent);
+        resource.setSchema(Schema.fromJson(
+                new File(TestUtil.getBasePath().toFile(), "/schema/population_schema.json"), true));
+
+        resource.setShouldSerializeToFile(true);
+        resource.setSerializationFormat(Resource.FORMAT_CSV);
+        resource.setDialect(Dialect.fromCsvFormat(csvFormat));
+        resource.setProfile(PROFILE_TABULAR_DATA_RESOURCE);
+        resource.setEncoding("utf-8");
+        pkg.addResource(resource);
+        Path tempDirPath = Files.createTempDirectory("datapackage-");
+        File createdFile = new File(tempDirPath.toFile(), "test_save_datapackage.zip");
+        pkg.write(createdFile, true);
+
+        // create new Package from the serialized form and check they are equal
+        Package testPkg = new Package(createdFile.toPath(), true);
+        String json = testPkg.asJson();
+        Assertions.assertEquals(
+                descriptorContent.replaceAll("[\n\r]+", "\n"),
+                json.replaceAll("[\n\r]+", "\n")
+        );
     }
 
     private static String createCSV(String[] headers, List<Object[]> data) {
@@ -111,4 +152,103 @@ private static String createCSV(String[] headers, List<Object[]> data) {
         }
         return sb.toString();
     }
+
+
+    private static final String resourceContent = "[\n" +
+            "    {\n" +
+            "\t  \"city\": \"london\",\n" +
+            "\t  \"year\": 2017,\n" +
+            "\t  \"population\": 8780000\n" +
+            "\t},\n" +
+            "\t{\n" +
+            "\t  \"city\": \"paris\",\n" +
+            "\t  \"year\": 2017,\n" +
+            "\t  \"population\": 2240000\n" +
+            "\t},\n" +
+            "\t{\n" +
+            "\t  \"city\": \"rome\",\n" +
+            "\t  \"year\": 2017,\n" +
+            "\t  \"population\": 2860000\n" +
+            "\t}\n" +
+            "  ]";
+
+    private static final String descriptorContent = "{\n" +
+            "  \"name\" : \"foreign-keys\",\n" +
+            "  \"profile\" : \"data-package\",\n" +
+            "  \"resources\" : [ {\n" +
+            "    \"name\" : \"test2\",\n" +
+            "    \"profile\" : \"data-resource\",\n" +
+            "    \"schema\" : \"schema/test2.json\",\n" +
+            "    \"path\" : \"data/test2.csv\"\n" +
+            "  }, {\n" +
+            "    \"name\" : \"test3\",\n" +
+            "    \"profile\" : \"tabular-data-resource\",\n" +
+            "    \"encoding\" : \"utf-8\",\n" +
+            "    \"dialect\" : \"dialect/test3.json\",\n" +
+            "    \"schema\" : \"schema/population_schema.json\",\n" +
+            "    \"path\" : \"data/test3.csv\"\n" +
+            "  } ]\n" +
+            "}";
+
+
+    private static String descriptorContentInlined ="{\n" +
+            "  \"name\" : \"foreign-keys\",\n" +
+            "  \"profile\" : \"data-package\",\n" +
+            "  \"resources\" : [ {\n" +
+            "    \"name\" : \"test2\",\n" +
+            "    \"profile\" : \"data-resource\",\n" +
+            "    \"schema\" : \"schema/test2.json\",\n" +
+            "    \"path\" : \"data/test2.csv\"\n" +
+            "  }, {\n" +
+            "    \"name\" : \"test3\",\n" +
+            "    \"profile\" : \"tabular-data-resource\",\n" +
+            "    \"encoding\" : \"utf-8\",\n" +
+            "    \"format\" : \"json\",\n" +
+            "    \"dialect\" : {\n" +
+            "      \"caseSensitiveHeader\" : false,\n" +
+            "      \"quoteChar\" : \"\\\"\",\n" +
+            "      \"doubleQuote\" : true,\n" +
+            "      \"delimiter\" : \"\\t\",\n" +
+            "      \"lineTerminator\" : \"\\r\\n\",\n" +
+            "      \"nullSequence\" : \"\",\n" +
+            "      \"header\" : true,\n" +
+            "      \"csvddfVersion\" : 1.2,\n" +
+            "      \"skipInitialSpace\" : true\n" +
+            "    },\n" +
+            "    \"schema\" : {\n" +
+            "      \"fields\" : [ {\n" +
+            "        \"name\" : \"city\",\n" +
+            "        \"title\" : \"city\",\n" +
+            "        \"type\" : \"string\",\n" +
+            "        \"format\" : \"default\",\n" +
+            "        \"description\" : \"The city.\"\n" +
+            "      }, {\n" +
+            "        \"name\" : \"year\",\n" +
+            "        \"title\" : \"year\",\n" +
+            "        \"type\" : \"year\",\n" +
+            "        \"format\" : \"default\",\n" +
+            "        \"description\" : \"The year.\"\n" +
+            "      }, {\n" +
+            "        \"name\" : \"population\",\n" +
+            "        \"title\" : \"population\",\n" +
+            "        \"type\" : \"integer\",\n" +
+            "        \"format\" : \"default\",\n" +
+            "        \"description\" : \"The population.\"\n" +
+            "      } ]\n" +
+            "    },\n" +
+            "    \"data\" : [ {\n" +
+            "      \"city\" : \"london\",\n" +
+            "      \"year\" : 2017,\n" +
+            "      \"population\" : 8780000\n" +
+            "    }, {\n" +
+            "      \"city\" : \"paris\",\n" +
+            "      \"year\" : 2017,\n" +
+            "      \"population\" : 2240000\n" +
+            "    }, {\n" +
+            "      \"city\" : \"rome\",\n" +
+            "      \"year\" : 2017,\n" +
+            "      \"population\" : 2860000\n" +
+            "    } ]\n" +
+            "  } ]\n" +
+            "}";
 }