Skip to content

Commit b1ed2d3

Browse files
JohannesJanderJohannesJander
committed
Fixed bugs in setting profiles on Resources and Packages; created tests for that
Allow for reading resource raw data; created tests for that
1 parent 8fc919a commit b1ed2d3

File tree

12 files changed

+193
-48
lines changed

12 files changed

+193
-48
lines changed

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
<modelVersion>4.0.0</modelVersion>
55
<groupId>io.frictionlessdata</groupId>
66
<artifactId>datapackage-java</artifactId>
7-
<version>0.6.9-SNAPSHOT</version>
7+
<version>0.6.10-SNAPSHOT</version>
88
<packaging>jar</packaging>
99
<issueManagement>
1010
<url>https://github.com/frictionlessdata/datapackage-java/issues</url>

‎src/main/java/io/frictionlessdata/datapackage/Package.java‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,7 @@ public Package(URL urlSource, boolean strict) throws Exception {
185185
*
186186
* @param descriptorFile local file path that points to the DataPackage Descriptor (if it's in
187187
* a local directory) or the ZIP file if it's a ZIP-based
188-
* package.
188+
* package or to the parent directory
189189
* @param strict whether to use strict schema parsing
190190
* @throws DataPackageFileOrUrlNotFoundException if the path is invalid
191191
* @throws IOException thrown if I/O operations fail
@@ -197,14 +197,21 @@ public Package(Path descriptorFile, boolean strict) throws Exception {
197197
if (!descriptorFile.toFile().exists()) {
198198
throw new DataPackageFileOrUrlNotFoundException("File " + descriptorFile + "does not exist");
199199
}
200-
if (isArchive(descriptorFile.toFile())) {
201-
isArchivePackage = true;
200+
if (descriptorFile.toFile().isDirectory()) {
202201
basePath = descriptorFile;
203-
sourceJsonNode = createNode(JSONBase.getZipFileContentAsString(descriptorFile, DATAPACKAGE_FILENAME));
204-
} else {
205-
basePath = descriptorFile.getParent();
206-
String sourceJsonString = getFileContentAsString(descriptorFile);
202+
File realDescriptor = new File(descriptorFile.toFile(), DATAPACKAGE_FILENAME);
203+
String sourceJsonString = getFileContentAsString(realDescriptor.toPath());
207204
sourceJsonNode = createNode(sourceJsonString);
205+
} else {
206+
if (isArchive(descriptorFile.toFile())) {
207+
isArchivePackage = true;
208+
basePath = descriptorFile;
209+
sourceJsonNode = createNode(JSONBase.getZipFileContentAsString(descriptorFile, DATAPACKAGE_FILENAME));
210+
} else {
211+
basePath = descriptorFile.getParent();
212+
String sourceJsonString = getFileContentAsString(descriptorFile);
213+
sourceJsonNode = createNode(sourceJsonString);
214+
}
208215
}
209216
this.setJson((ObjectNode) sourceJsonNode);
210217
}
@@ -854,7 +861,7 @@ private void addResource(AbstractDataResource resource, boolean validate)
854861
throws IOException, ValidationException, DataPackageException{
855862
DataPackageException dpe = null;
856863
// If a name property isn't given...
857-
if ((resource.getDataProperty() == null) || (resource).getFormat() == null) {
864+
if ((resource.getRawData() == null) || (resource).getFormat() == null) {
858865
dpe = new DataPackageValidationException("Invalid Resource. The data and format properties cannot be null.");
859866
} else {
860867
dpe = checkDuplicates(resource);

‎src/main/java/io/frictionlessdata/datapackage/resource/AbstractDataResource.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import io.frictionlessdata.datapackage.exceptions.DataPackageException;
66
import io.frictionlessdata.tableschema.Table;
77

8+
import java.io.IOException;
89
import java.nio.file.Path;
910
import java.util.ArrayList;
1011
import java.util.HashSet;
@@ -33,7 +34,8 @@ public abstract class AbstractDataResource<T,C> extends AbstractResource<T,C> {
3334
* @return the data
3435
*/
3536
@JsonIgnore
36-
public T getDataProperty() {
37+
@Override
38+
public Object getRawData() throws IOException {
3739
return data;
3840
}
3941

‎src/main/java/io/frictionlessdata/datapackage/resource/AbstractReferencebasedResource.java‎

Lines changed: 38 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,11 @@
66
import io.frictionlessdata.tableschema.tabledatasource.TableDataSource;
77
import io.frictionlessdata.tableschema.util.JsonUtil;
88

9-
import java.util.ArrayList;
10-
import java.util.Collection;
11-
import java.util.List;
12-
import java.util.Set;
9+
import java.io.ByteArrayOutputStream;
10+
import java.io.File;
11+
import java.io.IOException;
12+
import java.io.InputStream;
13+
import java.util.*;
1314
import java.util.stream.Collectors;
1415

1516
public abstract class AbstractReferencebasedResource<T,C> extends AbstractResource<T,C> {
@@ -29,6 +30,27 @@ public Collection<String> getReferencesAsStrings() {
2930
return strings;
3031
}
3132

33+
@JsonIgnore
34+
@Override
35+
public Object getRawData() throws IOException {
36+
// If the path(s) of data file/URLs has been set.
37+
if (paths != null){
38+
Iterator<T> iter = paths.iterator();
39+
if (paths.size() == 1) {
40+
return getRawData(iter.next());
41+
} else {
42+
// this is probably not very useful, but it's the spec...
43+
byte[][] retVal = new byte[paths.size()][];
44+
for (int i = 0; i < paths.size(); i++) {
45+
byte[] bytes = getRawData(iter.next());
46+
retVal[i] = bytes;
47+
}
48+
return retVal;
49+
}
50+
}
51+
return null;
52+
}
53+
3254
/*
3355
if more than one path in our paths object, return a JSON array,
3456
else just that one object.
@@ -68,4 +90,16 @@ public Set<String> getDatafileNamesForWriting() {
6890

6991
abstract String getStringRepresentation(T reference);
7092

93+
abstract byte[] getRawData(T input) throws IOException;
94+
byte[] getRawData(InputStream inputStream) throws IOException {
95+
byte[] retVal = null;
96+
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
97+
for (int b; (b = inputStream.read()) != -1; ) {
98+
out.write(b);
99+
}
100+
retVal = out.toByteArray();
101+
}
102+
return retVal;
103+
}
104+
71105
}

‎src/main/java/io/frictionlessdata/datapackage/resource/AbstractResource.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -298,14 +298,18 @@ public String getJson(){
298298
}
299299
json.set(JSON_KEY_DATA, data);
300300
} catch (Exception ex) {
301-
throw new RuntimeException(ex);
301+
throw new DataPackageException(ex);
302302
}
303303
}
304304
} else if ((this instanceof AbstractDataResource)) {
305305
if (this.shouldSerializeToFile()) {
306306
//TODO implement storing only the path - and where to get it
307307
} else {
308-
json.set(JSON_KEY_DATA, JsonUtil.getInstance().createNode(((AbstractDataResource) this).getDataProperty()));
308+
try {
309+
json.set(JSON_KEY_DATA, JsonUtil.getInstance().createNode(this.getRawData()));
310+
} catch (IOException e) {
311+
throw new DataPackageException(e);
312+
}
309313
}
310314
}
311315

‎src/main/java/io/frictionlessdata/datapackage/resource/FilebasedResource.java‎

Lines changed: 12 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,16 @@
33
import com.fasterxml.jackson.annotation.JsonIgnore;
44
import com.fasterxml.jackson.annotation.JsonInclude;
55
import com.fasterxml.jackson.annotation.JsonInclude.Include;
6+
import com.google.common.io.ByteStreams;
67
import io.frictionlessdata.datapackage.exceptions.DataPackageException;
78
import io.frictionlessdata.datapackage.exceptions.DataPackageValidationException;
89
import io.frictionlessdata.tableschema.Table;
910
import io.frictionlessdata.tableschema.tabledatasource.TableDataSource;
1011

11-
import java.io.File;
12+
import java.io.*;
13+
import java.net.URL;
1214
import java.nio.charset.Charset;
15+
import java.nio.file.Files;
1316
import java.nio.file.Path;
1417
import java.util.*;
1518

@@ -91,6 +94,14 @@ public File getBasePath() {
9194
return basePath;
9295
}
9396

97+
@Override
98+
byte[] getRawData(File input) throws IOException {
99+
File f = new File(this.basePath, input.getPath());
100+
try (InputStream inputStream = Files.newInputStream(f.toPath())) {
101+
return getRawData(inputStream);
102+
}
103+
}
104+
94105
@Override
95106
Table createTable(File reference) throws Exception {
96107
return Table.fromSource(reference, basePath, schema, getCsvFormat());
@@ -140,33 +151,7 @@ private List<Table> readfromOrdinaryFile() throws Exception {
140151
}
141152
return tables;
142153
}
143-
/*
144-
@Override
145-
public void writeDataAsCsv(Path outputDir, Dialect dialect) throws Exception {
146-
Dialect lDialect = (null != dialect) ? dialect : Dialect.DEFAULT;
147-
List<String> paths = new ArrayList<>(getReferencesAsStrings());
148-
int cnt = 0;
149-
for (String fileName : paths) {
150-
List<Table> tables = getTables();
151-
Table t = tables.get(cnt++);
152-
Path p;
153-
if (outputDir.toString().isEmpty()) {
154-
p = outputDir.getFileSystem().getPath(fileName);
155-
if (!Files.exists(p)) {
156-
Files.createDirectories(p);
157-
}
158-
} else {
159-
if (!Files.exists(outputDir)) {
160-
Files.createDirectories(outputDir);
161-
}
162-
p = outputDir.resolve(fileName);
163-
}
164154

165-
Files.deleteIfExists(p);
166-
writeTableAsCsv(t, lDialect, p);
167-
}
168-
}
169-
*/
170155
public void setIsInArchive(boolean isInArchive) {
171156
this.isInArchive = isInArchive;
172157
}

‎src/main/java/io/frictionlessdata/datapackage/resource/Resource.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,15 @@ public interface Resource<T,C> {
5050

5151
String getJson();
5252

53+
/**
54+
* Read all data from a Resource, unmapped and not transformed. This is useful for non-tabular resources
55+
*
56+
* @return Contents of the resource file or URL.
57+
* @throws IOException if reading the data fails
58+
*
59+
*/
60+
public Object getRawData() throws IOException;
61+
5362
/**
5463
* Read all data from a Resource, each row as String arrays. This can be used for smaller datapackages,
5564
* but for huge or unknown sizes, reading via iterator is preferred, as this method loads all data into RAM.

‎src/main/java/io/frictionlessdata/datapackage/resource/URLbasedResource.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,18 @@
11
package io.frictionlessdata.datapackage.resource;
22

3+
import com.fasterxml.jackson.annotation.JsonIgnore;
4+
import com.google.common.io.ByteStreams;
35
import io.frictionlessdata.tableschema.Table;
46

7+
import java.io.ByteArrayOutputStream;
8+
import java.io.File;
9+
import java.io.IOException;
10+
import java.io.InputStream;
511
import java.net.URL;
12+
import java.nio.file.Files;
613
import java.util.ArrayList;
714
import java.util.Collection;
15+
import java.util.Iterator;
816
import java.util.List;
917

1018
public class URLbasedResource<C> extends AbstractReferencebasedResource<URL, C> {
@@ -19,6 +27,13 @@ Table createTable(URL reference) throws Exception {
1927
return Table.fromSource(reference, schema, getCsvFormat());
2028
}
2129

30+
@Override
31+
byte[] getRawData(URL input) throws IOException {
32+
try (InputStream inputStream = input.openStream()) {
33+
return getRawData(inputStream);
34+
}
35+
}
36+
2237
@Override
2338
String getStringRepresentation(URL reference) {
2439
return reference.toExternalForm();

‎src/test/java/io/frictionlessdata/datapackage/PackageTest.java‎

Lines changed: 35 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -370,6 +370,40 @@ public void testAddValidResource() throws Exception{
370370
Assertions.assertNotNull(gotResource);
371371
}
372372

373+
@Test
374+
@DisplayName("Test getting resource data from a non-tabular datapackage, file based")
375+
public void testNonTabularPackage() throws Exception{
376+
String pathName = "/fixtures/datapackages/non-tabular";
377+
Path resourcePath = TestUtil.getResourcePath(pathName);
378+
Package dp = new Package(resourcePath, true);
379+
380+
Resource<?,?> resource = dp.getResource("logo-svg");
381+
Assertions.assertTrue(resource instanceof FilebasedResource);
382+
Object rawData = resource.getRawData();
383+
384+
byte[] testData = TestUtil.getResourceContent("/fixtures/files/frictionless-color-full-logo.svg");
385+
Assertions.assertArrayEquals(testData, (byte[])rawData);
386+
}
387+
388+
/*
389+
@Test
390+
@DisplayName("Test getting resource data from a non-tabular datapackage, URL based")
391+
public void testNonTabularPackageUrl() throws Exception{
392+
URL input = new URL("https://raw.githubusercontent.com/frictionlessdata/datapackage-java" +
393+
"/master/src/test/resources//fixtures/datapackages/non-tabular/datapackage.json");
394+
395+
Package dp = new Package(input, true);
396+
397+
Resource<?,?> resource = dp.getResource("logo-svg");
398+
Assertions.assertTrue(resource instanceof FilebasedResource);
399+
Object rawData = resource.getRawData();
400+
401+
byte[] testData = TestUtil.getResourceContent("/fixtures/files/frictionless-color-full-logo.svg");
402+
Assertions.assertArrayEquals(testData, (byte[])rawData);
403+
}
404+
405+
*/
406+
373407

374408
@Test
375409
@DisplayName("Test setting the 'profile' property")
@@ -816,11 +850,7 @@ private Package getDataPackageFromFilePath(boolean strict) throws Exception {
816850

817851
private static String getFileContents(String fileName) {
818852
try {
819-
// Create file-URL of source file:
820-
URL sourceFileUrl = PackageTest.class.getResource(fileName);
821-
// Get path of URL
822-
Path path = Paths.get(sourceFileUrl.toURI());
823-
return new String(Files.readAllBytes(path));
853+
return new String(TestUtil.getResourceContent(fileName));
824854
} catch (Exception ex) {
825855
throw new RuntimeException(ex);
826856
}

‎src/test/java/io/frictionlessdata/datapackage/TestUtil.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,11 @@
11
package io.frictionlessdata.datapackage;
22

3+
import java.io.ByteArrayOutputStream;
34
import java.io.File;
5+
import java.io.IOException;
6+
import java.io.InputStream;
47
import java.net.URL;
8+
import java.nio.file.Files;
59
import java.nio.file.Path;
610
import java.nio.file.Paths;
711

@@ -37,4 +41,22 @@ public static Path getResourcePath (String fileName) {
3741
throw new RuntimeException(ex);
3842
}
3943
}
44+
45+
public static byte[] getResourceContent (String fileName) throws IOException {
46+
String locFileName = fileName;
47+
if (fileName.startsWith("/")){
48+
locFileName = fileName.substring(1);
49+
}
50+
51+
try (InputStream inputStream = Package.class.getClassLoader().getResourceAsStream(locFileName);
52+
ByteArrayOutputStream out = new ByteArrayOutputStream()) {
53+
for (int b; (b = inputStream.read()) != -1; ) {
54+
out.write(b);
55+
}
56+
return out.toByteArray();
57+
} catch (IOException e) {
58+
throw new RuntimeException(e);
59+
}
60+
}
61+
4062
}

0 commit comments

Comments
 (0)