Introducing Resource.RawData. Already updating README.md with section about reading raw data. #13

Author: danielfireman

README.md

@@ -8,12 +8,13 @@ A Go library for working with [Data Packages](http://specs.frictionlessdata.io/d
 - [datapackage-go](#datapackage-go)
     - [Install](#install)
     - [Main Features](#main-features)
-        - [Loading and validating data package descriptors](#loading-and-validating-data-package-descriptors)
+        - [Loading and validating tabular data package descriptors](#loading-and-validating-tabular-data-package-descriptors)
         - [Accessing data package resources](#accessing-data-package-resources)
         - [Loading zip bundles](#loading-zip-bundles)
         - [Creating a zip bundle with the data package.](#creating-a-zip-bundle-with-the-data-package)
         - [CSV dialect support](#csv-dialect-support)
         - [Loading multipart resources](#loading-multipart-resources)
+        - [Loading non-tabular resources](#loading-non-tabular-resources)
         - [Manipulating data packages programatically](#manipulating-data-packages-programatically)
 
 <!-- /TOC -->
@@ -26,7 +27,7 @@ $ go get -u github.com/frictionlessdata/datapackage-go/...
 
 ## Main Features
 
-### Loading and validating data package descriptors
+### Loading and validating tabular data package descriptors
 
 A [data package](http://frictionlessdata.io/specs/data-package/) is a collection of [resources](http://frictionlessdata.io/specs/data-resource/). The [datapackage.Package](https://godoc.org/github.com/frictionlessdata/datapackage-go/datapackage#Package) provides various capabilities like loading local or remote data package, saving a data package descriptor and many more.
 
@@ -192,6 +193,59 @@ And all the rest of the code would still be working.
 A complete example can be found [here](https://github.com/frictionlessdata/datapackage-go/tree/master/examples/multipart).
 
 
+### Loading non-tabular resources
+
+A [Data package](https://frictionlessdata.io/data-packages/) is a container format used to describe and package a collection of data. Even though there is additional support for dealing with tabular resources, it can be used to package any kind of data.
+
+For instance, lets say an user needs to load JSON-LD information along with some tabular data (for more on this use case, please take a look at [this](https://github.com/frictionlessdata/datapackage-go/issues/13) issue). That can be packed together in a data package descriptor:
+
+```json
+{
+    "name": "carp-lake",
+    "title": "Carp Lake Title",
+    "description": "Tephra and Lithology from Carp Lake",
+    "resources": [
+      {
+        "name":"data",
+        "path": "data/carpLakeCoreStratigraphy.csv",
+        "format": "csv",
+        "schema": {
+          "fields": [
+            {"name": "depth", "type": "number"},
+            {"name": "notes", "type": "text"},
+            {"name": "core_segments", "type": "text"}
+          ]
+        }
+      },
+      {
+        "name": "schemaorg",
+        "path": "data/schemaorg-ld.json",
+        "format": "application/ld+json"
+      }
+    ]
+}
+```
+
+The package loading proceeds as usual.
+
+
+```go
+pkg, err := datapackage.Load("data/datapackage.json")
+// Check error.
+```
+
+Once the data package is loaded, we could use the [Resource.RawRead](https://godoc.org/github.com/frictionlessdata/datapackage-go/datapackage#Resource.GetSchema) method to access `schemaorg` resource contents as a byte slice.
+
+```go
+so := pkg.GetResource("schemaorg")
+soContents, _ := so.RawRead()
+// Use contents. For instance, one could validate the JSON-LD schema and unmarshal it into a data structure.
+
+data := pkg.GetResource("data")
+dataContents, err := data.ReadAll()
+// As data is a tabular resource, its content can be loaded as [][]string.
+```
+
 ### Manipulating data packages programatically
 
 The datapackage-go library also makes it easy to save packages. Let's say you're creating a program that produces data packages and would like to add or remove resource:

datapackage/package_test.go

@@ -56,6 +56,28 @@ func ExampleLoad_readAll() {
 	// Output: [[foo] [bar]]
 }
 
+func ExampleLoad_readRaw() {
+	dir, _ := ioutil.TempDir("", "datapackage_exampleload")
+	defer os.Remove(dir)
+	descriptorPath := filepath.Join(dir, "pkg.json")
+	descriptorContents := `{"resources": [{ 
+		  "name": "res1",
+		  "path": "schemaorg.json",
+		  "format": "application/ld+json",
+		  "profile": "data-resource"
+		}]}`
+	ioutil.WriteFile(descriptorPath, []byte(descriptorContents), 0666)
+
+	resPath := filepath.Join(dir, "schemaorg.json")
+	resContent := []byte(`{"@context": {"@vocab": "http://schema.org/"}}`)
+	ioutil.WriteFile(resPath, resContent, 0666)
+
+	pkg, _ := Load(descriptorPath, validator.InMemoryLoader())
+	contents, _ := pkg.GetResource("res1").RawRead()
+	fmt.Println(string(contents))
+	// Output: {"@context": {"@vocab": "http://schema.org/"}}
+}
+
 func ExampleLoad_readAllRemote() {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		// If the request is for data, returns the content.

datapackage/resource.go

@@ -6,10 +6,14 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"net/http"
 	"net/url"
+	"os"
 	"path"
 	"path/filepath"
 	"strings"
+	"sync"
+	"time"
 
 	"github.com/frictionlessdata/datapackage-go/clone"
 	"github.com/frictionlessdata/datapackage-go/validator"
@@ -190,19 +194,67 @@ func (r *Resource) GetTable(opts ...csv.CreationOpts) (table.Table, error) {
 			return nil, fmt.Errorf("only csv and string is supported for inlining data")
 		}
 	}
-	// Paths: create a table from the concatenation of the data in all paths.
-	var buf bytes.Buffer
-	for _, p := range r.path {
-		if r.basePath != "" {
-			p = joinPaths(r.basePath, p)
+	buf, err := loadContents(r.basePath, r.path, csvLoadFunc)
+	if err != nil {
+		return nil, err
+	}
+	t, err := csv.NewTable(func() (io.ReadCloser, error) { return ioutil.NopCloser(bytes.NewReader(buf)), nil }, fullOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return t, nil
+}
+
+func csvLoadFunc(p string) func() (io.ReadCloser, error) {
+	if strings.HasPrefix(p, "http") {
+		return csv.Remote(p)
+	}
+	return csv.FromFile(p)
+}
+
+const (
+	remoteFetchTimeout = 15 * time.Second
+)
+
+var (
+	httpClient      *http.Client
+	startHTTPClient sync.Once
+)
+
+func binaryLoadFunc(p string) func() (io.ReadCloser, error) {
+	if strings.HasPrefix(p, "http") {
+		return func() (io.ReadCloser, error) {
+			startHTTPClient.Do(func() {
+				httpClient = &http.Client{
+					Timeout: remoteFetchTimeout,
+				}
+			})
+			resp, err := httpClient.Get(p)
+			if err != nil {
+				return nil, err
+			}
+			defer resp.Body.Close()
+			b, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				return nil, err
+			}
+			return ioutil.NopCloser(bytes.NewReader(b)), nil
 		}
-		var source csv.Source
-		if strings.HasPrefix(p, "http") {
-			source = csv.Remote(p)
-		} else {
-			source = csv.FromFile(p)
+	}
+	return func() (io.ReadCloser, error) {
+		return os.Open(p)
+	}
+}
+
+type loadFunc func(string) func() (io.ReadCloser, error)
+
+func loadContents(basePath string, path []string, f loadFunc) ([]byte, error) {
+	var buf bytes.Buffer
+	for _, p := range path {
+		if basePath != "" {
+			p = joinPaths(basePath, p)
 		}
-		rc, err := source()
+		rc, err := f(p)()
 		if err != nil {
 			return nil, err
 		}
@@ -212,13 +264,11 @@ func (r *Resource) GetTable(opts ...csv.CreationOpts) (table.Table, error) {
 			return nil, err
 		}
 		buf.Write(b)
-		buf.WriteRune('\n')
-	}
-	t, err := csv.NewTable(func() (io.ReadCloser, error) { return ioutil.NopCloser(strings.NewReader(buf.String())), nil }, fullOpts...)
-	if err != nil {
-		return nil, err
+		if len(path) > 1 {
+			buf.WriteRune('\n')
+		}
 	}
-	return t, nil
+	return buf.Bytes(), nil
 }
 
 func joinPaths(basePath, path string) string {
@@ -239,6 +289,15 @@ func (r *Resource) ReadAll(opts ...csv.CreationOpts) ([][]string, error) {
 	return t.ReadAll()
 }
 
+// RawRead reads all resource contents and return it as byte slice.
+// It can be used to access the content of non-tabular resources.
+func (r *Resource) RawRead() ([]byte, error) {
+	if r.data != nil {
+		return []byte(r.data.(string)), nil
+	}
+	return loadContents(r.basePath, r.path, binaryLoadFunc)
+}
+
 // Iter returns an Iterator to read the tabular resource. Iter returns an error
 // if the table physical source can not be iterated.
 // The iteration process always start at the beginning of the table.

datapackage/resource_test.go

@@ -418,3 +418,36 @@ func TestResource_Cast(t *testing.T) {
 		}
 	})
 }
+
+func TestResource_RawRead(t *testing.T) {
+	t.Run("Remote", func(t *testing.T) {
+		is := is.New(t)
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			fmt.Fprint(w, "1234")
+		}))
+		defer ts.Close()
+		resStr := fmt.Sprintf(`
+		{
+			"name":    "ids",
+			"path":    "%s/id1",
+			"profile": "data-resource"
+		}`, ts.URL)
+		res, err := NewResourceFromString(resStr, validator.MustInMemoryRegistry())
+		is.NoErr(err)
+		contents, err := res.RawRead()
+		is.NoErr(err)
+		is.Equal(string(contents), "1234")
+	})
+	t.Run("Inline", func(t *testing.T) {
+		is := is.New(t)
+		resStr := `
+			{
+				"name": "ids", "data": "{\"foo\":\"1234\"}", "profile":"data-resource", "mediatype":"application/json"
+			}`
+		res, err := NewResourceFromString(resStr, validator.MustInMemoryRegistry())
+		is.NoErr(err)
+		contents, err := res.RawRead()
+		is.NoErr(err)
+		is.Equal(string(contents), "{\"foo\":\"1234\"}")
+	})
+}