Final API implementation of RawRead returning an io.ReadCloser (file like) object. #13

Author: danielfireman

README.md

@@ -238,7 +238,9 @@ Once the data package is loaded, we could use the [Resource.RawRead](https://god
 
 ```go
 so := pkg.GetResource("schemaorg")
-soContents, _ := so.RawRead()
+rc, _ := so.RawRead()
+defer rc.Close()
+contents, _ := ioutil.ReadAll(rc)
 // Use contents. For instance, one could validate the JSON-LD schema and unmarshal it into a data structure.
 
 data := pkg.GetResource("data")

datapackage/package_test.go

@@ -73,7 +73,9 @@ func ExampleLoad_readRaw() {
 	ioutil.WriteFile(resPath, resContent, 0666)
 
 	pkg, _ := Load(descriptorPath, validator.InMemoryLoader())
-	contents, _ := pkg.GetResource("res1").RawRead()
+	rc, _ := pkg.GetResource("res1").RawRead()
+	defer rc.Close()
+	contents, _ := ioutil.ReadAll(rc)
 	fmt.Println(string(contents))
 	// Output: {"@context": {"@vocab": "http://schema.org/"}}
 }

datapackage/resource.go

@@ -194,15 +194,7 @@ func (r *Resource) GetTable(opts ...csv.CreationOpts) (table.Table, error) {
 			return nil, fmt.Errorf("only csv and string is supported for inlining data")
 		}
 	}
-	buf, err := loadContents(r.basePath, r.path, csvLoadFunc)
-	if err != nil {
-		return nil, err
-	}
-	t, err := csv.NewTable(func() (io.ReadCloser, error) { return ioutil.NopCloser(bytes.NewReader(buf)), nil }, fullOpts...)
-	if err != nil {
-		return nil, err
-	}
-	return t, nil
+	return csv.NewTable(func() (io.ReadCloser, error) { return loadContents(r.basePath, r.path, csvLoadFunc) }, fullOpts...)
 }
 
 func csvLoadFunc(p string) func() (io.ReadCloser, error) {
@@ -233,23 +225,39 @@ func binaryLoadFunc(p string) func() (io.ReadCloser, error) {
 			if err != nil {
 				return nil, err
 			}
-			defer resp.Body.Close()
-			b, err := ioutil.ReadAll(resp.Body)
-			if err != nil {
-				return nil, err
-			}
-			return ioutil.NopCloser(bytes.NewReader(b)), nil
+			return resp.Body, nil
 		}
 	}
 	return func() (io.ReadCloser, error) {
 		return os.Open(p)
 	}
 }
 
-type loadFunc func(string) func() (io.ReadCloser, error)
+type multiReadCloser struct {
+	io.Reader
+	rcs []io.ReadCloser
+}
+
+func (m *multiReadCloser) Close() error {
+	var err error
+	for _, rc := range m.rcs {
+		if e := rc.Close(); e != nil {
+			err = e
+		}
+	}
+	return err
+}
+
+func newMultiReadCloser(rcs []io.ReadCloser) io.ReadCloser {
+	readers := make([]io.Reader, len(rcs))
+	for i := range rcs {
+		readers[i] = io.Reader(rcs[i])
+	}
+	return &multiReadCloser{io.MultiReader(readers...), rcs}
+}
 
-func loadContents(basePath string, path []string, f loadFunc) ([]byte, error) {
-	var buf bytes.Buffer
+func loadContents(basePath string, path []string, f func(string) func() (io.ReadCloser, error)) (io.ReadCloser, error) {
+	var rcs []io.ReadCloser
 	for _, p := range path {
 		if basePath != "" {
 			p = joinPaths(basePath, p)
@@ -258,17 +266,12 @@ func loadContents(basePath string, path []string, f loadFunc) ([]byte, error) {
 		if err != nil {
 			return nil, err
 		}
-		defer rc.Close()
-		b, err := ioutil.ReadAll(rc)
-		if err != nil {
-			return nil, err
-		}
-		buf.Write(b)
+		rcs = append(rcs, rc)
 		if len(path) > 1 {
-			buf.WriteRune('\n')
+			rcs = append(rcs, ioutil.NopCloser(bytes.NewReader([]byte{'\n'})))
 		}
 	}
-	return buf.Bytes(), nil
+	return newMultiReadCloser(rcs), nil
 }
 
 func joinPaths(basePath, path string) string {
@@ -289,11 +292,11 @@ func (r *Resource) ReadAll(opts ...csv.CreationOpts) ([][]string, error) {
 	return t.ReadAll()
 }
 
-// RawRead reads all resource contents and return it as byte slice.
+// RawRead returns an io.ReaderCloser associated to the resource contents.
 // It can be used to access the content of non-tabular resources.
-func (r *Resource) RawRead() ([]byte, error) {
+func (r *Resource) RawRead() (io.ReadCloser, error) {
 	if r.data != nil {
-		return []byte(r.data.(string)), nil
+		return ioutil.NopCloser(bytes.NewReader([]byte(r.data.(string)))), nil
 	}
 	return loadContents(r.basePath, r.path, binaryLoadFunc)
 }

datapackage/resource_test.go

@@ -434,7 +434,10 @@ func TestResource_RawRead(t *testing.T) {
 		}`, ts.URL)
 		res, err := NewResourceFromString(resStr, validator.MustInMemoryRegistry())
 		is.NoErr(err)
-		contents, err := res.RawRead()
+		rc, err := res.RawRead()
+		is.NoErr(err)
+		defer rc.Close()
+		contents, err := ioutil.ReadAll(rc)
 		is.NoErr(err)
 		is.Equal(string(contents), "1234")
 	})
@@ -446,7 +449,10 @@ func TestResource_RawRead(t *testing.T) {
 			}`
 		res, err := NewResourceFromString(resStr, validator.MustInMemoryRegistry())
 		is.NoErr(err)
-		contents, err := res.RawRead()
+		rc, err := res.RawRead()
+		is.NoErr(err)
+		defer rc.Close()
+		contents, err := ioutil.ReadAll(rc)
 		is.NoErr(err)
 		is.Equal(string(contents), "{\"foo\":\"1234\"}")
 	})