Merge pull request #134 from MichaelMure/querysize

add a Size field to Query's Result

Author: Stebalien

autobatch/autobatch.go

@@ -8,7 +8,7 @@ import (
 	dsq "github.com/ipfs/go-datastore/query"
 )
 
-// Datastore implements a go-datatsore.
+// Datastore implements a go-datastore.
 type Datastore struct {
 	child ds.Batching
 

basic_ds.go

@@ -61,7 +61,7 @@ func (d *MapDatastore) Delete(key Key) (err error) {
 func (d *MapDatastore) Query(q dsq.Query) (dsq.Results, error) {
 	re := make([]dsq.Entry, 0, len(d.values))
 	for k, v := range d.values {
-		e := dsq.Entry{Key: k.String()}
+		e := dsq.Entry{Key: k.String(), Size: len(v)}
 		if !q.KeysOnly {
 			e.Value = v
 		}

mount/mount.go

@@ -227,6 +227,7 @@ func (d *Datastore) Query(master query.Query) (query.Results, error) {
 		Orders:            master.Orders,
 		KeysOnly:          master.KeysOnly,
 		ReturnExpirations: master.ReturnExpirations,
+		ReturnsSizes:      master.ReturnsSizes,
 	}
 
 	prefix := ds.NewKey(childQuery.Prefix)

namespace/namespace_test.go

@@ -101,9 +101,9 @@ func (ks *DSSuite) TestQuery(c *C) {
 	c.Check(err, Equals, nil)
 
 	expect := []dsq.Entry{
-		{Key: "/bar", Value: []byte("/foo/bar")},
-		{Key: "/bar/baz", Value: []byte("/foo/bar/baz")},
-		{Key: "/baz/abc", Value: []byte("/foo/baz/abc")},
+		{Key: "/bar", Size: len([]byte("/foo/bar")), Value: []byte("/foo/bar")},
+		{Key: "/bar/baz", Size: len([]byte("/foo/bar/baz")), Value: []byte("/foo/bar/baz")},
+		{Key: "/baz/abc", Size: len([]byte("/foo/baz/abc")), Value: []byte("/foo/baz/abc")},
 	}
 
 	results, err := qres.Rest()
@@ -122,8 +122,8 @@ func (ks *DSSuite) TestQuery(c *C) {
 	c.Check(err, Equals, nil)
 
 	expect = []dsq.Entry{
-		{Key: "/bar", Value: []byte("/foo/bar")},
-		{Key: "/bar/baz", Value: []byte("/foo/bar/baz")},
+		{Key: "/bar", Size: len([]byte("/foo/bar")), Value: []byte("/foo/bar")},
+		{Key: "/bar/baz", Size: len([]byte("/foo/bar/baz")), Value: []byte("/foo/bar/baz")},
 	}
 
 	results, err = qres.Rest()

query/query.go

@@ -66,6 +66,9 @@ type Query struct {
 	Offset            int      // skip given number of results
 	KeysOnly          bool     // return only keys.
 	ReturnExpirations bool     // return expirations (see TTLDatastore)
+	ReturnsSizes      bool     // always return sizes. If not set, datastore impl can return
+	//                         // it anyway if it doesn't involve a performance cost. If KeysOnly
+	//                         // is not set, Size should always be set.
 }
 
 // String returns a string represenation of the Query for debugging/validation
@@ -117,6 +120,8 @@ type Entry struct {
 	Key        string    // cant be ds.Key because circular imports ...!!!
 	Value      []byte    // Will be nil if KeysOnly has been passed.
 	Expiration time.Time // Entry expiration timestamp if requested and supported (see TTLDatastore).
+	Size       int       // Might be -1 if the datastore doesn't support listing the size with KeysOnly
+	//                   // or if ReturnsSizes is not set
 }
 
 // Result is a special entry that includes an error, so that the client

query/query_impl.go

@@ -136,7 +136,7 @@ func NaiveQueryApply(q Query, qr Results) Results {
 func ResultEntriesFrom(keys []string, vals [][]byte) []Entry {
 	re := make([]Entry, len(keys))
 	for i, k := range keys {
-		re[i] = Entry{Key: k, Value: vals[i]}
+		re[i] = Entry{Key: k, Size: len(vals[i]), Value: vals[i]}
 	}
 	return re
 }

test/basic_tests.go

@@ -301,6 +301,10 @@ func SubtestFilter(t *testing.T, ds dstore.Datastore) {
 	test(new(testFilter))
 }
 
+func SubtestReturnSizes(t *testing.T, ds dstore.Datastore) {
+	subtestQuery(t, ds, dsq.Query{ReturnsSizes: true}, 100)
+}
+
 func randValue() []byte {
 	value := make([]byte, 64)
 	rand.Read(value)
@@ -315,6 +319,7 @@ func subtestQuery(t *testing.T, ds dstore.Datastore, q dsq.Query, count int) {
 		value := randValue()
 		input = append(input, dsq.Entry{
 			Key:   key,
+			Size:  len(value),
 			Value: value,
 		})
 	}
@@ -375,7 +380,9 @@ func subtestQuery(t *testing.T, ds dstore.Datastore, q dsq.Query, count int) {
 		if !q.KeysOnly && !bytes.Equal(actual[i].Value, expected[i].Value) {
 			t.Errorf("value mismatch for result %d (key=%q)", i, expected[i].Key)
 		}
-
+		if q.ReturnsSizes && actual[i].Size <= 0 {
+			t.Errorf("for result %d, expected size > 0 with ReturnsSizes", i)
+		}
 	}
 
 	t.Log("deleting all keys")

test/suite.go

@@ -18,6 +18,7 @@ var BasicSubtests = []func(t *testing.T, ds dstore.Datastore){
 	SubtestLimit,
 	SubtestFilter,
 	SubtestManyKeysAndQuery,
+	SubtestReturnSizes,
 }
 
 // BatchSubtests is a list of all basic batching datastore tests.