feat: add Features + datastore scoping

The motivation for this is to enable "dispatching" datastores that
dynamically implement the type of the datastore they are dispatching
to, so that type assertions behave equivalently on the dispatcher as
on the dispatchee. We also want this to be backwards-compatible with
existing code using type assertions.

At a high level, this works by generating a concrete implementation of
every possible combination of "features", and then picking the right
implementation at runtime. This is necessary due to language
constraints in Go--it is currently impossible to create a concrete
type dynamically with reflection that implements an interface.

"Features" are introduced here, which are supplemental, optional
interfaces that datastores may implement. These are
backwards-compatible with existing "features", which are:

* Batching
* CheckedDatastore
* GCDatastore
* PersistentDatastore
* ScrubbedDatastore
* TTLDatastore
* TxnDatastore

New features can also be added in a backwards-compatible way. E.g. if
datastore A is scoped down to datastore B, a new feature F is added,
and then implemented on B, then A will continue to implement the same
set of features since it hasn't implemented F yet (and vice versa if F
is implemented on A but not B).

Examples of things this enables:

* Allow us to deprecate ErrBatchUnsupported
* Allow existing dispatching datastores to support all
features (keytransform, retrystore, MutexDatastore, autobatch, etc.)
* Features supported by a Mount datastore could be scoped down to the
intersection of all children
* Communication with data about what functionality a datastore
supports (e.g. for cross-language/RPC support)

Some related issues:

* #160
* #88

Author: guseggert

basic_ds.go

@@ -89,62 +89,6 @@ func (d *MapDatastore) Close() error {
 	return nil
 }
 
-// NullDatastore stores nothing, but conforms to the API.
-// Useful to test with.
-type NullDatastore struct {
-}
-
-var _ Datastore = (*NullDatastore)(nil)
-var _ Batching = (*NullDatastore)(nil)
-
-// NewNullDatastore constructs a null datastore
-func NewNullDatastore() *NullDatastore {
-	return &NullDatastore{}
-}
-
-// Put implements Datastore.Put
-func (d *NullDatastore) Put(ctx context.Context, key Key, value []byte) (err error) {
-	return nil
-}
-
-// Sync implements Datastore.Sync
-func (d *NullDatastore) Sync(ctx context.Context, prefix Key) error {
-	return nil
-}
-
-// Get implements Datastore.Get
-func (d *NullDatastore) Get(ctx context.Context, key Key) (value []byte, err error) {
-	return nil, ErrNotFound
-}
-
-// Has implements Datastore.Has
-func (d *NullDatastore) Has(ctx context.Context, key Key) (exists bool, err error) {
-	return false, nil
-}
-
-// Has implements Datastore.GetSize
-func (d *NullDatastore) GetSize(ctx context.Context, key Key) (size int, err error) {
-	return -1, ErrNotFound
-}
-
-// Delete implements Datastore.Delete
-func (d *NullDatastore) Delete(ctx context.Context, key Key) (err error) {
-	return nil
-}
-
-// Query implements Datastore.Query
-func (d *NullDatastore) Query(ctx context.Context, q dsq.Query) (dsq.Results, error) {
-	return dsq.ResultsWithEntries(q, nil), nil
-}
-
-func (d *NullDatastore) Batch(ctx context.Context) (Batch, error) {
-	return NewBasicBatch(d), nil
-}
-
-func (d *NullDatastore) Close() error {
-	return nil
-}
-
 // LogDatastore logs all accesses through the datastore.
 type LogDatastore struct {
 	Name  string

basic_ds_test.go

@@ -5,24 +5,18 @@ import (
 	"log"
 	"testing"
 
-	dstore "github.com/ipfs/go-datastore"
+	"github.com/ipfs/go-datastore"
 	dstest "github.com/ipfs/go-datastore/test"
 )
 
 func TestMapDatastore(t *testing.T) {
-	ds := dstore.NewMapDatastore()
+	ds := datastore.NewMapDatastore()
 	dstest.SubtestAll(t, ds)
 }
 
-func TestNullDatastore(t *testing.T) {
-	ds := dstore.NewNullDatastore()
-	// The only test that passes. Nothing should be found.
-	dstest.SubtestNotFounds(t, ds)
-}
-
 func TestLogDatastore(t *testing.T) {
 	defer log.SetOutput(log.Writer())
 	log.SetOutput(ioutil.Discard)
-	ds := dstore.NewLogDatastore(dstore.NewMapDatastore(), "")
+	ds := datastore.NewLogDatastore(datastore.NewMapDatastore(), "")
 	dstest.SubtestAll(t, ds)
 }

datastore.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"io"
-	"time"
 
 	query "github.com/ipfs/go-datastore/query"
 )
@@ -103,8 +102,7 @@ type Read interface {
 // capabilities of a `Batch`, but the reverse is NOT true.
 type Batching interface {
 	Datastore
-
-	Batch(ctx context.Context) (Batch, error)
+	BatchingFeature
 }
 
 // ErrBatchUnsupported is returned if the by Batch if the Datastore doesn't
@@ -115,34 +113,29 @@ var ErrBatchUnsupported = errors.New("this datastore does not support batching")
 // which may need checking on-disk data integrity.
 type CheckedDatastore interface {
 	Datastore
-
-	Check(ctx context.Context) error
+	CheckedFeature
 }
 
 // ScrubbedDatastore is an interface that should be implemented by datastores
 // which want to provide a mechanism to check data integrity and/or
 // error correction.
 type ScrubbedDatastore interface {
 	Datastore
-
-	Scrub(ctx context.Context) error
+	ScrubbedFeature
 }
 
 // GCDatastore is an interface that should be implemented by datastores which
 // don't free disk space by just removing data from them.
 type GCDatastore interface {
 	Datastore
-
-	CollectGarbage(ctx context.Context) error
+	GCFeature
 }
 
 // PersistentDatastore is an interface that should be implemented by datastores
 // which can report disk usage.
 type PersistentDatastore interface {
 	Datastore
-
-	// DiskUsage returns the space used by a datastore, in bytes.
-	DiskUsage(ctx context.Context) (uint64, error)
+	PersistentFeature
 }
 
 // DiskUsage checks if a Datastore is a
@@ -163,13 +156,6 @@ type TTLDatastore interface {
 	TTL
 }
 
-// TTL encapulates the methods that deal with entries with time-to-live.
-type TTL interface {
-	PutWithTTL(ctx context.Context, key Key, value []byte, ttl time.Duration) error
-	SetTTL(ctx context.Context, key Key, ttl time.Duration) error
-	GetExpiration(ctx context.Context, key Key) (time.Time, error)
-}
-
 // Txn extends the Datastore type. Txns allow users to batch queries and
 // mutations to the Datastore into atomic groups, or transactions. Actions
 // performed on a transaction will not take hold until a successful call to
@@ -194,8 +180,7 @@ type Txn interface {
 // support transactions.
 type TxnDatastore interface {
 	Datastore
-
-	NewTransaction(ctx context.Context, readOnly bool) (Txn, error)
+	TxnFeature
 }
 
 // Errors

features.go

@@ -0,0 +1,132 @@
+package datastore
+
+import (
+	"context"
+	"reflect"
+	"time"
+)
+
+const (
+	FeatureNameBatching    = "Batching"
+	FeatureNameChecked     = "Checked"
+	FeatureNameGC          = "GC"
+	FeatureNamePersistent  = "Persistent"
+	FeatureNameScrubbed    = "Scrubbed"
+	FeatureNameTTL         = "TTL"
+	FeatureNameTransaction = "Transaction"
+)
+
+type BatchingFeature interface {
+	Batch(ctx context.Context) (Batch, error)
+}
+
+type CheckedFeature interface {
+	Check(ctx context.Context) error
+}
+
+type ScrubbedFeature interface {
+	Scrub(ctx context.Context) error
+}
+
+type GCFeature interface {
+	CollectGarbage(ctx context.Context) error
+}
+
+type PersistentFeature interface {
+	// DiskUsage returns the space used by a datastore, in bytes.
+	DiskUsage(ctx context.Context) (uint64, error)
+}
+
+// TTL encapulates the methods that deal with entries with time-to-live.
+type TTL interface {
+	PutWithTTL(ctx context.Context, key Key, value []byte, ttl time.Duration) error
+	SetTTL(ctx context.Context, key Key, ttl time.Duration) error
+	GetExpiration(ctx context.Context, key Key) (time.Time, error)
+}
+
+type TxnFeature interface {
+	NewTransaction(ctx context.Context, readOnly bool) (Txn, error)
+}
+
+// Feature contains metadata about a datastore Feature.
+type Feature struct {
+	Name string
+	// Interface is the nil interface of the feature.
+	Interface interface{}
+	// DatastoreInterface is the nil interface of the feature's corresponding datastore interface.
+	DatastoreInterface interface{}
+}
+
+var featuresByName map[string]Feature
+
+func init() {
+	featuresByName = map[string]Feature{}
+	for _, f := range Features() {
+		featuresByName[f.Name] = f
+	}
+}
+
+// Features returns a list of all known datastore features.
+// This serves both to provide an authoritative list of features,
+// and to define a canonical ordering of features.
+func Features() []Feature {
+	// for backwards compatibility, only append to this list
+	return []Feature{
+		{
+			Name:               FeatureNameBatching,
+			Interface:          (*BatchingFeature)(nil),
+			DatastoreInterface: (*Batching)(nil),
+		},
+		{
+			Name:               FeatureNameChecked,
+			Interface:          (*CheckedFeature)(nil),
+			DatastoreInterface: (*CheckedDatastore)(nil),
+		},
+		{
+			Name:               FeatureNameGC,
+			Interface:          (*GCFeature)(nil),
+			DatastoreInterface: (*GCDatastore)(nil),
+		},
+		{
+			Name:               FeatureNamePersistent,
+			Interface:          (*PersistentFeature)(nil),
+			DatastoreInterface: (*PersistentDatastore)(nil),
+		},
+		{
+			Name:               FeatureNameScrubbed,
+			Interface:          (*ScrubbedFeature)(nil),
+			DatastoreInterface: (*ScrubbedDatastore)(nil),
+		},
+		{
+			Name:               FeatureNameTTL,
+			Interface:          (*TTL)(nil),
+			DatastoreInterface: (*TTLDatastore)(nil),
+		},
+		{
+			Name:               FeatureNameTransaction,
+			Interface:          (*TxnFeature)(nil),
+			DatastoreInterface: (*TxnDatastore)(nil),
+		},
+	}
+}
+
+// FeatureByName returns the feature with the given name, if known.
+func FeatureByName(name string) (Feature, bool) {
+	feat, known := featuresByName[name]
+	return feat, known
+}
+
+// FeaturesForDatastore returns the features supported by the given datastore.
+func FeaturesForDatastore(dstore Datastore) (features []Feature) {
+	if dstore == nil {
+		return nil
+	}
+	dstoreType := reflect.TypeOf(dstore)
+	for _, f := range Features() {
+		fType := reflect.TypeOf(f.Interface).Elem()
+		if dstoreType.Implements(fType) {
+			features = append(features, f)
+		}
+	}
+	return
+}

features_test.go

@@ -0,0 +1,77 @@
+package datastore
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+)
+
+func TestFeatureByName(t *testing.T) {
+	feat, ok := FeatureByName(FeatureNameBatching)
+	if !ok {
+		t.Fatalf("expected a batching feature")
+	}
+	if feat.Name != FeatureNameBatching ||
+		feat.Interface != (*BatchingFeature)(nil) ||
+		feat.DatastoreInterface != (*Batching)(nil) {
+		t.Fatalf("expected a batching feature, got %v", feat)
+	}
+
+	feat, ok = FeatureByName("UnknownFeature")
+	if ok {
+		t.Fatalf("expected UnknownFeature not to be found")
+	}
+}
+
+func featuresByNames(names []string) (fs []Feature) {
+	for _, n := range names {
+		f, ok := FeatureByName(n)
+		if !ok {
+			panic(fmt.Sprintf("unknown feature %s", n))
+		}
+		fs = append(fs, f)
+	}
+	return
+}
+
+func TestFeaturesForDatastore(t *testing.T) {
+	cases := []struct {
+		name             string
+		d                Datastore
+		expectedFeatures []string
+	}{
+		{
+			name:             "MapDatastore",
+			d:                &MapDatastore{},
+			expectedFeatures: []string{"Batching"},
+		},
+		{
+			name:             "NullDatastore",
+			d:                &NullDatastore{},
+			expectedFeatures: []string{"Batching", "Checked", "GC", "Persistent", "Scrubbed", "Transaction"},
+		},
+		{
+			name:             "LogDatastore",
+			d:                &LogDatastore{},
+			expectedFeatures: []string{"Batching", "Checked", "GC", "Persistent", "Scrubbed"},
+		},
+		{
+			name:             "nil datastore",
+			d:                nil,
+			expectedFeatures: nil,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			feats := FeaturesForDatastore(c.d)
+			if len(feats) != len(c.expectedFeatures) {
+				t.Fatalf("expected %d features, got %v", len(c.expectedFeatures), feats)
+			}
+			expectedFeats := featuresByNames(c.expectedFeatures)
+			if !reflect.DeepEqual(expectedFeats, feats) {
+				t.Fatalf("expected features %v, got %v", c.expectedFeatures, feats)
+			}
+		})
+	}
+}