Large refactoring of the bundle store in order to handle ids in commands

Signed-off-by: Jean-Christophe Sirot <[email protected]>

Author: Jean-Christophe Sirot

internal/cnab/cnab.go

@@ -10,7 +10,6 @@ import (
 	"github.com/docker/app/internal"
 	"github.com/docker/app/internal/log"
 	"github.com/docker/app/internal/packager"
-	"github.com/docker/app/internal/store"
 	appstore "github.com/docker/app/internal/store"
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/cnab-to-oci/remotes"
@@ -90,7 +89,9 @@ func ResolveBundle(dockerCli command.Cli, bundleStore appstore.BundleStore, name
 
 // GetBundle searches for the bundle locally and tries to pull it if not found
 func GetBundle(dockerCli command.Cli, bundleStore appstore.BundleStore, name string) (*bundle.Bundle, reference.Reference, error) {
-	ref, err := store.StringToRef(name)
+	//ref, err := store.StringToRef(name)
+	ref, err := bundleStore.LookUp(name)
+	fmt.Println(ref)
 	if err != nil {
 		return nil, nil, err
 	}

internal/store/app.go

@@ -74,7 +74,8 @@ func (a ApplicationStore) BundleStore() (BundleStore, error) {
 	if err := os.MkdirAll(path, 0755); err != nil {
 		return nil, errors.Wrapf(err, "failed to create bundle store directory %q", path)
 	}
-	return &bundleStore{path: path}, nil
+	//return &bundleStore{path: path}, nil
+	return NewBundleStore(path)
 }
 
 func makeDigestedDirectory(context string) string {

internal/store/bundle.go

@@ -11,7 +11,6 @@ import (
 
 	"github.com/deislabs/cnab-go/bundle"
 	"github.com/docker/distribution/reference"
-	"github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
 )
 
@@ -22,12 +21,27 @@ type BundleStore interface {
 	Read(ref reference.Reference) (*bundle.Bundle, error)
 	List() ([]reference.Reference, error)
 	Remove(ref reference.Reference) error
+	LookUp(refOrID string) (reference.Reference, error)
 }
 
 var _ BundleStore = &bundleStore{}
 
 type bundleStore struct {
-	path string
+	path         string
+	referenceMap map[ID][]reference.Reference
+}
+
+func NewBundleStore(path string) (BundleStore, error) {
+	bundleStore := &bundleStore{
+		path:         path,
+		referenceMap: make(map[ID][]reference.Reference),
+	}
+	err := bundleStore.scanAllBundles()
+	if err != nil {
+		return nil, err
+	}
+	fmt.Println(bundleStore.referenceMap)
+	return bundleStore, nil
 }
 
 // We store bundles either by image:tags, image:digest or by unique ID (actually, bundle's sha256).
@@ -72,56 +86,27 @@ func (b *bundleStore) Store(ref reference.Reference, bndle *bundle.Bundle) (refe
 }
 
 func (b *bundleStore) Read(ref reference.Reference) (*bundle.Bundle, error) {
-	path, err := b.storePath(ref)
+	paths, err := b.storePaths(ref)
+	fmt.Println(paths)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to read bundle %q", ref)
 	}
 
-	data, err := ioutil.ReadFile(filepath.Join(path, "bundle.json"))
+	bndl, err := b.fetchBundleJSON(filepath.Join(paths[0], "bundle.json"))
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to read bundle %q", ref)
 	}
-	var bndle bundle.Bundle
-	if err := json.Unmarshal(data, &bndle); err != nil {
-		return nil, errors.Wrapf(err, "failed to read bundle %q", ref)
-	}
-	return &bndle, nil
+	return bndl, nil
 }
 
 // Returns the list of all bundles present in the bundle store
 func (b *bundleStore) List() ([]reference.Reference, error) {
 	var references []reference.Reference
-	digests := filepath.Join(b.path, "_ids")
-	if err := filepath.Walk(b.path, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-
-		if info.IsDir() {
-			return nil
-		}
-
-		if !strings.HasSuffix(info.Name(), ".json") {
-			return nil
-		}
-
-		if strings.HasPrefix(path, digests) {
-			rel := path[len(digests)+1:]
-			dg := strings.Split(filepath.ToSlash(rel), "/")[0]
-			references = append(references, ID{digest.NewDigestFromEncoded(digest.SHA256, dg)})
-			return nil
-		}
 
-		ref, err := b.pathToReference(path)
-		if err != nil {
-			return err
+	for _, refAliases := range b.referenceMap {
+		for _, ref := range refAliases {
+			references = append(references, ref)
 		}
-
-		references = append(references, ref)
-
-		return nil
-	}); err != nil {
-		return nil, err
 	}
 
 	sort.Slice(references, func(i, j int) bool {
@@ -143,6 +128,75 @@ func (b *bundleStore) Remove(ref reference.Reference) error {
 	return os.RemoveAll(path)
 }
 
+func (b *bundleStore) LookUp(refOrID string) (reference.Reference, error) {
+	ref, err := FromString(refOrID)
+	if err == nil {
+		return ref, nil
+	}
+	if isShortID(refOrID) {
+		ref, err := b.matchShortID(refOrID)
+		if err == nil {
+			return ref, nil
+		}
+	}
+	return reference.ParseNormalizedNamed(refOrID)
+}
+
+func (b *bundleStore) matchShortID(shortID string) (reference.Reference, error) {
+	var found reference.Reference
+	for id := range b.referenceMap {
+		if strings.HasPrefix(id.String(), shortID) {
+			if found != nil && found != id {
+				return nil, fmt.Errorf("Ambiguous reference found")
+			}
+			found = id
+		}
+	}
+	if found == nil {
+		return nil, fmt.Errorf("Could not find reference")
+	}
+	fmt.Printf("found = %s\n", found)
+	return found, nil
+}
+
+func (b *bundleStore) referenceToID(ref reference.Reference) (ID, error) {
+	if id, ok := ref.(ID); ok {
+		return id, nil
+	}
+	for id, refs := range b.referenceMap {
+		for _, r := range refs {
+			if r == ref {
+				return id, nil
+			}
+		}
+	}
+	return ID{}, fmt.Errorf("%s: reference not found", ref.String())
+}
+
+func (b *bundleStore) storePaths(ref reference.Reference) ([]string, error) {
+	var paths []string
+
+	id, err := b.referenceToID(ref)
+	if err != nil {
+		return nil, err
+	}
+
+	if refs, exist := b.referenceMap[id]; exist {
+		for _, rf := range refs {
+			path, err := b.storePath(rf)
+			if err != nil {
+				continue
+			}
+			paths = append(paths, path)
+		}
+	}
+
+	if len(paths) == 0 {
+		return nil, fmt.Errorf("%s: reference not found", ref.String())
+	}
+	return paths, nil
+}
+
 func (b *bundleStore) storePath(ref reference.Reference) (string, error) {
 	named, ok := ref.(reference.Named)
 	if !ok {
@@ -175,6 +229,72 @@ func (b *bundleStore) storePath(ref reference.Reference) (string, error) {
 	return storeDir, nil
 }
 
+// Returns the list of all bundles present in the bundle store
+func (b *bundleStore) scanAllBundles() error {
+	digests := filepath.Join(b.path, "_ids")
+	if err := filepath.Walk(b.path, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if info.IsDir() {
+			return nil
+		}
+
+		if !strings.HasSuffix(info.Name(), ".json") {
+			return nil
+		}
+
+		if strings.HasPrefix(path, digests) {
+			rel := path[len(digests)+1:]
+			dg := strings.Split(filepath.ToSlash(rel), "/")[0]
+			//references = append(references, ID{dg})
+			id := ID{dg}
+			if _, ok := b.referenceMap[id]; !ok {
+				b.referenceMap[id] = []reference.Reference{id}
+			} else {
+				b.referenceMap[id] = append(b.referenceMap[id], id)
+			}
+			return nil
+		}
+
+		ref, err := b.pathToReference(path)
+		if err != nil {
+			return err
+		}
+		bndl, err := b.fetchBundleJSON(path)
+		if err != nil {
+			return err
+		}
+		id, err := FromBundle(bndl)
+		if err != nil {
+			return err
+		}
+		if _, ok := b.referenceMap[id]; !ok {
+			b.referenceMap[id] = []reference.Reference{ref}
+		} else {
+			b.referenceMap[id] = append(b.referenceMap[id], ref)
+		}
+
+		return nil
+	}); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (b *bundleStore) fetchBundleJSON(bundlePath string) (*bundle.Bundle, error) {
+	data, err := ioutil.ReadFile(bundlePath)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to read file %s", bundlePath)
+	}
+	var bndle bundle.Bundle
+	if err := json.Unmarshal(data, &bndle); err != nil {
+		return nil, errors.Wrapf(err, "failed to read file %s", bundlePath)
+	}
+	return &bndle, nil
+}
+
 func (b *bundleStore) pathToReference(path string) (reference.Named, error) {
 	// Clean the path and remove the local bundle store path
 	cleanpath := filepath.ToSlash(path)

internal/store/digest.go

@@ -11,6 +11,16 @@ import (
 	"github.com/opencontainers/go-digest"
 )
 
+var (
+	identifierRegexp = regexp.MustCompile(`^([a-f0-9]{64})$`)
+
+	shortIdentifierRegexp = regexp.MustCompile(`^([a-f0-9]{1,64})$`)
+)
+
+func isShortID(ref string) bool {
+	return shortIdentifierRegexp.MatchString(ref)
+}
+
 // ComputeDigest takes a bundle and produce a unigue reference.Digested
 func ComputeDigest(bundle io.WriterTo) (digest.Digest, error) {
 	b := bytes.Buffer{}
@@ -29,7 +39,7 @@ func StringToRef(s string) (reference.Reference, error) {
 }
 
 func FromString(s string) (ID, error) {
-	if ok, _ := regexp.MatchString("[a-z0-9]{64}", s); !ok {
+	if ok := identifierRegexp.MatchString(s); !ok {
 		return ID{}, fmt.Errorf("could not parse '%s' as a valid reference", s)
 	}
 	digest := digest.NewDigestFromEncoded(digest.SHA256, s)

internal/store/installation.go

@@ -34,7 +34,7 @@ func NewInstallation(name string, reference string) (*Installation, error) {
 	}, nil
 }
 
-// SetParameters sets the parameter value if the installation bundle has
+// SetParameter sets the parameter value if the installation bundle has
 // a defined parameter with that name.
 func (i Installation) SetParameter(name string, value string) {
 	if _, ok := i.Bundle.Parameters[name]; ok {