image/refs: Add a generic name-based reference interface

And implement that interface for tarballs based on the specs
image-layout.  I plan on adding other backends later, but this is
enough for a proof of concept.

Also add a new oci-refs command so folks can access the new read
functionality from the command line.

The Engine.List interface uses a callback instead of returning
channels or a slice.  Benefits vs. returning a slice of names:

* There's no need to allocate a slice for the results, so calls with
  large (or negative) 'size' values can be made without consuming
  large amounts of memory.

* The name collection and processing can happen concurrently, so:
  * We don't waste cycles collecting names we won't use.
  * Slow collection can happen in the background if/when the consumer
    is blocked on something else.

The benefit of using callbacks vs. returning name and error channels
(as discussed in [1]) is more of a trade-off.  Stephen Day [2] and JT
Olds [3] don't like channel's internal locks.  Dave Cheney doesn't
have a problem with them [4].  Which approach is more efficient for a
given situation depends on how expensive it is for the engine to find
the next key and how expensive it is to act on a returned name.  If
both are expensive, you want goroutines in there somewhere to get
concurrent execution, and channels will help those goroutines
communicate.  When either action is fast (or both are fast), channels
are unnecessary overhead.  By using a callback in the interface, we
avoid baking in the overhead.  Folks who want concurrent execution can
initialize their own channel, launch List in a goroutine, and use the
callback to inject names into their channel.

In a subsequent commit, I'll replace the image/walker.go functionality
with this new API.

I'd prefer casLayout for the imported package, but Stephen doesn't
want camelCase for package names [5].

[1]: https://blog.golang.org/pipelines
[2]: opencontainers/image-spec#159 (comment)
[3]: http://www.jtolds.com/writing/2016/03/go-channels-are-bad-and-you-should-feel-bad/
[4]: https://groups.google.com/d/msg/golang-nuts/LM648yrPpck/idyupwodAwAJ
     Subject: Re: [go-nuts] Re: "Go channels are bad and you should feel bad"
     Date: Wed, 2 Mar 2016 16:04:13 -0800 (PST)
     Message-Id: <[email protected]>
[5]: opencontainers/image-spec#159 (comment)

Signed-off-by: W. Trevor King <[email protected]>

Author: wking

.gitignore

@@ -1,4 +1,5 @@
 /oci-cas
 /oci-create-runtime-bundle
 /oci-image-validate
+/oci-refs
 /oci-unpack

Makefile

@@ -6,6 +6,7 @@ TOOLS := \
 	oci-cas \
 	oci-create-runtime-bundle \
 	oci-image-validate \
+	oci-refs \
 	oci-unpack
 
 default: help

cmd/oci-refs/get.go

@@ -0,0 +1,78 @@
+// Copyright 2016 The Linux Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/opencontainers/image-tools/image/refs/layout"
+	"github.com/spf13/cobra"
+	"golang.org/x/net/context"
+)
+
+type getCmd struct {
+	path string
+	name string
+}
+
+func newGetCmd() *cobra.Command {
+	state := &getCmd{}
+
+	return &cobra.Command{
+		Use:   "get PATH NAME",
+		Short: "Retrieve a reference from the store",
+		Run:   state.Run,
+	}
+}
+
+func (state *getCmd) Run(cmd *cobra.Command, args []string) {
+	if len(args) != 2 {
+		fmt.Fprintln(os.Stderr, "both PATH and NAME must be provided")
+		if err := cmd.Usage(); err != nil {
+			fmt.Fprintln(os.Stderr, err)
+		}
+		os.Exit(1)
+	}
+
+	state.path = args[0]
+	state.name = args[1]
+
+	err := state.run()
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+
+	os.Exit(0)
+}
+
+func (state *getCmd) run() (err error) {
+	ctx := context.Background()
+
+	engine, err := layout.NewEngine(state.path)
+	if err != nil {
+		return err
+	}
+	defer engine.Close()
+
+	descriptor, err := engine.Get(ctx, state.name)
+	if err != nil {
+		return err
+	}
+
+	return json.NewEncoder(os.Stdout).Encode(&descriptor)
+}

cmd/oci-refs/list.go

@@ -0,0 +1,78 @@
+// Copyright 2016 The Linux Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/opencontainers/image-tools/image/refs/layout"
+	"github.com/spf13/cobra"
+	"golang.org/x/net/context"
+)
+
+type listCmd struct {
+	path string
+}
+
+func newListCmd() *cobra.Command {
+	state := &listCmd{}
+
+	return &cobra.Command{
+		Use:   "list PATH",
+		Short: "Return available names from the store.",
+		Run:   state.Run,
+	}
+}
+
+func (state *listCmd) Run(cmd *cobra.Command, args []string) {
+	if len(args) != 1 {
+		fmt.Fprintln(os.Stderr, "PATH must be provided")
+		if err := cmd.Usage(); err != nil {
+			fmt.Fprintln(os.Stderr, err)
+		}
+		os.Exit(1)
+	}
+
+	state.path = args[0]
+
+	err := state.run()
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+
+	os.Exit(0)
+}
+
+func (state *listCmd) run() (err error) {
+	ctx := context.Background()
+
+	engine, err := layout.NewEngine(state.path)
+	if err != nil {
+		return err
+	}
+	defer engine.Close()
+
+	return engine.List(ctx, "", -1, 0, state.printName)
+}
+
+func (state *listCmd) printName(ctx context.Context, name string) (err error) {
+	n, err = fmt.Fprintln(os.Stdout, name)
+	if n < len(name) {
+		return fmt.Errorf("wrote %d of %d name", n, len(name))
+	}
+	return err
+}

cmd/oci-refs/main.go

@@ -0,0 +1,38 @@
+// Copyright 2016 The Linux Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+)
+
+func main() {
+	cmd := &cobra.Command{
+		Use:   "oci-refs",
+		Short: "Name-based reference manipulation",
+	}
+
+	cmd.AddCommand(newGetCmd())
+	cmd.AddCommand(newListCmd())
+
+	err := cmd.Execute()
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}

image/refs/interface.go

@@ -0,0 +1,78 @@
+// Copyright 2016 The Linux Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package refs implements generic name-based reference access.
+package refs
+
+import (
+	"github.com/opencontainers/image-spec/specs-go"
+	"golang.org/x/net/context"
+)
+
+// ListNameCallback templates an Engine.List callback used for
+// processing names.  See Engine.List for more details.
+type ListNameCallback func(ctx context.Context, name string) (err error)
+
+// Engine represents a name-based reference storage engine.
+type Engine interface {
+
+	// Put adds a new reference to the store.  The action is idempotent;
+	// a nil return means "that descriptor is stored at NAME" without
+	// implying "because of your Put()".
+	Put(ctx context.Context, name string, descriptor *specs.Descriptor) (err error)
+
+	// Get returns a reference from the store.  Returns os.ErrNotExist
+	// if the name is not found.
+	Get(ctx context.Context, name string) (descriptor *specs.Descriptor, err error)
+
+	// List returns available names from the store.
+	//
+	// Results are sorted alphabetically.
+	//
+	// Arguments:
+	//
+	// * ctx: gives callers a way to gracefully cancel a long-running
+	//   list.
+	// * prefix: limits the result set to names starting with that
+	//   value.
+	// * size: limits the length of the result set to the first 'size'
+	//   matches.  A value of -1 means "all results".
+	// * from: shifts the result set to start from the 'from'th match.
+	// * nameCallback: called for every matching name.  List returns any
+	//   errors returned by nameCallback and aborts further listing.
+	//
+	// For example, a store with names like:
+	//
+	// * 123
+	// * abcd
+	// * abce
+	// * abcf
+	// * abcg
+	//
+	// will have the following call/result pairs:
+	//
+	// * List(ctx, "", -1, 0, printName) -> "123", "abcd", "abce", "abcf", "abcg"
+	// * List(ctx, "", 2, 0, printName) -> "123", "abcd"
+	// * List(ctx, "", 2, 1, printName) -> "abcd", "abce"
+	// * List(ctx,"abc", 2, 1, printName) -> "abce", "abcf"
+	List(ctx context.Context, prefix string, size int, from int, nameCallback ListNameCallback) (err error)
+
+	// Delete removes a reference from the store.  Returns
+	// os.ErrNotExist if the name is not found.
+	Delete(ctx context.Context, name string) (err error)
+
+	// Close releases resources held by the engine.  Subsequent engine
+	// method calls will fail.
+	Close() (err error)
+}

image/refs/layout/main.go

@@ -0,0 +1,36 @@
+// Copyright 2016 The Linux Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package layout implements the refs interface using the image-spec's
+// image-layout [1].
+//
+// [1]: https://github.com/opencontainers/image-spec/blob/master/image-layout.md
+package layout
+
+import (
+	"os"
+
+	"github.com/opencontainers/image-tools/image/refs"
+)
+
+// NewEngine instantiates an engine with the appropriate backend (tar,
+// HTTP, ...).
+func NewEngine(path string) (engine refs.Engine, err error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+
+	return NewTarEngine(file)
+}