Merge pull request #91 from github/use-pipe-package

Use the new `pipe` package for running external commands

Author: mhagger

git/batch_header.go

@@ -0,0 +1,47 @@
+package git
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/github/git-sizer/counts"
+)
+
+type BatchHeader struct {
+	OID        OID
+	ObjectType ObjectType
+	ObjectSize counts.Count32
+}
+
+var missingHeader = BatchHeader{
+	ObjectType: "missing",
+}
+
+// Parse a `cat-file --batch[-check]` output header line (including
+// the trailing LF). `spec`, if not "", is used in error messages.
+func ParseBatchHeader(spec string, header string) (BatchHeader, error) {
+	header = header[:len(header)-1]
+	words := strings.Split(header, " ")
+	if words[len(words)-1] == "missing" {
+		if spec == "" {
+			spec = words[0]
+		}
+		return missingHeader, fmt.Errorf("missing object %s", spec)
+	}
+
+	oid, err := NewOID(words[0])
+	if err != nil {
+		return missingHeader, err
+	}
+
+	size, err := strconv.ParseUint(words[2], 10, 0)
+	if err != nil {
+		return missingHeader, err
+	}
+	return BatchHeader{
+		OID:        oid,
+		ObjectType: ObjectType(words[1]),
+		ObjectSize: counts.NewCount32(size),
+	}, nil
+}

git/batch_obj_iter.go

@@ -0,0 +1,156 @@
+package git
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/github/git-sizer/internal/pipe"
+)
+
+type ObjectRecord struct {
+	BatchHeader
+	Data []byte
+}
+
+// BatchObjectIter iterates over objects whose names are fed into its
+// stdin. The output is buffered, so it has to be closed before you
+// can be sure that you have gotten all of the objects.
+type BatchObjectIter struct {
+	ctx   context.Context
+	p     *pipe.Pipeline
+	oidCh chan OID
+	objCh chan ObjectRecord
+	errCh chan error
+}
+
+// NewBatchObjectIter returns a `*BatchObjectIterator` and an
+// `io.WriteCloser`. The iterator iterates over objects whose names
+// are fed into the `io.WriteCloser`, one per line. The
+// `io.WriteCloser` should normally be closed and the iterator's
+// output drained before `Close()` is called.
+func (repo *Repository) NewBatchObjectIter(ctx context.Context) (*BatchObjectIter, error) {
+	iter := BatchObjectIter{
+		ctx:   ctx,
+		p:     pipe.New(),
+		oidCh: make(chan OID),
+		objCh: make(chan ObjectRecord),
+		errCh: make(chan error),
+	}
+
+	iter.p.Add(
+		// Read OIDs from `iter.oidCh` and write them to `git
+		// cat-file`:
+		pipe.Function(
+			"request-objects",
+			func(ctx context.Context, _ pipe.Env, _ io.Reader, stdout io.Writer) error {
+				out := bufio.NewWriter(stdout)
+
+				for {
+					select {
+					case oid, ok := <-iter.oidCh:
+						if !ok {
+							return out.Flush()
+						}
+						if _, err := fmt.Fprintln(out, oid.String()); err != nil {
+							return fmt.Errorf("writing to 'git cat-file': %w", err)
+						}
+					case <-ctx.Done():
+						return ctx.Err()
+					}
+				}
+			},
+		),
+
+		// Read OIDs from `stdin` and output a header line followed by
+		// the contents of the corresponding Git objects:
+		pipe.CommandStage(
+			"git-cat-file",
+			repo.GitCommand("cat-file", "--batch", "--buffer"),
+		),
+
+		// Parse the object headers and read the object contents, and
+		// shove both into `objCh`:
+		pipe.Function(
+			"object-reader",
+			func(ctx context.Context, _ pipe.Env, stdin io.Reader, _ io.Writer) error {
+				defer close(iter.objCh)
+
+				f := bufio.NewReader(stdin)
+
+				for {
+					header, err := f.ReadString('\n')
+					if err != nil {
+						if err == io.EOF {
+							return nil
+						}
+						return fmt.Errorf("reading from 'git cat-file': %w", err)
+					}
+					batchHeader, err := ParseBatchHeader("", header)
+					if err != nil {
+						return fmt.Errorf("parsing output of 'git cat-file': %w", err)
+					}
+
+					// Read the object contents plus the trailing LF
+					// (which is discarded below while creating the
+					// `ObjectRecord`):
+					data := make([]byte, batchHeader.ObjectSize+1)
+					if _, err := io.ReadFull(f, data); err != nil {
+						return fmt.Errorf(
+							"reading object data from 'git cat-file' for %s '%s': %w",
+							batchHeader.ObjectType, batchHeader.OID, err,
+						)
+					}
+
+					select {
+					case iter.objCh <- ObjectRecord{
+						BatchHeader: batchHeader,
+						Data:        data[:batchHeader.ObjectSize],
+					}:
+					case <-iter.ctx.Done():
+						return iter.ctx.Err()
+					}
+				}
+			},
+		),
+	)
+
+	if err := iter.p.Start(ctx); err != nil {
+		return nil, err
+	}
+
+	return &iter, nil
+}
+
+// RequestObject requests that the object with the specified `oid` be
+// processed. The objects registered via this method can be read using
+// `Next()` in the order that they were requested.
+func (iter *BatchObjectIter) RequestObject(oid OID) error {
+	select {
+	case iter.oidCh <- oid:
+		return nil
+	case <-iter.ctx.Done():
+		return iter.ctx.Err()
+	}
+}
+
+// Close closes the iterator and frees up resources. Close must be
+// called exactly once.
+func (iter *BatchObjectIter) Close() {
+	close(iter.oidCh)
+}
+
+// Next either returns the next object (its header and contents), or a
+// `false` boolean value if no more objects are left. Objects need to
+// be read asynchronously, but the last objects won't necessarily show
+// up here until `Close()` has been called.
+func (iter *BatchObjectIter) Next() (ObjectRecord, bool, error) {
+	obj, ok := <-iter.objCh
+	if !ok {
+		return ObjectRecord{
+			BatchHeader: missingHeader,
+		}, false, iter.p.Wait()
+	}
+	return obj, true, nil
+}

git/commit.go

@@ -0,0 +1,57 @@
+package git
+
+import (
+	"fmt"
+
+	"github.com/github/git-sizer/counts"
+)
+
+// Commit represents the parts of a commit object that we need.
+type Commit struct {
+	Size    counts.Count32
+	Parents []OID
+	Tree    OID
+}
+
+// ParseCommit parses the commit object whose contents are in `data`.
+// `oid` is used only in error messages.
+func ParseCommit(oid OID, data []byte) (*Commit, error) {
+	var parents []OID
+	var tree OID
+	var treeFound bool
+	iter, err := NewObjectHeaderIter(oid.String(), data)
+	if err != nil {
+		return nil, err
+	}
+	for iter.HasNext() {
+		key, value, err := iter.Next()
+		if err != nil {
+			return nil, err
+		}
+		switch key {
+		case "parent":
+			parent, err := NewOID(value)
+			if err != nil {
+				return nil, fmt.Errorf("malformed parent header in commit %s", oid)
+			}
+			parents = append(parents, parent)
+		case "tree":
+			if treeFound {
+				return nil, fmt.Errorf("multiple trees found in commit %s", oid)
+			}
+			tree, err = NewOID(value)
+			if err != nil {
+				return nil, fmt.Errorf("malformed tree header in commit %s", oid)
+			}
+			treeFound = true
+		}
+	}
+	if !treeFound {
+		return nil, fmt.Errorf("no tree found in commit %s", oid)
+	}
+	return &Commit{
+		Size:    counts.NewCount32(uint64(len(data))),
+		Parents: parents,
+		Tree:    tree,
+	}, nil
+}