Add initial "multi-stage" support in bashbrew

This allows bashbrew to properly handle cross-repository and cross-tag dependencies even in the face of multiple `FROM` instructions or `COPY --from=`.

This also provides the scaffolding necessary to implement this in scripts using `bashbrew cat`.

As fallback behavior, the `*DockerFrom` functions should return the `FROM` of the last stage in the `Dockerfile` (which is essentially the `FROM` of the final image).

Also, the output of `bashbrew from` is now a space-separated list.

Author: tianon

go/src/bashbrew/cmd-build.go

@@ -44,27 +44,29 @@ func cmdBuild(c *cli.Context) error {
 				continue
 			}
 
-			from, err := r.DockerFrom(entry)
+			meta, err := r.DockerfileMetadata(entry)
 			if err != nil {
 				return cli.NewMultiError(fmt.Errorf(`failed fetching/scraping FROM for %q (tags %q)`, r.RepoName, entry.TagsString()), err)
 			}
 
-			if from != "scratch" && pull != "never" {
-				doPull := false
-				switch pull {
-				case "always":
-					doPull = true
-				case "missing":
-					_, err := dockerInspect("{{.Id}}", from)
-					doPull = (err != nil)
-				default:
-					return fmt.Errorf(`unexpected value for --pull: %s`, pull)
-				}
-				if doPull {
-					// TODO detect if "from" is something we've built (ie, "python:3-onbuild" is "FROM python:3" but we don't want to pull "python:3" if we "bashbrew build python")
-					fmt.Printf("Pulling %s (%s)\n", from, r.EntryIdentifier(entry))
-					if !dryRun {
-						dockerPull(from)
+			for _, from := range meta.Froms {
+				if from != "scratch" && pull != "never" {
+					doPull := false
+					switch pull {
+					case "always":
+						doPull = true
+					case "missing":
+						_, err := dockerInspect("{{.Id}}", from)
+						doPull = (err != nil)
+					default:
+						return fmt.Errorf(`unexpected value for --pull: %s`, pull)
+					}
+					if doPull {
+						// TODO detect if "from" is something we've built (ie, "python:3-onbuild" is "FROM python:3" but we don't want to pull "python:3" if we "bashbrew build python")
+						fmt.Printf("Pulling %s (%s)\n", from, r.EntryIdentifier(entry))
+						if !dryRun {
+							dockerPull(from)
+						}
 					}
 				}
 			}
@@ -90,6 +92,8 @@ func cmdBuild(c *cli.Context) error {
 					}
 					defer archive.Close()
 
+					// TODO use "meta.StageNames" to do "docker build --target" so we can tag intermediate stages too for cache (streaming "git archive" directly to "docker build" makes that a little hard to accomplish without re-streaming)
+
 					err = dockerBuild(cacheTag, entry.ArchFile(arch), archive)
 					if err != nil {
 						return cli.NewMultiError(fmt.Errorf(`failed building %q (tags %q)`, r.RepoName, entry.TagsString()), err)

go/src/bashbrew/cmd-deps.go

@@ -67,12 +67,14 @@ func cmdFamily(parents bool, c *cli.Context) error {
 				continue
 			}
 
-			from, err := r.DockerFrom(entry)
+			meta, err := r.DockerfileMetadata(entry)
 			if err != nil {
 				return cli.NewMultiError(fmt.Errorf(`failed fetching/scraping FROM for %q (tags %q)`, r.RepoName, entry.TagsString()), err)
 			}
-			for _, tag := range r.Tags("", false, entry) {
-				network.AddEdge(from, tag)
+			for _, from := range meta.Froms {
+				for _, tag := range r.Tags("", false, entry) {
+					network.AddEdge(from, tag)
+				}
 			}
 		}
 	}

go/src/bashbrew/cmd-from.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/codegangsta/cli"
 )
@@ -27,13 +28,14 @@ func cmdFrom(c *cli.Context) error {
 				continue
 			}
 
-			from, err := r.DockerFrom(entry)
+			meta, err := r.DockerfileMetadata(entry)
 			if err != nil {
 				return cli.NewMultiError(fmt.Errorf(`failed fetching/scraping FROM for %q (tags %q)`, r.RepoName, entry.TagsString()), err)
 			}
 
+			froms := strings.Join(meta.Froms, " ")
 			for _, tag := range r.Tags(namespace, uniq, entry) {
-				fmt.Printf("%s: %s\n", tag, from)
+				fmt.Printf("%s: %s\n", tag, froms)
 			}
 		}
 	}

go/src/bashbrew/docker.go

@@ -10,81 +10,169 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"strconv"
 	"strings"
 
 	"github.com/codegangsta/cli"
 	"github.com/docker-library/go-dockerlibrary/manifest"
 )
 
-var dockerFromCache = map[string]string{}
+type DockerfileMetadata struct {
+	StageFroms []string              // every image "FROM" instruction value (or the parent stage's FROM value in the case of a named stage)
+	StageNames []string              // the name of any named stage (in order)
+	StageNameFroms map[string]string // map of stage names to FROM values (or the parent stage's FROM value in the case of a named stage), useful for resolving stage names to FROM values
+
+	Froms  []string // every "FROM" or "COPY --from=xxx" value (minus named and/or numbered stages in the case of "--from=")
+}
 
 func (r Repo) DockerFrom(entry *manifest.Manifest2822Entry) (string, error) {
 	return r.ArchDockerFrom(arch, entry)
 }
 
 func (r Repo) ArchDockerFrom(arch string, entry *manifest.Manifest2822Entry) (string, error) {
-	commit, err := r.fetchGitRepo(arch, entry)
+	dockerfileMeta, err := r.ArchDockerfileMetadata(arch, entry)
 	if err != nil {
 		return "", err
 	}
 
+	return dockerfileMeta.StageFroms[len(dockerfileMeta.StageFroms)-1], nil
+}
+
+func (r Repo) DockerfileMetadata(entry *manifest.Manifest2822Entry) (*DockerfileMetadata, error) {
+	return r.ArchDockerfileMetadata(arch, entry)
+}
+
+var dockerfileMetadataCache = map[string]*DockerfileMetadata{}
+
+func (r Repo) ArchDockerfileMetadata(arch string, entry *manifest.Manifest2822Entry) (*DockerfileMetadata, error) {
+	commit, err := r.fetchGitRepo(arch, entry)
+	if err != nil {
+		return nil, err
+	}
+
 	dockerfileFile := path.Join(entry.ArchDirectory(arch), entry.ArchFile(arch))
 
 	cacheKey := strings.Join([]string{
 		commit,
 		dockerfileFile,
 	}, "\n")
-	if from, ok := dockerFromCache[cacheKey]; ok {
-		return from, nil
+	if meta, ok := dockerfileMetadataCache[cacheKey]; ok {
+		return meta, nil
 	}
 
 	dockerfile, err := gitShow(commit, dockerfileFile)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 	defer dockerfile.Close()
 
-	from, err := dockerfileFrom(dockerfile)
+	meta, err := parseDockerfileMetadata(dockerfile)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 
 	if err := dockerfile.Close(); err != nil {
-		return "", err
+		return nil, err
 	}
 
-	// make sure to add ":latest" if it's implied
-	from = latestizeRepoTag(from)
-
-	dockerFromCache[cacheKey] = from
-	return from, nil
+	dockerfileMetadataCache[cacheKey] = meta
+	return meta, nil
 }
 
-// TODO determine multi-stage-builds impact here (once official images are willing/able to support them; post-17.06 at the earliest)
-func dockerfileFrom(dockerfile io.Reader) (string, error) {
+func parseDockerfileMetadata(dockerfile io.Reader) (*DockerfileMetadata, error) {
+	meta := &DockerfileMetadata{
+		// panic: assignment to entry in nil map
+		StageNameFroms: map[string]string{},
+		// (nil slices work fine)
+	}
+
 	scanner := bufio.NewScanner(dockerfile)
 	for scanner.Scan() {
 		line := strings.TrimSpace(scanner.Text())
+
 		if line == "" {
 			// ignore blank lines
 			continue
 		}
+
 		if line[0] == '#' {
+			// TODO handle "escape" parser directive
+			// TODO handle "syntax" parser directive -- explode appropriately (since custom syntax invalidates our Dockerfile parsing)
 			// ignore comments
 			continue
 		}
+
+		// handle line continuations
+		// (TODO see note above regarding "escape" parser directive)
+		for line[len(line)-1] == '\\' && scanner.Scan() {
+			nextLine := strings.TrimSpace(scanner.Text())
+			if nextLine == "" || nextLine[0] == '#' {
+				// ignore blank lines and comments
+				continue
+			}
+			line = line[0:len(line)-1]+nextLine
+		}
+
 		fields := strings.Fields(line)
 		if len(fields) < 1 {
+			// must be a much more complex empty line??
 			continue
 		}
-		if strings.ToUpper(fields[0]) == "FROM" {
-			return fields[1], nil
+		instruction := strings.ToUpper(fields[0])
+
+		// TODO balk at ARG / $ in from values
+
+		switch instruction {
+		case "FROM":
+			from := fields[1]
+
+			if stageFrom, ok := meta.StageNameFroms[from]; ok {
+				// if this is a valid stage name, we should resolve it back to the original FROM value of that previous stage (we don't care about inter-stage dependencies for the purposes of either tag dependency calculation or tag building -- just how many there are and what external things they require)
+				from = stageFrom
+			}
+
+			// make sure to add ":latest" if it's implied
+			from = latestizeRepoTag(from)
+
+			meta.StageFroms = append(meta.StageFroms, from)
+			meta.Froms = append(meta.Froms, from)
+
+			if len(fields) == 4 && strings.ToUpper(fields[2]) == "AS" {
+				stageName := fields[3]
+				meta.StageNames = append(meta.StageNames, stageName)
+				meta.StageNameFroms[stageName] = from
+			}
+		case "ADD", "COPY":
+			for _, arg := range fields[1:] {
+				if !strings.HasPrefix(arg, "--") {
+					// doesn't appear to be a "flag"; time to bail!
+					break
+				}
+				if !strings.HasPrefix(arg, "--from=") {
+					// ignore any flags we're not interested in
+					continue
+				}
+				from := arg[len("--from="):]
+
+				if stageFrom, ok := meta.StageNameFroms[from]; ok {
+					// see note above regarding stage names in FROM
+					from = stageFrom
+				} else if stageNumber, err := strconv.Atoi(from); err == nil && stageNumber < len(meta.StageFroms) {
+					// must be a stage number, we should resolve it too
+					from = meta.StageFroms[stageNumber]
+				}
+
+				// make sure to add ":latest" if it's implied
+				from = latestizeRepoTag(from)
+
+				meta.Froms = append(meta.Froms, from)
+			}
 		}
 	}
 	if err := scanner.Err(); err != nil {
-		return "", err
+		return nil, err
 	}
-	return "", nil
+	return meta, nil
 }
 
 func (r Repo) DockerCacheName(entry *manifest.Manifest2822Entry) (string, error) {
@@ -121,26 +209,29 @@ var dockerFromIdCache = map[string]string{
 }
 
 func (r Repo) dockerBuildUniqueBits(entry *manifest.Manifest2822Entry) ([]string, error) {
-	from, err := r.DockerFrom(entry)
+	uniqueBits := []string{
+		entry.ArchGitRepo(arch),
+		entry.ArchGitCommit(arch),
+		entry.ArchDirectory(arch),
+		entry.ArchFile(arch),
+	}
+	meta, err := r.DockerfileMetadata(entry)
 	if err != nil {
 		return nil, err
 	}
-	fromId, ok := dockerFromIdCache[from]
-	if !ok {
-		var err error
-		fromId, err = dockerInspect("{{.Id}}", from)
-		if err != nil {
-			return nil, err
+	for _, from := range meta.Froms {
+		fromId, ok := dockerFromIdCache[from]
+		if !ok {
+			var err error
+			fromId, err = dockerInspect("{{.Id}}", from)
+			if err != nil {
+				return nil, err
+			}
+			dockerFromIdCache[from] = fromId
 		}
-		dockerFromIdCache[from] = fromId
+		uniqueBits = append(uniqueBits, fromId)
 	}
-	return []string{
-		entry.ArchGitRepo(arch),
-		entry.ArchGitCommit(arch),
-		entry.ArchDirectory(arch),
-		entry.ArchFile(arch),
-		fromId,
-	}, nil
+	return uniqueBits, nil
 }
 
 func dockerBuild(tag string, file string, context io.Reader) error {

go/src/bashbrew/sort.go

@@ -101,27 +101,28 @@ func sortRepoObjects(rs []*Repo, applyConstraints bool) ([]*Repo, error) {
 				continue
 			}
 
-			from, err := r.DockerFrom(entry)
+			meta, err := r.DockerfileMetadata(entry)
 			if err != nil {
 				return nil, err
 			}
-			from = latestizeRepoTag(from)
 
-			fromNode, ok := canonicalNodes[from]
-			if !ok {
-				// if our FROM isn't in the list of things we're sorting, it isn't relevant in this context
-				continue
-			}
+			for _, from := range meta.Froms {
+				fromNode, ok := canonicalNodes[from]
+				if !ok {
+					// if our FROM isn't in the list of things we're sorting, it isn't relevant in this context
+					continue
+				}
 
-			// TODO somehow reconcile/avoid "a:a -> b:b, b:b -> a:c" (which will exhibit here as cyclic)
-			for _, tag := range r.Tags("", false, entry) {
-				if tagNode, ok := canonicalNodes[tag]; ok {
-					if tagNode == fromNode {
-						// don't be cyclic
-						continue
-					}
-					if err := network.AddEdge(fromNode, tagNode); err != nil {
-						return nil, err
+				// TODO somehow reconcile/avoid "a:a -> b:b, b:b -> a:c" (which will exhibit here as cyclic)
+				for _, tag := range r.Tags("", false, entry) {
+					if tagNode, ok := canonicalNodes[tag]; ok {
+						if tagNode == fromNode {
+							// don't be cyclic
+							continue
+						}
+						if err := network.AddEdge(fromNode, tagNode); err != nil {
+							return nil, err
+						}
 					}
 				}
 			}