estimate: Use the default GOPATH to cache downloads

This avoids re-downloading the same modules (potentially hundreds of
megabytes for moderately sized binaries) over and over.  The downside
is wasted disk space that the user must manually clear (`go clean
-modcache`).

Why keeping the downloaded modules is believed to be more useful than
automatically deleting them:

  * It makes offline use possible.
  * Packaging a complex module with lots of unpackaged dependencies
    means a long slow process of chipping away at the dependencies.
    It would be nice to be able to re-run `dh-make-golang estimate`
    several times to track progress and plan next steps without
    incurring a huge download cost.
  * Developing `dh-make-golang estimate` itself involves re-running it
    over and over to test changes.  The interesting modules to test
    are the complex ones, and those take a while to download.
  * I believe that `golang-*-dev` package maintainers are already
    accustomed to running `go clean -modcache`, an occasional
    necessity with serious Go development.
  * The modules don't actually need to be fully downloaded and
    unpacked; only their `go.mod` files are necessary to construct a
    dependency graph.  Changing `dh-make-golang estimate` in a future
    commit to only download just `go.mod` would address disk usage
    concerns.  It would also mean that re-downloading over and over
    would be less of a problem, but keeping the `go.mod` files avoids
    abusing proxy.golang.org and sum.golang.org.

This change makes `progressSize` not useful, so `go get`'s stdout and
stderr are passed through, and `-x` is passed to `go get` to get some
progress indication.

Author: rhansen

estimate.go

@@ -55,10 +55,9 @@ func getSourcesInNew() (map[string]string, error) {
 	return sourcesInNew, nil
 }
 
-func get(gopath, repodir, repo, rev string) error {
+func get(repodir, repo, rev string) error {
 	done := make(chan struct{})
 	defer close(done)
-	go progressSize("go get", gopath, done)
 
 	// As per https://groups.google.com/forum/#!topic/golang-nuts/N5apfenE4m4,
 	// the arguments to “go get” are packages, not repositories. Hence, we
@@ -71,43 +70,31 @@ func get(gopath, repodir, repo, rev string) error {
 	if rev != "" {
 		packages += "@" + rev
 	}
-	cmd := exec.Command("go", "get", "-t", packages)
+	cmd := exec.Command("go", "get", "-x", "-t", packages)
 
-	out := bytes.Buffer{}
 	cmd.Dir = repodir
-	cmd.Stderr = &out
-	cmd.Env = append([]string{
-		"GOPATH=" + gopath,
-	}, passthroughEnv()...)
-	err := cmd.Run()
-	if err != nil {
-		fmt.Fprint(os.Stderr, "\n", out.String())
-	}
-	return err
+	cmd.Stderr = os.Stderr
+	cmd.Stdout = os.Stdout
+	return cmd.Run()
 }
 
-// getModuleDir returns the path of the directory containing a module for the
-// given GOPATH and repository dir values.
-func getModuleDir(gopath, repodir, module string) (string, error) {
+// getModuleDir returns the path of the directory containing a module for the given repository dir.
+func getModuleDir(repodir, module string) (string, error) {
 	cmd := exec.Command("go", "list", "-m", "-f", "{{.Dir}}", module)
 	cmd.Dir = repodir
 	cmd.Stderr = os.Stderr
-	cmd.Env = append([]string{
-		"GOPATH=" + gopath,
-	}, passthroughEnv()...)
 	out, err := cmd.Output()
 	if err != nil {
 		return "", fmt.Errorf("go list: args: %v; error: %w", cmd.Args, err)
 	}
 	return string(bytes.TrimSpace(out)), nil
 }
 
-// getDirectDependencies returns a set of all the direct dependencies of a
-// module for the given GOPATH and repository dir values. It first finds the
-// directory that contains this module, then uses go list in this directory
-// to get its direct dependencies.
-func getDirectDependencies(gopath, repodir, module string) (map[string]bool, error) {
-	dir, err := getModuleDir(gopath, repodir, module)
+// getDirectDependencies returns a set of all the direct dependencies of a module for the given
+// repository dir. It first finds the directory that contains this module, then uses go list in this
+// directory to get its direct dependencies.
+func getDirectDependencies(repodir, module string) (map[string]bool, error) {
+	dir, err := getModuleDir(repodir, module)
 	if err != nil {
 		return nil, fmt.Errorf("get module dir: %w", err)
 	}
@@ -120,9 +107,6 @@ func getDirectDependencies(gopath, repodir, module string) (map[string]bool, err
 	cmd := exec.Command("go", "list", "-m", "-f", "{{if not .Indirect}}{{.Path}}{{end}}", "all")
 	cmd.Dir = dir
 	cmd.Stderr = os.Stderr
-	cmd.Env = append([]string{
-		"GOPATH=" + gopath,
-	}, passthroughEnv()...)
 	out, err := cmd.Output()
 	if err != nil {
 		return nil, fmt.Errorf("go list: args: %v; error: %w", cmd.Args, err)
@@ -207,13 +191,7 @@ func estimate(importpath, revision string) error {
 		}
 	}
 
-	// construct a separate GOPATH in a temporary directory
-	gopath, err := os.MkdirTemp("", "dh-make-golang")
-	if err != nil {
-		return fmt.Errorf("create temp dir: %w", err)
-	}
-	defer removeTemp(gopath)
-	// second temporary directosy for the repo sources
+	// temporary directory for the repo sources
 	repodir, err := os.MkdirTemp("", "dh-make-golang")
 	if err != nil {
 		return fmt.Errorf("create temp dir: %w", err)
@@ -226,7 +204,7 @@ func estimate(importpath, revision string) error {
 		return fmt.Errorf("create dummymod: %w", err)
 	}
 
-	if err := get(gopath, repodir, importpath, revision); err != nil {
+	if err := get(repodir, importpath, revision); err != nil {
 		return fmt.Errorf("go get: %w", err)
 	}
 
@@ -237,7 +215,7 @@ func estimate(importpath, revision string) error {
 
 	if found {
 		// Fetch un-vendored dependencies
-		if err := get(gopath, repodir, importpath, revision); err != nil {
+		if err := get(repodir, importpath, revision); err != nil {
 			return fmt.Errorf("fetch un-vendored: go get: %w", err)
 		}
 	}
@@ -246,16 +224,13 @@ func estimate(importpath, revision string) error {
 	cmd := exec.Command("go", "mod", "graph")
 	cmd.Dir = repodir
 	cmd.Stderr = os.Stderr
-	cmd.Env = append([]string{
-		"GOPATH=" + gopath,
-	}, passthroughEnv()...)
 	out, err := cmd.Output()
 	if err != nil {
 		return fmt.Errorf("go mod graph: args: %v; error: %w", cmd.Args, err)
 	}
 
 	// Get direct dependencies, to filter out indirect ones from go mod graph output
-	directDeps, err := getDirectDependencies(gopath, repodir, importpath)
+	directDeps, err := getDirectDependencies(repodir, importpath)
 	if err != nil {
 		return fmt.Errorf("get direct dependencies: %w", err)
 	}