Add DownloadToGopathBin for archived files (#5)

Add DownloadToGopathBin for archived files

🚨 THIS HAS BREAKING CHANGES!

* Move GOPATH related functions to github.com/carolynvs/magex/gopath
* Add package github.com/carolynvs/magex/archive to handle downloading
an archive to your GOPATH/bin directory. It is in it's own package so
that you don't have to add the dependencies to support archives unless
you are using it.

Author: carolynvs

go.mod

@@ -4,6 +4,7 @@ go 1.15
 
 require (
 	github.com/magefile/mage v1.11.0
+	github.com/mholt/archiver/v3 v3.5.0
 	github.com/pkg/errors v0.9.1
 	github.com/stretchr/testify v1.6.1
 )

go.sum

@@ -1,14 +1,38 @@
+github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4=
+github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
+github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
+github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I=
+github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/pgzip v1.2.4 h1:TQ7CNpYKovDOmqzRHKxJh0BeaBI7UdQZYc6p7pMQh1A=
+github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
 github.com/magefile/mage v1.11.0 h1:C/55Ywp9BpgVVclD3lRnSYCwXTYxmSppIgLeDYlNuls=
 github.com/magefile/mage v1.11.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
+github.com/mholt/archiver/v3 v3.5.0 h1:nE8gZIrw66cu4osS/U7UW7YDuGMHssxKutU8IfWxwWE=
+github.com/mholt/archiver/v3 v3.5.0/go.mod h1:qqTTPUK/HZPFgFQ/TJ3BzvTpF/dPtFVJXdQbCmeMxwc=
+github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ=
+github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0=
+github.com/pierrec/lz4/v4 v4.0.3 h1:vNQKSVZNYUEAvRY9FaUXAF1XPbSOHJtDTiP41kzDz2E=
+github.com/pierrec/lz4/v4 v4.0.3/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
+github.com/ulikunitz/xz v0.5.7 h1:YvTNdFzX6+W5m9msiYg/zpkSURPPtOlzbqYjrFn7Yt4=
+github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
+github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=
+github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=

pkg/archive/doc.go

@@ -0,0 +1,6 @@
+// Helper methods for working with archived/compressed files.
+//
+// These functions are separated into their own package to
+// limit the dependencies pulled in when using magex if you
+// are not using archived files.
+package archive

pkg/archive/install.go

@@ -0,0 +1,75 @@
+package archive
+
+import (
+	"log"
+	"os"
+	"path/filepath"
+	"runtime"
+
+	"github.com/carolynvs/magex/pkg/downloads"
+	"github.com/carolynvs/magex/xplat"
+	"github.com/mholt/archiver/v3"
+	_ "github.com/mholt/archiver/v3"
+	"github.com/pkg/errors"
+)
+
+// DownloadArchiveOptions are the set of options available for DownloadToGopathBin.
+type DownloadArchiveOptions struct {
+	downloads.DownloadOptions
+
+	// ArchiveExtensions maps from the GOOS to the expected extension. Required.
+	// For example, windows may use .zip while darwin/linux uses .tgz.
+	ArchiveExtensions map[string]string
+
+	// TargetFileTemplate specifies the path to the target binary in the archive. Required.
+	// Supports the same templating as downloads.DownloadOptions.UrlTemplate.
+	TargetFileTemplate string
+}
+
+// DownloadToGopathBin downloads an archived file to GOPATH/bin.
+func DownloadToGopathBin(opts DownloadArchiveOptions) error {
+	// determine the appropriate file extension based on the OS, e.g. windows gets .zip, otherwise .tgz
+	opts.Ext = opts.ArchiveExtensions[runtime.GOOS]
+	if opts.Ext == "" {
+		return errors.Errorf("no archive file extension was specified for the current GOOS (%s)", runtime.GOOS)
+	}
+
+	if opts.Hook == nil {
+		opts.Hook = ExtractBinaryFromArchiveHook(opts)
+	}
+
+	return downloads.DownloadToGopathBin(opts.DownloadOptions)
+}
+
+// ExtractBinaryFromArchiveHook is the default hook for DownloadToGopathBin.
+func ExtractBinaryFromArchiveHook(opts DownloadArchiveOptions) downloads.PostDownloadHook {
+	return func(archiveFile string) (binPath string, err error) {
+		// Save the binary next to the archive file in the temp directory
+		outDir := filepath.Dir(archiveFile)
+
+		// Render the name of the file in the archive
+		opts.Ext = xplat.FileExt()
+		targetFile, err := downloads.RenderTemplate(opts.TargetFileTemplate, opts.DownloadOptions)
+		if err != nil {
+			return "", errors.Wrapf(err, "error rendering TargetFileTemplate")
+		}
+
+		log.Printf("extracting %s from %s...\n", targetFile, archiveFile)
+
+		// Extract the binary
+		err = archiver.Extract(archiveFile, targetFile, outDir)
+		if err != nil {
+			return "", errors.Wrapf(err, "unable to unpack %s", archiveFile)
+		}
+
+		// The extracted file may be nested depending on its position in the archive
+		binFile := filepath.Join(outDir, targetFile)
+
+		// Check that file was extracted, Extract doesn't error out if you give it a missing targetFile
+		if _, err := os.Stat(binFile); os.IsNotExist(err) {
+			return "", errors.Errorf("could not find %s in the archive", targetFile)
+		}
+
+		return binFile, nil
+	}
+}

pkg/archive/install_example_test.go

@@ -0,0 +1,33 @@
+package archive_test
+
+import (
+	"log"
+
+	"github.com/carolynvs/magex/pkg/archive"
+	"github.com/carolynvs/magex/pkg/downloads"
+	"github.com/carolynvs/magex/pkg/gopath"
+)
+
+func ExampleDownloadToGopathBin() {
+	opts := archive.DownloadArchiveOptions{
+		DownloadOptions: downloads.DownloadOptions{
+			UrlTemplate: "https://get.helm.sh/helm-{{.VERSION}}-{{.GOOS}}-{{.GOARCH}}{{.EXT}}",
+			Name:        "helm",
+			Version:     "v3.5.3",
+		},
+		ArchiveExtensions: map[string]string{
+			"darwin":  ".tar.gz",
+			"linux":   ".tar.gz",
+			"windows": ".zip",
+		},
+		TargetFileTemplate: "{{.GOOS}}-{{.GOARCH}}/helm{{.EXT}}",
+	}
+	err := archive.DownloadToGopathBin(opts)
+	if err != nil {
+		log.Fatal("could not download helm")
+	}
+
+	// Add GOPATH/bin to PATH if necessary so that we can immediately
+	// use the installed tool
+	gopath.EnsureGopathBin()
+}

pkg/archive/install_test.go

@@ -0,0 +1,50 @@
+package archive
+
+import (
+	"os"
+	"os/exec"
+	"runtime"
+	"testing"
+
+	"github.com/carolynvs/magex/pkg/downloads"
+	"github.com/carolynvs/magex/pkg/gopath"
+	"github.com/carolynvs/magex/xplat"
+	"github.com/magefile/mage/mg"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDownloadArchiveToGopathBin(t *testing.T) {
+	os.Setenv(mg.VerboseEnv, "true")
+	err, cleanup := gopath.UseTempGopath()
+	require.NoError(t, err, "Failed to set up a temporary GOPATH")
+	defer cleanup()
+
+	// gh cli unfortunately uses a different archive schema depending on the OS
+	tmpl := "gh_{{.VERSION}}_{{.GOOS}}_{{.GOARCH}}/bin/gh{{.EXT}}"
+	if runtime.GOOS == "windows" {
+		tmpl = "bin/gh.exe"
+	}
+
+	opts := DownloadArchiveOptions{
+		DownloadOptions: downloads.DownloadOptions{
+			UrlTemplate: "https://github.com/cli/cli/releases/download/v{{.VERSION}}/gh_{{.VERSION}}_{{.GOOS}}_{{.GOARCH}}{{.EXT}}",
+			Name:        "gh",
+			Version:     "1.8.1",
+			OsReplacement: map[string]string{
+				"darwin": "macOS",
+			},
+		},
+		ArchiveExtensions: map[string]string{
+			"linux":   ".tar.gz",
+			"darwin":  ".tar.gz",
+			"windows": ".zip",
+		},
+		TargetFileTemplate: tmpl,
+	}
+
+	err = DownloadToGopathBin(opts)
+	require.NoError(t, err)
+
+	_, err = exec.LookPath("gh" + xplat.FileExt())
+	require.NoError(t, err)
+}

pkg/build.go

@@ -0,0 +1,161 @@
+package downloads
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"runtime"
+	"text/template"
+
+	"github.com/carolynvs/magex/pkg/gopath"
+	"github.com/carolynvs/magex/shx"
+	"github.com/carolynvs/magex/xplat"
+	"github.com/pkg/errors"
+)
+
+// PostDownloadHook is the handler called after downloading a file, which returns the absolute path to the binary.
+type PostDownloadHook func(archivePath string) (string, error)
+
+// DownloadOptions
+type DownloadOptions struct {
+	// UrlTemplate is the Go template for the URL to download. Required.
+	// Available Template Variables:
+	//   - {{.GOOS}}
+	//   - {{.GOARCH}}
+	//   - {{.EXT}}
+	//   - {{.VERSION}}
+	UrlTemplate string
+
+	// Name of the binary, excluding OS specific file extension. Required.
+	Name string
+
+	// Version to replace {{.VERSION}} in the URL template. Optional depending on whether or not the version is in the UrlTemplate.
+	Version string
+
+	// Ext to replace {{.EXT}} in the URL template. Optional, defaults to xplat.FileExt().
+	Ext string
+
+	// OsReplacement maps from a GOOS to the os keyword used for the download. Optional, defaults to empty.
+	OsReplacement map[string]string
+
+	// ArchReplacement maps from a GOARCH to the arch keyword used for the download. Optional, defaults to empty.
+	ArchReplacement map[string]string
+
+	// Hook to call after downloading the file.
+	Hook PostDownloadHook
+}
+
+// DownloadToGopathBin takes a Go templated URL and expands template variables
+// - srcTemplate is the URL
+// - version is the version to substitute into the template
+// - ext is the file extension to substitute into the template
+//
+// Template Variables:
+// - {{.GOOS}}
+// - {{.GOARCH}}
+// - {{.EXT}}
+// - {{.VERSION}}
+func DownloadToGopathBin(opts DownloadOptions) error {
+	src, err := RenderTemplate(opts.UrlTemplate, opts)
+	if err != nil {
+		return err
+	}
+	log.Printf("Downloading %s...", src)
+
+	err = gopath.EnsureGopathBin()
+	if err != nil {
+		return err
+	}
+
+	// Download to a temp file
+	tmpDir, err := ioutil.TempDir("", "magex")
+	if err != nil {
+		return errors.Wrap(err, "could not create temporary directory")
+	}
+	defer os.RemoveAll(tmpDir)
+	tmpFile := filepath.Join(tmpDir, filepath.Base(src))
+
+	r, err := http.Get(src)
+	if err != nil {
+		return errors.Wrapf(err, "could not resolve %s", src)
+	}
+	defer r.Body.Close()
+
+	f, err := os.OpenFile(tmpFile, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0755)
+	if err != nil {
+		return errors.Wrapf(err, "could not open %s", tmpFile)
+	}
+	defer f.Close()
+
+	// Download to the temp file
+	_, err = io.Copy(f, r.Body)
+	if err != nil {
+		errors.Wrapf(err, "error downloading %s", src)
+	}
+	f.Close()
+
+	// Call a hook to allow for extracting or modifying the downloaded file
+	var tmpBin = tmpFile
+	if opts.Hook != nil {
+		tmpBin, err = opts.Hook(tmpFile)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Make the binary executable
+	err = os.Chmod(tmpBin, 0755)
+	if err != nil {
+		return errors.Wrapf(err, "could not make %s executable", tmpBin)
+	}
+
+	// Move it to GOPATH/bin
+	dest := filepath.Join(gopath.GetGopathBin(), opts.Name+xplat.FileExt())
+	err = shx.Copy(tmpBin, dest)
+	return errors.Wrapf(err, "error copying %s to %s", tmpBin, dest)
+}
+
+// RenderTemplate takes a Go templated string and expands template variables
+// Available Template Variables:
+// - {{.GOOS}}
+// - {{.GOARCH}}
+// - {{.EXT}}
+// - {{.VERSION}}
+func RenderTemplate(tmplContents string, opts DownloadOptions) (string, error) {
+	tmpl, err := template.New("url").Parse(tmplContents)
+	if err != nil {
+		return "", errors.Wrapf(err, "error parsing %s as a Go template", opts.UrlTemplate)
+	}
+
+	srcData := struct {
+		GOOS    string
+		GOARCH  string
+		EXT     string
+		VERSION string
+	}{
+		GOOS:    runtime.GOOS,
+		GOARCH:  runtime.GOARCH,
+		EXT:     opts.Ext,
+		VERSION: opts.Version,
+	}
+
+	if overrideGoos, ok := opts.OsReplacement[runtime.GOOS]; ok {
+		srcData.GOOS = overrideGoos
+	}
+
+	if overrideGoarch, ok := opts.ArchReplacement[runtime.GOARCH]; ok {
+		srcData.GOARCH = overrideGoarch
+	}
+
+	buf := &bytes.Buffer{}
+	err = tmpl.Execute(buf, srcData)
+	if err != nil {
+		return "", errors.Wrapf(err, "error rendering %s as a Go template with data: %#v", opts.UrlTemplate, srcData)
+	}
+
+	return buf.String(), nil
+}