Making sure that plugin-manager works in disconnected env with local crane-plugins (#51)

Author: JaydipGabani

cmd/plugin-manager/add/add.go

@@ -171,13 +171,24 @@ func (o *Options) run(args []string) error {
 }
 
 func downloadBinary(filepath string, filename string, url string, log *logrus.Logger) error {
-
-	// Get the data
-	resp, err := http.Get(url)
-	if err != nil {
-		return err
+	var binaryContents io.Reader
+	isUrl, url := plugin.IsUrl(url)
+	if !isUrl {
+		srcPlugin, err := os.Open(url)
+		if err != nil {
+			return err
+		}
+		defer srcPlugin.Close()
+		binaryContents = srcPlugin
+	} else {
+		// Get the data
+		resp, err := http.Get(url)
+		if err != nil {
+			return err
+		}
+		defer resp.Body.Close()
+		binaryContents = resp.Body
 	}
-	defer resp.Body.Close()
 	// Create dir if not exists
 	if _, err := os.Stat(filepath); os.IsNotExist(err) {
 		err = os.MkdirAll(filepath, os.ModePerm)
@@ -187,20 +198,25 @@ func downloadBinary(filepath string, filename string, url string, log *logrus.Lo
 	}
 
 	// Create the file
-	out, err := os.OpenFile(filepath+"/"+filename, syscall.O_RDWR|syscall.O_CREAT|syscall.O_TRUNC, 0777)
+	plugin, err := os.OpenFile(filepath+"/"+filename, syscall.O_RDWR|syscall.O_CREAT|syscall.O_TRUNC, 0777)
 	if err != nil {
 		return err
 	}
-	defer out.Close()
+	defer plugin.Close()
 
 	// Write the body to filePluginDir
-	_, err = io.Copy(out, resp.Body)
-	if err == nil {
-		log.Infof("plugin %s added to the path - %s", filename, filepath)
+	_, err = io.Copy(plugin, binaryContents)
+	if err != nil {
+		return err
 	}
+	err = plugin.Sync()
+	if err != nil {
+		return err
+	}
+	log.Infof("plugin %s added to the path - %s", filename, filepath)
 	return err
 }
 
 func (o *Options) ManagedPluginDir() string {
 	return fmt.Sprintf("%v/%v", o.PluginDir, plugin.MANAGED_DIR)
-}
+}

go.mod

@@ -4,6 +4,7 @@ go 1.16
 
 require (
 	github.com/ghodss/yaml v1.0.0
+	github.com/jarcoal/httpmock v1.0.8
 	github.com/konveyor/crane-lib v0.0.4
 	github.com/mitchellh/mapstructure v1.4.1
 	github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5
@@ -12,6 +13,7 @@ require (
 	github.com/spf13/cobra v1.2.1
 	github.com/spf13/viper v1.8.1
 	github.com/vmware-tanzu/velero v1.6.3
+	gotest.tools v2.2.0+incompatible
 	k8s.io/api v0.22.2
 	k8s.io/apimachinery v0.22.2
 	k8s.io/cli-runtime v0.22.2

go.sum

@@ -430,6 +430,8 @@ github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
 github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jarcoal/httpmock v1.0.8 h1:8kI16SoO6LQKgPE7PvQuV+YuD/inwHd7fOOe2zMbo4k=
+github.com/jarcoal/httpmock v1.0.8/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jimstudt/http-authentication v0.0.0-20140401203705-3eca13d6893a/go.mod h1:wK6yTYYcgjHE1Z1QtXACPDjcFJyBskHEdagmnq3vsP8=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
@@ -1200,6 +1202,7 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=

internal/plugin/plugin_manager_helper.go

@@ -8,6 +8,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"io/ioutil"
 	"net/http"
+	"net/url"
 	"os"
 	"runtime"
 	"strings"
@@ -86,45 +87,33 @@ func BuildManifestMap(log *logrus.Logger, name string, repoName string) (map[str
 }
 
 // takes url as input and returns index.yml for plugin repository
-func GetYamlFromUrl(url string) (map[string]interface{}, error) {
-	res, err := http.Get(url)
+func GetYamlFromUrl(URL string) (map[string]interface{}, error) {
+	var manifest map[string]interface{}
+	index, err := getData(URL)
 	if err != nil {
 		return nil, err
 	}
-
-	defer res.Body.Close()
-
-	body, err := ioutil.ReadAll(res.Body)
-
+	err = yaml.Unmarshal(index, &manifest)
 	if err != nil {
 		return nil, err
 	}
-	var manifest map[string]interface{}
-	err = yaml.Unmarshal(body, &manifest)
-	if err != nil {
-		panic(err)
-	}
 	return manifest, nil
 }
 
 // takes url as input and fetches the manifest of a plugin
-func YamlToManifest(url string) (Manifest, error) {
+func YamlToManifest(URL string) (Manifest, error) {
 	plugin := Manifest{}
-	res, err := http.Get(url)
-	if err != nil {
-		return plugin, err
-	}
 
-	defer res.Body.Close()
-
-	body, err := ioutil.ReadAll(res.Body)
+	body, err := getData(URL)
 	if err != nil {
 		return plugin, err
 	}
+
 	err = yaml.Unmarshal(body, &plugin)
 	if err != nil {
 		return Manifest{}, err
 	}
+
 	isPluginAvailable := FilterPluginForOsArch(&plugin)
 	if isPluginAvailable {
 		return plugin, nil
@@ -180,3 +169,34 @@ func LocateBinaryInPluginDir(pluginDir string, name string, files []os.FileInfo)
 	}
 	return paths, nil
 }
+
+func IsUrl(URL string) (bool, string) {
+	URL = strings.TrimPrefix(URL, "file://")
+	u, err := url.Parse(URL)
+	return err == nil && u.Scheme != "" && u.Host != "", URL
+}
+
+func getData(URL string) ([]byte, error) {
+	var index []byte
+	var err error
+	isUrl, URL := IsUrl(URL)
+	if !isUrl {
+		index, err = ioutil.ReadFile(URL)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		res, err := http.Get(URL)
+		if err != nil {
+			return nil, err
+		}
+
+		defer res.Body.Close()
+
+		index, err = ioutil.ReadAll(res.Body)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return index, nil
+}

internal/plugin/plugin_manager_helper_test.go

@@ -0,0 +1,121 @@
+package plugin
+
+import (
+	"github.com/ghodss/yaml"
+	"github.com/jarcoal/httpmock"
+	"gotest.tools/assert"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"testing"
+)
+
+func TestGetYamlFromUrlWithUrl(t *testing.T) {
+	URL := "https://test.com/index.yml"
+	httpmock.Activate()
+	defer httpmock.DeactivateAndReset()
+	index := map[string]interface{}{
+		"foo-0.0.1": "https://test.com/foo-0.0.1.yml",
+	}
+	httpmock.RegisterResponder("GET", URL,
+		func(req *http.Request) (*http.Response, error) {
+			// Get ID from request
+			return httpmock.NewJsonResponse(200, index)
+		},
+	)
+
+	resp, _ := GetYamlFromUrl(URL)
+	assert.DeepEqual(t, index, resp)
+}
+
+func TestYamlToManifestWithUrl(t *testing.T) {
+	URL := "https://test.com/foo-0.0.1.yml"
+	httpmock.Activate()
+	defer httpmock.DeactivateAndReset()
+	index := Manifest{
+		Name:             "foo",
+		Version:          "0.0.1",
+		Description:      "Description of foo plugin",
+		ShortDescription: "Short description of foo plugin",
+		Binaries: []Binary{
+			{
+				OS:   "linux",
+				Arch: "amd64",
+				URI:  "https://test.com/download/foo",
+			},
+		},
+	}
+	httpmock.RegisterResponder("GET", URL,
+		func(req *http.Request) (*http.Response, error) {
+			// Get ID from request
+			return httpmock.NewJsonResponse(200, index)
+		},
+	)
+
+	resp, _ := YamlToManifest(URL)
+	assert.DeepEqual(t, index, resp)
+}
+
+func TestGetYamlFromUrlWithFile(t *testing.T) {
+	index := map[string]interface{}{
+		"foo-0.0.1": "file://tmp/foo-0.0.1.yml",
+	}
+	tempFile, err := ioutil.TempFile(os.TempDir(), "index.yml")
+	if err != nil {
+		panic(err)
+	}
+	defer os.Remove(tempFile.Name())
+
+	data, err := yaml.Marshal(&index)
+	if err != nil {
+		panic(err)
+	}
+
+	_, err = tempFile.Write(data)
+	if err != nil {
+		panic(err)
+	}
+
+	resp, _ := GetYamlFromUrl(tempFile.Name())
+	assert.DeepEqual(t, index, resp)
+
+	resp, _ = GetYamlFromUrl("file://" + tempFile.Name())
+	assert.DeepEqual(t, index, resp)
+}
+
+func TestYamlToManifestWithFile(t *testing.T) {
+	plugin := Manifest {
+		Name:             "foo",
+		Version:          "0.0.1",
+		Description:      "Description of foo plugin",
+		ShortDescription: "Short description of foo plugin",
+		Binaries: []Binary{
+			{
+				OS:   "linux",
+				Arch: "amd64",
+				URI:  "https://test.com/download/foo",
+			},
+		},
+	}
+	tempFile, err := ioutil.TempFile(os.TempDir(), "index.yml")
+	if err != nil {
+		panic(err)
+	}
+	defer os.Remove(tempFile.Name())
+
+	data, err := yaml.Marshal(&plugin)
+	if err != nil {
+		panic(err)
+	}
+
+	_, err = tempFile.Write(data)
+	if err != nil {
+		panic(err)
+	}
+
+	resp, _ := YamlToManifest(tempFile.Name())
+	assert.DeepEqual(t, plugin, resp)
+
+	resp, _ = YamlToManifest("file://" + tempFile.Name())
+	assert.DeepEqual(t, plugin, resp)
+}