add Jsonnet importer

Author: svenwltr

Gopkg.lock

@@ -68,4 +68,4 @@
 
 [[constraint]]
   name = "github.com/google/go-jsonnet"
-  version = "^0.10.0"
+  version = "^0.11.2"

Gopkg.toml

@@ -33,6 +33,10 @@ func (app *App) decode(files []gh.File, vars templates.Variables) ([]runtime.Obj
 	var objects []runtime.Object
 
 	for _, file := range files {
+		log.WithFields(log.Fields{
+			"Location": file.Location,
+		}).Debug("decoding file")
+
 		switch {
 		case strings.HasSuffix(file.Name(), ".yml"):
 			fallthrough
@@ -97,19 +101,15 @@ func (app *App) decodeYAML(file gh.File, vars templates.Variables) ([]runtime.Ob
 func (app *App) decodeJsonnet(file gh.File, vars templates.Variables, all []gh.File) ([]runtime.Object, error) {
 	var objects []runtime.Object
 
-	importer := new(jsonnet.MemoryImporter)
-	importer.Data = make(map[string]string)
-	for _, f := range all {
-		importer.Data[f.Name()] = f.Content
-	}
+	importer := gh.NewJsonnetImporter(app.Clients.GitHub)
 
 	vm := jsonnet.MakeVM()
 	vm.Importer(importer)
 	for k, v := range vars {
 		vm.ExtVar(k, v)
 	}
 
-	docs, err := vm.EvaluateSnippetStream(file.Name(), file.Content)
+	docs, err := vm.EvaluateSnippetStream(file.Location.String(), file.Content)
 	if err != nil {
 		return nil, errors.WithStack(err)
 	}

pkg/api/render.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 
 	"github.com/rebuy-de/kubernetes-deployment/pkg/gh"
+	fakeGH "github.com/rebuy-de/kubernetes-deployment/pkg/gh/fake"
 	"github.com/rebuy-de/kubernetes-deployment/pkg/interceptors"
 	"github.com/rebuy-de/kubernetes-deployment/pkg/templates"
 	"github.com/rebuy-de/rebuy-go-sdk/testutil"
@@ -73,18 +74,43 @@ func TestDecode(t *testing.T) {
 
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
-			files := []gh.File{}
+			files := fakeGH.Files{}
 			for _, fname := range tc.files {
 				files = append(files, gh.File{
-					Path:    fname,
+					Location: &gh.Location{
+						Owner: "rebuy-de",
+						Repo:  "test",
+						Path:  fname,
+						Ref:   "master",
+					},
 					Content: readFile(t, path.Join("test-fixtures", fname)),
 				})
 			}
 
-			app := App{Interceptors: &interceptors.Multi{}}
+			github := &fakeGH.GitHub{
+				"rebuy-de": fakeGH.Repos{
+					"test": fakeGH.Branches{
+						"master": fakeGH.Branch{
+							Meta: gh.Branch{
+								Name: "master",
+								SHA:  "5a5369823a2a9a6ad9c241b404be39f802d41d48",
+							},
+							Files: files,
+						},
+					},
+				},
+			}
+
+			app := App{
+				Interceptors: &interceptors.Multi{},
+				Clients: &Clients{
+					GitHub: github,
+				},
+			}
+
 			objects, err := app.decode(files, vars)
 			if err != nil {
-				t.Fatal(err)
+				t.Fatalf("%+v", err)
 			}
 			g := path.Join("test-fixtures", fmt.Sprintf("render-golden-%s.json", tc.name))
 			testutil.AssertGoldenJSON(t, g, objects)

pkg/api/render_test.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"net/http"
 	"path"
-	"regexp"
 	"strings"
 	"time"
 
@@ -20,10 +19,6 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-var (
-	ContentLocationRE = regexp.MustCompile(`^github.com/([^/]+)/([^/]+)/(.*)$`)
-)
-
 type Interface interface {
 	GetBranch(location *Location) (*Branch, error)
 	GetFile(location *Location) (File, error)
@@ -186,7 +181,7 @@ func (gh *API) GetFile(location *Location) (File, error) {
 
 	gh.statsd.Gauge("github.rate.remaining", resp.Rate.Remaining)
 
-	return File{location.Path, content}, nil
+	return File{Location: location, Content: content}, nil
 }
 
 func (gh *API) GetFiles(location *Location) ([]File, error) {
@@ -244,11 +239,16 @@ func (gh *API) GetFiles(location *Location) ([]File, error) {
 			_, notAFile := err.(ErrNotAFile)
 			if !notAFile {
 				return nil, errors.Wrapf(err,
-					"unable to decode file '%v'",
+					"unable to download file '%v'",
 					location)
 			}
 
+			log.WithFields(log.Fields{
+				"Location": file.Location,
+			}).Debug("skipping path, because it is not a file")
+			continue
 		}
+
 		files = append(files, file)
 	}
 

pkg/gh/client.go

@@ -57,18 +57,18 @@ func (d *GitHub) GetBranch(l *gh.Location) (*gh.Branch, error) {
 
 func (d *GitHub) GetFile(l *gh.Location) (gh.File, error) {
 	for _, file := range (*d)[l.Owner][l.Repo][l.Ref].Files {
-		if file.Path == l.Path {
+		if file.Location.Path == l.Path {
 			return file, nil
 		}
 	}
-	return gh.File{}, nil
+	return gh.File{}, fmt.Errorf("File %s not found", l.String())
 }
 
 func (d *GitHub) GetFiles(l *gh.Location) ([]gh.File, error) {
 	var files []gh.File
 
 	for _, file := range (*d)[l.Owner][l.Repo][l.Ref].Files {
-		dir, _ := path.Split("/" + file.Path)
+		dir, _ := path.Split("/" + file.Location.Path)
 		if path.Clean("/"+dir+"/") == path.Clean("/"+l.Path+"/") {
 			files = append(files, file)
 		}

pkg/gh/fake/impl.go

@@ -22,10 +22,10 @@ var (
 				"master": Branch{
 					Meta: ExampleBranch,
 					Files: Files{
-						{Path: "deployments.yaml", Content: YAML([]string{"foo", "bar"})},
-						{Path: "README.md", Content: "blubber"},
-						{Path: "sub/foo.txt", Content: "bar"},
-						{Path: "sub/bim.txt", Content: "baz"},
+						{Location: &gh.Location{Path: "deployments.yaml"}, Content: YAML([]string{"foo", "bar"})},
+						{Location: &gh.Location{Path: "README.md"}, Content: "blubber"},
+						{Location: &gh.Location{Path: "sub/foo.txt"}, Content: "bar"},
+						{Location: &gh.Location{Path: "sub/bim.txt"}, Content: "baz"},
 					},
 				},
 			},
@@ -89,8 +89,8 @@ func TestGetFiles(t *testing.T) {
 	}
 
 	expected := []gh.File{
-		{Path: "deployments.yaml", Content: "- foo\n- bar\n"},
-		{Path: "README.md", Content: "blubber"},
+		{Location: &gh.Location{Path: "deployments.yaml"}, Content: "- foo\n- bar\n"},
+		{Location: &gh.Location{Path: "README.md"}, Content: "blubber"},
 	}
 
 	if !reflect.DeepEqual(files, expected) {
@@ -107,8 +107,8 @@ func TestGetSubdirectoryFiles(t *testing.T) {
 	}
 
 	expected := []gh.File{
-		{Path: "sub/foo.txt", Content: "bar"},
-		{Path: "sub/bim.txt", Content: "baz"},
+		{Location: &gh.Location{Path: "sub/foo.txt"}, Content: "bar"},
+		{Location: &gh.Location{Path: "sub/bim.txt"}, Content: "baz"},
 	}
 
 	if !reflect.DeepEqual(files, expected) {

pkg/gh/fake/package_test.go

@@ -36,7 +36,7 @@ func ScanFiles(root string) Files {
 		}
 
 		raw, err := ioutil.ReadFile(path)
-		files = append(files, gh.File{Path: relPath, Content: string(raw)})
+		files = append(files, gh.File{Location: &gh.Location{Path: relPath}, Content: string(raw)})
 		return err
 	})
 	if err != nil {

pkg/gh/fake/utils.go

@@ -3,11 +3,11 @@ package gh
 import "path"
 
 type File struct {
-	Path    string
-	Content string
+	Location *Location
+	Content  string
 }
 
 func (f *File) Name() string {
-	_, name := path.Split(f.Path)
+	_, name := path.Split(f.Location.Path)
 	return name
 }

pkg/gh/file.go

@@ -0,0 +1,61 @@
+package gh
+
+import (
+	"path"
+	"path/filepath"
+	"strings"
+
+	jsonnet "github.com/google/go-jsonnet"
+)
+
+type jsonnetImporter struct {
+	client Interface
+}
+
+func NewJsonnetImporter(client Interface) jsonnet.Importer {
+	return &jsonnetImporter{
+		client: client,
+	}
+}
+
+func (i *jsonnetImporter) Import(importedFrom, importedPath string) (contents jsonnet.Contents, foundAt string, err error) {
+	fromLocation, err := NewLocation(importedFrom)
+	if err != nil {
+		return jsonnet.MakeContents(""), "", err
+	}
+
+	var location *Location
+	if strings.HasPrefix(importedPath, "github.com/") {
+		// location constrution for absolute paths
+		location, err = NewLocation(importedPath)
+		if err != nil {
+			return jsonnet.MakeContents(""), "", err
+		}
+	} else {
+		// location constrution for relative paths
+		fromDir := filepath.Dir(fromLocation.String())
+		absolute := path.Clean(path.Join(fromDir, importedPath))
+
+		location, err = NewLocation(absolute)
+		if err != nil {
+			return jsonnet.MakeContents(""), "", err
+		}
+	}
+
+	// having no ref means same ref, if inside same repo and "master" if it is in another repo
+	if location.Ref == "" {
+		if location.Owner == fromLocation.Owner && location.Repo == fromLocation.Repo {
+			location.Ref = fromLocation.Ref
+		} else {
+			location.Ref = "master"
+		}
+
+	}
+
+	file, err := i.client.GetFile(location)
+	if err != nil {
+		return jsonnet.MakeContents(""), "", err
+	}
+
+	return jsonnet.MakeContents(file.Content), location.String(), nil
+}