Merge pull request #98 from rebuy-de/jsonnet

add Jsonnet support

Author: svenwltr

Gopkg.lock

@@ -65,3 +65,7 @@
 [[constraint]]
   branch = "master"
   name = "github.com/benbjohnson/clock"
+
+[[constraint]]
+  name = "github.com/google/go-jsonnet"
+  version = "^0.10.0"

Gopkg.toml

@@ -2,6 +2,10 @@ package api
 
 import (
 	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+
 	"github.com/rebuy-de/kubernetes-deployment/pkg/gh"
 	"github.com/rebuy-de/kubernetes-deployment/pkg/interceptors"
 	"github.com/rebuy-de/kubernetes-deployment/pkg/interceptors/annotater"
@@ -13,9 +17,6 @@ import (
 	"github.com/rebuy-de/kubernetes-deployment/pkg/kubectl"
 	"github.com/rebuy-de/kubernetes-deployment/pkg/settings"
 	"github.com/rebuy-de/kubernetes-deployment/pkg/statsdw"
-	log "github.com/sirupsen/logrus"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/clientcmd"
 )
 
 type Clients struct {

pkg/api/app.go

@@ -1,15 +1,17 @@
 package api
 
 import (
-	"github.com/rebuy-de/kubernetes-deployment/pkg/gh"
 	"regexp"
 	"strings"
 
+	jsonnet "github.com/google/go-jsonnet"
 	"github.com/pkg/errors"
-	"github.com/rebuy-de/kubernetes-deployment/pkg/templates"
 	log "github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/kubernetes/scheme"
+
+	"github.com/rebuy-de/kubernetes-deployment/pkg/gh"
+	"github.com/rebuy-de/kubernetes-deployment/pkg/templates"
 )
 
 func (app *App) Render(fetched *FetchResult) ([]runtime.Object, error) {
@@ -24,48 +26,108 @@ func (app *App) Render(fetched *FetchResult) ([]runtime.Object, error) {
 		"Values": fetched.Service.Variables,
 	}).Debug("collected template values")
 
-	rendered, err := templates.RenderAll(fetched.Templates, fetched.Service.Variables)
-	if err != nil {
-		return nil, errors.WithStack(err)
+	return app.decode(fetched.Templates, fetched.Service.Variables)
+}
+
+func (app *App) decode(files []gh.File, vars templates.Variables) ([]runtime.Object, error) {
+	var objects []runtime.Object
+
+	for _, file := range files {
+		switch {
+		case strings.HasSuffix(file.Name(), ".yml"):
+			fallthrough
+
+		case strings.HasSuffix(file.Name(), ".yaml"):
+			objs, err := app.decodeYAML(file, vars)
+			if err != nil {
+				return nil, errors.WithStack(err)
+			}
+			objects = append(objects, objs...)
+
+		case strings.HasSuffix(file.Name(), ".jsonnet"):
+			objs, err := app.decodeJsonnet(file, vars, files)
+			if err != nil {
+				return nil, errors.WithStack(err)
+			}
+			objects = append(objects, objs...)
+
+		default:
+			log.WithFields(log.Fields{
+				"Name": file.Name(),
+			}).Debug("Ignoring file with wrong extension.")
+		}
 	}
 
-	return app.decode(rendered)
+	return objects, nil
 }
 
-func (app *App) decode(files []gh.File) ([]runtime.Object, error) {
+func (app *App) decodeYAML(file gh.File, vars templates.Variables) ([]runtime.Object, error) {
+	rendered, err := templates.Render(file.Content, vars)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
 	var objects []runtime.Object
+	splitted := regexp.MustCompile("[\n\r]---").Split(rendered, -1)
 	decode := scheme.Codecs.UniversalDeserializer().Decode
 
-	for _, file := range files {
-		if !strings.HasSuffix(file.Name(), ".yaml") && !strings.HasSuffix(file.Name(), ".yml") {
+	for _, part := range splitted {
+		if strings.TrimSpace(part) == "" {
 			log.WithFields(log.Fields{
 				"Name": file.Name(),
-			}).Debug("Ignoring file with wrong extension.")
+			}).Debug("Ignoring empty file.")
 			continue
 		}
 
-		splitted := regexp.MustCompile("[\n\r]---").Split(file.Content, -1)
+		obj, _, err := decode([]byte(part), nil, nil)
+		if err != nil {
+			return nil, errors.Wrapf(err, "unable to decode file '%s'", file.Name())
+		}
 
-		for _, part := range splitted {
-			if strings.TrimSpace(part) == "" {
-				log.WithFields(log.Fields{
-					"Name": file.Name(),
-				}).Debug("Ignoring empty file.")
-				continue
-			}
+		obj, err = app.Interceptors.PostManifestRender(obj)
+		if err != nil {
+			return nil, errors.WithStack(err)
+		}
 
-			obj, _, err := decode([]byte(part), nil, nil)
-			if err != nil {
-				return nil, errors.Wrapf(err, "unable to decode file '%s'", file.Name())
-			}
+		objects = append(objects, obj)
+	}
 
-			obj, err = app.Interceptors.PostManifestRender(obj)
-			if err != nil {
-				return nil, errors.WithStack(err)
-			}
+	return objects, nil
+}
 
-			objects = append(objects, obj)
+func (app *App) decodeJsonnet(file gh.File, vars templates.Variables, all []gh.File) ([]runtime.Object, error) {
+	var objects []runtime.Object
+	decode := scheme.Codecs.UniversalDeserializer().Decode
+
+	importer := new(jsonnet.MemoryImporter)
+	importer.Data = make(map[string]string)
+	for _, f := range all {
+		importer.Data[f.Name()] = f.Content
+	}
+
+	vm := jsonnet.MakeVM()
+	vm.Importer(importer)
+	for k, v := range vars {
+		vm.ExtVar(k, v)
+	}
+
+	docs, err := vm.EvaluateSnippetStream(file.Name(), file.Content)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	for _, doc := range docs {
+		obj, _, err := decode([]byte(doc), nil, nil)
+		if err != nil {
+			return nil, errors.Wrapf(err, "unable to decode file '%s'", file.Name())
 		}
+
+		obj, err = app.Interceptors.PostManifestRender(obj)
+		if err != nil {
+			return nil, errors.WithStack(err)
+		}
+
+		objects = append(objects, obj)
 	}
 
 	return objects, nil

pkg/api/render.go

@@ -1,12 +1,14 @@
 package api
 
 import (
+	"fmt"
 	"io/ioutil"
 	"path"
 	"testing"
 
 	"github.com/rebuy-de/kubernetes-deployment/pkg/gh"
 	"github.com/rebuy-de/kubernetes-deployment/pkg/interceptors"
+	"github.com/rebuy-de/kubernetes-deployment/pkg/templates"
 	"github.com/rebuy-de/rebuy-go-sdk/testutil"
 )
 
@@ -19,24 +21,61 @@ func readFile(t *testing.T, path string) string {
 }
 
 func TestDecode(t *testing.T) {
-	dir := "test-fixtures"
-	tcs := []gh.File{
+	cases := []struct {
+		name  string
+		files []string
+	}{
 		{
-			Path:    "manifest-deployment.yaml",
-			Content: readFile(t, path.Join(dir, "manifest-deployment.yaml")),
+			name: "multi-yaml",
+			files: []string{
+				"render-deployment.yaml",
+				"render-podpreset.yaml",
+			},
 		},
 		{
-			Path:    "manifest-podpreset.yaml",
-			Content: readFile(t, path.Join(dir, "manifest-podpreset.yaml")),
+			name: "simple-jsonnet",
+			files: []string{
+				"render-pdb.jsonnet",
+			},
+		},
+		{
+			name: "complex-jsonnet",
+			files: []string{
+				"render-job.jsonnet",
+			},
+		},
+		{
+			name: "local-import-jsonnet",
+			files: []string{
+				"render-silo.jsonnet",
+				"render-silo.libsonnet",
+			},
 		},
 	}
-	golden := "decoded-golden.yaml"
 
-	app := App{Interceptors: &interceptors.Multi{}}
-	objects, err := app.decode(tcs)
-	if err != nil {
-		t.Fatal(err)
+	vars := templates.Variables{
+		"testString":  "bish-bash-bosh",
+		"gitCommitID": "ffffff",
+		"clusterName": "staging",
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			files := []gh.File{}
+			for _, fname := range tc.files {
+				files = append(files, gh.File{
+					Path:    fname,
+					Content: readFile(t, path.Join("test-fixtures", fname)),
+				})
+			}
+
+			app := App{Interceptors: &interceptors.Multi{}}
+			objects, err := app.decode(files, vars)
+			if err != nil {
+				t.Fatal(err)
+			}
+			g := path.Join("test-fixtures", fmt.Sprintf("render-golden-%s.json", tc.name))
+			testutil.AssertGoldenJSON(t, g, objects)
+		})
 	}
-	g := path.Join(dir, golden)
-	testutil.AssertGoldenJSON(t, g, objects)
 }

pkg/api/render_test.go

@@ -7,6 +7,7 @@ metadata:
   labels:
     team: platform
     app: maintenance
+    test: {{.testString}}
 
 spec:
   revisionHistoryLimit: 1