Common Spec schema and dependency ordering across all resource / group /plugin types (#473)

Signed-off-by: David Chung <[email protected]>

Author: David Chung

docs/playbooks/intro/gcp/README.md

@@ -0,0 +1,3 @@
+Running InfraKit on GCP
+=======================
+

docs/playbooks/intro/gcp/index.ikb

@@ -0,0 +1,3 @@
+
+# Create a single instance
+provision-instance : provision-instance.ikt

docs/playbooks/intro/gcp/provision-instance.ikt

@@ -0,0 +1,40 @@
+# Input to create instance using the GCP instance plugin
+{{/* =% sh %= */}}
+
+{{ $user := flag "user" "string" "owner" | prompt "Owner?" "string" (env "USER") nil }}
+{{ $prefix := flag "prefix" "string" "Prefix to use" | prompt "Prefix for hostname:" "string" (env "USER") }}
+{{ $diskSize := flag "disk-size" "int" "Disk size in mb" | prompt "Disk size in MB?" "int" 100 }}
+{{ $machineType := flag "machine-type" "string" "Machine type" | prompt "Machine type?" "string" "n1-standard-1"}}
+{{ $privateIP := flag "private-ip" "string" "Private IP" | prompt "Private IP address (IPv4)?" "string" "10.128.0.10" nil }}
+{{ $image := flag "image" "string" "Image" | prompt "Image to boot?" "string" "https://www.googleapis.com/compute/v1/projects/ubuntu-os-cloud/global/images/ubuntu-1404-trusty-v20161205" }}
+
+echo "===> Running with {{$user}}, {{$image}}, {{$privateIP}}:"
+
+infrakit --log 3 --log-stack --name instance-gcp instance provision -y - <<EOF
+
+LogicalID : {{ $privateIP }}
+Tags:
+  infrakit-user: {{ $user }}
+  infrakit-created: {{ now | htmlDate }}
+
+Init: |
+  #!/bin/bash
+  sudo apt-get update -y
+  sudo apt-get install wget curl
+  wget -q0- https://get.docker.com | sh
+
+Properties:
+  NamePrefix: {{ $prefix }}
+  PrivateIP: {{ $privateIP }}
+  Description: Some description
+  Network: default
+  MachineType: {{ $machineType }}
+  DiskSizeMb: {{ $diskSize }}
+  DiskImage: {{ $image }}
+  Scopes:
+    - https://www.googleapis.com/auth/cloudruntimeconfig
+    - https://www.googleapis.com/auth/logging.write
+  Tags:
+    - {{ $user }}
+
+EOF

docs/playbooks/intro/gcp/start-plugin.ikt

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+{{/* =% sh %= */}}
+
+{{ $gcp := flag "start-gcp" "bool" "Start GCP plugin" | prompt "Start GCP plugin?" "bool" "no" }}
+{{ $defaultCred := cat (env "HOME") "/.config/gcloud/application_default_credentials.json" | nospace }}
+{{ $credentials := flag "credential-path" "string" "Path to credentials.json" | cond $gcp | prompt "Credentials JSON path?" "string" $defaultCred }}
+{{ $zone := flag "zone" "string" "GCP zone" | cond $gcp | prompt "What's the zone?" "string" "us-centra1-a"}}
+
+{{ $gcpImage := flag "gcp-plugin" "string" "Image of the plugin" |  cond $gcp | prompt "What's the GCP plugin image?" "string" "infrakit/gcp:dev" }}
+
+
+{{ if $gcp }}
+
+{{ $project := flag "project" "string" "Project name" | cond $gcp | prompt "What's the name of the project?" "string" }}
+
+{{ $gcpCredentials :=  (cat $credentials ":/infrakit/platforms/gcp/credentials.json" | nospace) }}
+
+
+# Starting docker container for instance plugin
+docker run -d --volumes-from infrakit --name instance-plugin \
+       -v {{$gcpCredentials}} {{$gcpImage}} infrakit-instance-gcp  \
+       --namespace-tags {{cat "infrakit.scope=" $project | nospace}} \
+       --zone {{ $zone }} --log 5 --project {{ $project }}
+
+{{ end }}

docs/playbooks/intro/start-infrakit.ikt

@@ -36,3 +36,5 @@ docker run -d --volumes-from infrakit --name manager \
 
 {{/* Here we just source a file in the same folder for more reuse and modularity */}}
 {{ source "aws/start-plugin.ikt" }}
+
+{{ source "gcp/start-plugin.ikt" }}

pkg/cli/remote/remote.go

@@ -109,18 +109,18 @@ loop:
 			}
 		}
 
+		// Documentation -- look for a README.md at the given dir
+		readme := path.Join(path.Dir(string(url)), "README.md")
+		if t, err := template.NewTemplate(readme, template.Options{}); err == nil {
+			if view, err := t.Render(nil); err == nil {
+				cmd.SetHelpTemplate(fmt.Sprintf(helpTemplate, view))
+			}
+		}
+
 		// if we can parse it as a map, then we have a 'directory'
 		mods, err := dir(url)
 		if err == nil {
 
-			// Documentation -- look for a README.md at the given dir
-			readme := path.Join(path.Dir(string(url)), "README.md")
-			if t, err := template.NewTemplate(readme, template.Options{}); err == nil {
-				if view, err := t.Render(nil); err == nil {
-					cmd.SetHelpTemplate(fmt.Sprintf(helpTemplate, view))
-				}
-			}
-
 			copy := url
 			subs, err := list(plugins, mods, input, cmd, &copy)
 			if err != nil {

pkg/core/depend.go

@@ -0,0 +1,221 @@
+package core
+
+import (
+	"encoding/json"
+	"reflect"
+
+	"github.com/docker/infrakit/pkg/types"
+	"github.com/twmb/algoimpl/go/graph"
+)
+
+// converts a map to a Spec, nil if it cannot be done
+func mapToSpec(m map[string]interface{}) *types.Spec {
+	// This is hacky -- generate a string representation
+	// and try to parse it as struct
+	buff, err := json.Marshal(m)
+	if err != nil {
+		return nil
+	}
+	s := types.Spec{}
+	err = json.Unmarshal(buff, &s)
+	if err != nil {
+		return nil
+	}
+	if s.Validate() == nil {
+		return &s
+	}
+	return nil
+}
+
+// findSpecs parses the bytes and returns a Spec, if the Spec can be parsed
+// from the buffer.  Some fields are verified and must be present for the
+// buffer to be considered a representation of a Spec.
+func findSpecs(v interface{}) []*types.Spec {
+
+	result := []*types.Spec{}
+
+	switch v := v.(type) {
+
+	case []*types.Spec:
+		for _, x := range v {
+			c := *x
+			result = append(result, findSpecs(&c)...)
+		}
+
+	case []types.Spec:
+		for _, x := range v {
+			c := x
+			result = append(result, findSpecs(&c)...)
+		}
+
+	case []interface{}:
+		for _, x := range v {
+			c := x
+			result = append(result, findSpecs(c)...)
+		}
+
+	case map[string]interface{}:
+		// convert to Spec?
+		result = append(result, findSpecs(mapToSpec(v))...)
+
+	case *types.Any:
+
+		if v == nil {
+			return result
+		}
+
+		spec := types.Spec{}
+		if err := v.Decode(&spec); err == nil {
+
+			if spec.Validate() == nil {
+				result = append(result, findSpecs(&spec)...)
+				return result
+			}
+
+		}
+
+		// now decode as regular struct - map or []interface{}
+		var vv interface{}
+		if err := v.Decode(&vv); err != nil {
+			return nil
+		}
+
+		switch vv := vv.(type) {
+		case []interface{}:
+			for _, x := range vv {
+				result = append(result, findSpecs(x)...)
+			}
+		case map[string]interface{}:
+			for _, x := range vv {
+				result = append(result, findSpecs(x)...)
+			}
+		}
+
+	case types.Spec:
+		c := v
+		result = append(result, &c)
+		result = append(result, findSpecs(c.Properties)...)
+
+	case *types.Spec:
+
+		if v == nil {
+			return result
+		}
+
+		c := *v
+		result = append(result, &c)
+		result = append(result, findSpecs(c.Properties)...)
+
+	default:
+		value := reflect.Indirect(reflect.ValueOf(v))
+		if value.Type().Kind() == reflect.Struct {
+			for i := 0; i < value.NumField(); i++ {
+				fv := value.Field(i)
+				if fv.IsValid() {
+					result = append(result, findSpecs(fv.Interface())...)
+				}
+			}
+		}
+	}
+	return result
+}
+
+// nested recurses through the Properties of the spec and returns any nested specs.
+func nested(s *types.Spec) []*types.Spec {
+	if s.Properties == nil {
+		return nil
+	}
+	return findSpecs(s.Properties)
+}
+
+type key struct {
+	class string
+	name  string
+}
+
+func indexSpecs(specs []*types.Spec, g *graph.Graph) map[key]*graph.Node {
+	index := map[key]*graph.Node{}
+	for _, spec := range specs {
+
+		node := g.MakeNode()
+		*(node.Value) = spec
+
+		index[key{class: spec.Class, name: spec.Metadata.Name}] = &node
+	}
+	return index
+}
+
+func indexGet(index map[key]*graph.Node, class, name string) *graph.Node {
+	if v, has := index[key{class: class, name: name}]; has {
+		return v
+	}
+	return nil
+}
+
+// AllSpecs returns a list of all the specs given plus any nested specs
+func AllSpecs(specs []*types.Spec) []*types.Spec {
+	all := []*types.Spec{}
+	for _, s := range specs {
+		all = append(all, s)
+		all = append(all, nested(s)...)
+	}
+	return all
+}
+
+// OrderByDependency returns the given specs in dependency order.  The input is assume to be exhaustive in that
+// all nested specs are part of the list.
+func OrderByDependency(specs []*types.Spec) ([]*types.Spec, error) {
+
+	g := graph.New(graph.Directed)
+	if g == nil {
+		return nil, nil
+	}
+
+	index := indexSpecs(specs, g)
+
+	for _, spec := range specs {
+
+		from := indexGet(index, spec.Class, spec.Metadata.Name)
+		if from == nil {
+			return nil, errNotFound{class: spec.Class, name: spec.Metadata.Name}
+		}
+
+		for _, depend := range spec.Depends {
+
+			to := indexGet(index, depend.Class, depend.Name)
+			if to == nil {
+				return nil, errBadDependency(depend)
+			}
+
+			if from == to {
+
+				a := (*from.Value).(*types.Spec)
+				b := (*to.Value).(*types.Spec)
+				return nil, errCircularDependency([]*types.Spec{a, b})
+			}
+
+			if err := g.MakeEdge(*to, *from); err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	// cycle detection
+	for _, connected := range g.StronglyConnectedComponents() {
+		if len(connected) > 1 {
+
+			cycle := []*types.Spec{}
+			for _, n := range connected {
+				cycle = append(cycle, (*n.Value).(*types.Spec))
+			}
+			return nil, errCircularDependency(cycle)
+		}
+	}
+
+	ordered := []*types.Spec{}
+	for _, n := range g.TopologicalSort() {
+		ordered = append(ordered, (*n.Value).(*types.Spec))
+	}
+
+	return ordered, nil
+}