Allow playbook templates to call instance plugins directly (#487)

Signed-off-by: David Chung <[email protected]>

Author: David Chung

cmd/cli/manager/manager.go

@@ -56,13 +56,13 @@ func Command(plugins func() discovery.Plugins) *cobra.Command {
 						return err
 					}
 
-					log.Info("Found manager", "name", name, "leader", isLeader)
+					log.Debug("Found manager", "name", name, "leader", isLeader)
 					if isLeader {
 
 						groupPlugin = group_plugin.Adapt(rpcClient)
 						groupPluginName = name
 
-						log.Info("Found manager", "name", name, "addr", endpoint.Address)
+						log.Debug("Found manager", "name", name, "addr", endpoint.Address)
 
 						break
 					}

docs/playbooks/intro/aws/provision-instance.ikt

@@ -1,26 +1,26 @@
-# Input to create instance using the AWS instance plugin
-{{/* =% sh %= */}}
+{{/* Input to create instance using the AWS instance plugin */}}
+{{/* =% instanceProvision "instance-aws/ec2-instance" true  %= */}}
 
 {{ $user := flag "user" "string" "username" | prompt "Please enter your user name:" "string" (env "USER")}}
-{{ $name := flag "name" "string" "name" | prompt "Name?" "string"}}
-{{ $imageId := flag "ami" "string" "ami" | prompt "AMI?" "string"}}
+{{ $imageId := flag "image-id" "string" "Image ID" | prompt "AMI?" "string"}}
 {{ $instanceType := flag "instance-type" "string" "instance type" | prompt "Instance type?" "string"}}
+{{ $privateIP := flag "private-ip" "string" "Private IP" | prompt "Private IP address (IPv4)?" "string" "" nil }}
+
+
 {{ $keyName := flag "key" "string" "ssh key name" | prompt "SSH key?" "string"}}
 {{ $az := flag "az" "string" "availability zone" | prompt "Availability zone?" "string"}}
 {{ $subnetId := flag "subnet" "string" "subnet id" | prompt "Subnet ID?" "string"}}
 {{ $securityGroupId := flag "security-group-id" "string" "security group id" | prompt "Security group ID?" "string" }}
 
-infrakit --log 3 --log-stack --name instance-aws/ec2-instance instance provision -y - <<EOF
-
 Tags:
-  infrakit.name: {{ $name }}
   infrakit.created: {{ now | htmlDate }}
   infrakit.user: {{ $user }}
 
 Init: |
   #!/bin/bash
   sudo apt-get update -y
   sudo apt-get install wget curl
+  wget -q0- https://get.docker.com | sh
 
 Properties:
   RunInstancesInput:
@@ -34,6 +34,7 @@ Properties:
     KeyName: {{ $keyName }}
     NetworkInterfaces:
     - AssociatePublicIpAddress: true
+      PrivateIpAddress: {{ $privateIP }}
       DeleteOnTermination: true
       DeviceIndex: 0
       Groups:
@@ -48,6 +49,4 @@ Properties:
     SubnetId: null
     UserData: null
   Tags:
-    infrakit.name: {{ $name }}
-
-EOF
+    infrakit.user: {{ $user }}

docs/playbooks/intro/aws/start-plugin.ikt

@@ -3,11 +3,11 @@
 {{ $aws := flag "start-aws" "bool" "Start AWS plugin" | prompt "Start AWS plugin?" "bool" "no" }}
 
 {{ $profile := flag "aws-cred-profile" "string" "Profile name for credentials" | cond $aws |  prompt "What's the profile for your .aws/credentials?" "string" "default" }}
-{{ $project := flag "project" "string" "Project name" | cond $aws | prompt "What's the name of the project?" "string" "testproject"}}
 {{ $region := flag "region" "string" "aws region" | cond $aws | prompt "What's the region?" "string" "us-west-1"}}
 
 {{ $awsImage := flag "aws-plugin" "string" "Image of the plugin" |  cond $aws | prompt "What's the AWS plugin image?" "string" "infrakit/aws:dev" }}
 
+{{ $project := var "/project" }}
 
 {{ if $aws }}
 
@@ -17,10 +17,6 @@ echo "Starting AWS plugin"
 {{ $creds := (source (cat "file://" (env "HOME") "/.aws/credentials" | nospace) | iniDecode | k $profile ) }}
 
 
-{{ $infrakit := (cat (env "HOME") "/.infrakit/" | nospace) }}
-{{ $dockerEnvs := "-e INFRAKIT_HOME=/infrakit -e INFRAKIT_PLUGINS_DIR=/infrakit/plugins" }}
-
-
 {{/* Show helpful message to user if we can find the credentials */}}
 {{ if not (empty $creds) }}{{ echo "Found your credential for profile" $profile }}
 {{ end }}

docs/playbooks/intro/gcp/index.ikb

@@ -1,3 +1,6 @@
 
 # Create a single instance
 provision-instance : provision-instance.ikt
+
+# Starts the plugin as a daemon (not container)
+start-daemon : start-plugin-daemon.ikt

docs/playbooks/intro/gcp/provision-instance.ikt

@@ -1,21 +1,18 @@
-# Input to create instance using the GCP instance plugin
-{{/* =% sh %= */}}
+{{/* Input to create instance using the GCP instance plugin */}}
+{{/* =% instanceProvision "instance-gcp" true  %= */}}
 
 {{ $user := flag "user" "string" "owner" | prompt "Owner?" "string" (env "USER") nil }}
-{{ $prefix := flag "prefix" "string" "Prefix to use" | prompt "Prefix for hostname:" "string" (env "USER") }}
-{{ $diskSize := flag "disk-size" "int" "Disk size in mb" | prompt "Disk size in MB?" "int" 100 }}
-{{ $machineType := flag "machine-type" "string" "Machine type" | prompt "Machine type?" "string" "n1-standard-1"}}
+{{ $image := flag "image-id" "string" "Image  ID" | prompt "Image to boot?" "string" "https://www.googleapis.com/compute/v1/projects/ubuntu-os-cloud/global/images/ubuntu-1404-trusty-v20161205" }}
+{{ $instanceType := flag "instance-type" "string" "Machine type" | prompt "Machine type?" "string" "n1-standard-1"}}
 {{ $privateIP := flag "private-ip" "string" "Private IP" | prompt "Private IP address (IPv4)?" "string" "10.128.0.10" nil }}
-{{ $image := flag "image" "string" "Image" | prompt "Image to boot?" "string" "https://www.googleapis.com/compute/v1/projects/ubuntu-os-cloud/global/images/ubuntu-1404-trusty-v20161205" }}
 
-echo "===> Running with {{$user}}, {{$image}}, {{$privateIP}}:"
+{{ $diskSize := flag "disk-size" "int" "Disk size in mb" | prompt "Disk size in MB?" "int" 100 }}
 
-infrakit --log 3 --log-stack --name instance-gcp instance provision -y - <<EOF
 
 LogicalID : {{ $privateIP }}
 Tags:
-  infrakit-user: {{ $user }}
-  infrakit-created: {{ now | htmlDate }}
+  infrakit.created: {{ now | htmlDate }}
+  infrakit.user: {{ $user }}
 
 Init: |
   #!/bin/bash
@@ -24,17 +21,15 @@ Init: |
   wget -q0- https://get.docker.com | sh
 
 Properties:
-  NamePrefix: {{ $prefix }}
+  NamePrefix: {{ $user }}
   PrivateIP: {{ $privateIP }}
   Description: Some description
   Network: default
-  MachineType: {{ $machineType }}
+  MachineType: {{ $instanceType }}
   DiskSizeMb: {{ $diskSize }}
   DiskImage: {{ $image }}
   Scopes:
     - https://www.googleapis.com/auth/cloudruntimeconfig
     - https://www.googleapis.com/auth/logging.write
   Tags:
     - {{ $user }}
-
-EOF

docs/playbooks/intro/gcp/start-plugin-daemon.ikt

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+{{/* =% sh %= */}}
+
+{{ $defaultCred := cat (env "HOME") "/.config/gcloud/application_default_credentials.json" | nospace }}
+{{ $credentials := flag "credential-path" "string" "Path to credentials.json" | prompt "Credentials JSON path?" "string" $defaultCred }}
+{{ $zone := flag "zone" "string" "GCP zone" | prompt "What's the zone?" "string" }}
+{{ $project := flag "project" "string" "Project name" | prompt "What's the name of the project?" "string" }}
+{{ $instanceImage := "infrakit/gcp:dev" }}
+{{ $gcpCredentials :=  (cat $credentials ":/infrakit/platforms/gcp/credentials.json" | nospace) }}
+
+echo "Starting daemon"
+infrakit-instance-gcp \
+       --namespace-tags {{cat "infrakit.scope=" $project | nospace}} \
+       --zone {{ $zone }} --log 5 --project {{ $project }} \
+       > {{env "INFRAKIT_HOME"}}/logs/instance-gcp.log 2>&1  &
+
+echo "Tailing log"
+tail -f {{env "INFRAKIT_HOME"}}/logs/*.log

docs/playbooks/intro/gcp/start-plugin.ikt

@@ -5,20 +5,21 @@
 {{ $gcp := flag "start-gcp" "bool" "Start GCP plugin" | prompt "Start GCP plugin?" "bool" "no" }}
 {{ $defaultCred := cat (env "HOME") "/.config/gcloud/application_default_credentials.json" | nospace }}
 {{ $credentials := flag "credential-path" "string" "Path to credentials.json" | cond $gcp | prompt "Credentials JSON path?" "string" $defaultCred }}
-{{ $zone := flag "zone" "string" "GCP zone" | cond $gcp | prompt "What's the zone?" "string" "us-centra1-a"}}
+{{ $zone := flag "zone" "string" "GCP zone" | cond $gcp | prompt "What's the zone?" "string" "us-central1-f"}}
 
 {{ $gcpImage := flag "gcp-plugin" "string" "Image of the plugin" |  cond $gcp | prompt "What's the GCP plugin image?" "string" "infrakit/gcp:dev" }}
 
+{{ $project := var "/project" }}
 
 {{ if $gcp }}
 
-{{ $project := flag "project" "string" "Project name" | cond $gcp | prompt "What's the name of the project?" "string" }}
+echo "Starting GCP plugin"
 
 {{ $gcpCredentials :=  (cat $credentials ":/infrakit/platforms/gcp/credentials.json" | nospace) }}
 
 
 # Starting docker container for instance plugin
-docker run -d --volumes-from infrakit --name instance-plugin \
+docker run -d --volumes-from infrakit --name instance-gcp \
        -v {{$gcpCredentials}} {{$gcpImage}} infrakit-instance-gcp  \
        --namespace-tags {{cat "infrakit.scope=" $project | nospace}} \
        --zone {{ $zone }} --log 5 --project {{ $project }}

docs/playbooks/intro/start-infrakit.ikt

@@ -5,6 +5,9 @@
 {{ $image := flag "infrakit-image" "string" "Infrakit image" | prompt "Infrakit image?" "string" "infrakit/devbundle:dev" }}
 {{ $port := flag "infrakit-port" "int" "Infrakit mux port" | prompt "Infrakit port for remote access?" "int" 24864 }}
 
+{{ $project := flag "project" "string" "Project name" | prompt "What's the name of the project?" "string" "testproject"}}
+
+
 {{/* optional plugins */}}
 
 
@@ -34,6 +37,8 @@ docker run -d --volumes-from infrakit --name manager \
        {{ $image }} infrakit-manager --proxy-for-group group-stateless os
 
 
+{{ var "/project" $project }}
+
 {{/* Here we just source a file in the same folder for more reuse and modularity */}}
 {{ source "aws/start-plugin.ikt" }}
 

docs/playbooks/intro/stop-infrakit.ikt

@@ -16,6 +16,9 @@ docker ps -f ancestor=infrakit/devbundle:dev -qa | xargs docker rm
 docker ps -f ancestor=infrakit/aws:dev -qa | xargs docker stop
 docker ps -f ancestor=infrakit/aws:dev -qa | xargs docker rm
 
+docker ps -f ancestor=infrakit/gcp:dev -qa | xargs docker stop
+docker ps -f ancestor=infrakit/gcp:dev -qa | xargs docker rm
+
 docker network rm infrakit
 
 {{ else }}

pkg/cli/context.go

@@ -9,7 +9,13 @@ import (
 	"strings"
 
 	"github.com/docker/infrakit/pkg/discovery"
+	"github.com/docker/infrakit/pkg/plugin"
+	group_plugin "github.com/docker/infrakit/pkg/rpc/group"
+	instance_plugin "github.com/docker/infrakit/pkg/rpc/instance"
+	"github.com/docker/infrakit/pkg/spi/group"
+	"github.com/docker/infrakit/pkg/spi/instance"
 	"github.com/docker/infrakit/pkg/template"
+	"github.com/docker/infrakit/pkg/types"
 	"github.com/docker/infrakit/pkg/util/exec"
 	"github.com/spf13/cobra"
 )
@@ -338,6 +344,102 @@ func (c *Context) loadBackends() error {
 			}
 			return ""
 		})
+	t.AddFunc("instanceProvision",
+		func(name string, isYAML bool) string {
+			c.run = func(script string) error {
+
+				// locate the plugin
+				endpoint, err := c.plugins().Find(plugin.Name(name))
+				if err != nil {
+					return err
+				}
+
+				plugin, err := instance_plugin.NewClient(plugin.Name(name), endpoint.Address)
+				if err != nil {
+					return err
+				}
+
+				spec := instance.Spec{}
+				if isYAML {
+					y, err := types.AnyYAML([]byte(script))
+					if err != nil {
+						return err
+					}
+					if err := y.Decode(&spec); err != nil {
+						return err
+					}
+				} else {
+					if err := types.AnyString(script).Decode(&spec); err != nil {
+						return err
+					}
+				}
+
+				id, err := plugin.Provision(spec)
+				if err != nil {
+					return err
+				}
+				fmt.Println(id)
+				return nil
+			}
+			return ""
+		})
+
+	t.AddFunc("managerCommit",
+		func(isYAML, pretend bool) string {
+			c.run = func(script string) error {
+
+				groups := []plugin.Spec{}
+				if isYAML {
+					y, err := types.AnyYAML([]byte(script))
+					if err != nil {
+						return err
+					}
+					if err := y.Decode(&groups); err != nil {
+						return err
+					}
+				} else {
+					if err := types.AnyString(script).Decode(&groups); err != nil {
+						return err
+					}
+				}
+
+				// Check the list of plugins
+				for _, gp := range groups {
+
+					endpoint, err := c.plugins().Find(gp.Plugin)
+					if err != nil {
+						return err
+					}
+
+					// unmarshal the group spec
+					spec := group.Spec{}
+					if gp.Properties != nil {
+						err = gp.Properties.Decode(&spec)
+						if err != nil {
+							return err
+						}
+					}
+
+					target, err := group_plugin.NewClient(endpoint.Address)
+
+					log.Debug("commit", "plugin", gp.Plugin, "address", endpoint.Address, "err", err, "spec", spec)
+
+					if err != nil {
+						return err
+					}
+
+					plan, err := target.CommitGroup(spec, pretend)
+					if err != nil {
+						return err
+					}
+
+					fmt.Println("Group", spec.ID, "with plugin", gp.Plugin, "plan:", plan)
+				}
+				return nil
+			}
+			return ""
+		})
+
 	t.AddFunc("sh",
 		func(opts ...string) string {
 			c.run = func(script string) error {