Updates to the GCP docs and logging (#758)

Signed-off-by: French Ben <[email protected]>

Author: FrenchBen

docs/playbooks/intro/README.md

@@ -8,7 +8,7 @@ If you don't have infrakit or go compiler installed locally, just
 
 ```shell
 
-docker run --rm -v `pwd`:/build infrakit/installer build-infrakit darwin
+docker run --rm -v `pwd`:/build -e GOARCH=amd64 -e GOOS=darwin infrakit/installer build-infrakit
 sudo cp ./infrakit /usr/local/bin
 ```
 This will cross-compile the `infrakit` cli for Mac OSX.  For Linux, there's no need to set the `GOOS` and `GOARCH`
@@ -17,7 +17,7 @@ environment variables.
 ### Linux
 
 ```shell
-$ docker run --rm -v `pwd`:/build infrakit/installer build-infrakit linux
+$ docker run --rm -v `pwd`:/build -e GOARCH=amd64 -e GOOS=linux infrakit/installer build-infrakit
 $ sudo cp ./infrakit /usr/local/bin/
 ```
 

docs/playbooks/intro/gcp/README.md

@@ -1,3 +1,78 @@
 Running InfraKit on GCP
 =======================
 
+The `infrakit` cli is a dynamic command line tool.  You can add 'playbooks' which contain pre-built features.
+This directory is in fact a playbook (see `index.ikb`).  To add this playbook, do this:
+
+```shell
+
+infrakit playbook add gcp https://docker.github.io/infrakit/playbooks/intro/gcp/index.ikb
+```
+
+Verify that the playbook `gcp` has been added:
+
+```shell
+
+infrakit playbook gcp -h
+
+
+___  ________   ________ ________  ________  ___  __    ___  _________
+|\  \|\   ___  \|\  _____\\   __  \|\   __  \|\  \|\  \ |\  \|\___   ___\
+\ \  \ \  \\ \  \ \  \__/\ \  \|\  \ \  \|\  \ \  \/  /|\ \  \|___ \  \_|
+ \ \  \ \  \\ \  \ \   __\\ \   _  _\ \   __  \ \   ___  \ \  \   \ \  \
+  \ \  \ \  \\ \  \ \  \_| \ \  \\  \\ \  \ \  \ \  \\ \  \ \  \   \ \  \
+   \ \__\ \__\\ \__\ \__\   \ \__\\ _\\ \__\ \__\ \__\\ \__\ \__\   \ \__\
+    \|__|\|__| \|__|\|__|    \|__|\|__|\|__|\|__|\|__| \|__|\|__|    \|__|
+
+
+gcp
+
+Usage:
+  infrakit playbook gcp [command]
+
+Available Commands:
+  provision-instance provision-instance
+  start-daemon       start-daemon
+
+...
+
+Use "infrakit playbook gcp [command] --help" for more information about a command.
+```
+
+## Prerequisites:
+
+This tutorial uses Google Cloud Platform.  Be sure you have the API token; you will be prompted to provide it
+if you don't provide it in the command line flag.  You can also authenticate via the `gcloud` tool and thus
+have a `~/.config/gcloud/application_default_credentials.json` file which has the access token.
+
+## Start up Infrakit controller daemons and plugins
+
+The CLI will guide you through starting up infrakit controllers if command line flags were not given.
+In the example below, we start up only the Google Cloud Platform plugin along with the rest of Infrakit controllers.
+All the controllers and plugins are running as Docker containers:
+
+```shell
+
+$ infrakit playbook gcp start-daemon
+Credentials JSON path? [/home/frenchben/.config/gcloud/application_default_credentials.json]:
+What's the zone? : us-west1-a
+What's the name of the project? : my-gcp-project
+Starting daemon
+Tailing log
+
+```
+
+## Provision an Instance
+
+To provision an instance on GCP, use the flags or follow the prompts:
+
+```shell
+
+$ infrakit playbook gcp provision-instance
+Owner? [frenchben]:
+Image to boot? [https://www.googleapis.com/compute/v1/projects/ubuntu-os-cloud/global/images/ubuntu-1404-trusty-v20161205]:
+Machine type? [n1-standard-1]:
+Private IP address (IPv4)? [10.128.0.10]:
+Disk size in GB? [100]:
+
+```

docs/playbooks/intro/gcp/provision-instance.ikt

@@ -1,12 +1,12 @@
 {{/* Input to create instance using the GCP instance plugin */}}
-{{/* =% instanceProvision "instance-gcp" true  %= */}}
+{{/* =% instanceProvision "google/compute" true  %= */}}
 
 {{ $user := flag "user" "string" "owner" | prompt "Owner?" "string" (env "USER") nil }}
 {{ $image := flag "image-id" "string" "Image  ID" | prompt "Image to boot?" "string" "https://www.googleapis.com/compute/v1/projects/ubuntu-os-cloud/global/images/ubuntu-1404-trusty-v20161205" }}
 {{ $instanceType := flag "instance-type" "string" "Machine type" | prompt "Machine type?" "string" "n1-standard-1"}}
 {{ $privateIP := flag "private-ip" "string" "Private IP" | prompt "Private IP address (IPv4)?" "string" "10.128.0.10" nil }}
 
-{{ $diskSize := flag "disk-size" "int" "Disk size in mb" | prompt "Disk size in MB?" "int" 100 }}
+{{ $diskSize := flag "disk-size" "int" "Disk size in gb" | prompt "Disk size in GB?" "int" 100 }}
 
 
 LogicalID : {{ $privateIP }}
@@ -26,8 +26,9 @@ Properties:
   Description: Some description
   Network: default
   MachineType: {{ $instanceType }}
-  DiskSizeMb: {{ $diskSize }}
-  DiskImage: {{ $image }}
+  Disks:
+    - Image: {{ $image }}
+      SizeGb: {{ $diskSize }}
   Scopes:
     - https://www.googleapis.com/auth/cloudruntimeconfig
     - https://www.googleapis.com/auth/logging.write

docs/playbooks/intro/gcp/start-plugin-daemon.ikt

@@ -10,9 +10,10 @@
 {{ $gcpCredentials :=  (cat $credentials ":/infrakit/platforms/gcp/credentials.json" | nospace) }}
 
 echo "Starting daemon"
-infrakit-instance-gcp \
-       --namespace-tags {{cat "infrakit.scope=" $project | nospace}} \
-       --zone {{ $zone }} --log 5 --project {{ $project }} \
+INFRAKIT_GOOGLE_ZONE={{ $zone }} \
+  INFRAKIT_GOOGLE_PROJECT={{ $project }} \
+  INFRAKIT_GOOGLE_NAMESPACE_TAGS={{cat "infrakit.scope=" $project | nospace}} \
+  infrakit plugin start google --log 5 \
        > {{env "INFRAKIT_HOME"}}/logs/instance-gcp.log 2>&1  &
 
 echo "Tailing log"

docs/playbooks/intro/gcp/start-plugin.ikt

@@ -20,8 +20,9 @@ echo "Starting GCP plugin"
 
 # Starting docker container for instance plugin
 docker run -d --volumes-from infrakit --name instance-gcp \
-       -v {{$gcpCredentials}} {{$gcpImage}} infrakit-instance-gcp  \
-       --namespace-tags {{cat "infrakit.scope=" $project | nospace}} \
-       --zone {{ $zone }} --log 5 --project {{ $project }}
+ -e INFRAKIT_GOOGLE_ZONE={{ $zone }} \
+ -e INFRAKIT_GOOGLE_PROJECT={{ $project }} \
+ -e INFRAKIT_GOOGLE_NAMESPACE_TAGS={{cat "infrakit.scope=" $project | nospace}} \
+ -v {{$gcpCredentials}} {{$gcpImage}} infrakit plugin start google --log 5
 
 {{ end }}

pkg/provider/google/README.md

@@ -111,6 +111,7 @@ $ cat << EOF > gcp-vanilla.json
     }
   }
 }
+EOF
 ```
 
 Finally, instruct the Group plugin to start watching the group:
@@ -119,6 +120,12 @@ Finally, instruct the Group plugin to start watching the group:
 $ build/infrakit group-stateless commit gcp-vanilla.json
 ```
 
+### Permissions
+If running on a GCP instance, please make sure that the instance has service accounts enabled:
+https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances
+
+You can check that the instance has the correct permissions via: `gcloud compute instances list`
+
 ## Group plugin
 
 An InfraKit group plugin which wraps Google Compute Engine's managed instance

pkg/provider/google/plugin/gcloud/api.go

@@ -9,7 +9,7 @@ import (
 	"time"
 
 	"cloud.google.com/go/compute/metadata"
-	log "github.com/Sirupsen/logrus"
+	logutil "github.com/docker/infrakit/pkg/log"
 	"golang.org/x/oauth2/google"
 	"google.golang.org/api/compute/v1"
 	"google.golang.org/api/googleapi"
@@ -112,10 +112,12 @@ type computeServiceWrapper struct {
 	service *compute.Service
 }
 
+var log = logutil.New("module", "provider/google")
+
 // NewAPI creates a new API instance.
 func NewAPI(project, zone string) (API, error) {
 	if project == "" {
-		log.Debugln("Project not passed on the command line")
+		log.Debug("Project not passed on the command line", "project", project)
 
 		project = findProject()
 		if project == "" {
@@ -124,16 +126,16 @@ func NewAPI(project, zone string) (API, error) {
 	}
 
 	if zone == "" {
-		log.Debugln("Zone not passed on the command line")
+		log.Debug("Zone not passed on the command line")
 
 		zone = findZone()
 		if zone == "" {
 			return nil, errors.New("Missing zone")
 		}
 	}
 
-	log.Debugln("Project:", project)
-	log.Debugln("Zone:", zone)
+	log.Debug("Project:", "project", project)
+	log.Debug("Zone:", "zone", zone)
 
 	serviceProvider := func() (*compute.Service, error) {
 		client, err := google.DefaultClient(context.TODO(), compute.ComputeScope)
@@ -159,15 +161,15 @@ func NewAPI(project, zone string) (API, error) {
 
 func findProject() string {
 	if metadata.OnGCE() {
-		log.Debugln("- Query the metadata server...")
+		log.Debug("- Query the metadata server...")
 
 		projectID, err := metadata.ProjectID()
 		if err == nil {
 			return projectID
 		}
 	}
 
-	log.Debugln(" - Look for", EnvProject, "env variable...")
+	log.Debug(" - Look for env var", "project", EnvProject)
 
 	value, found := os.LookupEnv(EnvProject)
 	if found && value != "" {
@@ -179,15 +181,15 @@ func findProject() string {
 
 func findZone() string {
 	if metadata.OnGCE() {
-		log.Debugln("- Query the metadata server...")
+		log.Debug("- Query the metadata server...")
 
 		zone, err := metadata.Zone()
 		if err == nil {
 			return zone
 		}
 	}
 
-	log.Debugln(" - Look for", EnvZone, "env variable...")
+	log.Debug(" - Look for env var", "zone", EnvZone)
 
 	value, found := os.LookupEnv(EnvZone)
 	if found && value != "" {
@@ -290,6 +292,7 @@ func (g *computeServiceWrapper) CreateInstance(name string, settings *InstanceSe
 			Preemptible:       settings.Preemptible,
 		},
 	}
+	log.Debug("Creating instance", "instance", instance)
 
 	return g.doCall(g.service.Instances.Insert(g.project, g.zone, instance))
 }
@@ -325,26 +328,26 @@ func (g *computeServiceWrapper) attachedDisk(instanceName string, settings DiskS
 
 	var existingDisk *compute.Disk
 	if settings.ReuseExisting {
-		log.Debugln("Trying to reuse disk", diskName)
+		log.Debug("Trying to reuse disk", diskName)
 
 		disk, err := g.service.Disks.Get(g.project, g.zone, diskName).Do()
 		if err != nil || disk == nil {
-			log.Debugln("Couldn't find existing disk", diskName)
+			log.Debug("Couldn't find existing disk", diskName)
 		} else if disk.SourceImage != sourceImage {
-			log.Debugln("Found existing disk that uses a wrong image. Let's delete", diskName)
+			log.Debug("Found existing disk that uses a wrong image. Let's delete", diskName)
 			if err := g.doCall(g.service.Disks.Delete(g.project, g.zone, disk.Name)); err != nil {
 				return nil, err
 			}
 		} else {
-			log.Debugln("Found existing disk", diskName)
+			log.Debug("Found existing disk", diskName)
 			existingDisk = disk
 		}
 	}
 
 	if existingDisk != nil {
 		disk.Source = existingDisk.SelfLink
 	} else if settings.Image == "" {
-		log.Debugln("Creating standalone disk", diskName)
+		log.Debug("Creating standalone disk", diskName)
 
 		if err := g.doCall(g.service.Disks.Insert(g.project, g.zone, &compute.Disk{
 			Name:   diskName,