Add support for Option Envs parser and use in tf plugin (#694)

* Add plugin `Option` environment parser

Generic parser that handles the `Envs` values for a plugin's `Options`; the
value is a slice of "key=val" strings that can either be a simple key/value
pair or a reference a key in a local JSON file.

For example, a value of:
```
"Options": {
  "Envs": [
    "key1=val1",
    "key2=file:///run/secrets/my-secret:some-key"
  ]
}
```
Parses the the given `/run/secrets/my-secret` file as JSON, gets the `some-key` value,
and set the `key2` value in the slice. The parsed slice is:
```
["key1=val1","key2=<file-value-of-some-key>"]
```

Signed-off-by: Steven Kaufer <[email protected]>

* Add support to use custom env vars in terraform plugin

The new `Options|Envs` plugin Option is processed to create a slice of
env vars to use on the terraform plugin. These env vars are used on all
`terraform` execs.

The idea is that the env vars would have the specify cloud provider
credentials.

Signed-off-by: Steven Kaufer <[email protected]>

* Cleanup terraform execs to get the working dir from the plugin

Since the plugin receiver has the working directory we no longer need
to pass it on the function call.

Signed-off-by: Steven Kaufer <[email protected]>

* Update tf plugin Option.Envs parsing to use native templating

Instead of using custom logic with file-uri like values we can simply take
advantage of the fact that the plugin configuration file is already
templated.

Signed-off-by: Steven Kaufer <[email protected]>

Author: kaufers

pkg/provider/terraform/instance/apply.go

@@ -45,13 +45,16 @@ func (p *plugin) terraformApply() error {
 			if p.shouldApply() {
 				fns := tfFuncs{
 					tfRefresh: func() error {
-						command := exec.Command("terraform refresh").InheritEnvs(true).WithDir(p.Dir)
+						command := exec.Command("terraform refresh").
+							InheritEnvs(true).
+							WithEnvs(p.envs...).
+							WithDir(p.Dir)
 						if err := command.WithStdout(os.Stdout).WithStderr(os.Stdout).Start(); err != nil {
 							return err
 						}
 						return command.Wait()
 					},
-					tfStateList: doTerraformStateList,
+					tfStateList: p.doTerraformStateList,
 				}
 				// The trigger for an apply is typically from a group commit, sleep for a few seconds so
 				// that multiple .tf.json.new files have time to be created
@@ -90,7 +93,10 @@ func (p *plugin) terraformApply() error {
 // doTerraformApply executes "terraform apply"
 func (p *plugin) doTerraformApply() error {
 	log.Infoln("Applying plan")
-	command := exec.Command("terraform apply -refresh=false").InheritEnvs(true).WithDir(p.Dir)
+	command := exec.Command("terraform apply -refresh=false").
+		InheritEnvs(true).
+		WithEnvs(p.envs...).
+		WithDir(p.Dir)
 	err := command.WithStdout(os.Stdout).WithStderr(os.Stdout).Start()
 	if err == nil {
 		return command.Wait()
@@ -127,7 +133,7 @@ func (p *plugin) shouldApply() bool {
 // External functions to use during when pruning files; broken out for testing
 type tfFuncs struct {
 	tfRefresh   func() error
-	tfStateList func(string) (map[TResourceType]map[TResourceName]struct{}, error)
+	tfStateList func() (map[TResourceType]map[TResourceName]struct{}, error)
 }
 
 // handleFiles handles resource pruning and new resources via:
@@ -148,7 +154,7 @@ func (p *plugin) handleFiles(fns tfFuncs) error {
 	if err := fns.tfRefresh(); err != nil {
 		return err
 	}
-	tfStateResources, err := fns.tfStateList(p.Dir)
+	tfStateResources, err := fns.tfStateList()
 	if err != nil {
 		// TODO(kaufers): If it possible that not all of the .new files were moved to
 		//  .tf.json files (NFS connection could be lost) and this could make the refresh
@@ -249,9 +255,12 @@ func (p *plugin) handleFiles(fns tfFuncs) error {
 }
 
 // doTerraformStateList shells out to run `terraform state list` and parses the result
-func doTerraformStateList(dir string) (map[TResourceType]map[TResourceName]struct{}, error) {
+func (p *plugin) doTerraformStateList() (map[TResourceType]map[TResourceName]struct{}, error) {
 	result := map[TResourceType]map[TResourceName]struct{}{}
-	command := exec.Command("terraform state list -no-color").InheritEnvs(true).WithDir(dir)
+	command := exec.Command("terraform state list -no-color").
+		InheritEnvs(true).
+		WithEnvs(p.envs...).
+		WithDir(p.Dir)
 	command.StartWithHandlers(
 		nil,
 		func(r io.Reader) error {

pkg/provider/terraform/instance/apply_test.go

@@ -26,7 +26,7 @@ func TestRunTerraformApply(t *testing.T) {
 	dir, err := os.Getwd()
 	require.NoError(t, err)
 	dir = path.Join(dir, "aws-two-tier")
-	terraform := NewTerraformInstancePlugin(dir, 1*time.Second, false, nil)
+	terraform := NewTerraformInstancePlugin(dir, 1*time.Second, false, []string{}, nil)
 	p, _ := terraform.(*plugin)
 	err = p.doTerraformApply()
 	require.NoError(t, err)
@@ -36,7 +36,7 @@ func TestContinuePollingStandalone(t *testing.T) {
 	dir, err := ioutil.TempDir("", "infrakit-instance-terraform")
 	require.NoError(t, err)
 	defer os.RemoveAll(dir)
-	terraform := NewTerraformInstancePlugin(dir, 1*time.Second, true, nil)
+	terraform := NewTerraformInstancePlugin(dir, 1*time.Second, true, []string{}, nil)
 	p, _ := terraform.(*plugin)
 	shoudApply := p.shouldApply()
 	require.True(t, shoudApply)
@@ -156,8 +156,7 @@ func TestHandleFilesStateListFail(t *testing.T) {
 			refreshInvoked = true
 			return nil
 		},
-		tfStateList: func(dirArg string) (map[TResourceType]map[TResourceName]struct{}, error) {
-			require.Equal(t, dir, dirArg)
+		tfStateList: func() (map[TResourceType]map[TResourceName]struct{}, error) {
 			return nil, fmt.Errorf("Custom state list error")
 		},
 	}
@@ -173,7 +172,7 @@ func TestHandleFilesNoFiles(t *testing.T) {
 
 	fns := tfFuncs{
 		tfRefresh: func() error { return nil },
-		tfStateList: func(dir string) (map[TResourceType]map[TResourceName]struct{}, error) {
+		tfStateList: func() (map[TResourceType]map[TResourceName]struct{}, error) {
 			return map[TResourceType]map[TResourceName]struct{}{}, nil
 		},
 	}
@@ -205,7 +204,7 @@ func TestHandleFilesNoPruneNoNewFiles(t *testing.T) {
 
 	fns := tfFuncs{
 		tfRefresh: func() error { return nil },
-		tfStateList: func(dir string) (map[TResourceType]map[TResourceName]struct{}, error) {
+		tfStateList: func() (map[TResourceType]map[TResourceName]struct{}, error) {
 			// Return 5 files
 			result := map[TResourceName]struct{}{}
 			for i := 100; i < 105; i++ {
@@ -320,7 +319,7 @@ func TestHandleFilesDedicatedGlobalNoPruneNoNewFiles(t *testing.T) {
 
 	fns := tfFuncs{
 		tfRefresh: func() error { return nil },
-		tfStateList: func(dir string) (map[TResourceType]map[TResourceName]struct{}, error) {
+		tfStateList: func() (map[TResourceType]map[TResourceName]struct{}, error) {
 			// Return all VMs
 			vms := map[TResourceName]struct{}{}
 			for i := 100; i < 115; i++ {
@@ -395,7 +394,7 @@ func TestHandleFilesNoPruneWithNewFiles(t *testing.T) {
 
 	fns := tfFuncs{
 		tfRefresh: func() error { return nil },
-		tfStateList: func(dir string) (map[TResourceType]map[TResourceName]struct{}, error) {
+		tfStateList: func() (map[TResourceType]map[TResourceName]struct{}, error) {
 			// Return the 5 existing instances
 			result := map[TResourceName]struct{}{}
 			for i := 100; i < 105; i++ {
@@ -450,7 +449,7 @@ func TestHandleFilesPruneMultipleVMTypes(t *testing.T) {
 
 	fns := tfFuncs{
 		tfRefresh: func() error { return nil },
-		tfStateList: func(dir string) (map[TResourceType]map[TResourceName]struct{}, error) {
+		tfStateList: func() (map[TResourceType]map[TResourceName]struct{}, error) {
 			// Return all of 1 type, 1 of another type, 0 of the last type
 			return map[TResourceType]map[TResourceName]struct{}{
 				VMIBMCloud: {
@@ -504,7 +503,7 @@ func TestHandleFilesPruneWithNewFiles(t *testing.T) {
 
 	fns := tfFuncs{
 		tfRefresh: func() error { return nil },
-		tfStateList: func(dir string) (map[TResourceType]map[TResourceName]struct{}, error) {
+		tfStateList: func() (map[TResourceType]map[TResourceName]struct{}, error) {
 			// Return only 3 of the five existing
 			result := map[TResourceName]struct{}{
 				TResourceName("instance-102"): {},
@@ -659,7 +658,7 @@ func TestHandleFilesDedicatedGlobalNoPruneWithNewFiles(t *testing.T) {
 
 	fns := tfFuncs{
 		tfRefresh: func() error { return nil },
-		tfStateList: func(dir string) (map[TResourceType]map[TResourceName]struct{}, error) {
+		tfStateList: func() (map[TResourceType]map[TResourceName]struct{}, error) {
 			// Return only existing instances (ie, only odd ones)
 			vms := map[TResourceName]struct{}{}
 			for i := 100; i < 115; i++ {
@@ -842,7 +841,7 @@ func TestHandleFilesDedicatedGlobalPruneWithNewFiles(t *testing.T) {
 
 	fns := tfFuncs{
 		tfRefresh: func() error { return nil },
-		tfStateList: func(dir string) (map[TResourceType]map[TResourceName]struct{}, error) {
+		tfStateList: func() (map[TResourceType]map[TResourceName]struct{}, error) {
 			// Return only existing instances (ie, only odd ones) EXCEPT 1 in each group
 			vms := map[TResourceName]struct{}{}
 			for i := 100; i < 115; i++ {
@@ -909,7 +908,7 @@ func TestHandleFilesDuplicates(t *testing.T) {
 
 	fns := tfFuncs{
 		tfRefresh: func() error { return nil },
-		tfStateList: func(dir string) (map[TResourceType]map[TResourceName]struct{}, error) {
+		tfStateList: func() (map[TResourceType]map[TResourceName]struct{}, error) {
 			return map[TResourceType]map[TResourceName]struct{}{}, nil
 		},
 	}

pkg/provider/terraform/instance/cmd/main.go

@@ -95,7 +95,7 @@ func main() {
 		}
 		cli.SetLogLevel(*logLevel)
 		run.Plugin(plugin_base.DefaultTransport(*name), instance_plugin.PluginServer(
-			terraform.NewTerraformInstancePlugin(*dir, *pollInterval, *standalone, &importOpts)),
+			terraform.NewTerraformInstancePlugin(*dir, *pollInterval, *standalone, []string{}, &importOpts)),
 		)
 	}
 

pkg/provider/terraform/instance/plugin.go

@@ -70,6 +70,7 @@ type plugin struct {
 	pollInterval time.Duration
 	pollChannel  chan bool
 	pluginLookup func() discovery.Plugins
+	envs         []string
 }
 
 // ImportResource defines a resource that should be imported
@@ -94,7 +95,7 @@ type ImportOptions struct {
 }
 
 // NewTerraformInstancePlugin returns an instance plugin backed by disk files.
-func NewTerraformInstancePlugin(dir string, pollInterval time.Duration, standalone bool, importOpts *ImportOptions) instance.Plugin {
+func NewTerraformInstancePlugin(dir string, pollInterval time.Duration, standalone bool, envs []string, importOpts *ImportOptions) instance.Plugin {
 	log.Debugln("terraform instance plugin. dir=", dir)
 	fsLock, err := lockfile.New(filepath.Join(dir, "tf-apply.lck"))
 	if err != nil {
@@ -120,6 +121,7 @@ func NewTerraformInstancePlugin(dir string, pollInterval time.Duration, standalo
 		fsLock:       fsLock,
 		pollInterval: pollInterval,
 		pluginLookup: pluginLookup,
+		envs:         envs,
 	}
 	if err := p.processImport(importOpts); err != nil {
 		panic(err)
@@ -164,15 +166,18 @@ func (p *plugin) processImport(importOpts *ImportOptions) error {
 
 	// Define the functions for import and import the VM
 	fns := importFns{
-		tfShow: doTerraformShow,
+		tfShow: p.doTerraformShow,
 		tfImport: func(resType TResourceType, resName, id string) error {
-			command := exec.Command(fmt.Sprintf("terraform import %v.%v %s", resType, resName, id)).InheritEnvs(true).WithDir(p.Dir)
+			command := exec.Command(fmt.Sprintf("terraform import %v.%v %s", resType, resName, id)).
+				InheritEnvs(true).
+				WithEnvs(p.envs...).
+				WithDir(p.Dir)
 			if err := command.WithStdout(os.Stdout).WithStderr(os.Stdout).Start(); err != nil {
 				return err
 			}
 			return command.Wait()
 		},
-		tfShowInst: doTerraformShowForInstance,
+		tfShowInst: p.doTerraformShowForInstance,
 		tfClean:    p.cleanupFailedImport,
 	}
 	err := p.importResources(fns, importOpts.Resources, importOpts.InstanceSpec)
@@ -1161,7 +1166,7 @@ func (p *plugin) DescribeInstances(tags map[string]string, properties bool) ([]i
 		// TODO - not the most efficient, but here we assume we're usually just one vm type
 		for vmResourceType := range localSpecs {
 
-			if result, err := doTerraformShow(p.Dir, []TResourceType{vmResourceType}, nil); err == nil {
+			if result, err := p.doTerraformShow([]TResourceType{vmResourceType}, nil); err == nil {
 				terraformShowResult = result
 			} else {
 				// Don't blow up... just do best and show what we can find.
@@ -1282,9 +1287,9 @@ func terraformLogicalID(props TResourceProperties) *instance.LogicalID {
 
 // External functions using during import; broken out for testing
 type importFns struct {
-	tfShow     func(dir string, resTypes []TResourceType, propFilter []string) (map[TResourceType]map[TResourceName]TResourceProperties, error)
+	tfShow     func(resTypes []TResourceType, propFilter []string) (map[TResourceType]map[TResourceName]TResourceProperties, error)
 	tfImport   func(resType TResourceType, filename, resID string) error
-	tfShowInst func(dir, id string) (TResourceProperties, error)
+	tfShowInst func(id string) (TResourceProperties, error)
 	tfClean    func(resType TResourceType, resName string)
 }
 
@@ -1342,7 +1347,7 @@ func (p *plugin) importResources(fns importFns, resources []*ImportResource, spe
 	for resType := range showResourceTypesMap {
 		showResourceTypes = append(showResourceTypes, resType)
 	}
-	existingResources, err := fns.tfShow(p.Dir, showResourceTypes, []string{"id"})
+	existingResources, err := fns.tfShow(showResourceTypes, []string{"id"})
 	if err != nil {
 		return err
 	}
@@ -1433,7 +1438,7 @@ func (p *plugin) importResources(fns importFns, resources []*ImportResource, spe
 	// Parse the terraform show output
 	if errorToThrow == nil {
 		for _, r := range resources {
-			importedProps, err := fns.tfShowInst(p.Dir, fmt.Sprintf("%v.%v", string(*r.ResourceType), r.FinalResourceName))
+			importedProps, err := fns.tfShowInst(fmt.Sprintf("%v.%v", string(*r.ResourceType), r.FinalResourceName))
 			if err != nil {
 				errorToThrow = err
 				break
@@ -1493,7 +1498,10 @@ func (p *plugin) importResources(fns importFns, resources []*ImportResource, spe
 
 // cleanupFailedImport removes the resource from the terraform state file
 func (p *plugin) cleanupFailedImport(vmType TResourceType, vmName string) {
-	command := exec.Command(fmt.Sprintf("terraform state rm %v.%v", vmType, vmName)).InheritEnvs(true).WithDir(p.Dir)
+	command := exec.Command(fmt.Sprintf("terraform state rm %v.%v", vmType, vmName)).
+		InheritEnvs(true).
+		WithEnvs(p.envs...).
+		WithDir(p.Dir)
 	err := command.WithStdout(os.Stdout).WithStderr(os.Stdout).Start()
 	if err == nil {
 		command.Wait()

pkg/provider/terraform/instance/plugin_test.go

@@ -53,7 +53,7 @@ func TestProcessImportOptions(t *testing.T) {
 func getPlugin(t *testing.T) (*plugin, string) {
 	dir, err := ioutil.TempDir("", "infrakit-instance-terraform")
 	require.NoError(t, err)
-	tf := NewTerraformInstancePlugin(dir, 120*time.Second, false, nil)
+	tf := NewTerraformInstancePlugin(dir, 120*time.Second, false, []string{}, nil)
 	tf.(*plugin).pretend = true
 	p, is := tf.(*plugin)
 	require.True(t, is)
@@ -3701,8 +3701,7 @@ func TestImportTfShowError(t *testing.T) {
 	defer os.RemoveAll(dir)
 
 	fns := importFns{
-		tfShow: func(dirArg string, types []TResourceType, propFilter []string) (map[TResourceType]map[TResourceName]TResourceProperties, error) {
-			require.Equal(t, dir, dirArg)
+		tfShow: func(types []TResourceType, propFilter []string) (map[TResourceType]map[TResourceName]TResourceProperties, error) {
 			require.Equal(t, []TResourceType{VMAmazon}, types)
 			require.Equal(t, []string{"id"}, propFilter)
 			return nil, fmt.Errorf("Custom show error")
@@ -3735,8 +3734,7 @@ func TestImportAlreadyExists(t *testing.T) {
 	defer os.RemoveAll(dir)
 
 	fns := importFns{
-		tfShow: func(dirArg string, types []TResourceType, propFilter []string) (map[TResourceType]map[TResourceName]TResourceProperties, error) {
-			require.Equal(t, dir, dirArg)
+		tfShow: func(types []TResourceType, propFilter []string) (map[TResourceType]map[TResourceName]TResourceProperties, error) {
 			require.Equal(t, []TResourceType{VMAmazon}, types)
 			require.Equal(t, []string{"id"}, propFilter)
 			return map[TResourceType]map[TResourceName]TResourceProperties{
@@ -3783,8 +3781,7 @@ func TestImportTfImportError(t *testing.T) {
 
 	cleanVals := []string{}
 	fns := importFns{
-		tfShow: func(dirArg string, types []TResourceType, propFilter []string) (map[TResourceType]map[TResourceName]TResourceProperties, error) {
-			require.Equal(t, dir, dirArg)
+		tfShow: func(types []TResourceType, propFilter []string) (map[TResourceType]map[TResourceName]TResourceProperties, error) {
 			require.Equal(t, []TResourceType{VMAmazon}, types)
 			require.Equal(t, []string{"id"}, propFilter)
 			return map[TResourceType]map[TResourceName]TResourceProperties{}, nil
@@ -3835,8 +3832,7 @@ func TestImportTfShowInstError(t *testing.T) {
 
 	cleanVals := []string{}
 	fns := importFns{
-		tfShow: func(dirArg string, types []TResourceType, propFilter []string) (map[TResourceType]map[TResourceName]TResourceProperties, error) {
-			require.Equal(t, dir, dirArg)
+		tfShow: func(types []TResourceType, propFilter []string) (map[TResourceType]map[TResourceName]TResourceProperties, error) {
 			require.Equal(t, []TResourceType{VMAmazon}, types)
 			require.Equal(t, []string{"id"}, propFilter)
 			return map[TResourceType]map[TResourceName]TResourceProperties{}, nil
@@ -3847,8 +3843,7 @@ func TestImportTfShowInstError(t *testing.T) {
 			require.Equal(t, "123", id)
 			return nil
 		},
-		tfShowInst: func(dirArg, id string) (TResourceProperties, error) {
-			require.Equal(t, dir, dirArg)
+		tfShowInst: func(id string) (TResourceProperties, error) {
 			return nil, fmt.Errorf("Custom show inst error")
 		},
 		tfClean: func(resType TResourceType, name string) {
@@ -3892,8 +3887,7 @@ func TestImportResourceTagMap(t *testing.T) {
 	cleanInvoked := false
 	resourceName := ""
 	fns := importFns{
-		tfShow: func(dirArg string, types []TResourceType, propFilter []string) (map[TResourceType]map[TResourceName]TResourceProperties, error) {
-			require.Equal(t, dir, dirArg)
+		tfShow: func(types []TResourceType, propFilter []string) (map[TResourceType]map[TResourceName]TResourceProperties, error) {
 			require.Equal(t, []TResourceType{VMAmazon}, types)
 			require.Equal(t, []string{"id"}, propFilter)
 			return map[TResourceType]map[TResourceName]TResourceProperties{}, nil
@@ -3905,8 +3899,7 @@ func TestImportResourceTagMap(t *testing.T) {
 			require.Equal(t, "123", id)
 			return nil
 		},
-		tfShowInst: func(dirArg, id string) (TResourceProperties, error) {
-			require.Equal(t, dir, dirArg)
+		tfShowInst: func(id string) (TResourceProperties, error) {
 			prefix := "aws_instance.instance-"
 			require.True(t, strings.HasPrefix(id, prefix), fmt.Sprintf("'%v' should have prefix '%v'", id, prefix))
 			props := TResourceProperties{
@@ -3984,8 +3977,7 @@ func TestImportResourceTagSlice(t *testing.T) {
 	cleanInvoked := false
 	resourceName := ""
 	fns := importFns{
-		tfShow: func(dirArg string, types []TResourceType, propFilter []string) (map[TResourceType]map[TResourceName]TResourceProperties, error) {
-			require.Equal(t, dir, dirArg)
+		tfShow: func(types []TResourceType, propFilter []string) (map[TResourceType]map[TResourceName]TResourceProperties, error) {
 			require.Equal(t, []TResourceType{VMIBMCloud}, types)
 			require.Equal(t, []string{"id"}, propFilter)
 			return map[TResourceType]map[TResourceName]TResourceProperties{
@@ -4003,8 +3995,7 @@ func TestImportResourceTagSlice(t *testing.T) {
 			require.Equal(t, "123", id)
 			return nil
 		},
-		tfShowInst: func(dirArg, id string) (TResourceProperties, error) {
-			require.Equal(t, dir, dirArg)
+		tfShowInst: func(id string) (TResourceProperties, error) {
 			require.True(t, strings.HasPrefix(id, "ibm_compute_vm_instance.instance-"))
 			props := TResourceProperties{
 				"hostname": "actual-hostname",
@@ -4139,8 +4130,7 @@ func internalTestImportResourceDedicatedGlobal(t *testing.T, options importOptio
 	cleanInvoked := false
 	imports := []string{}
 	fns := importFns{
-		tfShow: func(dirArg string, types []TResourceType, propFilter []string) (map[TResourceType]map[TResourceName]TResourceProperties, error) {
-			require.Equal(t, dir, dirArg)
+		tfShow: func(types []TResourceType, propFilter []string) (map[TResourceType]map[TResourceName]TResourceProperties, error) {
 			require.Len(t, types, 3)
 			require.Contains(t, types, VMIBMCloud)
 			require.Contains(t, types, TResourceType("ibm_storage_file"))
@@ -4174,8 +4164,7 @@ func internalTestImportResourceDedicatedGlobal(t *testing.T, options importOptio
 			imports = append(imports, fmt.Sprintf("%v.%v.%v", resType, resName, id))
 			return nil
 		},
-		tfShowInst: func(dirArg, id string) (TResourceProperties, error) {
-			require.Equal(t, dir, dirArg)
+		tfShowInst: func(id string) (TResourceProperties, error) {
 			if strings.HasPrefix(id, "ibm_compute_vm_instance.instance-") {
 				props := TResourceProperties{
 					"hostname": "actual-hostname",