Add template and environment parsing to gophercloud

Openstack Heat expects the client to do some parsing client side,
specifically for nested templates and environments which refer
to local files. This patch adds a recursive parser for both the
template and environment files to gophercloud. The interfaces
are also changed to make use of the new parsing functionality.

Author: pratikmallya

openstack/orchestration/v1/stacks/environment.go

@@ -0,0 +1,121 @@
+package stacks
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+)
+
+// an interface to represent stack environments
+type Environment struct {
+	TE
+}
+
+// allowed sections in a stack environment file
+var EnvironmentSections = map[string]bool{
+	"parameters":         true,
+	"parameter_defaults": true,
+	"resource_registry":  true,
+}
+
+func (e *Environment) Validate() error {
+	if e.Parsed == nil {
+		if err := e.Parse(); err != nil {
+			return err
+		}
+	}
+	for key, _ := range e.Parsed {
+		if _, ok := EnvironmentSections[key]; !ok {
+			return errors.New(fmt.Sprintf("Environment has wrong section: %s", key))
+		}
+	}
+	return nil
+}
+
+// Parse environment file to resolve the urls of the resources
+func GetRRFileContents(e *Environment, ignoreIf igFunc) error {
+	if e.Files == nil {
+		e.Files = make(map[string]string)
+	}
+	if e.fileMaps == nil {
+		e.fileMaps = make(map[string]string)
+	}
+	rr := e.Parsed["resource_registry"]
+	// search the resource registry for URLs
+	switch rr.(type) {
+	case map[string]interface{}, map[interface{}]interface{}:
+		rr_map, err := toStringKeys(rr)
+		if err != nil {
+			return err
+		}
+		var baseURL string
+		if val, ok := rr_map["base_url"]; ok {
+			baseURL = val.(string)
+		} else {
+			baseURL = e.baseURL
+		}
+		// use a fake template to fetch contents from URLs
+		tempTemplate := new(Template)
+		tempTemplate.baseURL = baseURL
+		tempTemplate.client = e.client
+
+		if err = GetFileContents(tempTemplate, rr, ignoreIf, false); err != nil {
+			return err
+		}
+		// check the `resources` section (if it exists) for more URLs
+		if val, ok := rr_map["resources"]; ok {
+			switch val.(type) {
+			case map[string]interface{}, map[interface{}]interface{}:
+				resources_map, err := toStringKeys(val)
+				if err != nil {
+					return err
+				}
+				for _, v := range resources_map {
+					switch v.(type) {
+					case map[string]interface{}, map[interface{}]interface{}:
+						resource_map, err := toStringKeys(v)
+						if err != nil {
+							return err
+						}
+						var resourceBaseURL string
+						// if base_url for the resource type is defined, use it
+						if val, ok := resource_map["base_url"]; ok {
+							resourceBaseURL = val.(string)
+						} else {
+							resourceBaseURL = baseURL
+						}
+						tempTemplate.baseURL = resourceBaseURL
+						if err := GetFileContents(tempTemplate, v, ignoreIf, false); err != nil {
+							return err
+						}
+					}
+
+				}
+
+			}
+		}
+		e.Files = tempTemplate.Files
+		return nil
+	default:
+		return nil
+	}
+}
+
+// function to choose keys whose values are other environment files
+func ignoreIfEnvironment(key string, value interface{}) bool {
+	// base_url and hooks refer to components which cannot have urls
+	if key == "base_url" || key == "hooks" {
+		return true
+	}
+	// if value is not string, it cannot be a URL
+	valueString, ok := value.(string)
+	if !ok {
+		return true
+	}
+	// if value contains `::`, it must be a reference to another resource type
+	// e.g. OS::Nova::Server : Rackspace::Cloud::Server
+	if strings.Contains(valueString, "::") {
+		return true
+	}
+	return false
+}

openstack/orchestration/v1/stacks/environment_test.go

@@ -0,0 +1,184 @@
+package stacks
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"testing"
+
+	th "github.com/rackspace/gophercloud/testhelper"
+)
+
+func TestEnvironmentValidation(t *testing.T) {
+	environmentJSON := new(Environment)
+	environmentJSON.Bin = []byte(ValidJSONEnvironment)
+	err := environmentJSON.Validate()
+	th.AssertNoErr(t, err)
+
+	environmentYAML := new(Environment)
+	environmentYAML.Bin = []byte(ValidYAMLEnvironment)
+	err = environmentYAML.Validate()
+	th.AssertNoErr(t, err)
+
+	environmentInvalid := new(Environment)
+	environmentInvalid.Bin = []byte(InvalidEnvironment)
+	if err = environmentInvalid.Validate(); err == nil {
+		t.Error("environment validation did not catch invalid environment")
+	}
+}
+
+func TestEnvironmentParsing(t *testing.T) {
+	environmentJSON := new(Environment)
+	environmentJSON.Bin = []byte(ValidJSONEnvironment)
+	err := environmentJSON.Parse()
+	th.AssertNoErr(t, err)
+	th.AssertDeepEquals(t, ValidJSONEnvironmentParsed, environmentJSON.Parsed)
+
+	environmentYAML := new(Environment)
+	environmentYAML.Bin = []byte(ValidJSONEnvironment)
+	err = environmentYAML.Parse()
+	th.AssertNoErr(t, err)
+	th.AssertDeepEquals(t, ValidJSONEnvironmentParsed, environmentYAML.Parsed)
+
+	environmentInvalid := new(Environment)
+	environmentInvalid.Bin = []byte("Keep Austin Weird")
+	err = environmentInvalid.Parse()
+	if err == nil {
+		t.Error("environment parsing did not catch invalid environment")
+	}
+}
+
+func TestIgnoreIfEnvironment(t *testing.T) {
+	var keyValueTests = []struct {
+		key   string
+		value interface{}
+		out   bool
+	}{
+		{"base_url", "afksdf", true},
+		{"not_type", "hooks", false},
+		{"get_file", "::", true},
+		{"hooks", "dfsdfsd", true},
+		{"type", "sdfubsduf.yaml", false},
+		{"type", "sdfsdufs.environment", false},
+		{"type", "sdfsdf.file", false},
+		{"type", map[string]string{"key": "value"}, true},
+	}
+	var result bool
+	for _, kv := range keyValueTests {
+		result = ignoreIfEnvironment(kv.key, kv.value)
+		if result != kv.out {
+			t.Errorf("key: %v, value: %v expected: %v, actual: %v", kv.key, kv.value, kv.out, result)
+		}
+	}
+}
+
+func TestGetRRFileContents(t *testing.T) {
+	th.SetupHTTP()
+	defer th.TeardownHTTP()
+	environment_content := `
+heat_template_version: 2013-05-23
+
+description:
+  Heat WordPress template to support F18, using only Heat OpenStack-native
+  resource types, and without the requirement for heat-cfntools in the image.
+  WordPress is web software you can use to create a beautiful website or blog.
+  This template installs a single-instance WordPress deployment using a local
+  MySQL database to store the data.
+
+parameters:
+
+  key_name:
+    type: string
+    description : Name of a KeyPair to enable SSH access to the instance
+
+resources:
+  wordpress_instance:
+    type: OS::Nova::Server
+    properties:
+      image: { get_param: image_id }
+      flavor: { get_param: instance_type }
+      key_name: { get_param: key_name }`
+
+	db_content := `
+heat_template_version: 2014-10-16
+
+description:
+  Test template for Trove resource capabilities
+
+parameters:
+  db_pass:
+    type: string
+    hidden: true
+    description: Database access password
+    default: secrete
+
+resources:
+
+service_db:
+  type: OS::Trove::Instance
+  properties:
+    name: trove_test_db
+    datastore_type: mariadb
+    flavor: 1GB Instance
+    size: 10
+    databases:
+    - name: test_data
+    users:
+    - name: kitchen_sink
+      password: { get_param: db_pass }
+      databases: [ test_data ]`
+	baseurl, err := getBasePath()
+	th.AssertNoErr(t, err)
+
+	fakeEnvURL := strings.Join([]string{baseurl, "my_env.yaml"}, "/")
+	urlparsed, err := url.Parse(fakeEnvURL)
+	th.AssertNoErr(t, err)
+	// handler for my_env.yaml
+	th.Mux.HandleFunc(urlparsed.Path, func(w http.ResponseWriter, r *http.Request) {
+		th.TestMethod(t, r, "GET")
+		w.Header().Set("Content-Type", "application/jason")
+		w.WriteHeader(http.StatusOK)
+		fmt.Fprintf(w, environment_content)
+	})
+
+	fakeDBURL := strings.Join([]string{baseurl, "my_db.yaml"}, "/")
+	urlparsed, err = url.Parse(fakeDBURL)
+	th.AssertNoErr(t, err)
+
+	// handler for my_db.yaml
+	th.Mux.HandleFunc(urlparsed.Path, func(w http.ResponseWriter, r *http.Request) {
+		th.TestMethod(t, r, "GET")
+		w.Header().Set("Content-Type", "application/jason")
+		w.WriteHeader(http.StatusOK)
+		fmt.Fprintf(w, db_content)
+	})
+
+	client := fakeClient{BaseClient: getHTTPClient()}
+	env := new(Environment)
+	env.Bin = []byte(`{"resource_registry": {"My::WP::Server": "my_env.yaml", "resources": {"my_db_server": {"OS::DBInstance": "my_db.yaml"}}}}`)
+	env.client = client
+
+	err = env.Parse()
+	th.AssertNoErr(t, err)
+	err = GetRRFileContents(env, ignoreIfEnvironment)
+	th.AssertNoErr(t, err)
+	expected_env_files_content := "\nheat_template_version: 2013-05-23\n\ndescription:\n  Heat WordPress template to support F18, using only Heat OpenStack-native\n  resource types, and without the requirement for heat-cfntools in the image.\n  WordPress is web software you can use to create a beautiful website or blog.\n  This template installs a single-instance WordPress deployment using a local\n  MySQL database to store the data.\n\nparameters:\n\n  key_name:\n    type: string\n    description : Name of a KeyPair to enable SSH access to the instance\n\nresources:\n  wordpress_instance:\n    type: OS::Nova::Server\n    properties:\n      image: { get_param: image_id }\n      flavor: { get_param: instance_type }\n      key_name: { get_param: key_name }"
+	expected_db_files_content := "\nheat_template_version: 2014-10-16\n\ndescription:\n  Test template for Trove resource capabilities\n\nparameters:\n  db_pass:\n    type: string\n    hidden: true\n    description: Database access password\n    default: secrete\n\nresources:\n\nservice_db:\n  type: OS::Trove::Instance\n  properties:\n    name: trove_test_db\n    datastore_type: mariadb\n    flavor: 1GB Instance\n    size: 10\n    databases:\n    - name: test_data\n    users:\n    - name: kitchen_sink\n      password: { get_param: db_pass }\n      databases: [ test_data ]"
+
+	th.AssertEquals(t, expected_env_files_content, env.Files[fakeEnvURL])
+	th.AssertEquals(t, expected_db_files_content, env.Files[fakeDBURL])
+
+	env.FixFileRefs()
+	expected_parsed := map[string]interface{}{
+		"resource_registry": "2015-04-30",
+		"My::WP::Server":    fakeEnvURL,
+		"resources": map[string]interface{}{
+			"my_db_server": map[string]interface{}{
+				"OS::DBInstance": fakeDBURL,
+			},
+		},
+	}
+	env.Parse()
+	th.AssertDeepEquals(t, expected_parsed, env.Parsed)
+}