helper/resource: Additional Go documentation for acceptance testing environment variables and CLI installation behaviors (#890)

Reference: #830

This also removes two unused internal functions from the internal/plugintest package.

The missing terraform.io website documentation for this environment variable will be separately submitted as well.

Author: bflad

helper/resource/environment_variables.go

@@ -0,0 +1,32 @@
+package resource
+
+// Environment variables for acceptance testing. Additional environment
+// variable constants can be found in the internal/plugintest package.
+const (
+	// Environment variable to enable acceptance tests using this package's
+	// ParallelTest and Test functions whose TestCase does not enable the
+	// IsUnitTest field. Defaults to disabled, in which each test will call
+	// (*testing.T).Skip(). Can be set to any value to enable acceptance tests,
+	// however "1" is conventional.
+	EnvTfAcc = "TF_ACC"
+
+	// Environment variable with hostname for the provider under acceptance
+	// test. The hostname is the first portion of the full provider source
+	// address, such as "example.com" in example.com/myorg/myprovider. Defaults
+	// to "registry.terraform.io".
+	//
+	// Only required if any Terraform configuration set via the TestStep
+	// type Config field includes a provider source, such as the terraform
+	// configuration block required_providers attribute.
+	EnvTfAccProviderHost = "TF_ACC_PROVIDER_HOST"
+
+	// Environment variable with namespace for the provider under acceptance
+	// test. The namespace is the second portion of the full provider source
+	// address, such as "myorg" in registry.terraform.io/myorg/myprovider.
+	// Defaults to "-" for Terraform 0.12-0.13 compatibility and "hashicorp".
+	//
+	// Only required if any Terraform configuration set via the TestStep
+	// type Config field includes a provider source, such as the terraform
+	// configuration block required_providers attribute.
+	EnvTfAccProviderNamespace = "TF_ACC_PROVIDER_NAMESPACE"
+)

helper/resource/plugin.go

@@ -53,12 +53,12 @@ func runProviderCommand(t testing.T, f func() error, wd *plugintest.WorkingDir,
 	// the host or namespace using environment variables.
 	var namespaces []string
 	host := "registry.terraform.io"
-	if v := os.Getenv("TF_ACC_PROVIDER_NAMESPACE"); v != "" {
+	if v := os.Getenv(EnvTfAccProviderNamespace); v != "" {
 		namespaces = append(namespaces, v)
 	} else {
 		namespaces = append(namespaces, "-", "hashicorp")
 	}
-	if v := os.Getenv("TF_ACC_PROVIDER_HOST"); v != "" {
+	if v := os.Getenv(EnvTfAccProviderHost); v != "" {
 		host = v
 	}
 
@@ -327,10 +327,10 @@ func runProviderCommand(t testing.T, f func() error, wd *plugintest.WorkingDir,
 func getProviderAddr(name string) string {
 	host := "registry.terraform.io"
 	namespace := "hashicorp"
-	if v := os.Getenv("TF_ACC_PROVIDER_NAMESPACE"); v != "" {
+	if v := os.Getenv(EnvTfAccProviderNamespace); v != "" {
 		namespace = v
 	}
-	if v := os.Getenv("TF_ACC_PROVIDER_HOST"); v != "" {
+	if v := os.Getenv(EnvTfAccProviderHost); v != "" {
 		host = v
 	}
 	return strings.TrimSuffix(host, "/") + "/" +

helper/resource/testing.go

@@ -84,6 +84,27 @@ func AddTestSweepers(name string, s *Sweeper) {
 	sweeperFuncs[name] = s
 }
 
+// TestMain adds sweeper functionality to the "go test" command, otherwise
+// tests are executed as normal. Most provider acceptance tests are written
+// using the Test() function of this package, which imposes its own
+// requirements and Terraform CLI behavior. Refer to that function's
+// documentation for additional details.
+//
+// Sweepers enable infrastructure cleanup functions to be included with
+// resource definitions, typically so developers can remove all resources of
+// that resource type from testing infrastructure in case of failures that
+// prevented the normal resource destruction behavior of acceptance tests.
+// Use the AddTestSweepers() function to configure available sweepers.
+//
+// Sweeper flags added to the "go test" command:
+//
+// -sweep: Comma-separated list of locations/regions to run available sweepers.
+// -sweep-allow-failues: Enable to allow other sweepers to run after failures.
+// -sweep-run: Comma-separated list of resource type sweepers to run. Defaults
+//             to all sweepers.
+//
+// Refer to the Env prefixed constants for environment variables that further
+// control testing functionality.
 func TestMain(m interface {
 	Run() int
 }) {
@@ -252,7 +273,8 @@ func runSweeperWithRegion(region string, s *Sweeper, sweepers map[string]*Sweepe
 	return runE
 }
 
-const TestEnvVar = "TF_ACC"
+// Deprecated: Use EnvTfAcc instead.
+const TestEnvVar = EnvTfAcc
 
 // TestCheckFunc is the callback type used with acceptance tests to check
 // the state of a resource. The state passed in is the latest state known,
@@ -275,6 +297,9 @@ type ErrorCheckFunc func(error) error
 //
 // When the destroy plan is executed, the config from the last TestStep
 // is used to plan it.
+//
+// Refer to the Env prefixed constants for environment variables that further
+// control testing functionality.
 type TestCase struct {
 	// IsUnitTest allows a test to run regardless of the TF_ACC
 	// environment variable. This should be used with care - only for
@@ -376,6 +401,9 @@ type ExternalProvider struct {
 // Multiple TestSteps can be sequenced in a Test to allow testing
 // potentially complex update logic. In general, simply create/destroy
 // tests will only need one step.
+//
+// Refer to the Env prefixed constants for environment variables that further
+// control testing functionality.
 type TestStep struct {
 	// ResourceName should be set to the name of the resource
 	// that is being tested. Example: "aws_instance.foo". Various test
@@ -508,11 +536,13 @@ type TestStep struct {
 }
 
 // ParallelTest performs an acceptance test on a resource, allowing concurrency
-// with other ParallelTest.
+// with other ParallelTest. The number of concurrent tests is controlled by the
+// "go test" command -parallel flag.
 //
 // Tests will fail if they do not properly handle conditions to allow multiple
 // tests to occur against the same resource or service (e.g. random naming).
-// All other requirements of the Test function also apply to this function.
+//
+// Test() function requirements and documentation also apply to this function.
 func ParallelTest(t testing.T, c TestCase) {
 	t.Helper()
 	t.Parallel()
@@ -529,16 +559,37 @@ func ParallelTest(t testing.T, c TestCase) {
 // the "-test.v" flag) is set. Because some acceptance tests take quite
 // long, we require the verbose flag so users are able to see progress
 // output.
+//
+// Use the ParallelTest() function to automatically set (*testing.T).Parallel()
+// to enable testing concurrency. Use the UnitTest() function to automatically
+// set the TestCase type IsUnitTest field.
+//
+// This function will automatically find or install Terraform CLI into a
+// temporary directory, based on the following behavior:
+//
+// - If the TF_ACC_TERRAFORM_PATH environment variable is set, that Terraform
+//   CLI binary is used if found and executable. If not found or executable,
+//   an error will be returned unless the TF_ACC_TERRAFORM_VERSION environment
+//   variable is also set.
+// - If the TF_ACC_TERRAFORM_VERSION environment variable is set, install and
+//   use that Terraform CLI version.
+// - If both the TF_ACC_TERRAFORM_PATH and TF_ACC_TERRAFORM_VERSION environment
+//   variables are unset, perform a lookup for the Terraform CLI binary based
+//   on the operating system PATH. If not found, the latest available Terraform
+//   CLI binary is installed.
+//
+// Refer to the Env prefixed constants for additional details about these
+// environment variables, and others, that control testing functionality.
 func Test(t testing.T, c TestCase) {
 	t.Helper()
 
 	// We only run acceptance tests if an env var is set because they're
 	// slow and generally require some outside configuration. You can opt out
 	// of this with OverrideEnvVar on individual TestCases.
-	if os.Getenv(TestEnvVar) == "" && !c.IsUnitTest {
+	if os.Getenv(EnvTfAcc) == "" && !c.IsUnitTest {
 		t.Skip(fmt.Sprintf(
 			"Acceptance tests skipped unless env '%s' set",
-			TestEnvVar))
+			EnvTfAcc))
 		return
 	}
 
@@ -621,6 +672,8 @@ func testProviderConfig(c TestCase) (string, error) {
 // UnitTest is a helper to force the acceptance testing harness to run in the
 // normal unit test suite. This should only be used for resource that don't
 // have any external dependencies.
+//
+// Test() function requirements and documentation also apply to this function.
 func UnitTest(t testing.T, c TestCase) {
 	t.Helper()
 

helper/resource/testing_test.go

@@ -18,7 +18,7 @@ import (
 )
 
 func init() {
-	if err := os.Setenv(TestEnvVar, "1"); err != nil {
+	if err := os.Setenv(EnvTfAcc, "1"); err != nil {
 		panic(err)
 	}
 }

internal/plugintest/config.go

@@ -30,10 +30,10 @@ type Config struct {
 // DiscoverConfig uses environment variables and other means to automatically
 // discover a reasonable test helper configuration.
 func DiscoverConfig(sourceDir string) (*Config, error) {
-	tfVersion := strings.TrimPrefix(os.Getenv("TF_ACC_TERRAFORM_VERSION"), "v")
-	tfPath := os.Getenv("TF_ACC_TERRAFORM_PATH")
+	tfVersion := strings.TrimPrefix(os.Getenv(EnvTfAccTerraformVersion), "v")
+	tfPath := os.Getenv(EnvTfAccTerraformPath)
 
-	tempDir := os.Getenv("TF_ACC_TEMP_DIR")
+	tempDir := os.Getenv(EnvTfAccTempDir)
 	tfDir, err := ioutil.TempDir(tempDir, "plugintest-terraform")
 	if err != nil {
 		return nil, fmt.Errorf("failed to create temp dir: %w", err)

internal/plugintest/environment_variables.go

@@ -0,0 +1,52 @@
+package plugintest
+
+// Environment variables
+const (
+	// Environment variable with acceptance testing temporary directory for
+	// testing files and Terraform CLI installation, if installation is
+	// required. By default, the operating system temporary directory is used.
+	//
+	// Setting TF_ACC_TERRAFORM_PATH does not override this value for Terraform
+	// CLI installation, if installation is required.
+	EnvTfAccTempDir = "TF_ACC_TEMP_DIR"
+
+	// Environment variable with path to save Terraform logs during acceptance
+	// testing. This value sets TF_LOG_PATH in a safe manner when executing
+	// Terraform CLI commands, which would otherwise be ignored since it could
+	// interfere with how the underlying execution is performed.
+	EnvTfAccLogPath = "TF_ACC_LOG_PATH"
+
+	// Environment variable with acceptance testing Terraform CLI version to
+	// download from releases.hashicorp.com, checksum verify, and install. The
+	// value can be any valid Terraform CLI version, such as 1.1.6, with or
+	// without a prepended v character.
+	//
+	// Setting this value takes precedence over using an available Terraform
+	// binary in the operation system PATH, or if not found, installing the
+	// latest version according to checkpoint.hashicorp.com.
+	//
+	// By default, the binary is installed in the operating system temporary
+	// directory, however that directory can be overridden with the
+	// TF_ACC_TEMP_DIR environment variable.
+	//
+	// If TF_ACC_TERRAFORM_PATH is also set, this installation method is
+	// only invoked when a binary does not exist at that path. No version
+	// checks are performed against an existing TF_ACC_TERRAFORM_PATH.
+	EnvTfAccTerraformVersion = "TF_ACC_TERRAFORM_VERSION"
+
+	// Acceptance testing path to Terraform CLI binary.
+	//
+	// Setting this value takes precedence over using an available Terraform
+	// binary in the operation system PATH, or if not found, installing the
+	// latest version according to checkpoint.hashicorp.com. This value does
+	// not override TF_ACC_TEMP_DIR for Terraform CLI installation, if
+	// installation is required.
+	//
+	// If TF_ACC_TERRAFORM_VERSION is not set, the binary must exist and be
+	// executable, or an error will be returned.
+	//
+	// If TF_ACC_TERRAFORM_VERSION is also set, that Terraform CLI version
+	// will be installed if a binary is not found at the given path. No version
+	// checks are performed against an existing binary.
+	EnvTfAccTerraformPath = "TF_ACC_TERRAFORM_PATH"
+)

internal/plugintest/guard.go

@@ -2,54 +2,8 @@ package plugintest
 
 import (
 	"fmt"
-	"os"
-	"testing"
 )
 
-// AcceptanceTest is a test guard that will produce a log and call SkipNow on
-// the given TestControl if the environment variable TF_ACC isn't set to
-// indicate that the caller wants to run acceptance tests.
-//
-// Call this immediately at the start of each acceptance test function to
-// signal that it may cost money and thus requires this opt-in enviromment
-// variable.
-//
-// For the purpose of this function, an "acceptance test" is any est that
-// reaches out to services that are not directly controlled by the test program
-// itself, particularly if those requests may lead to service charges. For any
-// system where it is possible and realistic to run a local instance of the
-// service for testing (e.g. in a daemon launched by the test program itself),
-// prefer to do this and _don't_ call AcceptanceTest, thus allowing tests to be
-// run more easily and without external cost by contributors.
-func AcceptanceTest(t TestControl) {
-	t.Helper()
-	if os.Getenv("TF_ACC") != "" {
-		t.Log("TF_ACC is not set")
-		t.SkipNow()
-	}
-}
-
-// LongTest is a test guard that will produce a log and call SkipNow on the
-// given TestControl if the test harness is currently running in "short mode".
-//
-// What is considered a "long test" will always be pretty subjective, but test
-// implementers should think of this in terms of what seems like it'd be
-// inconvenient to run repeatedly for quick feedback while testing a new feature
-// under development.
-//
-// When testing resource types that always take several minutes to complete
-// operations, consider having a single general test that covers the basic
-// functionality and then mark any other more specific tests as long tests so
-// that developers can quickly smoke-test a particular feature when needed
-// but can still run the full set of tests for a feature when needed.
-func LongTest(t TestControl) {
-	t.Helper()
-	if testing.Short() {
-		t.Log("skipping long test because of short mode")
-		t.SkipNow()
-	}
-}
-
 // TestControl is an interface requiring a subset of *testing.T which is used
 // by the test guards and helpers in this package. Most callers can simply
 // pass their *testing.T value here, but the interface allows other

internal/plugintest/helper.go

@@ -65,7 +65,7 @@ func AutoInitHelper(sourceDir string) (*Helper, error) {
 // behind in the system's temporary directory. There is currently no way to
 // automatically clean those up.
 func InitHelper(config *Config) (*Helper, error) {
-	tempDir := os.Getenv("TF_ACC_TEMP_DIR")
+	tempDir := os.Getenv(EnvTfAccTempDir)
 	baseDir, err := ioutil.TempDir(tempDir, "plugintest")
 	if err != nil {
 		return nil, fmt.Errorf("failed to create temporary directory for test helper: %s", err)

internal/plugintest/working_dir.go

@@ -99,7 +99,7 @@ func (wd *WorkingDir) SetConfig(cfg string) error {
 		return err
 	}
 
-	if p := os.Getenv("TF_ACC_LOG_PATH"); p != "" {
+	if p := os.Getenv(EnvTfAccLogPath); p != "" {
 		if err := wd.tf.SetLogPath(p); err != nil {
 			return fmt.Errorf("unable to set log path: %w", err)
 		}