query: refactor query test step logic into testStepNewQuery()

helper/resource/query/examplecloud_list_test.go

@@ -6,6 +6,7 @@ package query_test
 import (
 	"github.com/hashicorp/terraform-plugin-go/tfprotov6"
 	"github.com/hashicorp/terraform-plugin-go/tftypes"
+
 	"github.com/hashicorp/terraform-plugin-testing/internal/testing/testprovider"
 	"github.com/hashicorp/terraform-plugin-testing/internal/testing/testsdk/list"
 	"github.com/hashicorp/terraform-plugin-testing/internal/teststep"

helper/resource/query/query_test.go

@@ -4,11 +4,15 @@
 package query_test
 
 import (
+	"regexp"
 	"testing"
 
 	"github.com/hashicorp/terraform-plugin-go/tfprotov6"
+	"github.com/hashicorp/terraform-plugin-go/tftypes"
+
 	r "github.com/hashicorp/terraform-plugin-testing/helper/resource"
 	"github.com/hashicorp/terraform-plugin-testing/internal/testing/testprovider"
+	"github.com/hashicorp/terraform-plugin-testing/internal/testing/testsdk/list"
 	"github.com/hashicorp/terraform-plugin-testing/internal/testing/testsdk/providerserver"
 	"github.com/hashicorp/terraform-plugin-testing/knownvalue"
 	"github.com/hashicorp/terraform-plugin-testing/querycheck"
@@ -100,3 +104,88 @@ func TestQuery(t *testing.T) {
 		},
 	})
 }
+
+func TestQuery_ExpectError_ValidationError(t *testing.T) {
+	t.Parallel()
+
+	r.UnitTest(t, r.TestCase{
+		TerraformVersionChecks: []tfversion.TerraformVersionCheck{
+			tfversion.SkipBelow(tfversion.Version1_14_0),
+		},
+		ProtoV6ProviderFactories: map[string]func() (tfprotov6.ProviderServer, error){
+			"examplecloud": providerserver.NewProviderServer(testprovider.Provider{
+				ListResources: map[string]testprovider.ListResource{
+					"examplecloud_containerette": {
+						IncludeResource: true,
+						SchemaResponse: &list.SchemaResponse{
+							Schema: &tfprotov6.Schema{
+								Block: &tfprotov6.SchemaBlock{
+									Attributes: []*tfprotov6.SchemaAttribute{
+										{
+											Name:     "resource_group_name",
+											Type:     tftypes.String,
+											Required: true,
+										},
+									},
+								},
+							},
+						},
+						ValidateListConfigResponse: &list.ValidateListConfigResponse{
+							Diagnostics: []*tfprotov6.Diagnostic{
+								{
+									Severity: tfprotov6.DiagnosticSeverityError,
+									Summary:  "Diagnostic summary",
+									Detail:   "Diagnostic details",
+								},
+							},
+						},
+					},
+				},
+				Resources: map[string]testprovider.Resource{
+					"examplecloud_containerette": examplecloudResource(),
+				},
+			}),
+		},
+		Steps: []r.TestStep{
+			{ // config mode step 1 needs tf file with terraform providers block
+				// this step should provision all the resources that the query is support to list
+				// for simplicity we're only "provisioning" one here
+				Config: `
+				resource "examplecloud_containerette" "primary" {
+					name                = "banana"
+					resource_group_name = "foo"
+					location  			= "westeurope"
+
+					instances = 5
+				}`,
+			},
+			{ // Query mode step 2, operates on .tfquery.hcl files (needs tf file with terraform providers block)
+				// ```provider "examplecloud" {}``` has a slightly different syntax for a .tfquery.hcl file
+				// provider bock simulates a real providers workflow
+				// "config" in this case means configuration of the list resource/filters
+
+				Query: true,
+				Config: `
+				provider "examplecloud" {} 
+
+				list "examplecloud_containerette" "test" {
+					provider = examplecloud
+
+					config {
+						resource_group_name = "foo"
+ 					}
+				}
+
+				list "examplecloud_containerette" "test2" {
+					provider = examplecloud
+
+					config {
+						resource_group_name = "bar"
+					}
+				}
+				`,
+				ExpectError: regexp.MustCompile(`Diagnostic summary`),
+			},
+		},
+	})
+}

helper/resource/testing_new.go

@@ -15,7 +15,6 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/hashicorp/go-version"
 	tfjson "github.com/hashicorp/terraform-json"
-	"github.com/hashicorp/terraform-plugin-testing/helper/resource/query"
 	"github.com/mitchellh/go-testing-interface"
 
 	"github.com/hashicorp/terraform-plugin-testing/config"
@@ -362,33 +361,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest
 		if step.Query {
 			logging.HelperResourceTrace(ctx, "TestStep is Query mode")
 
-			queryConfigRequest := teststep.ConfigurationRequest{
-				Raw: &step.Config,
-			}
-			err := wd.SetQuery(ctx, teststep.Configuration(queryConfigRequest), step.ConfigVariables)
-			if err != nil {
-				t.Fatalf("Step %d/%d error setting query: %s", stepNumber, len(c.Steps), err)
-			}
-
-			err = runProviderCommand(ctx, t, wd, providers, func() error {
-				return wd.Init(ctx)
-			})
-			if err != nil {
-				t.Fatalf("Step %d/%d error running init: %s", stepNumber, len(c.Steps), err)
-			}
-
-			var queryOut []tfjson.LogMsg
-			err = runProviderCommand(ctx, t, wd, providers, func() error {
-				var err error
-				queryOut, err = wd.Query(ctx)
-				return err
-			})
-			if err != nil {
-				fmt.Printf("Step %d/%d Query Output:\n%s\n", stepNumber, len(c.Steps), queryOut)
-				t.Fatalf("Step %d/%d error running query: %s", stepNumber, len(c.Steps), err)
-			}
-
-			err = query.RunQueryChecks(ctx, t, queryOut, step.QueryResultChecks)
+			err := testStepNewQuery(ctx, t, wd, step, providers)
 
 			if step.ExpectError != nil {
 				logging.HelperResourceDebug(ctx, "Checking TestStep ExpectError")
@@ -414,7 +387,6 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest
 					)
 					t.Fatalf("Step %d/%d error running query checks: %s", stepNumber, len(c.Steps), err)
 				}
-				fmt.Printf("Step %d/%d Query Output:\n%s\n", stepNumber, len(c.Steps), queryOut)
 			}
 
 			logging.HelperResourceDebug(ctx, "Finished TestStep")

helper/resource/testing_new_query.go

@@ -0,0 +1,47 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package resource
+
+import (
+	"context"
+	"fmt"
+
+	tfjson "github.com/hashicorp/terraform-json"
+	"github.com/mitchellh/go-testing-interface"
+
+	"github.com/hashicorp/terraform-plugin-testing/helper/resource/query"
+	"github.com/hashicorp/terraform-plugin-testing/internal/plugintest"
+	"github.com/hashicorp/terraform-plugin-testing/internal/teststep"
+)
+
+func testStepNewQuery(ctx context.Context, t testing.T, wd *plugintest.WorkingDir, step TestStep, providers *providerFactories) error {
+	t.Helper()
+
+	queryConfigRequest := teststep.ConfigurationRequest{
+		Raw: &step.Config,
+	}
+	err := wd.SetQuery(ctx, teststep.Configuration(queryConfigRequest), step.ConfigVariables)
+	if err != nil {
+		return fmt.Errorf("Error setting query config: %w", err)
+	}
+
+	err = runProviderCommand(ctx, t, wd, providers, func() error {
+		return wd.Init(ctx)
+	})
+	if err != nil {
+		t.Fatalf("Error getting init: %s", err)
+	}
+
+	var queryOut []tfjson.LogMsg
+	err = runProviderCommand(ctx, t, wd, providers, func() error {
+		var err error
+		queryOut, err = wd.Query(ctx)
+		return err
+	})
+	if err != nil {
+		return err
+	}
+
+	return query.RunQueryChecks(ctx, t, queryOut, step.QueryResultChecks)
+}

internal/plugintest/working_dir.go

@@ -6,12 +6,13 @@ package plugintest
 import (
 	"context"
 	"fmt"
-	"github.com/hashicorp/terraform-exec/tfexec"
-	tfjson "github.com/hashicorp/terraform-json"
 	"io"
 	"os"
 	"path/filepath"
 
+	"github.com/hashicorp/terraform-exec/tfexec"
+	tfjson "github.com/hashicorp/terraform-json"
+
 	"github.com/hashicorp/terraform-plugin-testing/config"
 	"github.com/hashicorp/terraform-plugin-testing/internal/logging"
 	"github.com/hashicorp/terraform-plugin-testing/internal/teststep"

internal/testing/testprovider/list_resource.go

@@ -18,6 +18,12 @@ type ListResource struct {
 	ValidateListConfigResponse *list.ValidateListConfigResponse
 }
 
+func (r ListResource) ValidateListConfig(ctx context.Context, req list.ValidateListConfigRequest, resp *list.ValidateListConfigResponse) {
+	if r.ValidateListConfigResponse != nil {
+		resp.Diagnostics = r.ValidateListConfigResponse.Diagnostics
+	}
+}
+
 func (r ListResource) Schema(ctx context.Context, req list.SchemaRequest, resp *list.SchemaResponse) {
 	if r.SchemaResponse != nil {
 		resp.Diagnostics = r.SchemaResponse.Diagnostics

internal/testing/testsdk/list/list_resource.go

@@ -14,6 +14,7 @@ import (
 type ListResource interface {
 	Schema(context.Context, SchemaRequest, *SchemaResponse)
 	List(context.Context, ListRequest, *ListResultsStream)
+	ValidateListConfig(context.Context, ValidateListConfigRequest, *ValidateListConfigResponse)
 }
 
 type ListRequest struct {
@@ -45,7 +46,12 @@ type ListResult struct {
 	Diagnostics []*tfprotov6.Diagnostic
 }
 
+type ValidateListConfigRequest struct {
+	Config tftypes.Value
+}
+
 type ValidateListConfigResponse struct {
+	Diagnostics []*tfprotov6.Diagnostic
 }
 
 type SchemaRequest struct{}

internal/testing/testsdk/providerserver/providerserver.go

@@ -1104,7 +1104,56 @@ func (s ProviderServer) ListResource(ctx context.Context, req *tfprotov6.ListRes
 }
 
 func (s ProviderServer) ValidateListResourceConfig(ctx context.Context, req *tfprotov6.ValidateListResourceConfigRequest) (*tfprotov6.ValidateListResourceConfigResponse, error) {
-	return &tfprotov6.ValidateListResourceConfigResponse{}, nil
+	// Copy over identity if it's supported
+	identitySchemaReq := resource.IdentitySchemaRequest{}
+	identitySchemaResp := &resource.IdentitySchemaResponse{}
+
+	r, err := ProviderResource(s.Provider, req.TypeName)
+	if err != nil {
+		return nil, fmt.Errorf("failed to retrieve resource: %v", err)
+	}
+	r.IdentitySchema(ctx, identitySchemaReq, identitySchemaResp)
+	if len(identitySchemaResp.Diagnostics) > 0 {
+		return nil, fmt.Errorf("failed to retrieve resource schema: %v", identitySchemaResp.Diagnostics)
+	}
+
+	listresource, diag := ProviderListResource(s.Provider, req.TypeName)
+	if diag != nil {
+		return nil, fmt.Errorf("failed to retrieve resource identity schema: %v", err)
+	}
+
+	configSchemaReq := list.SchemaRequest{}
+	configSchemaResp := &list.SchemaResponse{}
+
+	listresource.Schema(ctx, configSchemaReq, configSchemaResp)
+	if len(configSchemaResp.Diagnostics) > 0 {
+		return nil, fmt.Errorf("failed to retrieve resource schema: %v", configSchemaResp.Diagnostics)
+	}
+
+	resourceSchemaResp := &resource.SchemaResponse{}
+	r.Schema(ctx, resource.SchemaRequest{}, resourceSchemaResp)
+	if resourceSchemaResp.Schema == nil {
+		return nil, fmt.Errorf("failed to retrieve resource schema: %v", resourceSchemaResp.Schema)
+	}
+
+	var config tftypes.Value
+	config, diag = DynamicValueToValue(configSchemaResp.Schema, req.Config)
+	if diag != nil {
+		return nil, fmt.Errorf("failed to convert config to value: %v", err)
+	}
+
+	validateReq := list.ValidateListConfigRequest{
+		Config: config,
+	}
+	validateResp := &list.ValidateListConfigResponse{}
+
+	listresource.ValidateListConfig(ctx, validateReq, validateResp)
+
+	resp := &tfprotov6.ValidateListResourceConfigResponse{
+		Diagnostics: validateResp.Diagnostics,
+	}
+
+	return resp, nil
 }
 
 func processListResults(req list.ListRequest, stream iter.Seq[list.ListResult]) iter.Seq[tfprotov6.ListResourceResult] {