query: refactor query test step logic into testStepNewQuery()

helper/resource/query/examplecloud_list_test.go

@@ -6,6 +6,7 @@ package query_test
 import (
 	"github.com/hashicorp/terraform-plugin-go/tfprotov6"
 	"github.com/hashicorp/terraform-plugin-go/tftypes"
+
 	"github.com/hashicorp/terraform-plugin-testing/internal/testing/testprovider"
 	"github.com/hashicorp/terraform-plugin-testing/internal/testing/testsdk/list"
 	"github.com/hashicorp/terraform-plugin-testing/internal/teststep"

helper/resource/query/query_checks.go

@@ -28,6 +28,13 @@ func RunQueryChecks(ctx context.Context, t testing.T, query []tfjson.LogMsg, que
 
 	for _, msg := range query {
 		switch v := msg.(type) {
+		case tfjson.DiagnosticLogMessage:
+			if v.Diagnostic.Severity == tfjson.DiagnosticSeverityError {
+				// TODO reimplement missing .tf config error
+				result = append(result, fmt.Errorf("query error diagnostic - summary: %s, detail: %s", v.Diagnostic.Summary, v.Diagnostic.Detail))
+				continue
+			}
+
 		case tfjson.ListResourceFoundMessage:
 			found = append(found, v.ListResourceFound)
 		case tfjson.ListCompleteMessage:

helper/resource/query/query_test.go

@@ -4,11 +4,15 @@
 package query_test
 
 import (
+	"regexp"
 	"testing"
 
 	"github.com/hashicorp/terraform-plugin-go/tfprotov6"
+	"github.com/hashicorp/terraform-plugin-go/tftypes"
+
 	r "github.com/hashicorp/terraform-plugin-testing/helper/resource"
 	"github.com/hashicorp/terraform-plugin-testing/internal/testing/testprovider"
+	"github.com/hashicorp/terraform-plugin-testing/internal/testing/testsdk/list"
 	"github.com/hashicorp/terraform-plugin-testing/internal/testing/testsdk/providerserver"
 	"github.com/hashicorp/terraform-plugin-testing/knownvalue"
 	"github.com/hashicorp/terraform-plugin-testing/querycheck"
@@ -100,3 +104,88 @@ func TestQuery(t *testing.T) {
 		},
 	})
 }
+
+func TestQuery_ExpectError_ValidationError(t *testing.T) {
+	t.Parallel()
+
+	r.UnitTest(t, r.TestCase{
+		TerraformVersionChecks: []tfversion.TerraformVersionCheck{
+			tfversion.SkipBelow(tfversion.Version1_14_0),
+		},
+		ProtoV6ProviderFactories: map[string]func() (tfprotov6.ProviderServer, error){
+			"examplecloud": providerserver.NewProviderServer(testprovider.Provider{
+				ListResources: map[string]testprovider.ListResource{
+					"examplecloud_containerette": {
+						IncludeResource: true,
+						SchemaResponse: &list.SchemaResponse{
+							Schema: &tfprotov6.Schema{
+								Block: &tfprotov6.SchemaBlock{
+									Attributes: []*tfprotov6.SchemaAttribute{
+										{
+											Name:     "resource_group_name",
+											Type:     tftypes.String,
+											Required: true,
+										},
+									},
+								},
+							},
+						},
+						ValidateListConfigResponse: &list.ValidateListConfigResponse{
+							Diagnostics: []*tfprotov6.Diagnostic{
+								{
+									Severity: tfprotov6.DiagnosticSeverityError,
+									Summary:  "Diagnostic summary",
+									Detail:   "Diagnostic details",
+								},
+							},
+						},
+					},
+				},
+				Resources: map[string]testprovider.Resource{
+					"examplecloud_containerette": examplecloudResource(),
+				},
+			}),
+		},
+		Steps: []r.TestStep{
+			{ // config mode step 1 needs tf file with terraform providers block
+				// this step should provision all the resources that the query is support to list
+				// for simplicity we're only "provisioning" one here
+				Config: `
+				resource "examplecloud_containerette" "primary" {
+					name                = "banana"
+					resource_group_name = "foo"
+					location  			= "westeurope"
+
+					instances = 5
+				}`,
+			},
+			{ // Query mode step 2, operates on .tfquery.hcl files (needs tf file with terraform providers block)
+				// ```provider "examplecloud" {}``` has a slightly different syntax for a .tfquery.hcl file
+				// provider bock simulates a real providers workflow
+				// "config" in this case means configuration of the list resource/filters
+
+				Query: true,
+				Config: `
+				provider "examplecloud" {} 
+
+				list "examplecloud_containerette" "test" {
+					provider = examplecloud
+
+					config {
+						resource_group_name = "foo"
+ 					}
+				}
+
+				list "examplecloud_containerette" "test2" {
+					provider = examplecloud
+
+					config {
+						resource_group_name = "bar"
+					}
+				}
+				`,
+				ExpectError: regexp.MustCompile(`Diagnostic summary`),
+			},
+		},
+	})
+}

helper/resource/testing_new.go

@@ -15,9 +15,10 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/hashicorp/go-version"
 	tfjson "github.com/hashicorp/terraform-json"
-	"github.com/hashicorp/terraform-plugin-testing/helper/resource/query"
 	"github.com/mitchellh/go-testing-interface"
 
+	"github.com/hashicorp/terraform-plugin-testing/helper/resource/query"
+
 	"github.com/hashicorp/terraform-plugin-testing/config"
 	"github.com/hashicorp/terraform-plugin-testing/internal/logging"
 	"github.com/hashicorp/terraform-plugin-testing/internal/plugintest"
@@ -384,7 +385,6 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest
 				return err
 			})
 			if err != nil {
-				fmt.Printf("Step %d/%d Query Output:\n%s\n", stepNumber, len(c.Steps), queryOut)
 				t.Fatalf("Step %d/%d error running query: %s", stepNumber, len(c.Steps), err)
 			}
 
@@ -414,7 +414,6 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest
 					)
 					t.Fatalf("Step %d/%d error running query checks: %s", stepNumber, len(c.Steps), err)
 				}
-				fmt.Printf("Step %d/%d Query Output:\n%s\n", stepNumber, len(c.Steps), queryOut)
 			}
 
 			logging.HelperResourceDebug(ctx, "Finished TestStep")

internal/plugintest/working_dir.go

@@ -6,12 +6,13 @@ package plugintest
 import (
 	"context"
 	"fmt"
-	"github.com/hashicorp/terraform-exec/tfexec"
-	tfjson "github.com/hashicorp/terraform-json"
 	"io"
 	"os"
 	"path/filepath"
 
+	"github.com/hashicorp/terraform-exec/tfexec"
+	tfjson "github.com/hashicorp/terraform-json"
+
 	"github.com/hashicorp/terraform-plugin-testing/config"
 	"github.com/hashicorp/terraform-plugin-testing/internal/logging"
 	"github.com/hashicorp/terraform-plugin-testing/internal/teststep"
@@ -525,7 +526,6 @@ func (wd *WorkingDir) Schemas(ctx context.Context) (*tfjson.ProviderSchemas, err
 
 func (wd *WorkingDir) Query(ctx context.Context) ([]tfjson.LogMsg, error) {
 	var messages []tfjson.LogMsg
-	var diags []tfjson.LogMsg
 
 	logging.HelperResourceTrace(ctx, "Calling Terraform CLI providers query command")
 
@@ -546,18 +546,9 @@ func (wd *WorkingDir) Query(ctx context.Context) ([]tfjson.LogMsg, error) {
 			return nil, fmt.Errorf("retrieving message: %w", msg.Err)
 		}
 
-		if msg.Msg.Level() == tfjson.Error {
-			// TODO reimplement missing .tf config error
-			diags = append(diags, msg.Msg)
-			continue
-		}
 		messages = append(messages, msg.Msg)
 	}
 
-	if len(diags) > 0 {
-		return nil, fmt.Errorf("running terraform query command returned diagnostics: %+v", diags)
-	}
-
 	logging.HelperResourceTrace(ctx, "Called Terraform CLI providers query command")
 
 	return messages, nil

internal/testing/testprovider/list_resource.go

@@ -18,6 +18,12 @@ type ListResource struct {
 	ValidateListConfigResponse *list.ValidateListConfigResponse
 }
 
+func (r ListResource) ValidateListConfig(ctx context.Context, req list.ValidateListConfigRequest, resp *list.ValidateListConfigResponse) {
+	if r.ValidateListConfigResponse != nil {
+		resp.Diagnostics = r.ValidateListConfigResponse.Diagnostics
+	}
+}
+
 func (r ListResource) Schema(ctx context.Context, req list.SchemaRequest, resp *list.SchemaResponse) {
 	if r.SchemaResponse != nil {
 		resp.Diagnostics = r.SchemaResponse.Diagnostics

internal/testing/testsdk/list/list_resource.go

@@ -14,6 +14,7 @@ import (
 type ListResource interface {
 	Schema(context.Context, SchemaRequest, *SchemaResponse)
 	List(context.Context, ListRequest, *ListResultsStream)
+	ValidateListConfig(context.Context, ValidateListConfigRequest, *ValidateListConfigResponse)
 }
 
 type ListRequest struct {
@@ -45,7 +46,12 @@ type ListResult struct {
 	Diagnostics []*tfprotov6.Diagnostic
 }
 
+type ValidateListConfigRequest struct {
+	Config tftypes.Value
+}
+
 type ValidateListConfigResponse struct {
+	Diagnostics []*tfprotov6.Diagnostic
 }
 
 type SchemaRequest struct{}

internal/testing/testsdk/providerserver/providerserver.go

@@ -1104,7 +1104,56 @@ func (s ProviderServer) ListResource(ctx context.Context, req *tfprotov6.ListRes
 }
 
 func (s ProviderServer) ValidateListResourceConfig(ctx context.Context, req *tfprotov6.ValidateListResourceConfigRequest) (*tfprotov6.ValidateListResourceConfigResponse, error) {
-	return &tfprotov6.ValidateListResourceConfigResponse{}, nil
+	// Copy over identity if it's supported
+	identitySchemaReq := resource.IdentitySchemaRequest{}
+	identitySchemaResp := &resource.IdentitySchemaResponse{}
+
+	r, err := ProviderResource(s.Provider, req.TypeName)
+	if err != nil {
+		return nil, fmt.Errorf("failed to retrieve resource: %v", err)
+	}
+	r.IdentitySchema(ctx, identitySchemaReq, identitySchemaResp)
+	if len(identitySchemaResp.Diagnostics) > 0 {
+		return nil, fmt.Errorf("failed to retrieve resource schema: %v", identitySchemaResp.Diagnostics)
+	}
+
+	listresource, diag := ProviderListResource(s.Provider, req.TypeName)
+	if diag != nil {
+		return nil, fmt.Errorf("failed to retrieve resource identity schema: %v", err)
+	}
+
+	configSchemaReq := list.SchemaRequest{}
+	configSchemaResp := &list.SchemaResponse{}
+
+	listresource.Schema(ctx, configSchemaReq, configSchemaResp)
+	if len(configSchemaResp.Diagnostics) > 0 {
+		return nil, fmt.Errorf("failed to retrieve resource schema: %v", configSchemaResp.Diagnostics)
+	}
+
+	resourceSchemaResp := &resource.SchemaResponse{}
+	r.Schema(ctx, resource.SchemaRequest{}, resourceSchemaResp)
+	if resourceSchemaResp.Schema == nil {
+		return nil, fmt.Errorf("failed to retrieve resource schema: %v", resourceSchemaResp.Schema)
+	}
+
+	var config tftypes.Value
+	config, diag = DynamicValueToValue(configSchemaResp.Schema, req.Config)
+	if diag != nil {
+		return nil, fmt.Errorf("failed to convert config to value: %v", err)
+	}
+
+	validateReq := list.ValidateListConfigRequest{
+		Config: config,
+	}
+	validateResp := &list.ValidateListConfigResponse{}
+
+	listresource.ValidateListConfig(ctx, validateReq, validateResp)
+
+	resp := &tfprotov6.ValidateListResourceConfigResponse{
+		Diagnostics: validateResp.Diagnostics,
+	}
+
+	return resp, nil
 }
 
 func processListResults(req list.ListRequest, stream iter.Seq[list.ListResult]) iter.Seq[tfprotov6.ListResourceResult] {