Implement Dynamic Type serialization (#3099)

This PR adds support for correctly serializing and de-serializing
properties with a Dynamic Type schema. This ensures that properties of
this schema type can be correctly saved to state and then read back with
any provider version.

There are a few changes which had to be made to make this all work:
1. When marshalling values to JSON we now need to pass in the schema
type in order to check if they are Dynamic type. Previously we were
using the value type and assumed the two match. This is not correct for
properties with a Dynamic schema type. We also thread this information
through various places.
2. When recording `RawStateDeltas` (these are the "diffs" between the TF
state and Pulumi state), we need to make sure that properties with a
Dynamic schema type always get a `Replace` delta. This is a full
recording of the TF value, instead of a "diff" with the pulumi state.
This is because the JSON representation of properties with Dynamic
schema type need to contain their value type as well as the value
itself. This is then used to read the state back into the correct
runtime type.
3. Added a `cty` and `hcty` valueshim implementations for the `cty`
package and the Hashi fork of the `cty` package. This is necessary
because the SDKv1 uses the `cty` package while the SDKv2 uses the `hcty`
package. The two implementations are practically identical.
4. Some unit and integration tests to make sure this all works correctly
and stays that way.

fixes #3078

---------

Co-authored-by: Anton Tayanovskyy <[email protected]>

Author: VenelinMartinov

pkg/pf/internal/pfutils/raw_state.go

@@ -15,9 +15,14 @@
 package schemashim
 
 import (
+	"context"
+
+	"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
+
 	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/internal/internalinter"
 	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/pf/internal/runtypes"
 	shim "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim"
+	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/valueshim"
 )
 
 type schemaOnlyDataSource struct {
@@ -27,6 +32,12 @@ type schemaOnlyDataSource struct {
 
 var _ shim.Resource = (*schemaOnlyDataSource)(nil)
 
+func (r *schemaOnlyDataSource) SchemaType() valueshim.Type {
+	protoSchema, err := r.tf.ResourceProtoSchema(context.Background())
+	contract.AssertNoErrorf(err, "ResourceProtoSchema failed")
+	return valueshim.FromTType(protoSchema.ValueType())
+}
+
 func (r *schemaOnlyDataSource) Schema() shim.SchemaMap {
 	return r.tf.Shim()
 }

pkg/pf/internal/schemashim/datasource.go

@@ -28,6 +28,7 @@ import (
 	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/internal/internalinter"
 	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/pf/internal/pfutils"
 	shim "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim"
+	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/valueshim"
 )
 
 // An Object type that masquerades as a Resource. This is a workaround to reusing tfgen code for generating schemas,
@@ -77,6 +78,10 @@ func (r *objectPseudoResource) Schema() shim.SchemaMap {
 	return r
 }
 
+func (r *objectPseudoResource) SchemaType() valueshim.Type {
+	return valueshim.FromTType(tftypes.Object{})
+}
+
 func (*objectPseudoResource) SchemaVersion() int {
 	panic("This is an Object type encoded as a shim.Resource, and " +
 		"SchemaVersion() should not be called on this entity during schema generation")
@@ -200,6 +205,10 @@ func newTuplePseudoResource(t attr.TypeWithElementTypes) shim.Resource {
 	}
 }
 
+func (r *tuplePseudoResource) SchemaType() valueshim.Type {
+	return valueshim.FromTType(tftypes.Object{})
+}
+
 func (*tuplePseudoResource) SchemaVersion() int         { panic("TODO") }
 func (*tuplePseudoResource) DeprecationMessage() string { panic("TODO") }
 

pkg/pf/internal/schemashim/object_pseudoresource.go

@@ -15,9 +15,14 @@
 package schemashim
 
 import (
+	"context"
+
+	"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
+
 	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/internal/internalinter"
 	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/pf/internal/runtypes"
 	shim "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim"
+	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/valueshim"
 )
 
 type schemaOnlyResource struct {
@@ -43,6 +48,12 @@ func (r *schemaOnlyResource) DeprecationMessage() string {
 	return r.tf.DeprecationMessage()
 }
 
+func (r *schemaOnlyResource) SchemaType() valueshim.Type {
+	s, err := r.tf.ResourceProtoSchema(context.Background())
+	contract.AssertNoErrorf(err, "failed to extract schema")
+	return valueshim.FromTType(s.ValueType())
+}
+
 func (*schemaOnlyResource) Importer() shim.ImportFunc {
 	panic("schemaOnlyResource does not implement runtime operation ImporterFunc")
 }

pkg/pf/internal/schemashim/resource.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/internal/internalinter"
 	shim "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim"
+	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/valueshim"
 )
 
 var (
@@ -113,6 +114,10 @@ func (o elementObject) Schema() shim.SchemaMap {
 	return elementObjectMap(o.typ)
 }
 
+func (o elementObject) SchemaType() valueshim.Type {
+	return valueshim.FromTType(o.typ)
+}
+
 func (m elementObjectMap) Len() int { return len(m.AttributeTypes) }
 
 func (m elementObjectMap) Get(key string) shim.Schema { return getSchemaMap(m, key) }

pkg/pf/proto/element.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/internal/internalinter"
 	shim "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim"
+	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/valueshim"
 )
 
 var (
@@ -33,6 +34,11 @@ type object struct {
 	internalinter.Internal
 }
 
+func (o object) SchemaType() valueshim.Type {
+	ty := o.obj.ValueType()
+	return valueshim.FromTType(ty)
+}
+
 func (o object) Schema() shim.SchemaMap {
 	contract.Assertf(o.obj.Nesting != tfprotov6.SchemaObjectNestingModeMap,
 		"%T cannot be a map, since that would require `o` to represent a Map<Object> type", o)

pkg/pf/proto/object.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/internal/internalinter"
 	shim "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim"
+	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/valueshim"
 )
 
 var (
@@ -70,6 +71,11 @@ func newResource(r *tfprotov6.Schema) *resource {
 	return &resource{r, internalinter.Internal{}}
 }
 
+func (r resource) SchemaType() valueshim.Type {
+	ty := r.r.Block.ValueType()
+	return valueshim.FromTType(ty)
+}
+
 func (r resource) Schema() shim.SchemaMap {
 	return blockMap{r.r.Block}
 }

pkg/pf/proto/resource.go

@@ -15,14 +15,19 @@
 package proto
 
 import (
-	// "github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
+	"github.com/hashicorp/terraform-plugin-go/tftypes"
 
 	shim "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim"
+	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/valueshim"
 )
 
 // pseudoResource represents a type that must pretent to be a [shim.Resource], but does not represent a resource.
 type pseudoResource struct{}
 
+func (pseudoResource) SchemaType() valueshim.Type {
+	return valueshim.FromTType(tftypes.Object{}) // not a top-level resource
+}
+
 func (pseudoResource) SchemaVersion() int              { return 0 }
 func (pseudoResource) Importer() shim.ImportFunc       { return nil }
 func (pseudoResource) Timeouts() *shim.ResourceTimeout { return nil }

pkg/pf/proto/schema.go

@@ -3,15 +3,15 @@ package tfbridgetests
 import (
 	"context"
 	"math/big"
-	"os"
 	"testing"
 
+	"github.com/hashicorp/terraform-plugin-framework/attr"
 	"github.com/hashicorp/terraform-plugin-framework/resource"
 	rschema "github.com/hashicorp/terraform-plugin-framework/resource/schema"
 	"github.com/hashicorp/terraform-plugin-framework/resource/schema/planmodifier"
 	"github.com/hashicorp/terraform-plugin-framework/resource/schema/stringplanmodifier"
+	"github.com/hashicorp/terraform-plugin-framework/types"
 	"github.com/hexops/autogold/v2"
-	"github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil"
 	"github.com/zclconf/go-cty/cty"
 
 	pb "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/pf/internal/providerbuilder"
@@ -184,10 +184,6 @@ func TestPFDetailedDiffStringAttribute(t *testing.T) {
 
 func TestPFDetailedDiffDynamicType(t *testing.T) {
 	t.Parallel()
-	if d, ok := os.LookupEnv("PULUMI_RAW_STATE_DELTA_ENABLED"); ok && cmdutil.IsTruthy(d) {
-		// TODO[pulumi/pulumi-terraform-bridge#3078]
-		t.Skip("Does not work with PULUMI_RAW_STATE_DELTA_ENABLED=true")
-	}
 
 	attributeSchema := rschema.Schema{
 		Attributes: map[string]rschema.Attribute{
@@ -222,7 +218,7 @@ func TestPFDetailedDiffDynamicType(t *testing.T) {
 	})
 
 	t.Run("type change", func(t *testing.T) {
-		// TODO[pulumi/pulumi-terraform-bridge#3078]
+		// TODO[pulumi/pulumi-terraform-bridge#3122]
 		t.Skip(`Error converting tftypes.Number<"1"> (value2) at "AttributeName(\"key\")": can't unmarshal tftypes.Number into *string, expected string`)
 		crosstests.Diff(t, res,
 			map[string]cty.Value{"key": cty.StringVal("value")},
@@ -231,10 +227,56 @@ func TestPFDetailedDiffDynamicType(t *testing.T) {
 	})
 }
 
+func TestPFDetailedDiffNestedDynamicType(t *testing.T) {
+	t.Parallel()
+
+	attributeSchema := rschema.Schema{
+		Attributes: map[string]rschema.Attribute{
+			"key": rschema.ObjectAttribute{
+				Optional: true,
+				AttributeTypes: map[string]attr.Type{
+					"nested": types.DynamicType,
+				},
+			},
+		},
+	}
+	res := pb.NewResource(pb.NewResourceArgs{
+		ResourceSchema: attributeSchema,
+	})
+
+	t.Run("no change", func(t *testing.T) {
+		crosstests.Diff(t, res,
+			map[string]cty.Value{"key": cty.ObjectVal(map[string]cty.Value{"nested": cty.StringVal("value")})},
+			map[string]cty.Value{"key": cty.ObjectVal(map[string]cty.Value{"nested": cty.StringVal("value")})},
+		)
+	})
+
+	t.Run("change", func(t *testing.T) {
+		crosstests.Diff(t, res,
+			map[string]cty.Value{"key": cty.ObjectVal(map[string]cty.Value{"nested": cty.StringVal("value")})},
+			map[string]cty.Value{"key": cty.ObjectVal(map[string]cty.Value{"nested": cty.StringVal("value1")})},
+		)
+	})
+
+	t.Run("int no change", func(t *testing.T) {
+		crosstests.Diff(t, res,
+			map[string]cty.Value{"key": cty.ObjectVal(map[string]cty.Value{"nested": cty.NumberVal(big.NewFloat(1))})},
+			map[string]cty.Value{"key": cty.ObjectVal(map[string]cty.Value{"nested": cty.NumberVal(big.NewFloat(1))})},
+		)
+	})
+
+	t.Run("type change", func(t *testing.T) {
+		// TODO[pulumi/pulumi-terraform-bridge#3122]
+		t.Skip(`Error converting tftypes.Number<"1"> (value2) at "AttributeName(\"key\")": can't unmarshal tftypes.Number into *string, expected string`)
+		crosstests.Diff(t, res,
+			map[string]cty.Value{"key": cty.ObjectVal(map[string]cty.Value{"nested": cty.StringVal("value")})},
+			map[string]cty.Value{"key": cty.ObjectVal(map[string]cty.Value{"nested": cty.NumberVal(big.NewFloat(1))})},
+		)
+	})
+}
+
 func TestPFDetailedDiffDynamicTypeWithMigration(t *testing.T) {
 	t.Parallel()
-	// TODO[pulumi/pulumi-terraform-bridge#3078]
-	t.Skip("DynamicPseudoType is not supported")
 
 	attributeSchema := rschema.Schema{
 		Attributes: map[string]rschema.Attribute{

pkg/pf/tests/diff_test/diff_test.go

@@ -339,7 +339,8 @@ func (p *provider) returnTerraformConfig() (resource.PropertyMap, error) {
 	}
 	// use valueshim package to marshal tfConfigValue into raw json,
 	// which can be unmarshaled into a map[string]interface{}
-	configJSONMessage, err := valueshim.FromTValue(tfConfigValue).Marshal()
+	value := valueshim.FromTValue(tfConfigValue)
+	configJSONMessage, err := value.Marshal(value.Type())
 	if err != nil {
 		return nil, fmt.Errorf("error marshaling into raw JSON message: %v", err)
 	}