From e1d0c57483da99d73f00da12cb1b922d4050fd04 Mon Sep 17 00:00:00 2001 From: Venelin Date: Thu, 29 May 2025 13:25:15 +0300 Subject: [PATCH 01/31] fix parsing pf dynamic type From aaa4cc5bf41b318727022ae26b11651c0151cc15 Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Fri, 16 May 2025 14:14:11 -0400 Subject: [PATCH 02/31] Add a test for DynamicPseudoType reproducing the error --- pkg/pf/tests/upgrade_state_cross_test.go | 56 ++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/pkg/pf/tests/upgrade_state_cross_test.go b/pkg/pf/tests/upgrade_state_cross_test.go index 3594c830a..7f421d2f4 100644 --- a/pkg/pf/tests/upgrade_state_cross_test.go +++ b/pkg/pf/tests/upgrade_state_cross_test.go @@ -1092,6 +1092,62 @@ func TestPFUpgrade_Downgrading(t *testing.T) { }) } +// Test when a dynamic pseudo type value is being sent through a state upgrader. +func TestPFUpgrade_DynamicPseudoType(t *testing.T) { + t.Parallel() + ct.SkipUnlessLinux(t) + //skipUnlessDeltasEnabled(t) + + resourceBefore := pb.NewResource(pb.NewResourceArgs{ + ResourceSchema: schema.Schema{ + Attributes: map[string]schema.Attribute{ + "dyn": schema.DynamicAttribute{Optional: true}, + }, + }, + }) + + resourceAfter := pb.NewResource(pb.NewResourceArgs{ + UpgradeStateFunc: func(ctx context.Context) map[int64]resource.StateUpgrader { + return map[int64]resource.StateUpgrader{ + 0: { + PriorSchema: &resourceBefore.ResourceSchema, + StateUpgrader: ct.NopUpgrader, + }, + } + }, + ResourceSchema: schema.Schema{ + Attributes: map[string]schema.Attribute{ + "dyn": schema.DynamicAttribute{Optional: true}, + }, + Version: 1, + }, + }) + + tfInputsBefore := cty.ObjectVal(map[string]cty.Value{"dyn": cty.StringVal("str")}) + tfInputsAfter := cty.ObjectVal(map[string]cty.Value{"dyn": cty.NumberIntVal(42)}) + pmBefore := presource.NewPropertyMapFromMap(map[string]any{"dyn": "str"}) + pmAfter := presource.NewPropertyMapFromMap(map[string]any{"dyn": 42}) + + testCase := ct.UpgradeStateTestCase{ + Resource1: &resourceBefore, + Resource2: &resourceAfter, + Inputs1: tfInputsBefore, + InputsMap1: pmBefore, + Inputs2: tfInputsAfter, + InputsMap2: pmAfter, + //ExpectedRawStateType: resourceBeforeAndAfter.ResourceSchema.Type().TerraformType(context.Background()), + SkipPulumi: "TODO", + } + + _ = testCase.Run(t) + + // autogold.Expect(map[apitype.OpType]int{apitype.OpType("same"): 2}).Equal(t, result.PulumiPreviewResult.ChangeSummary) + // autogold.Expect(&map[string]int{"same": 2}).Equal(t, result.PulumiUpResult.Summary.ResourceChanges) + + // autogold.Expect([]ct.UpgradeStateTrace{}).Equal(t, result.PulumiUpgrades) + // autogold.Expect([]ct.UpgradeStateTrace{}).Equal(t, result.TFUpgrades) +} + func skipUnlessDeltasEnabled(t *testing.T) { if d, ok := os.LookupEnv("PULUMI_RAW_STATE_DELTA_ENABLED"); !ok || !cmdutil.IsTruthy(d) { t.Skip("This test requires PULUMI_RAW_STATE_DELTA_ENABLED=true environment") From c626c8be23740363fc20136a0b7aa794ba124387 Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Fri, 16 May 2025 15:13:30 -0400 Subject: [PATCH 03/31] Update valueshim to handle DynamicPseudoType --- pkg/valueshim/convert.go | 41 +++++++++ pkg/valueshim/cty.go | 15 ++- pkg/valueshim/cty_test.go | 49 +++++++++- pkg/valueshim/shim.go | 11 ++- pkg/valueshim/tfvalue.go | 168 +++------------------------------- pkg/valueshim/tfvalue_test.go | 52 ++++++++++- 6 files changed, 172 insertions(+), 164 deletions(-) create mode 100644 pkg/valueshim/convert.go diff --git a/pkg/valueshim/convert.go b/pkg/valueshim/convert.go new file mode 100644 index 000000000..e5f33ef75 --- /dev/null +++ b/pkg/valueshim/convert.go @@ -0,0 +1,41 @@ +// Copyright 2016-2025, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package valueshim + +import ( + "encoding/json" + + "github.com/hashicorp/go-cty/cty" + ctyjson "github.com/hashicorp/go-cty/cty/json" + ctymsgpack "github.com/hashicorp/go-cty/cty/msgpack" + "github.com/hashicorp/terraform-plugin-go/tfprotov6" + "github.com/hashicorp/terraform-plugin-go/tftypes" +) + +func toCtyType(t tftypes.Type) (cty.Type, error) { + typeBytes, err := json.Marshal(t) + if err != nil { + return cty.NilType, err + } + return ctyjson.UnmarshalType(typeBytes) +} + +func toCtyValue(schemaType tftypes.Type, schemaCtyType cty.Type, value tftypes.Value) (cty.Value, error) { + dv, err := tfprotov6.NewDynamicValue(schemaType, value) + if err != nil { + return cty.NilVal, err + } + return ctymsgpack.Unmarshal(dv.MsgPack, schemaCtyType) +} diff --git a/pkg/valueshim/cty.go b/pkg/valueshim/cty.go index 30992664c..ebe16d755 100644 --- a/pkg/valueshim/cty.go +++ b/pkg/valueshim/cty.go @@ -16,6 +16,7 @@ package valueshim import ( "encoding/json" + "fmt" "math/big" "github.com/hashicorp/go-cty/cty" @@ -103,18 +104,24 @@ func (v ctyValueShim) Remove(key string) Value { } } -func (v ctyValueShim) Marshal() (json.RawMessage, error) { +func (v ctyValueShim) Marshal(schemaType Type) (json.RawMessage, error) { vv := v.val() - raw, err := ctyjson.Marshal(vv, vv.Type()) + tt, ok := schemaType.(ctyTypeShim) + if !ok { + return nil, fmt.Errorf("Cannot marshal to RawState: "+ + "expected schemaType to be of type ctyTypeShim, got %#T", + schemaType) + } + raw, err := ctyjson.Marshal(vv, tt.ty()) if err != nil { - return nil, err + return nil, fmt.Errorf("Cannot marshal to RawState: %w", err) } return json.RawMessage(raw), nil } type ctyTypeShim cty.Type -var _ Type = (*ctyTypeShim)(nil) +var _ Type = ctyTypeShim{} func (t ctyTypeShim) ty() cty.Type { return cty.Type(t) diff --git a/pkg/valueshim/cty_test.go b/pkg/valueshim/cty_test.go index 7db8af729..2a3cf93ac 100644 --- a/pkg/valueshim/cty_test.go +++ b/pkg/valueshim/cty_test.go @@ -109,8 +109,10 @@ func Test_HCtyValue_Marshal(t *testing.T) { tupType := cty.Tuple([]cty.Type{cty.String, cty.Number}) type testCase struct { - v cty.Value - expect autogold.Value + v cty.Value + expect autogold.Value + schemaType cty.Type + hasSchemaType bool } testCases := []testCase{ @@ -210,10 +212,51 @@ func Test_HCtyValue_Marshal(t *testing.T) { v: cty.TupleVal([]cty.Value{ok, n42}), expect: autogold.Expect(`["OK",42]`), }, + { + v: cty.NullVal(cty.String), + schemaType: cty.DynamicPseudoType, + hasSchemaType: true, + expect: autogold.Expect(`{"value":null,"type":"string"}`), + }, + { + v: cty.StringVal("foo"), + schemaType: cty.DynamicPseudoType, + hasSchemaType: true, + expect: autogold.Expect(`{"value":"foo","type":"string"}`), + }, + { + v: cty.NumberIntVal(42), + schemaType: cty.DynamicPseudoType, + hasSchemaType: true, + expect: autogold.Expect(`{"value":42,"type":"number"}`), + }, + { + v: cty.BoolVal(true), + schemaType: cty.DynamicPseudoType, + hasSchemaType: true, + expect: autogold.Expect(`{"value":true,"type":"bool"}`), + }, + { + v: cty.ListVal([]cty.Value{cty.StringVal("A")}), + schemaType: cty.DynamicPseudoType, + hasSchemaType: true, + expect: autogold.Expect(`{"value":["A"],"type":["list","string"]}`), + }, + { + v: cty.MapVal(map[string]cty.Value{"x": ok, "y": ok2}), + schemaType: cty.DynamicPseudoType, + hasSchemaType: true, + expect: autogold.Expect(`{"value":{"x":"OK","y":"OK2"},"type":["map","string"]}`), + }, } for _, tc := range testCases { - raw, err := valueshim.FromHCtyValue(tc.v).Marshal() + ty := tc.schemaType + if !tc.hasSchemaType { + ty = tc.v.Type() + } + vv := valueshim.FromHCtyValue(tc.v) + raw, err := vv.Marshal(valueshim.FromHCtyType(ty)) require.NoError(t, err) tc.expect.Equal(t, string(raw)) } diff --git a/pkg/valueshim/shim.go b/pkg/valueshim/shim.go index 00e33d146..9dbbcbe01 100644 --- a/pkg/valueshim/shim.go +++ b/pkg/valueshim/shim.go @@ -29,7 +29,16 @@ type Value interface { AsValueMap() map[string]Value // Marshals into the "raw state" JSON representation. - Marshal() (json.RawMessage, error) + // + // This is the representation expected on the TF protocol UpgradeResourceState method. + // + // For correctly encoding DynamicPseudoType values to {"type": "...", "value": "..."} structures, the + // schemaType is needed. This encoding will be used when the schema a type is a DynamicPseudoType but + // the value type is a concrete type. + // + // In situations where the DynamicPseudoType encoding is not needed, you can also call Marshal with + // value.Type() to assume the intrinsic type of the value. + Marshal(schemaType Type) (json.RawMessage, error) // Removes a top-level property from an Object. Remove(key string) Value diff --git a/pkg/valueshim/tfvalue.go b/pkg/valueshim/tfvalue.go index 35e27e06d..3d8ae961f 100644 --- a/pkg/valueshim/tfvalue.go +++ b/pkg/valueshim/tfvalue.go @@ -16,6 +16,8 @@ package valueshim import ( "encoding/json" + "errors" + "fmt" "math/big" "github.com/hashicorp/terraform-plugin-go/tftypes" @@ -74,12 +76,20 @@ func (v tValueShim) AsValueMap() map[string]Value { return res } -func (v tValueShim) Marshal() (json.RawMessage, error) { - inmem, err := jsonMarshal(v.val(), tftypes.NewAttributePath()) +func (v tValueShim) Marshal(schemaType Type) (json.RawMessage, error) { + tt, ok := schemaType.(tTypeShim) + if !ok { + return nil, errors.New("Cannot marshal to RawState: expected schemaType to be of type tTypeShim") + } + ctyType, err := toCtyType(tt.ty()) + if err != nil { + return nil, fmt.Errorf("Cannot marshal to RawState. Error converting to cty.Type: %w", err) + } + cty, err := toCtyValue(tt.ty(), ctyType, v.val()) if err != nil { - return nil, err + return nil, fmt.Errorf("Cannot marshal to RawState. Error converting to cty.Value: %w", err) } - return json.Marshal(inmem) + return FromHCtyValue(cty).Marshal(FromHCtyType(ctyType)) } func (v tValueShim) Remove(prop string) Value { @@ -144,159 +154,11 @@ func (v tValueShim) StringValue() string { return result } -func jsonMarshal(v tftypes.Value, p *tftypes.AttributePath) (interface{}, error) { - if v.IsNull() { - return nil, nil - } - if !v.IsKnown() { - return nil, p.NewErrorf("unknown values cannot be serialized to JSON") - } - typ := v.Type() - switch { - case typ.Is(tftypes.String): - return jsonMarshalString(v, p) - case typ.Is(tftypes.Number): - return jsonMarshalNumber(v, p) - case typ.Is(tftypes.Bool): - return jsonMarshalBool(v, p) - case typ.Is(tftypes.List{}): - return jsonMarshalList(v, p) - case typ.Is(tftypes.Set{}): - return jsonMarshalSet(v, p) - case typ.Is(tftypes.Map{}): - return jsonMarshalMap(v, p) - case typ.Is(tftypes.Tuple{}): - return jsonMarshalTuple(v, p) - case typ.Is(tftypes.Object{}): - return jsonMarshalObject(v, p) - } - - return nil, p.NewErrorf("unknown type %s", typ) -} - -func jsonMarshalString(v tftypes.Value, p *tftypes.AttributePath) (interface{}, error) { - var stringValue string - err := v.As(&stringValue) - if err != nil { - return nil, p.NewError(err) - } - return stringValue, nil -} - -func jsonMarshalNumber(v tftypes.Value, p *tftypes.AttributePath) (interface{}, error) { - var n big.Float - err := v.As(&n) - if err != nil { - return nil, p.NewError(err) - } - return json.Number(n.Text('f', -1)), nil -} - -func jsonMarshalBool(v tftypes.Value, p *tftypes.AttributePath) (interface{}, error) { - var b bool - err := v.As(&b) - if err != nil { - return nil, p.NewError(err) - } - return b, nil -} - -func jsonMarshalList(v tftypes.Value, p *tftypes.AttributePath) (interface{}, error) { - var vs []tftypes.Value - err := v.As(&vs) - if err != nil { - return nil, p.NewError(err) - } - res := make([]any, len(vs)) - for i, v := range vs { - ep := p.WithElementKeyInt(i) - e, err := jsonMarshal(v, ep) - if err != nil { - return nil, ep.NewError(err) - } - res[i] = e - } - return res, nil -} - -// Important to preserve original order of tftypes.Value here. -func jsonMarshalSet(v tftypes.Value, p *tftypes.AttributePath) (interface{}, error) { - var vs []tftypes.Value - err := v.As(&vs) - if err != nil { - return nil, p.NewError(err) - } - res := make([]any, len(vs)) - for i, v := range vs { - ep := p.WithElementKeyValue(v) - e, err := jsonMarshal(v, ep) - if err != nil { - return nil, ep.NewError(err) - } - res[i] = e - } - return res, nil -} - -func jsonMarshalMap(v tftypes.Value, p *tftypes.AttributePath) (interface{}, error) { - var vs map[string]tftypes.Value - err := v.As(&vs) - if err != nil { - return nil, p.NewError(err) - } - res := make(map[string]any, len(vs)) - for k, v := range vs { - ep := p.WithElementKeyValue(v) - e, err := jsonMarshal(v, ep) - if err != nil { - return nil, ep.NewError(err) - } - res[k] = e - } - return res, nil -} - -func jsonMarshalTuple(v tftypes.Value, p *tftypes.AttributePath) (interface{}, error) { - var vs []tftypes.Value - err := v.As(&vs) - if err != nil { - return nil, p.NewError(err) - } - res := make([]any, len(vs)) - for i, v := range vs { - ep := p.WithElementKeyInt(i) - e, err := jsonMarshal(v, ep) - if err != nil { - return nil, ep.NewError(err) - } - res[i] = e - } - return res, nil -} - -func jsonMarshalObject(v tftypes.Value, p *tftypes.AttributePath) (interface{}, error) { - var vs map[string]tftypes.Value - err := v.As(&vs) - if err != nil { - return nil, p.NewError(err) - } - res := make(map[string]any, len(vs)) - for k, v := range vs { - ep := p.WithAttributeName(k) - e, err := jsonMarshal(v, ep) - if err != nil { - return nil, ep.NewError(err) - } - res[k] = e - } - return res, nil -} - type tTypeShim struct { t tftypes.Type } -var _ Type = (*tTypeShim)(nil) +var _ Type = tTypeShim{} func (t tTypeShim) ty() tftypes.Type { return t.t diff --git a/pkg/valueshim/tfvalue_test.go b/pkg/valueshim/tfvalue_test.go index a30df0a30..184d4c462 100644 --- a/pkg/valueshim/tfvalue_test.go +++ b/pkg/valueshim/tfvalue_test.go @@ -104,8 +104,10 @@ func Test_TValue_Marshal(t *testing.T) { tupType := tftypes.Tuple{ElementTypes: []tftypes.Type{tftypes.String, tftypes.Number}} type testCase struct { - v tftypes.Value - expect autogold.Value + v tftypes.Value + schemaType tftypes.Type + hasSchemaType bool + expect autogold.Value } testCases := []testCase{ @@ -207,10 +209,54 @@ func Test_TValue_Marshal(t *testing.T) { v: tftypes.NewValue(tupType, []tftypes.Value{ok, n42}), expect: autogold.Expect(`["OK",42]`), }, + { + v: tftypes.NewValue(tftypes.String, nil), + schemaType: tftypes.DynamicPseudoType, + hasSchemaType: true, + expect: autogold.Expect(`{"value":null,"type":"string"}`), + }, + { + v: tftypes.NewValue(tftypes.String, "foo"), + schemaType: tftypes.DynamicPseudoType, + hasSchemaType: true, + expect: autogold.Expect(`{"value":"foo","type":"string"}`), + }, + { + v: tftypes.NewValue(tftypes.Number, 42), + schemaType: tftypes.DynamicPseudoType, + hasSchemaType: true, + expect: autogold.Expect(`{"value":42,"type":"number"}`), + }, + { + v: tftypes.NewValue(tftypes.Bool, true), + schemaType: tftypes.DynamicPseudoType, + hasSchemaType: true, + expect: autogold.Expect(`{"value":true,"type":"bool"}`), + }, + { + v: tftypes.NewValue(tftypes.List{ElementType: tftypes.String}, []tftypes.Value{ + tftypes.NewValue(tftypes.String, "OK"), + }), + schemaType: tftypes.DynamicPseudoType, + hasSchemaType: true, + expect: autogold.Expect(`{"value":["OK"],"type":["list","string"]}`), + }, + { + v: tftypes.NewValue(tftypes.Map{ElementType: tftypes.String}, map[string]tftypes.Value{ + "foo": tftypes.NewValue(tftypes.String, "OK"), + }), + schemaType: tftypes.DynamicPseudoType, + hasSchemaType: true, + expect: autogold.Expect(`{"value":{"foo":"OK"},"type":["map","string"]}`), + }, } for _, tc := range testCases { - raw, err := valueshim.FromTValue(tc.v).Marshal() + ty := tc.schemaType + if !tc.hasSchemaType { + ty = tc.v.Type() + } + raw, err := valueshim.FromTValue(tc.v).Marshal(valueshim.FromTType(ty)) require.NoError(t, err) tc.expect.Equal(t, string(raw)) } From fdbec2a252b8f7c0718b4c728c2c33a0d7cecc94 Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Fri, 16 May 2025 16:02:46 -0400 Subject: [PATCH 04/31] Allow walking types --- pkg/valueshim/cty.go | 19 +++++++++++++++++++ pkg/valueshim/cty_test.go | 28 ++++++++++++++++++++++++++++ pkg/valueshim/shim.go | 2 ++ pkg/valueshim/tfvalue.go | 25 +++++++++++++++++++++++++ pkg/valueshim/tfvalue_test.go | 28 ++++++++++++++++++++++++++++ 5 files changed, 102 insertions(+) diff --git a/pkg/valueshim/cty.go b/pkg/valueshim/cty.go index ebe16d755..cda094a06 100644 --- a/pkg/valueshim/cty.go +++ b/pkg/valueshim/cty.go @@ -155,6 +155,25 @@ func (t ctyTypeShim) IsObjectType() bool { return t.ty().IsObjectType() } +func (t ctyTypeShim) AttributeType(name string) (Type, bool) { + tt := t.ty() + if !tt.IsObjectType() { + return nil, false + } + if !tt.HasAttribute(name) { + return nil, false + } + return FromHCtyType(tt.AttributeType(name)), true +} + +func (t ctyTypeShim) ElementType() (Type, bool) { + tt := t.ty() + if !tt.IsCollectionType() { + return nil, false + } + return FromHCtyType(tt.ElementType()), true +} + func (t ctyTypeShim) GoString() string { return t.ty().GoString() } diff --git a/pkg/valueshim/cty_test.go b/pkg/valueshim/cty_test.go index 2a3cf93ac..ed1f148a1 100644 --- a/pkg/valueshim/cty_test.go +++ b/pkg/valueshim/cty_test.go @@ -314,3 +314,31 @@ func Test_HCty_ToX(t *testing.T) { assert.Equal(t, 42.41, valueshim.FromHCtyValue(cty.NumberFloatVal(42.41)).NumberValue()) assert.Equal(t, true, valueshim.FromHCtyValue(cty.BoolVal(true)).BoolValue()) } + +func Test_HCtyType_AttributeType(t *testing.T) { + objTy := cty.Object(map[string]cty.Type{"x": cty.String}) + + ty, ok := valueshim.FromHCtyType(objTy).AttributeType("x") + assert.True(t, ok) + assert.Equal(t, valueshim.FromHCtyType(cty.String), ty) + + _, ok = valueshim.FromHCtyType(objTy).AttributeType("y") + assert.False(t, ok) +} + +func Test_HCtyType_ElementType(t *testing.T) { + ty, ok := valueshim.FromHCtyType(cty.Set(cty.Number)).ElementType() + assert.True(t, ok) + assert.Equal(t, valueshim.FromHCtyType(cty.Number), ty) + + ty, ok = valueshim.FromHCtyType(cty.List(cty.Number)).ElementType() + assert.True(t, ok) + assert.Equal(t, valueshim.FromHCtyType(cty.Number), ty) + + ty, ok = valueshim.FromHCtyType(cty.Map(cty.Number)).ElementType() + assert.True(t, ok) + assert.Equal(t, valueshim.FromHCtyType(cty.Number), ty) + + ty, ok = valueshim.FromHCtyType(cty.String).ElementType() + assert.False(t, ok) +} diff --git a/pkg/valueshim/shim.go b/pkg/valueshim/shim.go index 9dbbcbe01..3e235b434 100644 --- a/pkg/valueshim/shim.go +++ b/pkg/valueshim/shim.go @@ -57,5 +57,7 @@ type Type interface { IsMapType() bool IsSetType() bool IsObjectType() bool + AttributeType(name string) (Type, bool) + ElementType() (Type, bool) GoString() string } diff --git a/pkg/valueshim/tfvalue.go b/pkg/valueshim/tfvalue.go index 3d8ae961f..3864696fa 100644 --- a/pkg/valueshim/tfvalue.go +++ b/pkg/valueshim/tfvalue.go @@ -195,3 +195,28 @@ func (t tTypeShim) IsObjectType() bool { func (t tTypeShim) GoString() string { return t.ty().String() } + +func (t tTypeShim) AttributeType(name string) (Type, bool) { + if !t.IsObjectType() { + return nil, false + } + tt := t.ty() + attr, ok := tt.(tftypes.Object).AttributeTypes[name] + if !ok { + return nil, false + } + return FromTType(attr), true +} + +func (t tTypeShim) ElementType() (Type, bool) { + tt := t.ty() + switch { + case tt.Is(tftypes.Map{}): + return FromTType(tt.(tftypes.Map).ElementType), true + case tt.Is(tftypes.List{}): + return FromTType(tt.(tftypes.List).ElementType), true + case tt.Is(tftypes.Set{}): + return FromTType(tt.(tftypes.Set).ElementType), true + } + return nil, false +} diff --git a/pkg/valueshim/tfvalue_test.go b/pkg/valueshim/tfvalue_test.go index 184d4c462..68eabcb14 100644 --- a/pkg/valueshim/tfvalue_test.go +++ b/pkg/valueshim/tfvalue_test.go @@ -313,6 +313,34 @@ func Test_TType(t *testing.T) { assert.Equal(t, "tftypes.Object[]", valueshim.FromTType(tftypes.Object{}).GoString()) } +func Test_TType_AttributeType(t *testing.T) { + objTy := tftypes.Object{AttributeTypes: map[string]tftypes.Type{"x": tftypes.String}} + + ty, ok := valueshim.FromTType(objTy).AttributeType("x") + assert.True(t, ok) + assert.Equal(t, valueshim.FromTType(tftypes.String), ty) + + _, ok = valueshim.FromTType(objTy).AttributeType("y") + assert.False(t, ok) +} + +func Test_TType_ElementType(t *testing.T) { + ty, ok := valueshim.FromTType(tftypes.Set{ElementType: tftypes.Number}).ElementType() + assert.True(t, ok) + assert.Equal(t, valueshim.FromTType(tftypes.Number), ty) + + ty, ok = valueshim.FromTType(tftypes.List{ElementType: tftypes.Number}).ElementType() + assert.True(t, ok) + assert.Equal(t, valueshim.FromTType(tftypes.Number), ty) + + ty, ok = valueshim.FromTType(tftypes.Map{ElementType: tftypes.Number}).ElementType() + assert.True(t, ok) + assert.Equal(t, valueshim.FromTType(tftypes.Number), ty) + + ty, ok = valueshim.FromTType(tftypes.String).ElementType() + assert.False(t, ok) +} + func Test_TValue_ToX(t *testing.T) { t.Parallel() From fe899a597906a30591c3ba94e76ba1761626003e Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Fri, 16 May 2025 16:03:07 -0400 Subject: [PATCH 05/31] Type walker --- pkg/tfshim/walk/walk.go | 27 +++++++++ pkg/tfshim/walk/walk_test.go | 103 +++++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+) diff --git a/pkg/tfshim/walk/walk.go b/pkg/tfshim/walk/walk.go index c0388874f..0170cd8a1 100644 --- a/pkg/tfshim/walk/walk.go +++ b/pkg/tfshim/walk/walk.go @@ -27,6 +27,7 @@ import ( shim "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim" "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim/schema" + "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/valueshim" ) // Represents locations in a tfshim.Schema value as a sequence of steps to locate it. @@ -174,6 +175,32 @@ func LookupSchemaMapPath(path SchemaPath, schemaMap shim.SchemaMap) (shim.Schema return LookupSchemaPath(path, wrapSchemaMap(schemaMap)) } +// Finds a nested Type at a given path. +func LookupType(path SchemaPath, ty valueshim.Type) (valueshim.Type, error) { + current := ty + for i, step := range path { + var subPath SchemaPath = path[0:i] + switch step := step.(type) { + case GetAttrStep: + attr, ok := current.AttributeType(step.Name) + if !ok { + return nil, fmt.Errorf("LookupType mismatch: no attribute %q at path %v", + step.Name, subPath) + } + current = attr + case ElementStep: + el, ok := current.ElementType() + if !ok { + return nil, fmt.Errorf("LookupType mismatch: no element type at path %v", subPath) + } + current = el + default: + contract.Failf("unexpected SchemaPathStep case") + } + } + return current, nil +} + // Represents elements of a SchemaPath. // // This interface is closed, the only the implementations given in the current package are allowed. diff --git a/pkg/tfshim/walk/walk_test.go b/pkg/tfshim/walk/walk_test.go index e618e9b7b..f22086d1b 100644 --- a/pkg/tfshim/walk/walk_test.go +++ b/pkg/tfshim/walk/walk_test.go @@ -18,11 +18,13 @@ import ( "fmt" "testing" + "github.com/hashicorp/terraform-plugin-go/tftypes" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim" "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim/schema" + "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/valueshim" ) var strSchema = (&schema.Schema{ @@ -168,3 +170,104 @@ func TestEncodeDecodeSchemaPath(t *testing.T) { }) } } + +func TestLookupType(t *testing.T) { + t.Parallel() + + type testCase struct { + p SchemaPath + toplevelType valueshim.Type + expectedType valueshim.Type + isErr bool + } + + testCases := []testCase{ + { + p: NewSchemaPath(), + toplevelType: valueshim.FromTType(tftypes.Object{ + AttributeTypes: map[string]tftypes.Type{ + "x": tftypes.String, + }, + }), + expectedType: valueshim.FromTType(tftypes.Object{ + AttributeTypes: map[string]tftypes.Type{ + "x": tftypes.String, + }, + }), + }, + { + p: NewSchemaPath().GetAttr("x"), + toplevelType: valueshim.FromTType(tftypes.Object{ + AttributeTypes: map[string]tftypes.Type{ + "x": tftypes.String, + }, + }), + expectedType: valueshim.FromTType(tftypes.String), + }, + { + p: NewSchemaPath().GetAttr("y"), + toplevelType: valueshim.FromTType(tftypes.Object{ + AttributeTypes: map[string]tftypes.Type{ + "x": tftypes.String, + }, + }), + isErr: true, + }, + { + p: NewSchemaPath().GetAttr("x").GetAttr("y"), + toplevelType: valueshim.FromTType(tftypes.Object{ + AttributeTypes: map[string]tftypes.Type{ + "x": tftypes.Object{ + AttributeTypes: map[string]tftypes.Type{ + "y": tftypes.String, + }, + }, + }, + }), + expectedType: valueshim.FromTType(tftypes.String), + }, + { + p: NewSchemaPath().Element(), + toplevelType: valueshim.FromTType(tftypes.Object{ + AttributeTypes: map[string]tftypes.Type{ + "x": tftypes.String, + }, + }), + isErr: true, + }, + { + p: NewSchemaPath().Element(), + toplevelType: valueshim.FromTType(tftypes.Map{ + ElementType: tftypes.String, + }), + expectedType: valueshim.FromTType(tftypes.String), + }, + { + p: NewSchemaPath().Element(), + toplevelType: valueshim.FromTType(tftypes.Set{ + ElementType: tftypes.String, + }), + expectedType: valueshim.FromTType(tftypes.String), + }, + { + p: NewSchemaPath().Element(), + toplevelType: valueshim.FromTType(tftypes.List{ + ElementType: tftypes.String, + }), + expectedType: valueshim.FromTType(tftypes.String), + }, + } + + for i, tc := range testCases { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + actualType, err := LookupType(tc.p, tc.toplevelType) + if tc.isErr { + require.NotNil(t, err) + t.Logf("ERROR: %v", err) + } else { + require.NoError(t, err) + require.Equal(t, tc.expectedType, actualType) + } + }) + } +} From 7cd12de7f99aa368e8c0d9a5911e02a24e849d20 Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Fri, 16 May 2025 16:34:43 -0400 Subject: [PATCH 06/31] Link schema type propagation together --- pkg/pf/internal/schemashim/datasource.go | 10 + .../schemashim/object_pseudoresource.go | 9 + pkg/pf/internal/schemashim/resource.go | 10 + pkg/pf/proto/element.go | 5 + pkg/pf/proto/object.go | 6 + pkg/pf/proto/resource.go | 6 + pkg/pf/proto/schema.go | 6 + pkg/pf/tfbridge/provider.go | 3 +- pkg/pf/tfbridge/resource_state.go | 3 +- pkg/tfbridge/provider.go | 9 +- pkg/tfbridge/rawstate.go | 20 +- pkg/tfshim/schema/resource.go | 6 + pkg/tfshim/sdk-v1/resource.go | 5 + pkg/tfshim/sdk-v2/provider2.go | 4 + pkg/tfshim/sdk-v2/resource.go | 5 + pkg/tfshim/sdk-v2/resource_config.go | 3 +- pkg/tfshim/shim.go | 1 + pkg/valueshim/cty.go | 14 +- pkg/valueshim/hcty.go | 173 ++++++++++++++++++ 19 files changed, 280 insertions(+), 18 deletions(-) create mode 100644 pkg/valueshim/hcty.go diff --git a/pkg/pf/internal/schemashim/datasource.go b/pkg/pf/internal/schemashim/datasource.go index dcfe3ffe0..7b3f5c718 100644 --- a/pkg/pf/internal/schemashim/datasource.go +++ b/pkg/pf/internal/schemashim/datasource.go @@ -15,9 +15,13 @@ package schemashim import ( + "context" + "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/internal/internalinter" "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/pf/internal/runtypes" shim "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim" + "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/valueshim" + "github.com/pulumi/pulumi/sdk/v3/go/common/util/contract" ) type schemaOnlyDataSource struct { @@ -27,6 +31,12 @@ type schemaOnlyDataSource struct { var _ shim.Resource = (*schemaOnlyDataSource)(nil) +func (r *schemaOnlyDataSource) SchemaType() valueshim.Type { + protoSchema, err := r.tf.ResourceProtoSchema(context.Background()) + contract.AssertNoErrorf(err, "ResourceProtoSchema failed") + return valueshim.FromTType(protoSchema.ValueType()) +} + func (r *schemaOnlyDataSource) Schema() shim.SchemaMap { return r.tf.Shim() } diff --git a/pkg/pf/internal/schemashim/object_pseudoresource.go b/pkg/pf/internal/schemashim/object_pseudoresource.go index 40db2a67c..c70ce9e30 100644 --- a/pkg/pf/internal/schemashim/object_pseudoresource.go +++ b/pkg/pf/internal/schemashim/object_pseudoresource.go @@ -28,6 +28,7 @@ import ( "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/internal/internalinter" "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/pf/internal/pfutils" shim "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim" + "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/valueshim" ) // An Object type that masquerades as a Resource. This is a workaround to reusing tfgen code for generating schemas, @@ -77,6 +78,10 @@ func (r *objectPseudoResource) Schema() shim.SchemaMap { return r } +func (r *objectPseudoResource) SchemaType() valueshim.Type { + return valueshim.FromTType(tftypes.Object{}) +} + func (*objectPseudoResource) SchemaVersion() int { panic("This is an Object type encoded as a shim.Resource, and " + "SchemaVersion() should not be called on this entity during schema generation") @@ -200,6 +205,10 @@ func newTuplePseudoResource(t attr.TypeWithElementTypes) shim.Resource { } } +func (r *tuplePseudoResource) SchemaType() valueshim.Type { + return valueshim.FromTType(tftypes.Object{}) +} + func (*tuplePseudoResource) SchemaVersion() int { panic("TODO") } func (*tuplePseudoResource) DeprecationMessage() string { panic("TODO") } diff --git a/pkg/pf/internal/schemashim/resource.go b/pkg/pf/internal/schemashim/resource.go index 99807bf4b..341a798c6 100644 --- a/pkg/pf/internal/schemashim/resource.go +++ b/pkg/pf/internal/schemashim/resource.go @@ -15,9 +15,13 @@ package schemashim import ( + "context" + "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/internal/internalinter" "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/pf/internal/runtypes" shim "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim" + "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/valueshim" + "github.com/pulumi/pulumi/sdk/v3/go/common/util/contract" ) type schemaOnlyResource struct { @@ -43,6 +47,12 @@ func (r *schemaOnlyResource) DeprecationMessage() string { return r.tf.DeprecationMessage() } +func (r *schemaOnlyResource) SchemaType() valueshim.Type { + s, err := r.tf.ResourceProtoSchema(context.Background()) + contract.AssertNoErrorf(err, "failed to extract schema") + return valueshim.FromTType(s.ValueType()) +} + func (*schemaOnlyResource) Importer() shim.ImportFunc { panic("schemaOnlyResource does not implement runtime operation ImporterFunc") } diff --git a/pkg/pf/proto/element.go b/pkg/pf/proto/element.go index 9e8d421f7..d1e7a324b 100644 --- a/pkg/pf/proto/element.go +++ b/pkg/pf/proto/element.go @@ -19,6 +19,7 @@ import ( "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/internal/internalinter" shim "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim" + "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/valueshim" ) var ( @@ -113,6 +114,10 @@ func (o elementObject) Schema() shim.SchemaMap { return elementObjectMap(o.typ) } +func (o elementObject) SchemaType() valueshim.Type { + return valueshim.FromTType(o.typ) +} + func (m elementObjectMap) Len() int { return len(m.AttributeTypes) } func (m elementObjectMap) Get(key string) shim.Schema { return getSchemaMap(m, key) } diff --git a/pkg/pf/proto/object.go b/pkg/pf/proto/object.go index d3dc6132f..74bd9d4db 100644 --- a/pkg/pf/proto/object.go +++ b/pkg/pf/proto/object.go @@ -20,6 +20,7 @@ import ( "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/internal/internalinter" shim "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim" + "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/valueshim" ) var ( @@ -33,6 +34,11 @@ type object struct { internalinter.Internal } +func (o object) SchemaType() valueshim.Type { + ty := o.obj.ValueType() + return valueshim.FromTType(ty) +} + func (o object) Schema() shim.SchemaMap { contract.Assertf(o.obj.Nesting != tfprotov6.SchemaObjectNestingModeMap, "%T cannot be a map, since that would require `o` to represent a Map type", o) diff --git a/pkg/pf/proto/resource.go b/pkg/pf/proto/resource.go index 9555457b5..cbb309075 100644 --- a/pkg/pf/proto/resource.go +++ b/pkg/pf/proto/resource.go @@ -20,6 +20,7 @@ import ( "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/internal/internalinter" shim "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim" + "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/valueshim" ) var ( @@ -70,6 +71,11 @@ func newResource(r *tfprotov6.Schema) *resource { return &resource{r, internalinter.Internal{}} } +func (r resource) SchemaType() valueshim.Type { + ty := r.r.Block.ValueType() + return valueshim.FromTType(ty) +} + func (r resource) Schema() shim.SchemaMap { return blockMap{r.r.Block} } diff --git a/pkg/pf/proto/schema.go b/pkg/pf/proto/schema.go index 576f7c607..9701c8ef5 100644 --- a/pkg/pf/proto/schema.go +++ b/pkg/pf/proto/schema.go @@ -17,12 +17,18 @@ package proto import ( // "github.com/pulumi/pulumi/sdk/v3/go/common/util/contract" + "github.com/hashicorp/terraform-plugin-go/tftypes" shim "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim" + "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/valueshim" ) // pseudoResource represents a type that must pretent to be a [shim.Resource], but does not represent a resource. type pseudoResource struct{} +func (pseudoResource) SchemaType() valueshim.Type { + return valueshim.FromTType(tftypes.Object{}) // not a top-level resource +} + func (pseudoResource) SchemaVersion() int { return 0 } func (pseudoResource) Importer() shim.ImportFunc { return nil } func (pseudoResource) Timeouts() *shim.ResourceTimeout { return nil } diff --git a/pkg/pf/tfbridge/provider.go b/pkg/pf/tfbridge/provider.go index f43c5d869..6ae562ff6 100644 --- a/pkg/pf/tfbridge/provider.go +++ b/pkg/pf/tfbridge/provider.go @@ -339,7 +339,8 @@ func (p *provider) returnTerraformConfig() (resource.PropertyMap, error) { } // use valueshim package to marshal tfConfigValue into raw json, // which can be unmarshaled into a map[string]interface{} - configJSONMessage, err := valueshim.FromTValue(tfConfigValue).Marshal() + value := valueshim.FromTValue(tfConfigValue) + configJSONMessage, err := value.Marshal(value.Type()) if err != nil { return nil, fmt.Errorf("error marshaling into raw JSON message: %v", err) } diff --git a/pkg/pf/tfbridge/resource_state.go b/pkg/pf/tfbridge/resource_state.go index 4fb1219fd..4867b3b24 100644 --- a/pkg/pf/tfbridge/resource_state.go +++ b/pkg/pf/tfbridge/resource_state.go @@ -185,7 +185,8 @@ func insertRawStateDelta(ctx context.Context, rh *resourceHandle, pm resource.Pr schemaInfos := rh.pulumiResourceInfo.GetFields() v := valueshim.FromTValue(state) - delta, err := tfbridge.RawStateComputeDelta(ctx, rh.schema.Shim(), schemaInfos, pm, v) + st := valueshim.FromTType(rh.schema.Type(ctx)) + delta, err := tfbridge.RawStateComputeDelta(ctx, rh.schema.Shim(), schemaInfos, pm, st, v) if err != nil { return err } diff --git a/pkg/tfbridge/provider.go b/pkg/tfbridge/provider.go index e778c2d01..44af91262 100644 --- a/pkg/tfbridge/provider.go +++ b/pkg/tfbridge/provider.go @@ -1371,7 +1371,8 @@ func (p *Provider) Create(ctx context.Context, req *pulumirpc.CreateRequest) (*p } if p.info.RawStateDeltaEnabled() { - if err := RawStateInjectDelta(ctx, res.TF.Schema(), res.Schema.Fields, props, newstate); err != nil { + s := res.TF.SchemaType() + if err := RawStateInjectDelta(ctx, res.TF.Schema(), res.Schema.Fields, props, s, newstate); err != nil { return nil, err } } @@ -1521,7 +1522,8 @@ func (p *Provider) Read(ctx context.Context, req *pulumirpc.ReadRequest) (*pulum } if p.info.RawStateDeltaEnabled() { - err := RawStateInjectDelta(ctx, res.TF.Schema(), res.Schema.Fields, props, newstate) + s := res.TF.SchemaType() + err := RawStateInjectDelta(ctx, res.TF.Schema(), res.Schema.Fields, props, s, newstate) if err != nil { return nil, err } @@ -1750,7 +1752,8 @@ func (p *Provider) Update(ctx context.Context, req *pulumirpc.UpdateRequest) (*p } if p.info.RawStateDeltaEnabled() { - if err := RawStateInjectDelta(ctx, res.TF.Schema(), res.Schema.Fields, props, newstate); err != nil { + s := res.TF.SchemaType() + if err := RawStateInjectDelta(ctx, res.TF.Schema(), res.Schema.Fields, props, s, newstate); err != nil { return nil, err } } diff --git a/pkg/tfbridge/rawstate.go b/pkg/tfbridge/rawstate.go index 45d22ea90..00bcd8674 100644 --- a/pkg/tfbridge/rawstate.go +++ b/pkg/tfbridge/rawstate.go @@ -454,6 +454,7 @@ func RawStateInjectDelta( schemaMap shim.SchemaMap, // top-level schema for a resource schemaInfos map[string]*SchemaInfo, // top-level schema overrides for a resource outMap resource.PropertyMap, + schemaType valueshim.Type, instanceState shim.InstanceState, ) error { // If called in a pulumi preview e.g. Create(preview=true) or in a continue-on-error scenario, bail because the @@ -467,7 +468,7 @@ func RawStateInjectDelta( return nil } v := instanceStateCty.Value() - d, err := RawStateComputeDelta(ctx, schemaMap, schemaInfos, outMap, v) + d, err := RawStateComputeDelta(ctx, schemaMap, schemaInfos, outMap, schemaType, v) if err != nil { return err } @@ -480,6 +481,7 @@ func RawStateComputeDelta( schemaMap shim.SchemaMap, // top-level schema for a resource schemaInfos map[string]*SchemaInfo, // top-level schema overrides for a resource outMap resource.PropertyMap, + schemaType valueshim.Type, v valueshim.Value, ) (RawStateDelta, error) { ih := &rawStateDeltaHelper{ @@ -491,7 +493,7 @@ func RawStateComputeDelta( vWithoutTimeouts := v.Remove("timeouts") delta := ih.delta(pv, vWithoutTimeouts) - err := delta.turnaroundCheck(ctx, newRawStateFromValue(vWithoutTimeouts), pv) + err := delta.turnaroundCheck(ctx, newRawStateFromValue(schemaType, vWithoutTimeouts), pv) if err != nil { return RawStateDelta{}, err } @@ -571,6 +573,7 @@ type rawStateDeltaHelper struct { schemaMap shim.SchemaMap // top-level schema for a resource schemaInfos map[string]*SchemaInfo // top-level schema overrides for a resource logger log.Logger + schemaType valueshim.Type } func (ih *rawStateDeltaHelper) delta(pv resource.PropertyValue, v valueshim.Value) RawStateDelta { @@ -603,7 +606,14 @@ func (ih *rawStateDeltaHelper) replaceDeltaAt( err, )) } - return RawStateDelta{Replace: &replaceDelta{Raw: newRawStateFromValue(v)}} + + relevantSchemaType, err := walk.LookupType(path, ih.schemaType) + if err != nil { + // If lookup failed, ignore the schema type; will only affect DynamicPseudoType. + relevantSchemaType = v.Type() + } + + return RawStateDelta{Replace: &replaceDelta{Raw: newRawStateFromValue(relevantSchemaType, v)}} } // Errors returned from this inner function are simply missed opportunities for optimization, as [deltaAt] will always @@ -834,8 +844,8 @@ func (ih *rawStateDeltaHelper) computeDeltaAt( } } -func newRawStateFromValue(v valueshim.Value) rawstate.RawState { - raw, err := v.Marshal() +func newRawStateFromValue(schemaType valueshim.Type, v valueshim.Value) rawstate.RawState { + raw, err := v.Marshal(schemaType) contract.AssertNoErrorf(err, "v.Marshal() failed") return rawstate.RawState(raw) } diff --git a/pkg/tfshim/schema/resource.go b/pkg/tfshim/schema/resource.go index 8edb3d5be..e5566305f 100644 --- a/pkg/tfshim/schema/resource.go +++ b/pkg/tfshim/schema/resource.go @@ -5,6 +5,7 @@ import ( "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/internal/internalinter" shim "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim" + "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/valueshim" ) var _ = shim.ResourceMap(ResourceMap{}) @@ -15,6 +16,7 @@ type Resource struct { Importer shim.ImportFunc DeprecationMessage string Timeouts *shim.ResourceTimeout + SchemaType valueshim.Type } func (r *Resource) Shim() shim.Resource { @@ -42,6 +44,10 @@ func (r ResourceShim) DeprecationMessage() string { return r.V.DeprecationMessage } +func (r ResourceShim) SchemaType() valueshim.Type { + return r.V.SchemaType +} + func (r ResourceShim) Timeouts() *shim.ResourceTimeout { return r.V.Timeouts } diff --git a/pkg/tfshim/sdk-v1/resource.go b/pkg/tfshim/sdk-v1/resource.go index caa754269..60c7f74f1 100644 --- a/pkg/tfshim/sdk-v1/resource.go +++ b/pkg/tfshim/sdk-v1/resource.go @@ -8,6 +8,7 @@ import ( "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/internal/internalinter" shim "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim" + "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/valueshim" ) var ( @@ -32,6 +33,10 @@ func (r v1Resource) SchemaVersion() int { return r.tf.SchemaVersion } +func (r v1Resource) SchemaType() valueshim.Type { + return valueshim.FromCtyType(r.tf.CoreConfigSchema().ImpliedType()) +} + func (r v1Resource) Importer() shim.ImportFunc { if r.tf.Importer == nil { return nil diff --git a/pkg/tfshim/sdk-v2/provider2.go b/pkg/tfshim/sdk-v2/provider2.go index b01230dd7..554a39d6c 100644 --- a/pkg/tfshim/sdk-v2/provider2.go +++ b/pkg/tfshim/sdk-v2/provider2.go @@ -75,6 +75,10 @@ func (r *v2Resource2) DecodeTimeouts(config shim.ResourceConfig) (*shim.Resource return v2Resource{tf: r.tf}.DecodeTimeouts(config) } +func (r *v2Resource2) SchemaType() valueshim.Type { + return valueshim.FromHCtyType(r.tf.CoreConfigSchema().ImpliedType()) +} + type v2InstanceState2 struct { resourceType string stateValue cty.Value diff --git a/pkg/tfshim/sdk-v2/resource.go b/pkg/tfshim/sdk-v2/resource.go index 606b24636..2cdb7322e 100644 --- a/pkg/tfshim/sdk-v2/resource.go +++ b/pkg/tfshim/sdk-v2/resource.go @@ -9,6 +9,7 @@ import ( "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/internal/internalinter" shim "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim" + "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/valueshim" ) var ( @@ -33,6 +34,10 @@ func (r v2Resource) SchemaVersion() int { return r.tf.SchemaVersion } +func (r v2Resource) SchemaType() valueshim.Type { + return valueshim.FromHCtyType(r.tf.CoreConfigSchema().ImpliedType()) +} + func (r v2Resource) Importer() shim.ImportFunc { // When v2Resource represents resources, it is wrapped in v2Resource2 and v2Resource2.Importer() is called. // The residual use case is v2Resource representing data sources, but those do not support importers. diff --git a/pkg/tfshim/sdk-v2/resource_config.go b/pkg/tfshim/sdk-v2/resource_config.go index 68b09ec8c..6cdae4201 100644 --- a/pkg/tfshim/sdk-v2/resource_config.go +++ b/pkg/tfshim/sdk-v2/resource_config.go @@ -42,7 +42,8 @@ func (c v2ResourceConfig) GetRawConfigMap() (map[string]any, error) { msg := "ConfigMap contains unknowns" return nil, fmt.Errorf("%s", msg) } - configJSONMessage, err := valueshim.FromHCtyValue(ctyValue).Marshal() + value := valueshim.FromHCtyValue(ctyValue) + configJSONMessage, err := value.Marshal(value.Type()) if err != nil { return nil, fmt.Errorf("error marshaling into raw JSON message: %v", err) } diff --git a/pkg/tfshim/shim.go b/pkg/tfshim/shim.go index 3bba20a91..ab6b56a80 100644 --- a/pkg/tfshim/shim.go +++ b/pkg/tfshim/shim.go @@ -263,6 +263,7 @@ type Resource interface { DecodeTimeouts(config ResourceConfig) (*ResourceTimeout, error) + SchemaType() valueshim.Type // This is a no-op internal interface to prevent external users from implementing the interface. internalinter.InternalInterface } diff --git a/pkg/valueshim/cty.go b/pkg/valueshim/cty.go index cda094a06..c546b098d 100644 --- a/pkg/valueshim/cty.go +++ b/pkg/valueshim/cty.go @@ -19,17 +19,17 @@ import ( "fmt" "math/big" - "github.com/hashicorp/go-cty/cty" - ctyjson "github.com/hashicorp/go-cty/cty/json" + "github.com/zclconf/go-cty/cty" + ctyjson "github.com/zclconf/go-cty/cty/json" ) // Wrap a cty.Value as Value. -func FromHCtyValue(v cty.Value) Value { +func FromCtyValue(v cty.Value) Value { return ctyValueShim(v) } // Wrap a cty.Type as Type. -func FromHCtyType(v cty.Type) Type { +func FromCtyType(v cty.Type) Type { return ctyTypeShim(v) } @@ -50,7 +50,7 @@ func (v ctyValueShim) GoString() string { } func (v ctyValueShim) Type() Type { - return FromHCtyType(v.val().Type()) + return FromCtyType(v.val().Type()) } func (v ctyValueShim) StringValue() string { @@ -163,7 +163,7 @@ func (t ctyTypeShim) AttributeType(name string) (Type, bool) { if !tt.HasAttribute(name) { return nil, false } - return FromHCtyType(tt.AttributeType(name)), true + return FromCtyType(tt.AttributeType(name)), true } func (t ctyTypeShim) ElementType() (Type, bool) { @@ -171,7 +171,7 @@ func (t ctyTypeShim) ElementType() (Type, bool) { if !tt.IsCollectionType() { return nil, false } - return FromHCtyType(tt.ElementType()), true + return FromCtyType(tt.ElementType()), true } func (t ctyTypeShim) GoString() string { diff --git a/pkg/valueshim/hcty.go b/pkg/valueshim/hcty.go new file mode 100644 index 000000000..9eee89a75 --- /dev/null +++ b/pkg/valueshim/hcty.go @@ -0,0 +1,173 @@ +// Copyright 2016-2025, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package valueshim + +import ( + "encoding/json" + "fmt" + + "github.com/hashicorp/go-cty/cty" + ctyjson "github.com/hashicorp/go-cty/cty/json" +) + +// Wrap a cty.Value as Value. +func FromHCtyValue(v cty.Value) Value { + return hctyValueShim(v) +} + +// Wrap a cty.Type as Type. +func FromHCtyType(v cty.Type) Type { + return hctyTypeShim(v) +} + +type hctyValueShim cty.Value + +var _ Value = (*hctyValueShim)(nil) + +func (v hctyValueShim) val() cty.Value { + return cty.Value(v) +} + +func (v hctyValueShim) IsNull() bool { + return v.val().IsNull() +} + +func (v hctyValueShim) GoString() string { + return v.val().GoString() +} + +func (v hctyValueShim) Type() Type { + return FromHCtyType(v.val().Type()) +} + +func (v hctyValueShim) StringValue() string { + return v.val().AsString() +} + +func (v hctyValueShim) BoolValue() bool { + return v.val().True() +} + +func (v hctyValueShim) NumberValue() float64 { + f, _ := v.val().AsBigFloat().Float64() + return f +} + +func (v hctyValueShim) AsValueSlice() []Value { + s := v.val().AsValueSlice() + res := make([]Value, len(s)) + for i, v := range s { + res[i] = hctyValueShim(v) + } + return res +} + +func (v hctyValueShim) AsValueMap() map[string]Value { + m := v.val().AsValueMap() + res := make(map[string]Value, len(m)) + + for k, v := range m { + res[k] = hctyValueShim(v) + } + return res +} + +func (v hctyValueShim) Remove(key string) Value { + switch { + case v.val().Type().IsObjectType(): + m := v.val().AsValueMap() + delete(m, key) + if len(m) == 0 { + return hctyValueShim(cty.EmptyObjectVal) + } + return hctyValueShim(cty.ObjectVal(m)) + default: + return v + } +} + +func (v hctyValueShim) Marshal(schemaType Type) (json.RawMessage, error) { + vv := v.val() + tt, ok := schemaType.(hctyTypeShim) + if !ok { + return nil, fmt.Errorf("Cannot marshal to RawState: "+ + "expected schemaType to be of type hctyTypeShim, got %#T", + schemaType) + } + raw, err := ctyjson.Marshal(vv, tt.ty()) + if err != nil { + return nil, fmt.Errorf("Cannot marshal to RawState: %w", err) + } + return json.RawMessage(raw), nil +} + +type hctyTypeShim cty.Type + +var _ Type = hctyTypeShim{} + +func (t hctyTypeShim) ty() cty.Type { + return cty.Type(t) +} + +func (t hctyTypeShim) IsNumberType() bool { + return t.ty().Equals(cty.Number) +} + +func (t hctyTypeShim) IsBooleanType() bool { + return t.ty().Equals(cty.Bool) +} + +func (t hctyTypeShim) IsStringType() bool { + return t.ty().Equals(cty.String) +} + +func (t hctyTypeShim) IsListType() bool { + return t.ty().IsListType() +} + +func (t hctyTypeShim) IsMapType() bool { + return t.ty().IsMapType() +} + +func (t hctyTypeShim) IsSetType() bool { + return t.ty().IsSetType() +} + +func (t hctyTypeShim) IsObjectType() bool { + return t.ty().IsObjectType() +} + +func (t hctyTypeShim) AttributeType(name string) (Type, bool) { + tt := t.ty() + if !tt.IsObjectType() { + return nil, false + } + if !tt.HasAttribute(name) { + return nil, false + } + return FromHCtyType(tt.AttributeType(name)), true +} + +func (t hctyTypeShim) ElementType() (Type, bool) { + tt := t.ty() + if !tt.IsCollectionType() { + return nil, false + } + return FromHCtyType(tt.ElementType()), true +} + +func (t hctyTypeShim) GoString() string { + return t.ty().GoString() +} From fe7ba20bc88361c3b46082b8ce07a50639f0f7cd Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Fri, 16 May 2025 16:37:37 -0400 Subject: [PATCH 07/31] Replace pfutils --- pkg/pf/internal/pfutils/raw_state.go | 28 ---- pkg/pf/internal/pfutils/value_to_json.go | 189 ----------------------- pkg/pf/tfbridge/resource_state.go | 5 +- 3 files changed, 3 insertions(+), 219 deletions(-) delete mode 100644 pkg/pf/internal/pfutils/raw_state.go delete mode 100644 pkg/pf/internal/pfutils/value_to_json.go diff --git a/pkg/pf/internal/pfutils/raw_state.go b/pkg/pf/internal/pfutils/raw_state.go deleted file mode 100644 index 07082f535..000000000 --- a/pkg/pf/internal/pfutils/raw_state.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2016-2023, Pulumi Corporation. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pfutils - -import ( - "github.com/hashicorp/terraform-plugin-go/tfprotov6" - "github.com/hashicorp/terraform-plugin-go/tftypes" -) - -func NewRawState(t tftypes.Type, v tftypes.Value) (*tfprotov6.RawState, error) { - json, err := ValueToJSON(t, v) - if err != nil { - return nil, err - } - return &tfprotov6.RawState{JSON: json}, nil -} diff --git a/pkg/pf/internal/pfutils/value_to_json.go b/pkg/pf/internal/pfutils/value_to_json.go deleted file mode 100644 index 9b2cb3ef8..000000000 --- a/pkg/pf/internal/pfutils/value_to_json.go +++ /dev/null @@ -1,189 +0,0 @@ -// Copyright 2016-2023, Pulumi Corporation. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pfutils - -import ( - "encoding/json" - "fmt" - "math/big" - - "github.com/hashicorp/terraform-plugin-go/tftypes" -) - -// Inverse of tftypes.ValueFromJson. -func ValueToJSON(typ tftypes.Type, v tftypes.Value) ([]byte, error) { - raw, err := jsonMarshal(v, typ, tftypes.NewAttributePath()) - if err != nil { - return nil, err - } - return json.Marshal(raw) -} - -func jsonMarshal(v tftypes.Value, typ tftypes.Type, p *tftypes.AttributePath) (interface{}, error) { - if v.IsNull() { - return nil, nil - } - if !v.IsKnown() { - return nil, p.NewErrorf("unknown values cannot be serialized to JSON") - } - switch { - case typ.Is(tftypes.String): - return jsonMarshalString(v, typ, p) - case typ.Is(tftypes.Number): - return jsonMarshalNumber(v, typ, p) - case typ.Is(tftypes.Bool): - return jsonMarshalBool(v, typ, p) - case typ.Is(tftypes.DynamicPseudoType): - return jsonMarshalDynamicPseudoType(v, typ, p) - case typ.Is(tftypes.List{}): - return jsonMarshalList(v, typ.(tftypes.List).ElementType, p) - case typ.Is(tftypes.Set{}): - return jsonMarshalSet(v, typ.(tftypes.Set).ElementType, p) - case typ.Is(tftypes.Map{}): - return jsonMarshalMap(v, typ.(tftypes.Map).ElementType, p) - case typ.Is(tftypes.Tuple{}): - return jsonMarshalTuple(v, typ.(tftypes.Tuple).ElementTypes, p) - case typ.Is(tftypes.Object{}): - return jsonMarshalObject(v, typ.(tftypes.Object).AttributeTypes, p) - } - - return nil, p.NewErrorf("unknown type %s", typ) -} - -func jsonMarshalString(v tftypes.Value, typ tftypes.Type, p *tftypes.AttributePath) (interface{}, error) { - var stringValue string - err := v.As(&stringValue) - if err != nil { - return nil, p.NewError(err) - } - return stringValue, nil -} - -func jsonMarshalNumber(v tftypes.Value, typ tftypes.Type, p *tftypes.AttributePath) (interface{}, error) { - var n big.Float - err := v.As(&n) - if err != nil { - return nil, p.NewError(err) - } - f64, _ := n.Float64() - return f64, nil -} - -func jsonMarshalBool(v tftypes.Value, typ tftypes.Type, p *tftypes.AttributePath) (interface{}, error) { - var b bool - err := v.As(&b) - if err != nil { - return nil, p.NewError(err) - } - return b, nil -} - -func jsonMarshalDynamicPseudoType(v tftypes.Value, typ tftypes.Type, p *tftypes.AttributePath) ([]byte, error) { - return nil, fmt.Errorf("DynamicPseudoType is not yet supported") -} - -func jsonMarshalList(v tftypes.Value, elementType tftypes.Type, p *tftypes.AttributePath) (interface{}, error) { - var vs []tftypes.Value - err := v.As(&vs) - if err != nil { - return nil, p.NewError(err) - } - var res []interface{} - for i, v := range vs { - ep := p.WithElementKeyInt(i) - e, err := jsonMarshal(v, elementType, ep) - if err != nil { - return nil, ep.NewError(err) - } - res = append(res, e) - } - return res, nil -} - -func jsonMarshalSet(v tftypes.Value, elementType tftypes.Type, p *tftypes.AttributePath) (interface{}, error) { - var vs []tftypes.Value - err := v.As(&vs) - if err != nil { - return nil, p.NewError(err) - } - var res []interface{} - for _, v := range vs { - ep := p.WithElementKeyValue(v) - e, err := jsonMarshal(v, elementType, ep) - if err != nil { - return nil, ep.NewError(err) - } - res = append(res, e) - } - return res, nil -} - -func jsonMarshalMap(v tftypes.Value, elementType tftypes.Type, p *tftypes.AttributePath) (interface{}, error) { - var vs map[string]tftypes.Value - err := v.As(&vs) - if err != nil { - return nil, p.NewError(err) - } - res := map[string]interface{}{} - for k, v := range vs { - ep := p.WithElementKeyValue(v) - e, err := jsonMarshal(v, elementType, ep) - if err != nil { - return nil, ep.NewError(err) - } - res[k] = e - } - return res, nil -} - -func jsonMarshalTuple(v tftypes.Value, elementTypes []tftypes.Type, p *tftypes.AttributePath) (interface{}, error) { - var vs []tftypes.Value - err := v.As(&vs) - if err != nil { - return nil, p.NewError(err) - } - var res []interface{} - for i, v := range vs { - ep := p.WithElementKeyInt(i) - e, err := jsonMarshal(v, elementTypes[i], ep) - if err != nil { - return nil, ep.NewError(err) - } - res = append(res, e) - } - return res, nil -} - -func jsonMarshalObject( - v tftypes.Value, - elementTypes map[string]tftypes.Type, - p *tftypes.AttributePath, -) (interface{}, error) { - var vs map[string]tftypes.Value - err := v.As(&vs) - if err != nil { - return nil, p.NewError(err) - } - res := map[string]interface{}{} - for k, v := range vs { - ep := p.WithAttributeName(k) - e, err := jsonMarshal(v, elementTypes[k], ep) - if err != nil { - return nil, ep.NewError(err) - } - res[k] = e - } - return res, nil -} diff --git a/pkg/pf/tfbridge/resource_state.go b/pkg/pf/tfbridge/resource_state.go index 4867b3b24..7fff16711 100644 --- a/pkg/pf/tfbridge/resource_state.go +++ b/pkg/pf/tfbridge/resource_state.go @@ -29,7 +29,6 @@ import ( "github.com/pulumi/pulumi/sdk/v3/go/common/util/contract" "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/convert" - "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/pf/internal/pfutils" "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/reservedkeys" "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfbridge" "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/valueshim" @@ -257,6 +256,7 @@ func (p *provider) parseAndUpgradeResourceState( return nil, fmt.Errorf("[pf/tfbridge] Error calling EncodePropertyMap: %w", err) } + // Before EnableRawStateDelta rollout, the behavior used to be to skip the upgrade method in case of an exact // version match. This seems incorrect, but to derisk fixing this problem it is flagged together with // EnableRawStateDelta so it participates in the phased rollout. Remove once rollout completes. @@ -269,10 +269,11 @@ func (p *provider) parseAndUpgradeResourceState( } tfType := rh.schema.Type(ctx).(tftypes.Object) - rawState, err := pfutils.NewRawState(tfType, value) + rawStateBytes, err := valueshim.FromTValue(value).Marshal(valueshim.FromTType(tfType)) if err != nil { return nil, fmt.Errorf("[pf/tfbridge] Error calling NewRawState: %w", err) } + rawState := &tfprotov6.RawState{JSON: []byte(rawStateBytes)} return p.upgradeResourceState(ctx, rh, rawState, parsedMeta.PrivateState, stateVersion) } From b3b7f98095ca9947581ff2ee4a0b5e5715c6a3d6 Mon Sep 17 00:00:00 2001 From: Venelin Date: Thu, 29 May 2025 15:08:05 +0300 Subject: [PATCH 08/31] fix function calls --- pkg/pf/tests/upgrade_state_cross_test.go | 12 ++++++------ pkg/tfbridge/rawstate_test.go | 2 +- pkg/valueshim/hcty.go | 5 +++++ 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/pkg/pf/tests/upgrade_state_cross_test.go b/pkg/pf/tests/upgrade_state_cross_test.go index 7f421d2f4..9f0d37eed 100644 --- a/pkg/pf/tests/upgrade_state_cross_test.go +++ b/pkg/pf/tests/upgrade_state_cross_test.go @@ -1099,9 +1099,9 @@ func TestPFUpgrade_DynamicPseudoType(t *testing.T) { //skipUnlessDeltasEnabled(t) resourceBefore := pb.NewResource(pb.NewResourceArgs{ - ResourceSchema: schema.Schema{ - Attributes: map[string]schema.Attribute{ - "dyn": schema.DynamicAttribute{Optional: true}, + ResourceSchema: rschema.Schema{ + Attributes: map[string]rschema.Attribute{ + "dyn": rschema.DynamicAttribute{Optional: true}, }, }, }) @@ -1115,9 +1115,9 @@ func TestPFUpgrade_DynamicPseudoType(t *testing.T) { }, } }, - ResourceSchema: schema.Schema{ - Attributes: map[string]schema.Attribute{ - "dyn": schema.DynamicAttribute{Optional: true}, + ResourceSchema: rschema.Schema{ + Attributes: map[string]rschema.Attribute{ + "dyn": rschema.DynamicAttribute{Optional: true}, }, Version: 1, }, diff --git a/pkg/tfbridge/rawstate_test.go b/pkg/tfbridge/rawstate_test.go index d0f2f6d8d..989a020ba 100644 --- a/pkg/tfbridge/rawstate_test.go +++ b/pkg/tfbridge/rawstate_test.go @@ -1168,7 +1168,7 @@ func Test_rawstate_against_MakeTerraformOutputs(t *testing.T) { require.NoError(t, err) tc.infl.Equal(t, string(deltaJSON)) - err = delta.turnaroundCheck(ctx, newRawStateFromValue(stateValue), pv) + err = delta.turnaroundCheck(ctx, newRawStateFromValue(stateValue.Type(), stateValue), pv) assert.NoError(t, err) }) } diff --git a/pkg/valueshim/hcty.go b/pkg/valueshim/hcty.go index 9eee89a75..e4e258fd9 100644 --- a/pkg/valueshim/hcty.go +++ b/pkg/valueshim/hcty.go @@ -17,6 +17,7 @@ package valueshim import ( "encoding/json" "fmt" + "math/big" "github.com/hashicorp/go-cty/cty" ctyjson "github.com/hashicorp/go-cty/cty/json" @@ -65,6 +66,10 @@ func (v hctyValueShim) NumberValue() float64 { return f } +func (v hctyValueShim) BigFloatValue() *big.Float { + return v.val().AsBigFloat() +} + func (v hctyValueShim) AsValueSlice() []Value { s := v.val().AsValueSlice() res := make([]Value, len(s)) From 97127f91f2bd0f7f21c5ccc001a2f87168fa250e Mon Sep 17 00:00:00 2001 From: Venelin Date: Thu, 29 May 2025 15:13:40 +0300 Subject: [PATCH 09/31] lint --- pkg/pf/internal/schemashim/datasource.go | 3 ++- pkg/pf/internal/schemashim/resource.go | 3 ++- pkg/pf/proto/schema.go | 3 +-- pkg/pf/tests/upgrade_state_cross_test.go | 4 ++-- pkg/pf/tfbridge/resource_state.go | 1 - pkg/tfshim/walk/walk.go | 2 +- pkg/valueshim/cty_test.go | 4 +++- pkg/valueshim/tfvalue_test.go | 4 +++- 8 files changed, 14 insertions(+), 10 deletions(-) diff --git a/pkg/pf/internal/schemashim/datasource.go b/pkg/pf/internal/schemashim/datasource.go index 7b3f5c718..ad4626293 100644 --- a/pkg/pf/internal/schemashim/datasource.go +++ b/pkg/pf/internal/schemashim/datasource.go @@ -17,11 +17,12 @@ package schemashim import ( "context" + "github.com/pulumi/pulumi/sdk/v3/go/common/util/contract" + "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/internal/internalinter" "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/pf/internal/runtypes" shim "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim" "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/valueshim" - "github.com/pulumi/pulumi/sdk/v3/go/common/util/contract" ) type schemaOnlyDataSource struct { diff --git a/pkg/pf/internal/schemashim/resource.go b/pkg/pf/internal/schemashim/resource.go index 341a798c6..d69295d7e 100644 --- a/pkg/pf/internal/schemashim/resource.go +++ b/pkg/pf/internal/schemashim/resource.go @@ -17,11 +17,12 @@ package schemashim import ( "context" + "github.com/pulumi/pulumi/sdk/v3/go/common/util/contract" + "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/internal/internalinter" "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/pf/internal/runtypes" shim "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim" "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/valueshim" - "github.com/pulumi/pulumi/sdk/v3/go/common/util/contract" ) type schemaOnlyResource struct { diff --git a/pkg/pf/proto/schema.go b/pkg/pf/proto/schema.go index 9701c8ef5..8e98328cb 100644 --- a/pkg/pf/proto/schema.go +++ b/pkg/pf/proto/schema.go @@ -15,9 +15,8 @@ package proto import ( - // "github.com/pulumi/pulumi/sdk/v3/go/common/util/contract" - "github.com/hashicorp/terraform-plugin-go/tftypes" + shim "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim" "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/valueshim" ) diff --git a/pkg/pf/tests/upgrade_state_cross_test.go b/pkg/pf/tests/upgrade_state_cross_test.go index 9f0d37eed..16636e0bf 100644 --- a/pkg/pf/tests/upgrade_state_cross_test.go +++ b/pkg/pf/tests/upgrade_state_cross_test.go @@ -1096,7 +1096,7 @@ func TestPFUpgrade_Downgrading(t *testing.T) { func TestPFUpgrade_DynamicPseudoType(t *testing.T) { t.Parallel() ct.SkipUnlessLinux(t) - //skipUnlessDeltasEnabled(t) + // skipUnlessDeltasEnabled(t) resourceBefore := pb.NewResource(pb.NewResourceArgs{ ResourceSchema: rschema.Schema{ @@ -1135,7 +1135,7 @@ func TestPFUpgrade_DynamicPseudoType(t *testing.T) { InputsMap1: pmBefore, Inputs2: tfInputsAfter, InputsMap2: pmAfter, - //ExpectedRawStateType: resourceBeforeAndAfter.ResourceSchema.Type().TerraformType(context.Background()), + // ExpectedRawStateType: resourceBeforeAndAfter.ResourceSchema.Type().TerraformType(context.Background()), SkipPulumi: "TODO", } diff --git a/pkg/pf/tfbridge/resource_state.go b/pkg/pf/tfbridge/resource_state.go index 7fff16711..3cb45fedb 100644 --- a/pkg/pf/tfbridge/resource_state.go +++ b/pkg/pf/tfbridge/resource_state.go @@ -256,7 +256,6 @@ func (p *provider) parseAndUpgradeResourceState( return nil, fmt.Errorf("[pf/tfbridge] Error calling EncodePropertyMap: %w", err) } - // Before EnableRawStateDelta rollout, the behavior used to be to skip the upgrade method in case of an exact // version match. This seems incorrect, but to derisk fixing this problem it is flagged together with // EnableRawStateDelta so it participates in the phased rollout. Remove once rollout completes. diff --git a/pkg/tfshim/walk/walk.go b/pkg/tfshim/walk/walk.go index 0170cd8a1..eb7a44f69 100644 --- a/pkg/tfshim/walk/walk.go +++ b/pkg/tfshim/walk/walk.go @@ -179,7 +179,7 @@ func LookupSchemaMapPath(path SchemaPath, schemaMap shim.SchemaMap) (shim.Schema func LookupType(path SchemaPath, ty valueshim.Type) (valueshim.Type, error) { current := ty for i, step := range path { - var subPath SchemaPath = path[0:i] + subPath := path[0:i] switch step := step.(type) { case GetAttrStep: attr, ok := current.AttributeType(step.Name) diff --git a/pkg/valueshim/cty_test.go b/pkg/valueshim/cty_test.go index ed1f148a1..8184ca86b 100644 --- a/pkg/valueshim/cty_test.go +++ b/pkg/valueshim/cty_test.go @@ -316,6 +316,7 @@ func Test_HCty_ToX(t *testing.T) { } func Test_HCtyType_AttributeType(t *testing.T) { + t.Parallel() objTy := cty.Object(map[string]cty.Type{"x": cty.String}) ty, ok := valueshim.FromHCtyType(objTy).AttributeType("x") @@ -327,6 +328,7 @@ func Test_HCtyType_AttributeType(t *testing.T) { } func Test_HCtyType_ElementType(t *testing.T) { + t.Parallel() ty, ok := valueshim.FromHCtyType(cty.Set(cty.Number)).ElementType() assert.True(t, ok) assert.Equal(t, valueshim.FromHCtyType(cty.Number), ty) @@ -339,6 +341,6 @@ func Test_HCtyType_ElementType(t *testing.T) { assert.True(t, ok) assert.Equal(t, valueshim.FromHCtyType(cty.Number), ty) - ty, ok = valueshim.FromHCtyType(cty.String).ElementType() + _, ok = valueshim.FromHCtyType(cty.String).ElementType() assert.False(t, ok) } diff --git a/pkg/valueshim/tfvalue_test.go b/pkg/valueshim/tfvalue_test.go index 68eabcb14..fc22ef6ca 100644 --- a/pkg/valueshim/tfvalue_test.go +++ b/pkg/valueshim/tfvalue_test.go @@ -314,6 +314,7 @@ func Test_TType(t *testing.T) { } func Test_TType_AttributeType(t *testing.T) { + t.Parallel() objTy := tftypes.Object{AttributeTypes: map[string]tftypes.Type{"x": tftypes.String}} ty, ok := valueshim.FromTType(objTy).AttributeType("x") @@ -325,6 +326,7 @@ func Test_TType_AttributeType(t *testing.T) { } func Test_TType_ElementType(t *testing.T) { + t.Parallel() ty, ok := valueshim.FromTType(tftypes.Set{ElementType: tftypes.Number}).ElementType() assert.True(t, ok) assert.Equal(t, valueshim.FromTType(tftypes.Number), ty) @@ -337,7 +339,7 @@ func Test_TType_ElementType(t *testing.T) { assert.True(t, ok) assert.Equal(t, valueshim.FromTType(tftypes.Number), ty) - ty, ok = valueshim.FromTType(tftypes.String).ElementType() + _, ok = valueshim.FromTType(tftypes.String).ElementType() assert.False(t, ok) } From 01f750d781903f8d6837d23b0f5e2ca6cdbcc364 Mon Sep 17 00:00:00 2001 From: Venelin Date: Thu, 29 May 2025 17:22:29 +0300 Subject: [PATCH 10/31] fix type propagation --- pkg/tfbridge/rawstate.go | 1 + pkg/tfbridge/rawstate_test.go | 1 + 2 files changed, 2 insertions(+) diff --git a/pkg/tfbridge/rawstate.go b/pkg/tfbridge/rawstate.go index 00bcd8674..3b90c3111 100644 --- a/pkg/tfbridge/rawstate.go +++ b/pkg/tfbridge/rawstate.go @@ -488,6 +488,7 @@ func RawStateComputeDelta( schemaMap: schemaMap, schemaInfos: schemaInfos, logger: log.TryGetLogger(ctx), + schemaType: schemaType, } pv := resource.NewObjectProperty(outMap) diff --git a/pkg/tfbridge/rawstate_test.go b/pkg/tfbridge/rawstate_test.go index 989a020ba..a790f2ed6 100644 --- a/pkg/tfbridge/rawstate_test.go +++ b/pkg/tfbridge/rawstate_test.go @@ -416,6 +416,7 @@ func Test_rawstate_delta_turnaround(t *testing.T) { "prop": {Name: "prop"}, } } + ih.schemaType = valueshim.FromHCtyType(cv.Type()) t.Logf("pv: %v", pv.String()) t.Logf("cv: %v", cv.GoString()) From b877eda04b35a8c14bd04f747e2773875a4e60b4 Mon Sep 17 00:00:00 2001 From: Venelin Date: Fri, 6 Jun 2025 16:57:03 +0300 Subject: [PATCH 11/31] ensure dynamic types always get replace deltas --- pkg/tfbridge/rawstate.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/pkg/tfbridge/rawstate.go b/pkg/tfbridge/rawstate.go index 3b90c3111..292876a4d 100644 --- a/pkg/tfbridge/rawstate.go +++ b/pkg/tfbridge/rawstate.go @@ -643,14 +643,16 @@ func (ih *rawStateDeltaHelper) computeDeltaAt( } } + tfs, ps, err := LookupSchemas(path, ih.schemaMap, ih.schemaInfos) + contract.AssertNoErrorf(err, "LookupSchemas failed") + // For assets and archives, save their AssetTranslation, so that at read time this AssetTranslation can be // invoked to TranslateAsset or TranslateArchive. if pv.IsAsset() || pv.IsArchive() { - schemaInfo := LookupSchemaInfoMapPath(path, ih.schemaInfos) - contract.Assertf(schemaInfo != nil && schemaInfo.Asset != nil, + contract.Assertf(ps != nil && ps.Asset != nil, "Assets must be matched with SchemaInfo with AssetTranslation [%q]", path.MustEncodeSchemaPath()) - at := schemaInfo.Asset + at := ps.Asset return RawStateDelta{Asset: &assetDelta{ Kind: at.Kind, Format: at.Format, @@ -658,6 +660,13 @@ func (ih *rawStateDeltaHelper) computeDeltaAt( }}, nil } + if tfs.Type() == shim.TypeDynamic { + relevantSchemaType, err := walk.LookupType(path, ih.schemaType) + contract.AssertNoErrorf(err, "LookupType failed") + + return RawStateDelta{Replace: &replaceDelta{Raw: newRawStateFromValue(relevantSchemaType, v)}}, nil + } + switch { case v.IsNull() && pv.IsNull(): return RawStateDelta{}, nil From e82d4f695a3eff9963b580b70411910e961bac21 Mon Sep 17 00:00:00 2001 From: Venelin Date: Fri, 6 Jun 2025 16:57:29 +0300 Subject: [PATCH 12/31] re-enable deltas tests --- pkg/pf/tests/diff_test/diff_test.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pkg/pf/tests/diff_test/diff_test.go b/pkg/pf/tests/diff_test/diff_test.go index 9fb5e48d5..9e6b9c696 100644 --- a/pkg/pf/tests/diff_test/diff_test.go +++ b/pkg/pf/tests/diff_test/diff_test.go @@ -3,7 +3,6 @@ package tfbridgetests import ( "context" "math/big" - "os" "testing" "github.com/hashicorp/terraform-plugin-framework/resource" @@ -11,7 +10,6 @@ import ( "github.com/hashicorp/terraform-plugin-framework/resource/schema/planmodifier" "github.com/hashicorp/terraform-plugin-framework/resource/schema/stringplanmodifier" "github.com/hexops/autogold/v2" - "github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil" "github.com/zclconf/go-cty/cty" pb "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/pf/internal/providerbuilder" @@ -184,10 +182,6 @@ func TestPFDetailedDiffStringAttribute(t *testing.T) { func TestPFDetailedDiffDynamicType(t *testing.T) { t.Parallel() - if d, ok := os.LookupEnv("PULUMI_RAW_STATE_DELTA_ENABLED"); ok && cmdutil.IsTruthy(d) { - // TODO[pulumi/pulumi-terraform-bridge#3078] - t.Skip("Does not work with PULUMI_RAW_STATE_DELTA_ENABLED=true") - } attributeSchema := rschema.Schema{ Attributes: map[string]rschema.Attribute{ @@ -233,8 +227,6 @@ func TestPFDetailedDiffDynamicType(t *testing.T) { func TestPFDetailedDiffDynamicTypeWithMigration(t *testing.T) { t.Parallel() - // TODO[pulumi/pulumi-terraform-bridge#3078] - t.Skip("DynamicPseudoType is not supported") attributeSchema := rschema.Schema{ Attributes: map[string]rschema.Attribute{ From e18d7754f14313925320981e938118388bdc6237 Mon Sep 17 00:00:00 2001 From: Venelin Date: Fri, 6 Jun 2025 17:19:53 +0300 Subject: [PATCH 13/31] nested dynamic test --- pkg/pf/tests/diff_test/diff_test.go | 50 +++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/pkg/pf/tests/diff_test/diff_test.go b/pkg/pf/tests/diff_test/diff_test.go index 9e6b9c696..fc981e2d1 100644 --- a/pkg/pf/tests/diff_test/diff_test.go +++ b/pkg/pf/tests/diff_test/diff_test.go @@ -5,10 +5,12 @@ import ( "math/big" "testing" + "github.com/hashicorp/terraform-plugin-framework/attr" "github.com/hashicorp/terraform-plugin-framework/resource" rschema "github.com/hashicorp/terraform-plugin-framework/resource/schema" "github.com/hashicorp/terraform-plugin-framework/resource/schema/planmodifier" "github.com/hashicorp/terraform-plugin-framework/resource/schema/stringplanmodifier" + "github.com/hashicorp/terraform-plugin-framework/types" "github.com/hexops/autogold/v2" "github.com/zclconf/go-cty/cty" @@ -225,6 +227,54 @@ func TestPFDetailedDiffDynamicType(t *testing.T) { }) } +func TestPFDetailedDiffNestedDynamicType(t *testing.T) { + t.Parallel() + + attributeSchema := rschema.Schema{ + Attributes: map[string]rschema.Attribute{ + "key": rschema.ObjectAttribute{ + Optional: true, + AttributeTypes: map[string]attr.Type{ + "nested": types.DynamicType, + }, + }, + }, + } + res := pb.NewResource(pb.NewResourceArgs{ + ResourceSchema: attributeSchema, + }) + + t.Run("no change", func(t *testing.T) { + crosstests.Diff(t, res, + map[string]cty.Value{"key": cty.ObjectVal(map[string]cty.Value{"nested": cty.StringVal("value")})}, + map[string]cty.Value{"key": cty.ObjectVal(map[string]cty.Value{"nested": cty.StringVal("value")})}, + ) + }) + + t.Run("change", func(t *testing.T) { + crosstests.Diff(t, res, + map[string]cty.Value{"key": cty.ObjectVal(map[string]cty.Value{"nested": cty.StringVal("value")})}, + map[string]cty.Value{"key": cty.ObjectVal(map[string]cty.Value{"nested": cty.StringVal("value1")})}, + ) + }) + + t.Run("int no change", func(t *testing.T) { + crosstests.Diff(t, res, + map[string]cty.Value{"key": cty.ObjectVal(map[string]cty.Value{"nested": cty.NumberVal(big.NewFloat(1))})}, + map[string]cty.Value{"key": cty.ObjectVal(map[string]cty.Value{"nested": cty.NumberVal(big.NewFloat(1))})}, + ) + }) + + t.Run("type change", func(t *testing.T) { + // TODO[pulumi/pulumi-terraform-bridge#3078] + t.Skip(`Error converting tftypes.Number<"1"> (value2) at "AttributeName(\"key\")": can't unmarshal tftypes.Number into *string, expected string`) + crosstests.Diff(t, res, + map[string]cty.Value{"key": cty.ObjectVal(map[string]cty.Value{"nested": cty.StringVal("value")})}, + map[string]cty.Value{"key": cty.ObjectVal(map[string]cty.Value{"nested": cty.NumberVal(big.NewFloat(1))})}, + ) + }) +} + func TestPFDetailedDiffDynamicTypeWithMigration(t *testing.T) { t.Parallel() From 70974d6744e6a7f6072336172a44856f25a29c4c Mon Sep 17 00:00:00 2001 From: Venelin Date: Fri, 6 Jun 2025 17:34:53 +0300 Subject: [PATCH 14/31] fix tests --- pkg/tfbridge/rawstate.go | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/pkg/tfbridge/rawstate.go b/pkg/tfbridge/rawstate.go index 292876a4d..bc8e0f338 100644 --- a/pkg/tfbridge/rawstate.go +++ b/pkg/tfbridge/rawstate.go @@ -643,16 +643,24 @@ func (ih *rawStateDeltaHelper) computeDeltaAt( } } - tfs, ps, err := LookupSchemas(path, ih.schemaMap, ih.schemaInfos) - contract.AssertNoErrorf(err, "LookupSchemas failed") + tfs, _, err := LookupSchemas(path, ih.schemaMap, ih.schemaInfos) + if err == nil { + if tfs.Type() == shim.TypeDynamic { + relevantSchemaType, err := walk.LookupType(path, ih.schemaType) + contract.AssertNoErrorf(err, "LookupType failed") + + return RawStateDelta{Replace: &replaceDelta{Raw: newRawStateFromValue(relevantSchemaType, v)}}, nil + } + } // For assets and archives, save their AssetTranslation, so that at read time this AssetTranslation can be // invoked to TranslateAsset or TranslateArchive. if pv.IsAsset() || pv.IsArchive() { - contract.Assertf(ps != nil && ps.Asset != nil, + schemaInfo := LookupSchemaInfoMapPath(path, ih.schemaInfos) + contract.Assertf(schemaInfo != nil && schemaInfo.Asset != nil, "Assets must be matched with SchemaInfo with AssetTranslation [%q]", path.MustEncodeSchemaPath()) - at := ps.Asset + at := schemaInfo.Asset return RawStateDelta{Asset: &assetDelta{ Kind: at.Kind, Format: at.Format, @@ -660,13 +668,6 @@ func (ih *rawStateDeltaHelper) computeDeltaAt( }}, nil } - if tfs.Type() == shim.TypeDynamic { - relevantSchemaType, err := walk.LookupType(path, ih.schemaType) - contract.AssertNoErrorf(err, "LookupType failed") - - return RawStateDelta{Replace: &replaceDelta{Raw: newRawStateFromValue(relevantSchemaType, v)}}, nil - } - switch { case v.IsNull() && pv.IsNull(): return RawStateDelta{}, nil From 653b53edf7b5d8229f8cad2c139de62440f015d2 Mon Sep 17 00:00:00 2001 From: Venelin Date: Fri, 6 Jun 2025 17:42:59 +0300 Subject: [PATCH 15/31] add isDynamicType to value shim --- pkg/valueshim/cty.go | 4 ++++ pkg/valueshim/hcty.go | 4 ++++ pkg/valueshim/shim.go | 1 + pkg/valueshim/tfvalue.go | 4 ++++ 4 files changed, 13 insertions(+) diff --git a/pkg/valueshim/cty.go b/pkg/valueshim/cty.go index c546b098d..e6ac47b01 100644 --- a/pkg/valueshim/cty.go +++ b/pkg/valueshim/cty.go @@ -155,6 +155,10 @@ func (t ctyTypeShim) IsObjectType() bool { return t.ty().IsObjectType() } +func (t ctyTypeShim) IsDynamicType() bool { + return t.ty().Equals(cty.DynamicPseudoType) +} + func (t ctyTypeShim) AttributeType(name string) (Type, bool) { tt := t.ty() if !tt.IsObjectType() { diff --git a/pkg/valueshim/hcty.go b/pkg/valueshim/hcty.go index e4e258fd9..9623a6f50 100644 --- a/pkg/valueshim/hcty.go +++ b/pkg/valueshim/hcty.go @@ -154,6 +154,10 @@ func (t hctyTypeShim) IsObjectType() bool { return t.ty().IsObjectType() } +func (t hctyTypeShim) IsDynamicType() bool { + return t.ty().Equals(cty.DynamicPseudoType) +} + func (t hctyTypeShim) AttributeType(name string) (Type, bool) { tt := t.ty() if !tt.IsObjectType() { diff --git a/pkg/valueshim/shim.go b/pkg/valueshim/shim.go index 3e235b434..9b4054a84 100644 --- a/pkg/valueshim/shim.go +++ b/pkg/valueshim/shim.go @@ -57,6 +57,7 @@ type Type interface { IsMapType() bool IsSetType() bool IsObjectType() bool + IsDynamicType() bool AttributeType(name string) (Type, bool) ElementType() (Type, bool) GoString() string diff --git a/pkg/valueshim/tfvalue.go b/pkg/valueshim/tfvalue.go index 3864696fa..71f11c12c 100644 --- a/pkg/valueshim/tfvalue.go +++ b/pkg/valueshim/tfvalue.go @@ -192,6 +192,10 @@ func (t tTypeShim) IsObjectType() bool { return t.ty().Is(tftypes.Object{}) } +func (t tTypeShim) IsDynamicType() bool { + return t.ty().Is(tftypes.DynamicPseudoType) +} + func (t tTypeShim) GoString() string { return t.ty().String() } From 2cefb8d4dc4577b2d4660ce53b4d6fce3d5ac9c6 Mon Sep 17 00:00:00 2001 From: Venelin Date: Fri, 6 Jun 2025 17:43:29 +0300 Subject: [PATCH 16/31] use the valueshim type instead of the schema map --- pkg/tfbridge/rawstate.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/tfbridge/rawstate.go b/pkg/tfbridge/rawstate.go index bc8e0f338..d634910c1 100644 --- a/pkg/tfbridge/rawstate.go +++ b/pkg/tfbridge/rawstate.go @@ -643,13 +643,10 @@ func (ih *rawStateDeltaHelper) computeDeltaAt( } } - tfs, _, err := LookupSchemas(path, ih.schemaMap, ih.schemaInfos) + schType, err := walk.LookupType(path, ih.schemaType) if err == nil { - if tfs.Type() == shim.TypeDynamic { - relevantSchemaType, err := walk.LookupType(path, ih.schemaType) - contract.AssertNoErrorf(err, "LookupType failed") - - return RawStateDelta{Replace: &replaceDelta{Raw: newRawStateFromValue(relevantSchemaType, v)}}, nil + if schType.IsDynamicType() { + return RawStateDelta{Replace: &replaceDelta{Raw: newRawStateFromValue(schType, v)}}, nil } } From daab9181b09edb6dbb4720087177220b68dd4b5d Mon Sep 17 00:00:00 2001 From: Venelin Date: Fri, 6 Jun 2025 17:51:41 +0300 Subject: [PATCH 17/31] try without the timeout removal --- pkg/tfbridge/rawstate.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/tfbridge/rawstate.go b/pkg/tfbridge/rawstate.go index d634910c1..15c1bc767 100644 --- a/pkg/tfbridge/rawstate.go +++ b/pkg/tfbridge/rawstate.go @@ -492,9 +492,8 @@ func RawStateComputeDelta( } pv := resource.NewObjectProperty(outMap) - vWithoutTimeouts := v.Remove("timeouts") - delta := ih.delta(pv, vWithoutTimeouts) - err := delta.turnaroundCheck(ctx, newRawStateFromValue(schemaType, vWithoutTimeouts), pv) + delta := ih.delta(pv, v) + err := delta.turnaroundCheck(ctx, newRawStateFromValue(schemaType, v), pv) if err != nil { return RawStateDelta{}, err } From 1ffc615f51007066954963c48f2e1b2f9671c154 Mon Sep 17 00:00:00 2001 From: Venelin Date: Fri, 6 Jun 2025 18:17:03 +0300 Subject: [PATCH 18/31] fix test --- pkg/tfbridge/rawstate.go | 1 + pkg/tfbridge/rawstate_test.go | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/tfbridge/rawstate.go b/pkg/tfbridge/rawstate.go index 15c1bc767..f6ed5542e 100644 --- a/pkg/tfbridge/rawstate.go +++ b/pkg/tfbridge/rawstate.go @@ -644,6 +644,7 @@ func (ih *rawStateDeltaHelper) computeDeltaAt( schType, err := walk.LookupType(path, ih.schemaType) if err == nil { + contract.Assertf(schType != nil, "schType is nil") if schType.IsDynamicType() { return RawStateDelta{Replace: &replaceDelta{Raw: newRawStateFromValue(schType, v)}}, nil } diff --git a/pkg/tfbridge/rawstate_test.go b/pkg/tfbridge/rawstate_test.go index a790f2ed6..33d4cce9b 100644 --- a/pkg/tfbridge/rawstate_test.go +++ b/pkg/tfbridge/rawstate_test.go @@ -1155,9 +1155,11 @@ func Test_rawstate_against_MakeTerraformOutputs(t *testing.T) { outMap := MakeTerraformOutputs(ctx, p, stateObj, tfs, tc.ps, assets, supportsSecrets) + schemaType := p.ResourcesMap().Get(tok).SchemaType() ih := &rawStateDeltaHelper{ schemaMap: tfs, schemaInfos: tc.ps, + schemaType: schemaType, } pv := resource.NewObjectProperty(outMap) @@ -1168,8 +1170,7 @@ func Test_rawstate_against_MakeTerraformOutputs(t *testing.T) { deltaJSON, err := json.MarshalIndent(deltaPV.Mappable(), "", " ") require.NoError(t, err) tc.infl.Equal(t, string(deltaJSON)) - - err = delta.turnaroundCheck(ctx, newRawStateFromValue(stateValue.Type(), stateValue), pv) + err = delta.turnaroundCheck(ctx, newRawStateFromValue(schemaType, stateValue), pv) assert.NoError(t, err) }) } From b55904891af179fd9e7876bd47a6dd57b8bf6e7a Mon Sep 17 00:00:00 2001 From: Venelin Date: Mon, 9 Jun 2025 12:49:05 +0300 Subject: [PATCH 19/31] bring back tftype serialization, add tests --- pkg/valueshim/tftype_json.go | 200 +++++++++++ pkg/valueshim/tftype_json_test.go | 576 ++++++++++++++++++++++++++++++ pkg/valueshim/tfvalue.go | 11 +- 3 files changed, 779 insertions(+), 8 deletions(-) create mode 100644 pkg/valueshim/tftype_json.go create mode 100644 pkg/valueshim/tftype_json_test.go diff --git a/pkg/valueshim/tftype_json.go b/pkg/valueshim/tftype_json.go new file mode 100644 index 000000000..77e2d7695 --- /dev/null +++ b/pkg/valueshim/tftype_json.go @@ -0,0 +1,200 @@ +// Copyright 2016-2025, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package valueshim + +import ( + "encoding/json" + "math/big" + + "github.com/hashicorp/terraform-plugin-go/tftypes" +) + +// Inverse of tftypes.ValueFromJson. +func tftypeValueToJSON(typ tftypes.Type, v tftypes.Value) ([]byte, error) { + raw, err := jsonMarshal(v, typ, tftypes.NewAttributePath()) + if err != nil { + return nil, err + } + return json.Marshal(raw) +} + +func jsonMarshal(v tftypes.Value, typ tftypes.Type, p *tftypes.AttributePath) (interface{}, error) { + if v.IsNull() { + return nil, nil + } + if !v.IsKnown() { + return nil, p.NewErrorf("unknown values cannot be serialized to JSON") + } + switch { + case typ.Is(tftypes.String): + return jsonMarshalString(v, typ, p) + case typ.Is(tftypes.Number): + return jsonMarshalNumber(v, typ, p) + case typ.Is(tftypes.Bool): + return jsonMarshalBool(v, typ, p) + case typ.Is(tftypes.DynamicPseudoType): + return jsonMarshalDynamicPseudoType(v, typ, p) + case typ.Is(tftypes.List{}): + return jsonMarshalList(v, typ.(tftypes.List).ElementType, p) + case typ.Is(tftypes.Set{}): + return jsonMarshalSet(v, typ.(tftypes.Set).ElementType, p) + case typ.Is(tftypes.Map{}): + return jsonMarshalMap(v, typ.(tftypes.Map).ElementType, p) + case typ.Is(tftypes.Tuple{}): + return jsonMarshalTuple(v, typ.(tftypes.Tuple).ElementTypes, p) + case typ.Is(tftypes.Object{}): + return jsonMarshalObject(v, typ.(tftypes.Object).AttributeTypes, p) + } + + return nil, p.NewErrorf("unknown type %s", typ) +} + +func jsonMarshalString(v tftypes.Value, typ tftypes.Type, p *tftypes.AttributePath) (interface{}, error) { + var stringValue string + err := v.As(&stringValue) + if err != nil { + return nil, p.NewError(err) + } + return stringValue, nil +} + +func jsonMarshalNumber(v tftypes.Value, typ tftypes.Type, p *tftypes.AttributePath) (interface{}, error) { + var n big.Float + err := v.As(&n) + if err != nil { + return nil, p.NewError(err) + } + f64, _ := n.Float64() + return f64, nil +} + +func jsonMarshalBool(v tftypes.Value, typ tftypes.Type, p *tftypes.AttributePath) (interface{}, error) { + var b bool + err := v.As(&b) + if err != nil { + return nil, p.NewError(err) + } + return b, nil +} + +func jsonMarshalDynamicPseudoType(v tftypes.Value, _ tftypes.Type, p *tftypes.AttributePath) (interface{}, error) { + valType := v.Type() + typeJSON, err := valType.MarshalJSON() + if err != nil { + return nil, p.NewError(err) + } + valJSON, err := jsonMarshal(v, valType, p) + if err != nil { + return nil, p.NewError(err) + } + return map[string]interface{}{ + "type": string(typeJSON), + "value": valJSON, + }, nil +} + +func jsonMarshalList(v tftypes.Value, elementType tftypes.Type, p *tftypes.AttributePath) (interface{}, error) { + var vs []tftypes.Value + err := v.As(&vs) + if err != nil { + return nil, p.NewError(err) + } + var res []interface{} + for i, v := range vs { + ep := p.WithElementKeyInt(i) + e, err := jsonMarshal(v, elementType, ep) + if err != nil { + return nil, ep.NewError(err) + } + res = append(res, e) + } + return res, nil +} + +func jsonMarshalSet(v tftypes.Value, elementType tftypes.Type, p *tftypes.AttributePath) (interface{}, error) { + var vs []tftypes.Value + err := v.As(&vs) + if err != nil { + return nil, p.NewError(err) + } + var res []interface{} + for _, v := range vs { + ep := p.WithElementKeyValue(v) + e, err := jsonMarshal(v, elementType, ep) + if err != nil { + return nil, ep.NewError(err) + } + res = append(res, e) + } + return res, nil +} + +func jsonMarshalMap(v tftypes.Value, elementType tftypes.Type, p *tftypes.AttributePath) (interface{}, error) { + var vs map[string]tftypes.Value + err := v.As(&vs) + if err != nil { + return nil, p.NewError(err) + } + res := map[string]interface{}{} + for k, v := range vs { + ep := p.WithElementKeyValue(v) + e, err := jsonMarshal(v, elementType, ep) + if err != nil { + return nil, ep.NewError(err) + } + res[k] = e + } + return res, nil +} + +func jsonMarshalTuple(v tftypes.Value, elementTypes []tftypes.Type, p *tftypes.AttributePath) (interface{}, error) { + var vs []tftypes.Value + err := v.As(&vs) + if err != nil { + return nil, p.NewError(err) + } + var res []interface{} + for i, v := range vs { + ep := p.WithElementKeyInt(i) + e, err := jsonMarshal(v, elementTypes[i], ep) + if err != nil { + return nil, ep.NewError(err) + } + res = append(res, e) + } + return res, nil +} + +func jsonMarshalObject( + v tftypes.Value, + elementTypes map[string]tftypes.Type, + p *tftypes.AttributePath, +) (interface{}, error) { + var vs map[string]tftypes.Value + err := v.As(&vs) + if err != nil { + return nil, p.NewError(err) + } + res := map[string]interface{}{} + for k, v := range vs { + ep := p.WithAttributeName(k) + e, err := jsonMarshal(v, elementTypes[k], ep) + if err != nil { + return nil, ep.NewError(err) + } + res[k] = e + } + return res, nil +} diff --git a/pkg/valueshim/tftype_json_test.go b/pkg/valueshim/tftype_json_test.go new file mode 100644 index 000000000..b41ec127f --- /dev/null +++ b/pkg/valueshim/tftype_json_test.go @@ -0,0 +1,576 @@ +// Copyright 2016-2025, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package valueshim + +import ( + "encoding/json" + "testing" + + "github.com/hashicorp/terraform-plugin-go/tftypes" + "github.com/hexops/autogold/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValueToJSON(t *testing.T) { + t.Parallel() + type testCase struct { + name string + typ tftypes.Type + value tftypes.Value + expectJSON autogold.Value + expectError string + } + + testCases := []testCase{ + // String types + { + name: "string null", + typ: tftypes.String, + value: tftypes.NewValue(tftypes.String, nil), + expectJSON: autogold.Expect("null"), + }, + { + name: "string value", + typ: tftypes.String, + value: tftypes.NewValue(tftypes.String, "hello world"), + expectJSON: autogold.Expect(`"hello world"`), + }, + { + name: "string empty", + typ: tftypes.String, + value: tftypes.NewValue(tftypes.String, ""), + expectJSON: autogold.Expect(`""`), + }, + { + name: "string with special characters", + typ: tftypes.String, + value: tftypes.NewValue(tftypes.String, "hello\nworld\t\"quoted\""), + expectJSON: autogold.Expect(`"hello\nworld\t\"quoted\""`), + }, + + // Number types + { + name: "number null", + typ: tftypes.Number, + value: tftypes.NewValue(tftypes.Number, nil), + expectJSON: autogold.Expect("null"), + }, + { + name: "number integer", + typ: tftypes.Number, + value: tftypes.NewValue(tftypes.Number, 42), + expectJSON: autogold.Expect("42"), + }, + { + name: "number float", + typ: tftypes.Number, + value: tftypes.NewValue(tftypes.Number, 3.14159), + expectJSON: autogold.Expect("3.14159"), + }, + { + name: "number zero", + typ: tftypes.Number, + value: tftypes.NewValue(tftypes.Number, 0), + expectJSON: autogold.Expect("0"), + }, + { + name: "number negative", + typ: tftypes.Number, + value: tftypes.NewValue(tftypes.Number, -123.45), + expectJSON: autogold.Expect("-123.45"), + }, + + // Bool types + { + name: "bool null", + typ: tftypes.Bool, + value: tftypes.NewValue(tftypes.Bool, nil), + expectJSON: autogold.Expect("null"), + }, + { + name: "bool true", + typ: tftypes.Bool, + value: tftypes.NewValue(tftypes.Bool, true), + expectJSON: autogold.Expect("true"), + }, + { + name: "bool false", + typ: tftypes.Bool, + value: tftypes.NewValue(tftypes.Bool, false), + expectJSON: autogold.Expect("false"), + }, + + // List types + { + name: "list null", + typ: tftypes.List{ElementType: tftypes.String}, + value: tftypes.NewValue(tftypes.List{ElementType: tftypes.String}, nil), + expectJSON: autogold.Expect("null"), + }, + { + name: "list empty", + typ: tftypes.List{ElementType: tftypes.String}, + value: tftypes.NewValue(tftypes.List{ElementType: tftypes.String}, []tftypes.Value{}), + // Empty lists serialize to null, not [] + expectJSON: autogold.Expect("null"), + }, + { + name: "list of strings", + typ: tftypes.List{ElementType: tftypes.String}, + value: tftypes.NewValue(tftypes.List{ElementType: tftypes.String}, []tftypes.Value{ + tftypes.NewValue(tftypes.String, "first"), + tftypes.NewValue(tftypes.String, "second"), + tftypes.NewValue(tftypes.String, "third"), + }), + expectJSON: autogold.Expect(`["first","second","third"]`), + }, + { + name: "list of numbers", + typ: tftypes.List{ElementType: tftypes.Number}, + value: tftypes.NewValue(tftypes.List{ElementType: tftypes.Number}, []tftypes.Value{ + tftypes.NewValue(tftypes.Number, 1), + tftypes.NewValue(tftypes.Number, 2), + tftypes.NewValue(tftypes.Number, 3), + }), + expectJSON: autogold.Expect(`[1,2,3]`), + }, + { + name: "list with null element", + typ: tftypes.List{ElementType: tftypes.String}, + value: tftypes.NewValue(tftypes.List{ElementType: tftypes.String}, []tftypes.Value{ + tftypes.NewValue(tftypes.String, "first"), + tftypes.NewValue(tftypes.String, nil), + tftypes.NewValue(tftypes.String, "third"), + }), + expectJSON: autogold.Expect(`["first",null,"third"]`), + }, + + // Set types + { + name: "set null", + typ: tftypes.Set{ElementType: tftypes.String}, + value: tftypes.NewValue(tftypes.Set{ElementType: tftypes.String}, nil), + expectJSON: autogold.Expect("null"), + }, + { + name: "set empty", + typ: tftypes.Set{ElementType: tftypes.String}, + value: tftypes.NewValue(tftypes.Set{ElementType: tftypes.String}, []tftypes.Value{}), + // Empty sets serialize to null, not [] + expectJSON: autogold.Expect("null"), + }, + { + name: "set of strings", + typ: tftypes.Set{ElementType: tftypes.String}, + value: tftypes.NewValue(tftypes.Set{ElementType: tftypes.String}, []tftypes.Value{ + tftypes.NewValue(tftypes.String, "alpha"), + tftypes.NewValue(tftypes.String, "beta"), + }), + expectJSON: autogold.Expect(`["alpha","beta"]`), + }, + + // Map types + { + name: "map null", + typ: tftypes.Map{ElementType: tftypes.String}, + value: tftypes.NewValue(tftypes.Map{ElementType: tftypes.String}, nil), + expectJSON: autogold.Expect("null"), + }, + { + name: "map empty", + typ: tftypes.Map{ElementType: tftypes.String}, + value: tftypes.NewValue(tftypes.Map{ElementType: tftypes.String}, map[string]tftypes.Value{}), + expectJSON: autogold.Expect("{}"), + }, + { + name: "map of strings", + typ: tftypes.Map{ElementType: tftypes.String}, + value: tftypes.NewValue(tftypes.Map{ElementType: tftypes.String}, map[string]tftypes.Value{ + "key1": tftypes.NewValue(tftypes.String, "value1"), + "key2": tftypes.NewValue(tftypes.String, "value2"), + }), + expectJSON: autogold.Expect(`{"key1":"value1","key2":"value2"}`), + }, + { + name: "map with null value", + typ: tftypes.Map{ElementType: tftypes.String}, + value: tftypes.NewValue(tftypes.Map{ElementType: tftypes.String}, map[string]tftypes.Value{ + "key1": tftypes.NewValue(tftypes.String, "value1"), + "key2": tftypes.NewValue(tftypes.String, nil), + }), + expectJSON: autogold.Expect(`{"key1":"value1","key2":null}`), + }, + + // Object types + { + name: "object null", + typ: tftypes.Object{AttributeTypes: map[string]tftypes.Type{ + "name": tftypes.String, + "age": tftypes.Number, + }}, + value: tftypes.NewValue(tftypes.Object{AttributeTypes: map[string]tftypes.Type{ + "name": tftypes.String, + "age": tftypes.Number, + }}, nil), + expectJSON: autogold.Expect("null"), + }, + { + name: "object simple", + typ: tftypes.Object{AttributeTypes: map[string]tftypes.Type{ + "name": tftypes.String, + "age": tftypes.Number, + }}, + value: tftypes.NewValue(tftypes.Object{AttributeTypes: map[string]tftypes.Type{ + "name": tftypes.String, + "age": tftypes.Number, + }}, map[string]tftypes.Value{ + "name": tftypes.NewValue(tftypes.String, "John"), + "age": tftypes.NewValue(tftypes.Number, 30), + }), + expectJSON: autogold.Expect(`{"age":30,"name":"John"}`), + }, + { + name: "object with null attribute", + typ: tftypes.Object{AttributeTypes: map[string]tftypes.Type{ + "name": tftypes.String, + "email": tftypes.String, + }}, + value: tftypes.NewValue(tftypes.Object{AttributeTypes: map[string]tftypes.Type{ + "name": tftypes.String, + "email": tftypes.String, + }}, map[string]tftypes.Value{ + "name": tftypes.NewValue(tftypes.String, "John"), + "email": tftypes.NewValue(tftypes.String, nil), + }), + expectJSON: autogold.Expect(`{"email":null,"name":"John"}`), + }, + + // Tuple types + { + name: "tuple null", + typ: tftypes.Tuple{ElementTypes: []tftypes.Type{tftypes.String, tftypes.Number}}, + value: tftypes.NewValue(tftypes.Tuple{ElementTypes: []tftypes.Type{ + tftypes.String, + tftypes.Number, + }}, nil), + expectJSON: autogold.Expect("null"), + }, + { + name: "tuple simple", + typ: tftypes.Tuple{ElementTypes: []tftypes.Type{tftypes.String, tftypes.Number, tftypes.Bool}}, + value: tftypes.NewValue(tftypes.Tuple{ElementTypes: []tftypes.Type{ + tftypes.String, + tftypes.Number, + tftypes.Bool, + }}, []tftypes.Value{ + tftypes.NewValue(tftypes.String, "hello"), + tftypes.NewValue(tftypes.Number, 42), + tftypes.NewValue(tftypes.Bool, true), + }), + expectJSON: autogold.Expect(`["hello",42,true]`), + }, + { + name: "tuple with null element", + typ: tftypes.Tuple{ElementTypes: []tftypes.Type{tftypes.String, tftypes.Number}}, + value: tftypes.NewValue(tftypes.Tuple{ElementTypes: []tftypes.Type{ + tftypes.String, + tftypes.Number, + }}, []tftypes.Value{ + tftypes.NewValue(tftypes.String, "hello"), + tftypes.NewValue(tftypes.Number, nil), + }), + expectJSON: autogold.Expect(`["hello",null]`), + }, + + // Nested structures + { + name: "nested list of objects", + typ: tftypes.List{ElementType: tftypes.Object{AttributeTypes: map[string]tftypes.Type{"name": tftypes.String}}}, + value: tftypes.NewValue( + tftypes.List{ElementType: tftypes.Object{AttributeTypes: map[string]tftypes.Type{"name": tftypes.String}}}, + []tftypes.Value{ + tftypes.NewValue(tftypes.Object{AttributeTypes: map[string]tftypes.Type{"name": tftypes.String}}, + map[string]tftypes.Value{"name": tftypes.NewValue(tftypes.String, "Alice")}), + tftypes.NewValue(tftypes.Object{AttributeTypes: map[string]tftypes.Type{"name": tftypes.String}}, + map[string]tftypes.Value{"name": tftypes.NewValue(tftypes.String, "Bob")}), + }, + ), + expectJSON: autogold.Expect(`[{"name":"Alice"},{"name":"Bob"}]`), + }, + { + name: "object with nested list", + typ: tftypes.Object{AttributeTypes: map[string]tftypes.Type{ + "name": tftypes.String, + "items": tftypes.List{ElementType: tftypes.String}, + }}, + value: tftypes.NewValue(tftypes.Object{AttributeTypes: map[string]tftypes.Type{ + "name": tftypes.String, + "items": tftypes.List{ElementType: tftypes.String}, + }}, map[string]tftypes.Value{ + "name": tftypes.NewValue(tftypes.String, "shopping"), + "items": tftypes.NewValue(tftypes.List{ElementType: tftypes.String}, []tftypes.Value{ + tftypes.NewValue(tftypes.String, "milk"), + tftypes.NewValue(tftypes.String, "bread"), + }), + }), + expectJSON: autogold.Expect(`{"items":["milk","bread"],"name":"shopping"}`), + }, + + // Error cases + { + name: "unknown value", + typ: tftypes.String, + value: tftypes.NewValue(tftypes.String, tftypes.UnknownValue), + expectError: "unknown values cannot be serialized to JSON", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + result, err := tftypeValueToJSON(tc.typ, tc.value) + + if tc.expectError != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.expectError) + } else { + require.NoError(t, err) + tc.expectJSON.Equal(t, string(result)) + } + }) + } +} + +func TestValueToJSON_DynamicPseudoType(t *testing.T) { + t.Parallel() + + // Test dynamic pseudo type which requires special handling + // DynamicPseudoType is used to wrap values of any concrete type + // We create a string value but tell the marshal function to treat it as dynamic + stringValue := tftypes.NewValue(tftypes.String, "dynamic content") + + // When marshaling with DynamicPseudoType as the schema type, + // it should produce {"type": "...", "value": "..."} + result, err := tftypeValueToJSON(tftypes.DynamicPseudoType, stringValue) + require.NoError(t, err) + + // The result should be a JSON object with type and value fields + expected := `{"type":"\"string\"","value":"dynamic content"}` + assert.JSONEq(t, expected, string(result)) + + // Also test with other types + numberValue := tftypes.NewValue(tftypes.Number, 42) + result, err = tftypeValueToJSON(tftypes.DynamicPseudoType, numberValue) + require.NoError(t, err) + + expected = `{"type":"\"number\"","value":42}` + assert.JSONEq(t, expected, string(result)) + + boolValue := tftypes.NewValue(tftypes.Bool, true) + result, err = tftypeValueToJSON(tftypes.DynamicPseudoType, boolValue) + require.NoError(t, err) + + expected = `{"type":"\"bool\"","value":true}` + assert.JSONEq(t, expected, string(result)) +} + +func TestValueToJSON_ComplexNested(t *testing.T) { + t.Parallel() + + // Test a complex nested structure + complexType := tftypes.Object{AttributeTypes: map[string]tftypes.Type{ + "metadata": tftypes.Object{AttributeTypes: map[string]tftypes.Type{ + "name": tftypes.String, + "version": tftypes.Number, + }}, + "items": tftypes.List{ElementType: tftypes.Object{AttributeTypes: map[string]tftypes.Type{ + "id": tftypes.String, + "active": tftypes.Bool, + "tags": tftypes.Set{ElementType: tftypes.String}, + }}}, + "config": tftypes.Map{ElementType: tftypes.String}, + }} + + complexValue := tftypes.NewValue(complexType, map[string]tftypes.Value{ + "metadata": tftypes.NewValue(tftypes.Object{AttributeTypes: map[string]tftypes.Type{ + "name": tftypes.String, + "version": tftypes.Number, + }}, map[string]tftypes.Value{ + "name": tftypes.NewValue(tftypes.String, "test-app"), + "version": tftypes.NewValue(tftypes.Number, 1.5), + }), + "items": tftypes.NewValue(tftypes.List{ElementType: tftypes.Object{AttributeTypes: map[string]tftypes.Type{ + "id": tftypes.String, + "active": tftypes.Bool, + "tags": tftypes.Set{ElementType: tftypes.String}, + }}}, []tftypes.Value{ + tftypes.NewValue(tftypes.Object{AttributeTypes: map[string]tftypes.Type{ + "id": tftypes.String, + "active": tftypes.Bool, + "tags": tftypes.Set{ElementType: tftypes.String}, + }}, map[string]tftypes.Value{ + "id": tftypes.NewValue(tftypes.String, "item1"), + "active": tftypes.NewValue(tftypes.Bool, true), + "tags": tftypes.NewValue(tftypes.Set{ElementType: tftypes.String}, []tftypes.Value{ + tftypes.NewValue(tftypes.String, "prod"), + tftypes.NewValue(tftypes.String, "web"), + }), + }), + }), + "config": tftypes.NewValue(tftypes.Map{ElementType: tftypes.String}, map[string]tftypes.Value{ + "env": tftypes.NewValue(tftypes.String, "production"), + "debug": tftypes.NewValue(tftypes.String, "false"), + "optional": tftypes.NewValue(tftypes.String, nil), + }), + }) + + result, err := tftypeValueToJSON(complexType, complexValue) + require.NoError(t, err) + + // Verify it's valid JSON + var parsed interface{} + err = json.Unmarshal(result, &parsed) + require.NoError(t, err) + + // Convert back to pretty JSON for readability in test output + prettyResult, err := json.MarshalIndent(parsed, "", " ") + require.NoError(t, err) + + autogold.Expect(`{ + "config": { + "debug": "false", + "env": "production", + "optional": null + }, + "items": [ + { + "active": true, + "id": "item1", + "tags": [ + "prod", + "web" + ] + } + ], + "metadata": { + "name": "test-app", + "version": 1.5 + } +}`).Equal(t, string(prettyResult)) +} + +func TestValueToJSON_EdgeCases(t *testing.T) { + t.Parallel() + + type testCase struct { + name string + typ tftypes.Type + value tftypes.Value + expectError string + } + + testCases := []testCase{ + { + name: "unknown string value", + typ: tftypes.String, + value: tftypes.NewValue(tftypes.String, tftypes.UnknownValue), + expectError: "unknown values cannot be serialized to JSON", + }, + { + name: "unknown number value", + typ: tftypes.Number, + value: tftypes.NewValue(tftypes.Number, tftypes.UnknownValue), + expectError: "unknown values cannot be serialized to JSON", + }, + { + name: "unknown bool value", + typ: tftypes.Bool, + value: tftypes.NewValue(tftypes.Bool, tftypes.UnknownValue), + expectError: "unknown values cannot be serialized to JSON", + }, + { + name: "list with unknown element", + typ: tftypes.List{ElementType: tftypes.String}, + value: tftypes.NewValue(tftypes.List{ElementType: tftypes.String}, []tftypes.Value{ + tftypes.NewValue(tftypes.String, "known"), + tftypes.NewValue(tftypes.String, tftypes.UnknownValue), + }), + expectError: "unknown values cannot be serialized to JSON", + }, + { + name: "object with unknown attribute", + typ: tftypes.Object{AttributeTypes: map[string]tftypes.Type{ + "known": tftypes.String, + "unknown": tftypes.String, + }}, + value: tftypes.NewValue(tftypes.Object{AttributeTypes: map[string]tftypes.Type{ + "known": tftypes.String, + "unknown": tftypes.String, + }}, map[string]tftypes.Value{ + "known": tftypes.NewValue(tftypes.String, "value"), + "unknown": tftypes.NewValue(tftypes.String, tftypes.UnknownValue), + }), + expectError: "unknown values cannot be serialized to JSON", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + result, err := tftypeValueToJSON(tc.typ, tc.value) + + require.Error(t, err) + assert.Contains(t, err.Error(), tc.expectError) + assert.Nil(t, result) + }) + } +} + +func TestValueToJSON_RoundTrip(t *testing.T) { + t.Parallel() + + // Test that we can round-trip simple values through JSON + testCases := []struct { + name string + typ tftypes.Type + value tftypes.Value + }{ + {"string", tftypes.String, tftypes.NewValue(tftypes.String, "test")}, + {"number", tftypes.Number, tftypes.NewValue(tftypes.Number, 42)}, + {"bool", tftypes.Bool, tftypes.NewValue(tftypes.Bool, true)}, + {"null string", tftypes.String, tftypes.NewValue(tftypes.String, nil)}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + // Convert to JSON + jsonBytes, err := tftypeValueToJSON(tc.typ, tc.value) + require.NoError(t, err) + + // Convert back from JSON + reconstructed, err := tftypes.ValueFromJSON(jsonBytes, tc.typ) + require.NoError(t, err) + + // Should be equal + assert.True(t, tc.value.Equal(reconstructed), + "Original: %s, Reconstructed: %s", tc.value.String(), reconstructed.String()) + }) + } +} diff --git a/pkg/valueshim/tfvalue.go b/pkg/valueshim/tfvalue.go index 71f11c12c..24c67d408 100644 --- a/pkg/valueshim/tfvalue.go +++ b/pkg/valueshim/tfvalue.go @@ -17,7 +17,6 @@ package valueshim import ( "encoding/json" "errors" - "fmt" "math/big" "github.com/hashicorp/terraform-plugin-go/tftypes" @@ -81,15 +80,11 @@ func (v tValueShim) Marshal(schemaType Type) (json.RawMessage, error) { if !ok { return nil, errors.New("Cannot marshal to RawState: expected schemaType to be of type tTypeShim") } - ctyType, err := toCtyType(tt.ty()) + raw, err := tftypeValueToJSON(tt.ty(), v.val()) if err != nil { - return nil, fmt.Errorf("Cannot marshal to RawState. Error converting to cty.Type: %w", err) + return nil, err } - cty, err := toCtyValue(tt.ty(), ctyType, v.val()) - if err != nil { - return nil, fmt.Errorf("Cannot marshal to RawState. Error converting to cty.Value: %w", err) - } - return FromHCtyValue(cty).Marshal(FromHCtyType(ctyType)) + return json.RawMessage(raw), nil } func (v tValueShim) Remove(prop string) Value { From 2a50b2bdd42bdd1a979935fc8bb14b96294d864d Mon Sep 17 00:00:00 2001 From: Venelin Date: Mon, 9 Jun 2025 12:50:00 +0300 Subject: [PATCH 20/31] remove unused --- pkg/pf/tests/upgrade_state_cross_test.go | 56 ------------------------ pkg/valueshim/convert.go | 41 ----------------- 2 files changed, 97 deletions(-) delete mode 100644 pkg/valueshim/convert.go diff --git a/pkg/pf/tests/upgrade_state_cross_test.go b/pkg/pf/tests/upgrade_state_cross_test.go index 16636e0bf..3594c830a 100644 --- a/pkg/pf/tests/upgrade_state_cross_test.go +++ b/pkg/pf/tests/upgrade_state_cross_test.go @@ -1092,62 +1092,6 @@ func TestPFUpgrade_Downgrading(t *testing.T) { }) } -// Test when a dynamic pseudo type value is being sent through a state upgrader. -func TestPFUpgrade_DynamicPseudoType(t *testing.T) { - t.Parallel() - ct.SkipUnlessLinux(t) - // skipUnlessDeltasEnabled(t) - - resourceBefore := pb.NewResource(pb.NewResourceArgs{ - ResourceSchema: rschema.Schema{ - Attributes: map[string]rschema.Attribute{ - "dyn": rschema.DynamicAttribute{Optional: true}, - }, - }, - }) - - resourceAfter := pb.NewResource(pb.NewResourceArgs{ - UpgradeStateFunc: func(ctx context.Context) map[int64]resource.StateUpgrader { - return map[int64]resource.StateUpgrader{ - 0: { - PriorSchema: &resourceBefore.ResourceSchema, - StateUpgrader: ct.NopUpgrader, - }, - } - }, - ResourceSchema: rschema.Schema{ - Attributes: map[string]rschema.Attribute{ - "dyn": rschema.DynamicAttribute{Optional: true}, - }, - Version: 1, - }, - }) - - tfInputsBefore := cty.ObjectVal(map[string]cty.Value{"dyn": cty.StringVal("str")}) - tfInputsAfter := cty.ObjectVal(map[string]cty.Value{"dyn": cty.NumberIntVal(42)}) - pmBefore := presource.NewPropertyMapFromMap(map[string]any{"dyn": "str"}) - pmAfter := presource.NewPropertyMapFromMap(map[string]any{"dyn": 42}) - - testCase := ct.UpgradeStateTestCase{ - Resource1: &resourceBefore, - Resource2: &resourceAfter, - Inputs1: tfInputsBefore, - InputsMap1: pmBefore, - Inputs2: tfInputsAfter, - InputsMap2: pmAfter, - // ExpectedRawStateType: resourceBeforeAndAfter.ResourceSchema.Type().TerraformType(context.Background()), - SkipPulumi: "TODO", - } - - _ = testCase.Run(t) - - // autogold.Expect(map[apitype.OpType]int{apitype.OpType("same"): 2}).Equal(t, result.PulumiPreviewResult.ChangeSummary) - // autogold.Expect(&map[string]int{"same": 2}).Equal(t, result.PulumiUpResult.Summary.ResourceChanges) - - // autogold.Expect([]ct.UpgradeStateTrace{}).Equal(t, result.PulumiUpgrades) - // autogold.Expect([]ct.UpgradeStateTrace{}).Equal(t, result.TFUpgrades) -} - func skipUnlessDeltasEnabled(t *testing.T) { if d, ok := os.LookupEnv("PULUMI_RAW_STATE_DELTA_ENABLED"); !ok || !cmdutil.IsTruthy(d) { t.Skip("This test requires PULUMI_RAW_STATE_DELTA_ENABLED=true environment") diff --git a/pkg/valueshim/convert.go b/pkg/valueshim/convert.go deleted file mode 100644 index e5f33ef75..000000000 --- a/pkg/valueshim/convert.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2016-2025, Pulumi Corporation. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package valueshim - -import ( - "encoding/json" - - "github.com/hashicorp/go-cty/cty" - ctyjson "github.com/hashicorp/go-cty/cty/json" - ctymsgpack "github.com/hashicorp/go-cty/cty/msgpack" - "github.com/hashicorp/terraform-plugin-go/tfprotov6" - "github.com/hashicorp/terraform-plugin-go/tftypes" -) - -func toCtyType(t tftypes.Type) (cty.Type, error) { - typeBytes, err := json.Marshal(t) - if err != nil { - return cty.NilType, err - } - return ctyjson.UnmarshalType(typeBytes) -} - -func toCtyValue(schemaType tftypes.Type, schemaCtyType cty.Type, value tftypes.Value) (cty.Value, error) { - dv, err := tfprotov6.NewDynamicValue(schemaType, value) - if err != nil { - return cty.NilVal, err - } - return ctymsgpack.Unmarshal(dv.MsgPack, schemaCtyType) -} From a8ad0b664d52466fefed98524eefc585bcfa8ce9 Mon Sep 17 00:00:00 2001 From: Venelin Date: Mon, 9 Jun 2025 13:01:45 +0300 Subject: [PATCH 21/31] add todo for type changes --- pkg/pf/tests/diff_test/diff_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/pf/tests/diff_test/diff_test.go b/pkg/pf/tests/diff_test/diff_test.go index fc981e2d1..157c60550 100644 --- a/pkg/pf/tests/diff_test/diff_test.go +++ b/pkg/pf/tests/diff_test/diff_test.go @@ -218,7 +218,7 @@ func TestPFDetailedDiffDynamicType(t *testing.T) { }) t.Run("type change", func(t *testing.T) { - // TODO[pulumi/pulumi-terraform-bridge#3078] + // TODO[pulumi/pulumi-terraform-bridge#3122] t.Skip(`Error converting tftypes.Number<"1"> (value2) at "AttributeName(\"key\")": can't unmarshal tftypes.Number into *string, expected string`) crosstests.Diff(t, res, map[string]cty.Value{"key": cty.StringVal("value")}, @@ -266,7 +266,7 @@ func TestPFDetailedDiffNestedDynamicType(t *testing.T) { }) t.Run("type change", func(t *testing.T) { - // TODO[pulumi/pulumi-terraform-bridge#3078] + // TODO[pulumi/pulumi-terraform-bridge#3122] t.Skip(`Error converting tftypes.Number<"1"> (value2) at "AttributeName(\"key\")": can't unmarshal tftypes.Number into *string, expected string`) crosstests.Diff(t, res, map[string]cty.Value{"key": cty.ObjectVal(map[string]cty.Value{"nested": cty.StringVal("value")})}, From ecfb99d14ce351067a1ea93556308fb69314077e Mon Sep 17 00:00:00 2001 From: Venelin Date: Mon, 9 Jun 2025 13:03:21 +0300 Subject: [PATCH 22/31] lint --- pkg/valueshim/tftype_json.go | 1 + pkg/valueshim/tftype_json_test.go | 34 ------------------------------- 2 files changed, 1 insertion(+), 34 deletions(-) diff --git a/pkg/valueshim/tftype_json.go b/pkg/valueshim/tftype_json.go index 77e2d7695..a8e9a779e 100644 --- a/pkg/valueshim/tftype_json.go +++ b/pkg/valueshim/tftype_json.go @@ -91,6 +91,7 @@ func jsonMarshalBool(v tftypes.Value, typ tftypes.Type, p *tftypes.AttributePath func jsonMarshalDynamicPseudoType(v tftypes.Value, _ tftypes.Type, p *tftypes.AttributePath) (interface{}, error) { valType := v.Type() + //nolint:staticcheck // the method isn't really deprecated but rather internal typeJSON, err := valType.MarshalJSON() if err != nil { return nil, p.NewError(err) diff --git a/pkg/valueshim/tftype_json_test.go b/pkg/valueshim/tftype_json_test.go index b41ec127f..57685f2ab 100644 --- a/pkg/valueshim/tftype_json_test.go +++ b/pkg/valueshim/tftype_json_test.go @@ -540,37 +540,3 @@ func TestValueToJSON_EdgeCases(t *testing.T) { }) } } - -func TestValueToJSON_RoundTrip(t *testing.T) { - t.Parallel() - - // Test that we can round-trip simple values through JSON - testCases := []struct { - name string - typ tftypes.Type - value tftypes.Value - }{ - {"string", tftypes.String, tftypes.NewValue(tftypes.String, "test")}, - {"number", tftypes.Number, tftypes.NewValue(tftypes.Number, 42)}, - {"bool", tftypes.Bool, tftypes.NewValue(tftypes.Bool, true)}, - {"null string", tftypes.String, tftypes.NewValue(tftypes.String, nil)}, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - // Convert to JSON - jsonBytes, err := tftypeValueToJSON(tc.typ, tc.value) - require.NoError(t, err) - - // Convert back from JSON - reconstructed, err := tftypes.ValueFromJSON(jsonBytes, tc.typ) - require.NoError(t, err) - - // Should be equal - assert.True(t, tc.value.Equal(reconstructed), - "Original: %s, Reconstructed: %s", tc.value.String(), reconstructed.String()) - }) - } -} From 9de5088609b20e524bdef37bfcef4237721c2fa8 Mon Sep 17 00:00:00 2001 From: Venelin Date: Mon, 9 Jun 2025 13:15:41 +0300 Subject: [PATCH 23/31] fix float serialization --- pkg/valueshim/tftype_json.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/valueshim/tftype_json.go b/pkg/valueshim/tftype_json.go index a8e9a779e..148df4bc2 100644 --- a/pkg/valueshim/tftype_json.go +++ b/pkg/valueshim/tftype_json.go @@ -76,8 +76,7 @@ func jsonMarshalNumber(v tftypes.Value, typ tftypes.Type, p *tftypes.AttributePa if err != nil { return nil, p.NewError(err) } - f64, _ := n.Float64() - return f64, nil + return json.Number(n.Text('f', -1)), nil } func jsonMarshalBool(v tftypes.Value, typ tftypes.Type, p *tftypes.AttributePath) (interface{}, error) { From fd00d4d9438d287c0488fdd13d86e7f2aa5e433c Mon Sep 17 00:00:00 2001 From: Venelin Date: Mon, 9 Jun 2025 13:47:21 +0300 Subject: [PATCH 24/31] fix serialization of empty lists and sets --- pkg/valueshim/tftype_json.go | 12 ++++++------ pkg/valueshim/tftype_json_test.go | 6 ++---- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/pkg/valueshim/tftype_json.go b/pkg/valueshim/tftype_json.go index 148df4bc2..592f706a7 100644 --- a/pkg/valueshim/tftype_json.go +++ b/pkg/valueshim/tftype_json.go @@ -111,14 +111,14 @@ func jsonMarshalList(v tftypes.Value, elementType tftypes.Type, p *tftypes.Attri if err != nil { return nil, p.NewError(err) } - var res []interface{} + res := make([]interface{}, len(vs)) for i, v := range vs { ep := p.WithElementKeyInt(i) e, err := jsonMarshal(v, elementType, ep) if err != nil { return nil, ep.NewError(err) } - res = append(res, e) + res[i] = e } return res, nil } @@ -129,14 +129,14 @@ func jsonMarshalSet(v tftypes.Value, elementType tftypes.Type, p *tftypes.Attrib if err != nil { return nil, p.NewError(err) } - var res []interface{} - for _, v := range vs { + res := make([]interface{}, len(vs)) + for i, v := range vs { ep := p.WithElementKeyValue(v) e, err := jsonMarshal(v, elementType, ep) if err != nil { return nil, ep.NewError(err) } - res = append(res, e) + res[i] = e } return res, nil } @@ -147,7 +147,7 @@ func jsonMarshalMap(v tftypes.Value, elementType tftypes.Type, p *tftypes.Attrib if err != nil { return nil, p.NewError(err) } - res := map[string]interface{}{} + res := make(map[string]interface{}, len(vs)) for k, v := range vs { ep := p.WithElementKeyValue(v) e, err := jsonMarshal(v, elementType, ep) diff --git a/pkg/valueshim/tftype_json_test.go b/pkg/valueshim/tftype_json_test.go index 57685f2ab..5ef8d7961 100644 --- a/pkg/valueshim/tftype_json_test.go +++ b/pkg/valueshim/tftype_json_test.go @@ -124,8 +124,7 @@ func TestValueToJSON(t *testing.T) { name: "list empty", typ: tftypes.List{ElementType: tftypes.String}, value: tftypes.NewValue(tftypes.List{ElementType: tftypes.String}, []tftypes.Value{}), - // Empty lists serialize to null, not [] - expectJSON: autogold.Expect("null"), + expectJSON: autogold.Expect("[]"), }, { name: "list of strings", @@ -169,8 +168,7 @@ func TestValueToJSON(t *testing.T) { name: "set empty", typ: tftypes.Set{ElementType: tftypes.String}, value: tftypes.NewValue(tftypes.Set{ElementType: tftypes.String}, []tftypes.Value{}), - // Empty sets serialize to null, not [] - expectJSON: autogold.Expect("null"), + expectJSON: autogold.Expect("[]"), }, { name: "set of strings", From a580ec9228434de5a0c337dfb13915985a41ba1d Mon Sep 17 00:00:00 2001 From: Venelin Date: Mon, 9 Jun 2025 13:51:37 +0300 Subject: [PATCH 25/31] remove redundant cty shim --- pkg/valueshim/cty.go | 19 +++-- pkg/valueshim/hcty.go | 182 ------------------------------------------ 2 files changed, 9 insertions(+), 192 deletions(-) delete mode 100644 pkg/valueshim/hcty.go diff --git a/pkg/valueshim/cty.go b/pkg/valueshim/cty.go index e6ac47b01..d70724ebd 100644 --- a/pkg/valueshim/cty.go +++ b/pkg/valueshim/cty.go @@ -19,17 +19,17 @@ import ( "fmt" "math/big" - "github.com/zclconf/go-cty/cty" - ctyjson "github.com/zclconf/go-cty/cty/json" + "github.com/hashicorp/go-cty/cty" + ctyjson "github.com/hashicorp/go-cty/cty/json" ) // Wrap a cty.Value as Value. -func FromCtyValue(v cty.Value) Value { +func FromHCtyValue(v cty.Value) Value { return ctyValueShim(v) } // Wrap a cty.Type as Type. -func FromCtyType(v cty.Type) Type { +func FromHCtyType(v cty.Type) Type { return ctyTypeShim(v) } @@ -50,7 +50,7 @@ func (v ctyValueShim) GoString() string { } func (v ctyValueShim) Type() Type { - return FromCtyType(v.val().Type()) + return FromHCtyType(v.val().Type()) } func (v ctyValueShim) StringValue() string { @@ -62,8 +62,7 @@ func (v ctyValueShim) BoolValue() bool { } func (v ctyValueShim) NumberValue() float64 { - bf := v.BigFloatValue() - f, _ := bf.Float64() + f, _ := v.val().AsBigFloat().Float64() return f } @@ -109,7 +108,7 @@ func (v ctyValueShim) Marshal(schemaType Type) (json.RawMessage, error) { tt, ok := schemaType.(ctyTypeShim) if !ok { return nil, fmt.Errorf("Cannot marshal to RawState: "+ - "expected schemaType to be of type ctyTypeShim, got %#T", + "expected schemaType to be of type hctyTypeShim, got %#T", schemaType) } raw, err := ctyjson.Marshal(vv, tt.ty()) @@ -167,7 +166,7 @@ func (t ctyTypeShim) AttributeType(name string) (Type, bool) { if !tt.HasAttribute(name) { return nil, false } - return FromCtyType(tt.AttributeType(name)), true + return FromHCtyType(tt.AttributeType(name)), true } func (t ctyTypeShim) ElementType() (Type, bool) { @@ -175,7 +174,7 @@ func (t ctyTypeShim) ElementType() (Type, bool) { if !tt.IsCollectionType() { return nil, false } - return FromCtyType(tt.ElementType()), true + return FromHCtyType(tt.ElementType()), true } func (t ctyTypeShim) GoString() string { diff --git a/pkg/valueshim/hcty.go b/pkg/valueshim/hcty.go deleted file mode 100644 index 9623a6f50..000000000 --- a/pkg/valueshim/hcty.go +++ /dev/null @@ -1,182 +0,0 @@ -// Copyright 2016-2025, Pulumi Corporation. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package valueshim - -import ( - "encoding/json" - "fmt" - "math/big" - - "github.com/hashicorp/go-cty/cty" - ctyjson "github.com/hashicorp/go-cty/cty/json" -) - -// Wrap a cty.Value as Value. -func FromHCtyValue(v cty.Value) Value { - return hctyValueShim(v) -} - -// Wrap a cty.Type as Type. -func FromHCtyType(v cty.Type) Type { - return hctyTypeShim(v) -} - -type hctyValueShim cty.Value - -var _ Value = (*hctyValueShim)(nil) - -func (v hctyValueShim) val() cty.Value { - return cty.Value(v) -} - -func (v hctyValueShim) IsNull() bool { - return v.val().IsNull() -} - -func (v hctyValueShim) GoString() string { - return v.val().GoString() -} - -func (v hctyValueShim) Type() Type { - return FromHCtyType(v.val().Type()) -} - -func (v hctyValueShim) StringValue() string { - return v.val().AsString() -} - -func (v hctyValueShim) BoolValue() bool { - return v.val().True() -} - -func (v hctyValueShim) NumberValue() float64 { - f, _ := v.val().AsBigFloat().Float64() - return f -} - -func (v hctyValueShim) BigFloatValue() *big.Float { - return v.val().AsBigFloat() -} - -func (v hctyValueShim) AsValueSlice() []Value { - s := v.val().AsValueSlice() - res := make([]Value, len(s)) - for i, v := range s { - res[i] = hctyValueShim(v) - } - return res -} - -func (v hctyValueShim) AsValueMap() map[string]Value { - m := v.val().AsValueMap() - res := make(map[string]Value, len(m)) - - for k, v := range m { - res[k] = hctyValueShim(v) - } - return res -} - -func (v hctyValueShim) Remove(key string) Value { - switch { - case v.val().Type().IsObjectType(): - m := v.val().AsValueMap() - delete(m, key) - if len(m) == 0 { - return hctyValueShim(cty.EmptyObjectVal) - } - return hctyValueShim(cty.ObjectVal(m)) - default: - return v - } -} - -func (v hctyValueShim) Marshal(schemaType Type) (json.RawMessage, error) { - vv := v.val() - tt, ok := schemaType.(hctyTypeShim) - if !ok { - return nil, fmt.Errorf("Cannot marshal to RawState: "+ - "expected schemaType to be of type hctyTypeShim, got %#T", - schemaType) - } - raw, err := ctyjson.Marshal(vv, tt.ty()) - if err != nil { - return nil, fmt.Errorf("Cannot marshal to RawState: %w", err) - } - return json.RawMessage(raw), nil -} - -type hctyTypeShim cty.Type - -var _ Type = hctyTypeShim{} - -func (t hctyTypeShim) ty() cty.Type { - return cty.Type(t) -} - -func (t hctyTypeShim) IsNumberType() bool { - return t.ty().Equals(cty.Number) -} - -func (t hctyTypeShim) IsBooleanType() bool { - return t.ty().Equals(cty.Bool) -} - -func (t hctyTypeShim) IsStringType() bool { - return t.ty().Equals(cty.String) -} - -func (t hctyTypeShim) IsListType() bool { - return t.ty().IsListType() -} - -func (t hctyTypeShim) IsMapType() bool { - return t.ty().IsMapType() -} - -func (t hctyTypeShim) IsSetType() bool { - return t.ty().IsSetType() -} - -func (t hctyTypeShim) IsObjectType() bool { - return t.ty().IsObjectType() -} - -func (t hctyTypeShim) IsDynamicType() bool { - return t.ty().Equals(cty.DynamicPseudoType) -} - -func (t hctyTypeShim) AttributeType(name string) (Type, bool) { - tt := t.ty() - if !tt.IsObjectType() { - return nil, false - } - if !tt.HasAttribute(name) { - return nil, false - } - return FromHCtyType(tt.AttributeType(name)), true -} - -func (t hctyTypeShim) ElementType() (Type, bool) { - tt := t.ty() - if !tt.IsCollectionType() { - return nil, false - } - return FromHCtyType(tt.ElementType()), true -} - -func (t hctyTypeShim) GoString() string { - return t.ty().GoString() -} From 50d6045e5b346c2883e410afb767e7d68dd70c4f Mon Sep 17 00:00:00 2001 From: Venelin Date: Mon, 9 Jun 2025 13:59:00 +0300 Subject: [PATCH 26/31] Revert "remove redundant cty shim" This reverts commit a580ec9228434de5a0c337dfb13915985a41ba1d. --- pkg/valueshim/cty.go | 19 ++--- pkg/valueshim/hcty.go | 182 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 192 insertions(+), 9 deletions(-) create mode 100644 pkg/valueshim/hcty.go diff --git a/pkg/valueshim/cty.go b/pkg/valueshim/cty.go index d70724ebd..e6ac47b01 100644 --- a/pkg/valueshim/cty.go +++ b/pkg/valueshim/cty.go @@ -19,17 +19,17 @@ import ( "fmt" "math/big" - "github.com/hashicorp/go-cty/cty" - ctyjson "github.com/hashicorp/go-cty/cty/json" + "github.com/zclconf/go-cty/cty" + ctyjson "github.com/zclconf/go-cty/cty/json" ) // Wrap a cty.Value as Value. -func FromHCtyValue(v cty.Value) Value { +func FromCtyValue(v cty.Value) Value { return ctyValueShim(v) } // Wrap a cty.Type as Type. -func FromHCtyType(v cty.Type) Type { +func FromCtyType(v cty.Type) Type { return ctyTypeShim(v) } @@ -50,7 +50,7 @@ func (v ctyValueShim) GoString() string { } func (v ctyValueShim) Type() Type { - return FromHCtyType(v.val().Type()) + return FromCtyType(v.val().Type()) } func (v ctyValueShim) StringValue() string { @@ -62,7 +62,8 @@ func (v ctyValueShim) BoolValue() bool { } func (v ctyValueShim) NumberValue() float64 { - f, _ := v.val().AsBigFloat().Float64() + bf := v.BigFloatValue() + f, _ := bf.Float64() return f } @@ -108,7 +109,7 @@ func (v ctyValueShim) Marshal(schemaType Type) (json.RawMessage, error) { tt, ok := schemaType.(ctyTypeShim) if !ok { return nil, fmt.Errorf("Cannot marshal to RawState: "+ - "expected schemaType to be of type hctyTypeShim, got %#T", + "expected schemaType to be of type ctyTypeShim, got %#T", schemaType) } raw, err := ctyjson.Marshal(vv, tt.ty()) @@ -166,7 +167,7 @@ func (t ctyTypeShim) AttributeType(name string) (Type, bool) { if !tt.HasAttribute(name) { return nil, false } - return FromHCtyType(tt.AttributeType(name)), true + return FromCtyType(tt.AttributeType(name)), true } func (t ctyTypeShim) ElementType() (Type, bool) { @@ -174,7 +175,7 @@ func (t ctyTypeShim) ElementType() (Type, bool) { if !tt.IsCollectionType() { return nil, false } - return FromHCtyType(tt.ElementType()), true + return FromCtyType(tt.ElementType()), true } func (t ctyTypeShim) GoString() string { diff --git a/pkg/valueshim/hcty.go b/pkg/valueshim/hcty.go new file mode 100644 index 000000000..9623a6f50 --- /dev/null +++ b/pkg/valueshim/hcty.go @@ -0,0 +1,182 @@ +// Copyright 2016-2025, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package valueshim + +import ( + "encoding/json" + "fmt" + "math/big" + + "github.com/hashicorp/go-cty/cty" + ctyjson "github.com/hashicorp/go-cty/cty/json" +) + +// Wrap a cty.Value as Value. +func FromHCtyValue(v cty.Value) Value { + return hctyValueShim(v) +} + +// Wrap a cty.Type as Type. +func FromHCtyType(v cty.Type) Type { + return hctyTypeShim(v) +} + +type hctyValueShim cty.Value + +var _ Value = (*hctyValueShim)(nil) + +func (v hctyValueShim) val() cty.Value { + return cty.Value(v) +} + +func (v hctyValueShim) IsNull() bool { + return v.val().IsNull() +} + +func (v hctyValueShim) GoString() string { + return v.val().GoString() +} + +func (v hctyValueShim) Type() Type { + return FromHCtyType(v.val().Type()) +} + +func (v hctyValueShim) StringValue() string { + return v.val().AsString() +} + +func (v hctyValueShim) BoolValue() bool { + return v.val().True() +} + +func (v hctyValueShim) NumberValue() float64 { + f, _ := v.val().AsBigFloat().Float64() + return f +} + +func (v hctyValueShim) BigFloatValue() *big.Float { + return v.val().AsBigFloat() +} + +func (v hctyValueShim) AsValueSlice() []Value { + s := v.val().AsValueSlice() + res := make([]Value, len(s)) + for i, v := range s { + res[i] = hctyValueShim(v) + } + return res +} + +func (v hctyValueShim) AsValueMap() map[string]Value { + m := v.val().AsValueMap() + res := make(map[string]Value, len(m)) + + for k, v := range m { + res[k] = hctyValueShim(v) + } + return res +} + +func (v hctyValueShim) Remove(key string) Value { + switch { + case v.val().Type().IsObjectType(): + m := v.val().AsValueMap() + delete(m, key) + if len(m) == 0 { + return hctyValueShim(cty.EmptyObjectVal) + } + return hctyValueShim(cty.ObjectVal(m)) + default: + return v + } +} + +func (v hctyValueShim) Marshal(schemaType Type) (json.RawMessage, error) { + vv := v.val() + tt, ok := schemaType.(hctyTypeShim) + if !ok { + return nil, fmt.Errorf("Cannot marshal to RawState: "+ + "expected schemaType to be of type hctyTypeShim, got %#T", + schemaType) + } + raw, err := ctyjson.Marshal(vv, tt.ty()) + if err != nil { + return nil, fmt.Errorf("Cannot marshal to RawState: %w", err) + } + return json.RawMessage(raw), nil +} + +type hctyTypeShim cty.Type + +var _ Type = hctyTypeShim{} + +func (t hctyTypeShim) ty() cty.Type { + return cty.Type(t) +} + +func (t hctyTypeShim) IsNumberType() bool { + return t.ty().Equals(cty.Number) +} + +func (t hctyTypeShim) IsBooleanType() bool { + return t.ty().Equals(cty.Bool) +} + +func (t hctyTypeShim) IsStringType() bool { + return t.ty().Equals(cty.String) +} + +func (t hctyTypeShim) IsListType() bool { + return t.ty().IsListType() +} + +func (t hctyTypeShim) IsMapType() bool { + return t.ty().IsMapType() +} + +func (t hctyTypeShim) IsSetType() bool { + return t.ty().IsSetType() +} + +func (t hctyTypeShim) IsObjectType() bool { + return t.ty().IsObjectType() +} + +func (t hctyTypeShim) IsDynamicType() bool { + return t.ty().Equals(cty.DynamicPseudoType) +} + +func (t hctyTypeShim) AttributeType(name string) (Type, bool) { + tt := t.ty() + if !tt.IsObjectType() { + return nil, false + } + if !tt.HasAttribute(name) { + return nil, false + } + return FromHCtyType(tt.AttributeType(name)), true +} + +func (t hctyTypeShim) ElementType() (Type, bool) { + tt := t.ty() + if !tt.IsCollectionType() { + return nil, false + } + return FromHCtyType(tt.ElementType()), true +} + +func (t hctyTypeShim) GoString() string { + return t.ty().GoString() +} From 990858965103cde56ebcf9238e53b6219d7d0e61 Mon Sep 17 00:00:00 2001 From: Venelin Date: Mon, 9 Jun 2025 14:00:55 +0300 Subject: [PATCH 27/31] lint --- pkg/valueshim/tftype_json_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/valueshim/tftype_json_test.go b/pkg/valueshim/tftype_json_test.go index 5ef8d7961..c5441c7fd 100644 --- a/pkg/valueshim/tftype_json_test.go +++ b/pkg/valueshim/tftype_json_test.go @@ -121,9 +121,9 @@ func TestValueToJSON(t *testing.T) { expectJSON: autogold.Expect("null"), }, { - name: "list empty", - typ: tftypes.List{ElementType: tftypes.String}, - value: tftypes.NewValue(tftypes.List{ElementType: tftypes.String}, []tftypes.Value{}), + name: "list empty", + typ: tftypes.List{ElementType: tftypes.String}, + value: tftypes.NewValue(tftypes.List{ElementType: tftypes.String}, []tftypes.Value{}), expectJSON: autogold.Expect("[]"), }, { @@ -165,9 +165,9 @@ func TestValueToJSON(t *testing.T) { expectJSON: autogold.Expect("null"), }, { - name: "set empty", - typ: tftypes.Set{ElementType: tftypes.String}, - value: tftypes.NewValue(tftypes.Set{ElementType: tftypes.String}, []tftypes.Value{}), + name: "set empty", + typ: tftypes.Set{ElementType: tftypes.String}, + value: tftypes.NewValue(tftypes.Set{ElementType: tftypes.String}, []tftypes.Value{}), expectJSON: autogold.Expect("[]"), }, { From 478e3cbc48c00283d86caab52cafd6c880dc28aa Mon Sep 17 00:00:00 2001 From: Venelin Date: Mon, 9 Jun 2025 14:21:04 +0300 Subject: [PATCH 28/31] fix marshalling of dynamic type values --- pkg/valueshim/tftype_json.go | 21 ++++++++++++--------- pkg/valueshim/tftype_json_test.go | 6 +++--- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/pkg/valueshim/tftype_json.go b/pkg/valueshim/tftype_json.go index 592f706a7..830948798 100644 --- a/pkg/valueshim/tftype_json.go +++ b/pkg/valueshim/tftype_json.go @@ -16,6 +16,7 @@ package valueshim import ( "encoding/json" + "fmt" "math/big" "github.com/hashicorp/terraform-plugin-go/tftypes" @@ -31,12 +32,15 @@ func tftypeValueToJSON(typ tftypes.Type, v tftypes.Value) ([]byte, error) { } func jsonMarshal(v tftypes.Value, typ tftypes.Type, p *tftypes.AttributePath) (interface{}, error) { - if v.IsNull() { - return nil, nil - } if !v.IsKnown() { return nil, p.NewErrorf("unknown values cannot be serialized to JSON") } + if typ.Is(tftypes.DynamicPseudoType) { + return jsonMarshalDynamicPseudoType(v, typ, p) + } + if v.IsNull() { + return nil, nil + } switch { case typ.Is(tftypes.String): return jsonMarshalString(v, typ, p) @@ -44,8 +48,6 @@ func jsonMarshal(v tftypes.Value, typ tftypes.Type, p *tftypes.AttributePath) (i return jsonMarshalNumber(v, typ, p) case typ.Is(tftypes.Bool): return jsonMarshalBool(v, typ, p) - case typ.Is(tftypes.DynamicPseudoType): - return jsonMarshalDynamicPseudoType(v, typ, p) case typ.Is(tftypes.List{}): return jsonMarshalList(v, typ.(tftypes.List).ElementType, p) case typ.Is(tftypes.Set{}): @@ -99,10 +101,11 @@ func jsonMarshalDynamicPseudoType(v tftypes.Value, _ tftypes.Type, p *tftypes.At if err != nil { return nil, p.NewError(err) } - return map[string]interface{}{ - "type": string(typeJSON), - "value": valJSON, - }, nil + marshalledValJSON, err := json.Marshal(valJSON) + if err != nil { + return nil, p.NewError(err) + } + return json.RawMessage(fmt.Sprintf(`{"value": %s, "type": %s}`, marshalledValJSON, typeJSON)), nil } func jsonMarshalList(v tftypes.Value, elementType tftypes.Type, p *tftypes.AttributePath) (interface{}, error) { diff --git a/pkg/valueshim/tftype_json_test.go b/pkg/valueshim/tftype_json_test.go index c5441c7fd..0032ea338 100644 --- a/pkg/valueshim/tftype_json_test.go +++ b/pkg/valueshim/tftype_json_test.go @@ -367,7 +367,7 @@ func TestValueToJSON_DynamicPseudoType(t *testing.T) { require.NoError(t, err) // The result should be a JSON object with type and value fields - expected := `{"type":"\"string\"","value":"dynamic content"}` + expected := `{"type":"string","value":"dynamic content"}` assert.JSONEq(t, expected, string(result)) // Also test with other types @@ -375,14 +375,14 @@ func TestValueToJSON_DynamicPseudoType(t *testing.T) { result, err = tftypeValueToJSON(tftypes.DynamicPseudoType, numberValue) require.NoError(t, err) - expected = `{"type":"\"number\"","value":42}` + expected = `{"type":"number","value":42}` assert.JSONEq(t, expected, string(result)) boolValue := tftypes.NewValue(tftypes.Bool, true) result, err = tftypeValueToJSON(tftypes.DynamicPseudoType, boolValue) require.NoError(t, err) - expected = `{"type":"\"bool\"","value":true}` + expected = `{"type":"bool","value":true}` assert.JSONEq(t, expected, string(result)) } From e14db5845cd15876a75f86c47909e76b7ff175aa Mon Sep 17 00:00:00 2001 From: Venelin Date: Mon, 9 Jun 2025 14:50:32 +0300 Subject: [PATCH 29/31] fix null dynamic type handling --- pkg/valueshim/tftype_json.go | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/pkg/valueshim/tftype_json.go b/pkg/valueshim/tftype_json.go index 830948798..b1e46fc24 100644 --- a/pkg/valueshim/tftype_json.go +++ b/pkg/valueshim/tftype_json.go @@ -97,14 +97,22 @@ func jsonMarshalDynamicPseudoType(v tftypes.Value, _ tftypes.Type, p *tftypes.At if err != nil { return nil, p.NewError(err) } - valJSON, err := jsonMarshal(v, valType, p) - if err != nil { - return nil, p.NewError(err) - } - marshalledValJSON, err := json.Marshal(valJSON) - if err != nil { - return nil, p.NewError(err) + + var marshalledValJSON []byte + // The null case is handled separately to prevent infinite recursion. + if v.IsNull() { + marshalledValJSON = []byte("null") + } else { + valJSON, err := jsonMarshal(v, v.Type(), p) + if err != nil { + return nil, p.NewError(err) + } + marshalledValJSON, err = json.Marshal(valJSON) + if err != nil { + return nil, p.NewError(err) + } } + return json.RawMessage(fmt.Sprintf(`{"value": %s, "type": %s}`, marshalledValJSON, typeJSON)), nil } From 6276979aa967d72168b6650e27408d2bbb8e74f1 Mon Sep 17 00:00:00 2001 From: Venelin Date: Tue, 10 Jun 2025 13:10:42 +0300 Subject: [PATCH 30/31] restore timeouts removal in turnaround check --- pkg/tfbridge/rawstate.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/tfbridge/rawstate.go b/pkg/tfbridge/rawstate.go index f6ed5542e..d4cdb275b 100644 --- a/pkg/tfbridge/rawstate.go +++ b/pkg/tfbridge/rawstate.go @@ -493,7 +493,8 @@ func RawStateComputeDelta( pv := resource.NewObjectProperty(outMap) delta := ih.delta(pv, v) - err := delta.turnaroundCheck(ctx, newRawStateFromValue(schemaType, v), pv) + vWithoutTimeouts := v.Remove("timeouts") + err := delta.turnaroundCheck(ctx, newRawStateFromValue(schemaType, vWithoutTimeouts), pv) if err != nil { return RawStateDelta{}, err } From 2c92446d109f016d71b9c06c709a64feadbebc61 Mon Sep 17 00:00:00 2001 From: Venelin Date: Tue, 10 Jun 2025 15:26:27 +0300 Subject: [PATCH 31/31] handle sdkv2 timeouts as before --- pkg/tests/schema_pulumi_test.go | 47 +++++++++++++++++++++++++++++++++ pkg/tfbridge/rawstate.go | 12 ++++++--- pkg/tfshim/sdk-v2/provider2.go | 2 +- pkg/tfshim/sdk-v2/resource.go | 2 +- pkg/valueshim/hcty.go | 13 +++++++++ 5 files changed, 71 insertions(+), 5 deletions(-) diff --git a/pkg/tests/schema_pulumi_test.go b/pkg/tests/schema_pulumi_test.go index 4e10512da..9d25f7106 100644 --- a/pkg/tests/schema_pulumi_test.go +++ b/pkg/tests/schema_pulumi_test.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/hashicorp/terraform-plugin-sdk/v2/diag" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" @@ -1016,3 +1017,49 @@ Resources: pt.Up(t) }) } + +func TestTimeoutsHandling(t *testing.T) { + t.Parallel() + + res := &schema.Resource{ + Schema: map[string]*schema.Schema{ + "test": { + Type: schema.TypeString, + Optional: true, + }, + }, + Timeouts: &schema.ResourceTimeout{ + Create: schema.DefaultTimeout(time.Second * 10), + Update: schema.DefaultTimeout(time.Second * 10), + }, + } + + tfp := &schema.Provider{ + ResourcesMap: map[string]*schema.Resource{ + "prov_test": res, + }, + } + + bridgedProvider := pulcheck.BridgedProvider(t, "prov", tfp) + + pt := pulcheck.PulCheck(t, bridgedProvider, ` + name: test + runtime: yaml + resources: + mainRes: + type: prov:index:Test + properties: + test: hello`) + // we just check that no errors occur. + pt.Up(t) + + pt.WritePulumiYaml(t, ` + name: test + runtime: yaml + resources: + mainRes: + type: prov:index:Test + properties: + test: hello1`) + pt.Up(t) +} diff --git a/pkg/tfbridge/rawstate.go b/pkg/tfbridge/rawstate.go index d4cdb275b..6797ee7a3 100644 --- a/pkg/tfbridge/rawstate.go +++ b/pkg/tfbridge/rawstate.go @@ -828,9 +828,15 @@ func (ih *rawStateDeltaHelper) computeDeltaAt( if subPV, isIntersectingKey := pvElements[key]; isIntersectingKey { delta = ih.deltaAt(subPath, subPV, v) } else { - // Missing matching PropertyValue for key, generate a replace delta. - n := resource.NewNullProperty() - delta = ih.replaceDeltaAt(subPath, n, v, fmt.Errorf("No PropertyValue at key")) + if len(path) == 0 && key == "timeouts" { + // Timeouts are a special property that accidentally gets pushed here for historical reasons; it is not + // relevant for the permanent RawState storage. Ignore it for now. + delta = RawStateDelta{} + } else { + // Missing matching PropertyValue for key, generate a replace delta. + n := resource.NewNullProperty() + delta = ih.replaceDeltaAt(subPath, n, v, fmt.Errorf("No PropertyValue at key")) + } } oDelta.set(k, key, delta) handledKeys[key] = struct{}{} diff --git a/pkg/tfshim/sdk-v2/provider2.go b/pkg/tfshim/sdk-v2/provider2.go index 554a39d6c..1abed38e5 100644 --- a/pkg/tfshim/sdk-v2/provider2.go +++ b/pkg/tfshim/sdk-v2/provider2.go @@ -76,7 +76,7 @@ func (r *v2Resource2) DecodeTimeouts(config shim.ResourceConfig) (*shim.Resource } func (r *v2Resource2) SchemaType() valueshim.Type { - return valueshim.FromHCtyType(r.tf.CoreConfigSchema().ImpliedType()) + return valueshim.FromHctyResourceType(r.tf.CoreConfigSchema().ImpliedType()) } type v2InstanceState2 struct { diff --git a/pkg/tfshim/sdk-v2/resource.go b/pkg/tfshim/sdk-v2/resource.go index 2cdb7322e..9bb8548fa 100644 --- a/pkg/tfshim/sdk-v2/resource.go +++ b/pkg/tfshim/sdk-v2/resource.go @@ -35,7 +35,7 @@ func (r v2Resource) SchemaVersion() int { } func (r v2Resource) SchemaType() valueshim.Type { - return valueshim.FromHCtyType(r.tf.CoreConfigSchema().ImpliedType()) + return valueshim.FromHctyResourceType(r.tf.CoreConfigSchema().ImpliedType()) } func (r v2Resource) Importer() shim.ImportFunc { diff --git a/pkg/valueshim/hcty.go b/pkg/valueshim/hcty.go index 9623a6f50..d2b27ce9a 100644 --- a/pkg/valueshim/hcty.go +++ b/pkg/valueshim/hcty.go @@ -28,6 +28,19 @@ func FromHCtyValue(v cty.Value) Value { return hctyValueShim(v) } +func FromHctyResourceType(v cty.Type) Type { + if v.IsObjectType() { + // remove the timeouts attribute + attrsCopy := make(map[string]cty.Type, len(v.AttributeTypes())) + for k, v := range v.AttributeTypes() { + attrsCopy[k] = v + } + delete(attrsCopy, "timeouts") + v = cty.Object(attrsCopy) + } + return hctyTypeShim(v) +} + // Wrap a cty.Type as Type. func FromHCtyType(v cty.Type) Type { return hctyTypeShim(v)