Preserve TF raw state as deltas in Pulumi state (#2945)

This change fixes a long-standing problem of resource state data being
distorted by the bridge, which is particularly sensitive around provider
upgrade scenarios where schemas are changing and the TF provider expects
to receive the older state record verbatim to auto-migrate it to the new
schema. Before the change states were stored as PropertyMap and
converted to TF representation on-read against the currently known
provider schema, which may be different from the schema at the write
time. The fix is two-fold:

- at write time, compute the delta between TF raw state and Pulumi
PropertyMap and store it in the PropertyMap
- at read time, if the delta is found, reconstruct the original
byte-for-byte Terraform RawState and pass it to Terraform to migrate.

The change is feature-flagged and we aim for a gradual rollout but
eventually make the new behavior the default.

- Set EnableRawStateDelta: true on info.Provider to enable for a
provider
- Set PULUMI_RAW_STATE_DELTA_ENABLED=true to dynamically enable (or
disable) this behavior

One particular technical complication is around Set structures, which
Pulumi does not model. For the deltas to work correctly the set element
ordering in Pulumi array representation needs to be consistent between
the TF and Pulumi states. This is accomplished for the SDKv2 bridge by
preserving the cty.Value-canonical ordering but for the PF bridge by
preserving the "pulumi-natural" ordering of set elements.

Fixes #1667

Author: t0yv0

.github/workflows/build-and-test.yml

@@ -26,7 +26,7 @@ jobs:
         # go version in our go.mod files.
         go-version: [1.22.x, 1.23.x]
         platform: [ubuntu-latest, macos-latest, windows-latest]
-        feature-flags: ["DEFAULT"]
+        feature-flags: ["DEFAULT", "PULUMI_RAW_STATE_DELTA_ENABLED"]
         # Needs to match TOTAL_SHARDS
         shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
         exclude:

dynamic/provider_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/hexops/autogold/v2"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/resource/plugin"
+	"github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
 	pulumirpc "github.com/pulumi/pulumi/sdk/v3/proto/go"
 	"github.com/stretchr/testify/assert"
@@ -65,6 +66,8 @@ func TestStacktraceDisplayed(t *testing.T) {
 func TestPrimitiveTypes(t *testing.T) {
 	t.Parallel()
 	skipWindows(t)
+	skipUnlessDeltasEnabled(t)
+
 	ctx := context.Background()
 
 	grpc := grpcTestServer(ctx, t)
@@ -206,6 +209,12 @@ func TestPrimitiveTypes(t *testing.T) {
 	}, noParallel))
 }
 
+func skipUnlessDeltasEnabled(t *testing.T) {
+	if d, ok := os.LookupEnv("PULUMI_RAW_STATE_DELTA_ENABLED"); !ok || !cmdutil.IsTruthy(d) {
+		t.Skip("This test requires PULUMI_RAW_STATE_DELTA_ENABLED=true environment")
+	}
+}
+
 func TestConfigure(t *testing.T) {
 	t.Parallel()
 	skipWindows(t)

dynamic/testdata/TestPrimitiveTypes/create.golden

@@ -1,6 +1,9 @@
 {
   "id": "example-id",
   "properties": {
+    "__pulumi_raw_state_delta": {
+      "obj": {}
+    },
     "attrBoolComputed": false,
     "attrBoolRequired": true,
     "attrIntComputed": 128,

dynamic/testdata/TestPrimitiveTypes/import.golden

@@ -7,6 +7,37 @@
     "attrStringRequired": "imported value"
   },
   "properties": {
+    "__pulumi_raw_state_delta": {
+      "obj": {
+        "ps": {
+          "attrBoolComputed": {
+            "replace": {
+              "raw": null
+            }
+          },
+          "attrNumberComputed": {
+            "replace": {
+              "raw": null
+            }
+          },
+          "attrStringComputed": {
+            "replace": {
+              "raw": null
+            }
+          },
+          "attrStringDefault": {
+            "replace": {
+              "raw": null
+            }
+          },
+          "attrStringDefaultOverridden": {
+            "replace": {
+              "raw": null
+            }
+          }
+        }
+      }
+    },
     "attrBoolRequired": true,
     "attrIntComputed": 10000,
     "attrIntRequired": 1234,

dynamic/testdata/TestPrimitiveTypes/read.golden

@@ -2,6 +2,22 @@
   "id": "output-id",
   "inputs": {},
   "properties": {
+    "__pulumi_raw_state_delta": {
+      "obj": {
+        "ps": {
+          "attrBoolComputed": {
+            "replace": {
+              "raw": null
+            }
+          },
+          "attrNumberComputed": {
+            "replace": {
+              "raw": null
+            }
+          }
+        }
+      }
+    },
     "attrBoolRequired": true,
     "attrIntComputed": 10000,
     "attrIntRequired": 64,

dynamic/testdata/TestPrimitiveTypes/update.golden

@@ -1,5 +1,26 @@
 {
   "properties": {
+    "__pulumi_raw_state_delta": {
+      "obj": {
+        "ps": {
+          "attrBoolComputed": {
+            "replace": {
+              "raw": null
+            }
+          },
+          "attrNumberComputed": {
+            "replace": {
+              "raw": null
+            }
+          },
+          "attrStringComputed": {
+            "replace": {
+              "raw": null
+            }
+          }
+        }
+      }
+    },
     "attrBoolRequired": true,
     "attrIntComputed": 256,
     "attrIntRequired": 64,

go.mod

@@ -1,8 +1,8 @@
 module github.com/pulumi/pulumi-terraform-bridge/v3
 
-go 1.22.3
+go 1.23
 
-toolchain go1.23.2
+toolchain go1.23.7
 
 require (
 	github.com/apparentlymart/go-cidr v1.1.0
@@ -44,7 +44,7 @@ require (
 	github.com/opentofu/registry-address v0.0.0-20230922120653-901b9ae4061a
 	github.com/pkg/errors v0.9.1
 	github.com/pulumi/inflector v0.1.1
-	github.com/pulumi/providertest v0.1.3
+	github.com/pulumi/providertest v0.3.0
 	github.com/pulumi/pulumi-java/pkg v1.8.0
 	github.com/pulumi/pulumi-yaml v1.15.1
 	github.com/pulumi/schema-tools v0.1.2

go.sum

@@ -2198,8 +2198,8 @@ github.com/pulumi/esc v0.13.0 h1:O2MPR2koScaQ2fXwyer8Q3Dd7z+DCnaDfsgNl5mVNMk=
 github.com/pulumi/esc v0.13.0/go.mod h1:IIQo6W6Uzajt6f1RW4QvNxIRDlbK3TNQysnrwBHNo3U=
 github.com/pulumi/inflector v0.1.1 h1:dvlxlWtXwOJTUUtcYDvwnl6Mpg33prhK+7mzeF+SobA=
 github.com/pulumi/inflector v0.1.1/go.mod h1:HUFCjcPTz96YtTuUlwG3i3EZG4WlniBvR9bd+iJxCUY=
-github.com/pulumi/providertest v0.1.3 h1:GpNKRy/haNjRHiUA9bi4diU4Op2zf3axYXbga5AepHg=
-github.com/pulumi/providertest v0.1.3/go.mod h1:GcsqEGgSngwaNOD+kICJPIUQlnA911fGBU8HDlJvVL0=
+github.com/pulumi/providertest v0.3.0 h1:Dv10aoFaNQBDRtK8cRbd2uk9vEhpC/NryN+0y7NX4/o=
+github.com/pulumi/providertest v0.3.0/go.mod h1:aTxVfdxP/Pe0iPbokjBp99x0XaY4fkZB2CjIS5wysMs=
 github.com/pulumi/pulumi-java/pkg v1.8.0 h1:xCTQqTGxDj1f+VmCR//V0x355rAkc2b2VCIig1Ln/n8=
 github.com/pulumi/pulumi-java/pkg v1.8.0/go.mod h1:VH4YGMcPEYuMyOJjohMTepAqPSFPgmz4I3U4q5sJ89o=
 github.com/pulumi/pulumi-yaml v1.15.1 h1:4T36uwbJlQMbcK/X3U9BuqMZFEN4lnAIysPtqDvm0Tg=

pkg/internal/tests/cross-tests/diff_cross_test.go

@@ -1836,6 +1836,7 @@ func TestStateUpgradeSet(t *testing.T) {
 
 func TestDiffProviderUpgradeMaxItemsOneChanged(t *testing.T) {
 	t.Parallel()
+	skipUnlessDeltasEnabled(t)
 
 	resWithMaxItemsOne := &schema.Resource{
 		Schema: map[string]*schema.Schema{"prop": {
@@ -1871,8 +1872,6 @@ func TestDiffProviderUpgradeMaxItemsOneChanged(t *testing.T) {
 			map[string]cty.Value{"prop": cty.ListVal([]cty.Value{cty.StringVal("a")})},
 			map[string]cty.Value{"prop": cty.ListVal([]cty.Value{cty.StringVal("a")})},
 			DiffProviderUpgradedSchema(resWithMaxItemsOne),
-			// Note we produce a diff here while TF does not.
-			DiffSkipDiffEquivalenceCheck(),
 		)
 
 		autogold.ExpectFile(t, res.PulumiOut)

pkg/internal/tests/cross-tests/testdata/TestDiffProviderUpgradeMaxItemsOneChanged/max_items_one_added.golden

@@ -1,11 +1,6 @@
 `Previewing update (test):
   pulumi:pulumi:Stack: (same)
     [urn=urn:pulumi:test::project::pulumi:pulumi:Stack::project-test]
-    ~ crossprovider:index/testRes:TestRes: (update)
-        [id=newid]
-        [urn=urn:pulumi:test::project::crossprovider:index/testRes:TestRes::example]
-      + prop: "a"
 Resources:
-    ~ 1 to update
-    1 unchanged
+    2 unchanged
 `