OmitType and TypeName to support sharing types (#2409)

Introduce OmitType and TypeName flags to enforce type sharing.

Some of the upstream providers generate very large concrete schemata. TF
is not being materially affected, just high RAM demands for in-memory
processing. The example is inspired by QuickSight types in AWS. Pulumi
is affected significantly. In Pulumi the default projection is going to
generate named types for every instance of the shared schema. This leads
to SDK bloat and issues with "filename too long."

With this change it is possible for the provider maintainer opt into
explicit sharing of types, and ensure that the type names for the shared
types have shorter meaningful prefixes.

At definition type the user can specify the type name to generate, which
can be very short, and replace the automatically implied
ReallyLongPrefixedTypeName like this:

```go
"visuals": {
	Elem: &info.Schema{
		TypeName: tfbridge.Ref("Visual"),
	},
},
```

At reference time in another resource, the user can reuse an already
generated type by token. This already worked before this change but had
the downside of still generating unused helper types and causing SDK
bloat.

```go
"visuals": {
	Elem: &info.Schema{
		Type:     "testprov:index/Visual:Visual",
	},
},
```

With this change it is possible to instruct the bridge to stop
generating the unused helper types:

```go
"visuals": {
	Elem: &info.Schema{
		Type:     "testprov:index/Visual:Visual",
		OmitType: true
	},
},
```

Author: t0yv0

pkg/tfbridge/info.go

@@ -318,11 +318,18 @@ func MakeResource(pkg string, mod string, res string) tokens.Type {
 	return tokens.NewTypeToken(modT, tokens.TypeName(res))
 }
 
-// BoolRef returns a reference to the bool argument.
+// BoolRef returns a reference to the bool argument. Retained for backwards compatibility. Prefer [Ref] for new usage
+// and other types like strings.
 func BoolRef(b bool) *bool {
 	return &b
 }
 
+// Fluently construct a reference to the argument. This utility function is needed to ease configuring the bridge where
+// references are expected instead of plain boolean or string literals.
+func Ref[T any](x T) *T {
+	return &x
+}
+
 // StringValue gets a string value from a property map if present, else ""
 func StringValue(vars resource.PropertyMap, prop resource.PropertyKey) string {
 	val, ok := vars[prop]

pkg/tfbridge/info/info.go

@@ -441,9 +441,19 @@ type Schema struct {
 	// a name to override the default when targeting C#; "" uses the default.
 	CSharpName string
 
-	// a type to override the default; "" uses the default.
+	// An optional Pulumi type token to use for the Pulumi type projection of the current property. When unset, the
+	// default behavior is to generate fresh named Pulumi types as needed to represent the schema. To force the use
+	// of a known type and avoid generating unnecessary types, use both [Type] and [OmitType].
 	Type tokens.Type
 
+	// Used together with [Type] to omit generating any Pulumi types whatsoever for the current property, and
+	// instead use the object type identified by the token setup in [Type].
+	//
+	// It is an error to set [OmitType] to true without specifying [Type].
+	//
+	// Experimental.
+	OmitType bool
+
 	// alternative types that can be used instead of the override.
 	AltTypes []tokens.Type
 
@@ -502,6 +512,24 @@ type Schema struct {
 
 	// whether or not to treat this property as secret
 	Secret *bool
+
+	// Specifies the exact name to use for the generated type.
+	//
+	// When generating types for properties, by default Pulumi picks reasonable names based on the property path
+	// prefix and the name of the property. Use [TypeName] to override this decision when the default names for
+	// nested properties are too long or otherwise undesirable. The choice will further affect the automatically
+	// generated names for any properties nested under the current one.
+	//
+	// Example use:
+	//
+	//     TypeName: tfbridge.Ref("Visual")
+	//
+	// Note that the type name, and not the full token like "aws:quicksight/Visual:Visual" is specified. The token
+	// will be picked based on the current module ("quicksight" in the above example) where the parent resource or
+	// data source is found.
+	//
+	// Experimental.
+	TypeName *string
 }
 
 // Config represents a synthetic configuration variable that is Pulumi-only, and not passed to Terraform.

pkg/tfgen/generate.go

@@ -342,6 +342,8 @@ type propertyType struct {
 	nestedType tokens.Type
 	altTypes   []tokens.Type
 	asset      *tfbridge.AssetTranslation
+
+	typeName *string
 }
 
 func (g *Generator) Sink() diag.Sink {
@@ -353,6 +355,9 @@ func (g *Generator) makePropertyType(typePath paths.TypePath,
 	entityDocs entityDocs) (*propertyType, error) {
 
 	t := &propertyType{}
+	if info != nil {
+		t.typeName = info.TypeName
+	}
 
 	var elemInfo *tfbridge.SchemaInfo
 	if info != nil {
@@ -456,10 +461,23 @@ func getDocsFromSchemaMap(key string, schemaMap shim.SchemaMap) string {
 func (g *Generator) makeObjectPropertyType(typePath paths.TypePath,
 	res shim.Resource, info *tfbridge.SchemaInfo,
 	out bool, entityDocs entityDocs) (*propertyType, error) {
+
+	// If the user supplied an explicit Type token override, omit generating types and short-circuit.
+	if info != nil && info.OmitType {
+		if info.Type == "" {
+			return nil, fmt.Errorf("Cannot set info.OmitType without also setting info.Type")
+		}
+		return &propertyType{typ: info.Type}, nil
+	}
+
 	t := &propertyType{
 		kind: kindObject,
 	}
 
+	if info != nil {
+		t.typeName = info.TypeName
+	}
+
 	if info != nil {
 		t.typ = info.Type
 		t.nestedType = info.NestedType
@@ -549,6 +567,15 @@ func (t *propertyType) equals(other *propertyType) bool {
 	if len(t.properties) != len(other.properties) {
 		return false
 	}
+	switch {
+	case t.typeName != nil && other.typeName == nil:
+		return false
+	case t.typeName == nil && other.typeName != nil:
+		return false
+	case t.typeName != nil && other.typeName != nil &&
+		*t.typeName != *other.typeName:
+		return false
+	}
 	for i, p := range t.properties {
 		o := other.properties[i]
 		if p.name != o.name {

pkg/tfgen/generate_schema.go

@@ -120,7 +120,15 @@ func (nt *schemaNestedTypes) declareType(typePath paths.TypePath, declarer decla
 	typ *propertyType, isInput bool) string {
 
 	// Generate a name for this nested type.
-	typeName := namePrefix + cases.Title(language.Und, cases.NoLower).String(name)
+	var typeName string
+
+	if typ.typeName != nil {
+		// Use an explicit name if provided.
+		typeName = *typ.typeName
+	} else {
+		// Otherwise build one based on the current property name and prefix.
+		typeName = namePrefix + cases.Title(language.Und, cases.NoLower).String(name)
+	}
 
 	// Override the nested type name, if necessary.
 	if typ.nestedType.Name().String() != "" {

pkg/tfgen/generate_schema_test.go

@@ -19,10 +19,13 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"runtime"
+	"sort"
 	"testing"
 	"text/template"
 
 	"github.com/hashicorp/hcl/v2"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 	"github.com/hexops/autogold/v2"
 	csgen "github.com/pulumi/pulumi/pkg/v3/codegen/dotnet"
 	gogen "github.com/pulumi/pulumi/pkg/v3/codegen/go"
@@ -35,7 +38,9 @@ import (
 
 	bridgetesting "github.com/pulumi/pulumi-terraform-bridge/v3/internal/testing"
 	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfbridge"
+	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfbridge/info"
 	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfgen/internal/testprovider"
+	sdkv2 "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim/sdk-v2"
 	"github.com/pulumi/pulumi-terraform-bridge/v3/unstable/metadata"
 	"github.com/pulumi/pulumi-terraform-bridge/x/muxer"
 )
@@ -116,6 +121,167 @@ func TestCSharpMiniRandom(t *testing.T) {
 	bridgetesting.AssertEqualsJSONFile(t, "test_data/minirandom-schema-csharp.json", schema)
 }
 
+// Test the ability to force type sharing. Some of the upstream providers generate very large concrete schemata in Go,
+// with TF not being materially affected. The example is inspired by QuickSight types in AWS. In Pulumi the default
+// projection is going to generate named types for every instance of the shared schema. This may lead to SDK bloat. Test
+// the ability of the provider author to curb the bloat and force an explicit sharing.
+func TestTypeSharing(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skipf("Skipping on Windows due to a test setup issue")
+	}
+
+	tmpdir := t.TempDir()
+	barCharVisualSchema := func() *schema.Schema {
+		return &schema.Schema{
+			Type:     schema.TypeList,
+			Optional: true,
+			MinItems: 1,
+			MaxItems: 1,
+			Elem: &schema.Resource{
+				Schema: map[string]*schema.Schema{
+					"nest": {
+						Type:     schema.TypeList,
+						MaxItems: 1,
+						Optional: true,
+						Elem: &schema.Resource{
+							Schema: map[string]*schema.Schema{
+								"nested_prop": {
+									Type:     schema.TypeBool,
+									Optional: true,
+								},
+							},
+						},
+					},
+				},
+			},
+		}
+	}
+	visualsSchema := func() *schema.Schema {
+		return &schema.Schema{
+			Type:     schema.TypeList,
+			MinItems: 1,
+			MaxItems: 50,
+			Optional: true,
+			Elem: &schema.Resource{
+				Schema: map[string]*schema.Schema{
+					"bar_chart_visual": barCharVisualSchema(),
+					"box_plot_visual":  barCharVisualSchema(),
+				},
+			},
+		}
+	}
+	provider := info.Provider{
+		Name: "testprov",
+		P: sdkv2.NewProvider(&schema.Provider{
+			ResourcesMap: map[string]*schema.Resource{
+				"testprov_r1": {
+					Schema: map[string]*schema.Schema{
+						"sheets": {
+							Type:     schema.TypeList,
+							MinItems: 1,
+							MaxItems: 20,
+							Optional: true,
+							Elem: &schema.Resource{
+								Schema: map[string]*schema.Schema{
+									"visuals": visualsSchema(),
+								},
+							},
+						},
+					},
+				},
+				"testprov_r2": {
+					Schema: map[string]*schema.Schema{
+						"x": {
+							Type:     schema.TypeInt,
+							Optional: true,
+						},
+						"sheets": {
+							Type:     schema.TypeList,
+							MinItems: 1,
+							MaxItems: 20,
+							Optional: true,
+							Elem: &schema.Resource{
+								Schema: map[string]*schema.Schema{
+									"y": {
+										Type:     schema.TypeBool,
+										Optional: true,
+									},
+									"visuals": visualsSchema(),
+								},
+							},
+						},
+					},
+				},
+			},
+		}),
+		UpstreamRepoPath: tmpdir,
+		Resources: map[string]*info.Resource{
+			"testprov_r1": {
+				Tok: "testprov:index:R1",
+				Fields: map[string]*info.Schema{
+					"sheets": {
+						Elem: &info.Schema{
+							Fields: map[string]*info.Schema{
+								"visuals": {
+									Elem: &info.Schema{
+										TypeName: tfbridge.Ref("Visual"),
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			"testprov_r2": {
+				Tok: "testprov:index:R2",
+				Fields: map[string]*info.Schema{
+					"sheets": {
+						Elem: &info.Schema{
+							Fields: map[string]*info.Schema{
+								"visuals": {
+									Elem: &info.Schema{
+										Type:     "testprov:index/Visual:Visual",
+										OmitType: true,
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	var buf bytes.Buffer
+	schema, err := GenerateSchema(provider, diag.DefaultSink(&buf, &buf, diag.FormatOptions{
+		Color: colors.Never,
+	}))
+	require.NoError(t, err)
+
+	t.Logf("%s", buf.String())
+
+	keys := []string{}
+	for k := range schema.Types {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	// Note that there is only one set of helper types, and they are not prefixed by any of the resource names.
+	autogold.Expect([]string{
+		"testprov:index/R1Sheet:R1Sheet", "testprov:index/R2Sheet:R2Sheet",
+		"testprov:index/Visual:Visual",
+		"testprov:index/VisualBarChartVisual:VisualBarChartVisual",
+		"testprov:index/VisualBarChartVisualNest:VisualBarChartVisualNest",
+		"testprov:index/VisualBoxPlotVisual:VisualBoxPlotVisual",
+		"testprov:index/VisualBoxPlotVisualNest:VisualBoxPlotVisualNest",
+	}).Equal(t, keys)
+
+	bytes, err := json.MarshalIndent(schema, "", "  ")
+	require.NoError(t, err)
+
+	autogold.ExpectFile(t, autogold.Raw(string(bytes)))
+}
+
 // TestPropertyDocumentationEdits tests that documentation edits are applied to
 // individual properties. This includes both the property description and
 // deprecation message. This tests the following workflow