Add type exclusion filters for buf generate

.golangci.yml

@@ -349,12 +349,6 @@ issues:
       # from the fileDescriptor set, the operation should be safe.
       path: private/bufpkg/bufprotosource/paths.go
       text: "G115:"
-    - linters:
-        - gosec
-      # bufimageutil is handling images and converting loop indices to int32. Since it is
-      # parsing from an Image, the operation should be safe.
-      path: private/bufpkg/bufimage/bufimageutil/bufimageutil.go
-      text: "G115:"
     - linters:
         - gosec
       # Bounds checks have been added with assertion statements to ensure safe int -> int32

CHANGELOG.md

@@ -4,6 +4,9 @@
 
 - Fix `buf convert` to allow for zero length for `binpb`, `txtpb`, and `yaml` formats.
 - Fix use of deprecated flag `--include-types` for `buf generate`.
+- Improve type filtering for `buf generate`. Adds the ability to exclude types with the parameter
+  `exclude_types` in `buf.gen.yaml` and a flag `--exclude-types` in the CLI.
+  Type filters may now also be specified as plugin parameters in `buf.gen.yaml`.
 
 ## [v1.50.1] - 2025-03-10
 

private/buf/bufctl/controller.go

@@ -1337,8 +1337,15 @@ func filterImage(
 	if functionOptions.imageExcludeImports {
 		newImage = bufimage.ImageWithoutImports(newImage)
 	}
-	if len(functionOptions.imageTypes) > 0 {
-		newImage, err = bufimageutil.ImageFilteredByTypes(newImage, functionOptions.imageTypes...)
+	includeTypes := functionOptions.imageTypes
+	excludeTypes := functionOptions.imageExcludeTypes
+	if len(includeTypes) > 0 || len(excludeTypes) > 0 {
+		newImage, err = bufimageutil.FilterImage(
+			newImage,
+			bufimageutil.WithIncludeTypes(includeTypes...),
+			bufimageutil.WithExcludeTypes(excludeTypes...),
+			bufimageutil.WithMutateInPlace(),
+		)
 		if err != nil {
 			return nil, err
 		}

private/buf/bufctl/option.go

@@ -72,6 +72,12 @@ func WithImageTypes(imageTypes []string) FunctionOption {
 	}
 }
 
+func WithImageExcludeTypes(imageExcludeTypes []string) FunctionOption {
+	return func(functionOptions *functionOptions) {
+		functionOptions.imageExcludeTypes = imageExcludeTypes
+	}
+}
+
 func WithImageAsFileDescriptorSet(imageAsFileDescriptorSet bool) FunctionOption {
 	return func(functionOptions *functionOptions) {
 		functionOptions.imageAsFileDescriptorSet = imageAsFileDescriptorSet
@@ -134,6 +140,7 @@ type functionOptions struct {
 	imageExcludeSourceInfo          bool
 	imageExcludeImports             bool
 	imageTypes                      []string
+	imageExcludeTypes               []string
 	imageAsFileDescriptorSet        bool
 	configOverride                  string
 	ignoreAndDisallowV1BufWorkYAMLs bool

private/buf/bufgen/generator.go

@@ -20,12 +20,14 @@ import (
 	"fmt"
 	"log/slog"
 	"path/filepath"
+	"sort"
 
 	connect "connectrpc.com/connect"
 	"github.com/bufbuild/buf/private/buf/bufprotopluginexec"
 	"github.com/bufbuild/buf/private/bufpkg/bufconfig"
 	"github.com/bufbuild/buf/private/bufpkg/bufimage"
 	"github.com/bufbuild/buf/private/bufpkg/bufimage/bufimagemodify"
+	"github.com/bufbuild/buf/private/bufpkg/bufimage/bufimageutil"
 	"github.com/bufbuild/buf/private/bufpkg/bufprotoplugin"
 	"github.com/bufbuild/buf/private/bufpkg/bufprotoplugin/bufprotopluginos"
 	"github.com/bufbuild/buf/private/bufpkg/bufremoteplugin"
@@ -204,72 +206,91 @@ func (g *generator) execPlugins(
 	includeImportsOverride *bool,
 	includeWellKnownTypesOverride *bool,
 ) ([]*pluginpb.CodeGeneratorResponse, error) {
-	imageProvider := newImageProvider(image)
 	// Collect all of the plugin jobs so that they can be executed in parallel.
 	jobs := make([]func(context.Context) error, 0, len(pluginConfigs))
 	responses := make([]*pluginpb.CodeGeneratorResponse, len(pluginConfigs))
 	requiredFeatures := computeRequiredFeatures(image)
-	remotePluginConfigTable := make(map[string][]*remotePluginExecArgs, len(pluginConfigs))
-	for i, pluginConfig := range pluginConfigs {
-		index := i
-		currentPluginConfig := pluginConfig
-		// We're using this as a proxy for Type() == PluginConfigTypeRemote.
-		//
-		// We should be using the enum here.
-		remote := currentPluginConfig.RemoteHost()
-		if remote != "" {
-			remotePluginConfigTable[remote] = append(
-				remotePluginConfigTable[remote],
-				&remotePluginExecArgs{
-					Index:        index,
-					PluginConfig: currentPluginConfig,
-				},
+
+	// Group the pluginConfigs by similar properties to batch image processing.
+	pluginConfigsForImage := slicesext.ToIndexedValuesMap(pluginConfigs, createPluginConfigKeyForImage)
+	for _, indexedPluginConfigs := range pluginConfigsForImage {
+		image := image
+		hashPluginConfig := indexedPluginConfigs[0].Value
+
+		// Apply per-plugin filters.
+		includeTypes := hashPluginConfig.Types()
+		excludeTypes := hashPluginConfig.ExcludeTypes()
+		if len(includeTypes) > 0 || len(excludeTypes) > 0 {
+			var err error
+			image, err = bufimageutil.FilterImage(
+				image,
+				bufimageutil.WithIncludeTypes(includeTypes...),
+				bufimageutil.WithExcludeTypes(excludeTypes...),
 			)
-		} else {
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		// Batch for each remote.
+		if remote := hashPluginConfig.RemoteHost(); remote != "" {
 			jobs = append(jobs, func(ctx context.Context) error {
-				includeImports := currentPluginConfig.IncludeImports()
-				if includeImportsOverride != nil {
-					includeImports = *includeImportsOverride
-				}
-				includeWellKnownTypes := currentPluginConfig.IncludeWKT()
-				if includeWellKnownTypesOverride != nil {
-					includeWellKnownTypes = *includeWellKnownTypesOverride
-				}
-				response, err := g.execLocalPlugin(
+				results, err := g.execRemotePluginsV2(
 					ctx,
 					container,
-					imageProvider,
-					currentPluginConfig,
-					includeImports,
-					includeWellKnownTypes,
+					image,
+					remote,
+					indexedPluginConfigs,
+					includeImportsOverride,
+					includeWellKnownTypesOverride,
 				)
 				if err != nil {
 					return err
 				}
-				responses[index] = response
+				for _, result := range results {
+					responses[result.Index] = result.Value
+				}
 				return nil
 			})
+			continue
 		}
-	}
-	// Batch for each remote.
-	for remote, indexedPluginConfigs := range remotePluginConfigTable {
-		if len(indexedPluginConfigs) > 0 {
+
+		// Local plugins.
+		var images []bufimage.Image
+		switch Strategy(hashPluginConfig.Strategy()) {
+		case StrategyAll:
+			images = []bufimage.Image{image}
+		case StrategyDirectory:
+			var err error
+			images, err = bufimage.ImageByDir(image)
+			if err != nil {
+				return nil, err
+			}
+		default:
+			return nil, fmt.Errorf("unknown strategy: %v", hashPluginConfig.Strategy())
+		}
+		for _, indexedPluginConfig := range indexedPluginConfigs {
 			jobs = append(jobs, func(ctx context.Context) error {
-				results, err := g.execRemotePluginsV2(
+				includeImports := hashPluginConfig.IncludeImports()
+				if includeImportsOverride != nil {
+					includeImports = *includeImportsOverride
+				}
+				includeWellKnownTypes := hashPluginConfig.IncludeWKT()
+				if includeWellKnownTypesOverride != nil {
+					includeWellKnownTypes = *includeWellKnownTypesOverride
+				}
+				response, err := g.execLocalPlugin(
 					ctx,
 					container,
-					image,
-					remote,
-					indexedPluginConfigs,
-					includeImportsOverride,
-					includeWellKnownTypesOverride,
+					images,
+					indexedPluginConfig.Value,
+					includeImports,
+					includeWellKnownTypes,
 				)
 				if err != nil {
 					return err
 				}
-				for _, result := range results {
-					responses[result.Index] = result.CodeGeneratorResponse
-				}
+				responses[indexedPluginConfig.Index] = response
 				return nil
 			})
 		}
@@ -305,15 +326,11 @@ func (g *generator) execPlugins(
 func (g *generator) execLocalPlugin(
 	ctx context.Context,
 	container app.EnvStdioContainer,
-	imageProvider *imageProvider,
+	pluginImages []bufimage.Image,
 	pluginConfig bufconfig.GeneratePluginConfig,
 	includeImports bool,
 	includeWellKnownTypes bool,
 ) (*pluginpb.CodeGeneratorResponse, error) {
-	pluginImages, err := imageProvider.GetImages(Strategy(pluginConfig.Strategy()))
-	if err != nil {
-		return nil, err
-	}
 	requests, err := bufimage.ImagesToCodeGeneratorRequests(
 		pluginImages,
 		pluginConfig.Opt(),
@@ -338,37 +355,27 @@ func (g *generator) execLocalPlugin(
 	return response, nil
 }
 
-type remotePluginExecArgs struct {
-	Index        int
-	PluginConfig bufconfig.GeneratePluginConfig
-}
-
-type remotePluginExecutionResult struct {
-	CodeGeneratorResponse *pluginpb.CodeGeneratorResponse
-	Index                 int
-}
-
 func (g *generator) execRemotePluginsV2(
 	ctx context.Context,
 	container app.EnvStdioContainer,
 	image bufimage.Image,
 	remote string,
-	pluginConfigs []*remotePluginExecArgs,
+	indexedPluginConfigs []slicesext.Indexed[bufconfig.GeneratePluginConfig],
 	includeImportsOverride *bool,
 	includeWellKnownTypesOverride *bool,
-) ([]*remotePluginExecutionResult, error) {
-	requests := make([]*registryv1alpha1.PluginGenerationRequest, len(pluginConfigs))
-	for i, pluginConfig := range pluginConfigs {
-		includeImports := pluginConfig.PluginConfig.IncludeImports()
+) ([]slicesext.Indexed[*pluginpb.CodeGeneratorResponse], error) {
+	requests := make([]*registryv1alpha1.PluginGenerationRequest, len(indexedPluginConfigs))
+	for i, pluginConfig := range indexedPluginConfigs {
+		includeImports := pluginConfig.Value.IncludeImports()
 		if includeImportsOverride != nil {
 			includeImports = *includeImportsOverride
 		}
-		includeWellKnownTypes := pluginConfig.PluginConfig.IncludeWKT()
+		includeWellKnownTypes := pluginConfig.Value.IncludeWKT()
 		if includeWellKnownTypesOverride != nil {
 			includeWellKnownTypes = *includeWellKnownTypesOverride
 		}
 		request, err := getPluginGenerationRequest(
-			pluginConfig.PluginConfig,
+			pluginConfig.Value,
 			includeImports,
 			includeWellKnownTypes,
 		)
@@ -398,15 +405,15 @@ func (g *generator) execRemotePluginsV2(
 	if len(responses) != len(requests) {
 		return nil, fmt.Errorf("unexpected number of responses received, got %d, wanted %d", len(responses), len(requests))
 	}
-	result := make([]*remotePluginExecutionResult, 0, len(responses))
+	result := make([]slicesext.Indexed[*pluginpb.CodeGeneratorResponse], 0, len(responses))
 	for i := range requests {
 		codeGeneratorResponse := responses[i].GetResponse()
 		if codeGeneratorResponse == nil {
 			return nil, errors.New("expected code generator response")
 		}
-		result = append(result, &remotePluginExecutionResult{
-			CodeGeneratorResponse: codeGeneratorResponse,
-			Index:                 pluginConfigs[i].Index,
+		result = append(result, slicesext.Indexed[*pluginpb.CodeGeneratorResponse]{
+			Value: codeGeneratorResponse,
+			Index: indexedPluginConfigs[i].Index,
 		})
 	}
 	return result, nil
@@ -482,3 +489,29 @@ type generateOptions struct {
 func newGenerateOptions() *generateOptions {
 	return &generateOptions{}
 }
+
+type pluginConfigKeyForImage struct {
+	includeTypes string // string representation of []string
+	excludeTypes string // string representation of []string
+	strategy     Strategy
+	remoteHost   string
+}
+
+// createPluginConfigKeyForImage returns a key of the plugin config with
+// a subset of properties. This is used to batch plugins that have similar
+// configuration. The key is based on the following properties:
+//   - Types
+//   - ExcludeTypes
+//   - Strategy
+//   - RemoteHost
+func createPluginConfigKeyForImage(pluginConfig bufconfig.GeneratePluginConfig) pluginConfigKeyForImage {
+	// Sort the types and excludeTypes so that the key is deterministic.
+	sort.Strings(pluginConfig.Types())
+	sort.Strings(pluginConfig.ExcludeTypes())
+	return pluginConfigKeyForImage{
+		includeTypes: fmt.Sprintf("%v", pluginConfig.Types()),
+		excludeTypes: fmt.Sprintf("%v", pluginConfig.ExcludeTypes()),
+		strategy:     Strategy(pluginConfig.Strategy()),
+		remoteHost:   pluginConfig.RemoteHost(),
+	}
+}